feat: aggregate tag list with database info (#7)

Because

- Distribution registry doesn't contain push timestamps nor digests for
tags

This commit

- Aggregates the tag list information from the registry with the
database information (if present)

cmd/main/main.go

@@ -137,11 +137,11 @@ func main() {
  grpc_zap.WithDecider(func(fullMethodName string, err error) bool {
  // will not log gRPC calls if it was a call to liveness or readiness and no error was raised
  if err == nil {
- if match, _ := regexp.MatchString("vdp.artifact.v1beta.ArtifactPublicService/.*ness$", fullMethodName); match {
+ if match, _ := regexp.MatchString("artifact.artifact.v1alpha.ArtifactPublicService/.*ness$", fullMethodName); match {
  return false
  }
  // stop logging successful private function calls
- if match, _ := regexp.MatchString("vdp.artifact.v1beta.ArtifactPrivateService/.*Admin$", fullMethodName); match {
+ if match, _ := regexp.MatchString("artifact.artifact.v1alpha.ArtifactPrivateService/.*$", fullMethodName); match {
  return false
  }
  }
@@ -204,7 +204,7 @@ func main() {
 
  repository := repository.NewRepository(db)
 
- service := service.NewService(httpclient.NewRegistryClient(ctx))
+ service := service.NewService(repository, httpclient.NewRegistryClient(ctx))
 
  publicGrpcS := grpc.NewServer(grpcServerOpts...)
  reflection.Register(publicGrpcS)
@@ -245,7 +245,7 @@ func main() {
  logger.Info("try to start usage reporter")
  go func() {
  for {
- usg = usage.NewUsage(ctx, repository, mgmtPrivateServiceClient, redisClient, usageServiceClient)
+ usg = usage.NewUsage(ctx, mgmtPrivateServiceClient, redisClient, usageServiceClient)
  if usg != nil {
  usg.StartReporter(ctx)
  logger.Info("usage reporter started")

cmd/migration/main.go

@@ -3,8 +3,11 @@ package main
 import (
  "database/sql"
  "fmt"
+ "os"
 
+ "github.com/golang-migrate/migrate/v4"
  _ "github.com/golang-migrate/migrate/v4/database/postgres"
+ _ "github.com/golang-migrate/migrate/v4/source/file"
 
  "github.com/instill-ai/artifact-backend/config"
 )
@@ -55,8 +58,52 @@ func main() {
  }
 
  databaseConfig := config.Config.Database
-
  if err := dbExistsOrCreate(databaseConfig); err != nil {
  panic(err)
  }
+
+ dsn := fmt.Sprintf("postgres:https://%s:%s@%s:%d/%s?%s",
+ databaseConfig.Username,
+ databaseConfig.Password,
+ databaseConfig.Host,
+ databaseConfig.Port,
+ databaseConfig.Name,
+ "sslmode=disable",
+ )
+
+ migrateFolder, _ := os.Getwd()
+ m, err := migrate.New(fmt.Sprintf("file:https:///%s/pkg/db/migration", migrateFolder), dsn)
+ if err != nil {
+ panic(err)
+ }
+
+ expectedVersion := databaseConfig.Version
+ curVersion, dirty, err := m.Version()
+ if err != nil && curVersion != 0 {
+ panic(err)
+ }
+
+ fmt.Printf("Expected migration version is %d\n", expectedVersion)
+ fmt.Printf("The current schema version is %d, and dirty flag is %t\n", curVersion, dirty)
+ if dirty {
+ panic("the database's dirty flag is set, please fix it")
+ }
+
+ step := curVersion
+ for {
+ if expectedVersion <= step {
+ fmt.Printf("Migration to version %d complete\n", expectedVersion)
+ break
+ }
+
+ fmt.Printf("Step up to version %d\n", step+1)
+ if err := m.Steps(1); err != nil {
+ panic(err)
+ }
+
+ step, _, err = m.Version()
+ if err != nil {
+ panic(err)
+ }
+ }
 }

cmd/worker/main.go

@@ -17,7 +17,6 @@ import (
  "github.com/instill-ai/artifact-backend/config"
  httpclient "github.com/instill-ai/artifact-backend/pkg/client/http"
  "github.com/instill-ai/artifact-backend/pkg/logger"
- "github.com/instill-ai/artifact-backend/pkg/repository"
  "github.com/instill-ai/x/temporal"
  "github.com/instill-ai/x/zapadapter"
 
@@ -98,7 +97,6 @@ func main() {
 
  db := database.GetSharedConnection()
  defer database.Close(db)
- repository := repository.NewRepository(db)
 
  var err error
 
@@ -149,7 +147,7 @@ func main() {
  }
  }()
 
- _ = artifactWorker.NewWorker(repository, redisClient, influxDBWriteClient)
+ _ = artifactWorker.NewWorker(redisClient, influxDBWriteClient)
 
  w := worker.New(temporalClient, artifactWorker.TaskQueue, worker.Options{
  MaxConcurrentActivityExecutionSize: 2,

config/config.yaml

@@ -22,7 +22,7 @@ database:
  host: pg-sql
  port: 5432
  name: artifact
- version: 0
+ version: 1
  timezone: Etc/UTC
  pool:
  idleconnections: 5

pkg/db/migration/000001_init.down.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+DROP TABLE IF EXISTS repository_tag CASCADE;
+
+COMMIT;

pkg/db/migration/000001_init.up.sql

@@ -0,0 +1,11 @@
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS repository_tag (
+ -- "<repository>:tag", e.g. "melancholic-wombat/llava-34b:latest"
+ name VARCHAR(255) PRIMARY KEY,
+ digest VARCHAR(255) NOT NULL,
+ create_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL,
+ update_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+
+COMMIT;

pkg/middleware/interceptor.go

@@ -17,7 +17,6 @@ import (
 
  "github.com/instill-ai/artifact-backend/pkg/acl"
  "github.com/instill-ai/artifact-backend/pkg/handler"
- "github.com/instill-ai/artifact-backend/pkg/repository"
  "github.com/instill-ai/artifact-backend/pkg/service"
  "github.com/instill-ai/x/errmsg"
 )
@@ -90,15 +89,11 @@ func AsGRPCError(err error) error {
  code = codes.AlreadyExists
  case
  errors.Is(err, gorm.ErrRecordNotFound),
- errors.Is(err, repository.ErrNoDataDeleted),
- errors.Is(err, repository.ErrNoDataUpdated),
  errors.Is(err, service.ErrNotFound),
  errors.Is(err, acl.ErrMembershipNotFound):
 
  code = codes.NotFound
  case
- errors.Is(err, repository.ErrOwnerTypeNotMatch),
- errors.Is(err, repository.ErrPageTokenDecode),
  errors.Is(err, bcrypt.ErrMismatchedHashAndPassword),
  errors.Is(err, handler.ErrCheckUpdateImmutableFields),
  errors.Is(err, handler.ErrCheckOutputOnlyFields),

pkg/mock/generator.go

@@ -1,3 +1,4 @@
 package mock
 
 //go:generate minimock -g -i github.com/instill-ai/artifact-backend/pkg/service.RegistryClient -o ./ -s "_mock.gen.go"
+//go:generate minimock -g -i github.com/instill-ai/artifact-backend/pkg/service.Repository -o ./ -s "_mock.gen.go"