feat: fetch repository tags from registry (#6)

Because

- We want to list the versions of a model.
- Registry has a
[tags](https://distribution.github.io/distribution/#listing-image-tags)
endpoint but it doesn't expose any timestamp.

![CleanShot 2024-03-19 at 20 24
14](https://github.com/instill-ai/protobufs/assets/3977183/38f408d1-0096-4f39-b3fe-92991104fa66)

This commit

- Serves the `ListRepositoryTags` endpoint in the Artifact Private API.
- Fetches the tag list from the `registry` service.
- Simplifies part of the code. Part of the complexity was copied as
boilerplate from other services, but it's actually not necessary yet.

# QA 🔨 

```sh
$ docker pull hello-world
$ # add some tags
$ docker tag hello-world localhost:5001/admin/hello-world:0.1.4
$ docker push hello-world localhost:5001/admin/hello-world:0.1.4 
$ # ... repeat for different tags
$ grpcurl -plaintext \
-proto ./artifact/artifact/v1alpha/artifact_private_service.proto \
-import-path ~/Code/go/src/github.com/instill-ai/pipeline-backend/integration-test/proto/vdp/pipeline/v1beta/ \
-import-path . \
-d '{"parent": "repositories/admin/hello-world", "page_size":2,"page": 1}' \
localhost:8085 artifact.artifact.v1alpha.ArtifactPrivateService/ListRepositoryTags | jq
{
  "tags": [
    {
      "name": "repositories/admin/hello-world/tags/0.1.3",
      "id": "0.1.3"
    },
    {
      "name": "repositories/admin/hello-world/tags/0.1.1",
      "id": "0.1.1"
    }
  ]
}
$ grpcurl -plaintext \
-proto ./artifact/artifact/v1alpha/artifact_private_service.proto \
-import-path ~/Code/go/src/github.com/instill-ai/pipeline-backend/integration-test/proto/vdp/pipeline/v1beta/ \
-import-path . \
-d '{"parent": "repositories/admin/hello-world"}' \
localhost:8085 artifact.artifact.v1alpha.ArtifactPrivateService/ListRepositoryTags | jq
{
  "tags": [
    {
      "name": "repositories/admin/hello-world/tags/0.1.1-beta",
      "id": "0.1.1-beta"
    },
    {
      "name": "repositories/admin/hello-world/tags/0.1.0",
      "id": "0.1.0"
    },
    {
      "name": "repositories/admin/hello-world/tags/0.1.3",
      "id": "0.1.3"
    },
    {
      "name": "repositories/admin/hello-world/tags/0.1.1",
      "id": "0.1.1"
    },
    {
      "name": "repositories/admin/hello-world/tags/0.1.4",
      "id": "0.1.4"
    }
  ]
}
```

# ⏭️ Next steps
- I tried to set up integration tests throuth `k6s`, replicating the
previous section, but it's the first time setting the tests up and I was
losing quite some time with the network setup and with broken import
paths. As other devs are relying on this feature for testing the model
deployment on d0, I created a ticket to add the e2e tests later.
- Another ticket was created to include `artifact-backend` in the core
and cloud ecosystems.
- The next feature will consist of opening the `CreateRepositoryTag`
endpoint, which will store some extra information in the database, and
to aggregate the information from both sources in the List endpoint.

Makefile

@@ -53,9 +53,10 @@ go-gen: ## Generate codes
 .PHONY: unit-test
 unit-test: ## Run unit test
  @go test -v -race -coverpkg=./... -coverprofile=coverage.out ./...
- @go tool cover -func=coverage.out
- @go tool cover -html=coverage.out
- @rm coverage.out
+ @cat coverage.out | grep -v "mock" > coverage.final.out
+ @go tool cover -func=coverage.final.out
+ @go tool cover -html=coverage.final.out
+ @rm coverage.out coverage.final.out
 
 .PHONY: integration-test
 integration-test: ## Run integration test

cmd/main/main.go

@@ -30,12 +30,11 @@ import (
  grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
  grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
  grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
- openfgaClient "github.com/openfga/go-sdk/client"
 
  "github.com/instill-ai/artifact-backend/config"
- "github.com/instill-ai/artifact-backend/pkg/acl"
+ grpcclient "github.com/instill-ai/artifact-backend/pkg/client/grpc"
+ httpclient "github.com/instill-ai/artifact-backend/pkg/client/http"
  "github.com/instill-ai/artifact-backend/pkg/constant"
- "github.com/instill-ai/artifact-backend/pkg/external"
  "github.com/instill-ai/artifact-backend/pkg/handler"
  "github.com/instill-ai/artifact-backend/pkg/logger"
  "github.com/instill-ai/artifact-backend/pkg/middleware"
@@ -71,22 +70,22 @@ func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler) http.Handl
 }
 
 func main() {
-
  if err := config.Init(); err != nil {
  log.Fatal(err.Error())
  }
 
  // setup tracing and metrics
  ctx, cancel := context.WithCancel(context.Background())
 
- if tp, err := custom_otel.SetupTracing(ctx, "artifact-backend"); err != nil {
+ tp, err := custom_otel.SetupTracing(ctx, "artifact-backend")
+ if err != nil {
  panic(err)
- } else {
- defer func() {
- err = tp.Shutdown(ctx)
- }()
  }
 
+ defer func() {
+ err = tp.Shutdown(ctx)
+ }()
+
  ctx, span := otel.Tracer("main-tracer").Start(ctx,
  "main",
  )
@@ -105,7 +104,6 @@ func main() {
  defer database.Close(db)
 
  var temporalClientOptions client.Options
- var err error
  if config.Config.Temporal.Ca != "" && config.Config.Temporal.Cert != "" && config.Config.Temporal.Key != "" {
  if temporalClientOptions, err = temporal.GetTLSClientOption(
  config.Config.Temporal.HostPort,
@@ -165,29 +163,6 @@ func main() {
  )),
  }
 
- fgaClient, err := openfgaClient.NewSdkClient(&openfgaClient.ClientConfiguration{
- ApiScheme: "http",
- ApiHost: fmt.Sprintf("%s:%d", config.Config.OpenFGA.Host, config.Config.OpenFGA.Port),
- })
-
- if err != nil {
- panic(err)
- }
-
- var aclClient acl.ACLClient
- if stores, err := fgaClient.ListStores(context.Background()).Execute(); err == nil {
- fgaClient.SetStoreId(*(*stores.Stores)[0].Id)
- if models, err := fgaClient.ReadAuthorizationModels(context.Background()).Execute(); err == nil {
- aclClient = acl.NewACLClient(fgaClient, (*models.AuthorizationModels)[0].Id)
- }
- if err != nil {
- panic(err)
- }
-
- } else {
- panic(err)
- }
-
  // Create tls based credential.
  var creds credentials.TransportCredentials
  var tlsConfig *tls.Config
@@ -205,19 +180,19 @@ func main() {
  grpcServerOpts = append(grpcServerOpts, grpc.MaxRecvMsgSize(constant.MaxPayloadSize))
  grpcServerOpts = append(grpcServerOpts, grpc.MaxSendMsgSize(constant.MaxPayloadSize))
 
- mgmtPrivateServiceClient, mgmtPrivateServiceClientConn := external.InitMgmtPrivateServiceClient(ctx)
+ mgmtPrivateServiceClient, mgmtPrivateServiceClientConn := grpcclient.NewMGMTPrivateClient(ctx)
  if mgmtPrivateServiceClientConn != nil {
  defer mgmtPrivateServiceClientConn.Close()
  }
- _, mgmtPublicServiceClientConn := external.InitMgmtPublicServiceClient(ctx)
+ _, mgmtPublicServiceClientConn := grpcclient.NewMGMTPublicClient(ctx)
  if mgmtPublicServiceClientConn != nil {
  defer mgmtPublicServiceClientConn.Close()
  }
 
  redisClient := redis.NewClient(&config.Config.Cache.Redis.RedisOptions)
  defer redisClient.Close()
 
- influxDBClient, influxDBWriteClient := external.InitInfluxDBServiceClient(ctx)
+ influxDBClient, influxDBWriteClient := httpclient.NewInfluxDBClient(ctx)
  defer influxDBClient.Close()
 
  influxErrCh := influxDBWriteClient.Errors()
@@ -229,39 +204,20 @@ func main() {
 
  repository := repository.NewRepository(db)
 
- service := service.NewService(
- repository,
- redisClient,
- temporalClient,
- influxDBWriteClient,
- &aclClient,
- )
-
- privateGrpcS := grpc.NewServer(grpcServerOpts...)
- reflection.Register(privateGrpcS)
+ service := service.NewService(httpclient.NewRegistryClient(ctx))
 
  publicGrpcS := grpc.NewServer(grpcServerOpts...)
  reflection.Register(publicGrpcS)
-
  artifactPB.RegisterArtifactPublicServiceServer(
  publicGrpcS,
  handler.NewPublicHandler(ctx, service),
  )
 
- privateServeMux := runtime.NewServeMux(
- runtime.WithForwardResponseOption(middleware.HTTPResponseModifier),
- runtime.WithErrorHandler(middleware.ErrorHandler),
- runtime.WithIncomingHeaderMatcher(middleware.CustomMatcher),
- runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{
- MarshalOptions: protojson.MarshalOptions{
- UseProtoNames: true,
- EmitUnpopulated: true,
- UseEnumNumbers: false,
- },
- UnmarshalOptions: protojson.UnmarshalOptions{
- DiscardUnknown: true,
- },
- }),
+ privateGrpcS := grpc.NewServer(grpcServerOpts...)
+ reflection.Register(privateGrpcS)
+ artifactPB.RegisterArtifactPrivateServiceServer(
+ publicGrpcS,
+ handler.NewPrivateHandler(ctx, service),
  )
 
  publicServeMux := runtime.NewServeMux(
@@ -283,7 +239,7 @@ func main() {
  // Start usage reporter
  var usg usage.Usage
  if config.Config.Server.Usage.Enabled {
- usageServiceClient, usageServiceClientConn := external.InitUsageServiceClient(ctx)
+ usageServiceClient, usageServiceClientConn := grpcclient.NewUsageClient(ctx)
  if usageServiceClientConn != nil {
  defer usageServiceClientConn.Close()
  logger.Info("try to start usage reporter")
@@ -314,12 +270,6 @@ func main() {
  logger.Fatal(err.Error())
  }
 
- privateHTTPServer := &http.Server{
- Addr: fmt.Sprintf(":%v", config.Config.Server.PrivatePort),
- Handler: grpcHandlerFunc(privateGrpcS, privateServeMux),
- TLSConfig: tlsConfig,
- }
-
  publicHTTPServer := &http.Server{
  Addr: fmt.Sprintf(":%v", config.Config.Server.PublicPort),
  Handler: grpcHandlerFunc(publicGrpcS, publicServeMux),
@@ -330,22 +280,12 @@ func main() {
  quitSig := make(chan os.Signal, 1)
  errSig := make(chan error)
  if config.Config.Server.HTTPS.Cert != "" && config.Config.Server.HTTPS.Key != "" {
- go func() {
- if err := privateHTTPServer.ListenAndServeTLS(config.Config.Server.HTTPS.Cert, config.Config.Server.HTTPS.Key); err != nil {
- errSig <- err
- }
- }()
  go func() {
  if err := publicHTTPServer.ListenAndServeTLS(config.Config.Server.HTTPS.Cert, config.Config.Server.HTTPS.Key); err != nil {
  errSig <- err
  }
  }()
  } else {
- go func() {
- if err := privateHTTPServer.ListenAndServe(); err != nil {
- errSig <- err
- }
- }()
  go func() {
  if err := publicHTTPServer.ListenAndServe(); err != nil {
  errSig <- err

cmd/worker/main.go

@@ -15,7 +15,7 @@ import (
  "go.temporal.io/sdk/worker"
 
  "github.com/instill-ai/artifact-backend/config"
- "github.com/instill-ai/artifact-backend/pkg/external"
+ httpclient "github.com/instill-ai/artifact-backend/pkg/client/http"
  "github.com/instill-ai/artifact-backend/pkg/logger"
  "github.com/instill-ai/artifact-backend/pkg/repository"
  "github.com/instill-ai/x/temporal"
@@ -139,7 +139,7 @@ func main() {
  redisClient := redis.NewClient(&config.Config.Cache.Redis.RedisOptions)
  defer redisClient.Close()
 
- influxDBClient, influxDBWriteClient := external.InitInfluxDBServiceClient(ctx)
+ influxDBClient, influxDBWriteClient := httpclient.NewInfluxDBClient(ctx)
  defer influxDBClient.Close()
 
  influxErrCh := influxDBWriteClient.Errors()

config/config.go

@@ -26,6 +26,7 @@ type AppConfig struct {
  Cache CacheConfig `koanf:"cache"`
  Log LogConfig `koanf:"log"`
  MgmtBackend MgmtBackendConfig `koanf:"mgmtbackend"`
+ Registry RegistryConfig `koanf:"registry"`
  OpenFGA OpenFGAConfig `koanf:"openfga"`
 }
 
@@ -141,6 +142,12 @@ type MgmtBackendConfig struct {
  }
 }
 
+// RegistryConfig is the registry configuration.
+type RegistryConfig struct {
+ Host string `koanf:"host"`
+ Port int `koanf:"port"`
+}
+
 // Init - Assign global config to decoded config struct
 func Init() error {
  k := koanf.New(".")

config/config.yaml

@@ -61,6 +61,9 @@ mgmtbackend:
  https:
  cert:
  key:
+registry:
+ host: registry
+ port: 5000
 openfga:
  host: openfga
  port: 8080

go.mod

@@ -5,13 +5,17 @@ go 1.21.1
 toolchain go1.21.3
 
 require (
+ github.com/frankban/quicktest v1.14.6
  github.com/go-redis/redis/v9 v9.0.0-beta.2
+ github.com/go-resty/resty/v2 v2.12.0
  github.com/gofrs/uuid v4.4.0+incompatible
+ github.com/gojuno/minimock/v3 v3.3.6
  github.com/golang-migrate/migrate/v4 v4.17.0
+ github.com/google/go-cmp v0.6.0
  github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
  github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0
  github.com/influxdata/influxdb-client-go/v2 v2.12.3
- github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240213113306-651642106db5
+ github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240326081344-9f99c3127134
  github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
  github.com/instill-ai/x v0.3.0-alpha.0.20231219052200-6230a89e386c
  github.com/knadh/koanf v1.5.0
@@ -28,7 +32,7 @@ require (
  go.temporal.io/api v1.16.0
  go.temporal.io/sdk v1.21.0
  go.uber.org/zap v1.26.0
- golang.org/x/net v0.21.0
+ golang.org/x/net v0.22.0
  google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405
  google.golang.org/grpc v1.59.0
  google.golang.org/protobuf v1.33.0
@@ -39,9 +43,10 @@ require (
 require (
  github.com/gogo/status v1.1.1 // indirect
  github.com/golang/mock v1.6.0 // indirect
- github.com/google/go-cmp v0.6.0 // indirect
  github.com/hashicorp/errwrap v1.1.0 // indirect
  github.com/hashicorp/go-multierror v1.1.1 // indirect
+ github.com/kr/pretty v0.3.1 // indirect
+ github.com/kr/text v0.2.0 // indirect
  github.com/pelletier/go-toml v1.9.3 // indirect
  golang.org/x/sync v0.5.0 // indirect
  google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
@@ -91,10 +96,10 @@ require (
  go.opentelemetry.io/proto/otlp v0.19.0 // indirect
  go.uber.org/atomic v1.10.0 // indirect
  go.uber.org/multierr v1.10.0 // indirect
- golang.org/x/crypto v0.20.0
- golang.org/x/sys v0.17.0 // indirect
+ golang.org/x/crypto v0.21.0
+ golang.org/x/sys v0.18.0 // indirect
  golang.org/x/text v0.14.0 // indirect
- golang.org/x/time v0.3.0 // indirect
+ golang.org/x/time v0.5.0 // indirect
  google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
  gopkg.in/yaml.v3 v3.0.1 // indirect
 )