feat(artifact): add tag create / list endpoints (#286)

Because

- We want to list the versions of a model.
- Registry has a
[tags](https://distribution.github.io/distribution/#listing-image-tags)
endpoint but it doesn't expose any timestamp.

![CleanShot 2024-03-19 at 20 24
14](https://github.com/instill-ai/protobufs/assets/3977183/38f408d1-0096-4f39-b3fe-92991104fa66)

This commit

- Adds and documents a private API for the repository tags in artifact.
- Only `api-gateway` (when it detects the end of the image push
operation) should communicate the creation of a tag.
- The tags won't be exposed directly to the client. `model-backend` will
request this information to expose the model versions in the same family
as the rest of the model endpoints.

## 🗒️ Notes

As an initial addition to the Artifact domain, I made some naming
choices. Please review and provide feedback about them.

## ⏭️ Next steps

- `artifact-backend` will implement a first version of this endpoint
that will fetch the tag list from the `registry` instance. This endpoint
only returns the tag list so the digest and update time will be empty.
- In order to record the update time, `api-gateway` will call the
creation endpoint when we intercept a `PUT manifest` request. We'll
store the digest and update time in the `artifact` database and we'll
aggregate that info to the `registry` tag list (the registry should
remain the source of truth).

---------

Co-authored-by: droplet-bot <[email protected]>

artifact/artifact/v1alpha/artifact.proto

@@ -5,6 +5,9 @@ package artifact.artifact.v1alpha;
 import "common/healthcheck/v1beta/healthcheck.proto";
 // Google API
 import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+// Protocol Buffers Well-Known Types
+import "google/protobuf/timestamp.proto";
 
 // LivenessRequest represents a request to check a service liveness status
 message LivenessRequest {
@@ -29,3 +32,104 @@ message ReadinessResponse {
  // HealthCheckResponse message
  common.healthcheck.v1beta.HealthCheckResponse health_check_response = 1;
 }
+
+/*
+
+This API is under development and, therefore, some of its entitites and
+entpoints are not implemented yet. This section aims to give context about the
+current interface and how it fits in the Artifact vision.
+
+# Artifact
+
+The Artifact domain is responsible of storing data that will later be used for
+processing unstructured data. Artifact will support the following types of
+data:
+
+- Repositories
+- Objects
+- Vectors
+
+## Repositories
+
+An implementation of the [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec?tab=readme-ov-file)
+is used to manage versioned content. The main use for repositories is storing
+container images that can be used to deploy AI models or VDP pipelines.
+
+The ID of a repository has 2 segments, the owner (an Instill user or
+organization) and the content ID (the AI model or pipeline ID), e.g.
+`curious-wombat/llava-34b`.
+
+## Objects
+
+Raw data is stored in binary blobs. Object storage allows users to upload data
+(e.g. images, audio) that can be used by pipelines or to store the results or a
+pipeline trigger.
+
+## Vectors
+
+Vector embeddings have their own storage, which allows fast retrieval and similarity search.
+
+*/
+
+// RepositoryTag contains information about the version of some content in a
+// repository.
+message RepositoryTag {
+ // The name of the tag, defined by its parent repository and ID.
+ // - Format: `repositories/{repository.id}/tags/{tag.id}`.
+ string name = 1 [(google.api.field_behavior) = IMMUTABLE];
+ // The tag identifier.
+ string id = 2 [(google.api.field_behavior) = IMMUTABLE];
+ // Unique identifier, computed from the manifest the tag refers to.
+ string digest = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+ // Tag update time, i.e. timestamp of the last push.
+ google.protobuf.Timestamp update_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// ListRepositoryTagsRequest represents a request to list the tags of a
+// repository.
+message ListRepositoryTagsRequest {
+ // The maximum number of tags to return. The default and cap values are 10
+ // and 100, respectively.
+ optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
+ // Page number.
+ optional int32 page = 2 [(google.api.field_behavior) = OPTIONAL];
+ // The repository holding the different versions of a given content.
+ // - Format: `repositories/{repository.id}`.
+ // - Example: `repository/flaming-wombat/llama-2-7b`.
+ string parent = 3 [
+ (google.api.field_behavior) = REQUIRED,
+ (google.api.resource_reference) = {type: "api.instill.tech/Repository"}
+ ];
+}
+
+// ListRepositoryTagsResponse contains a list of container image tags.
+message ListRepositoryTagsResponse {
+ // A list of repository tags.
+ repeated RepositoryTag tags = 1;
+ // Total number of tags.
+ int32 total_size = 2;
+ // The requested page size.
+ int32 page_size = 3;
+ // The requested page offset.
+ int32 page = 4;
+}
+
+// CreateRepositoryTagRequest represents a request to add a tag to a given
+// repository.
+message CreateRepositoryTagRequest {
+ // The tag information.
+ RepositoryTag tag = 1;
+ // The repository holding the different versions of a given content.
+ // - Format: `repositories/{repository.id}`.
+ // - Example: `repository/flaming-wombat/llama-2-7b`.
+ string parent = 2 [
+ (google.api.field_behavior) = REQUIRED,
+ (google.api.resource_reference) = {type: "api.instill.tech/Repository"}
+ ];
+}
+
+// CreateRepositoryTagResponse contains the created tag.
+message CreateRepositoryTagResponse {
+ // The created tag.
+ RepositoryTag tag = 1;
+}

artifact/artifact/v1alpha/artifact_private_service.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package artifact.artifact.v1alpha;
+
+import "google/api/visibility.proto";
+import "artifact/artifact/v1alpha/artifact.proto";
+
+// ArtifactPrivateService exposes the private endpoints that allow clients to
+// manage artifacts.
+service ArtifactPrivateService {
+ // List the tags in a repository.
+ //
+ // Returns a portion of the versions that the specified repository holds.
+ rpc ListRepositoryTags(ListRepositoryTagsRequest) returns (ListRepositoryTagsResponse);
+
+ // Create a new repository tag.
+ //
+ // Adds a tag to a given repository. Note that this operation is only
+ // intended to register the information of an *already created* tag. This
+ // method should be called as part of the content push operation, right after
+ // the [PUT Manifest](https://distribution.github.io/distribution/#put-manifest) has
+ // succeeded. The distribution registry won't hold data such as the push time
+ // or the tag digest, so `artifact-backend` will hold this information locally.
+ rpc CreateRepositoryTag(CreateRepositoryTagRequest) returns (CreateRepositoryTagResponse);
+
+ option (google.api.api_visibility).restriction = "INTERNAL";
+}

artifact/artifact/v1alpha/artifact_public_service.proto

@@ -10,8 +10,6 @@ import "google/api/visibility.proto";
 // OpenAPI definition
 import "protoc-gen-openapiv2/options/annotations.proto";
 
-// Artifact
-//
 // ArtifactPublicService exposes the public endpoints that allow clients to
 // manage artifacts.
 service ArtifactPublicService {

openapiv2/artifact/service.swagger.yaml

@@ -30,3 +30,55 @@ definitions:
  items:
  type: object
  $ref: '#/definitions/protobufAny'
+ v1alphaCreateRepositoryTagResponse:
+ type: object
+ properties:
+ tag:
+ $ref: '#/definitions/v1alphaRepositoryTag'
+ description: The created tag.
+ description: CreateRepositoryTagResponse contains the created tag.
+ v1alphaListRepositoryTagsResponse:
+ type: object
+ properties:
+ tags:
+ type: array
+ items:
+ type: object
+ $ref: '#/definitions/v1alphaRepositoryTag'
+ description: A list of repository tags.
+ total_size:
+ type: integer
+ format: int32
+ description: Total number of tags.
+ page_size:
+ type: integer
+ format: int32
+ description: The requested page size.
+ page:
+ type: integer
+ format: int32
+ description: The requested page offset.
+ description: ListRepositoryTagsResponse contains a list of container image tags.
+ v1alphaRepositoryTag:
+ type: object
+ properties:
+ name:
+ type: string
+ description: |-
+ The name of the tag, defined by its parent repository and ID.
+ - Format: `repositories/{repository.id}/tags/{tag.id}`.
+ id:
+ type: string
+ description: The tag identifier.
+ digest:
+ type: string
+ description: Unique identifier, computed from the manifest the tag refers to.
+ readOnly: true
+ update_time:
+ type: string
+ format: date-time
+ description: Tag update time, i.e. timestamp of the last push.
+ readOnly: true
+ description: |-
+ RepositoryTag contains information about the version of some content in a
+ repository.

openapiv2/vdp/service.swagger.yaml

@@ -868,7 +868,7 @@ paths:
  - name: pipeline_release_name
  description: |-
  The name of the release, defined by its parent and ID.
- - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+ - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
  in: path
  required: true
  type: string
@@ -1836,7 +1836,7 @@ paths:
  - name: pipeline_release_name_1
  description: |-
  The name of the release, defined by its parent and ID.
- - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+ - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
  in: path
  required: true
  type: string
@@ -5369,9 +5369,9 @@ definitions:
  properties:
  name:
  type: string
- title: |-
+ description: |-
  The name of the release, defined by its parent and ID.
- - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+ - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
  readOnly: true
  uid:
  type: string

vdp/pipeline/v1beta/pipeline.proto

@@ -345,7 +345,7 @@ message PipelineRelease {
  };
 
  // The name of the release, defined by its parent and ID.
- // - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+ // - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
  string name = 1 [
  (google.api.field_behavior) = OUTPUT_ONLY,
  (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {