feat(kb): support namespace(owner)

config/config.yaml

@@ -22,7 +22,7 @@ database:
  host: pg-sql
  port: 5432
  name: artifact
- version: 4
+ version: 5
  timezone: Etc/UTC
  pool:
  idleconnections: 5

go.mod

@@ -15,7 +15,7 @@ require (
  github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
  github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
  github.com/influxdata/influxdb-client-go/v2 v2.12.3
- github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240612051236-4bd57e13ff0b
+ github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240620083254-e05817f35b4d
  github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61
  github.com/instill-ai/x v0.3.0-alpha.0.20231219052200-6230a89e386c
  github.com/knadh/koanf v1.5.0

go.sum

@@ -286,8 +286,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
 github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240612051236-4bd57e13ff0b h1:YsH/M29+hdGLU5GS6Ty/JcLY1RMnEPT2lhfE43JUR5E=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240612051236-4bd57e13ff0b/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240620083254-e05817f35b4d h1:gJqn9yDo+1vyL6I9In667Y8LM0iVRg0ExH7Kd97OZvw=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240620083254-e05817f35b4d/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
 github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61 h1:smPTvmXDhn/QC7y/TPXyMTqbbRd0gvzmFgWBChwTfhE=
 github.com/instill-ai/usage-client v0.3.0-alpha.0.20240319060111-4a3a39f2fd61/go.mod h1:/TAHs4ybuylk5icuy+MQtHRc4XUnIyXzeNKxX9qDFhw=
 github.com/instill-ai/x v0.3.0-alpha.0.20231219052200-6230a89e386c h1:a2RVkpIV2QcrGnSHAou+t/L+vBsaIfFvk5inVg5Uh4s=

pkg/db/migration/000005_add_creator_column_to_knowledge_base_tables.down.sql

@@ -0,0 +1,6 @@
+BEGIN;
+-- Remove the comment on the creator_uid column
+COMMENT ON COLUMN knowledge_base.creator_uid IS NULL;
+-- Drop the creator_uid column from the knowledge_base table
+ALTER TABLE knowledge_base DROP COLUMN creator_uid;
+COMMIT;

pkg/db/migration/000005_add_creator_column_to_knowledge_base_tables.up.sql

@@ -0,0 +1,9 @@
+BEGIN;
+-- Add the new column to the knowledge_base table
+ALTER TABLE knowledge_base
+ADD COLUMN creator_uid UUID NOT NULL;
+
+-- Add comment for the new column
+COMMENT ON COLUMN knowledge_base.creator_uid IS 'UUID of the creator of the knowledge base';
+
+COMMIT;

pkg/handler/knowledgebase.go

@@ -6,10 +6,13 @@
  "regexp"
  "strings"
 
+ "go.uber.org/zap"
  "google.golang.org/grpc/metadata"
 
+ "github.com/google/uuid"
  "github.com/instill-ai/artifact-backend/pkg/constant"
  "github.com/instill-ai/artifact-backend/pkg/customerror"
+ "github.com/instill-ai/artifact-backend/pkg/logger"
  "github.com/instill-ai/artifact-backend/pkg/repository"
  artifactpb "github.com/instill-ai/protogen-go/artifact/artifact/v1alpha"
 )
@@ -22,147 +25,226 @@
 const ErrorDeleteKnowledgeBaseMsg = "failed to delete knowledge base: %w"
 
 func (ph *PublicHandler) CreateKnowledgeBase(ctx context.Context, req *artifactpb.CreateKnowledgeBaseRequest) (*artifactpb.CreateKnowledgeBaseResponse, error) {
-
- uid, err := getUserUIDFromContext(ctx)
+ log, _ := logger.GetZapLogger(ctx)
+ authUid, err := getUserUIDFromContext(ctx)
  if err != nil {
  err := fmt.Errorf("failed to get user id from header: %v. err: %w", err, customerror.ErrUnauthenticated)
  return nil, err
  }
+
+ // TODO: check user's permission to create knowledge base in the user or org context
+ // ....
+
  // check name if it is empty
  if req.Name == "" {
  err := fmt.Errorf("name is required. err: %w", ErrCheckRequiredFields)
  return nil, err
  }
  nameOk := isValidName(req.Name)
  if !nameOk {
- msg := "name is invalid: %v. err: %w"
+ msg := "kb name is invalid: %v. err: %w"
  return nil, fmt.Errorf(msg, req.Name, customerror.ErrInvalidArgument)
  }
 
+ // get the owner uid from the mgmt service
+ var ownerUUID string
+ {
+ // get the owner uid from the mgmt service
+ ownerUUID, err = ph.getOwnerUID(ctx, req.OwnerId)
+ if err != nil {
+ log.Error("failed to get owner uid", zap.Error(err))
+ return nil, err
+ }
+ }
+
+ creatorUUID, err := uuid.Parse(authUid)
+ if err != nil {
+ log.Error("failed to parse creator uid", zap.String("uid", authUid), zap.Error(err))
+ return nil, err
+ }
+
  dbData, err := ph.service.Repository.CreateKnowledgeBase(ctx,
  repository.KnowledgeBase{
- Name: req.Name,
- ID: toIDStyle(req.Name),
+ Name: req.Name,
+ // make name as kbID
+ KbID: req.Name,
  Description: req.Description,
  Tags: req.Tags,
- Owner: uid,
+ Owner: ownerUUID,
+ CreatorUID: creatorUUID,
  },
  )
  if err != nil {
  return nil, err
  }
-
  // TODO: ACL - set the owner of the knowledge base
  // ....
-
  return &artifactpb.CreateKnowledgeBaseResponse{
- Body: &artifactpb.KnowledgeBase{
- Name: dbData.Name,
- Id: dbData.ID,
- Description: dbData.Description,
- Tags: dbData.Tags,
- OwnerName: dbData.Owner,
- CreateTime: dbData.CreateTime.String(),
- UpdateTime: dbData.UpdateTime.String(),
- }, ErrorMsg: "", StatusCode: 0,
+ KnowledgeBase: &artifactpb.KnowledgeBase{
+ Name: dbData.Name,
+ KbId: dbData.KbID,
+ Description: dbData.Description,
+ Tags: dbData.Tags,
+ OwnerName: dbData.Owner,
+ CreateTime: dbData.CreateTime.String(),
+ UpdateTime: dbData.UpdateTime.String(),
+ ConvertingPipelines: []string{"leo/fake-pipeline-1", "leo/fake-pipeline-2"},
+ SplittingPipelines: []string{"leo/fake-pipeline-3", "leo/fake-pipeline-4"},
+ EmbeddingPipelines: []string{"leo/fake-pipeline-5", "leo/fake-pipeline-6"},
+ // DownstreamApps: []string{"leo/fake-app-1", "leo/fake-app-2"},
+ },
  }, nil
 }
-func (ph *PublicHandler) ListKnowledgeBases(ctx context.Context, _ *artifactpb.ListKnowledgeBasesRequest) (*artifactpb.ListKnowledgeBasesResponse, error) {
 
+func (ph *PublicHandler) ListKnowledgeBases(ctx context.Context, req *artifactpb.ListKnowledgeBasesRequest) (*artifactpb.ListKnowledgeBasesResponse, error) {
+ log, _ := logger.GetZapLogger(ctx)
  // get user id from context
- uid, err := getUserUIDFromContext(ctx)
+ _, err := getUserUIDFromContext(ctx)
  if err != nil {
 
  return nil, fmt.Errorf(ErrorListKnowledgeBasesMsg, err)
  }
 
- // TODO: ACL - check user's permission to list knowledge bases
+ // get the owner uid from the mgmt service
+ var ownerUUID string
+ {
+ // get the owner uid from the mgmt service
+ ownerUUID, err = ph.getOwnerUID(ctx, req.OwnerId)
+ if err != nil {
+ log.Error("failed to get owner uid", zap.Error(err))
+ return nil, err
+ }
+ }
+
+ // TODO: ACL - check user(authUid)'s permission to list knowledge bases
+ // ....
 
- dbData, err := ph.service.Repository.ListKnowledgeBases(ctx, uid)
+ dbData, err := ph.service.Repository.ListKnowledgeBases(ctx, ownerUUID)
  if err != nil {
+ log.Error("failed to get knowledge bases", zap.Error(err))
  return nil, fmt.Errorf(ErrorListKnowledgeBasesMsg, err)
  }
 
  kbs := make([]*artifactpb.KnowledgeBase, len(dbData))
  for i, kb := range dbData {
  kbs[i] = &artifactpb.KnowledgeBase{
- Name: kb.Name,
- Id: kb.ID,
- Description: kb.Description,
- Tags: kb.Tags,
- CreateTime: kb.CreateTime.String(),
- UpdateTime: kb.UpdateTime.String(),
- OwnerName: kb.Owner,
+ Name: kb.Name,
+ KbId: kb.KbID,
+ Description: kb.Description,
+ Tags: kb.Tags,
+ CreateTime: kb.CreateTime.String(),
+ UpdateTime: kb.UpdateTime.String(),
+ OwnerName: kb.Owner,
+ ConvertingPipelines: []string{"leo/fake-pipeline-1", "leo/fake-pipeline-2"},
+ SplittingPipelines: []string{"leo/fake-pipeline-3", "leo/fake-pipeline-4"},
+ EmbeddingPipelines: []string{"leo/fake-pipeline-5", "leo/fake-pipeline-6"},
+ // DownstreamApps: []string{"leo/fake-app-1", "leo/fake-app-2"},
  }
  }
  return &artifactpb.ListKnowledgeBasesResponse{
- Body: &artifactpb.KnowledgeBasesList{
- KnowledgeBases: kbs,
- },
- ErrorMsg: "", StatusCode: 0,
+ KnowledgeBases: kbs,
  }, nil
 }
 func (ph *PublicHandler) UpdateKnowledgeBase(ctx context.Context, req *artifactpb.UpdateKnowledgeBaseRequest) (*artifactpb.UpdateKnowledgeBaseResponse, error) {
- uid, err := getUserUIDFromContext(ctx)
+ log, _ := logger.GetZapLogger(ctx)
+ authUID, err := getUserUIDFromContext(ctx)
  if err != nil {
+ log.Error("failed to get user id from header", zap.Error(err))
  return nil, err
  }
  // check name if it is empty
- if req.Name == "" {
- return nil, fmt.Errorf("name is empty. err: %w", ErrCheckRequiredFields)
+ if req.KbId == "" {
+ log.Error("kb_id is empty", zap.Error(ErrCheckRequiredFields))
+ return nil, fmt.Errorf("kb_id is empty. err: %w", ErrCheckRequiredFields)
  }
- nameOk := isValidName(req.Name)
- if !nameOk {
- return nil, fmt.Errorf("name: %s is invalid. err: %w", req.Name, customerror.ErrInvalidArgument)
+
+ // get the owner uid from the mgmt service
+ var ownerUUID string
+ {
+ // get the owner uid from the mgmt service
+ ownerUUID, err = ph.getOwnerUID(ctx, req.OwnerId)
+ if err != nil {
+ log.Error("failed to get owner uid", zap.Error(err))
+ return nil, err
+ }
  }
 
  // TODO: ACL - check user's permission to update knowledge base
-
+ _ = authUID
  // check if knowledge base exists
  dbData, err := ph.service.Repository.UpdateKnowledgeBase(
  ctx,
- uid,
+ ownerUUID,
  repository.KnowledgeBase{
- Name: req.Name,
- ID:  req.Id,
+ // Name: req.KbId,
+ KbID: req.KbId,
  Description: req.Description,
  Tags: req.Tags,
- Owner: uid,
+ Owner: ownerUUID,
  },
  )
  if err != nil {
+ log.Error("failed to update knowledge base", zap.Error(err))
  return nil, err
  }
  // populate response
  return &artifactpb.UpdateKnowledgeBaseResponse{
- Body: &artifactpb.KnowledgeBase{
- Name: dbData.Name,
- Id: dbData.ID,
- Description: dbData.Description,
- Tags: dbData.Tags,
- CreateTime: dbData.CreateTime.String(),
- UpdateTime: dbData.UpdateTime.String(),
- OwnerName: dbData.Owner,
- }, ErrorMsg: "", StatusCode: 0,
+ KnowledgeBase: &artifactpb.KnowledgeBase{
+ Name: dbData.Name,
+ KbId: dbData.KbID,
+ Description: dbData.Description,
+ Tags: dbData.Tags,
+ CreateTime: dbData.CreateTime.String(),
+ UpdateTime: dbData.UpdateTime.String(),
+ OwnerName: dbData.Owner,
+ ConvertingPipelines: []string{"leo/fake-pipeline-1", "leo/fake-pipeline-2"},
+ SplittingPipelines: []string{"leo/fake-pipeline-3", "leo/fake-pipeline-4"},
+ EmbeddingPipelines: []string{"leo/fake-pipeline-5", "leo/fake-pipeline-6"},
+ // DownstreamApps: []string{"leo/fake-app-1", "leo/fake-app-2"},
+ },
  }, nil
 }
 func (ph *PublicHandler) DeleteKnowledgeBase(ctx context.Context, req *artifactpb.DeleteKnowledgeBaseRequest) (*artifactpb.DeleteKnowledgeBaseResponse, error) {
-
- uid, err := getUserUIDFromContext(ctx)
+ log, _ := logger.GetZapLogger(ctx)
+ authUID, err := getUserUIDFromContext(ctx)
  if err != nil {
 
  return nil, err
  }
-
+ // get the owner uid from the mgmt service
+ var ownerUUID string
+ {
+ // get the owner uid from the mgmt service
+ ownerUUID, err = ph.getOwnerUID(ctx, req.OwnerId)
+ if err != nil {
+ log.Error("failed to get owner uid", zap.Error(err))
+ return nil, err
+ }
+ }
  // TODO: ACL - check user's permission to delete knowledge base
+ _ = authUID
 
- err = ph.service.Repository.DeleteKnowledgeBase(ctx, uid, req.Id)
+ deletedKb, err := ph.service.Repository.DeleteKnowledgeBase(ctx, ownerUUID, req.KbId)
  if err != nil {
 
  return nil, err
  }
+ // populate response
+
  return &artifactpb.DeleteKnowledgeBaseResponse{
- ErrorMsg: "", StatusCode: 0,
+ KnowledgeBase: &artifactpb.KnowledgeBase{
+ Name: deletedKb.Name,
+ KbId: deletedKb.KbID,
+ Description: deletedKb.Description,
+ Tags: deletedKb.Tags,
+ CreateTime: deletedKb.CreateTime.String(),
+ UpdateTime: deletedKb.UpdateTime.String(),
+ OwnerName: deletedKb.Owner,
+ ConvertingPipelines: []string{"leo/fake-pipeline-1", "leo/fake-pipeline-2"},
+ SplittingPipelines: []string{"leo/fake-pipeline-3", "leo/fake-pipeline-4"},
+ EmbeddingPipelines: []string{"leo/fake-pipeline-5", "leo/fake-pipeline-6"},
+ // DownstreamApps: []string{"leo/fake-app-1", "leo/fake-app-2"}
+ },
  }, nil
 }
 func getUserUIDFromContext(ctx context.Context) (string, error) {
@@ -173,25 +255,14 @@
  return "", fmt.Errorf("user id not found in context. err: %w", customerror.ErrUnauthenticated)
 }
 
+// The ID should be lowercase without any space or special character besides
+// the hyphen, it can not start with number or hyphen, and should be less
+// than 32 characters.
 func isValidName(name string) bool {
- name = strings.ToLower(name) // Convert the name to lowercase for case-insensitive matching
  // Define the regular expression pattern
- pattern := `^[a-z0-9_-]+$`
+ pattern := `^[a-z]([a-z-]{0,30}[a-z])?$`
  // Compile the regular expression
  re := regexp.MustCompile(pattern)
  // Match the name against the regular expression
  return re.MatchString(name)
 }
-
-// toIDStyle converts a name to an ID style by replacing spaces with underscores
-// and ensuring it only contains lowercase letters, underscores, and hyphens.
-func toIDStyle(name string) string {
-
- // Replace spaces with underscores
- id := strings.ReplaceAll(name, " ", "_")
-
- // Convert to lowercase
- id = strings.ToLower(id)
-
- return id
-}

pkg/logger/logger.go

@@ -97,7 +97,7 @@ func GetZapLogger(ctx context.Context) (*zap.Logger, error) {
  return nil
  }),
  zap.AddCaller(),
- zap.AddStacktrace(zapcore.ErrorLevel),
+ // zap.AddStacktrace(zapcore.ErrorLevel),
  )
 
  return logger, err