Merge branch 'main' of github.com:bytebase/bytebase into chore/conflict

# Conflicts:
#	backend/api/v1/database_service.go
#	backend/plugin/parser/pg/completion.go
#	backend/runner/taskrun/executor.go
#	frontend/pnpm-lock.yaml
#	frontend/src/types/proto/google/protobuf/descriptor.ts
#	frontend/src/types/proto/store/common.ts
#	frontend/src/types/proto/v1/common.ts
#	frontend/src/types/sql-review-schema.yaml
#	go.sum
#	proto/generated-go/store/activity.pb.go
#	proto/generated-go/store/approval.pb.go
#	proto/generated-go/store/audit_log.pb.go
#	proto/generated-go/store/branch.pb.go
#	proto/generated-go/store/changelist.pb.go
#	proto/generated-go/store/common.pb.go
#	proto/generated-go/store/data_source.pb.go
#	proto/generated-go/store/database.pb.go
#	proto/generated-go/store/export_archive.pb.go
#	proto/generated-go/store/idp.pb.go
#	proto/generated-go/store/instance.pb.go
#	proto/generated-go/store/instance_change_history.pb.go
#	proto/generated-go/store/issue.pb.go
#	proto/generated-go/store/issue_comment.pb.go
#	proto/generated-go/store/plan.pb.go
#	proto/generated-go/store/plan_check_run.pb.go
#	proto/generated-go/store/policy.pb.go
#	proto/generated-go/store/project.pb.go
#	proto/generated-go/store/query_history.pb.go
#	proto/generated-go/store/role.pb.go
#	proto/generated-go/store/setting.pb.go
#	proto/generated-go/store/sheet.pb.go
#	proto/generated-go/store/slow_query.pb.go
#	proto/generated-go/store/task_run.pb.go
#	proto/generated-go/store/user.pb.go
#	proto/generated-go/store/vcs.pb.go
#	proto/generated-go/v1/actuator_service.pb.go
#	proto/generated-go/v1/anomaly_service.pb.go
#	proto/generated-go/v1/audit_log_service.pb.go
#	proto/generated-go/v1/auth_service.pb.go
#	proto/generated-go/v1/branch_service.pb.go
#	proto/generated-go/v1/cel_service.pb.go
#	proto/generated-go/v1/changelist_service.pb.go
#	proto/generated-go/v1/common.pb.go
#	proto/generated-go/v1/database_service.pb.go
#	proto/generated-go/v1/environment_service.pb.go
#	proto/generated-go/v1/iam_policy.pb.go
#	proto/generated-go/v1/idp_service.pb.go
#	proto/generated-go/v1/instance_role_service.pb.go
#	proto/generated-go/v1/instance_service.pb.go
#	proto/generated-go/v1/issue_service.pb.go
#	proto/generated-go/v1/logging_service.pb.go
#	proto/generated-go/v1/org_policy_service.pb.go
#	proto/generated-go/v1/project_service.pb.go
#	proto/generated-go/v1/risk_service.pb.go
#	proto/generated-go/v1/role_service.pb.go
#	proto/generated-go/v1/rollout_service.pb.go
#	proto/generated-go/v1/setting_service.pb.go
#	proto/generated-go/v1/sheet_service.pb.go
#	proto/generated-go/v1/sql_service.pb.go
#	proto/generated-go/v1/subscription_service.pb.go
#	proto/generated-go/v1/vcs_connector_service.pb.go
#	proto/generated-go/v1/vcs_provider_service.pb.go
#	proto/generated-go/v1/worksheet_service.pb.go

.github/workflows/backend-tests.yml

@@ -21,7 +21,7 @@ jobs:
  - uses: actions/checkout@v4
  - uses: actions/setup-go@v5
  with:
- go-version: 1.21.6
+ go-version: 1.22.3
  cache: false
  - name: Verify go.mod is tidy
  run: |
@@ -37,7 +37,7 @@ jobs:
  - name: Install dependencies
  run: go generate -tags mysql ./...
  - name: golangci-lint
- uses: golangci/golangci-lint-action@v4
+ uses: golangci/golangci-lint-action@v5
  with:
  version: v1.57.2
  args: -j 8 --verbose --timeout 20m --max-same-issues=30 --allow-parallel-runners

.github/workflows/build-artifacts-and-draft-release.yml

@@ -6,52 +6,10 @@ on:
  - "*.*.*"
 
 jobs:
- build-linux-binary:
- runs-on: self-hosted
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-go@v5
- with:
- go-version: 1.21.6
- - uses: pnpm/[email protected]
- with:
- version: 9
- - uses: actions/setup-node@v4
- with:
- node-version: "20.12.2"
- cache: pnpm
- cache-dependency-path: "frontend/pnpm-lock.yaml"
- - run: pnpm install --frozen-lockfile
- working-directory: frontend
- - run: pnpm release
- working-directory: frontend
- - name: Build
- uses: goreleaser/goreleaser-action@v5
- with:
- args: release --skip-publish --config scripts/.goreleaser-for-linux.yaml
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Upload
- uses: actions/upload-artifact@v4
- with:
- name: bytebase-linux
- path: dist/bytebase*
  draft-release:
- needs: [build-linux-binary]
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v4
- - uses: actions/setup-go@v5
- with:
- go-version: 1.21.6
- - name: Make directories
- run: |
- mkdir -p ./bytebase-build/linux
- - name: Download linux binaries
- uses: actions/download-artifact@v4
- with:
- name: bytebase-linux
- path: ./bytebase-build/linux
  - name: Release
  uses: goreleaser/goreleaser-action@v5
  with:

.github/workflows/test_link.yml

@@ -16,7 +16,7 @@ jobs:
  - uses: actions/checkout@v4
  - uses: actions/setup-go@v5
  with:
- go-version: 1.21.6
+ go-version: 1.22.3
  cache: false
  - name: Validate links
  run: go test -timeout 600s -v ./scripts/... | tee test.log; exit ${PIPESTATUS[0]}

README.md

@@ -257,7 +257,7 @@ Below diagram describes a typical mapping between an engineering org and the cor
 
 ### Prerequisites
 
-- [Go](https://golang.org/doc/install) (1.21.6 or later)
+- [Go](https://golang.org/doc/install) (1.22.3 or later)
 - [pnpm](https://pnpm.io/installation)
 - [Air](https://github.com/bytebase/air) (**our forked repo @87187cc with the proper signal handling**). This is for backend live reload.
  ```bash

backend/api/gitops/gitops.go

@@ -17,6 +17,7 @@ type Service struct {
  activityManager *activity.Manager
  stateCfg *state.State
  licenseService enterprise.LicenseService
+ planService *v1pb.PlanService
  rolloutService *v1pb.RolloutService
  issueService *v1pb.IssueService
 }
@@ -28,6 +29,7 @@ func NewService(
  activityManager *activity.Manager,
  stateCfg *state.State,
  licenseService enterprise.LicenseService,
+ planService *v1pb.PlanService,
  rolloutService *v1pb.RolloutService,
  issueService *v1pb.IssueService,
 ) *Service {
@@ -37,6 +39,7 @@ func NewService(
  activityManager: activityManager,
  stateCfg: stateCfg,
  licenseService: licenseService,
+ planService: planService,
  rolloutService: rolloutService,
  issueService: issueService,
  }

backend/api/gitops/webhook.go

@@ -19,12 +19,16 @@ import (
  "github.com/labstack/echo/v4"
  "github.com/pkg/errors"
 
+ "github.com/bytebase/bytebase/backend/utils"
+
  "github.com/bytebase/bytebase/backend/common"
  "github.com/bytebase/bytebase/backend/common/log"
  "github.com/bytebase/bytebase/backend/component/activity"
  api "github.com/bytebase/bytebase/backend/legacyapi"
  "github.com/bytebase/bytebase/backend/plugin/vcs"
 
+ sc "github.com/bytebase/bytebase/backend/component/sheet"
+
  "github.com/bytebase/bytebase/backend/store"
  storepb "github.com/bytebase/bytebase/proto/generated-go/store"
  v1pb "github.com/bytebase/bytebase/proto/generated-go/v1"
@@ -182,21 +186,32 @@ func validateGitHubWebhookSignature256(signature, key string, body []byte) (bool
 
 func (s *Service) createIssueFromPRInfo(ctx context.Context, project *store.ProjectMessage, vcsProvider *store.VCSProviderMessage, vcsConnector *store.VCSConnectorMessage, prInfo *pullRequestInfo) (*v1pb.Issue, error) {
  creatorID := api.SystemBotID
+ creatorName := common.FormatUserUID(api.SystemBotID)
  user, err := s.store.GetUser(ctx, &store.FindUserMessage{Email: &prInfo.email})
  if err != nil {
  slog.Error("failed to find user by email", slog.String("email", prInfo.email), log.BBError(err))
  }
  if user != nil {
  creatorID = user.ID
+ creatorName = common.FormatUserUID(user.ID)
+ }
+
+ engine, err := s.getDatabaseEngineSample(ctx, project, vcsConnector)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get database engine")
  }
 
  var sheets []int
  for _, change := range prInfo.changes {
- sheet, err := s.store.CreateSheet(ctx, &store.SheetMessage{
+ sheet, err := sc.CreateSheet(ctx, s.store, &store.SheetMessage{
  CreatorID: creatorID,
  ProjectUID: project.UID,
  Title: change.path,
  Statement: change.content,
+
+ Payload: &storepb.SheetPayload{
+ Engine: engine,
+ },
  })
  if err != nil {
  return nil, errors.Wrapf(err, "failed to create sheet for file %s", change.path)
@@ -212,7 +227,7 @@ func (s *Service) createIssueFromPRInfo(ctx context.Context, project *store.Proj
  childCtx := context.WithValue(ctx, common.PrincipalIDContextKey, creatorID)
  childCtx = context.WithValue(childCtx, common.UserContextKey, user)
  childCtx = context.WithValue(childCtx, common.LoopbackContextKey, true)
- plan, err := s.rolloutService.CreatePlan(childCtx, &v1pb.CreatePlanRequest{
+ plan, err := s.planService.CreatePlan(childCtx, &v1pb.CreatePlanRequest{
  Parent: fmt.Sprintf("projects/%s", project.ResourceID),
  Plan: &v1pb.Plan{
  Title: prInfo.title,
@@ -265,6 +280,19 @@ func (s *Service) createIssueFromPRInfo(ctx context.Context, project *store.Proj
  return nil, errors.Wrapf(err, "failed to create activity payload")
  }
 
+ if err := s.store.CreateAuditLog(ctx, &storepb.AuditLog{
+ Parent: project.GetName(),
+ Method: store.AuditLogMethodProjectRepositoryPush.String(),
+ Resource: issue.Name,
+ User: creatorName,
+ Severity: storepb.AuditLog_INFO,
+ Request: "",
+ Response: "",
+ Status: nil,
+ }); err != nil {
+ slog.Warn("failed to create audit log after creating issue from push event", "issueUID", issueUID)
+ }
+
  activityCreate := &store.ActivityMessage{
  CreatorUID: creatorID,
  ResourceContainer: project.GetName(),
@@ -280,6 +308,64 @@ func (s *Service) createIssueFromPRInfo(ctx context.Context, project *store.Proj
  return issue, nil
 }
 
+func (s *Service) getDatabaseEngineSample(
+ ctx context.Context,
+ project *store.ProjectMessage,
+ vcsConnector *store.VCSConnectorMessage,
+) (storepb.Engine, error) {
+ sample, err := func() (*store.DatabaseMessage, error) {
+ if dbg := vcsConnector.Payload.GetDatabaseGroup(); dbg != "" {
+ projectID, databaseGroupID, err := common.GetProjectIDDatabaseGroupID(dbg)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get project id and database group id from %q", dbg)
+ }
+ if projectID != project.ResourceID {
+ return nil, errors.Errorf("project id %q in databaseGroup %q does not match project id %q in plan config", projectID, dbg, project.ResourceID)
+ }
+ databaseGroup, err := s.store.GetDatabaseGroup(ctx, &store.FindDatabaseGroupMessage{ProjectUID: &project.UID, ResourceID: &databaseGroupID})
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get database group %q", databaseGroupID)
+ }
+ if databaseGroup == nil {
+ return nil, errors.Errorf("database group %q not found", databaseGroupID)
+ }
+ allDatabases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &project.ResourceID})
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to list databases for project %q", project.ResourceID)
+ }
+
+ matchedDatabases, _, err := utils.GetMatchedAndUnmatchedDatabasesInDatabaseGroup(ctx, databaseGroup, allDatabases)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to get matched and unmatched databases in database group %q", databaseGroupID)
+ }
+ if len(matchedDatabases) == 0 {
+ return nil, errors.Errorf("no matched databases found in database group %q", databaseGroupID)
+ }
+ return matchedDatabases[0], nil
+ }
+ allDatabases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{ProjectID: &project.ResourceID})
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to list databases for project %q", project.ResourceID)
+ }
+ if len(allDatabases) == 0 {
+ return nil, errors.Errorf("no database in the project %q", project.ResourceID)
+ }
+ return allDatabases[0], nil
+ }()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to get sample database")
+ }
+
+ instance, err := s.store.GetInstanceV2(ctx, &store.FindInstanceMessage{ResourceID: &sample.InstanceID})
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to get instance")
+ }
+ if instance == nil {
+ return 0, errors.Errorf("instance not found")
+ }
+ return instance.Engine, nil
+}
+
 func (s *Service) getChangeSteps(
  ctx context.Context,
  project *store.ProjectMessage,

backend/api/lsp/completion.go

@@ -15,6 +15,11 @@ import (
  storepb "github.com/bytebase/bytebase/proto/generated-go/store"
 )
 
+const (
+ // 1MB.
+ contentLengthLimit = 1024 * 1024
+)
+
 func newEmptyCompletionList() *lsp.CompletionList {
  return &lsp.CompletionList{
  IsIncomplete: false,
@@ -33,6 +38,10 @@ func (h *Handler) handleTextDocumentCompletion(ctx context.Context, _ *jsonrpc2.
  if err != nil {
  return nil, err
  }
+ if len(content) > contentLengthLimit {
+ // We don't want to parse a huge file.
+ return newEmptyCompletionList(), nil
+ }
  _, valid, why := offsetForPosition(content, params.Position)
  if !valid {
  return nil, errors.Errorf("invalid position %d:%d (%s)", params.Position.Line, params.Position.Character, why)
@@ -45,7 +54,8 @@ func (h *Handler) handleTextDocumentCompletion(ctx context.Context, _ *jsonrpc2.
  // Nothing.
  case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT, storepb.Engine_RISINGWAVE, storepb.Engine_GAUSSDB:
  // Nothing.
- case storepb.Engine_ORACLE, storepb.Engine_DM, storepb.Engine_OCEANBASE_ORACLE, storepb.Engine_SNOWFLAKE, storepb.Engine_MSSQL:
+ case storepb.Engine_MSSQL:
+ case storepb.Engine_ORACLE, storepb.Engine_DM, storepb.Engine_OCEANBASE_ORACLE, storepb.Engine_SNOWFLAKE:
  default:
  slog.Debug("Engine is not supported", slog.String("engine", engine.String()))
  return newEmptyCompletionList(), nil

backend/api/v1/acl_config.go

@@ -124,16 +124,17 @@ var methodPermissionMap = map[string]iam.Permission{
  v1pb.VCSProviderService_DeleteVCSProvider_FullMethodName: iam.PermissionVCSProvidersDelete,
  v1pb.VCSProviderService_SearchVCSProviderRepositories_FullMethodName: iam.PermissionVCSProvidersSearchProjects,
  v1pb.VCSProviderService_ListVCSConnectorsInProvider_FullMethodName: iam.PermissionVCSProvidersListProjects,
- v1pb.RolloutService_ListPlans_FullMethodName: iam.PermissionPlansList,
- v1pb.RolloutService_GetPlan_FullMethodName: iam.PermissionPlansGet,
- v1pb.RolloutService_CreatePlan_FullMethodName: iam.PermissionPlansCreate,
- v1pb.RolloutService_UpdatePlan_FullMethodName: iam.PermissionPlansUpdate,
  v1pb.RolloutService_GetRollout_FullMethodName: iam.PermissionRolloutsGet,
  v1pb.RolloutService_CreateRollout_FullMethodName: iam.PermissionRolloutsCreate,
  v1pb.RolloutService_PreviewRollout_FullMethodName: iam.PermissionRolloutsPreview,
  v1pb.RolloutService_ListTaskRuns_FullMethodName: iam.PermissionTaskRunsList,
- v1pb.RolloutService_ListPlanCheckRuns_FullMethodName: iam.PermissionPlanCheckRunsList,
- v1pb.RolloutService_RunPlanChecks_FullMethodName: iam.PermissionPlanCheckRunsRun,
+ v1pb.RolloutService_GetTaskRunLog_FullMethodName: iam.PermissionTaskRunsList,
+ v1pb.PlanService_ListPlans_FullMethodName: iam.PermissionPlansList,
+ v1pb.PlanService_GetPlan_FullMethodName: iam.PermissionPlansGet,
+ v1pb.PlanService_CreatePlan_FullMethodName: iam.PermissionPlansCreate,
+ v1pb.PlanService_UpdatePlan_FullMethodName: iam.PermissionPlansUpdate,
+ v1pb.PlanService_ListPlanCheckRuns_FullMethodName: iam.PermissionPlanCheckRunsList,
+ v1pb.PlanService_RunPlanChecks_FullMethodName: iam.PermissionPlanCheckRunsRun,
  v1pb.SettingService_ListSettings_FullMethodName: iam.PermissionSettingsList,
  v1pb.SettingService_GetSetting_FullMethodName: iam.PermissionSettingsGet,
  v1pb.SettingService_UpdateSetting_FullMethodName: iam.PermissionSettingsSet,

backend/api/v1/acl_iam.go

@@ -118,7 +118,7 @@ func isSkippedMethod(fullMethod string) bool {
  return true
  // handled in the method because we need to consider plan.Creator.
  case
- v1pb.RolloutService_UpdatePlan_FullMethodName:
+ v1pb.PlanService_UpdatePlan_FullMethodName:
  return true
  // handled in the method because we need to consider issue.Creator and issue type.
  // additional bb.plans.action and bb.rollouts.action permissions are required if the issue type is change database.
@@ -146,6 +146,8 @@ func isSkippedMethod(fullMethod string) bool {
  return true
  // handled in the method because checking is complex.
  case
+ v1pb.AuditLogService_SearchAuditLogs_FullMethodName,
+ v1pb.AuditLogService_ExportAuditLogs_FullMethodName,
  v1pb.InstanceService_SearchInstances_FullMethodName,
  v1pb.DatabaseService_ListSlowQueries_FullMethodName,
  v1pb.DatabaseService_ListDatabases_FullMethodName,
@@ -155,7 +157,7 @@ func isSkippedMethod(fullMethod string) bool {
  v1pb.ProjectService_ListDatabaseGroups_FullMethodName,
  v1pb.ProjectService_SearchProjects_FullMethodName,
  v1pb.ChangelistService_ListChangelists_FullMethodName,
- v1pb.RolloutService_ListPlans_FullMethodName,
+ v1pb.PlanService_ListPlans_FullMethodName,
  v1pb.ProjectService_ListSchemaGroups_FullMethodName:
  return true
  }