improve README and add fixes

README.md

@@ -1,16 +1,6 @@
 # Gitlab to Google SCIM
 
-To install dependencies:
-
-```bash
-bun install
-```
-
-To run:
-
-```bash
-bun run index.ts
-```
+Synchronizes your google users with gitlab users via SCIM. It supports mapping of different privileges for different groups & users.
 
 
 ## Configuration
@@ -19,7 +9,6 @@ You need a few items of configuration. One side from Gitlab, and the other from
 You will need the files produced by these steps for AWS Lambda deployment as well as locally running the sync tool.
 This how-to assumes you have Gitlab SSO configured and a Google SAML app to log in into Gitlab.
 
-
 ### Google
 
 First, you have to setup your API. In the project you want to use go to the Console and select API & Services > Enable APIs and Services. Search for Admin SDK and Enable the API.
@@ -73,7 +62,24 @@ To get an API token, [create a Group Access Token](https://docs.gitlab.com/ee/us
 | GITLAB_API_TOKEN_SECRET | no | AWS Secret name to retrieve the API token from |
 | GITLAB_API_TOKEN_FILE | no | Filepath to retrieve the API token from |
 | GITLAB_API_TOKEN | no | API token |
+| DEFAULT_MEMBERSHIP_ROLE | no | Default gitlab role. Defaults to Minimal Access |
 | ROLE_MAPPINGS_SECRET | no | AWS Secret name to retrieve the gitlab role mappings from |
 | ROLE_MAPPINGS_FILE | no | Filepath to retrieve the gitlab role mappings from |
 | ROLE_MAPPINGS | no | Role mappings for gitlab |
 | SLACK_WEBHOOK_URL | no | Slack Webhook url to send notifications to |
+| LOG_LEVEL | no | Level of logs to print. Defaults to info |
+| DRY_RUN | no | Whether to only retrieve information, not create anything. Defaults to false |
+
+# Development
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```

index.ts

@@ -6,20 +6,24 @@ import { GitlabScim } from './src/gitlab/scim';
 import { GitlabApi } from './src/gitlab/api';
 import { Google } from './src/google';
 import { logger } from "./src/utils/logging";
-import { PrivilegeMap, loadMappings } from "./src/utils/mappings";
+import { PrivilegeMap, getGroupPrivilege, getUserCustomMembership, loadMappings } from "./src/utils/mappings";
 import { Slack } from './src/utils/slack';
 
 
 const GOOGLE_GROUP_FILTER = process.env.GOOGLE_GROUP_FILTER || "*";
 const DEFAULT_MEMBERSHIP_ROLE: GitlabRole = GitlabRoleMapping[process.env.DEFAULT_MEMBERSHIP_ROLE || 'Minimal Access']
-const DRY_RUN = (process.env.DRY_RUN || "0") !== "0"
+const DRY_RUN = (process.env.DRY_RUN || "false") !== "false"
 const GITLAB_GROUP = process.env.GITLAB_GROUP!
 
 const google = new Google()
 await google.initialize()
 const gitlabScim = new GitlabScim()
 const gitlabApi = new GitlabApi()
-const slack = new Slack()
+let slack: Slack | null = null;
+
+if (process.env.SLACK_WEBHOOK_URL !== undefined) {
+ slack = new Slack()
+}
 
 
 async function getGoogleUserGroups(): Promise<{ [key: string]: string[] }> {
@@ -44,30 +48,6 @@ async function getGoogleUserGroups(): Promise<{ [key: string]: string[] }> {
  return userGroups
 }
 
-function getUserCustomMembership(email: string, groups: string[], mappings: PrivilegeMap): { [key: string]: GitlabRole } {
- logger.debug(`retrieve membership of user ${email}`)
-
- const membership: { [key: string]: GitlabRole } = {}
-
- if (mappings.users.hasOwnProperty(email)) {
- for (var gitlabGroup of Object.keys(mappings.users[email])) {
- membership[gitlabGroup] = GitlabRoleMapping[mappings.users[email][gitlabGroup]]
- }
- } else {
- for (const userGroup of groups) {
- if (mappings.groups[userGroup] !== undefined) {
- for (var gitlabGroup of Object.keys(mappings.groups[userGroup])) {
- if (!membership.hasOwnProperty(gitlabGroup)) {
- membership[gitlabGroup] = GitlabRoleMapping[mappings.groups[userGroup][gitlabGroup]]
- }
- }
- break
- }
- }
- }
-
- return membership
-}
 
 async function execute() {
  const mappings = await loadMappings()
@@ -78,106 +58,132 @@ async function execute() {
  const googleUsers = await google.listUsers(Object.keys(userGroups))
  const gitlabScimUsers = await gitlabScim.listScimUsers()
  const gitlabUsers = await gitlabApi.listGroupSamlMembers()
-
  // Compute updates to execute
 
  const membershipUpdates: GitlabAccessUpdate[] = []
  const userUpdates: GitlabUserUpdate[] = []
+ const leftOverUsers = Object.keys(gitlabScimUsers)
 
  // check which users need to be added
  for (const email of Object.keys(userGroups)) {
  if (Object.keys(gitlabScimUsers).includes(email)) {
- continue
- }
- // if a google user is not a scim user, add it and its custom memberships
+ if (gitlabScimUsers[email].active) { // user is active, we'll compare
+  continue
+  }
 
- userUpdates.push({
- user: googleUsers[email],
- op: GitlabUserUpdateOperation.ADD
- })
+ userUpdates.push({
+ user: gitlabScimUsers[email],
+ op: GitlabUserUpdateOperation.ACTIVATE,
+ notes: email
+ })
+ } else {
+ // if a google user is not a scim user, add it and its custom memberships
+
+ userUpdates.push({
+ user: googleUsers[email],
+ op: GitlabUserUpdateOperation.ADD,
+ notes: email,
+ })
+ }
 
  const membership = getUserCustomMembership(email, userGroups[email], mappings)
+ if (membership[GITLAB_GROUP] === undefined) {
+ membershipUpdates.push({
+ user: email,
+ group: GITLAB_GROUP,
+ role: getGroupPrivilege(DEFAULT_MEMBERSHIP_ROLE, GITLAB_GROUP, membership),
+ op: GitlabAccessUpdateOperation.ADD,
+ notes: email,
+ })
+ }
  for (const key of Object.keys(membership)) {
  membershipUpdates.push({
  user: email,
  group: key,
  role: membership[key],
- op: GitlabAccessUpdateOperation.ADD
+ op: GitlabAccessUpdateOperation.ADD,
+ notes: email,
  })
  }
+
+ leftOverUsers.splice(leftOverUsers.indexOf(email), 1)
  }
 
+
  // check which users need to be removed or activate
- for (const email of Object.keys(gitlabScimUsers)) {
- if (!Object.keys(googleUsers).includes(email) && gitlabScimUsers[email].active) { // user does not exist in google and is active in gitlab: remove
+ for (const email of leftOverUsers) {
+ if (!gitlabScimUsers[email].active) {
+ continue
+ }
+
+ // user does not exist in google, or is suspended in google, and is active in gitlab: remove
+ if (!Object.keys(googleUsers).includes(email) || googleUsers[email].suspended) {
  userUpdates.push({
  user: gitlabScimUsers[email],
- op: GitlabUserUpdateOperation.REMOVE
+ op: GitlabUserUpdateOperation.REMOVE,
+ notes: email,
  })
  continue
- } else if (
- Object.keys(googleUsers).includes(email) &&
- googleUsers[email].suspended != !gitlabScimUsers[email].active
- ) { // user exists in both gitlab and google and their status differ
- if (googleUsers[email].suspended) { // google user is suspended but gitlab user is active: remove
- userUpdates.push({
- user: gitlabScimUsers[email],
- op: GitlabUserUpdateOperation.REMOVE
- })
- continue
- } else if (!googleUsers[email].suspended) { // google user is active but gitlab user is inactive: activate
- userUpdates.push({
- user: gitlabScimUsers[email],
- op: GitlabUserUpdateOperation.ACTIVATE
- })
- }
  }
 
- // will only reach here if the user is either to be activated or active in both google and gitlab
-
  const expectedMembership = getUserCustomMembership(email, userGroups[email], mappings)
  const currentMembership = await gitlabApi.listUserMembership(gitlabUsers[email].id)
 
-
  if (currentMembership.length === 0 && Object.keys(expectedMembership).length > 0) {
- expectedMembership
  for (const key of Object.keys(expectedMembership)) {
  membershipUpdates.push({
  user: email,
  group: key,
  role: expectedMembership[key],
- op: GitlabAccessUpdateOperation.ADD
+ op: GitlabAccessUpdateOperation.ADD,
+ notes: email,
  })
  }
  continue
  }
 
  for (const item of currentMembership) {
  if (expectedMembership[item.source_full_name] === undefined) {
- if (item.access_level.integer_value !== DEFAULT_MEMBERSHIP_ROLE) {
+ const userDefaultMembership = getGroupPrivilege(DEFAULT_MEMBERSHIP_ROLE, item.source_full_name, expectedMembership)
+
+ if (item.access_level.integer_value !== userDefaultMembership) {
  membershipUpdates.push({
  user: gitlabUsers[email].id,
  group: item.source_full_name,
- role: DEFAULT_MEMBERSHIP_ROLE,
- op: shouldDelete(item.source_full_name, DEFAULT_MEMBERSHIP_ROLE)
+ role: userDefaultMembership,
+ op: shouldDelete(item.source_full_name, userDefaultMembership),
+ notes: email,
  })
  }
  } else if (expectedMembership[item.source_full_name] !== item.access_level.integer_value) {
  membershipUpdates.push({
  user: gitlabUsers[email].id,
  group: item.source_full_name,
  role: expectedMembership[item.source_full_name],
- op: shouldDelete(item.source_full_name, expectedMembership[item.source_full_name])
+ op: shouldDelete(item.source_full_name, expectedMembership[item.source_full_name]),
+ notes: email,
  })
  }
  }
  }
 
  logger.info(`User updates:`)
- logger.info(JSON.stringify(membershipUpdates))
+ userUpdates.forEach(update => {
+ console.log(JSON.stringify({
+ op: update.op,
+ notes: update.notes,
+ }))
+ })
 
  logger.info(`Membership updates:`)
- logger.info(JSON.stringify(membershipUpdates))
+ membershipUpdates.forEach(update => {
+ console.log(JSON.stringify({
+ notes: update.notes,
+ op: update.op,
+ role: update.role,
+ group: update.group,
+ }))
+ })
 
  if (DRY_RUN) {
  logger.info("Dry run is on, not committing any change")
@@ -191,12 +197,12 @@ async function execute() {
  }
 
  for (const update of membershipUpdates) {
- logger.debug(`changing ${update.user} access for group ${update.group} to ${update.role}`)
+ logger.debug(`changing ${update.notes} access for group ${update.group} to ${update.role}`)
 
  await gitlabApi.changeUserAccessLevel(update)
  }
 
- if (slack.active()) {
+ if (slack !== null) {
  slack.notify(userUpdates, membershipUpdates)
  }
 }

src/gitlab/api.ts

@@ -10,7 +10,12 @@ async function resolveGitlabApiToken(): Promise<string> {
  }
 
  if (process.env.GITLAB_API_TOKEN_FILE !== undefined) {
- return await Bun.file(process.env.GITLAB_API_TOKEN_FILE).text()
+ const f = Bun.file(process.env.GITLAB_API_TOKEN_FILE)
+
+ if (!f.exists()) {
+ throw Error(`Gitlab API token file does not exist: ${process.env.GITLAB_API_TOKEN_FILE}`)
+ }
+ return await f.text()
  }
 
  if (process.env.GITLAB_API_TOKEN !== undefined) {
@@ -87,8 +92,12 @@ export class GitlabApi extends Gitlab {
  }
 
  for (var user of (await resp.json()) as GitlabApiUser[]) {
- if (user.group_saml_identity !== undefined) {
- users[user.email!] = user
+ if (user.username.startsWith("group_")) {
+ continue
+ }
+
+ if (user.group_saml_identity !== null) {
+ users[user.group_saml_identity.extern_uid] = user
  }
  }
  }

src/gitlab/scim.ts

@@ -13,7 +13,12 @@ async function resolveGitlabScimToken(): Promise<string> {
  }
 
  if (process.env.GITLAB_SCIM_TOKEN_FILE !== undefined) {
- return await Bun.file(process.env.GITLAB_SCIM_TOKEN_FILE).text()
+ const f = Bun.file(process.env.GITLAB_SCIM_TOKEN_FILE)
+
+ if (!f.exists()) {
+ throw Error(`Gitlab SCIM token file does not exist: ${process.env.GITLAB_SCIM_TOKEN_FILE}`)
+ }
+ return await f.text()
  }
 
  if (process.env.GITLAB_SCIM_TOKEN !== undefined) {

src/gitlab/types.ts

@@ -21,13 +21,13 @@ export type GitlabScimUser = {
 
 export type GitlabApiUser = {
  id: string
+ username: string,
  state: string,
- email: string | undefined
  group_saml_identity: {
  extern_uid: string
  provider: string
  saml_provider_id: number
- } | undefined
+ } | null
 }
 
 export type GitlabGroup = {
@@ -75,6 +75,7 @@ export type GitlabAccessUpdate = {
  group: string
  role: GitlabRole
  op: GitlabAccessUpdateOperation
+ notes: string
 }
 
 export enum GitlabUserUpdateOperation {
@@ -86,4 +87,5 @@ export enum GitlabUserUpdateOperation {
 export type GitlabUserUpdate = {
  user: GoogleUser | GitlabScimUser
  op: GitlabUserUpdateOperation
+ notes?: string
 }

src/google/index.ts

@@ -15,7 +15,6 @@ const GOOGLE_ADMIN_EMAIL = process.env.GOOGLE_ADMIN_EMAIL
 const GOOGLE_TARGET_SA_FILE = GOOGLE_SA_KEY_FILE || "/tmp/service_account.json"
 
 async function resolveGoogleServiceAcccount() {
-
  if (process.env.GOOGLE_SA_KEY_SECRET !== undefined) {
  const secret = await getSecretFromAws(process.env.GOOGLE_SA_KEY_SECRET)
  await Bun.write(GOOGLE_TARGET_SA_FILE, secret)
@@ -28,8 +27,13 @@ async function resolveGoogleServiceAcccount() {
  }
 
  if (process.env.GOOGLE_SA_KEY_FILE !== undefined) {
+ const f = Bun.file(process.env.GOOGLE_SA_KEY_FILE)
+ if (!f.exists()) {
+ throw Error(`Google service account file does not exist: ${process.env.GOOGLE_SA_KEY_FILE}`)
+ }
  return
  }
+
  throw Error(`Google Service Account was not provided`)
 }