diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b37d3a843..7bb4ec8ecc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -177,7 +177,20 @@ jobs: --health-timeout 5s --health-retries 5 ports: - - 5432/tcp + - 5432:5432/tcp + postgres-seed: + image: postgres:${{ matrix.pg-version }} + env: + POSTGRES_DB: medplum_test + POSTGRES_USER: medplum + POSTGRES_PASSWORD: medplum + options: >- + --health-cmd pg_isready + --health-retries 5 + --health-interval 10s + --health-timeout 5s + ports: + - 5433:5432/tcp redis: image: redis:${{ matrix.redis-version }} options: >- @@ -186,7 +199,7 @@ jobs: --health-timeout 5s --health-retries 5 ports: - - 6379:6379 + - 6379:6379/tcp steps: - uses: actions/checkout@v4 @@ -224,7 +237,8 @@ jobs: run: ./scripts/test.sh env: POSTGRES_HOST: localhost - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_PORT: 5432 + POSTGRES_SEED_PORT: 5433 REDIS_PASSWORD_DISABLED_IN_TESTS: 1 - name: Upload code coverage if: ${{ matrix.node-version == 20 && matrix.pg-version == 14 && matrix.redis-version == 7 }} diff --git a/docker-compose.seed.yml b/docker-compose.seed.yml new file mode 100644 index 0000000000..17ecc8af88 --- /dev/null +++ b/docker-compose.seed.yml @@ -0,0 +1,15 @@ +version: '3.7' +services: + postgres-seed: + image: postgres:12 + restart: always + environment: + - POSTGRES_USER=medplum + - POSTGRES_PASSWORD=medplum + + volumes: + - ./postgres/postgres.conf:/usr/local/etc/postgres/postgres.conf + - ./postgres/:/docker-entrypoint-initdb.d/ + command: postgres -c config_file=/usr/local/etc/postgres/postgres.conf + ports: + - '5433:5432' diff --git a/package-lock.json b/package-lock.json index 50e1942337..f38510f949 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@types/node": "20.12.8", "babel-jest": "29.7.0", "babel-preset-vite": "1.1.3", + "concurrently": "8.2.2", "cross-env": "7.0.3", "danger": "12.2.0", "esbuild": "0.20.2", @@ -23836,6 +23837,92 @@ "typedarray": "^0.0.6" } }, + "node_modules/concurrently": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz", + "integrity": "sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.2", + "date-fns": "^2.30.0", + "lodash": "^4.17.21", + "rxjs": "^7.8.1", + "shell-quote": "^1.8.1", + "spawn-command": "0.0.2", + "supports-color": "^8.1.1", + "tree-kill": "^1.2.2", + "yargs": "^17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, + "engines": { + "node": "^14.13.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/concurrently/node_modules/date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.21.0" + }, + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/confbox": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.7.tgz", @@ -29563,6 +29650,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "dev": true, "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", @@ -29576,6 +29664,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -47632,6 +47721,15 @@ "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", "dev": true }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dev": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -49027,6 +49125,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/spawn-command": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", + "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==", + "dev": true + }, "node_modules/spawn-please": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/spawn-please/-/spawn-please-2.0.2.tgz", @@ -53934,6 +54038,7 @@ "version": "1.10.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "dev": true, "engines": { "node": ">= 6" } diff --git a/package.json b/package.json index b3a2f55c19..73bb5b39f4 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@types/node": "20.12.8", "babel-jest": "29.7.0", "babel-preset-vite": "1.1.3", + "concurrently": "8.2.2", "cross-env": "7.0.3", "danger": "12.2.0", "esbuild": "0.20.2", diff --git a/packages/docs/docs/contributing/testing.md b/packages/docs/docs/contributing/testing.md index 6c1f977a1d..430d800f20 100644 --- a/packages/docs/docs/contributing/testing.md +++ b/packages/docs/docs/contributing/testing.md @@ -16,6 +16,18 @@ We use the following tools for testing: Every pull request is analyzed by [Sonarcloud](https://sonarcloud.io/project/overview?id=medplum_medplum) and [Coveralls](https://coveralls.io/github/medplum/medplum?branch=main) for code coverage and other static analysis. +## Before testing +When you are testing for the first time with a fresh database, you first need to seed and migrate the database. +You can accomplish this by running the following command: + +```bash +npx turbo run test:seed:parallel --filter=@medplum/server +``` + +This process can take a minute or two as it migrates the database and then creates many FHIR resources required for Medplum to operate correctly. + +After the test passes and the database is migrated and seeded, you won't need to run it again unless the database schema changes or you destroy the volume attached to your `postgres` Docker container. Now you should be able to run the rest of the tests. + ## How to test To run all tests for all packages, use the build script: diff --git a/packages/server/README.md b/packages/server/README.md index 940c51e014..b54a14daf9 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -68,3 +68,19 @@ If so, create a new multiarch driver: ```bash docker buildx create --name multiarch --driver docker-container --use ``` + +## Testing + +### Seeding the database +Before running `npm run test` in `packages/server`, you will want to make sure the database is seeded. You are able to seed the database before running the tests by running: + +```bash +npm run test:seed:parallel +``` + +### Running the tests +To run the tests, simply issue the command: + +```bash +npm run test +``` diff --git a/packages/server/jest.config.json b/packages/server/jest.config.json deleted file mode 100644 index 8e2a0d7497..0000000000 --- a/packages/server/jest.config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "testEnvironment": "node", - "testTimeout": 600000, - "testSequencer": "/jest.sequencer.js", - "transform": { - "^.+\\.(js|jsx|ts|tsx)$": "babel-jest" - }, - "moduleFileExtensions": ["ts", "js", "json", "node"], - "testMatch": ["**/src/**/*.test.ts"], - "coverageDirectory": "coverage", - "coverageReporters": ["json", "text"], - "collectCoverageFrom": ["**/src/**/*", "!**/src/__mocks__/**/*.ts", "!**/src/migrations/**/*.ts"] -} diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts new file mode 100644 index 0000000000..a4178f579d --- /dev/null +++ b/packages/server/jest.config.ts @@ -0,0 +1,18 @@ +import type { Config } from 'jest'; + +export default { + testEnvironment: 'node', + testTimeout: 5000, + testSequencer: '/jest.sequencer.js', + transform: { + '^.+\\.(js|jsx|ts|tsx)$': 'babel-jest', + }, + // Oddly, the glob `/src/seed*.test.ts` correctly matches both seed tests in the positive case in `jest.seed.config.ts` + // But `!/src/seed*.test.ts` doesn't match both in the negative case, and only matches `seed-serial.test.ts` + // That's why we use `!**/src/seed*.test.ts` here + testMatch: ['/src/**/*.test.ts', '!**/src/seed*.test.ts'], + moduleFileExtensions: ['ts', 'js', 'json', 'node'], + coverageDirectory: 'coverage', + coverageReporters: ['json', 'text'], + collectCoverageFrom: ['**/src/**/*', '!**/src/__mocks__/**/*.ts', '!**/src/migrations/**/*.ts'], +} satisfies Config; diff --git a/packages/server/jest.seed.config.ts b/packages/server/jest.seed.config.ts new file mode 100644 index 0000000000..515d5d6f1d --- /dev/null +++ b/packages/server/jest.seed.config.ts @@ -0,0 +1,9 @@ +import type { Config } from 'jest'; +import defaultConfig from './jest.config'; + +export default { + ...defaultConfig, + testTimeout: 300000, + testMatch: ['/src/seed*.test.ts'], + collectCoverageFrom: ['/src/**/*', '!**/src/__mocks__/**/*.ts', '!**/src/migrations/**/*.ts'], +} satisfies Config; diff --git a/packages/server/package.json b/packages/server/package.json index 84df37d1ac..adbad6c411 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,8 +18,9 @@ "clean": "rimraf dist", "dev": "ts-node-dev --poll --respawn --transpile-only --require ./src/otel/instrumentation.ts src/index.ts", "start": "node --require ./dist/otel/instrumentation.js dist/index.js", - "test": "jest", - "test:seed": "jest seed.test.ts" + "test:seed:serial": "jest seed-serial.test.ts --config jest.seed.config.ts --coverageDirectory \"/coverage/seed/serial\"", + "test:seed:parallel": "jest seed.test.ts --config jest.seed.config.ts --coverageDirectory \"/coverage/seed/parallel\"", + "test": "jest" }, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "3.569.0", diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index 7ebb1062e0..f8f1cccc56 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -261,7 +261,7 @@ function addDefaults(config: MedplumServerConfig): MedplumServerConfig { config.accurateCountThreshold = config.accurateCountThreshold ?? 1000000; config.defaultBotRuntimeVersion = config.defaultBotRuntimeVersion ?? 'awslambda'; config.defaultProjectFeatures = config.defaultProjectFeatures ?? []; - config.emailProvider = config.emailProvider || (config.smtp ? 'smtp' : 'awsses'); + config.emailProvider = config.emailProvider ?? (config.smtp ? 'smtp' : 'awsses'); return config; } diff --git a/packages/server/src/seed-serial.test.ts b/packages/server/src/seed-serial.test.ts new file mode 100644 index 0000000000..767397713b --- /dev/null +++ b/packages/server/src/seed-serial.test.ts @@ -0,0 +1,56 @@ +import { Project } from '@medplum/fhirtypes'; +import { shutdownApp } from './app'; +import { loadTestConfig } from './config'; +import { AuthenticatedRequestContext, requestContextStore } from './context'; +import { getDatabasePool, initDatabase } from './database'; +import { SelectQuery } from './fhir/sql'; +import { loadStructureDefinitions } from './fhir/structure'; +import { initRedis } from './redis'; +import { seedDatabase } from './seed'; +import { withTestContext } from './test.setup'; + +describe('Seed Serial', () => { + beforeAll(async () => { + console.log = jest.fn(); + + const config = await loadTestConfig(); + config.database.port = process.env['POSTGRES_SEED_PORT'] + ? Number.parseInt(process.env['POSTGRES_SEED_PORT'], 10) + : 5433; + // Keep Redis separate so caches between main test suite and this are separate + config.redis.db = 8; + config.database.runMigrations = true; + + // We load the minimal required to get things running so this actually tests seeding the database + return withTestContext(() => + requestContextStore.run(AuthenticatedRequestContext.system(), async () => { + loadStructureDefinitions(); + initRedis(config.redis); + await initDatabase(config); + }) + ); + }); + + afterAll(async () => { + await shutdownApp(); + }); + + test('Seeder completes successfully -- serial version', async () => { + // First time, seeder should run + await seedDatabase({ parallel: false }); + + // Make sure the first project is a super admin + const rows = await new SelectQuery('Project') + .column('content') + .where('name', '=', 'Super Admin') + .execute(getDatabasePool()); + expect(rows.length).toBe(1); + + const project = JSON.parse(rows[0].content) as Project; + expect(project.superAdmin).toBe(true); + expect(project.strictMode).toBe(true); + + // Second time, seeder should silently ignore + await seedDatabase({ parallel: false }); + }, 240000); +}); diff --git a/packages/server/src/seed.test.ts b/packages/server/src/seed.test.ts index e4b8dbaa29..eaeba1e7f3 100644 --- a/packages/server/src/seed.test.ts +++ b/packages/server/src/seed.test.ts @@ -1,8 +1,11 @@ import { Project } from '@medplum/fhirtypes'; -import { initAppServices, shutdownApp } from './app'; +import { shutdownApp } from './app'; import { loadTestConfig } from './config'; -import { getDatabasePool } from './database'; +import { AuthenticatedRequestContext, requestContextStore } from './context'; +import { getDatabasePool, initDatabase } from './database'; import { SelectQuery } from './fhir/sql'; +import { loadStructureDefinitions } from './fhir/structure'; +import { initRedis } from './redis'; import { seedDatabase } from './seed'; import { withTestContext } from './test.setup'; @@ -12,7 +15,15 @@ describe('Seed', () => { const config = await loadTestConfig(); config.database.runMigrations = true; - return withTestContext(() => initAppServices(config)); + + // We load the minimal required to get things running so this actually tests seeding the database + return withTestContext(() => + requestContextStore.run(AuthenticatedRequestContext.system(), async () => { + loadStructureDefinitions(); + initRedis(config.redis); + await initDatabase(config); + }) + ); }); afterAll(async () => { diff --git a/packages/server/src/seed.ts b/packages/server/src/seed.ts index a63f1f1f9c..18016632e2 100644 --- a/packages/server/src/seed.ts +++ b/packages/server/src/seed.ts @@ -4,18 +4,28 @@ import { NIL as nullUuid, v5 } from 'uuid'; import { bcryptHashPassword } from './auth/utils'; import { getSystemRepo } from './fhir/repo'; import { globalLogger } from './logger'; +import { RebuildOptions } from './seeds/common'; import { rebuildR4SearchParameters } from './seeds/searchparameters'; import { rebuildR4StructureDefinitions } from './seeds/structuredefinitions'; import { rebuildR4ValueSets } from './seeds/valuesets'; export const r4ProjectId = v5('R4', nullUuid); -export async function seedDatabase(): Promise { +/** + * Seeds the database with system resources. + * + * @param options - Optional options for seeding the database. + * @returns A Promise that resolves when seeding is done. + */ +export async function seedDatabase(options?: RebuildOptions): Promise { if (await isSeeded()) { globalLogger.info('Already seeded'); return; } + performance.mark('Starting to seed'); + globalLogger.info('Seeding database...'); + const systemRepo = getSystemRepo(); const [firstName, lastName, email] = ['Medplum', 'Admin', 'admin@example.com']; @@ -70,9 +80,38 @@ export async function seedDatabase(): Promise { admin: true, }); - await rebuildR4StructureDefinitions(); - await rebuildR4ValueSets(); - await rebuildR4SearchParameters(); + globalLogger.info('Rebuilding system resources...'); + performance.mark('Starting rebuilds'); + + performance.mark('Starting rebuildR4StructureDefinitions'); + await rebuildR4StructureDefinitions({ parallel: true, ...options }); + const sdStats = performance.measure( + 'Finished rebuildR4StructureDefinitions', + 'Starting rebuildR4StructureDefinitions' + ); + globalLogger.info('Finished rebuildR4StructureDefinitions', { + duration: `${Math.ceil(sdStats.duration)} ms`, + }); + + performance.mark('Starting rebuildR4ValueSets'); + await rebuildR4ValueSets({ parallel: true, ...options }); + const valueSetsStats = performance.measure('Finished rebuildR4ValueSets', 'Starting rebuildR4ValueSets'); + globalLogger.info('Finished rebuildR4ValueSets', { duration: `${Math.ceil(valueSetsStats.duration)} ms` }); + + performance.mark('Starting rebuildR4SearchParameters'); + await rebuildR4SearchParameters({ parallel: true, ...options }); + const searchParamsStats = performance.measure( + 'Finished rebuildR4SearchParameters', + 'Starting rebuildR4SearchParameters' + ); + globalLogger.info('Finished rebuildR4SearchParameters', { + duration: `${Math.ceil(searchParamsStats.duration)} ms`, + }); + + const rebuildStats = performance.measure('Finished rebuilds', 'Starting rebuilds'); + globalLogger.info('Finished rebuilds', { duration: `${Math.ceil(rebuildStats.duration)} ms` }); + const seedingStats = performance.measure('Finished seeding', 'Starting to seed'); + globalLogger.info('Finished seeding', { duration: `${Math.ceil(seedingStats.duration)} ms` }); } /** diff --git a/packages/server/src/seeds/common.ts b/packages/server/src/seeds/common.ts new file mode 100644 index 0000000000..6d9db22550 --- /dev/null +++ b/packages/server/src/seeds/common.ts @@ -0,0 +1,16 @@ +export interface RebuildOptions { + /** + * Whether the resources should be created in parallel. + * + * **WARNING: Can be CPU intensive and/or clog up the connection pool.** + */ + parallel: boolean; +} + +const defaultOptions = { + parallel: false, +}; + +export function buildRebuildOptions(options?: Partial): RebuildOptions { + return { ...defaultOptions, ...options }; +} diff --git a/packages/server/src/seeds/searchparameters.ts b/packages/server/src/seeds/searchparameters.ts index 90496b08ef..85bb61dc78 100644 --- a/packages/server/src/seeds/searchparameters.ts +++ b/packages/server/src/seeds/searchparameters.ts @@ -4,19 +4,32 @@ import { getDatabasePool } from '../database'; import { Repository, getSystemRepo } from '../fhir/repo'; import { globalLogger } from '../logger'; import { r4ProjectId } from '../seed'; +import { RebuildOptions, buildRebuildOptions } from './common'; /** * Creates all SearchParameter resources. + * @param options - Optional options for how rebuild should be done. */ -export async function rebuildR4SearchParameters(): Promise { +export async function rebuildR4SearchParameters(options?: Partial): Promise { + const rebuildOptions = buildRebuildOptions(options); const client = getDatabasePool(); await client.query('DELETE FROM "SearchParameter" WHERE "projectId" = $1', [r4ProjectId]); const systemRepo = getSystemRepo(); - for (const filename of SEARCH_PARAMETER_BUNDLE_FILES) { - for (const entry of readJson(filename).entry as BundleEntry[]) { - await createParameter(systemRepo, entry.resource as SearchParameter); + if (rebuildOptions.parallel) { + const promises = []; + for (const filename of SEARCH_PARAMETER_BUNDLE_FILES) { + for (const entry of readJson(filename).entry as BundleEntry[]) { + promises.push(createParameter(systemRepo, entry.resource as SearchParameter)); + } + } + await Promise.all(promises); + } else { + for (const filename of SEARCH_PARAMETER_BUNDLE_FILES) { + for (const entry of readJson(filename).entry as BundleEntry[]) { + await createParameter(systemRepo, entry.resource as SearchParameter); + } } } } diff --git a/packages/server/src/seeds/structuredefinitions.ts b/packages/server/src/seeds/structuredefinitions.ts index a5595d78a1..6ea5085704 100644 --- a/packages/server/src/seeds/structuredefinitions.ts +++ b/packages/server/src/seeds/structuredefinitions.ts @@ -4,41 +4,69 @@ import { getDatabasePool } from '../database'; import { Repository, getSystemRepo } from '../fhir/repo'; import { globalLogger } from '../logger'; import { r4ProjectId } from '../seed'; +import { RebuildOptions, buildRebuildOptions } from './common'; /** * Creates all StructureDefinition resources. + * @param options - Optional options for how rebuild should be done. */ -export async function rebuildR4StructureDefinitions(): Promise { +export async function rebuildR4StructureDefinitions(options?: Partial): Promise { + const rebuildOptions = buildRebuildOptions(options); const client = getDatabasePool(); await client.query(`DELETE FROM "StructureDefinition" WHERE "projectId" = $1`, [r4ProjectId]); const systemRepo = getSystemRepo(); - await createStructureDefinitionsForBundle(systemRepo, readJson('fhir/r4/profiles-resources.json') as Bundle); - await createStructureDefinitionsForBundle(systemRepo, readJson('fhir/r4/profiles-medplum.json') as Bundle); - await createStructureDefinitionsForBundle(systemRepo, readJson('fhir/r4/profiles-others.json') as Bundle); + if (rebuildOptions.parallel) { + await Promise.all([ + createStructureDefinitionsForBundleParallel(systemRepo, readJson('fhir/r4/profiles-resources.json') as Bundle), + createStructureDefinitionsForBundleParallel(systemRepo, readJson('fhir/r4/profiles-medplum.json') as Bundle), + createStructureDefinitionsForBundleParallel(systemRepo, readJson('fhir/r4/profiles-others.json') as Bundle), + ]); + } else { + await createStructureDefinitionsForBundleSerial(systemRepo, readJson('fhir/r4/profiles-resources.json') as Bundle); + await createStructureDefinitionsForBundleSerial(systemRepo, readJson('fhir/r4/profiles-medplum.json') as Bundle); + await createStructureDefinitionsForBundleSerial(systemRepo, readJson('fhir/r4/profiles-others.json') as Bundle); + } } -async function createStructureDefinitionsForBundle( +async function createStructureDefinitionsForBundleParallel( systemRepo: Repository, structureDefinitions: Bundle ): Promise { + const promises = []; for (const entry of structureDefinitions.entry as BundleEntry[]) { const resource = entry.resource as Resource; + if (resource.resourceType === 'StructureDefinition' && resource.name) { + promises.push(createAndLogStructureDefinition(systemRepo, resource)); + } + } + await Promise.all(promises); +} +async function createStructureDefinitionsForBundleSerial( + systemRepo: Repository, + structureDefinitions: Bundle +): Promise { + for (const entry of structureDefinitions.entry as BundleEntry[]) { + const resource = entry.resource as Resource; if (resource.resourceType === 'StructureDefinition' && resource.name) { - globalLogger.debug('StructureDefinition: ' + resource.name); - const result = await systemRepo.createResource({ - ...resource, - meta: { - ...resource.meta, - project: r4ProjectId, - lastUpdated: undefined, - versionId: undefined, - }, - text: undefined, - differential: undefined, - }); - globalLogger.debug('Created: ' + result.id); + await createAndLogStructureDefinition(systemRepo, resource); } } } + +async function createAndLogStructureDefinition(systemRepo: Repository, resource: StructureDefinition): Promise { + globalLogger.debug('[StructureDefinition] creation started: ' + resource.name); + const result = await systemRepo.createResource({ + ...resource, + meta: { + ...resource.meta, + project: r4ProjectId, + lastUpdated: undefined, + versionId: undefined, + }, + text: undefined, + differential: undefined, + }); + globalLogger.debug(`[StructureDefinition] creation finished: ${result.name} - ID: ${result.id}`); +} diff --git a/packages/server/src/seeds/valuesets.ts b/packages/server/src/seeds/valuesets.ts index d6397f9e98..c7fb69462c 100644 --- a/packages/server/src/seeds/valuesets.ts +++ b/packages/server/src/seeds/valuesets.ts @@ -3,35 +3,54 @@ import { readJson } from '@medplum/definitions'; import { Bundle, BundleEntry, CodeSystem, ValueSet } from '@medplum/fhirtypes'; import { Repository, getSystemRepo } from '../fhir/repo'; import { r4ProjectId } from '../seed'; +import { RebuildOptions, buildRebuildOptions } from './common'; /** * Imports all built-in ValueSets and CodeSystems into the database. + * @param options - Optional options for how rebuild should be done. */ -export async function rebuildR4ValueSets(): Promise { +export async function rebuildR4ValueSets(options?: Partial): Promise { + const rebuildOptions = buildRebuildOptions(options); const systemRepo = getSystemRepo(); const files = ['v2-tables.json', 'v3-codesystems.json', 'valuesets.json', 'valuesets-medplum.json']; for (const file of files) { const bundle = readJson('fhir/r4/' + file) as Bundle; - for (const entry of bundle.entry as BundleEntry[]) { - const resource = entry.resource as CodeSystem | ValueSet; - await deleteExisting(systemRepo, resource, r4ProjectId); - await systemRepo.createResource({ - ...resource, - meta: { - ...resource.meta, - project: r4ProjectId, - lastUpdated: undefined, - versionId: undefined, - }, - }); + if (rebuildOptions.parallel) { + const promises = []; + for (const entry of bundle.entry as BundleEntry[]) { + promises.push(overwriteResource(systemRepo, entry.resource as CodeSystem | ValueSet, rebuildOptions)); + } + await Promise.all(promises); + } else { + for (const entry of bundle.entry as BundleEntry[]) { + await overwriteResource(systemRepo, entry.resource as CodeSystem | ValueSet, rebuildOptions); + } } } } +async function overwriteResource( + systemRepo: Repository, + resource: CodeSystem | ValueSet, + options: RebuildOptions +): Promise { + await deleteExisting(systemRepo, resource, r4ProjectId, options); + await systemRepo.createResource({ + ...resource, + meta: { + ...resource.meta, + project: r4ProjectId, + lastUpdated: undefined, + versionId: undefined, + }, + }); +} + async function deleteExisting( systemRepo: Repository, resource: CodeSystem | ValueSet, - projectId: string + projectId: string, + options: RebuildOptions ): Promise { const bundle = await systemRepo.search({ resourceType: resource.resourceType, @@ -41,9 +60,18 @@ async function deleteExisting( ], }); if (bundle.entry && bundle.entry.length > 0) { - for (const entry of bundle.entry) { - const existing = entry.resource as CodeSystem | ValueSet; - await systemRepo.deleteResource(existing.resourceType, existing.id as string); + if (options.parallel) { + const promises = []; + for (const entry of bundle.entry) { + const existing = entry.resource as CodeSystem | ValueSet; + promises.push(systemRepo.deleteResource(existing.resourceType, existing.id as string)); + } + await Promise.all(promises); + } else { + for (const entry of bundle.entry) { + const existing = entry.resource as CodeSystem | ValueSet; + await systemRepo.deleteResource(existing.resourceType, existing.id as string); + } } } } diff --git a/packages/server/turbo.json b/packages/server/turbo.json index 4562db1783..abd1098190 100644 --- a/packages/server/turbo.json +++ b/packages/server/turbo.json @@ -2,16 +2,16 @@ "$schema": "https://turborepo.org/schema.json", "extends": ["//"], "pipeline": { - "test:seed": { + "test:seed:serial": { "dependsOn": ["build"], "outputs": ["coverage/**"], - "inputs": ["src/**/*.tsx", "src/**/*.ts"], - "env": ["SHOULD_RUN_SEED_TEST"] + "inputs": ["src/**/*.tsx", "src/**/*.ts"] }, - "test": { - "dependsOn": ["test:seed"], + "test:seed:parallel": { + "dependsOn": ["build"], "outputs": ["coverage/**"], - "inputs": ["src/**/*.tsx", "src/**/*.ts"] + "inputs": ["src/**/*.tsx", "src/**/*.ts"], + "env": ["SHOULD_RUN_SEED_TEST"] } } } diff --git a/scripts/test-main.sh b/scripts/test-main.sh new file mode 100755 index 0000000000..b1d480c341 --- /dev/null +++ b/scripts/test-main.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Fail on error +set -e +# Echo commands +set -x + +# Seed the database before testing +# This is the parallel implementation so it's faster +SHOULD_RUN_SEED_TEST=$(date) time npx turbo run test:seed:parallel --filter=./packages/server -- --coverage +cp "packages/server/coverage/seed/parallel/coverage-final.json" "coverage/packages/coverage-server-seed-parallel.json" + +# Test +# Run them separately because code coverage is resource intensive + +for dir in `ls packages`; do + if test -f "packages/$dir/package.json" && grep -q "\"test\":" "packages/$dir/package.json"; then + npx turbo run test --filter=./packages/$dir -- --coverage + fi +done + +for dir in `ls examples`; do + if test -f "examples/$dir/package.json" && grep -q "\"test\":" "examples/$dir/package.json"; then + npx turbo run test --filter=./packages/$dir + fi +done diff --git a/scripts/test-seed.sh b/scripts/test-seed.sh new file mode 100755 index 0000000000..31757b8fa5 --- /dev/null +++ b/scripts/test-seed.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Fail on error +set -e +# Echo commands +set -x + +# Testing production path of seeding the database +# This is a special "test" which runs all of the seed logic, such as setting up structure definitions +# On a normal developer machine, this is run only rarely when setting up a new database +# We execute this in parallel with the main line of tests +time npx turbo run test:seed:serial --filter=./packages/server -- --coverage +cp "packages/server/coverage/seed/serial/coverage-final.json" "coverage/packages/coverage-server-seed-serial.json" diff --git a/scripts/test.sh b/scripts/test.sh index ef4bf26209..2a5838d632 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -7,35 +7,14 @@ set -e set -x # Set node options -export NODE_OPTIONS='--max-old-space-size=5120' +export NODE_OPTIONS='--max-old-space-size=8192' # Clear old code coverage data rm -rf coverage mkdir -p coverage/packages mkdir -p coverage/combined -# Seed the database -# This is a special "test" which runs all of the seed logic, such as setting up structure definitions -# On a normal developer machine, this is run only rarely when setting up a new database -# This test must be run first, and cannot be run concurrently with other tests -SHOULD_RUN_SEED_TEST=$(date) time npx turbo run test:seed --filter=./packages/server -- --coverage -cp "packages/server/coverage/coverage-final.json" "coverage/packages/coverage-server-seed.json" - -# Test -# Run them separately because code coverage is resource intensive - -for dir in `ls packages`; do - if test -f "packages/$dir/package.json" && grep -q "\"test\":" "packages/$dir/package.json"; then - npx turbo run test --filter=./packages/$dir -- --coverage - fi -done - -for dir in `ls examples`; do - if test -f "examples/$dir/package.json" && grep -q "\"test\":" "examples/$dir/package.json"; then - npx turbo run test --filter=./packages/$dir - fi -done - +npx concurrently -n seed,main --kill-others-on-fail "scripts/test-seed.sh" "scripts/test-main.sh" # Combine test coverage PACKAGES=(