From d89db56b8b23d9abec4f72e771677c0885b73a47 Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Thu, 8 Feb 2024 14:00:59 -0600 Subject: [PATCH] fix-3885 propagate `traceId` through asynchronous jobs (#3886) * support `Sentry-Trace` header * update doc * traceparent parser allowing some divergence from the spec and maintain full traceparent in logs * allow some grace on flags format * feedback * fix-3885 propagate traceId through asynchronous jobs * propagate traceId to vmcontext and lambda bot executions * fix tests --------- Co-authored-by: Cody Ebberson --- packages/server/src/context.ts | 4 +- .../server/src/fhir/operations/execute.ts | 7 +- packages/server/src/test.setup.ts | 4 +- packages/server/src/traceparent.test.ts | 10 +- packages/server/src/workers/download.test.ts | 80 ++-- packages/server/src/workers/download.ts | 32 +- .../server/src/workers/subscription.test.ts | 424 +++++++++--------- packages/server/src/workers/subscription.ts | 29 +- 8 files changed, 329 insertions(+), 261 deletions(-) diff --git a/packages/server/src/context.ts b/packages/server/src/context.ts index 7445308685..a29716fbad 100644 --- a/packages/server/src/context.ts +++ b/packages/server/src/context.ts @@ -59,10 +59,10 @@ export class AuthenticatedRequestContext extends RequestContext { this.repo.close(); } - static system(): AuthenticatedRequestContext { + static system(ctx?: { requestId?: string; traceId?: string }): AuthenticatedRequestContext { const systemLogger = new Logger(write, undefined, LogLevel.ERROR); return new AuthenticatedRequestContext( - new RequestContext('', ''), + new RequestContext(ctx?.requestId ?? '', ctx?.traceId ?? ''), {} as unknown as Login, {} as unknown as Project, {} as unknown as ProjectMembership, diff --git a/packages/server/src/fhir/operations/execute.ts b/packages/server/src/fhir/operations/execute.ts index 925bcaf62c..bab04ec337 100644 --- a/packages/server/src/fhir/operations/execute.ts +++ b/packages/server/src/fhir/operations/execute.ts @@ -56,6 +56,7 @@ export interface BotExecutionRequest { readonly remoteAddress?: string; readonly forwardedFor?: string; readonly requestTime?: string; + readonly traceId?: string; } export interface BotExecutionResult { @@ -269,7 +270,7 @@ async function writeBotInputToStorage(request: BotExecutionRequest): Promise { - const { bot, runAs, input, contentType } = request; + const { bot, runAs, input, contentType, traceId } = request; const config = getConfig(); const accessToken = await getBotAccessToken(runAs); const secrets = await getBotSecrets(bot); @@ -282,6 +283,7 @@ async function runInLambda(request: BotExecutionRequest): Promise { - const { bot, runAs, input, contentType } = request; + const { bot, runAs, input, contentType, traceId } = request; const config = getConfig(); if (!config.vmContextBotsEnabled) { @@ -409,6 +411,7 @@ async function runInVmContext(request: BotExecutionRequest): Promise(fn: () => T): T { - return requestContextStore.run(AuthenticatedRequestContext.system(), fn); +export function withTestContext(fn: () => T, ctx?: { requestId?: string; traceId?: string }): T { + return requestContextStore.run(AuthenticatedRequestContext.system(ctx), fn); } diff --git a/packages/server/src/traceparent.test.ts b/packages/server/src/traceparent.test.ts index 7cc00023eb..54f16a6a15 100644 --- a/packages/server/src/traceparent.test.ts +++ b/packages/server/src/traceparent.test.ts @@ -12,23 +12,23 @@ describe('parseTraceparent', () => { expect(parseTraceparent(`${tp.version}-${tp.traceId}-${tp.parentId}-${tp.flags}`)).toEqual(tp); }); - it('allow missing version', () => { + it('allows missing version', () => { expect(parseTraceparent(`${tp.traceId}-${tp.parentId}-${tp.flags}`)).toEqual({ ...tp, version: undefined }); }); - it('allow missing flags', () => { + it('allows missing flags', () => { expect(parseTraceparent(`${tp.version}-${tp.traceId}-${tp.parentId}`)).toEqual({ ...tp, flags: undefined }); }); - it('allow missing version and flags', () => { + it('allows missing version and flags', () => { expect(parseTraceparent(`${tp.traceId}-${tp.parentId}`)).toEqual({ ...tp, version: undefined, flags: undefined }); }); - it('allow 1 character for flags', () => { + it('allows 1 character for flags', () => { expect(parseTraceparent(`${tp.traceId}-${tp.parentId}-1`)).toEqual({ ...tp, version: undefined, flags: '1' }); }); - it('no more than 2 characters for flags', () => { + it('returns null for more than 2 characters for flags', () => { expect(parseTraceparent(`${tp.traceId}-${tp.parentId}-001`)).toEqual(null); }); diff --git a/packages/server/src/workers/download.test.ts b/packages/server/src/workers/download.test.ts index 2cd882e7b4..0f8e929eed 100644 --- a/packages/server/src/workers/download.test.ts +++ b/packages/server/src/workers/download.test.ts @@ -37,45 +37,53 @@ describe('Download Worker', () => { }); test('Download external URL', () => - withTestContext(async () => { - const url = 'https://example.com/download'; - - const queue = getDownloadQueue() as any; - queue.add.mockClear(); - - const media = await repo.createResource({ - resourceType: 'Media', - status: 'completed', - content: { - contentType: ContentType.TEXT, - url, - }, - }); - expect(media).toBeDefined(); - expect(queue.add).toHaveBeenCalled(); - - const body = new Readable(); - body.push('foo'); - body.push(null); - - (fetch as unknown as jest.Mock).mockImplementation(() => ({ - status: 200, - headers: { - get(name: string): string | undefined { - return { - 'content-disposition': 'attachment; filename=download', - 'content-type': ContentType.TEXT, - }[name]; + withTestContext( + async () => { + const url = 'https://example.com/download'; + + const queue = getDownloadQueue() as any; + queue.add.mockClear(); + + const media = await repo.createResource({ + resourceType: 'Media', + status: 'completed', + content: { + contentType: ContentType.TEXT, + url, }, - }, - body, - })); + }); + expect(media).toBeDefined(); + expect(queue.add).toHaveBeenCalled(); + + const body = new Readable(); + body.push('foo'); + body.push(null); + + (fetch as unknown as jest.Mock).mockImplementation(() => ({ + status: 200, + headers: { + get(name: string): string | undefined { + return { + 'content-disposition': 'attachment; filename=download', + 'content-type': ContentType.TEXT, + }[name]; + }, + }, + body, + })); - const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; - await execDownloadJob(job); + const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; + await execDownloadJob(job); - expect(fetch).toHaveBeenCalledWith(url); - })); + expect(fetch).toHaveBeenCalledWith(url, { + headers: { + 'x-trace-id': '00-12345678901234567890123456789012-3456789012345678-01', + traceparent: '00-12345678901234567890123456789012-3456789012345678-01', + }, + }); + }, + { traceId: '00-12345678901234567890123456789012-3456789012345678-01' } + )); test('Ignore media missing URL', () => withTestContext(async () => { diff --git a/packages/server/src/workers/download.ts b/packages/server/src/workers/download.ts index 28a3838b87..3bb3da9858 100644 --- a/packages/server/src/workers/download.ts +++ b/packages/server/src/workers/download.ts @@ -4,10 +4,11 @@ import { Job, Queue, QueueBaseOptions, Worker } from 'bullmq'; import fetch from 'node-fetch'; import { Readable } from 'stream'; import { getConfig, MedplumServerConfig } from '../config'; -import { getRequestContext } from '../context'; +import { getRequestContext, RequestContext, requestContextStore } from '../context'; import { getSystemRepo } from '../fhir/repo'; import { getBinaryStorage } from '../fhir/storage'; import { globalLogger } from '../logger'; +import { parseTraceparent } from '../traceparent'; /* * The download worker inspects resources, @@ -24,6 +25,8 @@ export interface DownloadJobData { readonly resourceType: string; readonly id: string; readonly url: string; + readonly requestId: string; + readonly traceId: string; } const queueName = 'DownloadQueue'; @@ -53,10 +56,15 @@ export function initDownloadWorker(config: MedplumServerConfig): void { }, }); - worker = new Worker(queueName, execDownloadJob, { - ...defaultOptions, - ...config.bullmq, - }); + worker = new Worker( + queueName, + (job) => + requestContextStore.run(new RequestContext(job.data.requestId, job.data.traceId), () => execDownloadJob(job)), + { + ...defaultOptions, + ...config.bullmq, + } + ); worker.on('completed', (job) => globalLogger.info(`Completed job ${job.id} successfully`)); worker.on('failed', (job, err) => globalLogger.info(`Failed job ${job?.id} with ${err}`)); } @@ -102,12 +110,15 @@ export function getDownloadQueue(): Queue | undefined { * @param resource - The resource that was created or updated. */ export async function addDownloadJobs(resource: Resource): Promise { + const ctx = getRequestContext(); for (const attachment of getAttachments(resource)) { if (isExternalUrl(attachment.url)) { await addDownloadJobData({ resourceType: resource.resourceType, id: resource.id as string, url: attachment.url, + requestId: ctx.requestId, + traceId: ctx.traceId, }); } } @@ -172,9 +183,18 @@ export async function execDownloadJob(job: Job): Promise return; } + const headers: HeadersInit = {}; + const traceId = job.data.traceId; + headers['x-trace-id'] = traceId; + if (parseTraceparent(traceId)) { + headers['traceparent'] = traceId; + } + try { ctx.logger.info('Requesting content at: ' + url); - const response = await fetch(url); + const response = await fetch(url, { + headers, + }); ctx.logger.info('Received status: ' + response.status); if (response.status >= 400) { diff --git a/packages/server/src/workers/subscription.test.ts b/packages/server/src/workers/subscription.test.ts index 20a2f181cf..b1b1c65441 100644 --- a/packages/server/src/workers/subscription.test.ts +++ b/packages/server/src/workers/subscription.test.ts @@ -182,49 +182,54 @@ describe('Subscription Worker', () => { })); test('Send subscription with custom headers', () => - withTestContext(async () => { - const url = 'https://example.com/subscription'; - - const subscription = await repo.createResource({ - resourceType: 'Subscription', - reason: 'test', - status: 'active', - criteria: 'Patient', - channel: { - type: 'rest-hook', - endpoint: url, - header: ['Authorization: Basic xyz'], - }, - }); - expect(subscription).toBeDefined(); - - const queue = getSubscriptionQueue() as any; - queue.add.mockClear(); - - const patient = await repo.createResource({ - resourceType: 'Patient', - name: [{ given: ['Alice'], family: 'Smith' }], - }); - expect(patient).toBeDefined(); - expect(queue.add).toHaveBeenCalled(); - - (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); - - const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; - await execSubscriptionJob(job); - - expect(fetch).toHaveBeenCalledWith( - url, - expect.objectContaining({ - method: 'POST', - body: stringify(patient), - headers: { - 'Content-Type': ContentType.FHIR_JSON, - Authorization: 'Basic xyz', + withTestContext( + async () => { + const url = 'https://example.com/subscription'; + + const subscription = await repo.createResource({ + resourceType: 'Subscription', + reason: 'test', + status: 'active', + criteria: 'Patient', + channel: { + type: 'rest-hook', + endpoint: url, + header: ['Authorization: Basic xyz'], }, - }) - ); - })); + }); + expect(subscription).toBeDefined(); + + const queue = getSubscriptionQueue() as any; + queue.add.mockClear(); + + const patient = await repo.createResource({ + resourceType: 'Patient', + name: [{ given: ['Alice'], family: 'Smith' }], + }); + expect(patient).toBeDefined(); + expect(queue.add).toHaveBeenCalled(); + + (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); + + const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; + await execSubscriptionJob(job); + + expect(fetch).toHaveBeenCalledWith( + url, + expect.objectContaining({ + method: 'POST', + body: stringify(patient), + headers: { + 'Content-Type': ContentType.FHIR_JSON, + Authorization: 'Basic xyz', + 'x-trace-id': '00-12345678901234567890123456789012-3456789012345678-01', + traceparent: '00-12345678901234567890123456789012-3456789012345678-01', + }, + }) + ); + }, + { traceId: '00-12345678901234567890123456789012-3456789012345678-01' } + )); test('Create-only subscription', () => withTestContext(async () => { @@ -291,173 +296,188 @@ describe('Subscription Worker', () => { })); test('Delete-only subscription', () => - withTestContext(async () => { - const url = 'https://example.com/subscription'; - - const subscription = await repo.createResource({ - resourceType: 'Subscription', - reason: 'test', - status: 'active', - criteria: 'Patient', - channel: { - type: 'rest-hook', - endpoint: url, - }, - extension: [ - { - url: 'https://medplum.com/fhir/StructureDefinition/subscription-supported-interaction', - valueCode: 'delete', + withTestContext( + async () => { + const url = 'https://example.com/subscription'; + + const subscription = await repo.createResource({ + resourceType: 'Subscription', + reason: 'test', + status: 'active', + criteria: 'Patient', + channel: { + type: 'rest-hook', + endpoint: url, }, - ], - }); - expect(subscription).toBeDefined(); - - // Clear the queue - const queue = getSubscriptionQueue() as any; - queue.add.mockClear(); - - // Create the patient - const patient = await repo.createResource({ - resourceType: 'Patient', - name: [{ given: ['Alice'], family: 'Smith' }], - }); - expect(patient).toBeDefined(); - - // Create should trigger the subscription - expect(queue.add).not.toHaveBeenCalled(); - - // Update the patient - await repo.updateResource({ ...patient, active: true }); - - // Update should not trigger the subscription - expect(queue.add).not.toHaveBeenCalled(); - - // Delete the patient - await repo.deleteResource('Patient', patient.id as string); - - expect(queue.add).toHaveBeenCalled(); - const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; - await execSubscriptionJob(job); - expect(fetch).toHaveBeenCalledWith( - url, - expect.objectContaining({ - method: 'POST', - body: '{}', - headers: { - 'Content-Type': ContentType.FHIR_JSON, - 'X-Medplum-Deleted-Resource': `Patient/${patient.id}`, - }, - }) - ); - })); + extension: [ + { + url: 'https://medplum.com/fhir/StructureDefinition/subscription-supported-interaction', + valueCode: 'delete', + }, + ], + }); + expect(subscription).toBeDefined(); + + // Clear the queue + const queue = getSubscriptionQueue() as any; + queue.add.mockClear(); + + // Create the patient + const patient = await repo.createResource({ + resourceType: 'Patient', + name: [{ given: ['Alice'], family: 'Smith' }], + }); + expect(patient).toBeDefined(); + + // Create should trigger the subscription + expect(queue.add).not.toHaveBeenCalled(); + + // Update the patient + await repo.updateResource({ ...patient, active: true }); + + // Update should not trigger the subscription + expect(queue.add).not.toHaveBeenCalled(); + + // Delete the patient + await repo.deleteResource('Patient', patient.id as string); + + expect(queue.add).toHaveBeenCalled(); + const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; + await execSubscriptionJob(job); + expect(fetch).toHaveBeenCalledWith( + url, + expect.objectContaining({ + method: 'POST', + body: '{}', + headers: { + 'Content-Type': ContentType.FHIR_JSON, + 'X-Medplum-Deleted-Resource': `Patient/${patient.id}`, + 'x-trace-id': '00-12345678901234567890123456789012-3456789012345678-01', + traceparent: '00-12345678901234567890123456789012-3456789012345678-01', + }, + }) + ); + }, + { traceId: '00-12345678901234567890123456789012-3456789012345678-01' } + )); test('Send subscriptions with signature', () => - withTestContext(async () => { - const url = 'https://example.com/subscription'; - const secret = '0123456789'; - - const subscription = await repo.createResource({ - resourceType: 'Subscription', - reason: 'test', - status: 'active', - criteria: 'Patient', - channel: { - type: 'rest-hook', - endpoint: url, - }, - extension: [ - { - url: 'https://www.medplum.com/fhir/StructureDefinition/subscription-secret', - valueString: secret, + withTestContext( + async () => { + const url = 'https://example.com/subscription'; + const secret = '0123456789'; + + const subscription = await repo.createResource({ + resourceType: 'Subscription', + reason: 'test', + status: 'active', + criteria: 'Patient', + channel: { + type: 'rest-hook', + endpoint: url, }, - ], - }); - expect(subscription).toBeDefined(); - - const queue = getSubscriptionQueue() as any; - queue.add.mockClear(); - - const patient = await repo.createResource({ - resourceType: 'Patient', - name: [{ given: ['Alice'], family: 'Smith' }], - }); - expect(patient).toBeDefined(); - expect(queue.add).toHaveBeenCalled(); - - (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); - - const body = stringify(patient); - const signature = createHmac('sha256', secret).update(body).digest('hex'); - - const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; - await execSubscriptionJob(job); - - expect(fetch).toHaveBeenCalledWith( - url, - expect.objectContaining({ - method: 'POST', - body, - headers: { - 'Content-Type': ContentType.FHIR_JSON, - 'X-Signature': signature, - }, - }) - ); - })); + extension: [ + { + url: 'https://www.medplum.com/fhir/StructureDefinition/subscription-secret', + valueString: secret, + }, + ], + }); + expect(subscription).toBeDefined(); + + const queue = getSubscriptionQueue() as any; + queue.add.mockClear(); + + const patient = await repo.createResource({ + resourceType: 'Patient', + name: [{ given: ['Alice'], family: 'Smith' }], + }); + expect(patient).toBeDefined(); + expect(queue.add).toHaveBeenCalled(); + + (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); + + const body = stringify(patient); + const signature = createHmac('sha256', secret).update(body).digest('hex'); + + const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; + await execSubscriptionJob(job); + + expect(fetch).toHaveBeenCalledWith( + url, + expect.objectContaining({ + method: 'POST', + body, + headers: { + 'Content-Type': ContentType.FHIR_JSON, + 'X-Signature': signature, + 'x-trace-id': '00-12345678901234567890123456789012-3456789012345678-01', + traceparent: '00-12345678901234567890123456789012-3456789012345678-01', + }, + }) + ); + }, + { traceId: '00-12345678901234567890123456789012-3456789012345678-01' } + )); test('Send subscriptions with legacy signature extension', () => - withTestContext(async () => { - const url = 'https://example.com/subscription'; - const secret = '0123456789'; - - const subscription = await repo.createResource({ - resourceType: 'Subscription', - reason: 'test', - status: 'active', - criteria: 'Patient', - channel: { - type: 'rest-hook', - endpoint: url, - }, - extension: [ - { - url: 'https://www.medplum.com/fhir/StructureDefinition-subscriptionSecret', - valueString: secret, - }, - ], - }); - expect(subscription).toBeDefined(); - - const queue = getSubscriptionQueue() as any; - queue.add.mockClear(); - - const patient = await repo.createResource({ - resourceType: 'Patient', - name: [{ given: ['Alice'], family: 'Smith' }], - }); - expect(patient).toBeDefined(); - expect(queue.add).toHaveBeenCalled(); - - (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); - - const body = stringify(patient); - const signature = createHmac('sha256', secret).update(body).digest('hex'); - - const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; - await execSubscriptionJob(job); - - expect(fetch).toHaveBeenCalledWith( - url, - expect.objectContaining({ - method: 'POST', - body, - headers: { - 'Content-Type': ContentType.FHIR_JSON, - 'X-Signature': signature, + withTestContext( + async () => { + const url = 'https://example.com/subscription'; + const secret = '0123456789'; + + const subscription = await repo.createResource({ + resourceType: 'Subscription', + reason: 'test', + status: 'active', + criteria: 'Patient', + channel: { + type: 'rest-hook', + endpoint: url, }, - }) - ); - })); + extension: [ + { + url: 'https://www.medplum.com/fhir/StructureDefinition-subscriptionSecret', + valueString: secret, + }, + ], + }); + expect(subscription).toBeDefined(); + + const queue = getSubscriptionQueue() as any; + queue.add.mockClear(); + + const patient = await repo.createResource({ + resourceType: 'Patient', + name: [{ given: ['Alice'], family: 'Smith' }], + }); + expect(patient).toBeDefined(); + expect(queue.add).toHaveBeenCalled(); + + (fetch as unknown as jest.Mock).mockImplementation(() => ({ status: 200 })); + + const body = stringify(patient); + const signature = createHmac('sha256', secret).update(body).digest('hex'); + + const job = { id: 1, data: queue.add.mock.calls[0][1] } as unknown as Job; + await execSubscriptionJob(job); + + expect(fetch).toHaveBeenCalledWith( + url, + expect.objectContaining({ + method: 'POST', + body, + headers: { + 'Content-Type': ContentType.FHIR_JSON, + 'X-Signature': signature, + 'x-trace-id': '00-12345678901234567890123456789012-3456789012345678-01', + traceparent: '00-12345678901234567890123456789012-3456789012345678-01', + }, + }) + ); + }, + { traceId: '00-12345678901234567890123456789012-3456789012345678-01' } + )); test('Ignore non-subscription subscriptions', () => withTestContext(async () => { diff --git a/packages/server/src/workers/subscription.ts b/packages/server/src/workers/subscription.ts index a66a994803..e322acff4b 100644 --- a/packages/server/src/workers/subscription.ts +++ b/packages/server/src/workers/subscription.ts @@ -18,7 +18,7 @@ import { createHmac } from 'crypto'; import fetch, { HeadersInit } from 'node-fetch'; import { URL } from 'url'; import { MedplumServerConfig } from '../config'; -import { getRequestContext } from '../context'; +import { getRequestContext, RequestContext, requestContextStore } from '../context'; import { executeBot } from '../fhir/operations/execute'; import { getSystemRepo, Repository } from '../fhir/repo'; import { globalLogger } from '../logger'; @@ -27,6 +27,7 @@ import { createSubEventNotification } from '../subscriptions/websockets'; import { AuditEventOutcome } from '../util/auditevent'; import { BackgroundJobContext, BackgroundJobInteraction } from './context'; import { createAuditEvent, findProjectMembership, isFhirCriteriaMet, isJobSuccessful } from './utils'; +import { parseTraceparent } from '../traceparent'; /** * The upper limit on the number of times a job can be retried. @@ -55,6 +56,8 @@ export interface SubscriptionJobData { readonly versionId: string; readonly interaction: 'create' | 'update' | 'delete'; readonly requestTime: string; + readonly requestId: string; + readonly traceId: string; } const queueName = 'SubscriptionQueue'; @@ -84,10 +87,15 @@ export function initSubscriptionWorker(config: MedplumServerConfig): void { }, }); - worker = new Worker(queueName, execSubscriptionJob, { - ...defaultOptions, - ...config.bullmq, - }); + worker = new Worker( + queueName, + (job) => + requestContextStore.run(new RequestContext(job.data.requestId, job.data.traceId), () => execSubscriptionJob(job)), + { + ...defaultOptions, + ...config.bullmq, + } + ); worker.on('completed', (job) => globalLogger.info(`Completed job ${job.id} successfully`)); worker.on('failed', (job, err) => globalLogger.info(`Failed job ${job?.id} with ${err}`)); } @@ -152,6 +160,8 @@ export async function addSubscriptionJobs(resource: Resource, context: Backgroun versionId: resource.meta?.versionId as string, interaction: context.interaction, requestTime, + requestId: ctx.requestId, + traceId: ctx.traceId, }); } } @@ -404,6 +414,11 @@ async function sendRestHook( if (interaction === 'delete') { headers['X-Medplum-Deleted-Resource'] = `${resource.resourceType}/${resource.id}`; } + const traceId = job.data.traceId; + headers['x-trace-id'] = traceId; + if (parseTraceparent(traceId)) { + headers['traceparent'] = traceId; + } const body = interaction === 'delete' ? '{}' : stringify(resource); let error: Error | undefined = undefined; @@ -487,10 +502,11 @@ async function execBot( interaction: BackgroundJobInteraction, requestTime: string ): Promise { + const ctx = getRequestContext(); const url = subscription.channel?.endpoint as string; if (!url) { // This can happen if a user updates the Subscription after the job is created. - getRequestContext().logger.debug(`Ignore rest hook missing URL`); + ctx.logger.debug(`Ignore rest hook missing URL`); return; } @@ -517,6 +533,7 @@ async function execBot( input: interaction === 'delete' ? { deletedResource: resource } : resource, contentType: ContentType.FHIR_JSON, requestTime, + traceId: ctx.traceId, }); }