From 05580176653b54ad7d0b90c0a6e14cada60c9c79 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Thu, 13 Jun 2024 18:42:51 -0400 Subject: [PATCH 01/13] feat(ihe): retries expo backoff jitter refactor Refs: #1667 Signed-off-by: Jonah Kaye --- .../carequality/ihe-gateway-v2/gateways.ts | 9 +- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 52 ++++---- .../ihe-gateway-v2/monitor/store.ts | 4 +- .../outbound/__tests__/process-dq.test.ts | 20 +-- .../outbound/__tests__/process-dr.test.ts | 20 +-- .../outbound/xca/create/iti38-envelope.ts | 6 +- .../outbound/xca/create/iti39-envelope.ts | 6 +- .../xca/orchestrate/send-process-retry.ts | 120 +++++++++++++++++ .../outbound/xca/process/dq-response.ts | 68 ++++++---- .../outbound/xca/process/dr-response.ts | 15 ++- .../outbound/xca/process/error.ts | 32 ++++- .../outbound/xca/send/dq-requests.ts | 122 +++++++++--------- .../outbound/xca/send/dr-requests.ts | 122 +++++++++--------- .../outbound/xcpd/process/xcpd-response.ts | 4 + packages/ihe-gateway-sdk/src/models/shared.ts | 2 + packages/utils/src/saml/pre-prod-tester.ts | 73 +++++++---- packages/utils/src/saml/saml-server.ts | 52 ++++---- 17 files changed, 467 insertions(+), 260 deletions(-) create mode 100644 packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/gateways.ts b/packages/core/src/external/carequality/ihe-gateway-v2/gateways.ts index a68e1eacf8..734d53ff75 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/gateways.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/gateways.ts @@ -21,13 +21,18 @@ const specialNamespaceRequiredUrl = const pointClickCareOid = "2.16.840.1.113883.3.6448"; const redoxOid = "2.16.840.1.113883.3.6147.458"; const redoxGatewayOid = "2.16.840.1.113883.3.6147.458.2"; +const surescriptsOid = "2.16.840.1.113883.3.2054.2.1.1"; /* * These gateways only accept a single document reference per request. */ -const gatewaysThatAcceptOneDocRefPerRequest = [pointClickCareOid, redoxOid, redoxGatewayOid]; +const gatewaysThatAcceptOneDocRefPerRequest = [ + pointClickCareOid, + redoxOid, + redoxGatewayOid, + surescriptsOid, +]; -const surescriptsOid = "2.16.840.1.113883.3.2054.2.1.1"; /* * These gateways require a urn:uuid prefix before document Unique ids formatted as lowercase uuids */ diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 51d90cd8dd..062efeb72c 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -10,11 +10,11 @@ import { createAndSignBulkXCPDRequests } from "./outbound/xcpd/create/iti55-enve import { createAndSignBulkDQRequests } from "./outbound/xca/create/iti38-envelope"; import { createAndSignBulkDRRequests } from "./outbound/xca/create/iti39-envelope"; import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; -import { sendSignedDQRequests } from "./outbound/xca/send/dq-requests"; -import { sendSignedDRRequests } from "./outbound/xca/send/dr-requests"; import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; -import { processDQResponse } from "./outbound/xca/process/dq-response"; -import { processDrResponse } from "./outbound/xca/process/dr-response"; +import { + sendProcessRetryDqRequests, + sendProcessRetryDrRequests, +} from "./outbound/xca/orchestrate/send-process-retry"; import { capture } from "../../../util/notifications"; export async function createSignSendProcessXCPDRequest({ @@ -75,17 +75,19 @@ export async function createSignSendProcessDQRequests({ bulkBodyData: dqRequestsGatewayV2, samlCertsAndKeys, }); - const responses = await sendSignedDQRequests({ - signedRequests, - samlCertsAndKeys, - patientId, - cxId, - }); - const results = responses.map(response => { - return processDQResponse({ - dqResponse: response, + + const resultPromises = signedRequests.map(async (signedRequest, index) => { + return sendProcessRetryDqRequests({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, }); }); + + const results = await Promise.all(resultPromises); + for (const result of results) { try { await axios.post(dqResponseUrl, result); @@ -117,19 +119,19 @@ export async function createSignSendProcessDRRequests({ bulkBodyData: drRequestsGatewayV2, samlCertsAndKeys, }); - const responses = await sendSignedDRRequests({ - signedRequests, - samlCertsAndKeys, - patientId, - cxId, + + const resultPromises = signedRequests.map(async (signedRequest, index) => { + return sendProcessRetryDrRequests({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + }); }); - const results = await Promise.all( - responses.map(async response => { - return await processDrResponse({ - drResponse: response, - }); - }) - ); + + const results = await Promise.all(resultPromises); + for (const result of results) { try { await axios.post(drResponseUrl, result); diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/monitor/store.ts b/packages/core/src/external/carequality/ihe-gateway-v2/monitor/store.ts index 5685d02a49..0f2b53d93e 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/monitor/store.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/monitor/store.ts @@ -56,7 +56,7 @@ export async function storeXcpdResponses({ } } -export async function storeDqResponses({ +export async function storeDqResponse({ response, outboundRequest, gateway, @@ -90,7 +90,7 @@ export async function storeDqResponses({ } } -export async function storeDrResponses({ +export async function storeDrResponse({ response, outboundRequest, gateway, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dq.test.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dq.test.ts index 0c20e1701a..318dedd7ba 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dq.test.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dq.test.ts @@ -1,13 +1,13 @@ import fs from "fs"; import path from "path"; -import { processDQResponse } from "../xca/process/dq-response"; +import { processDqResponse } from "../xca/process/dq-response"; import { outboundDqRequest, expectedDqDocumentReference } from "./constants"; -describe("processDQResponse", () => { +describe("processDqResponse", () => { it("should process the successful DQ response correctly", async () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dq_multiple_docs.xml"), "utf8"); - const response = processDQResponse({ - dqResponse: { + const response = await processDqResponse({ + response: { response: xmlString, success: true, gateway: outboundDqRequest.gateway, @@ -21,8 +21,8 @@ describe("processDQResponse", () => { }); it("should process the empty DQ response correctly", async () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dq_empty.xml"), "utf8"); - const response = processDQResponse({ - dqResponse: { + const response = await processDqResponse({ + response: { response: xmlString, success: true, gateway: outboundDqRequest.gateway, @@ -34,8 +34,8 @@ describe("processDQResponse", () => { it("should process the DQ response with registry error correctly", async () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dq_error.xml"), "utf8"); - const response = processDQResponse({ - dqResponse: { + const response = await processDqResponse({ + response: { response: xmlString, success: true, gateway: outboundDqRequest.gateway, @@ -48,8 +48,8 @@ describe("processDQResponse", () => { it("should process response that is not a string correctly", async () => { const randomResponse = "This is a bad response and is not xml"; - const response = processDQResponse({ - dqResponse: { + const response = await processDqResponse({ + response: { success: true, response: randomResponse, gateway: outboundDqRequest.gateway, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dr.test.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dr.test.ts index 70f8d6f8e0..89e3f1e335 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dr.test.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/process-dr.test.ts @@ -6,7 +6,7 @@ import { outboundDrRequest, testFiles, outboundDrRequestMtom } from "./constants import { S3Utils } from "../../../../aws/s3"; import { createMtomMessageWithAttachments, createMtomMessageWithoutAttachments } from "./mtom"; -describe("processDRResponse for MTOM with/without attachments and for different file types", () => { +describe("processDrResponse for MTOM with/without attachments and for different file types", () => { beforeEach(() => { jest.spyOn(S3Utils.prototype, "uploadFile").mockImplementation(() => { return Promise.resolve({ @@ -36,7 +36,7 @@ describe("processDRResponse for MTOM with/without attachments and for different const attachmentsData = [{ payload: fileBuffer, mimeType: mimeType }]; const mtomAttachments = await createMtomMessageWithAttachments(attachmentsData); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomAttachments, gateway: outboundDrRequestMtom.gateway, outboundRequest: outboundDrRequestMtom, @@ -63,7 +63,7 @@ describe("processDRResponse for MTOM with/without attachments and for different const mtomResponse = await createMtomMessageWithoutAttachments(modifiedXml); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: { @@ -104,7 +104,7 @@ describe("processDRResponse for MTOM with/without attachments and for different const mtomAttachments = await createMtomMessageWithAttachments(attachmentsData); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomAttachments, gateway: outboundDrRequestMtom.gateway, outboundRequest: outboundDrRequestMtom, @@ -120,7 +120,7 @@ describe("processDRResponse for MTOM with/without attachments and for different expect(response.documentReference?.[0]?.contentType).toBe(xmlFile.mimeType); }); }); -describe("processDRResponse", () => { +describe("processDrResponse", () => { beforeEach(() => { jest.spyOn(S3Utils.prototype, "uploadFile").mockImplementation(() => Promise.resolve({ @@ -140,7 +140,7 @@ describe("processDRResponse", () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dr_success.xml"), "utf8"); const mtomResponse = await createMtomMessageWithoutAttachments(xmlString); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: outboundDrRequest, @@ -166,7 +166,7 @@ describe("processDRResponse", () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dr_soap_error.xml"), "utf8"); const mtomResponse = await createMtomMessageWithoutAttachments(xmlString); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: outboundDrRequest, @@ -180,7 +180,7 @@ describe("processDRResponse", () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dr_registry_error.xml"), "utf8"); const mtomResponse = await createMtomMessageWithoutAttachments(xmlString); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: outboundDrRequest, @@ -193,7 +193,7 @@ describe("processDRResponse", () => { const xmlString = fs.readFileSync(path.join(__dirname, "xmls/dr_empty.xml"), "utf8"); const mtomResponse = await createMtomMessageWithoutAttachments(xmlString); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: outboundDrRequest, @@ -205,7 +205,7 @@ describe("processDRResponse", () => { const randomResponse = "This is a bad response and is not xml"; const mtomResponse = await createMtomMessageWithoutAttachments(randomResponse); const response = await processDrResponse({ - drResponse: { + response: { mtomResponse: mtomResponse, gateway: outboundDrRequest.gateway, outboundRequest: outboundDrRequest, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope.ts index 3d9030584e..b325a88d20 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope.ts @@ -15,7 +15,7 @@ const stableDocumentType = "7edca82f-054d-47f2-a032-9b2a5b5186c1"; const onDemandDocumentType = "34268e47-fdf5-41a6-ba33-82133c465248"; const dateFormat = "YYYYMMDDHHmmss"; -export type BulkSignedDQ = { +export type SignedDqRequest = { gateway: XCAGateway; signedRequest: string; outboundRequest: OutboundDocumentQueryReq; @@ -244,8 +244,8 @@ export function createAndSignBulkDQRequests({ }: { bulkBodyData: OutboundDocumentQueryReq[]; samlCertsAndKeys: SamlCertsAndKeys; -}): BulkSignedDQ[] { - const signedRequests: BulkSignedDQ[] = []; +}): SignedDqRequest[] { + const signedRequests: SignedDqRequest[] = []; for (const bodyData of bulkBodyData) { const signedRequest = createAndSignDQRequest(bodyData, samlCertsAndKeys); diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope.ts index e3dd233a65..458ad36e8b 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope.ts @@ -19,7 +19,7 @@ const action = "urn:ihe:iti:2007:CrossGatewayRetrieve"; const minDocumentReferencesPerDrRequest = 1; const maxDocumentReferencesPerDrRequest = 10; -export type BulkSignedDR = { +export type SignedDrRequest = { gateway: XCAGateway; signedRequest: string; outboundRequest: OutboundDocumentRetrievalReq; @@ -117,8 +117,8 @@ export function createAndSignBulkDRRequests({ }: { bulkBodyData: OutboundDocumentRetrievalReq[]; samlCertsAndKeys: SamlCertsAndKeys; -}): BulkSignedDR[] { - const signedRequests: BulkSignedDR[] = []; +}): SignedDrRequest[] { + const signedRequests: SignedDrRequest[] = []; for (const bodyData of bulkBodyData) { const documentReferencesPerRequest = requiresOnlyOneDocRefPerRequest(bodyData.gateway) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts new file mode 100644 index 0000000000..14bbd58f85 --- /dev/null +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts @@ -0,0 +1,120 @@ +import { isRetryableError } from "../process/error"; +import { SamlCertsAndKeys } from "../../../saml/security/types"; +import { + OutboundDocumentRetrievalResp, + OutboundDocumentQueryResp, +} from "@metriport/ihe-gateway-sdk"; +import { SignedDqRequest } from "../create/iti38-envelope"; +import { SignedDrRequest } from "../create/iti39-envelope"; +import { sendSignedDqRequest } from "../send/dq-requests"; +import { sendSignedDrRequest } from "../send/dr-requests"; +import { processDqResponse } from "../process/dq-response"; +import { processDrResponse } from "../process/dr-response"; +import { out } from "../../../../../../util/log"; + +const { log } = out("IHE Gateway V2"); + +function calculateBackoff(attempt: number, baseDelay = 2000, jitterRange = 2000): number { + const baseBackoffTime = Math.pow(2, attempt + 1) * baseDelay; + const jitter = (Math.random() - 0.5) * jitterRange; + return baseBackoffTime + jitter; +} + +export async function sendProcessRetryRequests({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + sendRequest, + processResponse, + maxRetries = 2, +}: { + signedRequest: T; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; + sendRequest: (params: { + request: T; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; + }) => Promise; + processResponse: (params: { + response: R; + attempt: number; + }) => Promise; + maxRetries?: number; +}): Promise { + let attempt = 0; + + while (attempt < maxRetries) { + const response = await sendRequest({ + request: signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + }); + const result = await processResponse({ + response, + attempt, + }); + + if (!isRetryableError(result)) { + return result; + } + + attempt++; + const backoffTime = calculateBackoff(attempt); + await new Promise(resolve => setTimeout(resolve, backoffTime)); + log(`Attempt ${attempt + 1} of ${maxRetries + 1}`); + } + + const finalBackoffTime = calculateBackoff(maxRetries); + await new Promise(resolve => setTimeout(resolve, finalBackoffTime)); + + log(`Attempt ${maxRetries + 1} of ${maxRetries + 1}`); + const finalResponse = await sendRequest({ + request: signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + }); + const result = await processResponse({ + response: finalResponse, + attempt: maxRetries, + }); + return result; +} + +export async function sendProcessRetryDrRequests(params: { + signedRequest: SignedDrRequest; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; +}): Promise { + return sendProcessRetryRequests({ + ...params, + sendRequest: sendSignedDrRequest, + processResponse: processDrResponse, + }); +} + +export async function sendProcessRetryDqRequests(params: { + signedRequest: SignedDqRequest; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; +}): Promise { + return sendProcessRetryRequests({ + ...params, + sendRequest: sendSignedDqRequest, + processResponse: processDqResponse, + }); +} diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts index 4493a6c1d0..f69e747d1f 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts @@ -17,12 +17,9 @@ import { } from "../../../../shared"; import { successStatus, partialSuccessStatus } from "./constants"; import { capture } from "../../../../../../util/notifications"; -import { out } from "../../../../../../util/log"; dayjs.extend(utc); -const { log } = out("DQ Processing"); - type Identifier = { _identificationScheme: string; _value: string; @@ -64,7 +61,6 @@ function getCreationTime(time: string | undefined): string | undefined { try { return time ? dayjs.utc(time).toISOString() : undefined; } catch (error) { - log(`Error parsing creation time: ${time}, error: ${error}`); return undefined; } } @@ -166,11 +162,13 @@ function handleSuccessResponse({ extrinsicObjects, outboundRequest, gateway, + attempt, }: { //eslint-disable-next-line @typescript-eslint/no-explicit-any extrinsicObjects: any; outboundRequest: OutboundDocumentQueryReq; gateway: XCAGateway; + attempt?: number | undefined; }): OutboundDocumentQueryResp { const documentReferences = Array.isArray(extrinsicObjects) ? extrinsicObjects.flatMap( @@ -186,21 +184,28 @@ function handleSuccessResponse({ gateway, documentReference: documentReferences, externalGatewayPatient: outboundRequest.externalGatewayPatient, + retried: attempt, + iheGatewayV2: true, }; return response; } -export function processDQResponse({ - dqResponse: { response, success, gateway, outboundRequest }, +export function processDqResponse({ + response: { response, success, gateway, outboundRequest }, + attempt, }: { - dqResponse: DQSamlClientResponse; -}): OutboundDocumentQueryResp { + response: DQSamlClientResponse; + attempt?: number | undefined; +}): Promise { if (success === false) { - return handleHttpErrorResponse({ - httpError: response, - outboundRequest, - gateway: gateway, - }); + return Promise.resolve( + handleHttpErrorResponse({ + httpError: response, + outboundRequest, + gateway: gateway, + attempt, + }) + ); } const parser = new XMLParser({ ignoreAttributes: false, @@ -217,21 +222,30 @@ export function processDQResponse({ const registryErrorList = jsonObj?.Envelope?.Body?.AdhocQueryResponse?.RegistryErrorList; if ((status === successStatus || status === partialSuccessStatus) && extrinsicObjects) { - return handleSuccessResponse({ - extrinsicObjects, - outboundRequest, - gateway, - }); + return Promise.resolve( + handleSuccessResponse({ + extrinsicObjects, + outboundRequest, + gateway, + attempt, + }) + ); } else if (registryErrorList) { - return handleRegistryErrorResponse({ - registryErrorList, - outboundRequest, - gateway, - }); + return Promise.resolve( + handleRegistryErrorResponse({ + registryErrorList, + outboundRequest, + gateway, + attempt, + }) + ); } else { - return handleEmptyResponse({ - outboundRequest, - gateway, - }); + return Promise.resolve( + handleEmptyResponse({ + outboundRequest, + gateway, + attempt, + }) + ); } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts index 286d09850f..b223031acd 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts @@ -163,11 +163,13 @@ async function handleSuccessResponse({ outboundRequest, gateway, mtomResponse, + attempt, }: { documentResponses: DocumentResponse[]; outboundRequest: OutboundDocumentRetrievalReq; gateway: XCAGateway; mtomResponse: MtomAttachments; + attempt?: number | undefined; }): Promise { try { const idMapping = generateIdMapping(outboundRequest.documentReference); @@ -193,6 +195,8 @@ async function handleSuccessResponse({ responseTimestamp: dayjs().toISOString(), gateway, documentReference: documentReferences, + retried: attempt, + iheGatewayV2: true, }; return response; } catch (error) { @@ -201,9 +205,11 @@ async function handleSuccessResponse({ } export async function processDrResponse({ - drResponse: { errorResponse, mtomResponse, gateway, outboundRequest }, + response: { errorResponse, mtomResponse, gateway, outboundRequest }, + attempt, }: { - drResponse: DrSamlClientResponse; + response: DrSamlClientResponse; + attempt?: number; }): Promise { if (!gateway || !outboundRequest) throw new Error("Missing gateway or outboundRequest"); if (errorResponse) { @@ -211,6 +217,7 @@ export async function processDrResponse({ httpError: errorResponse, outboundRequest, gateway, + attempt, }); } if (!mtomResponse) { @@ -240,23 +247,27 @@ export async function processDrResponse({ outboundRequest, gateway, mtomResponse, + attempt, }); } else if (registryErrorList) { return handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, + attempt, }); } else if (soapFault) { return handleSoapFaultResponse({ soapFault, outboundRequest, gateway, + attempt, }); } else { return handleEmptyResponse({ outboundRequest, gateway, + attempt, }); } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 84d622dc3b..e4b0bbd17e 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -11,6 +11,7 @@ import { capture } from "../../../../../../util/notifications"; import { out } from "../../../../../../util/log"; const { log } = out("XCA Error Handling"); +const knownNonRetryableErrors = ["No active consent for patient id"]; export function processRegistryErrorList( //eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -30,7 +31,7 @@ export function processRegistryErrorList( //eslint-disable-next-line @typescript-eslint/no-explicit-any registryErrors.forEach((entry: any) => { const issue = { - severity: entry?._severity?.toString().toLowerCase().split(":").pop(), + severity: "error", code: entry?._errorCode?.toString(), details: { text: entry?._codeContext?.toString(), @@ -67,11 +68,13 @@ export function handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, + attempt, }: { //eslint-disable-next-line @typescript-eslint/no-explicit-any registryErrorList: any; outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; + attempt?: number | undefined; }): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { const operationOutcome = processRegistryErrorList(registryErrorList, outboundRequest); return { @@ -81,6 +84,8 @@ export function handleRegistryErrorResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, + retried: attempt, + iheGatewayV2: true, }; } @@ -88,10 +93,12 @@ export function handleHttpErrorResponse({ httpError, outboundRequest, gateway, + attempt, }: { httpError: string; outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; + attempt?: number | undefined; }): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", @@ -113,15 +120,19 @@ export function handleHttpErrorResponse({ gateway: gateway, patientId: outboundRequest.patientId, operationOutcome: operationOutcome, + retried: attempt, + iheGatewayV2: true, }; } export function handleEmptyResponse({ outboundRequest, gateway, + attempt, }: { outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; + attempt?: number | undefined; }): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", @@ -143,6 +154,8 @@ export function handleEmptyResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, + retried: attempt, + iheGatewayV2: true, }; } @@ -150,11 +163,13 @@ export function handleSoapFaultResponse({ soapFault, outboundRequest, gateway, + attempt, }: { //eslint-disable-next-line @typescript-eslint/no-explicit-any soapFault: any; outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; + attempt?: number | undefined; }): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { const faultCode = soapFault?.Code?.Value?.toString() ?? "unknown_fault"; const faultReason = @@ -181,5 +196,20 @@ export function handleSoapFaultResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, + retried: attempt, + iheGatewayV2: true, }; } + +export function isRetryableError(outboundRequest: OutboundDocumentRetrievalResp): boolean { + return ( + outboundRequest.operationOutcome?.issue.some( + issue => + issue.severity === "error" && + !knownNonRetryableErrors.some( + nonRetryableError => + "text" in issue.details && issue.details.text.includes(nonRetryableError) + ) + ) ?? false + ); +} diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts index 216222f5a7..21dd36dad9 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts @@ -3,9 +3,9 @@ import { errorToString } from "../../../../../../util/error/shared"; import { capture } from "../../../../../../util/notifications"; import { SamlCertsAndKeys } from "../../../saml/security/types"; import { getTrustedKeyStore, SamlClientResponse, sendSignedXml } from "../../../saml/saml-client"; -import { BulkSignedDQ } from "../create/iti38-envelope"; +import { SignedDqRequest } from "../create/iti38-envelope"; import { out } from "../../../../../../util/log"; -import { storeDqResponses } from "../../../monitor/store"; +import { storeDqResponse } from "../../../monitor/store"; const { log } = out("Sending DQ Requests"); const context = "ihe-gateway-v2-dq-saml-client"; @@ -15,73 +15,71 @@ export type DQSamlClientResponse = SamlClientResponse & { outboundRequest: OutboundDocumentQueryReq; }; -export async function sendSignedDQRequests({ - signedRequests, +export async function sendSignedDqRequest({ + request, samlCertsAndKeys, patientId, cxId, + index, }: { - signedRequests: BulkSignedDQ[]; + request: SignedDqRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; -}): Promise { + index: number; +}): Promise { const trustedKeyStore = await getTrustedKeyStore(); - const requestPromises = signedRequests.map(async (request, index) => { - try { - const { response } = await sendSignedXml({ - signedXml: request.signedRequest, - url: request.gateway.url, - samlCertsAndKeys, - trustedKeyStore, - }); - log( - `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ - request.gateway.homeCommunityId - }` - ); - await storeDqResponses({ - response, - outboundRequest: request.outboundRequest, - gateway: request.gateway, - }); - return { - gateway: request.gateway, - response, - success: true, - outboundRequest: request.outboundRequest, - }; - //eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - const msg = "HTTP/SSL Failure Sending Signed DQ SAML Request"; - log( - `${msg}, cxId: ${cxId}, patientId: ${patientId}, gateway: ${request.gateway.homeCommunityId}, error: ${error}` - ); - if (error?.response?.data) { - log(`error details: ${JSON.stringify(error?.response?.data)}`); - } - const errorString: string = errorToString(error); - const extra = { - errorString, - request, - patientId, - cxId, - }; - capture.error(msg, { - extra: { - context, - extra, - error, - }, - }); - return { - gateway: request.gateway, - outboundRequest: request.outboundRequest, - response: errorString, - success: false, - }; + try { + const { response } = await sendSignedXml({ + signedXml: request.signedRequest, + url: request.gateway.url, + samlCertsAndKeys, + trustedKeyStore, + }); + log( + `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ + request.gateway.homeCommunityId + }` + ); + await storeDqResponse({ + response, + outboundRequest: request.outboundRequest, + gateway: request.gateway, + }); + return { + gateway: request.gateway, + response, + success: true, + outboundRequest: request.outboundRequest, + }; + //eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + const msg = `HTTP/SSL Failure Sending Signed DQ SAML Request ${index + 1}`; + log( + `${msg}, cxId: ${cxId}, patientId: ${patientId}, gateway: ${request.gateway.homeCommunityId}, error: ${error}` + ); + if (error?.response?.data) { + log(`error details: ${JSON.stringify(error?.response?.data)}`); } - }); - - return await Promise.all(requestPromises); + const errorString: string = errorToString(error); + const extra = { + errorString, + request, + patientId, + cxId, + }; + capture.error(msg, { + extra: { + context, + extra, + error, + }, + }); + return { + gateway: request.gateway, + outboundRequest: request.outboundRequest, + response: errorString, + success: false, + }; + } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts index 58b0022a9f..253e2fe270 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts @@ -4,9 +4,9 @@ import { capture } from "../../../../../../util/notifications"; import { SamlCertsAndKeys } from "../../../saml/security/types"; import { getTrustedKeyStore, sendSignedXmlMtom } from "../../../saml/saml-client"; import { MtomAttachments } from "../mtom/parser"; -import { BulkSignedDR } from "../create/iti39-envelope"; +import { SignedDrRequest } from "../create/iti39-envelope"; import { out } from "../../../../../../util/log"; -import { storeDrResponses } from "../../../monitor/store"; +import { storeDrResponse } from "../../../monitor/store"; const { log } = out("Sending DR Requests"); const context = "ihe-gateway-v2-dr-saml-client"; @@ -18,74 +18,72 @@ export type DrSamlClientResponse = { outboundRequest: OutboundDocumentRetrievalReq; }; -export async function sendSignedDRRequests({ - signedRequests, +export async function sendSignedDrRequest({ + request, samlCertsAndKeys, patientId, cxId, + index, }: { - signedRequests: BulkSignedDR[]; + request: SignedDrRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; -}): Promise { + index: number; +}): Promise { const trustedKeyStore = await getTrustedKeyStore(); - const requestPromises = signedRequests.map(async (request, index) => { - try { - const { mtomParts, rawResponse } = await sendSignedXmlMtom({ - signedXml: request.signedRequest, - url: request.gateway.url, - samlCertsAndKeys, - trustedKeyStore, - }); - log( - `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ - request.gateway.homeCommunityId - }` - ); - await storeDrResponses({ - response: rawResponse, - outboundRequest: request.outboundRequest, - gateway: request.gateway, - }); - return { - gateway: request.gateway, - mtomResponse: mtomParts, - outboundRequest: request.outboundRequest, - }; - //eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - const msg = "HTTP/SSL Failure Sending Signed DR SAML Request"; - log( - `${msg}, cxId: ${cxId}, patientId: ${patientId}, gateway: ${request.gateway.homeCommunityId}, error: ${error}` - ); - if (error?.response?.data) { - const errorDetails = Buffer.isBuffer(error?.response?.data) - ? error.response.data.toString("utf-8") - : JSON.stringify(error?.response?.data); - log(`error details: ${errorDetails}`); - } - - const errorString: string = errorToString(error); - const extra = { - errorString, - request, - patientId, - cxId, - }; - capture.error(msg, { - extra: { - context, - ...extra, - }, - }); - return { - gateway: request.gateway, - outboundRequest: request.outboundRequest, - errorResponse: errorString, - }; + try { + const { mtomParts, rawResponse } = await sendSignedXmlMtom({ + signedXml: request.signedRequest, + url: request.gateway.url, + samlCertsAndKeys, + trustedKeyStore, + }); + log( + `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ + request.gateway.homeCommunityId + }` + ); + await storeDrResponse({ + response: rawResponse, + outboundRequest: request.outboundRequest, + gateway: request.gateway, + }); + return { + gateway: request.gateway, + mtomResponse: mtomParts, + outboundRequest: request.outboundRequest, + }; + //eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + const msg = "HTTP/SSL Failure Sending Signed DR SAML Request"; + log( + `${msg}, cxId: ${cxId}, patientId: ${patientId}, gateway: ${request.gateway.homeCommunityId}, error: ${error}` + ); + if (error?.response?.data) { + const errorDetails = Buffer.isBuffer(error?.response?.data) + ? error.response.data.toString("utf-8") + : JSON.stringify(error?.response?.data); + log(`error details: ${errorDetails}`); } - }); - return await Promise.all(requestPromises); + const errorString: string = errorToString(error); + const extra = { + errorString, + request, + patientId, + cxId, + }; + capture.error(msg, { + extra: { + context, + ...extra, + }, + }); + return { + gateway: request.gateway, + outboundRequest: request.outboundRequest, + errorResponse: errorString, + }; + } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts index af922dafe7..2b0437a3d5 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts @@ -153,6 +153,7 @@ function handleHTTPErrorResponse({ patientId: outboundRequest?.patientId, patientMatch: null, operationOutcome: operationOutcome, + iheGatewayV2: true, }; } @@ -200,6 +201,7 @@ function handlePatientMatchResponse({ patientMatch: true, gatewayHomeCommunityId: outboundRequest.samlAttributes.homeCommunityId, patientResource: patientResource, + iheGatewayV2: true, }; return response; @@ -243,6 +245,7 @@ function handlePatientErrorResponse({ patientId: outboundRequest.patientId, patientMatch: null, operationOutcome: operationOutcome, + iheGatewayV2: true, }; return response; } @@ -274,6 +277,7 @@ function handlePatientNoMatchResponse({ patientId: outboundRequest.patientId, patientMatch: false, operationOutcome: operationOutcome, + iheGatewayV2: true, }; return response; } diff --git a/packages/ihe-gateway-sdk/src/models/shared.ts b/packages/ihe-gateway-sdk/src/models/shared.ts index 758f167e38..8e52857041 100644 --- a/packages/ihe-gateway-sdk/src/models/shared.ts +++ b/packages/ihe-gateway-sdk/src/models/shared.ts @@ -86,6 +86,8 @@ export const baseResponseSchema = z.object({ externalGatewayPatient: externalGatewayPatientSchema.optional(), patientId: z.string().nullish(), operationOutcome: operationOutcomeSchema.optional(), + retried: z.number().optional(), + iheGatewayV2: z.boolean().optional(), }); export type BaseResponse = z.infer; diff --git a/packages/utils/src/saml/pre-prod-tester.ts b/packages/utils/src/saml/pre-prod-tester.ts index 80ecd4325d..49e851fa42 100644 --- a/packages/utils/src/saml/pre-prod-tester.ts +++ b/packages/utils/src/saml/pre-prod-tester.ts @@ -12,14 +12,11 @@ import { OutboundDocumentRetrievalResp, } from "@metriport/ihe-gateway-sdk"; import { createAndSignBulkDQRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope"; -import { sendSignedDQRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests"; -import { processDQResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response"; +import { sendSignedDqRequest } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests"; +import { processDqResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response"; import { createAndSignBulkDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope"; -import { sendSignedDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests"; -import { - processDrResponse, - setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse, -} from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; +import { sendProcessRetryDrRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { Config } from "@metriport/core/util/config"; import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; @@ -36,6 +33,8 @@ const s3utils = new MockS3Utils(Config.getAWSRegion()); setS3UtilsInstanceForStoringDrResponse(s3utils); setS3UtilsInstanceForStoringIheResponse(s3utils); +const athenaOid = "2.16.840.1.113883.3.564.1"; + const samlAttributes = { subjectId: "System User", subjectRole: { @@ -76,6 +75,33 @@ async function queryDatabaseForDQs() { } } +export async function queryDatabaseForDqsFromFailedDrs() { + const sqlDBCreds = getEnvVarOrFail("DB_CREDS"); + const sequelize = initDbPool(sqlDBCreds); + const query = ` + SELECT dqr.data + FROM document_retrieval_result drr + JOIN document_query_result dqr ON drr.request_id = dqr.request_id + WHERE drr.status = 'failure' + AND dqr.status = 'success' + AND drr.data->'gateway'->>'homeCommunityId' = '${athenaOid}' + AND dqr.data->'gateway'->>'homeCommunityId' = '${athenaOid}' + ORDER BY RANDOM() + LIMIT 10; + `; + try { + const results = await sequelize.query(query, { + type: QueryTypes.SELECT, + }); + sequelize.close(); + return results; + } catch (error) { + console.error("Error executing SQL query:", error); + sequelize.close(); + throw error; + } +} + async function getDrUrl(id: string): Promise { const sqlDBCreds = getEnvVarOrFail("DB_CREDS"); const sequelize = initDbPool(sqlDBCreds); @@ -165,7 +191,7 @@ async function DRIntegrationTest() { let runTimeErrorCount = 0; console.log("Querrying DB for DQs..."); - const results = await queryDatabaseForDQs(); + const results = await queryDatabaseForDqsFromFailedDrs(); console.log("Sending DQs and DRs..."); const promises = results.map(async result => { //eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -265,15 +291,17 @@ async function queryDQ(dqRequest: OutboundDocumentQueryReq): Promise { + return sendProcessRetryDrRequests({ + signedRequest, + samlCertsAndKeys, + patientId: uuidv4(), + cxId: uuidv4(), + index, + }); }); + const results = await Promise.all(resultPromises); + return results[0]; } catch (error) { console.log("Erroring drRequest", drRequest); throw error; diff --git a/packages/utils/src/saml/saml-server.ts b/packages/utils/src/saml/saml-server.ts index eae7bbf116..577a37cc1c 100644 --- a/packages/utils/src/saml/saml-server.ts +++ b/packages/utils/src/saml/saml-server.ts @@ -10,14 +10,12 @@ import { createAndSignBulkXCPDRequests } from "@metriport/core/external/carequal import { createAndSignBulkDQRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti38-envelope"; import { createAndSignBulkDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope"; import { sendSignedXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests"; -import { sendSignedDQRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests"; -import { sendSignedDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests"; import { processXCPDResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response"; -import { processDQResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response"; import { - processDrResponse, - setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse, -} from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; + sendProcessRetryDrRequests, + sendProcessRetryDqRequests, +} from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { Config } from "@metriport/core/util/config"; @@ -87,22 +85,21 @@ app.post("/xcadq", async (req: Request, res: Response) => { } try { - const xmlResponses = createAndSignBulkDQRequests({ + const signedRequests = createAndSignBulkDQRequests({ bulkBodyData: req.body, samlCertsAndKeys, }); - const responses = await sendSignedDQRequests({ - signedRequests: xmlResponses, - samlCertsAndKeys, - patientId: uuidv4(), - cxId: uuidv4(), - }); - const results = responses.map(response => { - return processDQResponse({ - dqResponse: response, + const resultPromises = signedRequests.map(async (signedRequest, index) => { + return sendProcessRetryDqRequests({ + signedRequest, + samlCertsAndKeys, + patientId: uuidv4(), + cxId: uuidv4(), + index, }); }); + const results = await Promise.all(resultPromises); res.type("application/json").send(results); } catch (error) { @@ -121,24 +118,21 @@ app.post("/xcadr", async (req: Request, res: Response) => { })); try { - const xmlResponses = createAndSignBulkDRRequests({ + const signedRequests = createAndSignBulkDRRequests({ bulkBodyData: req.body, samlCertsAndKeys, }); - const response = await sendSignedDRRequests({ - signedRequests: xmlResponses, - samlCertsAndKeys, - patientId: uuidv4(), - cxId: uuidv4(), + const resultPromises = signedRequests.map(async (signedRequest, index) => { + return sendProcessRetryDrRequests({ + signedRequest, + samlCertsAndKeys, + patientId: uuidv4(), + cxId: uuidv4(), + index, + }); }); - const results = await Promise.all( - response.map(async response => { - return processDrResponse({ - drResponse: response, - }); - }) - ); + const results = await Promise.all(resultPromises); res.type("application/json").send(results); } catch (error) { res.status(500).send({ detail: "Internal Server Error" }); From f350f8f0b720cfa1503c06a7b2270557560a796e Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Thu, 13 Jun 2024 20:16:18 -0400 Subject: [PATCH 02/13] feat(ihe): all settled Refs: #1667 Signed-off-by: Jonah Kaye --- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 062efeb72c..702fa02229 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -86,18 +86,20 @@ export async function createSignSendProcessDQRequests({ }); }); - const results = await Promise.all(resultPromises); + const results = await Promise.allSettled(resultPromises); for (const result of results) { - try { - await axios.post(dqResponseUrl, result); - } catch (error) { - capture.error("Failed to send DQ response to Internal Carequality Endpoint", { - extra: { - error, - result, - }, - }); + if (result.status === "fulfilled") { + try { + await axios.post(dqResponseUrl, result.value); + } catch (error) { + capture.error("Failed to send DQ response to Internal Carequality Endpoint", { + extra: { + error, + result: result.value, + }, + }); + } } } } @@ -130,18 +132,20 @@ export async function createSignSendProcessDRRequests({ }); }); - const results = await Promise.all(resultPromises); + const results = await Promise.allSettled(resultPromises); for (const result of results) { - try { - await axios.post(drResponseUrl, result); - } catch (error) { - capture.error("Failed to send DR response to Internal Carequality Endpoint", { - extra: { - error, - result, - }, - }); + if (result.status === "fulfilled") { + try { + await axios.post(drResponseUrl, result.value); + } catch (error) { + capture.error("Failed to send DR response to Internal Carequality Endpoint", { + extra: { + error, + result: result.value, + }, + }); + } } } } From 2b45d2b0094e683b3cd241e2a4b7339c93dc4f1c Mon Sep 17 00:00:00 2001 From: Rafael Leite <2132564+leite08@users.noreply.github.com> Date: Sat, 15 Jun 2024 16:11:55 -0500 Subject: [PATCH 03/13] build: e2e tests on branch to staging Ref. metriport/metriport-internal#1040 Signed-off-by: Rafael Leite <2132564+leite08@users.noreply.github.com> --- .github/workflows/branch-to-staging.yml | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/branch-to-staging.yml b/.github/workflows/branch-to-staging.yml index a77425e151..3619a9b88b 100644 --- a/.github/workflows/branch-to-staging.yml +++ b/.github/workflows/branch-to-staging.yml @@ -81,3 +81,30 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} IHE_GW_KEYSTORE_STOREPASS: ${{ secrets.IHE_GW_KEYSTORE_STOREPASS_STAGING }} IHE_GW_KEYSTORE_KEYPASS: ${{ secrets.IHE_GW_KEYSTORE_KEYPASS_STAGING }} + + e2e-tests: + uses: ./.github/workflows/_e2e-tests.yml + needs: [api, infra-api-lambdas, infra-ihe-gw, ihe-gw-server] + # run even if one of the dependencies didn't + # can't use ${{ ! failure() && success() }} because `success()` "Returns true when none of the previous steps have failed or been canceled." + # can't use ${{ ! failure() && contains(needs.*.result, 'success') }} because if anything that came before succeeded, even if not a direct dependency, it will run + if: ${{ !failure() && (needs.api.result == 'success' || needs.infra-api-lambdas.result == 'success' || needs.infra-ihe-gw.result == 'success' || needs.ihe-gw-server.result == 'success') }} + with: + deploy_env: "staging" + api_url: ${{ vars.API_URL_STAGING }} + fhir_url: ${{ vars.FHIR_SERVER_URL_STAGING }} + test_patient_id: ${{ vars.TEST_PATIENT_ID }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TEST_API_KEY: ${{ secrets.TEST_API_KEY_STAGING }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + NGROK_AUTHTOKEN: ${{ secrets.NGROK_AUTHTOKEN }} + CW_CERTIFICATE: ${{ secrets.CW_CERTIFICATE_STAGING }} + CW_PRIVATE_KEY: ${{ secrets.CW_PRIVATE_KEY_STAGING }} + CW_MEMBER_CERTIFICATE: ${{ secrets.CW_MEMBER_CERTIFICATE_STAGING }} + CW_MEMBER_PRIVATE_KEY: ${{ secrets.CW_MEMBER_PRIVATE_KEY_STAGING }} + CW_MEMBER_NAME: ${{ secrets.CW_MEMBER_NAME_STAGING }} + CW_MEMBER_OID: ${{ secrets.CW_MEMBER_OID_STAGING }} + \ No newline at end of file From 3154b59de5a19cfa58b8fbd7a647a9700e8f5c15 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Sun, 16 Jun 2024 17:13:15 -0400 Subject: [PATCH 04/13] feat(ihe): fixes Refs: #1667 Signed-off-by: Jonah Kaye --- .../xca/orchestrate/send-process-retry.ts | 11 +++++----- .../outbound/xca/process/dq-response.ts | 6 +++--- .../outbound/xca/process/dr-response.ts | 2 +- .../outbound/xca/process/error.ts | 14 ++++++------- .../outbound/xcpd/process/xcpd-response.ts | 2 +- packages/utils/src/saml/pre-prod-tester.ts | 20 +++++++++---------- packages/utils/src/saml/saml-server.ts | 2 +- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts index 14bbd58f85..cda5099a59 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts @@ -1,15 +1,16 @@ -import { isRetryableError } from "../process/error"; -import { SamlCertsAndKeys } from "../../../saml/security/types"; import { OutboundDocumentRetrievalResp, OutboundDocumentQueryResp, } from "@metriport/ihe-gateway-sdk"; +import { sleep } from "@metriport/shared"; import { SignedDqRequest } from "../create/iti38-envelope"; import { SignedDrRequest } from "../create/iti39-envelope"; import { sendSignedDqRequest } from "../send/dq-requests"; import { sendSignedDrRequest } from "../send/dr-requests"; import { processDqResponse } from "../process/dq-response"; import { processDrResponse } from "../process/dr-response"; +import { isRetryable } from "../process/error"; +import { SamlCertsAndKeys } from "../../../saml/security/types"; import { out } from "../../../../../../util/log"; const { log } = out("IHE Gateway V2"); @@ -63,18 +64,18 @@ export async function sendProcessRetryRequests({ attempt, }); - if (!isRetryableError(result)) { + if (!isRetryable(result)) { return result; } attempt++; const backoffTime = calculateBackoff(attempt); - await new Promise(resolve => setTimeout(resolve, backoffTime)); + await sleep(backoffTime); log(`Attempt ${attempt + 1} of ${maxRetries + 1}`); } const finalBackoffTime = calculateBackoff(maxRetries); - await new Promise(resolve => setTimeout(resolve, finalBackoffTime)); + await sleep(finalBackoffTime); log(`Attempt ${maxRetries + 1} of ${maxRetries + 1}`); const finalResponse = await sendRequest({ diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts index 5b7c487cc6..7c19cf8a22 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts @@ -131,7 +131,7 @@ function parseDocumentReference({ return documentReference; } -function handleSuccessResponse({ +async function handleSuccessResponse({ extrinsicObjects, outboundRequest, gateway, @@ -141,7 +141,7 @@ function handleSuccessResponse({ outboundRequest: OutboundDocumentQueryReq; gateway: XCAGateway; attempt?: number | undefined; -}): OutboundDocumentQueryResp { +}): Promise { const documentReferences = extrinsicObjects.flatMap( extrinsicObject => parseDocumentReference({ extrinsicObject, outboundRequest }) ?? [] ); @@ -216,7 +216,7 @@ export function processDqResponse({ }); } } catch (error) { - log("Error processing DQ response", error); + log(`Error processing DQ response ${JSON.stringify(jsonObj)}`); return handleSchemaErrorResponse({ outboundRequest, gateway, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts index 1c60c52b1f..f16974b64e 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts @@ -245,7 +245,7 @@ export async function processDrResponse({ }); } } catch (error) { - log("Error processing DR response", error); + log(`Error processing DR response ${JSON.stringify(error)}`); return handleSchemaErrorResponse({ outboundRequest, gateway, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 48de7f4cbf..158c5a490b 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -62,7 +62,7 @@ export function processRegistryErrorList( return operationOutcome.issue.length > 0 ? operationOutcome : undefined; } -export function handleRegistryErrorResponse({ +export async function handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, @@ -72,7 +72,7 @@ export function handleRegistryErrorResponse({ outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; attempt?: number | undefined; -}): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { +}): Promise { const operationOutcome = processRegistryErrorList(registryErrorList, outboundRequest); return { id: outboundRequest.id, @@ -122,7 +122,7 @@ export function handleHttpErrorResponse({ }; } -export function handleEmptyResponse({ +export async function handleEmptyResponse({ outboundRequest, gateway, text = "No documents found", @@ -132,7 +132,7 @@ export function handleEmptyResponse({ gateway: XCAGateway; text?: string; attempt?: number | undefined; -}): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { +}): Promise { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", id: outboundRequest.id, @@ -158,7 +158,7 @@ export function handleEmptyResponse({ }; } -export function handleSchemaErrorResponse({ +export async function handleSchemaErrorResponse({ outboundRequest, gateway, attempt, @@ -168,7 +168,7 @@ export function handleSchemaErrorResponse({ gateway: XCAGateway; attempt?: number | undefined; text?: string; -}): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { +}): Promise { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", id: outboundRequest.id, @@ -194,7 +194,7 @@ export function handleSchemaErrorResponse({ }; } -export function isRetryableError(outboundRequest: OutboundDocumentRetrievalResp): boolean { +export function isRetryable(outboundRequest: OutboundDocumentRetrievalResp): boolean { return ( outboundRequest.operationOutcome?.issue.some( issue => diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts index 94e4c2be57..ac30cdb891 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response.ts @@ -252,7 +252,7 @@ export function processXCPDResponse({ }); } } catch (error) { - log(`Error processing XCPD response: ${error}`); + log(`Error processing XCPD response: ${JSON.stringify(error)}`); return handleSchemaErrorResponse({ outboundRequest, gateway, diff --git a/packages/utils/src/saml/pre-prod-tester.ts b/packages/utils/src/saml/pre-prod-tester.ts index 3cd16b52f3..387780a8af 100644 --- a/packages/utils/src/saml/pre-prod-tester.ts +++ b/packages/utils/src/saml/pre-prod-tester.ts @@ -21,7 +21,7 @@ import { createAndSignBulkDQRequests } from "@metriport/core/external/carequalit import { sendSignedDqRequest } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests"; import { processDqResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response"; import { createAndSignBulkDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope"; -import { sendProcessRetryDrRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +import { sendProcessRetryDrRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { Config } from "@metriport/core/util/config"; @@ -72,12 +72,12 @@ async function queryDatabaseForXcpds() { const results = await sequelize.query(query, { type: QueryTypes.SELECT, }); - sequelize.close(); return results; } catch (error) { console.error("Error executing SQL query:", error); - sequelize.close(); throw error; + } finally { + sequelize.close(); } } @@ -96,12 +96,12 @@ async function queryDatabaseForDQs() { const results = await sequelize.query(query, { type: QueryTypes.SELECT, }); - sequelize.close(); return results; } catch (error) { console.error("Error executing SQL query:", error); - sequelize.close(); throw error; + } finally { + sequelize.close(); } } @@ -123,12 +123,12 @@ export async function queryDatabaseForDqsFromFailedDrs() { const results = await sequelize.query(query, { type: QueryTypes.SELECT, }); - sequelize.close(); return results; } catch (error) { console.error("Error executing SQL query:", error); - sequelize.close(); throw error; + } finally { + sequelize.close(); } } @@ -145,12 +145,12 @@ async function getDrUrl(id: string): Promise { replacements: { id }, type: QueryTypes.SELECT, }); - sequelize.close(); return results[0].url_dr; } catch (error) { - console.error("Error executing SQL query:", error); - sequelize.close(); + console.log("Error executing SQL query:", error); throw error; + } finally { + sequelize.close(); } } diff --git a/packages/utils/src/saml/saml-server.ts b/packages/utils/src/saml/saml-server.ts index 577a37cc1c..628f8d23ba 100644 --- a/packages/utils/src/saml/saml-server.ts +++ b/packages/utils/src/saml/saml-server.ts @@ -14,7 +14,7 @@ import { processXCPDResponse } from "@metriport/core/external/carequality/ihe-ga import { sendProcessRetryDrRequests, sendProcessRetryDqRequests, -} from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +} from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; From c560f8f8a26ffa145b959c2449c1e392c199b2e9 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Sun, 16 Jun 2024 19:00:41 -0400 Subject: [PATCH 05/13] feat(ihe): refactor to use new shared retry w/ backoff Refs: #1667 Signed-off-by: Jonah Kaye --- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 102 ++++++++++++--- .../outbound/__tests__/constants.ts | 1 + .../xca/orchestrate/send-process-retry.ts | 121 ------------------ .../outbound/xca/process/error.ts | 8 +- packages/shared/src/common/retry.ts | 65 +++++++++- packages/shared/src/net/retry.ts | 5 +- packages/utils/src/saml/pre-prod-tester.ts | 4 +- packages/utils/src/saml/saml-server.ts | 10 +- 8 files changed, 160 insertions(+), 156 deletions(-) delete mode 100644 packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 1b7ecc106a..78ad776448 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -3,21 +3,93 @@ import { OutboundDocumentRetrievalReq, OutboundPatientDiscoveryReq, OutboundPatientDiscoveryResp, + OutboundDocumentRetrievalResp, + OutboundDocumentQueryResp, } from "@metriport/ihe-gateway-sdk"; -import { executeWithNetworkRetries } from "@metriport/shared"; +import { + executeWithNetworkRetries, + executeWithRetriesOnResult as executeWithOperationOutcomeRetries, +} from "@metriport/shared"; import axios from "axios"; import { capture } from "../../../util/notifications"; -import { createAndSignBulkDQRequests } from "./outbound/xca/create/iti38-envelope"; -import { createAndSignBulkDRRequests } from "./outbound/xca/create/iti39-envelope"; +import { createAndSignBulkDQRequests, SignedDqRequest } from "./outbound/xca/create/iti38-envelope"; +import { createAndSignBulkDRRequests, SignedDrRequest } from "./outbound/xca/create/iti39-envelope"; +import { sendSignedDqRequest } from "./outbound/xca/send/dq-requests"; +import { sendSignedDrRequest } from "./outbound/xca/send/dr-requests"; +import { processDqResponse } from "./outbound/xca/process/dq-response"; +import { processDrResponse } from "./outbound/xca/process/dr-response"; +import { isRetryable } from "./outbound/xca/process/error"; import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; -import { - sendProcessRetryDqRequests, - sendProcessRetryDrRequests, -} from "./outbound/xca/orchestrate/send-process-retry"; import { createAndSignBulkXCPDRequests } from "./outbound/xcpd/create/iti55-envelope"; import { SamlCertsAndKeys } from "./saml/security/types"; +export async function sendProcessRetryDqRequest({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, +}: { + signedRequest: SignedDqRequest; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; +}): Promise { + async function sendProcessDqRequest() { + const response = await sendSignedDqRequest({ + request: signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + }); + return await processDqResponse({ + response, + }); + } + + return await executeWithOperationOutcomeRetries(sendProcessDqRequest, { + initialDelay: 3000, + maxAttempts: 3, + shouldRetryResult: result => isRetryable(result), + }); +} + +export async function sendProcessRetryDrRequest({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, +}: { + signedRequest: SignedDrRequest; + samlCertsAndKeys: SamlCertsAndKeys; + patientId: string; + cxId: string; + index: number; +}): Promise { + async function sendProcessDrRequest() { + const response = await sendSignedDrRequest({ + request: signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + }); + return await processDrResponse({ + response, + }); + } + + return await executeWithOperationOutcomeRetries(sendProcessDrRequest, { + initialDelay: 3000, + maxAttempts: 3, + shouldRetryResult: result => isRetryable(result), + }); +} + export async function createSignSendProcessXCPDRequest({ pdResponseUrl, xcpdRequest, @@ -80,13 +152,7 @@ export async function createSignSendProcessDQRequests({ }); const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDqRequests({ - signedRequest, - samlCertsAndKeys, - patientId, - cxId, - index, - }); + return sendProcessRetryDqRequest({ signedRequest, samlCertsAndKeys, patientId, cxId, index }); }); const results = await Promise.allSettled(resultPromises); @@ -126,13 +192,7 @@ export async function createSignSendProcessDRRequests({ }); const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDrRequests({ - signedRequest, - samlCertsAndKeys, - patientId, - cxId, - index, - }); + return sendProcessRetryDrRequest({ signedRequest, samlCertsAndKeys, patientId, cxId, index }); }); const results = await Promise.allSettled(resultPromises); diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/constants.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/constants.ts index 21bf037605..ce24e78fe5 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/constants.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/__tests__/constants.ts @@ -121,6 +121,7 @@ export const expectedXcpdResponse: OutboundPatientDiscoveryRespSuccessfulSchema }, ], }, + iheGatewayV2: true, }; export const expectedMultiNameAddressResponse: OutboundPatientDiscoveryRespSuccessfulSchema = { diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts deleted file mode 100644 index cda5099a59..0000000000 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { - OutboundDocumentRetrievalResp, - OutboundDocumentQueryResp, -} from "@metriport/ihe-gateway-sdk"; -import { sleep } from "@metriport/shared"; -import { SignedDqRequest } from "../create/iti38-envelope"; -import { SignedDrRequest } from "../create/iti39-envelope"; -import { sendSignedDqRequest } from "../send/dq-requests"; -import { sendSignedDrRequest } from "../send/dr-requests"; -import { processDqResponse } from "../process/dq-response"; -import { processDrResponse } from "../process/dr-response"; -import { isRetryable } from "../process/error"; -import { SamlCertsAndKeys } from "../../../saml/security/types"; -import { out } from "../../../../../../util/log"; - -const { log } = out("IHE Gateway V2"); - -function calculateBackoff(attempt: number, baseDelay = 2000, jitterRange = 2000): number { - const baseBackoffTime = Math.pow(2, attempt + 1) * baseDelay; - const jitter = (Math.random() - 0.5) * jitterRange; - return baseBackoffTime + jitter; -} - -export async function sendProcessRetryRequests({ - signedRequest, - samlCertsAndKeys, - patientId, - cxId, - index, - sendRequest, - processResponse, - maxRetries = 2, -}: { - signedRequest: T; - samlCertsAndKeys: SamlCertsAndKeys; - patientId: string; - cxId: string; - index: number; - sendRequest: (params: { - request: T; - samlCertsAndKeys: SamlCertsAndKeys; - patientId: string; - cxId: string; - index: number; - }) => Promise; - processResponse: (params: { - response: R; - attempt: number; - }) => Promise; - maxRetries?: number; -}): Promise { - let attempt = 0; - - while (attempt < maxRetries) { - const response = await sendRequest({ - request: signedRequest, - samlCertsAndKeys, - patientId, - cxId, - index, - }); - const result = await processResponse({ - response, - attempt, - }); - - if (!isRetryable(result)) { - return result; - } - - attempt++; - const backoffTime = calculateBackoff(attempt); - await sleep(backoffTime); - log(`Attempt ${attempt + 1} of ${maxRetries + 1}`); - } - - const finalBackoffTime = calculateBackoff(maxRetries); - await sleep(finalBackoffTime); - - log(`Attempt ${maxRetries + 1} of ${maxRetries + 1}`); - const finalResponse = await sendRequest({ - request: signedRequest, - samlCertsAndKeys, - patientId, - cxId, - index, - }); - const result = await processResponse({ - response: finalResponse, - attempt: maxRetries, - }); - return result; -} - -export async function sendProcessRetryDrRequests(params: { - signedRequest: SignedDrRequest; - samlCertsAndKeys: SamlCertsAndKeys; - patientId: string; - cxId: string; - index: number; -}): Promise { - return sendProcessRetryRequests({ - ...params, - sendRequest: sendSignedDrRequest, - processResponse: processDrResponse, - }); -} - -export async function sendProcessRetryDqRequests(params: { - signedRequest: SignedDqRequest; - samlCertsAndKeys: SamlCertsAndKeys; - patientId: string; - cxId: string; - index: number; -}): Promise { - return sendProcessRetryRequests({ - ...params, - sendRequest: sendSignedDqRequest, - processResponse: processDqResponse, - }); -} diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 158c5a490b..dbdb6928fa 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -194,9 +194,13 @@ export async function handleSchemaErrorResponse({ }; } -export function isRetryable(outboundRequest: OutboundDocumentRetrievalResp): boolean { +/** + * Retries if the response has an error that is not in the known non-retryable errors list + * Will not retry if the response is successful and is not an error. + */ +export function isRetryable(outboundResponse: OutboundDocumentRetrievalResp): boolean { return ( - outboundRequest.operationOutcome?.issue.some( + outboundResponse.operationOutcome?.issue.some( issue => issue.severity === "error" && !knownNonRetryableErrors.some( diff --git a/packages/shared/src/common/retry.ts b/packages/shared/src/common/retry.ts index c156a04600..e7a99a93d6 100644 --- a/packages/shared/src/common/retry.ts +++ b/packages/shared/src/common/retry.ts @@ -3,17 +3,18 @@ import { MetriportError } from "../error/metriport-error"; import { errorToString } from "../error/shared"; import { sleep } from "./sleep"; -export const defaultOptions: Required = { +export const defaultOptions: Required> = { initialDelay: 10, maxDelay: Infinity, backoffMultiplier: 2, maxAttempts: 10, shouldRetry: () => true, + shouldRetryResult: () => true, getTimeToWait: defaultGetTimeToWait, log: console.log, }; -export type ExecuteWithRetriesOptions = { +export type ExecuteWithRetriesOptions = { /** The intitial delay in milliseconds. Defaults to 10ms. */ initialDelay?: number; /** The maximum delay in milliseconds. Defaults to Infinity. */ @@ -28,6 +29,8 @@ export type ExecuteWithRetriesOptions = { maxAttempts?: number; /** Function to determine if the error should be retried. Defaults to always retry. */ shouldRetry?: (error: unknown, attempt: number) => boolean | Promise; + /** Function to determine if the result should be retried. Defaults to always retry. */ + shouldRetryResult?: (result: T) => boolean | Promise; /** Function to determine how long to wait before the next retry. It should not be changed. */ getTimeToWait?: (params: GetTimeToWaitParams) => number; /** Function to log details about the execution */ @@ -55,7 +58,7 @@ export type GetTimeToWaitParams = { */ export async function executeWithRetries( fn: () => Promise, - options: ExecuteWithRetriesOptions = defaultOptions + options: ExecuteWithRetriesOptions = defaultOptions ): Promise { const actualOptions = { ...defaultOptions, ...options }; const { @@ -120,7 +123,7 @@ export function defaultGetTimeToWait({ */ export async function executeWithRetriesSafe( fn: () => Promise, - options?: Partial + options?: Partial> ): Promise { const actualOptions = { ...defaultOptions, ...options }; const { log } = actualOptions; @@ -131,3 +134,57 @@ export async function executeWithRetriesSafe( return undefined; } } + +/** + * Executes a function with retries and backoff with jitter. If the result of the function is + * retryable, it will retry. This function differs from `executeWithRetries` because it does not + * retry based on the function throwing errors, but rather based on the result of the function. + * If the function is not retryable on the last attempt, it will return the result. + * If the function succeeds, it will return the result. + * + * @param fn the function to be executed + * @param options the options to be used; see `ExecuteWithRetriesOptions` for components and + * `defaultOptions` for defaults. + * @returns the result of calling the `fn` function + * @see https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ +export async function executeWithRetriesOnResult( + fn: () => Promise, + options: ExecuteWithRetriesOptions = defaultOptions +): Promise { + const actualOptions = { ...defaultOptions, ...options }; + const { + initialDelay, + maxDelay, + backoffMultiplier, + maxAttempts: _maxAttempts, + shouldRetryResult, + getTimeToWait, + log, + } = actualOptions; + const maxAttempts = Math.max(_maxAttempts, 1); + let attempt = 0; + while (++attempt <= maxAttempts) { + const msg = `[executeWithRetriesErrorIndependent]`; + try { + const result = await fn(); + if (attempt >= maxAttempts) { + log(`${msg}, gave up.`); + return result; + } + if (await shouldRetryResult(result)) { + throw new Error("Retry"); + } + return result; + } catch (error) { + log(`${msg}, retrying... (attempt: ${attempt})`); + const timeToWait = getTimeToWait({ initialDelay, backoffMultiplier, attempt, maxDelay }); + await sleep(timeToWait); + } + } + throw new MetriportError("Unreachable code", undefined, { + attempt, + maxAttempts, + context: "executeWithRetries", + }); +} diff --git a/packages/shared/src/net/retry.ts b/packages/shared/src/net/retry.ts index 233f68d6fa..1e96c7246e 100644 --- a/packages/shared/src/net/retry.ts +++ b/packages/shared/src/net/retry.ts @@ -6,7 +6,10 @@ import { } from "../common/retry"; import { NetworkError } from "./error"; -export type ExecuteWithHttpRetriesOptions = Omit & { +export type ExecuteWithHttpRetriesOptions = Omit< + ExecuteWithRetriesOptions, + "shouldRetry" +> & { /** The network error codes to retry. See `defaultOptions` for defaults. */ httpCodesToRetry: NetworkError[]; }; diff --git a/packages/utils/src/saml/pre-prod-tester.ts b/packages/utils/src/saml/pre-prod-tester.ts index 387780a8af..5a57e2fa29 100644 --- a/packages/utils/src/saml/pre-prod-tester.ts +++ b/packages/utils/src/saml/pre-prod-tester.ts @@ -21,7 +21,7 @@ import { createAndSignBulkDQRequests } from "@metriport/core/external/carequalit import { sendSignedDqRequest } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests"; import { processDqResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response"; import { createAndSignBulkDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/create/iti39-envelope"; -import { sendProcessRetryDrRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry"; +import { sendProcessRetryDrRequest } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { Config } from "@metriport/core/util/config"; @@ -458,7 +458,7 @@ async function queryDR( }); const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDrRequests({ + return sendProcessRetryDrRequest({ signedRequest, samlCertsAndKeys, patientId: uuidv4(), diff --git a/packages/utils/src/saml/saml-server.ts b/packages/utils/src/saml/saml-server.ts index 628f8d23ba..af658e235a 100644 --- a/packages/utils/src/saml/saml-server.ts +++ b/packages/utils/src/saml/saml-server.ts @@ -12,9 +12,9 @@ import { createAndSignBulkDRRequests } from "@metriport/core/external/carequalit import { sendSignedXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests"; import { processXCPDResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response"; import { - sendProcessRetryDrRequests, - sendProcessRetryDqRequests, -} from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/orchestrate/send-process-retry"; + sendProcessRetryDrRequest, + sendProcessRetryDqRequest, +} from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; @@ -91,7 +91,7 @@ app.post("/xcadq", async (req: Request, res: Response) => { }); const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDqRequests({ + return sendProcessRetryDqRequest({ signedRequest, samlCertsAndKeys, patientId: uuidv4(), @@ -123,7 +123,7 @@ app.post("/xcadr", async (req: Request, res: Response) => { samlCertsAndKeys, }); const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDrRequests({ + return sendProcessRetryDrRequest({ signedRequest, samlCertsAndKeys, patientId: uuidv4(), From a6e5e2a838cd45f7ea10c8e6dc9aadac048e5de7 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Sun, 16 Jun 2024 19:59:36 -0400 Subject: [PATCH 06/13] feat(ihe): adding tests Refs: #1667 Signed-off-by: Jonah Kaye --- .../shared/src/common/__tests__/retry.test.ts | 56 ++++++++++++++++++- packages/shared/src/common/retry.ts | 2 +- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/common/__tests__/retry.test.ts b/packages/shared/src/common/__tests__/retry.test.ts index acc8115246..ba1dc78407 100644 --- a/packages/shared/src/common/__tests__/retry.test.ts +++ b/packages/shared/src/common/__tests__/retry.test.ts @@ -1,5 +1,10 @@ import { faker } from "@faker-js/faker"; -import { defaultGetTimeToWait, executeWithRetries, executeWithRetriesSafe } from "../retry"; +import { + defaultGetTimeToWait, + executeWithRetries, + executeWithRetriesSafe, + executeWithRetriesOnResult, +} from "../retry"; describe("retry", () => { describe("executeWithRetries", () => { @@ -115,6 +120,55 @@ describe("retry", () => { }); }); + describe("executeWithRetriesOnResult", () => { + const fn = jest.fn(); + beforeEach(() => { + fn.mockImplementation(() => { + throw new Error("error"); + }); + }); + afterEach(() => { + jest.resetAllMocks(); + }); + + it("returns the first successful execution", async () => { + const expectedResult = faker.lorem.word(); + fn.mockImplementationOnce(() => expectedResult); + const resp = await executeWithRetriesOnResult(fn, { + initialDelay: 1, + maxAttempts: 2, + }); + expect(resp).toEqual(expectedResult); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("keeps trying on retryable result and returns the first non-retryable result", async () => { + const retryableResult = faker.lorem.word(); + const expectedResult = faker.lorem.sentence(); + fn.mockImplementationOnce(() => retryableResult); + fn.mockImplementationOnce(() => expectedResult); + const resp = await executeWithRetriesOnResult(fn, { + initialDelay: 1, + maxAttempts: 3, + shouldRetryResult: result => result === retryableResult, + }); + expect(resp).toEqual(expectedResult); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it("returns the last retryable result after max attempts", async () => { + const retryableResult = faker.lorem.word(); + fn.mockImplementation(() => retryableResult); + const resp = await executeWithRetriesOnResult(fn, { + initialDelay: 1, + maxAttempts: 3, + shouldRetryResult: result => result === retryableResult, + }); + expect(resp).toEqual(retryableResult); + expect(fn).toHaveBeenCalledTimes(3); + }); + }); + describe("defaultGetTimeToWait", () => { it("returns initialDelay when backoffMultiplier is lower than 1", async () => { const initialDelay = faker.number.int({ min: 10, max: 30 }); diff --git a/packages/shared/src/common/retry.ts b/packages/shared/src/common/retry.ts index e7a99a93d6..12a92725c5 100644 --- a/packages/shared/src/common/retry.ts +++ b/packages/shared/src/common/retry.ts @@ -9,7 +9,7 @@ export const defaultOptions: Required> = { backoffMultiplier: 2, maxAttempts: 10, shouldRetry: () => true, - shouldRetryResult: () => true, + shouldRetryResult: () => false, getTimeToWait: defaultGetTimeToWait, log: console.log, }; From 068b6f486a166caa22cd38afae73397b34a1a3d5 Mon Sep 17 00:00:00 2001 From: Rafael Leite <2132564+leite08@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:45:08 -0500 Subject: [PATCH 07/13] refactor(core): executeWithRetries checks retry on non-error result Ref. metriport/metriport-internal#1667 Signed-off-by: Rafael Leite <2132564+leite08@users.noreply.github.com> --- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 25 ++--- .../outbound/xca/process/error.ts | 15 +-- .../shared/src/common/__tests__/retry.test.ts | 19 ++-- packages/shared/src/common/retry.ts | 101 ++++++------------ packages/shared/src/net/retry.ts | 2 +- 5 files changed, 61 insertions(+), 101 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 78ad776448..d9305b04f8 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -1,27 +1,24 @@ import { OutboundDocumentQueryReq, + OutboundDocumentQueryResp, OutboundDocumentRetrievalReq, + OutboundDocumentRetrievalResp, OutboundPatientDiscoveryReq, OutboundPatientDiscoveryResp, - OutboundDocumentRetrievalResp, - OutboundDocumentQueryResp, } from "@metriport/ihe-gateway-sdk"; -import { - executeWithNetworkRetries, - executeWithRetriesOnResult as executeWithOperationOutcomeRetries, -} from "@metriport/shared"; +import { executeWithNetworkRetries, executeWithRetries } from "@metriport/shared"; import axios from "axios"; import { capture } from "../../../util/notifications"; import { createAndSignBulkDQRequests, SignedDqRequest } from "./outbound/xca/create/iti38-envelope"; import { createAndSignBulkDRRequests, SignedDrRequest } from "./outbound/xca/create/iti39-envelope"; -import { sendSignedDqRequest } from "./outbound/xca/send/dq-requests"; -import { sendSignedDrRequest } from "./outbound/xca/send/dr-requests"; import { processDqResponse } from "./outbound/xca/process/dq-response"; import { processDrResponse } from "./outbound/xca/process/dr-response"; import { isRetryable } from "./outbound/xca/process/error"; -import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; -import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; +import { sendSignedDqRequest } from "./outbound/xca/send/dq-requests"; +import { sendSignedDrRequest } from "./outbound/xca/send/dr-requests"; import { createAndSignBulkXCPDRequests } from "./outbound/xcpd/create/iti55-envelope"; +import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; +import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; import { SamlCertsAndKeys } from "./saml/security/types"; export async function sendProcessRetryDqRequest({ @@ -50,10 +47,10 @@ export async function sendProcessRetryDqRequest({ }); } - return await executeWithOperationOutcomeRetries(sendProcessDqRequest, { + return await executeWithRetries(sendProcessDqRequest, { initialDelay: 3000, maxAttempts: 3, - shouldRetryResult: result => isRetryable(result), + shouldRetry: isRetryable, }); } @@ -83,10 +80,10 @@ export async function sendProcessRetryDrRequest({ }); } - return await executeWithOperationOutcomeRetries(sendProcessDrRequest, { + return await executeWithRetries(sendProcessDrRequest, { initialDelay: 3000, maxAttempts: 3, - shouldRetryResult: result => isRetryable(result), + shouldRetry: isRetryable, }); } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index dbdb6928fa..0563a72f41 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -1,16 +1,16 @@ -import dayjs from "dayjs"; import { + OperationOutcome, OutboundDocumentQueryReq, - OutboundDocumentRetrievalReq, OutboundDocumentQueryResp, + OutboundDocumentRetrievalReq, OutboundDocumentRetrievalResp, - OperationOutcome, XCAGateway, } from "@metriport/ihe-gateway-sdk"; -import { capture } from "../../../../../../util/notifications"; -import { out } from "../../../../../../util/log"; -import { RegistryErrorList, RegistryError } from "./schema"; import { toArray } from "@metriport/shared"; +import dayjs from "dayjs"; +import { out } from "../../../../../../util/log"; +import { capture } from "../../../../../../util/notifications"; +import { RegistryError, RegistryErrorList } from "./schema"; const { log } = out("XCA Error Handling"); const knownNonRetryableErrors = ["No active consent for patient id"]; @@ -198,7 +198,8 @@ export async function handleSchemaErrorResponse({ * Retries if the response has an error that is not in the known non-retryable errors list * Will not retry if the response is successful and is not an error. */ -export function isRetryable(outboundResponse: OutboundDocumentRetrievalResp): boolean { +export function isRetryable(outboundResponse: OutboundDocumentRetrievalResp | undefined): boolean { + if (!outboundResponse) return false; return ( outboundResponse.operationOutcome?.issue.some( issue => diff --git a/packages/shared/src/common/__tests__/retry.test.ts b/packages/shared/src/common/__tests__/retry.test.ts index ba1dc78407..5c4c27c65a 100644 --- a/packages/shared/src/common/__tests__/retry.test.ts +++ b/packages/shared/src/common/__tests__/retry.test.ts @@ -1,10 +1,5 @@ import { faker } from "@faker-js/faker"; -import { - defaultGetTimeToWait, - executeWithRetries, - executeWithRetriesSafe, - executeWithRetriesOnResult, -} from "../retry"; +import { defaultGetTimeToWait, executeWithRetries, executeWithRetriesSafe } from "../retry"; describe("retry", () => { describe("executeWithRetries", () => { @@ -55,7 +50,7 @@ describe("retry", () => { }); it("uses custom shouldRetry when provided", async () => { - const shouldRetry = (_: unknown, attempt: number): boolean => { + const shouldRetry = (_r: unknown, _e: unknown, attempt: number): boolean => { return attempt !== 2; }; await expect(async () => @@ -134,7 +129,7 @@ describe("retry", () => { it("returns the first successful execution", async () => { const expectedResult = faker.lorem.word(); fn.mockImplementationOnce(() => expectedResult); - const resp = await executeWithRetriesOnResult(fn, { + const resp = await executeWithRetries(fn, { initialDelay: 1, maxAttempts: 2, }); @@ -147,10 +142,10 @@ describe("retry", () => { const expectedResult = faker.lorem.sentence(); fn.mockImplementationOnce(() => retryableResult); fn.mockImplementationOnce(() => expectedResult); - const resp = await executeWithRetriesOnResult(fn, { + const resp = await executeWithRetries(fn, { initialDelay: 1, maxAttempts: 3, - shouldRetryResult: result => result === retryableResult, + shouldRetry: result => result === retryableResult, }); expect(resp).toEqual(expectedResult); expect(fn).toHaveBeenCalledTimes(2); @@ -159,10 +154,10 @@ describe("retry", () => { it("returns the last retryable result after max attempts", async () => { const retryableResult = faker.lorem.word(); fn.mockImplementation(() => retryableResult); - const resp = await executeWithRetriesOnResult(fn, { + const resp = await executeWithRetries(fn, { initialDelay: 1, maxAttempts: 3, - shouldRetryResult: result => result === retryableResult, + shouldRetry: result => result === retryableResult, }); expect(resp).toEqual(retryableResult); expect(fn).toHaveBeenCalledTimes(3); diff --git a/packages/shared/src/common/retry.ts b/packages/shared/src/common/retry.ts index 12a92725c5..8cb2853f30 100644 --- a/packages/shared/src/common/retry.ts +++ b/packages/shared/src/common/retry.ts @@ -3,13 +3,17 @@ import { MetriportError } from "../error/metriport-error"; import { errorToString } from "../error/shared"; import { sleep } from "./sleep"; +function defaultShouldRetry(_: T | undefined, error: unknown) { + if (error) return true; + return false; +} + export const defaultOptions: Required> = { initialDelay: 10, maxDelay: Infinity, backoffMultiplier: 2, maxAttempts: 10, - shouldRetry: () => true, - shouldRetryResult: () => false, + shouldRetry: defaultShouldRetry, getTimeToWait: defaultGetTimeToWait, log: console.log, }; @@ -28,9 +32,12 @@ export type ExecuteWithRetriesOptions = { /** The maximum number of retries. Defaults to 10. */ maxAttempts?: number; /** Function to determine if the error should be retried. Defaults to always retry. */ - shouldRetry?: (error: unknown, attempt: number) => boolean | Promise; + shouldRetry?: ( + result: T | undefined, + error: unknown, + attempt: number + ) => boolean | Promise; /** Function to determine if the result should be retried. Defaults to always retry. */ - shouldRetryResult?: (result: T) => boolean | Promise; /** Function to determine how long to wait before the next retry. It should not be changed. */ getTimeToWait?: (params: GetTimeToWaitParams) => number; /** Function to log details about the execution */ @@ -45,10 +52,14 @@ export type GetTimeToWaitParams = { }; /** - * Executes a function with retries and backoff with jitter. If the function throws an error, it will retry - * up to maxAttempts-1, waiting between each retry. + * Executes a function with retries and backoff with jitter. + * Decides whether or not to retry based on the `shouldRetry()` function parameter - by default + * it only retries on error. + * If the function doesn't throw errors and `shouldRetry` returns true, it will return the last + * function's result when it reaches the maximum attempts. + * It retries up to maxAttempts-1, waiting between each retry. * If the function throws an error on the last attempt, it will throw the error. - * If the function succeeds, it will return the result. + * If the function succeeds and `shouldRetry` returns false, it will return the function's result. * * @param fn the function to be executed * @param options the options to be used; see `ExecuteWithRetriesOptions` for components and @@ -70,19 +81,29 @@ export async function executeWithRetries( getTimeToWait, log, } = actualOptions; + const context = "executeWithRetries"; const maxAttempts = Math.max(_maxAttempts, 1); let attempt = 0; while (++attempt <= maxAttempts) { try { - return await fn(); + const result = await fn(); + if (await shouldRetry(result, undefined, attempt)) { + if (attempt >= maxAttempts) { + log(`[${context}] Gave up after ${attempt} attempts.`); + return result; + } + log(`[${context}] Retrying... (attempt: ${attempt})`); + continue; + } + return result; } catch (error) { - const msg = `[executeWithRetries] Error: ${errorToString(error)}`; + const msg = `[${context}] Error: ${errorToString(error)}`; if (attempt >= maxAttempts) { - log(`${msg}, gave up.`); + log(`${msg}, gave up after ${attempt} attempts.`); throw error; } - if (!(await shouldRetry(error, attempt))) { - log(`${msg}, should not retry.`); + if (!(await shouldRetry(undefined, error, attempt))) { + log(`${msg}, should not retry (after ${attempt} attempts).`); throw error; } log(`${msg}, retrying... (attempt: ${attempt})`); @@ -93,7 +114,7 @@ export async function executeWithRetries( throw new MetriportError("Unreachable code", undefined, { attempt, maxAttempts, - context: "executeWithRetries", + context, }); } @@ -134,57 +155,3 @@ export async function executeWithRetriesSafe( return undefined; } } - -/** - * Executes a function with retries and backoff with jitter. If the result of the function is - * retryable, it will retry. This function differs from `executeWithRetries` because it does not - * retry based on the function throwing errors, but rather based on the result of the function. - * If the function is not retryable on the last attempt, it will return the result. - * If the function succeeds, it will return the result. - * - * @param fn the function to be executed - * @param options the options to be used; see `ExecuteWithRetriesOptions` for components and - * `defaultOptions` for defaults. - * @returns the result of calling the `fn` function - * @see https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ - */ -export async function executeWithRetriesOnResult( - fn: () => Promise, - options: ExecuteWithRetriesOptions = defaultOptions -): Promise { - const actualOptions = { ...defaultOptions, ...options }; - const { - initialDelay, - maxDelay, - backoffMultiplier, - maxAttempts: _maxAttempts, - shouldRetryResult, - getTimeToWait, - log, - } = actualOptions; - const maxAttempts = Math.max(_maxAttempts, 1); - let attempt = 0; - while (++attempt <= maxAttempts) { - const msg = `[executeWithRetriesErrorIndependent]`; - try { - const result = await fn(); - if (attempt >= maxAttempts) { - log(`${msg}, gave up.`); - return result; - } - if (await shouldRetryResult(result)) { - throw new Error("Retry"); - } - return result; - } catch (error) { - log(`${msg}, retrying... (attempt: ${attempt})`); - const timeToWait = getTimeToWait({ initialDelay, backoffMultiplier, attempt, maxDelay }); - await sleep(timeToWait); - } - } - throw new MetriportError("Unreachable code", undefined, { - attempt, - maxAttempts, - context: "executeWithRetries", - }); -} diff --git a/packages/shared/src/net/retry.ts b/packages/shared/src/net/retry.ts index 1e96c7246e..6df93184e7 100644 --- a/packages/shared/src/net/retry.ts +++ b/packages/shared/src/net/retry.ts @@ -48,7 +48,7 @@ export async function executeWithNetworkRetries( const codesAsString = httpCodesToRetry.map(String); return executeWithRetries(fn, { ...actualOptions, - shouldRetry: (error: unknown) => { + shouldRetry: (_, error: unknown) => { const networkCode = axios.isAxiosError(error) ? error.code : undefined; if (!networkCode) return false; return codesAsString.includes(networkCode); From 57434b50244148368b5aab3288297a84f865d959 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Mon, 17 Jun 2024 16:53:50 -0400 Subject: [PATCH 08/13] feat(ihe): trustedkeystore, removing attempts, nits Refs: #1667 Signed-off-by: Jonah Kaye --- .../ihe-gateway-v2/ihe-gateway-v2-direct.ts | 8 ++-- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 39 ++++++++++++++++--- .../outbound/xca/process/dq-response.ts | 13 +------ .../outbound/xca/process/dr-response.ts | 10 ----- .../outbound/xca/process/error.ts | 9 ----- .../outbound/xca/send/dq-requests.ts | 5 ++- .../outbound/xca/send/dr-requests.ts | 5 ++- .../outbound/xcpd/send/xcpd-requests.ts | 5 ++- .../ihe-gateway-v2-outbound-document-query.ts | 4 +- ...-gateway-v2-outbound-document-retrieval.ts | 4 +- packages/utils/src/saml/bulk-saml.ts | 7 +++- packages/utils/src/saml/pre-prod-tester.ts | 31 ++++++++++----- packages/utils/src/saml/saml-server.ts | 13 ++++++- 13 files changed, 92 insertions(+), 61 deletions(-) diff --git a/packages/api/src/external/ihe-gateway-v2/ihe-gateway-v2-direct.ts b/packages/api/src/external/ihe-gateway-v2/ihe-gateway-v2-direct.ts index cdb42e7e76..3f30d36cbb 100644 --- a/packages/api/src/external/ihe-gateway-v2/ihe-gateway-v2-direct.ts +++ b/packages/api/src/external/ihe-gateway-v2/ihe-gateway-v2-direct.ts @@ -6,8 +6,8 @@ import { import { IHEGatewayV2 } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2"; import { createSignSendProcessXCPDRequest, - createSignSendProcessDQRequests, - createSignSendProcessDRRequests, + createSignSendProcessDqRequests, + createSignSendProcessDrRequests, } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { SamlCertsAndKeys } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/security/types"; import { Config } from "../../shared/config"; @@ -60,7 +60,7 @@ export class IHEGatewayV2Direct extends IHEGatewayV2 { patientId: string; cxId: string; }): Promise { - await createSignSendProcessDQRequests({ + await createSignSendProcessDqRequests({ dqResponseUrl: this.dqResponseUrl, dqRequestsGatewayV2, samlCertsAndKeys: this.samlCertsAndKeys, @@ -78,7 +78,7 @@ export class IHEGatewayV2Direct extends IHEGatewayV2 { patientId: string; cxId: string; }): Promise { - await createSignSendProcessDRRequests({ + await createSignSendProcessDrRequests({ drResponseUrl: this.drResponseUrl, drRequestsGatewayV2, samlCertsAndKeys: this.samlCertsAndKeys, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index d9305b04f8..1921e4cc20 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -20,6 +20,7 @@ import { createAndSignBulkXCPDRequests } from "./outbound/xcpd/create/iti55-enve import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; import { SamlCertsAndKeys } from "./saml/security/types"; +import { getTrustedKeyStore } from "./saml/saml-client"; export async function sendProcessRetryDqRequest({ signedRequest, @@ -27,12 +28,14 @@ export async function sendProcessRetryDqRequest({ patientId, cxId, index, + trustedKeyStore, }: { signedRequest: SignedDqRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; + trustedKeyStore: string; }): Promise { async function sendProcessDqRequest() { const response = await sendSignedDqRequest({ @@ -41,6 +44,7 @@ export async function sendProcessRetryDqRequest({ patientId, cxId, index, + trustedKeyStore, }); return await processDqResponse({ response, @@ -60,12 +64,14 @@ export async function sendProcessRetryDrRequest({ patientId, cxId, index, + trustedKeyStore, }: { signedRequest: SignedDrRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; + trustedKeyStore: string; }): Promise { async function sendProcessDrRequest() { const response = await sendSignedDrRequest({ @@ -74,6 +80,7 @@ export async function sendProcessRetryDrRequest({ patientId, cxId, index, + trustedKeyStore, }); return await processDrResponse({ response, @@ -101,11 +108,13 @@ export async function createSignSendProcessXCPDRequest({ cxId: string; }): Promise { const signedRequests = createAndSignBulkXCPDRequests(xcpdRequest, samlCertsAndKeys); + const trustedKeyStore = await getTrustedKeyStore(); const responses = await sendSignedXCPDRequests({ signedRequests, samlCertsAndKeys, patientId, cxId, + trustedKeyStore, }); const results: OutboundPatientDiscoveryResp[] = responses.map(response => { return processXCPDResponse({ @@ -130,7 +139,7 @@ export async function createSignSendProcessXCPDRequest({ } } -export async function createSignSendProcessDQRequests({ +export async function createSignSendProcessDqRequests({ dqResponseUrl, dqRequestsGatewayV2, samlCertsAndKeys, @@ -148,11 +157,20 @@ export async function createSignSendProcessDQRequests({ samlCertsAndKeys, }); + const trustedKeyStore = await getTrustedKeyStore(); + const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDqRequest({ signedRequest, samlCertsAndKeys, patientId, cxId, index }); + return sendProcessRetryDqRequest({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + trustedKeyStore, + }); }); - const results = await Promise.allSettled(resultPromises); + const results = await Promise.all(resultPromises); for (const result of results) { try { @@ -170,7 +188,7 @@ export async function createSignSendProcessDQRequests({ } } -export async function createSignSendProcessDRRequests({ +export async function createSignSendProcessDrRequests({ drResponseUrl, drRequestsGatewayV2, samlCertsAndKeys, @@ -188,11 +206,20 @@ export async function createSignSendProcessDRRequests({ samlCertsAndKeys, }); + const trustedKeyStore = await getTrustedKeyStore(); + const resultPromises = signedRequests.map(async (signedRequest, index) => { - return sendProcessRetryDrRequest({ signedRequest, samlCertsAndKeys, patientId, cxId, index }); + return sendProcessRetryDrRequest({ + signedRequest, + samlCertsAndKeys, + patientId, + cxId, + index, + trustedKeyStore, + }); }); - const results = await Promise.allSettled(resultPromises); + const results = await Promise.all(resultPromises); for (const result of results) { try { diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts index 7c19cf8a22..46420b4e80 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts @@ -115,7 +115,8 @@ function parseDocumentReference({ return undefined; } - const creationTime = String(findSlotValue("creationTime")); + const creationTimeValue = findSlotValue("creationTime"); + const creationTime = creationTimeValue ? String(creationTimeValue) : undefined; const documentReference: DocumentReference = { homeCommunityId: getHomeCommunityIdForDr(extrinsicObject), @@ -135,12 +136,10 @@ async function handleSuccessResponse({ extrinsicObjects, outboundRequest, gateway, - attempt, }: { extrinsicObjects: ExtrinsicObject[]; outboundRequest: OutboundDocumentQueryReq; gateway: XCAGateway; - attempt?: number | undefined; }): Promise { const documentReferences = extrinsicObjects.flatMap( extrinsicObject => parseDocumentReference({ extrinsicObject, outboundRequest }) ?? [] @@ -154,7 +153,6 @@ async function handleSuccessResponse({ gateway, documentReference: documentReferences, externalGatewayPatient: outboundRequest.externalGatewayPatient, - retried: attempt, iheGatewayV2: true, }; return response; @@ -162,10 +160,8 @@ async function handleSuccessResponse({ export function processDqResponse({ response: { response, success, gateway, outboundRequest }, - attempt, }: { response: DQSamlClientResponse; - attempt?: number | undefined; }): Promise { if (success === false) { return Promise.resolve( @@ -173,7 +169,6 @@ export function processDqResponse({ httpError: response, outboundRequest, gateway: gateway, - attempt, }) ); } @@ -199,20 +194,17 @@ export function processDqResponse({ extrinsicObjects: toArray(extrinsicObjects), outboundRequest, gateway, - attempt, }); } else if (registryErrorList) { return handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, - attempt, }); } else { return handleEmptyResponse({ outboundRequest, gateway, - attempt, }); } } catch (error) { @@ -221,7 +213,6 @@ export function processDqResponse({ outboundRequest, gateway, text: errorToString(error), - attempt, }); } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts index f16974b64e..434f210c4c 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response.ts @@ -150,13 +150,11 @@ async function handleSuccessResponse({ outboundRequest, gateway, mtomResponse, - attempt, }: { documentResponses: DocumentResponse[]; outboundRequest: OutboundDocumentRetrievalReq; gateway: XCAGateway; mtomResponse: MtomAttachments; - attempt?: number | undefined; }): Promise { try { const idMapping = generateIdMapping(outboundRequest.documentReference); @@ -173,7 +171,6 @@ async function handleSuccessResponse({ responseTimestamp: dayjs().toISOString(), gateway, documentReference: documentReferences, - retried: attempt, iheGatewayV2: true, }; return response; @@ -184,10 +181,8 @@ async function handleSuccessResponse({ export async function processDrResponse({ response: { errorResponse, mtomResponse, gateway, outboundRequest }, - attempt, }: { response: DrSamlClientResponse; - attempt?: number; }): Promise { if (!gateway || !outboundRequest) throw new Error("Missing gateway or outboundRequest"); if (errorResponse) { @@ -195,7 +190,6 @@ export async function processDrResponse({ httpError: errorResponse, outboundRequest, gateway, - attempt, }); } if (!mtomResponse) { @@ -228,20 +222,17 @@ export async function processDrResponse({ outboundRequest, gateway, mtomResponse, - attempt, }); } else if (registryErrorList) { return handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, - attempt, }); } else { return handleEmptyResponse({ outboundRequest, gateway, - attempt, }); } } catch (error) { @@ -250,7 +241,6 @@ export async function processDrResponse({ outboundRequest, gateway, text: errorToString(error), - attempt, }); } } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 0563a72f41..6bf6f7243e 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -66,12 +66,10 @@ export async function handleRegistryErrorResponse({ registryErrorList, outboundRequest, gateway, - attempt, }: { registryErrorList: RegistryErrorList; outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; - attempt?: number | undefined; }): Promise { const operationOutcome = processRegistryErrorList(registryErrorList, outboundRequest); return { @@ -81,7 +79,6 @@ export async function handleRegistryErrorResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, - retried: attempt, iheGatewayV2: true, }; } @@ -126,12 +123,10 @@ export async function handleEmptyResponse({ outboundRequest, gateway, text = "No documents found", - attempt, }: { outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; text?: string; - attempt?: number | undefined; }): Promise { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", @@ -153,7 +148,6 @@ export async function handleEmptyResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, - retried: attempt, iheGatewayV2: true, }; } @@ -161,12 +155,10 @@ export async function handleEmptyResponse({ export async function handleSchemaErrorResponse({ outboundRequest, gateway, - attempt, text = "Schema Error", }: { outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; - attempt?: number | undefined; text?: string; }): Promise { const operationOutcome: OperationOutcome = { @@ -189,7 +181,6 @@ export async function handleSchemaErrorResponse({ responseTimestamp: dayjs().toISOString(), gateway, operationOutcome, - retried: attempt, iheGatewayV2: true, }; } diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts index 21dd36dad9..f58c609fad 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts @@ -2,7 +2,7 @@ import { XCAGateway, OutboundDocumentQueryReq } from "@metriport/ihe-gateway-sdk import { errorToString } from "../../../../../../util/error/shared"; import { capture } from "../../../../../../util/notifications"; import { SamlCertsAndKeys } from "../../../saml/security/types"; -import { getTrustedKeyStore, SamlClientResponse, sendSignedXml } from "../../../saml/saml-client"; +import { SamlClientResponse, sendSignedXml } from "../../../saml/saml-client"; import { SignedDqRequest } from "../create/iti38-envelope"; import { out } from "../../../../../../util/log"; import { storeDqResponse } from "../../../monitor/store"; @@ -21,14 +21,15 @@ export async function sendSignedDqRequest({ patientId, cxId, index, + trustedKeyStore, }: { request: SignedDqRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; + trustedKeyStore: string; }): Promise { - const trustedKeyStore = await getTrustedKeyStore(); try { const { response } = await sendSignedXml({ signedXml: request.signedRequest, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts index 253e2fe270..d7ce78d343 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts @@ -2,7 +2,7 @@ import { XCAGateway, OutboundDocumentRetrievalReq } from "@metriport/ihe-gateway import { errorToString } from "../../../../../../util/error/shared"; import { capture } from "../../../../../../util/notifications"; import { SamlCertsAndKeys } from "../../../saml/security/types"; -import { getTrustedKeyStore, sendSignedXmlMtom } from "../../../saml/saml-client"; +import { sendSignedXmlMtom } from "../../../saml/saml-client"; import { MtomAttachments } from "../mtom/parser"; import { SignedDrRequest } from "../create/iti39-envelope"; import { out } from "../../../../../../util/log"; @@ -24,14 +24,15 @@ export async function sendSignedDrRequest({ patientId, cxId, index, + trustedKeyStore, }: { request: SignedDrRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; + trustedKeyStore: string; }): Promise { - const trustedKeyStore = await getTrustedKeyStore(); try { const { mtomParts, rawResponse } = await sendSignedXmlMtom({ signedXml: request.signedRequest, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts index 2334e9d40c..3cae8ce114 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts @@ -1,7 +1,7 @@ import { XCPDGateway, OutboundPatientDiscoveryReq } from "@metriport/ihe-gateway-sdk"; import { errorToString } from "../../../../../../util/error/shared"; import { SamlCertsAndKeys } from "../../../saml/security/types"; -import { getTrustedKeyStore, SamlClientResponse, sendSignedXml } from "../../../saml/saml-client"; +import { SamlClientResponse, sendSignedXml } from "../../../saml/saml-client"; import { BulkSignedXCPD } from "../create/iti55-envelope"; import { out } from "../../../../../../util/log"; import { storeXcpdResponses } from "../../../monitor/store"; @@ -18,13 +18,14 @@ export async function sendSignedXCPDRequests({ samlCertsAndKeys, patientId, cxId, + trustedKeyStore, }: { signedRequests: BulkSignedXCPD[]; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; + trustedKeyStore: string; }): Promise { - const trustedKeyStore = await getTrustedKeyStore(); const requestPromises = signedRequests.map(async (request, index) => { try { const { response } = await sendSignedXml({ diff --git a/packages/lambdas/src/ihe-gateway-v2-outbound-document-query.ts b/packages/lambdas/src/ihe-gateway-v2-outbound-document-query.ts index 1c48412c9a..3da4613954 100644 --- a/packages/lambdas/src/ihe-gateway-v2-outbound-document-query.ts +++ b/packages/lambdas/src/ihe-gateway-v2-outbound-document-query.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/serverless"; import { DQRequestGatewayV2Params } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2"; -import { createSignSendProcessDQRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +import { createSignSendProcessDqRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { getEnvVarOrFail, getEnvType } from "@metriport/core/util/env-var"; import { out } from "@metriport/core/util/log"; import { getSamlCertsAndKeys } from "./shared/secrets"; @@ -19,7 +19,7 @@ export const handler = Sentry.AWSLambda.wrapHandler( ); const samlCertsAndKeys = await getSamlCertsAndKeys(); - await createSignSendProcessDQRequests({ + await createSignSendProcessDqRequests({ dqResponseUrl: documentQueryResponseUrl, dqRequestsGatewayV2, samlCertsAndKeys, diff --git a/packages/lambdas/src/ihe-gateway-v2-outbound-document-retrieval.ts b/packages/lambdas/src/ihe-gateway-v2-outbound-document-retrieval.ts index ed49fa0094..382ea6f3d2 100644 --- a/packages/lambdas/src/ihe-gateway-v2-outbound-document-retrieval.ts +++ b/packages/lambdas/src/ihe-gateway-v2-outbound-document-retrieval.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/serverless"; import { DRRequestGatewayV2Params } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2"; -import { createSignSendProcessDRRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; +import { createSignSendProcessDrRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { getEnvVarOrFail, getEnvType } from "@metriport/core/util/env-var"; import { out } from "@metriport/core/util/log"; import { getSamlCertsAndKeys } from "./shared/secrets"; @@ -20,7 +20,7 @@ export const handler = Sentry.AWSLambda.wrapHandler( const samlCertsAndKeys = await getSamlCertsAndKeys(); - await createSignSendProcessDRRequests({ + await createSignSendProcessDrRequests({ drResponseUrl, drRequestsGatewayV2, samlCertsAndKeys, diff --git a/packages/utils/src/saml/bulk-saml.ts b/packages/utils/src/saml/bulk-saml.ts index d5de20598c..3837daa3bb 100644 --- a/packages/utils/src/saml/bulk-saml.ts +++ b/packages/utils/src/saml/bulk-saml.ts @@ -9,7 +9,10 @@ import { XCPDGateway } from "@metriport/ihe-gateway-sdk"; import { createAndSignBulkXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/create/iti55-envelope"; import { sendSignedXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests"; import { processXCPDResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response"; -import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { + setRejectUnauthorized, + getTrustedKeyStore, +} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { MockS3Utils } from "./mock-s3"; import { Config } from "@metriport/core/util/config"; @@ -52,11 +55,13 @@ async function main() { console.log("signing bulk requests...", body.gateways.length); const xmlResponses = createAndSignBulkXCPDRequests(body, samlCertsAndKeys); console.log("sending bulk requests..."); + const trustedKeyStore = await getTrustedKeyStore(); const responses = await sendSignedXCPDRequests({ signedRequests: xmlResponses, samlCertsAndKeys, patientId: uuidv4(), cxId: uuidv4(), + trustedKeyStore, }); console.log("processing bulk responses..."); const results = responses.map(response => { diff --git a/packages/utils/src/saml/pre-prod-tester.ts b/packages/utils/src/saml/pre-prod-tester.ts index 5a57e2fa29..3086d73a74 100644 --- a/packages/utils/src/saml/pre-prod-tester.ts +++ b/packages/utils/src/saml/pre-prod-tester.ts @@ -25,7 +25,10 @@ import { sendProcessRetryDrRequest } from "@metriport/core/external/carequality/ import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { Config } from "@metriport/core/util/config"; -import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { + setRejectUnauthorized, + getTrustedKeyStore, +} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { MockS3Utils } from "./mock-s3"; /** @@ -165,6 +168,7 @@ async function XcpdIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; + const trustedKeyStore = await getTrustedKeyStore(); console.log("Querrying DB for Xcpds..."); const results = await queryDatabaseForXcpds(); console.log("Sending Xcpds..."); @@ -188,7 +192,7 @@ async function XcpdIntegrationTest() { principalCareProviderIds: [""], }; try { - const xcpdResponse = await queryXcpd(xcpdRequest); + const xcpdResponse = await queryXcpd(xcpdRequest, trustedKeyStore); return { xcpdRequest, xcpdResponse }; } catch (error) { console.error("Runtime error:", error); @@ -232,6 +236,7 @@ async function DQIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; + const trustedKeyStore = await getTrustedKeyStore(); const results = await queryDatabaseForDQs(); const promises = results.map(async result => { //eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -250,7 +255,7 @@ async function DQIntegrationTest() { externalGatewayPatient: dqResult.externalGatewayPatient, }; try { - const dqResponse = await queryDQ(dqRequest); + const dqResponse = await queryDQ(dqRequest, trustedKeyStore); return { dqResult, dqResponse }; } catch (error) { console.error("Runtime error:", error); @@ -293,6 +298,7 @@ async function DRIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; + const trustedKeyStore = await getTrustedKeyStore(); console.log("Querrying DB for DQs..."); const results = await queryDatabaseForDqsFromFailedDrs(); console.log("Sending DQs and DRs..."); @@ -318,7 +324,7 @@ async function DRIntegrationTest() { samlAttributes: samlAttributes, externalGatewayPatient: dqResult.externalGatewayPatient, }; - const dqResponse = await queryDQ(dqRequest); + const dqResponse = await queryDQ(dqRequest, trustedKeyStore); if (!dqResponse.documentReference) { console.log("No document references found for DQ: ", dqRequest.id); return undefined; @@ -342,7 +348,7 @@ async function DRIntegrationTest() { })), }; try { - const drResponse = await queryDR(drRequest); + const drResponse = await queryDR(drRequest, trustedKeyStore); return { drRequest, drResponse }; } catch (error) { console.error("Runtime error:", error); @@ -381,7 +387,8 @@ async function DRIntegrationTest() { } async function queryXcpd( - xcpdRequest: OutboundPatientDiscoveryReq + xcpdRequest: OutboundPatientDiscoveryReq, + trustedKeyStore: string ): Promise { try { const samlCertsAndKeys = { @@ -398,6 +405,7 @@ async function queryXcpd( samlCertsAndKeys, patientId: xcpdRequest.patientId, cxId: xcpdRequest.cxId, + trustedKeyStore, }); return processXCPDResponse({ @@ -409,7 +417,10 @@ async function queryXcpd( } } -async function queryDQ(dqRequest: OutboundDocumentQueryReq): Promise { +async function queryDQ( + dqRequest: OutboundDocumentQueryReq, + trustedKeyStore: string +): Promise { try { const samlCertsAndKeys = { publicCert: getEnvVarOrFail("CQ_ORG_CERTIFICATE_PRODUCTION"), @@ -429,11 +440,11 @@ async function queryDQ(dqRequest: OutboundDocumentQueryReq): Promise { try { const samlCertsAndKeys = { @@ -464,6 +476,7 @@ async function queryDR( patientId: uuidv4(), cxId: uuidv4(), index, + trustedKeyStore, }); }); const results = await Promise.all(resultPromises); diff --git a/packages/utils/src/saml/saml-server.ts b/packages/utils/src/saml/saml-server.ts index af658e235a..5a00e64ffe 100644 --- a/packages/utils/src/saml/saml-server.ts +++ b/packages/utils/src/saml/saml-server.ts @@ -17,7 +17,10 @@ import { } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; -import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { + setRejectUnauthorized, + getTrustedKeyStore, +} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { Config } from "@metriport/core/util/config"; import { MockS3Utils } from "./mock-s3"; @@ -55,12 +58,14 @@ app.post("/xcpd", async (req: Request, res: Response) => { } try { + const trustedKeyStore = await getTrustedKeyStore(); const xmlResponses = createAndSignBulkXCPDRequests(req.body, samlCertsAndKeys); const response = await sendSignedXCPDRequests({ signedRequests: xmlResponses, samlCertsAndKeys, patientId: uuidv4(), cxId: uuidv4(), + trustedKeyStore, }); const results = response.map(response => { return processXCPDResponse({ @@ -90,6 +95,8 @@ app.post("/xcadq", async (req: Request, res: Response) => { samlCertsAndKeys, }); + const trustedKeyStore = await getTrustedKeyStore(); + const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDqRequest({ signedRequest, @@ -97,6 +104,7 @@ app.post("/xcadq", async (req: Request, res: Response) => { patientId: uuidv4(), cxId: uuidv4(), index, + trustedKeyStore, }); }); const results = await Promise.all(resultPromises); @@ -122,6 +130,8 @@ app.post("/xcadr", async (req: Request, res: Response) => { bulkBodyData: req.body, samlCertsAndKeys, }); + const trustedKeyStore = await getTrustedKeyStore(); + const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDrRequest({ signedRequest, @@ -129,6 +139,7 @@ app.post("/xcadr", async (req: Request, res: Response) => { patientId: uuidv4(), cxId: uuidv4(), index, + trustedKeyStore, }); }); From 54f299203c49eddcc861cd6fd9849b59519ca1f2 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Mon, 17 Jun 2024 16:58:09 -0400 Subject: [PATCH 09/13] feat(ihe): add executew with network retries to saml client + retries for 429s Refs: #1667 Signed-off-by: Jonah Kaye --- .../outbound/xca/process/error.ts | 1 + .../ihe-gateway-v2/saml/saml-client.ts | 54 +++++++++++++------ packages/shared/src/net/retry.ts | 13 +++-- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 6bf6f7243e..8b3e47acd9 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -195,6 +195,7 @@ export function isRetryable(outboundResponse: OutboundDocumentRetrievalResp | un outboundResponse.operationOutcome?.issue.some( issue => issue.severity === "error" && + issue.code !== "http-error" && !knownNonRetryableErrors.some( nonRetryableError => "text" in issue.details && issue.details.text.includes(nonRetryableError) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts index 0cbfcc0f5f..10f22a9688 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts @@ -7,6 +7,8 @@ import { Config } from "../../../../util/config"; import { out } from "../../../../util/log"; import { MetriportError } from "../../../../util/error/metriport-error"; import { createMtomContentTypeAndPayload } from "../outbound/xca/mtom/builder"; +import { executeWithNetworkRetries } from "@metriport/shared"; + import { parseMtomResponse, getBoundaryFromMtomResponse, @@ -76,15 +78,24 @@ export async function sendSignedXml({ secureOptions: constants.SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, }); - const response = await axios.post(url, signedXml, { - timeout: 120000, - headers: { - "Content-Type": "application/soap+xml;charset=UTF-8", - Accept: "application/soap+xml", - "Cache-Control": "no-cache", + const response = await executeWithNetworkRetries( + async () => { + return axios.post(url, signedXml, { + timeout: 120000, + headers: { + "Content-Type": "application/soap+xml;charset=UTF-8", + Accept: "application/soap+xml", + "Cache-Control": "no-cache", + }, + httpsAgent: agent, + }); }, - httpsAgent: agent, - }); + { + initialDelay: 3000, + maxAttempts: 3, + httpCodesToRetry: ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT"], + } + ); return { response: response.data, contentType: response.headers["content-type"] }; } @@ -112,16 +123,25 @@ export async function sendSignedXmlMtom({ }); const { contentType, payload } = createMtomContentTypeAndPayload(signedXml); - const response = await axios.post(url, payload, { - timeout: timeout, - headers: { - "Accept-Encoding": "gzip, deflate", - "Content-Type": contentType, - "Cache-Control": "no-cache", + const response = await executeWithNetworkRetries( + async () => { + return axios.post(url, payload, { + timeout: timeout, + headers: { + "Accept-Encoding": "gzip, deflate", + "Content-Type": contentType, + "Cache-Control": "no-cache", + }, + httpsAgent: agent, + responseType: "arraybuffer", + }); }, - httpsAgent: agent, - responseType: "arraybuffer", - }); + { + initialDelay: 3000, + maxAttempts: 3, + httpCodesToRetry: ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT"], + } + ); const binaryData: Buffer = Buffer.isBuffer(response.data) ? response.data diff --git a/packages/shared/src/net/retry.ts b/packages/shared/src/net/retry.ts index 6df93184e7..22ce7ba02e 100644 --- a/packages/shared/src/net/retry.ts +++ b/packages/shared/src/net/retry.ts @@ -12,6 +12,7 @@ export type ExecuteWithHttpRetriesOptions = Omit< > & { /** The network error codes to retry. See `defaultOptions` for defaults. */ httpCodesToRetry: NetworkError[]; + httpStatusCodesToRetry: number[]; }; const defaultOptions: ExecuteWithHttpRetriesOptions = { @@ -21,6 +22,7 @@ const defaultOptions: ExecuteWithHttpRetriesOptions = { "ECONNREFUSED", // (Connection refused): No connection could be made because the target machine actively refused it. This usually results from trying to connect to a service that is inactive on the foreign host. "ECONNRESET", // (Connection reset by peer): A connection was forcibly closed by a peer. This normally results from a loss of the connection on the remote socket due to a timeout or reboot. Commonly encountered via the http and net modules. ], + httpStatusCodesToRetry: [429], // 429 Too Many Requests }; /** @@ -44,14 +46,19 @@ export async function executeWithNetworkRetries( options?: Partial ): Promise { const actualOptions = { ...defaultOptions, ...options }; - const { httpCodesToRetry } = actualOptions; + const { httpCodesToRetry, httpStatusCodesToRetry } = actualOptions; const codesAsString = httpCodesToRetry.map(String); return executeWithRetries(fn, { ...actualOptions, shouldRetry: (_, error: unknown) => { const networkCode = axios.isAxiosError(error) ? error.code : undefined; - if (!networkCode) return false; - return codesAsString.includes(networkCode); + const networkStatus = axios.isAxiosError(error) ? error.response?.status : undefined; + if (!networkCode && !networkStatus) return false; + return ( + (networkCode && codesAsString.includes(networkCode)) || + (networkStatus && httpStatusCodesToRetry.includes(networkStatus)) || + false + ); }, }); } From 888fd353beba38385ad5277c37f82ab29e80e03e Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Mon, 17 Jun 2024 17:09:14 -0400 Subject: [PATCH 10/13] feat(ihe): improve logging Refs: #1667 Signed-off-by: Jonah Kaye --- .../carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts | 2 +- .../carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts index f58c609fad..e18eafd490 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts @@ -65,7 +65,7 @@ export async function sendSignedDqRequest({ const errorString: string = errorToString(error); const extra = { errorString, - request, + outboundRequest: request.outboundRequest, patientId, cxId, }; diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts index d7ce78d343..70a96c0b37 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts @@ -71,7 +71,7 @@ export async function sendSignedDrRequest({ const errorString: string = errorToString(error); const extra = { errorString, - request, + outboundRequest: request.outboundRequest, patientId, cxId, }; From 6602fab6527fdb87cf18f78cef186f7197d9e399 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Mon, 17 Jun 2024 21:31:49 -0400 Subject: [PATCH 11/13] feat(ihe): resloving comments Refs: #1667 Signed-off-by: Jonah Kaye --- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 15 ---------- .../outbound/xca/process/dq-response.ts | 12 ++++---- .../outbound/xca/process/error.ts | 4 +-- .../outbound/xca/send/dq-requests.ts | 3 -- .../outbound/xca/send/dr-requests.ts | 3 -- .../outbound/xcpd/send/xcpd-requests.ts | 3 -- .../ihe-gateway-v2/saml/saml-client.ts | 15 ++++++---- packages/shared/src/net/retry.ts | 2 ++ packages/utils/src/saml/bulk-saml.ts | 7 +---- packages/utils/src/saml/pre-prod-tester.ts | 30 +++++-------------- packages/utils/src/saml/saml-server.ts | 12 +------- 11 files changed, 29 insertions(+), 77 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 1921e4cc20..874e89ac2a 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -20,7 +20,6 @@ import { createAndSignBulkXCPDRequests } from "./outbound/xcpd/create/iti55-enve import { processXCPDResponse } from "./outbound/xcpd/process/xcpd-response"; import { sendSignedXCPDRequests } from "./outbound/xcpd/send/xcpd-requests"; import { SamlCertsAndKeys } from "./saml/security/types"; -import { getTrustedKeyStore } from "./saml/saml-client"; export async function sendProcessRetryDqRequest({ signedRequest, @@ -28,14 +27,12 @@ export async function sendProcessRetryDqRequest({ patientId, cxId, index, - trustedKeyStore, }: { signedRequest: SignedDqRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; - trustedKeyStore: string; }): Promise { async function sendProcessDqRequest() { const response = await sendSignedDqRequest({ @@ -44,7 +41,6 @@ export async function sendProcessRetryDqRequest({ patientId, cxId, index, - trustedKeyStore, }); return await processDqResponse({ response, @@ -64,14 +60,12 @@ export async function sendProcessRetryDrRequest({ patientId, cxId, index, - trustedKeyStore, }: { signedRequest: SignedDrRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; - trustedKeyStore: string; }): Promise { async function sendProcessDrRequest() { const response = await sendSignedDrRequest({ @@ -80,7 +74,6 @@ export async function sendProcessRetryDrRequest({ patientId, cxId, index, - trustedKeyStore, }); return await processDrResponse({ response, @@ -108,13 +101,11 @@ export async function createSignSendProcessXCPDRequest({ cxId: string; }): Promise { const signedRequests = createAndSignBulkXCPDRequests(xcpdRequest, samlCertsAndKeys); - const trustedKeyStore = await getTrustedKeyStore(); const responses = await sendSignedXCPDRequests({ signedRequests, samlCertsAndKeys, patientId, cxId, - trustedKeyStore, }); const results: OutboundPatientDiscoveryResp[] = responses.map(response => { return processXCPDResponse({ @@ -157,8 +148,6 @@ export async function createSignSendProcessDqRequests({ samlCertsAndKeys, }); - const trustedKeyStore = await getTrustedKeyStore(); - const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDqRequest({ signedRequest, @@ -166,7 +155,6 @@ export async function createSignSendProcessDqRequests({ patientId, cxId, index, - trustedKeyStore, }); }); @@ -206,8 +194,6 @@ export async function createSignSendProcessDrRequests({ samlCertsAndKeys, }); - const trustedKeyStore = await getTrustedKeyStore(); - const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDrRequest({ signedRequest, @@ -215,7 +201,6 @@ export async function createSignSendProcessDrRequests({ patientId, cxId, index, - trustedKeyStore, }); }); diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts index 46420b4e80..606cfd57d0 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/dq-response.ts @@ -164,13 +164,11 @@ export function processDqResponse({ response: DQSamlClientResponse; }): Promise { if (success === false) { - return Promise.resolve( - handleHttpErrorResponse({ - httpError: response, - outboundRequest, - gateway: gateway, - }) - ); + return handleHttpErrorResponse({ + httpError: response, + outboundRequest, + gateway: gateway, + }); } const parser = new XMLParser({ ignoreAttributes: false, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts index 8b3e47acd9..aea0484b30 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/process/error.ts @@ -83,7 +83,7 @@ export async function handleRegistryErrorResponse({ }; } -export function handleHttpErrorResponse({ +export async function handleHttpErrorResponse({ httpError, outboundRequest, gateway, @@ -93,7 +93,7 @@ export function handleHttpErrorResponse({ outboundRequest: OutboundDocumentQueryReq | OutboundDocumentRetrievalReq; gateway: XCAGateway; attempt?: number | undefined; -}): OutboundDocumentQueryResp | OutboundDocumentRetrievalResp { +}): Promise { const operationOutcome: OperationOutcome = { resourceType: "OperationOutcome", id: outboundRequest.id, diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts index e18eafd490..a54f0c2685 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dq-requests.ts @@ -21,21 +21,18 @@ export async function sendSignedDqRequest({ patientId, cxId, index, - trustedKeyStore, }: { request: SignedDqRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; - trustedKeyStore: string; }): Promise { try { const { response } = await sendSignedXml({ signedXml: request.signedRequest, url: request.gateway.url, samlCertsAndKeys, - trustedKeyStore, }); log( `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts index 70a96c0b37..7ac96f50f2 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xca/send/dr-requests.ts @@ -24,21 +24,18 @@ export async function sendSignedDrRequest({ patientId, cxId, index, - trustedKeyStore, }: { request: SignedDrRequest; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; index: number; - trustedKeyStore: string; }): Promise { try { const { mtomParts, rawResponse } = await sendSignedXmlMtom({ signedXml: request.signedRequest, url: request.gateway.url, samlCertsAndKeys, - trustedKeyStore, }); log( `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts index 3cae8ce114..95282571e4 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests.ts @@ -18,13 +18,11 @@ export async function sendSignedXCPDRequests({ samlCertsAndKeys, patientId, cxId, - trustedKeyStore, }: { signedRequests: BulkSignedXCPD[]; samlCertsAndKeys: SamlCertsAndKeys; patientId: string; cxId: string; - trustedKeyStore: string; }): Promise { const requestPromises = signedRequests.map(async (request, index) => { try { @@ -32,7 +30,6 @@ export async function sendSignedXCPDRequests({ signedXml: request.signedRequest, url: request.gateway.url, samlCertsAndKeys, - trustedKeyStore, }); log( `Request ${index + 1} sent successfully to: ${request.gateway.url} + oid: ${ diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts index 10f22a9688..8adf4aefb6 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts @@ -19,6 +19,11 @@ import { const { log } = out("Saml Client"); const timeout = 120000; let rejectUnauthorized = true; +let trustedStore: string | undefined = undefined; +async function getTrustedKeyStore(): Promise { + if (!trustedStore) trustedStore = await loadTrustedKeyStore(); + return trustedStore; +} /* * ONLY use this function for testing purposes. It will turn off SSL Verification of the server if set to false. @@ -36,7 +41,7 @@ export type SamlClientResponse = { success: boolean; }; -export async function getTrustedKeyStore(): Promise { +export async function loadTrustedKeyStore(): Promise { try { const s3 = new AWS.S3({ region: Config.getAWSRegion() }); const trustBundleBucketName = Config.getCqTrustBundleBucketName(); @@ -60,13 +65,12 @@ export async function sendSignedXml({ signedXml, url, samlCertsAndKeys, - trustedKeyStore, }: { signedXml: string; url: string; samlCertsAndKeys: SamlCertsAndKeys; - trustedKeyStore: string; }): Promise<{ response: string; contentType: string }> { + const trustedKeyStore = await getTrustedKeyStore(); const agent = new https.Agent({ rejectUnauthorized: getRejectUnauthorized(), requestCert: true, @@ -93,6 +97,7 @@ export async function sendSignedXml({ { initialDelay: 3000, maxAttempts: 3, + //TODO: This introduces retry on timeout without needing to specify the http Code: https://github.com/metriport/metriport/pull/2285. Remove once PR is merged httpCodesToRetry: ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT"], } ); @@ -104,13 +109,12 @@ export async function sendSignedXmlMtom({ signedXml, url, samlCertsAndKeys, - trustedKeyStore, }: { signedXml: string; url: string; samlCertsAndKeys: SamlCertsAndKeys; - trustedKeyStore: string; }): Promise<{ mtomParts: MtomAttachments; rawResponse: Buffer }> { + const trustedKeyStore = await getTrustedKeyStore(); const agent = new https.Agent({ rejectUnauthorized: getRejectUnauthorized(), requestCert: true, @@ -139,6 +143,7 @@ export async function sendSignedXmlMtom({ { initialDelay: 3000, maxAttempts: 3, + //TODO: This introduces retry on timeout without needing to specify the http Code: https://github.com/metriport/metriport/pull/2285. Remove once PR is merged httpCodesToRetry: ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT"], } ); diff --git a/packages/shared/src/net/retry.ts b/packages/shared/src/net/retry.ts index 22ce7ba02e..3cbd13c8d1 100644 --- a/packages/shared/src/net/retry.ts +++ b/packages/shared/src/net/retry.ts @@ -17,6 +17,7 @@ export type ExecuteWithHttpRetriesOptions = Omit< const defaultOptions: ExecuteWithHttpRetriesOptions = { ...defaultRetryWithBackoffOptions, + initialDelay: 1000, httpCodesToRetry: [ // https://nodejs.org/docs/latest-v18.x/api/errors.html#common-system-errors "ECONNREFUSED", // (Connection refused): No connection could be made because the target machine actively refused it. This usually results from trying to connect to a service that is inactive on the foreign host. @@ -33,6 +34,7 @@ const defaultOptions: ExecuteWithHttpRetriesOptions = { * This is a specialization of `executeWithRetries` for network errors. * By default it retries on ECONNREFUSED and ECONNRESET (customize the errors to retry * setting the option `httpCodesToRetry`). + * By default it also retries on HTTP status code 429 (Too Many Requests). * * @param fn the function to be executed * @param options the options to be used; see `ExecuteWithHttpRetriesOptions` for components and diff --git a/packages/utils/src/saml/bulk-saml.ts b/packages/utils/src/saml/bulk-saml.ts index 3837daa3bb..d5de20598c 100644 --- a/packages/utils/src/saml/bulk-saml.ts +++ b/packages/utils/src/saml/bulk-saml.ts @@ -9,10 +9,7 @@ import { XCPDGateway } from "@metriport/ihe-gateway-sdk"; import { createAndSignBulkXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/create/iti55-envelope"; import { sendSignedXCPDRequests } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/send/xcpd-requests"; import { processXCPDResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xcpd/process/xcpd-response"; -import { - setRejectUnauthorized, - getTrustedKeyStore, -} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { MockS3Utils } from "./mock-s3"; import { Config } from "@metriport/core/util/config"; @@ -55,13 +52,11 @@ async function main() { console.log("signing bulk requests...", body.gateways.length); const xmlResponses = createAndSignBulkXCPDRequests(body, samlCertsAndKeys); console.log("sending bulk requests..."); - const trustedKeyStore = await getTrustedKeyStore(); const responses = await sendSignedXCPDRequests({ signedRequests: xmlResponses, samlCertsAndKeys, patientId: uuidv4(), cxId: uuidv4(), - trustedKeyStore, }); console.log("processing bulk responses..."); const results = responses.map(response => { diff --git a/packages/utils/src/saml/pre-prod-tester.ts b/packages/utils/src/saml/pre-prod-tester.ts index 3086d73a74..102eedbaad 100644 --- a/packages/utils/src/saml/pre-prod-tester.ts +++ b/packages/utils/src/saml/pre-prod-tester.ts @@ -25,10 +25,7 @@ import { sendProcessRetryDrRequest } from "@metriport/core/external/carequality/ import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; import { Config } from "@metriport/core/util/config"; -import { - setRejectUnauthorized, - getTrustedKeyStore, -} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { MockS3Utils } from "./mock-s3"; /** @@ -168,7 +165,6 @@ async function XcpdIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; - const trustedKeyStore = await getTrustedKeyStore(); console.log("Querrying DB for Xcpds..."); const results = await queryDatabaseForXcpds(); console.log("Sending Xcpds..."); @@ -192,7 +188,7 @@ async function XcpdIntegrationTest() { principalCareProviderIds: [""], }; try { - const xcpdResponse = await queryXcpd(xcpdRequest, trustedKeyStore); + const xcpdResponse = await queryXcpd(xcpdRequest); return { xcpdRequest, xcpdResponse }; } catch (error) { console.error("Runtime error:", error); @@ -236,7 +232,6 @@ async function DQIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; - const trustedKeyStore = await getTrustedKeyStore(); const results = await queryDatabaseForDQs(); const promises = results.map(async result => { //eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -255,7 +250,7 @@ async function DQIntegrationTest() { externalGatewayPatient: dqResult.externalGatewayPatient, }; try { - const dqResponse = await queryDQ(dqRequest, trustedKeyStore); + const dqResponse = await queryDQ(dqRequest); return { dqResult, dqResponse }; } catch (error) { console.error("Runtime error:", error); @@ -298,7 +293,6 @@ async function DRIntegrationTest() { let failureCount = 0; let runTimeErrorCount = 0; - const trustedKeyStore = await getTrustedKeyStore(); console.log("Querrying DB for DQs..."); const results = await queryDatabaseForDqsFromFailedDrs(); console.log("Sending DQs and DRs..."); @@ -324,7 +318,7 @@ async function DRIntegrationTest() { samlAttributes: samlAttributes, externalGatewayPatient: dqResult.externalGatewayPatient, }; - const dqResponse = await queryDQ(dqRequest, trustedKeyStore); + const dqResponse = await queryDQ(dqRequest); if (!dqResponse.documentReference) { console.log("No document references found for DQ: ", dqRequest.id); return undefined; @@ -348,7 +342,7 @@ async function DRIntegrationTest() { })), }; try { - const drResponse = await queryDR(drRequest, trustedKeyStore); + const drResponse = await queryDR(drRequest); return { drRequest, drResponse }; } catch (error) { console.error("Runtime error:", error); @@ -387,8 +381,7 @@ async function DRIntegrationTest() { } async function queryXcpd( - xcpdRequest: OutboundPatientDiscoveryReq, - trustedKeyStore: string + xcpdRequest: OutboundPatientDiscoveryReq ): Promise { try { const samlCertsAndKeys = { @@ -405,7 +398,6 @@ async function queryXcpd( samlCertsAndKeys, patientId: xcpdRequest.patientId, cxId: xcpdRequest.cxId, - trustedKeyStore, }); return processXCPDResponse({ @@ -417,10 +409,7 @@ async function queryXcpd( } } -async function queryDQ( - dqRequest: OutboundDocumentQueryReq, - trustedKeyStore: string -): Promise { +async function queryDQ(dqRequest: OutboundDocumentQueryReq): Promise { try { const samlCertsAndKeys = { publicCert: getEnvVarOrFail("CQ_ORG_CERTIFICATE_PRODUCTION"), @@ -440,7 +429,6 @@ async function queryDQ( patientId: dqRequest.patientId, cxId: dqRequest.cxId, index: 0, - trustedKeyStore, }); return processDqResponse({ @@ -453,8 +441,7 @@ async function queryDQ( } async function queryDR( - drRequest: OutboundDocumentRetrievalReq, - trustedKeyStore: string + drRequest: OutboundDocumentRetrievalReq ): Promise { try { const samlCertsAndKeys = { @@ -476,7 +463,6 @@ async function queryDR( patientId: uuidv4(), cxId: uuidv4(), index, - trustedKeyStore, }); }); const results = await Promise.all(resultPromises); diff --git a/packages/utils/src/saml/saml-server.ts b/packages/utils/src/saml/saml-server.ts index 5a00e64ffe..8eb407e0e6 100644 --- a/packages/utils/src/saml/saml-server.ts +++ b/packages/utils/src/saml/saml-server.ts @@ -17,10 +17,7 @@ import { } from "@metriport/core/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringDrResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/outbound/xca/process/dr-response"; import { setS3UtilsInstance as setS3UtilsInstanceForStoringIheResponse } from "@metriport/core/external/carequality/ihe-gateway-v2/monitor/store"; -import { - setRejectUnauthorized, - getTrustedKeyStore, -} from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; +import { setRejectUnauthorized } from "@metriport/core/external/carequality/ihe-gateway-v2/saml/saml-client"; import { Config } from "@metriport/core/util/config"; import { MockS3Utils } from "./mock-s3"; @@ -58,14 +55,12 @@ app.post("/xcpd", async (req: Request, res: Response) => { } try { - const trustedKeyStore = await getTrustedKeyStore(); const xmlResponses = createAndSignBulkXCPDRequests(req.body, samlCertsAndKeys); const response = await sendSignedXCPDRequests({ signedRequests: xmlResponses, samlCertsAndKeys, patientId: uuidv4(), cxId: uuidv4(), - trustedKeyStore, }); const results = response.map(response => { return processXCPDResponse({ @@ -95,8 +90,6 @@ app.post("/xcadq", async (req: Request, res: Response) => { samlCertsAndKeys, }); - const trustedKeyStore = await getTrustedKeyStore(); - const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDqRequest({ signedRequest, @@ -104,7 +97,6 @@ app.post("/xcadq", async (req: Request, res: Response) => { patientId: uuidv4(), cxId: uuidv4(), index, - trustedKeyStore, }); }); const results = await Promise.all(resultPromises); @@ -130,7 +122,6 @@ app.post("/xcadr", async (req: Request, res: Response) => { bulkBodyData: req.body, samlCertsAndKeys, }); - const trustedKeyStore = await getTrustedKeyStore(); const resultPromises = signedRequests.map(async (signedRequest, index) => { return sendProcessRetryDrRequest({ @@ -139,7 +130,6 @@ app.post("/xcadr", async (req: Request, res: Response) => { patientId: uuidv4(), cxId: uuidv4(), index, - trustedKeyStore, }); }); From 298f63d010a047259f8bb6f4c3605a1c0115bdba Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Mon, 17 Jun 2024 21:45:34 -0400 Subject: [PATCH 12/13] feat(ihe): promise all settled and filtering Refs: #1667 Signed-off-by: Jonah Kaye --- .../ihe-gateway-v2/ihe-gateway-v2-logic.ts | 21 ++++++++++++++----- packages/shared/src/net/retry.ts | 1 - 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts index 874e89ac2a..c093664af0 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/ihe-gateway-v2-logic.ts @@ -158,9 +158,14 @@ export async function createSignSendProcessDqRequests({ }); }); - const results = await Promise.all(resultPromises); - - for (const result of results) { + const results = await Promise.allSettled(resultPromises); + const successfulResults = results + .filter( + (result): result is PromiseFulfilledResult => + result.status === "fulfilled" + ) + .map(result => result.value); + for (const result of successfulResults) { try { await executeWithNetworkRetries(async () => axios.post(dqResponseUrl, result)); } catch (error) { @@ -204,9 +209,15 @@ export async function createSignSendProcessDrRequests({ }); }); - const results = await Promise.all(resultPromises); + const results = await Promise.allSettled(resultPromises); + const successfulResults = results + .filter( + (result): result is PromiseFulfilledResult => + result.status === "fulfilled" + ) + .map(result => result.value); - for (const result of results) { + for (const result of successfulResults) { try { await executeWithNetworkRetries(async () => axios.post(drResponseUrl, result)); } catch (error) { diff --git a/packages/shared/src/net/retry.ts b/packages/shared/src/net/retry.ts index 3cbd13c8d1..7229e1264a 100644 --- a/packages/shared/src/net/retry.ts +++ b/packages/shared/src/net/retry.ts @@ -55,7 +55,6 @@ export async function executeWithNetworkRetries( shouldRetry: (_, error: unknown) => { const networkCode = axios.isAxiosError(error) ? error.code : undefined; const networkStatus = axios.isAxiosError(error) ? error.response?.status : undefined; - if (!networkCode && !networkStatus) return false; return ( (networkCode && codesAsString.includes(networkCode)) || (networkStatus && httpStatusCodesToRetry.includes(networkStatus)) || From 4a679bfe5867f7940a8ca5f93e678237bb2669a6 Mon Sep 17 00:00:00 2001 From: Jonah Kaye Date: Tue, 18 Jun 2024 08:53:53 -0400 Subject: [PATCH 13/13] feat(ihe): removing an export Refs: #1667 Signed-off-by: Jonah Kaye --- .../external/carequality/ihe-gateway-v2/saml/saml-client.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts index 8adf4aefb6..28dcde7980 100644 --- a/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts +++ b/packages/core/src/external/carequality/ihe-gateway-v2/saml/saml-client.ts @@ -40,8 +40,7 @@ export type SamlClientResponse = { response: string; success: boolean; }; - -export async function loadTrustedKeyStore(): Promise { +async function loadTrustedKeyStore(): Promise { try { const s3 = new AWS.S3({ region: Config.getAWSRegion() }); const trustBundleBucketName = Config.getCqTrustBundleBucketName();