Merge pull request #2261 from metriport/develop

RELEASE - 2209 - Prevent docs redownloads
metriport · Jun 13, 2024 · 9a3f1ed · 9a3f1ed
2 parents ab54b02 + 70a121a
commit 9a3f1ed
Show file tree

Hide file tree

Showing 4 changed files with 186 additions and 94 deletions.
diff --git a/packages/api/src/external/carequality/document/get-non-existent-doc-refs.ts b/packages/api/src/external/carequality/document/get-non-existent-doc-refs.ts
@@ -1,4 +1,4 @@
-import { createFileName } from "@metriport/core/domain/filename";
+import { createDocumentFilePath } from "@metriport/core/domain/document/filename";
 import { S3Utils } from "@metriport/core/external/aws/s3";
 import { executeAsynchronously } from "@metriport/core/util/concurrency";
 import { errorToString } from "@metriport/core/util/error/shared";
@@ -49,7 +49,12 @@ async function checkDocRefsExistInS3(
  documents,
  async doc => {
  try {
- const fileName = createFileName(cxId, patientId, doc.metriportId);
+ const fileName = createDocumentFilePath(
+ cxId,
+ patientId,
+ doc.metriportId,
+ doc.contentType || undefined
+ );
 
  const { exists } = await s3Utils.getFileInfoFromS3(fileName, s3BucketName);
 

diff --git a/packages/api/src/external/carequality/document/process-outbound-document-query-resps.ts b/packages/api/src/external/carequality/document/process-outbound-document-query-resps.ts
@@ -56,32 +56,7 @@ export async function processOutboundDocumentQueryResps({
  const cqData = getCQData(patient.data.externalData);
  const docQueryStartedAt = cqData?.documentQueryProgress?.startedAt;
  const duration = elapsedTimeFromNow(docQueryStartedAt);
-
- const addDocRefId = addMetriportDocRefID({ cxId, patientId, requestId });
- const resultsWithMetriportId: OutboundDocumentQueryResp[] = [];
-
- const updateResponseDocumentReferencesWithMetriportId = async (
- response: OutboundDocumentQueryResp
- ): Promise<void> => {
- const updatedDocumentReferences = response.documentReference
- ? await Promise.all(
- response.documentReference.map(async docRef => {
- return await addDocRefId(docRef);
- })
- )
- : response.documentReference;
-
- resultsWithMetriportId.push({
- ...response,
- documentReference: updatedDocumentReferences,
- });
- };
-
- await executeAsynchronously(response, updateResponseDocumentReferencesWithMetriportId, {
- numberOfParallelExecutions: 20,
- });
-
- const docRefs = resultsWithMetriportId.flatMap(result => result.documentReference ?? []);
+ const docRefs = response.flatMap(result => result.documentReference ?? []);
  const contentTypes = docRefs.map(getContentTypeOrUnknown);
  const contentTypeCounts = getDocumentReferenceContentTypeCounts(contentTypes);
 
@@ -98,9 +73,17 @@ export async function processOutboundDocumentQueryResps({
  },
  });
 
- const docRefsWithMetriportId = filterDocRefsWithMetriportId(docRefs);
+ const responsesWithDocsToDownload = await getRespWithDocsToDownload({
+ cxId,
+ patientId,
+ requestId,
+ response,
+ });
+
+ const docsToDownload = responsesWithDocsToDownload.flatMap(
+ result => result.documentReference ?? []
+ );
 
- const docsToDownload = await getNonExistentDocRefs(docRefsWithMetriportId, patientId, cxId);
  const convertibleDocCount = docsToDownload.filter(doc =>
  isConvertible(doc.contentType || undefined)
  ).length;
@@ -147,44 +130,14 @@ export async function processOutboundDocumentQueryResps({
  // Since we have most of the document contents when doing the document query,
  // we will store this in FHIR and then upsert the reference to the s3 object in FHIR
  // when doing the doc retrieval
- await storeInitDocRefInFHIR(docRefsWithMetriportId, cxId, patientId, log);
-
- const resultsWithMetriportIdAndDrUrl: OutboundDocumentQueryResp[] = [];
-
- const replaceDqUrlWithDrUrl = async (
- outboundDocumentQueryResp: OutboundDocumentQueryResp
- ): Promise<void> => {
- const gateway = await getCQDirectoryEntry(outboundDocumentQueryResp.gateway.homeCommunityId);
-
- if (!gateway) {
- const msg = `Gateway not found - Doc Retrieval`;
- log(`${msg}: ${outboundDocumentQueryResp.gateway.homeCommunityId} skipping...`);
- capture.message(msg, {
- extra: {
- context: `cq.dq.getCQDirectoryEntry`,
- patientId,
- requestId,
- cxId,
- gateway: outboundDocumentQueryResp.gateway,
- },
- });
- return;
- } else if (!gateway.urlDR) {
- log(`Gateway ${gateway.id} has no DR URL, skipping...`);
- return;
- }
-
- resultsWithMetriportIdAndDrUrl.push({
- ...outboundDocumentQueryResp,
- gateway: {
- ...outboundDocumentQueryResp.gateway,
- url: gateway.urlDR,
- },
- });
- };
+ await storeInitDocRefInFHIR(docsToDownload, cxId, patientId, log);
 
- await executeAsynchronously(resultsWithMetriportId, replaceDqUrlWithDrUrl, {
- numberOfParallelExecutions: 20,
+ const resultsWithMetriportIdAndDrUrl = await replaceDqUrlWithDrUrl({
+ patientId,
+ requestId,
+ cxId,
+ responsesWithDocsToDownload,
+ log,
  });
 
  const outboundDocumentQueryResultsV1: OutboundDocumentQueryResp[] = [];
@@ -298,6 +251,164 @@ function buildInterrupt({
  };
 }
 
+type DqRespWithDocRefsWithMetriportId = OutboundDocumentQueryResp & {
+ documentReference: DocumentReferenceWithMetriportId[];
+};
+
+async function getRespWithDocsToDownload({
+ cxId,
+ patientId,
+ requestId,
+ response,
+}: OutboundDocQueryRespParam): Promise<DqRespWithDocRefsWithMetriportId[]> {
+ const respWithDocsToDownload: DqRespWithDocRefsWithMetriportId[] = [];
+
+ await executeAsynchronously(
+ response,
+ async gwResp => {
+ const resultsWithMetriportId = await getDocumentReferencesWithMetriportId({
+ cxId,
+ patientId,
+ requestId,
+ response: gwResp,
+ });
+ const docRefs = resultsWithMetriportId.flatMap(result => result.documentReference ?? []);
+ const docRefsWithMetriportId = filterDocRefsWithMetriportId(docRefs);
+ const docsToDownload = await getNonExistentDocRefs(docRefsWithMetriportId, patientId, cxId);
+
+ if (docsToDownload.length === 0) {
+ return;
+ }
+
+ respWithDocsToDownload.push({
+ ...gwResp,
+ documentReference: docsToDownload,
+ });
+ },
+ {
+ numberOfParallelExecutions: 20,
+ }
+ );
+
+ return respWithDocsToDownload;
+}
+
+async function getDocumentReferencesWithMetriportId({
+ cxId,
+ patientId,
+ requestId,
+ response,
+}: {
+ cxId: string;
+ patientId: string;
+ requestId: string;
+ response: OutboundDocumentQueryResp;
+}): Promise<OutboundDocumentQueryResp[]> {
+ const resultsWithMetriportId: OutboundDocumentQueryResp[] = [];
+
+ const docRefs = response.documentReference ?? [];
+
+ const docRefsWithMetriportId = await Promise.all(
+ docRefs.map(docRef =>
+ addMetriportDocRefID({
+ cxId,
+ patientId,
+ requestId,
+ document: docRef,
+ })
+ )
+ );
+
+ resultsWithMetriportId.push({
+ ...response,
+ documentReference: docRefsWithMetriportId,
+ });
+
+ return resultsWithMetriportId;
+}
+
+async function addMetriportDocRefID({
+ cxId,
+ patientId,
+ requestId,
+ document,
+}: {
+ patientId: string;
+ cxId: string;
+ requestId: string;
+ document: DocumentReference;
+}) {
+ const documentId = document.docUniqueId;
+
+ const { metriportId, originalId } = await mapDocRefToMetriport({
+ cxId,
+ patientId,
+ documentId,
+ requestId,
+ source: MedicalDataSource.CAREQUALITY,
+ });
+
+ return {
+ ...document,
+ docUniqueId: originalId,
+ metriportId,
+ };
+}
+
+async function replaceDqUrlWithDrUrl({
+ patientId,
+ requestId,
+ cxId,
+ responsesWithDocsToDownload,
+ log,
+}: {
+ patientId: string;
+ requestId: string;
+ cxId: string;
+ responsesWithDocsToDownload: OutboundDocumentQueryResp[];
+ log: typeof console.log;
+}): Promise<OutboundDocumentQueryResp[]> {
+ const resultsWithMetriportIdAndDrUrl: OutboundDocumentQueryResp[] = [];
+
+ await executeAsynchronously(
+ responsesWithDocsToDownload,
+ async outboundDocumentQueryResp => {
+ const gateway = await getCQDirectoryEntry(outboundDocumentQueryResp.gateway.homeCommunityId);
+
+ if (!gateway) {
+ const msg = `Gateway not found - Doc Retrieval`;
+ log(`${msg}: ${outboundDocumentQueryResp.gateway.homeCommunityId} skipping...`);
+ capture.message(msg, {
+ extra: {
+ context: `cq.dq.getCQDirectoryEntry`,
+ patientId,
+ requestId,
+ cxId,
+ gateway: outboundDocumentQueryResp.gateway,
+ },
+ });
+ return;
+ } else if (!gateway.urlDR) {
+ log(`Gateway ${gateway.id} has no DR URL, skipping...`);
+ return;
+ }
+
+ resultsWithMetriportIdAndDrUrl.push({
+ ...outboundDocumentQueryResp,
+ gateway: {
+ ...outboundDocumentQueryResp.gateway,
+ url: gateway.urlDR,
+ },
+ });
+ },
+ {
+ numberOfParallelExecutions: 20,
+ }
+ );
+
+ return resultsWithMetriportIdAndDrUrl;
+}
+
 async function storeInitDocRefInFHIR(
  docRefs: DocumentReferenceWithMetriportId[],
  cxId: string,
@@ -331,30 +442,3 @@ async function storeInitDocRefInFHIR(
  { numberOfParallelExecutions: parallelUpsertsToFhir }
  );
 }
-
-function addMetriportDocRefID({
- cxId,
- patientId,
- requestId,
-}: {
- patientId: string;
- cxId: string;
- requestId: string;
-}) {
- return async (document: DocumentReference): Promise<DocumentReferenceWithMetriportId> => {
- const documentId = document.docUniqueId;
-
- const { metriportId, originalId } = await mapDocRefToMetriport({
- cxId,
- patientId,
- documentId,
- requestId,
- source: MedicalDataSource.CAREQUALITY,
- });
- return {
- ...document,
- docUniqueId: originalId,
- metriportId,
- };
- };
-}
diff --git a/packages/infra/lib/api-stack.ts b/packages/infra/lib/api-stack.ts
@@ -251,12 +251,14 @@ export class APIStack extends Stack {
  bucketName: props.config.medicalDocumentsBucketName,
  publicReadAccess: false,
  encryption: s3.BucketEncryption.S3_MANAGED,
+ versioned: true,
  });
 
  const medicalDocumentsUploadBucket = new s3.Bucket(this, "APIMedicalDocumentsUploadBucket", {
  bucketName: props.config.medicalDocumentsUploadBucketName,
  publicReadAccess: false,
  encryption: s3.BucketEncryption.S3_MANAGED,
+ versioned: true,
  });
 
  const getSandboxSeedDataBucket = (sandboxConfig: EnvConfigSandbox) => {

diff --git a/packages/infra/lib/shared/backup.ts b/packages/infra/lib/shared/backup.ts
@@ -122,6 +122,7 @@ export class DailyBackup extends Construct {
 
  this.backupPlan.addSelection("BackupSelection", {
  resources: props.resources,
+ allowRestores: true,
  });
 
  // Outputs