Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvements to DQ and DR PRs #1528

Draft
wants to merge 4 commits into
base: develop
Choose a base branch
from
Draft

Improvements to DQ and DR PRs #1528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fdb3373
feat(cq): fixing nits on dq pr
jonahkaye Jan 26, 2024
ca03831
feat(cq): nits on dr pr
jonahkaye Jan 28, 2024
f7038d3
feat(cq): tested changes local
jonahkaye Jan 28, 2024
c305351
feat(cq): better naming a function and better log
jonahkaye Jan 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/api/src/external/carequality/document/process-document-query-results.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { S3Utils, createS3FileName } from "@metriport/core/external/aws/s3";
import { S3Utils } from "@metriport/core/external/aws/s3";
import { createS3FileName } from "@metriport/core/domain/utils";
import { out } from "@metriport/core/util/log";
import { errorToString } from "@metriport/core/util/error/index";
import { capture } from "@metriport/core/util/notifications";
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/routes/medical/document.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { S3Utils, createS3FileName } from "@metriport/core/external/aws/s3";
import { S3Utils } from "@metriport/core/external/aws/s3";
import { createS3FileName } from "@metriport/core/domain/utils";
import { UploadDocumentResult } from "@metriport/api-sdk";
import { uuidv7 } from "@metriport/core/util/uuid-v7";
import { Request, Response } from "express";
Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/routes/medical/internal-docs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { createS3FileName, S3Utils } from "@metriport/core/external/aws/s3";
import { S3Utils } from "@metriport/core/external/aws/s3";
import { createS3FileName } from "@metriport/core/domain/utils";
import { uuidv7 } from "@metriport/core/util/uuid-v7";
import { Request, Response } from "express";
import Router from "express-promise-router";
Expand Down
7 changes: 5 additions & 2 deletions packages/core/src/command/patient-loader-metriport-api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,17 @@ export class PatientLoaderMetriportAPI implements PatientLoader {
lastNameInitial: data?.lastNameInitial,
},
});
// call convertToDomainObject(response) here
if (!response.data) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

response.data is of type any so checking if undefined to prevent runtime errors

console.log(`No patients found for ${JSON.stringify(data)}`);
throw new Error();
}
const patients: Patient[] = response.data.map((patient: PatientDTO) =>
getDomainFromDTO(patient)
);
patients.forEach(validatePatient);
return patients;
} catch (error) {
console.log("Failing on request to internal endpoint", error);
console.log(`Failing on request to internal endpoint ${JSON.stringify(error)}`);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no multiline errors

throw error;
}
}
Expand Down
22 changes: 22 additions & 0 deletions packages/core/src/domain/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
export const createS3FileName = (cxId: string, patientId: string, fileName: string): string => {
return `${cxId}/${patientId}/${cxId}_${patientId}_${fileName}`;
};

export const parseS3FileName = (
fileKey: string
): { cxId: string; patientId: string; docId: string } | undefined => {
if (fileKey.includes("/")) {
const keyParts = fileKey.split("/");
const docName = keyParts[keyParts.length - 1];
if (docName) {
const docNameParts = docName.split("_");
const cxId = docNameParts[0];
const patientId = docNameParts[1];
const docId = docNameParts[2];
if (cxId && patientId && docId) {
return { cxId, patientId, docId };
}
}
}
return;
};
20 changes: 15 additions & 5 deletions packages/core/src/external/aws/lambda-logic/document-uploader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import axios from "axios";
import { MetriportError } from "../../../util/error/metriport-error";
import { S3Utils, parseS3FileName, buildDestinationKeyMetadata } from "../s3";
import { S3Utils } from "../s3";
import { buildDestinationKeyMetadata } from "../../carequality/dq/utils";
import { parseS3FileName } from "../../../domain/utils";
import { DocumentReference } from "@medplum/fhirtypes";
import { createExtrinsicObjectXml } from "../../carequality/dq/create-metadata-xml";
import {
Expand Down Expand Up @@ -59,11 +61,17 @@ export async function documentUploaderHandler(
}

// Get file info from the copied file
const { size, contentType, eTag } = await s3Utils.getFileInfoFromS3(
const { exists, size, contentType, eTag } = await s3Utils.getFileInfoFromS3(
destinationKey,
destinationBucket
);

if (!exists) {
const message = `Failed to get file info from the copied file for cxId: ${cxId}, patientId: ${patientId}, docId: ${docId}`;
console.log(message);
throw new MetriportError(message, null, { destinationBucket, destinationKey });
}

const fileData: FileData = {
mimeType: contentType,
size,
Expand All @@ -74,8 +82,8 @@ export async function documentUploaderHandler(

try {
const docRef = await forwardCallToServer(cxId, apiServerURL, fileData);
const stringSize = size ? size.toString() : "";
const hash = eTag ? eTag : "";
const stringSize = size.toString();
const hash = eTag;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed unnessecary conditionals

if (!docRef) {
const message = "Failed with the call to update the doc-ref of an uploaded file";
console.log(`${message}: ${docRef}`);
Expand Down Expand Up @@ -166,6 +174,8 @@ async function createAndUploadMetadataFile({
title,
});

console.log(`Uploading metadata to S3 with key: ${s3MetadataFileName}`);
console.log(
`Uploading metadata to S3 with key: ${s3MetadataFileName}, cxId: ${cxId}, patientId: ${patientId}`
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

more descriptive logging

);
await s3Utils.uploadFile(destinationBucket, s3MetadataFileName, Buffer.from(extrinsicObjectXml));
}
117 changes: 42 additions & 75 deletions packages/core/src/external/aws/s3.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,12 @@ import * as stream from "stream";
import { capture } from "../../util/notifications";

dayjs.extend(duration);
const UPLOADS_FOLDER = "uploads";
const DEFAULT_SIGNED_URL_DURATION = dayjs.duration({ minutes: 3 }).asSeconds();

export function makeS3Client(region: string): AWS.S3 {
return new AWS.S3({ signatureVersion: "v4", region });
}

export const createS3FileName = (cxId: string, patientId: string, fileName: string): string => {
return `${cxId}/${patientId}/${cxId}_${patientId}_${fileName}`;
};

export const parseS3FileName = (
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved these files to metriport business logic files instead of s3.ts

fileKey: string
): { cxId: string; patientId: string; docId: string } | undefined => {
if (fileKey.includes("/")) {
const keyParts = fileKey.split("/");
const docName = keyParts[keyParts.length - 1];
if (docName) {
const docNameParts = docName.split("_");
const cxId = docNameParts[0];
const patientId = docNameParts[1];
const docId = docNameParts[2];
if (cxId && patientId && docId) {
return { cxId, patientId, docId };
}
}
}
return;
};

/**
* @deprecated Use `S3Utils.getSignedUrl()` instead
*/
Expand Down Expand Up @@ -98,8 +74,8 @@ export class S3Utils {
key: string,
bucket: string
): Promise<
| { exists: true; size: number; contentType: string; eTag?: string }
| { exists: false; size?: never; contentType?: never; eTag?: never }
| { exists: true; size: number; contentType: string; eTag: string }
| { exists: false; size?: undefined; contentType?: undefined; eTag?: undefined }
> {
try {
const head = await this.s3
Expand All @@ -108,12 +84,17 @@ export class S3Utils {
Key: key,
})
.promise();
return {
exists: true,
size: head.ContentLength ?? 0,
contentType: head.ContentType ?? "",
eTag: head.ETag ?? "",
};

if (head.ContentLength && head.ContentType && head.ETag) {
return {
exists: true,
size: head.ContentLength,
contentType: head.ContentType,
eTag: head.ETag,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed unnessecary conditionals

};
} else {
throw new Error("Missing properties in HeadObjectOutput");
}
} catch (err) {
return { exists: false };
}
Expand Down Expand Up @@ -220,70 +201,56 @@ export class S3Utils {
key: string,
file: Buffer
): Promise<AWS.S3.ManagedUpload.SendData> {
return new Promise((resolve, reject) => {
this._s3.upload(
{
try {
const data = await this._s3
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

modified to use await/async

.upload({
Bucket: bucket,
Key: key,
Body: file,
},
(err, data) => {
if (err) {
console.error("Error during upload:", err);
reject(err);
} else {
console.log("Upload successful");
resolve(data);
}
}
);
});
})
.promise();

console.log("Upload successful");
return data;
} catch (err) {
console.error("Error during upload:", err);
throw err;
}
}
async retrieveDocumentIdsFromS3(
cxId: string,
patientId: string,
bucketName: string
): Promise<string[] | undefined> {
const Prefix = `${cxId}/${patientId}/uploads/`;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removing metriport business logic from retrieveDocumentsContentFromS3


async retrieveDocumentsContentFromS3(
bucketName: string,
prefix: string,
endsWith: string
): Promise<string[]> {
const params = {
Bucket: bucketName,
Prefix,
Prefix: prefix,
};

try {
const data = await this._s3.listObjectsV2(params).promise();
const documentContents = (
await Promise.all(
data.Contents?.filter(item => item.Key && item.Key.endsWith("_metadata.xml")).map(
async item => {
if (item.Key) {
const params = {
Bucket: bucketName,
Key: item.Key,
};
data.Contents?.filter(item => item.Key && item.Key.endsWith(endsWith)).map(async item => {
if (item.Key) {
const params = {
Bucket: bucketName,
Key: item.Key,
};

const data = await this._s3.getObject(params).promise();
return data.Body?.toString();
}
return undefined;
const data = await this._s3.getObject(params).promise();
return data.Body?.toString();
}
) || []
return undefined;
}) || []
)
).filter((item): item is string => Boolean(item));

return documentContents;
} catch (error) {
console.error(`Error retrieving document IDs from S3: ${error}`);
return undefined;
throw error;
}
}
}

export function buildDestinationKeyMetadata(
cxId: string,
patientId: string,
docId: string
): string {
return `${cxId}/${patientId}/${UPLOADS_FOLDER}/${cxId}_${patientId}_${docId}_metadata.xml`;
}
32 changes: 23 additions & 9 deletions packages/core/src/external/carequality/dq/create-metadata-xml.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ import {
DEFAULT_HEALTHCARE_FACILITY_TYPE_CODE_NODE,
DEFAULT_HEALTHCARE_FACILITY_TYPE_CODE_DISPLAY,
METRIPORT_HOME_COMMUNITY_ID,
ON_DEMAND_OBJECT_TYPE,
CLASS_CODE_CLASSIFICATION_SCHEME,
CONDIDENTIALITY_CODE_CLASSIFICATION_SCHEME,
FORMAT_CODE_CLASSIFICATION_SCHEME,
PRACTICE_SETTING_CODE_CLASSIFICATION_SCHEME,
HEALTHCARE_FACILITY_TYPE_CODE_CLASSIFICATION_SCHEME,
TYPE_CODE_CLASSIFICATION_SCHEME,
PATIENT_ID_CLASSIFICATION_SCHEME,
DOCUMENT_ENTRY_CLASSIFICATION_SCHEME,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removing all hardcoding

createDocumentUniqueId,
} from "../shared";
import { uuidv7 } from "../../../util/uuid-v7";
Expand Down Expand Up @@ -59,7 +68,12 @@ export function createExtrinsicObjectXml({
healthcareFacilityTypeCode?.text ||
DEFAULT_HEALTHCARE_FACILITY_TYPE_CODE_DISPLAY;

const metadataXml = `<ExtrinsicObject home="${METRIPORT_HOME_COMMUNITY_ID}" id="${documentUUID}" isOpaque="false" mimeType="text/xml" objectType="urn:uuid:34268e47-fdf5-41a6-ba33-82133c465248" status="urn:oasis:names:tc:ebxml-regrep:StatusType:Approved">
const objectTypeClassification =
"urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification";
const externalIdentifierClassification =
"urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:ExternalIdentifier";

const metadataXml = `<ExtrinsicObject home="${METRIPORT_HOME_COMMUNITY_ID}" id="${documentUUID}" isOpaque="false" mimeType="text/xml" objectType="${ON_DEMAND_OBJECT_TYPE}" status="urn:oasis:names:tc:ebxml-regrep:StatusType:Approved">

<Slot name="creationTime">
<ValueList>
Expand Down Expand Up @@ -101,7 +115,7 @@ export function createExtrinsicObjectXml({
<LocalizedString charset="UTF-8" value="${title}"/>
</Name>

<Classification classificationScheme="urn:uuid:41a5887f-8865-4c09-adf7-e362475b143a" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${classCodeNode}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${CLASS_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${classCodeNode}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${LOINC_CODE}</Value>
Expand All @@ -112,7 +126,7 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<Classification classificationScheme="urn:uuid:f4f85eac-e6cb-4883-b524-f2705394840f" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${DEFAULT_CONFIDENTIALITY_CODE}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${CONDIDENTIALITY_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${DEFAULT_CONFIDENTIALITY_CODE}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${CONFIDENTIALITY_CODE_SYSTEM}</Value>
Expand All @@ -123,7 +137,7 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<Classification classificationScheme="urn:uuid:a09d5840-386c-46f2-b5ad-9c3699a4309d" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${DEFAULT_FORMAT_CODE_NODE}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${FORMAT_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${DEFAULT_FORMAT_CODE_NODE}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${DEFAULT_FORMAT_CODE_SYSTEM}</Value>
Expand All @@ -134,7 +148,7 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<Classification classificationScheme="urn:uuid:cccf5598-8b07-4b77-a05e-ae952c785ead" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${practiceSettingCodeNode}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${PRACTICE_SETTING_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${practiceSettingCodeNode}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${SNOMED_CODE}</Value>
Expand All @@ -145,7 +159,7 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<Classification classificationScheme="urn:uuid:f0306f51-975f-434e-a61c-c59651d33983" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${classCodeNode}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${TYPE_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${classCodeNode}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${LOINC_CODE}</Value>
Expand All @@ -156,7 +170,7 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<Classification classificationScheme="urn:uuid:f33fb8ac-18af-42cc-ae0e-ed0b0bdb91e1" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${healthcareFacilityTypeCodeNode}" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:Classification">
<Classification classificationScheme="${HEALTHCARE_FACILITY_TYPE_CODE_CLASSIFICATION_SCHEME}" classifiedObject="${documentUUID}" id="${uuidv7()}" nodeRepresentation="${healthcareFacilityTypeCodeNode}" objectType="${objectTypeClassification}">
<Slot name="codingScheme">
<ValueList>
<Value>${SNOMED_CODE}</Value>
Expand All @@ -167,14 +181,14 @@ export function createExtrinsicObjectXml({
</Name>
</Classification>

<ExternalIdentifier id="${uuidv7()}" identificationScheme="urn:uuid:58a6f841-87b3-4a3e-92fd-a8ffeff98427" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:ExternalIdentifier" registryObject="${documentUUID}" value="${patientId}^^^&amp;${homeCommunityId}&amp;ISO">
<ExternalIdentifier id="${uuidv7()}" identificationScheme="${PATIENT_ID_CLASSIFICATION_SCHEME}" objectType=""${externalIdentifierClassification}"}" registryObject="${documentUUID}" value="${patientId}^^^&amp;${homeCommunityId}&amp;ISO">
<Name>
<LocalizedString charset="UTF-8" value="XDSDocumentEntry.patientId"/>
</Name>
</ExternalIdentifier>

<!-- (IHE) REQUIRED - DocumentEntry.uniqueId - Globally unique identifier assigned to the document by its creator -->
<ExternalIdentifier id="${uuidv7()}" identificationScheme="urn:uuid:2e82c1f6-a085-4c72-9da3-8640a32e42ab" objectType="urn:oasis:names:tc:ebxml-regrep:ObjectType:RegistryObject:ExternalIdentifier" registryObject="${documentUUID}" value="${createDocumentUniqueId(
<ExternalIdentifier id="${uuidv7()}" identificationScheme="${DOCUMENT_ENTRY_CLASSIFICATION_SCHEME}" objectType=""${externalIdentifierClassification}"}" registryObject="${documentUUID}" value="${createDocumentUniqueId(
documentUniqueId
)}">
<Name>
Expand Down
Loading
Loading