feat(snomed): simplified filtering, conditions resource id mapping

Refs: #1442

packages/fhir-converter/src/templates/cda/Sections/Hospital_Discharge_Medications.hbs

@@ -5,7 +5,7 @@
  {{>Resources/Medication.hbs medication=medAdm.substanceAdministration.consumable.manufacturedProduct.manufacturedMaterial ID=(generateUUID (toJsonString medAdm.substanceAdministration.consumable.manufacturedProduct.manufacturedMaterial))}},
  {{>References/MedicationAdministration/medicationReference.hbs ID=(generateUUID (toJsonString medAdm.substanceAdministration)) REF=(concat 'Medication/' (generateUUID (toJsonString medAdm.substanceAdministration.consumable.manufacturedProduct.manufacturedMaterial)))}},
 
- {{#with (evaluate 'Utils/GenerateOrganizationId.hbs' obj=edAdm.substanceAdministration.performer.assignedEntity.representedOrganization) as |orgId|}}
+ {{#with (evaluate 'Utils/GenerateOrganizationId.hbs' obj=medAdm.substanceAdministration.performer.assignedEntity.representedOrganization) as |orgId|}}
  {{>Resources/Organization.hbs org=medAdm.substanceAdministration.performer.assignedEntity.representedOrganization ID=orgId.Id}},
  {{>References/MedicationAdministration/performer.actor.hbs ID=(generateUUID (toJsonString medAdm.substanceAdministration)) REF=(concat 'Organization/' orgId.Id)}},
  {{/with}}

packages/fhir-converter/src/templates/cda/Sections/Medication.hbs

@@ -42,7 +42,7 @@
  {{/with}}
  {{/if}}
  {{/if}}
-
+ 
  {{/each}}
  {{/if}}
 

packages/utils/src/customer-requests/medical-records-and-csvs.ts

@@ -0,0 +1,42 @@
+import { bundleToHtml } from "@metriport/core/external/aws/lambda-logic/bundle-to-html";
+import fs from "fs";
+import { convertHtmlTablesToCsv } from "./convert-html-to-csv";
+import * as path from "path";
+
+// Function to process each JSON file
+function processFile(filePath: string) {
+ const bundle = fs.readFileSync(filePath, "utf8");
+ const bundleParsed = JSON.parse(bundle);
+
+ // Convert JSON to HTML
+ const html = bundleToHtml(bundleParsed);
+ const htmlOutputFilePath = filePath.replace(".json", ".html");
+ fs.writeFileSync(htmlOutputFilePath, html);
+ console.log(`HTML file created at ${htmlOutputFilePath}`);
+
+ // Convert HTML to CSV
+ const csvContent = convertHtmlTablesToCsv(html);
+ const csvOutputFilePath = filePath.replace(".json", ".csv");
+ fs.writeFileSync(csvOutputFilePath, csvContent);
+ console.log(`CSV file created at ${csvOutputFilePath}`);
+}
+
+// Main function to iterate through the directory
+function main() {
+ const targetPath = process.argv[2];
+
+ if (!targetPath) {
+ console.log("Usage: node medical-records-local.js <path-to-directory>");
+ process.exit(1);
+ }
+
+ const files = fs.readdirSync(targetPath);
+ files.forEach(file => {
+ const fullPath = path.join(targetPath, file);
+ if (path.extname(fullPath) === ".json") {
+ processFile(fullPath);
+ }
+ });
+}
+
+main();

packages/utils/src/terminology-server/snomed-dedup-and-filter.ts

@@ -34,7 +34,8 @@ async function processDirectoryOrFile(
 
 async function computeHashTable(
  filePath: string,
- hashTable: Record<string, SnomedHierarchyTableEntry>
+ hashTable: Record<string, SnomedHierarchyTableEntry>,
+ conditionIdsDictionary: Record<string, Set<string>>
 ) {
  const data = JSON.parse(fs.readFileSync(filePath, "utf8"));
  const entries = data.bundle ? data.bundle.entry : data.entry;
@@ -47,7 +48,13 @@ async function computeHashTable(
  if (coding.system === "https://snomed.info/sct") {
  const codeDetails = await getCodeDetailsFull(coding.code, "SNOMEDCT_US");
  if (codeDetails) {
- await populateHashTableFromCodeDetails(hashTable, codeDetails, coding.code);
+ await populateHashTableFromCodeDetails(
+ hashTable,
+ conditionIdsDictionary,
+ codeDetails,
+ coding.code,
+ resource.id
+ );
  }
  }
  }
@@ -58,9 +65,10 @@ async function computeHashTable(
 async function processFileEntries(
  filePath: string,
  hashTable: Record<string, SnomedHierarchyTableEntry>,
- removalStats: RemovalStats,
- rxNormSet: Set<string>,
- cptSet: Set<string>
+ cptSet: Set<string>,
+ medicationConditionDict: Record<string, string>,
+ allRemainingEnries: Set<string>,
+ removalStats: RemovalStats
 ) {
  const data = JSON.parse(fs.readFileSync(filePath, "utf8"));
  const entries = data.bundle ? data.bundle.entry : data.entry;
@@ -86,19 +94,18 @@ async function processFileEntries(
  entries.splice(i, 1);
  break;
  } else {
- hashTable[coding.code].inserted = true;
  const codeDetails = await getCodeDisplay(coding.code, "SNOMEDCT_US");
- if (codeDetails) {
- if (codeDetails.category !== "disorder") {
-  removalStats.nonDisorderCodes.count += 1;
-  removalStats.nonDisorderCodes.codes.add(coding.code);
-  entries.splice(i, 1);
-  break;
-  } else {
-  const updatedText = `${codeDetails.display} (${codeDetails.category})`;
-  resource.code.text = updatedText;
-  coding.text = updatedText;
- }
+ if (codeDetails && codeDetails.category === "disorder") {
+ hashTable[coding.code].inserted = true;
+ const updatedText = `${codeDetails.display} (${codeDetails.category})`;
+ resource.code.text = updatedText;
+ coding.text = updatedText;
+ allRemainingEnries.add(resource.id);
+ } else {
+ removalStats.nonDisorderCodes.count += 1;
+ removalStats.nonDisorderCodes.codes.add(coding.code);
+ entries.splice(i, 1);
+ break;
  }
  }
  }
@@ -122,6 +129,7 @@ async function processFileEntries(
  entries.splice(i, 1);
  } else {
  cptSet.add(coding.code);
+ allRemainingEnries.add(resource.id);
  }
  break;
  } else {
@@ -136,19 +144,58 @@ async function processFileEntries(
  removalStats.entriesWithoutCptCodes.count += 1;
  entries.splice(i, 1);
  }
- } else if (resource && resource.resourceType === "Medication") {
- const codings = resource.code?.coding || [];
- for (const coding of codings) {
- if (coding.system === "https://www.nlm.nih.gov/research/umls/rxnorm") {
- if (rxNormSet.has(coding.code)) {
- entries.splice(i, 1);
- removalStats.rxnormDuplicates.count += 1;
- removalStats.rxnormDuplicates.codes.add(coding.code);
- break;
- } else {
- rxNormSet.add(coding.code);
+ } else if (
+ resource &&
+ resource.resourceType === "MedicationAdministration" &&
+ resource.reasonReference
+ ) {
+ const medicationRef = resource.medicationReference?.reference.split("/")[1];
+ const conditionRef = resource.reasonReference[0]?.reference.split("/")[1];
+
+ if (medicationRef && conditionRef) {
+ medicationConditionDict[medicationRef] = conditionRef;
+ allRemainingEnries.add(resource.id);
+ } else {
+ console.log("Removing MedicationAdministration entry without condition reference");
+ entries.splice(i, 1);
+ }
+ }
+ }
+ fs.writeFileSync(filePath, JSON.stringify(data, null, 2), "utf8");
+}
+
+async function filterMedicationsEntries(
+ filePath: string,
+ allRemainingEnries: Set<string>,
+ medicationConditionDict: Record<string, string>,
+ rxNormSet: Set<string>,
+ removalStats: RemovalStats
+) {
+ const data = JSON.parse(fs.readFileSync(filePath, "utf8"));
+ const entries = data.bundle ? data.bundle.entry : data.entry;
+
+ for (let i = entries.length - 1; i >= 0; i--) {
+ const entry = entries[i];
+ const resource = entry.resource;
+ if (resource && resource.resourceType === "Medication") {
+ const conditionRef = medicationConditionDict[resource.id];
+ if (allRemainingEnries.has(conditionRef)) {
+ const codings = resource.code?.coding || [];
+ for (const coding of codings) {
+ if (coding.system === "https://www.nlm.nih.gov/research/umls/rxnorm") {
+ if (rxNormSet.has(coding.code)) {
+ removalStats.rxnormDuplicates.count += 1;
+ removalStats.rxnormDuplicates.codes.add(coding.code);
+ entries.splice(i, 1);
+ } else {
+ rxNormSet.add(coding.code);
+ allRemainingEnries.add(resource.id);
+ }
  }
  }
+ } else {
+ removalStats.nonConditionLinkedMedications.count += 1;
+ entries.splice(i, 1);
  }
  }
  }
@@ -162,18 +209,45 @@ async function main() {
  process.exit(1);
  }
  const hashTable: Record<string, SnomedHierarchyTableEntry> = {};
+ const conditionIdsDictionary: Record<string, Set<string>> = {};
 
  await processDirectoryOrFile(directoryPath, async filePath => {
- await computeHashTable(filePath, hashTable);
+ await computeHashTable(filePath, hashTable, conditionIdsDictionary);
  });
 
+ const filteredConditionIdsDictionary = Object.fromEntries(
+ Object.entries(conditionIdsDictionary).filter(([, value]) => value.size > 0)
+ );
+ console.log("conditionIdsDictionary", filteredConditionIdsDictionary);
+
  prettyPrintHashTable(hashTable);
  const removalStats = createInitialRemovalStats();
 
- const rxNormSet = new Set<string>();
  const cptSet = new Set<string>();
+ const medicationConditionDict: Record<string, string> = {};
+
+ const allRemainingEnries = new Set<string>();
+ await processDirectoryOrFile(directoryPath, async filePath => {
+ await processFileEntries(
+ filePath,
+ hashTable,
+ cptSet,
+ medicationConditionDict,
+ allRemainingEnries,
+ removalStats
+ );
+ });
+
+ const rxNormSet = new Set<string>();
+
  await processDirectoryOrFile(directoryPath, async filePath => {
- await processFileEntries(filePath, hashTable, removalStats, rxNormSet, cptSet);
+ await filterMedicationsEntries(
+ filePath,
+ allRemainingEnries,
+ medicationConditionDict,
+ rxNormSet,
+ removalStats
+ );
  });
 
  prettyPrintRemovalStats(removalStats);

packages/utils/src/terminology-server/snomed-heirarchies.ts

@@ -2,85 +2,79 @@ import { CodeDetailsResponse } from "./term-server-api";
 
 export type SnomedHierarchyTableEntry = {
  found: boolean;
- children: Set<string>;
  parents: Set<string>;
  root: boolean;
  inserted: boolean;
+ resourceId?: string;
 };
 
 export async function populateHashTableFromCodeDetails(
  hashTable: Record<string, SnomedHierarchyTableEntry>,
+ conditionIdsDictionary: Record<string, Set<string>>,
  codeDetails: CodeDetailsResponse,
- queriedCode: string
+ queriedCode: string,
+ resourceId: string
 ): Promise<Record<string, SnomedHierarchyTableEntry>> {
- if (hashTable[queriedCode] && !hashTable[queriedCode].found) {
- // if we found a parent node, make it the root and its children non-root
- if (hashTable[queriedCode].parents.size == 0) {
- console.log(
- `Updating root node to ${queriedCode}, hashTable[queriedCode].children:`,
- hashTable[queriedCode].children
- );
- hashTable[queriedCode] = { ...hashTable[queriedCode], found: true, root: true };
- for (const child of hashTable[queriedCode].children) {
- hashTable[child].root = false;
- }
- }
- // if we found a child node, do nothing since it already has a parent which is found and higher
- else {
- console.log(
- `Updating root node for ${queriedCode}, hashTable[queriedCode].parents:`,
- hashTable[queriedCode].parents
- );
- hashTable[queriedCode] = { ...hashTable[queriedCode], found: true, root: false };
- }
- } else {
+ if (!hashTable[queriedCode]) {
  hashTable[queriedCode] = {
  found: true,
  parents: new Set(),
- children: new Set(),
  root: true,
  inserted: false,
+ resourceId: resourceId,
  };
- codeDetails.parameter.forEach(param => {
- if (param.name === "property") {
- const valuePart = param.part.find(part => part.name === "value");
- const value = valuePart?.valueCode || valuePart?.value;
+ // set a node that was previously an unfound child to be found
+ } else {
+ console.log(`Downgrading ${queriedCode} to non-root (current)`);
+ hashTable[queriedCode] = { ...hashTable[queriedCode], found: true, resourceId: resourceId };
+ // children Ids point to parent
+ conditionIdsDictionary[resourceId] = new Set(
+ Array.from(hashTable[queriedCode].parents)
+ .map(parentCode => hashTable[parentCode]?.resourceId)
+ .filter((id): id is string => id !== undefined)
+ );
+ }
+ codeDetails.parameter.forEach(param => {
+ if (param.name === "property") {
+ const valuePart = param.part.find(part => part.name === "value");
+ const value = valuePart?.valueCode || valuePart?.value;
 
-  const codePart = param.part.find(part => part.name === "code");
-  const code = codePart?.valueCode || codePart?.value;
+ const codePart = param.part.find(part => part.name === "code");
+ const code = codePart?.valueCode || codePart?.value;
 
- // parents
- if (value && code === "parent") {
- if (!hashTable[value]) {
- hashTable[value] = {
- found: false,
- parents: new Set(),
- children: new Set(),
- root: false,
- inserted: false,
- };
- hashTable[queriedCode].parents.add(value);
- // having children indicates your a parent node
- hashTable[value].children.add(queriedCode);
- }
- }
- // children
- else if (value && !code) {
- if (!hashTable[value]) {
- hashTable[value] = {
- found: false,
- parents: new Set(),
- children: new Set(),
- root: false,
- inserted: false,
- };
- hashTable[queriedCode].children.add(value);
- // having parents indicates your a chilld node
- hashTable[value].parents.add(queriedCode);
+ // downgrade all children
+ if (value && !code) {
+ if (!hashTable[value]) {
+ hashTable[value] = {
+ found: false,
+ parents: new Set(),
+ root: false,
+ inserted: false,
+ };
+ } else {
+ hashTable[value] = { ...hashTable[value], root: false };
+ const resourceId = hashTable[value].resourceId;
+ if (resourceId) {
+ console.log(`Downgrading ${value} to non-root (existing)`);
+ if (!conditionIdsDictionary[resourceId]) {
+ const queriedResourceId = hashTable[queriedCode].resourceId;
+ if (queriedResourceId) {
+ conditionIdsDictionary[resourceId] = new Set([queriedResourceId]);
+ } else {
+ conditionIdsDictionary[resourceId] = new Set();
+ }
+ } else {
+ const queriedResourceId = hashTable[queriedCode].resourceId;
+ if (queriedResourceId) {
+ conditionIdsDictionary[resourceId].add(queriedResourceId);
+ }
+ }
  }
  }
+ hashTable[value].parents.add(queriedCode);
  }
- });
- }
+ }
+ });
+
  return hashTable;
 }