SO-2902: Add multi-language RF2 import support. (#194)

* [rf2] Added tests for multi-language rf2 importer. * [rf2] Changed description, languagerefset file property to collection. * [rf2] Changed logic so it can work with collections. * [rf2] Added test import rf2 archives * [rf2] Added statusCode check for the concept id, renamed variables to match what they represent, members ids instead of concept ids * [rf2] Changed data binding properties. * [rf2] Changed loops to streams moved adding description and language refset files logic outside of loop * [rf2] Merged the two test cases into one. * [rf2] Changed logic of reading and writing of description files and language refset files for net4j indications. * [rf2] Added final modifiers. * [rf2] Fixed test import file changed language code and refset id. * [rf2] Fixed typo. * [rf2] Fixed conceptId of refset in the import file. * [rf2] Added placeholder properties for databinding. * [rf2] Added validation for definition text files, removed extra lines. * [rf2] Added proper monitor progress. Added logic to write textDefinitionFiles * [rf2] Added method to read collection of files and to monitor progress. * [rf2] Removed extra line. * [rf2] Removed extra line. Added logic to handle multiple text definition files. * [rf2] Removed unneeded iteration. * [rf2] Added logic to handle collection of text definition files. * [rf2] Removed extra line. * SO-2902: Simplify test case https://snowowl.atlassian.net/browse/SO-2902 * [snomed.import] Refactor ImportConfiguration - remove unnecessary fields from import config - refactored SnomedRefSetNameCollector: - do not parse description files for each refset URL, parse them only once - "prepare" resolvable refset labels for the client - do not send language refset files twice over the wire * [snomed.import] Fix cloning of stated relationship file * SO-2902: Update license header * SO-2902: Use Pattern.quote instead of Charmatcher.replace * SO-2902: Use try-with-resource for description readers * SO-2902: Fix NPE
b2ihealthcare · Mar 8, 2018 · b7e0973 · b7e0973
1 parent fc288db
commit b7e0973
Show file tree

Hide file tree

Showing 17 changed files with 465 additions and 553 deletions.
diff --git a/...d/api/rest/io/SnomedCT_Release_INT_20150201_descriptions_with_multiple_language_codes.zip b/...d/api/rest/io/SnomedCT_Release_INT_20150201_descriptions_with_multiple_language_codes.zip
diff --git a/...pi.rest.tests/src/com/b2international/snowowl/snomed/api/rest/io/SnomedImportApiTest.java b/...pi.rest.tests/src/com/b2international/snowowl/snomed/api/rest/io/SnomedImportApiTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2011-2017 B2i Healthcare Pte Ltd, http:https://b2i.sg
+ * Copyright 2011-2018 B2i Healthcare Pte Ltd, http:https://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -218,7 +218,7 @@ public void import11ExtensionConceptWithVersion() {
  getComponent(branchPath, SnomedComponentType.CONCEPT, "555231000005107").statusCode(200);
  getVersion("SNOMEDCT-NE", "2015-02-05").statusCode(200);
  }
-
+ 
  @Test
  public void import12OnlyPubContentWithVersioning() {
  validateBranchHeadtimestampUpdate(branchPath,
@@ -299,6 +299,24 @@ public void import24IncompleteTaxonomyMustBeImported() {
  getComponent(branchPath, SnomedComponentType.CONCEPT, "882169191000154107").statusCode(200);
  getComponent(branchPath, SnomedComponentType.RELATIONSHIP, "955630781000154129").statusCode(200);
  }
+
+ @Test
+ public void import25WithMultipleLanguageCodes() {
+ final String enDescriptionId = "41320138114";
+ final String svDescriptionId = "24688171113";
+ final String enLanguageRefsetMemberId = "34d07985-48a0-41e7-b6ec-b28e6b00adfc";
+ final String svLanguageRefsetMemberId = "34d07985-48a0-41e7-b6ec-b28e6b00adfb";
+
+ getComponent(branchPath, SnomedComponentType.DESCRIPTION, enDescriptionId).statusCode(404);
+ getComponent(branchPath, SnomedComponentType.DESCRIPTION, svDescriptionId).statusCode(404);
+ getComponent(branchPath, SnomedComponentType.MEMBER, enLanguageRefsetMemberId).statusCode(404);
+ getComponent(branchPath, SnomedComponentType.MEMBER, svLanguageRefsetMemberId).statusCode(404);
+ importArchive("SnomedCT_Release_INT_20150201_descriptions_with_multiple_language_codes.zip");
+ getComponent(branchPath, SnomedComponentType.DESCRIPTION, enDescriptionId).statusCode(200);
+ getComponent(branchPath, SnomedComponentType.DESCRIPTION, svDescriptionId).statusCode(200);
+ getComponent(branchPath, SnomedComponentType.MEMBER, enLanguageRefsetMemberId).statusCode(200);
+ getComponent(branchPath, SnomedComponentType.MEMBER, svLanguageRefsetMemberId).statusCode(200);
+ }
 
  private void validateBranchHeadtimestampUpdate(IBranchPath branch, String importArchiveFileName,
  boolean createVersions) {

diff --git a/....rf2/src/com/b2international/snowowl/snomed/importer/rf2/command/ImportRefSetCommand.java b/....rf2/src/com/b2international/snowowl/snomed/importer/rf2/command/ImportRefSetCommand.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2011-2016 B2i Healthcare Pte Ltd, http:https://b2i.sg
+ * Copyright 2011-2018 B2i Healthcare Pte Ltd, http:https://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -87,7 +87,7 @@ public void execute(final CommandInterpreter interpreter) {
  }
 
  try {
- configuration.addRefSetSource(refSetFile.toURI().toURL());
+ configuration.addRefSetURL(refSetFile.toURI().toURL());
  } catch (final MalformedURLException e) {
  interpreter.println(e);
  }
@@ -124,7 +124,7 @@ public void execute(final CommandInterpreter interpreter) {
  return;
  }
 
- configuration.setVersion(contentSubType);
+ configuration.setContentSubType(contentSubType);
  configuration.setSourceKind(ImportSourceKind.FILES);
 
  if ("-x".equals(arg)) {

diff --git a/...c/com/b2international/snowowl/snomed/importer/rf2/command/ListLanguageRefSetsCommand.java b/...c/com/b2international/snowowl/snomed/importer/rf2/command/ListLanguageRefSetsCommand.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2011-2016 B2i Healthcare Pte Ltd, http:https://b2i.sg
+ * Copyright 2011-2018 B2i Healthcare Pte Ltd, http:https://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,13 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Collection;
+import java.net.URL;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.osgi.framework.console.CommandInterpreter;
@@ -41,9 +43,8 @@
 public class ListLanguageRefSetsCommand extends AbstractRf2ImporterCommand {
 
  public ListLanguageRefSetsCommand() {
- super("rf2_languages", "<path>", "Lists all available language type reference set identifiers in a release archive", new String[] {
- "<path>\t\tSpecifies the release archive to scan."
- });
+ super("rf2_languages", "<path>", "Lists all available language type reference set identifiers in a release archive",
+ new String[] { "<path>\t\tSpecifies the release archive to scan." });
  }
 
  @Override
@@ -93,52 +94,58 @@ public void execute(final CommandInterpreter interpreter) {
  return;
  }
 
- final Collection<String> languageRefSetFileNames = archiveFileSet.getAllFileName(zipFiles, ReleaseComponentType.LANGUAGE_REFERENCE_SET, contentSubType);
+ final Map<String, String> languageRefsetToLabelMap = Maps.newHashMap();
+ final ImportConfiguration config = new ImportConfiguration(Branch.MAIN_PATH);
 
- final Map<String, String> $ = Maps.newHashMap();
+ config.setArchiveFile(archiveFile);
 
- for (final String languageRefSetFileName : languageRefSetFileNames) {
+ final Set<File> languageRefSetFiles = archiveFileSet
+ .getAllFileName(zipFiles, ReleaseComponentType.LANGUAGE_REFERENCE_SET, contentSubType)
+ .stream()
+ .map(fileName -> new File(fileName))
+ .collect(Collectors.toSet());
 
- final File languageRefSetFile = new File(languageRefSetFileName);
- interpreter.println("Searching for language type reference sets in '" + languageRefSetFile.getName() + "'...");
-
- final ImportConfiguration config = new ImportConfiguration(Branch.MAIN_PATH);
-
- // Setting up configuration only with the required fields
- config.setSourceKind(ImportSourceKind.ARCHIVE);
- config.setArchiveFile(archiveFile);
- config.setDescriptionsFile(new File(archiveFileSet.getFileName(zipFiles, ReleaseComponentType.DESCRIPTION, contentSubType)));
- config.setLanguageRefSetFile(languageRefSetFile);
-
- final SnomedRefSetNameCollector provider = new SnomedRefSetNameCollector(config, new NullProgressMonitor(), "");
-
- try {
- provider.parse(config.toURL(config.getLanguageRefSetFile()));
- } catch (final IOException e) {
- interpreter.println(e);
- return;
- }
+ final Set<File> descriptionFiles = archiveFileSet
+ .getAllFileName(zipFiles, ReleaseComponentType.DESCRIPTION, contentSubType)
+ .stream()
+ .map(fileName -> new File(fileName))
+ .collect(Collectors.toSet());
 
- for (final Entry<String, String> label : provider.getAvailableLabels().entrySet()) {
- $.put(label.getKey(), label.getValue());
+ descriptionFiles.forEach(file -> config.addDescriptionFile(file));
+
+ Set<URL> refsetUrls = languageRefSetFiles.stream().map(file -> {
+ try {
+ return config.toURL(file);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
  }
-
+ }).collect(Collectors.toSet());
+
+ final SnomedRefSetNameCollector provider = new SnomedRefSetNameCollector(refsetUrls, config, new NullProgressMonitor());
+ provider.parse();
+
+ // Setting up configuration only with the required fields
+ config.setSourceKind(ImportSourceKind.ARCHIVE);
+ config.setArchiveFile(archiveFile);
+
+ for (final Entry<String, String> label : provider.getRefsetIdToLabelMap().entrySet()) {
+ languageRefsetToLabelMap.put(label.getKey(), label.getValue());
  }
 
- if ($.isEmpty()) {
+ if (languageRefsetToLabelMap.isEmpty()) {
  interpreter.println("No language reference sets could be found in release archive.");
  return;
  }
 
  interpreter.println("\n---------------------------------------------------------------------\n");
 
- for (final Entry<String, String> label : $.entrySet()) {
+ for (final Entry<String, String> label : languageRefsetToLabelMap.entrySet()) {
  final StringBuilder sb = new StringBuilder();
  sb.append("\t");
  sb.append(label.getKey());
- sb.append("|");
+ sb.append(" | ");
  sb.append(label.getValue().trim());
- sb.append("|");
+ sb.append(" | ");
  interpreter.println(sb.toString());
  }
 

diff --git a/...rf2/src/com/b2international/snowowl/snomed/importer/rf2/net4j/SnomedImportIndication.java b/...rf2/src/com/b2international/snowowl/snomed/importer/rf2/net4j/SnomedImportIndication.java
@@ -19,6 +19,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.util.Set;
 
 import org.eclipse.net4j.signal.IndicationWithMonitoring;
 import org.eclipse.net4j.util.io.ExtendedDataInputStream;
@@ -36,6 +37,7 @@
 import com.b2international.snowowl.snomed.importer.net4j.SnomedImportResult;
 import com.b2international.snowowl.snomed.importer.net4j.SnomedValidationDefect;
 import com.b2international.snowowl.snomed.importer.rf2.util.ImportUtil;
+import com.google.common.collect.Sets;
 import com.google.common.io.Files;
 
 /**
@@ -64,16 +66,17 @@ protected int getIndicatingWorkPercent() {
  @Override
  protected void indicating(final ExtendedDataInputStream in, final OMMonitor monitor) throws Exception {
 
- monitor.begin(1 + 7 + 1);
+ monitor.begin(1 + 5 + 1);
  OMMonitor refSetSubmonitor = null;
 
- final ImportConfiguration importConfiguration = new ImportConfiguration(in.readUTF());
+ final ImportConfiguration importConfiguration = new ImportConfiguration(in.readUTF()); // branchPath
 
  try {
  // XXX: source kind is always FILES, since the server just receives a bunch of them
  importConfiguration.setSourceKind(ImportSourceKind.FILES);
+
  userId = in.readString();
- importConfiguration.setVersion(in.readEnum(ContentSubType.class));
+ importConfiguration.setContentSubType(in.readEnum(ContentSubType.class));
  importConfiguration.setCreateVersions(in.readBoolean());
 
  final int exludedRefSetIdCount = in.readInt();
@@ -85,27 +88,30 @@ protected void indicating(final ExtendedDataInputStream in, final OMMonitor moni
  String codeSystemShortName = in.readUTF();
  importConfiguration.setCodeSystemShortName(codeSystemShortName);
 
- monitor.worked();
+ monitor.worked(); // 1
 
  receivedFilesDirectory = Files.createTempDir();
  receivedFilesDirectory.deleteOnExit();
 
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setConceptsFile(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setDescriptionsFile(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setTextDefinitionFile(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setRelationshipsFile(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setStatedRelationshipsFile(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setDescriptionType(f); }}, monitor.fork());
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { importConfiguration.setLanguageRefSetFile(f); }}, monitor.fork());
+ int descriptionFilesSize = in.readInt();
+
+ readComponents(in, monitor, importConfiguration, descriptionFilesSize).forEach(file -> importConfiguration.addDescriptionFile(file));
+
+ int textDefinitionFilesSize = in.readInt();
+
+ readComponents(in, monitor, importConfiguration, textDefinitionFilesSize).forEach(file -> importConfiguration.addTextDefinitionFile(file));
+
+ readComponent(in, importConfiguration, receivedFilesDirectory, f -> importConfiguration.setConceptFile(f), monitor.fork());
+ readComponent(in, importConfiguration, receivedFilesDirectory, f -> importConfiguration.setRelationshipFile(f), monitor.fork());
+ readComponent(in, importConfiguration, receivedFilesDirectory, f -> importConfiguration.setStatedRelationshipFile(f), monitor.fork());
 
  final int refSetUrlCount = in.readInt();
 
  refSetSubmonitor = monitor.fork();
  refSetSubmonitor.begin(refSetUrlCount);
 
  for (int i = 0; i < refSetUrlCount; i++) {
- // XXX: assume that for the pre-determined number of additional refsets, a boolean value of "true" will always be sent 
- readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() { @Override public void setFile(final File f) { addRefSetUrl(importConfiguration, f); }}, refSetSubmonitor.fork());
+ readComponent(in, importConfiguration, receivedFilesDirectory, f -> addRefSetUrl(importConfiguration, f), refSetSubmonitor.fork());
  }
 
  this.configuration = importConfiguration;
@@ -120,9 +126,25 @@ protected void indicating(final ExtendedDataInputStream in, final OMMonitor moni
  }
  }
 
+ private Set<File> readComponents(final ExtendedDataInputStream in, final OMMonitor monitor, final ImportConfiguration importConfiguration, int numberOfFiles) throws IOException {
+ OMMonitor subMonitor = monitor.fork();
+ subMonitor.begin();
+ Set<File> releaseFiles = Sets.newHashSet();
+ for (int i = 0; i < numberOfFiles; i++) {
+ readComponent(in, importConfiguration, receivedFilesDirectory, new FileCallback() {
+ @Override
+ public void setFile(File f) {
+ releaseFiles.add(f);
+ }
+ }, subMonitor.fork());
+ }
+ subMonitor.done();
+ return releaseFiles;
+ }
+
  private void addRefSetUrl(final ImportConfiguration importConfiguration, final File f) {
  try {
- importConfiguration.addRefSetSource(f.toURI().toURL());
+ importConfiguration.addRefSetURL(f.toURI().toURL());
  } catch (final MalformedURLException e) {
  throw new IORuntimeException(e);
  }