Optimize algorithm for combining two data sets #64 (#82)

* Optimize algorithm for combining two data sets #64
cuba-platform · Nov 14, 2018 · 998fef3 · 998fef3
1 parent cd25bd3
commit 998fef3
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 15 deletions.
diff --git a/...re/src/com/haulmont/yarg/reporting/extraction/controller/DefaultExtractionController.java b/...re/src/com/haulmont/yarg/reporting/extraction/controller/DefaultExtractionController.java
@@ -156,27 +156,30 @@ protected List<Map<String, Object>> getQueriesResult(Iterator<ReportQuery> query
  List<Map<String, Object>> currentQueryData = getQueryData(context, reportQuery);
  String link = reportQuery.getLinkParameterName();
  if (StringUtils.isNotBlank(link)) {
+ Map<Object, Map<String, Object>> cacheMap = new HashMap<>();
+ for (Map<String, Object> resultRow : result) {
+ if (Thread.interrupted()) {
+ throw new ReportingInterruptedException("Data extraction interrupted");
+ }
+ Object linkObj = resultRow.get(link);
+ if (linkObj != null) {
+ cacheMap.putIfAbsent(linkObj, resultRow);
+ } else {
+ throw new DataLoadingException(String.format("An error occurred while loading data for band [%s]." +
+ " Query defines link parameter [%s] but result does not contain such field. Query [%s].",
+ context.getBand().getName(), link, firstReportQuery.getName()));
+ }
+ }
+
  for (Map<String, Object> currentRow : currentQueryData) {
  if (Thread.interrupted()) {
  throw new ReportingInterruptedException("Data extraction interrupted");
  }
  Object linkObj = currentRow.get(link);
  if (linkObj != null) {
- for (Map<String, Object> resultRow : result) {
- if (Thread.interrupted()) {
- throw new ReportingInterruptedException("Data extraction interrupted");
- }
- Object linkObj2 = resultRow.get(link);
- if (linkObj2 != null) {
- if (linkObj.equals(linkObj2)) {
- resultRow.putAll(currentRow);
- break;
- }
- } else {
- throw new DataLoadingException(String.format("An error occurred while loading data for band [%s]." +
- " Query defines link parameter [%s] but result does not contain such field. Query [%s].",
- context.getBand().getName(), link, firstReportQuery.getName()));
- }
+ Map<String, Object> resultRow = cacheMap.get(linkObj);
+ if (resultRow != null) {
+ resultRow.putAll(currentRow);
  }
  } else {
  throw new DataLoadingException(String.format("An error occurred while loading data for band [%s]." +

diff --git a/core/modules/core/test/extraction/controller/DefaultControllerTest.java b/core/modules/core/test/extraction/controller/DefaultControllerTest.java
@@ -22,11 +22,15 @@
 import com.haulmont.yarg.loaders.impl.GroovyDataLoader;
 import com.haulmont.yarg.loaders.impl.JsonDataLoader;
 import com.haulmont.yarg.loaders.impl.SqlDataLoader;
+import com.haulmont.yarg.reporting.DataExtractorImpl;
 import com.haulmont.yarg.reporting.extraction.DefaultExtractionContextFactory;
 import com.haulmont.yarg.reporting.extraction.DefaultExtractionControllerFactory;
 import com.haulmont.yarg.reporting.extraction.ExtractionContextFactory;
 import com.haulmont.yarg.structure.BandData;
+import com.haulmont.yarg.structure.Report;
 import com.haulmont.yarg.structure.ReportBand;
+import com.haulmont.yarg.structure.impl.BandBuilder;
+import com.haulmont.yarg.structure.impl.ReportBuilder;
 import com.haulmont.yarg.util.groovy.DefaultScriptingImpl;
 import org.apache.commons.lang.StringUtils;
 import org.junit.AfterClass;
@@ -162,4 +166,51 @@ public void testJsonExtractionForBand() throws IOException, URISyntaxException {
  checkMasterData(reportBandMap.get("master_data"), 2, 2,
  "id", "name", "value", "user_id");
  }
+
+ @Test
+ public void stressTest() throws IOException, URISyntaxException {
+ int queries = 100;
+ int recordsPerQuery = 10000;
+
+ Report report = createReport(queries, recordsPerQuery);
+
+ BandData rootBand = new BandData(BandData.ROOT_BAND_NAME);
+ rootBand.setData(new HashMap<>());
+ rootBand.addReportFieldFormats(report.getReportFieldFormats());
+ rootBand.setFirstLevelBandDefinitionNames(new HashSet<>());
+
+ long start = System.currentTimeMillis();
+ try {
+ new DataExtractorImpl(new DefaultLoaderFactory().setGroovyDataLoader(
+ new GroovyDataLoader(new DefaultScriptingImpl()))).extractData(report, new HashMap<>(), rootBand);
+ } finally {
+ System.out.println(
+ String.format("Report processing stress test (%d queries and %d records per query) took %d ms",
+ queries, recordsPerQuery, System.currentTimeMillis() - start)
+ );
+ }
+ }
+
+ private Report createReport(int queries, int recordsPerQuery) {
+ BandBuilder bandBuilder = new BandBuilder()
+ .name("band");
+ for (int i = 0; i < queries; i++) {
+ String script = "import java.util.*;\n" +
+ "int i = " + i + ";\n" +
+ "List result = new ArrayList<>(" + recordsPerQuery + ");\n" +
+ "for (int j = 0; j < " + recordsPerQuery + "; j++) {\n" +
+ " Map<String, Object> record = new LinkedHashMap<>();\n" +
+ " record.put(\"col\" + i + j, Integer.toString(i) + Integer.toString(j));\n" +
+ " record.put(\"link\", j);\n" +
+ " result.add(record);\n" +
+ "}\n" +
+ "return result;";
+ bandBuilder.query("q" + i, script, "groovy", "link");
+ }
+
+ return new ReportBuilder()
+ .band(bandBuilder.build())
+ .name("report")
+ .build();
+ }
 }