From a1a5a8fa540610d9508ca2c18f01a6cec9a4d7ca Mon Sep 17 00:00:00 2001 From: Ivan Gavrilov Date: Tue, 13 Dec 2022 12:46:08 +0400 Subject: [PATCH] Add option to post-process excel formulas for xlsx, html and pdf outputs #135 --- core/build.gradle | 1 + .../util/properties/PropertiesLoader.java | 1 + .../yarg/console/ReportEngineCreator.java | 3 + .../factory/DefaultFormatterFactory.java | 12 +++ .../yarg/formatters/impl/XlsxFormatter.java | 88 +++++++++++++------ 5 files changed, 77 insertions(+), 28 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 1db6494f..1a1b85a9 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -224,6 +224,7 @@ configure(core) { exclude(group: 'bouncycastle', module: 'bctsp-jdk14') } compile(group: 'org.apache.poi', name: 'poi', version: '4.1.1') + compile(group: 'org.apache.poi', name: 'poi-ooxml', version: '4.1.1') compile(group: 'org.apache.poi', name: 'poi-scratchpad', version: '4.1.1') { exclude(group: 'org.apache.poi', module: 'poi') } diff --git a/core/modules/api/src/com/haulmont/yarg/util/properties/PropertiesLoader.java b/core/modules/api/src/com/haulmont/yarg/util/properties/PropertiesLoader.java index 52671db9..5326e98f 100644 --- a/core/modules/api/src/com/haulmont/yarg/util/properties/PropertiesLoader.java +++ b/core/modules/api/src/com/haulmont/yarg/util/properties/PropertiesLoader.java @@ -34,6 +34,7 @@ public interface PropertiesLoader { String CUBA_REPORTING_PUT_EMPTY_ROW_IF_NO_DATA_SELECTED = "cuba.reporting.dataextractor.putEmptyRowIfNoDataSelected"; String CUBA_REPORTING_FONTS_DIRECTORY = "cuba.reporting.fontsDirectory"; String CUBA_REPORTING_OPEN_HTML_FOR_PDF_CONVERSION = "cuba.reporting.openHtmlForPdfConversion"; + String CUBA_REPORTING_FORMULAS_POST_PROCESSING_EVALUATION_ENABLED = "cuba.reporting.formulasPostProcessingEvaluationEnabled"; Properties load() throws IOException; } diff --git a/core/modules/console/src/com/haulmont/yarg/console/ReportEngineCreator.java b/core/modules/console/src/com/haulmont/yarg/console/ReportEngineCreator.java index a068af7e..ebd0a5b6 100644 --- a/core/modules/console/src/com/haulmont/yarg/console/ReportEngineCreator.java +++ b/core/modules/console/src/com/haulmont/yarg/console/ReportEngineCreator.java @@ -68,6 +68,9 @@ public Reporting createReportingEngine(PropertiesLoader propertiesLoader) throws } } + String formulasEvaluationEnabled = properties.getProperty(PropertiesLoader.CUBA_REPORTING_FORMULAS_POST_PROCESSING_EVALUATION_ENABLED, "true"); + formatterFactory.setFormulasPostProcessingEvaluationEnabled(Boolean.parseBoolean(formulasEvaluationEnabled)); + reporting.setFormatterFactory(formatterFactory); SqlDataLoader sqlDataLoader = new PropertiesSqlLoaderFactory(propertiesLoader).create(); GroovyDataLoader groovyDataLoader = new GroovyDataLoader(new DefaultScriptingImpl()); diff --git a/core/modules/core/src/com/haulmont/yarg/formatters/factory/DefaultFormatterFactory.java b/core/modules/core/src/com/haulmont/yarg/formatters/factory/DefaultFormatterFactory.java index a8cde579..fd216d18 100644 --- a/core/modules/core/src/com/haulmont/yarg/formatters/factory/DefaultFormatterFactory.java +++ b/core/modules/core/src/com/haulmont/yarg/formatters/factory/DefaultFormatterFactory.java @@ -47,9 +47,12 @@ public class DefaultFormatterFactory implements ReportFormatterFactory { protected ReportInlinersProvider inlinersProvider; + protected boolean formulasPostProcessingEvaluationEnabled; + public DefaultFormatterFactory() { htmlImportProcessor = new HtmlImportProcessorImpl(); htmlToPdfConverterFactory = new HtmlToPdfConverterFactory(); + formulasPostProcessingEvaluationEnabled = true; formattersMap.put("xls", factoryInput -> { XLSFormatter xlsFormatter = new XLSFormatter(factoryInput); xlsFormatter.setDocumentConverter(documentConverter); @@ -91,6 +94,7 @@ public DefaultFormatterFactory() { xlsxFormatter.setDefaultFormatProvider(defaultFormatProvider); xlsxFormatter.setDocumentConverter(documentConverter); xlsxFormatter.setScripting(scripting); + xlsxFormatter.setFormulasPostProcessingEvaluationEnabled(formulasPostProcessingEvaluationEnabled); return xlsxFormatter; }; formattersMap.put("xlsx", xlsxCreator); @@ -147,6 +151,14 @@ public void setScripting(Scripting scripting) { this.scripting = scripting; } + public boolean isFormulasPostProcessingEvaluationEnabled() { + return formulasPostProcessingEvaluationEnabled; + } + + public void setFormulasPostProcessingEvaluationEnabled(boolean formulasPostProcessingEvaluationEnabled) { + this.formulasPostProcessingEvaluationEnabled = formulasPostProcessingEvaluationEnabled; + } + public ReportFormatter createFormatter(FormatterFactoryInput factoryInput) { String templateExtension = factoryInput.templateExtension; BandData rootBand = factoryInput.rootBand; diff --git a/core/modules/core/src/com/haulmont/yarg/formatters/impl/XlsxFormatter.java b/core/modules/core/src/com/haulmont/yarg/formatters/impl/XlsxFormatter.java index e5f45a4e..09150b6b 100644 --- a/core/modules/core/src/com/haulmont/yarg/formatters/impl/XlsxFormatter.java +++ b/core/modules/core/src/com/haulmont/yarg/formatters/impl/XlsxFormatter.java @@ -32,6 +32,9 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.poi.hssf.usermodel.HSSFDateUtil; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.FormulaEvaluator; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.docx4j.XmlUtils; import org.docx4j.dml.chart.CTAxDataSource; import org.docx4j.dml.chart.CTChart; @@ -53,10 +56,7 @@ import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; +import java.io.*; import java.util.*; import java.util.regex.Matcher; @@ -82,6 +82,8 @@ public class XlsxFormatter extends AbstractFormatter { protected BandData previousRangeBandData; protected int previousRangesRightOffset; + protected boolean formulasPostProcessingEvaluationEnabled = true; + protected Unmarshaller unmarshaller; protected Marshaller marshaller; @@ -96,6 +98,10 @@ public void setDocumentConverter(DocumentConverter documentConverter) { this.documentConverter = documentConverter; } + public void setFormulasPostProcessingEvaluationEnabled(boolean formulasPostProcessingEvaluationEnabled) { + this.formulasPostProcessingEvaluationEnabled = formulasPostProcessingEvaluationEnabled; + } + @Override public void renderDocument() { init(); @@ -135,36 +141,40 @@ protected void validateTemplateContainsNamedRange() { protected void saveAndClose() { try { checkThreadInterrupted(); - if (ReportOutputType.xlsx.equals(outputType)) { - writeToOutputStream(result.getPackage(), outputStream); - outputStream.flush(); - } else if (ReportOutputType.csv.equals(outputType)) { + if (ReportOutputType.csv.equals(outputType)) { saveXlsxAsCsv(result, outputStream); outputStream.flush(); - } else if (ReportOutputType.pdf.equals(outputType)) { - if (documentConverter != null) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - writeToOutputStream(result.getPackage(), bos); - documentConverter.convertToPdf(DocumentConverter.FileType.SPREADSHEET, bos.toByteArray(), outputStream); - outputStream.flush(); - } else { - throw new UnsupportedOperationException( - "XlsxFormatter could not convert result to pdf without Libre/Open office connected. " + - "Please setup Libre/Open office connection details."); + } else { + ByteArrayOutputStream intermediateBos = new ByteArrayOutputStream(); + writeToOutputStream(result.getPackage(), intermediateBos); + if (formulasPostProcessingEvaluationEnabled) { + intermediateBos = evaluateFormulas(intermediateBos.toByteArray()); } - } else if (ReportOutputType.html.equals(outputType)) { - if (documentConverter != null) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - writeToOutputStream(result.getPackage(), bos); - documentConverter.convertToHtml(DocumentConverter.FileType.SPREADSHEET, bos.toByteArray(), outputStream); + + if (ReportOutputType.xlsx.equals(outputType)) { + outputStream.write(intermediateBos.toByteArray()); outputStream.flush(); + } else if (ReportOutputType.pdf.equals(outputType)) { + if (documentConverter != null) { + documentConverter.convertToPdf(DocumentConverter.FileType.SPREADSHEET, intermediateBos.toByteArray(), outputStream); + outputStream.flush(); + } else { + throw new UnsupportedOperationException( + "XlsxFormatter could not convert result to pdf without Libre/Open office connected. " + + "Please setup Libre/Open office connection details."); + } + } else if (ReportOutputType.html.equals(outputType)) { + if (documentConverter != null) { + documentConverter.convertToHtml(DocumentConverter.FileType.SPREADSHEET, intermediateBos.toByteArray(), outputStream); + outputStream.flush(); + } else { + throw new UnsupportedOperationException( + "XlsxFormatter could not convert result to html without Libre/Open office connected. " + + "Please setup Libre/Open office connection details."); + } } else { - throw new UnsupportedOperationException( - "XlsxFormatter could not convert result to html without Libre/Open office connected. " + - "Please setup Libre/Open office connection details."); + throw new UnsupportedOperationException(String.format("XlsxFormatter could not output file with type [%s]", outputType)); } - } else { - throw new UnsupportedOperationException(String.format("XlsxFormatter could not output file with type [%s]", outputType)); } } catch (Docx4JException e) { throw wrapWithReportingException("An error occurred while saving result report", e); @@ -175,6 +185,28 @@ protected void saveAndClose() { } } + protected ByteArrayOutputStream evaluateFormulas(byte[] content) { + try { + ByteArrayInputStream bis = new ByteArrayInputStream(content); + org.apache.poi.ss.usermodel.Workbook workbook = new XSSFWorkbook(bis); + FormulaEvaluator evaluator = workbook.getCreationHelper().createFormulaEvaluator(); + for (org.apache.poi.ss.usermodel.Sheet sheet : workbook) { + for (org.apache.poi.ss.usermodel.Row row : sheet) { + for (org.apache.poi.ss.usermodel.Cell cell : row) { + if (cell.getCellType() == CellType.FORMULA) { + evaluator.evaluateFormulaCell(cell); + } + } + } + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + workbook.write(bos); + return bos; + } catch (IOException e) { + throw new ReportingException(e); + } + } + protected void init() { try { template = Document.create(SpreadsheetMLPackage.load(reportTemplate.getDocumentContent()));