From e82c0cac5222e354e001a65a90aaccee7b17b186 Mon Sep 17 00:00:00 2001 From: Janos Haber Date: Tue, 17 Jan 2023 23:02:13 +0100 Subject: [PATCH] FINERACT-1707 - Export refactor [x] Refactor Datatable export [x] Fix CSV exceptions --- .../org.apache.fineract.dependencies.gradle | 1 + fineract-provider/dependencies.gradle | 1 + .../core/api/ApiParameterHelper.java | 18 ---- .../DatatableExportTargetParameter.java | 48 +++++++++ .../DatatableReportingProcessService.java | 98 +++++++++++-------- .../service/ReadReportingServiceImpl.java | 79 ++++----------- 6 files changed, 122 insertions(+), 123 deletions(-) create mode 100644 fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle index a58531cff..4d53d1334 100644 --- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle +++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle @@ -60,6 +60,7 @@ dependencyManagement { dependency ('org.mnode.ical4j:ical4j:3.2.8') { exclude 'com.sun.mail:javax.mail' } + dependency 'org.apache.commons:commons-csv:1.9.0' dependency 'org.quartz-scheduler:quartz:2.3.2' dependency 'software.amazon.awssdk:bom:2.15.0' dependency 'org.ehcache:ehcache:3.10.8' diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle index c5d85003a..6ad81837a 100644 --- a/fineract-provider/dependencies.gradle +++ b/fineract-provider/dependencies.gradle @@ -121,6 +121,7 @@ dependencies { exclude group: 'com.zaxxer', module: 'HikariCP-java7' } implementation platform('software.amazon.awssdk:bom') + implementation('org.apache.commons:commons-csv'){} implementation ('software.amazon.awssdk:s3') { } implementation ('software.amazon.awssdk:auth') { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java index b48d6c95b..89ce6c1d3 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java @@ -100,24 +100,6 @@ public final class ApiParameterHelper { return locale; } - public static boolean exportCsv(final MultivaluedMap queryParams) { - boolean exportCsv = false; - if (queryParams.getFirst("exportCSV") != null) { - final String exportCsvValue = queryParams.getFirst("exportCSV"); - exportCsv = "true".equalsIgnoreCase(exportCsvValue); - } - return exportCsv; - } - - public static boolean exportPdf(final MultivaluedMap queryParams) { - boolean exportPDF = false; - if (queryParams.getFirst("exportPDF") != null) { - final String exportPdfValue = queryParams.getFirst("exportPDF"); - exportPDF = "true".equalsIgnoreCase(exportPdfValue); - } - return exportPDF; - } - public static boolean parameterType(final MultivaluedMap queryParams) { boolean parameterType = false; if (queryParams.getFirst("parameterType") != null) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java new file mode 100644 index 000000000..e596b182e --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service; + +import javax.ws.rs.core.MultivaluedMap; + +public enum DatatableExportTargetParameter { + + CSV("exportCSV"), PDF("exportPDF"), S3("exportS3"), JSON("exportJSON"), PRETTY_JSON("pretty"); + + private final String value; + + DatatableExportTargetParameter(final String value) { + this.value = value; + } + + public String getValue() { + return this.value; + } + + public static DatatableExportTargetParameter checkTarget(final MultivaluedMap queryParams) { + for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) { + String parameterName = parameter.getValue(); + if (queryParams.getFirst(parameterName) != null) { + if ("true".equalsIgnoreCase(queryParams.getFirst(parameterName))) { + return parameter; + } + } + } + return JSON; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java index e7cb15653..b5f2e584b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java @@ -56,55 +56,67 @@ public class DatatableReportingProcessService implements ReportingProcessService public Response processRequest(String reportName, MultivaluedMap queryParams) { boolean isSelfServiceUserReport = Boolean.parseBoolean( queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0)); - final boolean prettyPrint = ApiParameterHelper.prettyPrint(queryParams); - final boolean exportCsv = ApiParameterHelper.exportCsv(queryParams); - final boolean exportPdf = ApiParameterHelper.exportPdf(queryParams); + + DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams); final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report"; - - // PDF format - if (exportPdf) { - final Map reportParams = getReportParams(queryParams); - final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams, - isSelfServiceUserReport); - - final File file = new File(pdfFileName); - - final ResponseBuilder response = Response.ok(file); - response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\""); - response.header("content-Type", "application/pdf"); - - return response.build(); - } - - // JSON format - if (!exportCsv) { - final Map reportParams = getReportParams(queryParams); - - final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, - parameterTypeValue, reportParams, isSelfServiceUserReport); - - String json; - final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams); - final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams); - if (genericResultSetIsPassed) { - if (genericResultSet) { - json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); - } else { - json = this.genericDataService.generateJsonFromGenericResultsetData(result); - } - } else { - json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); - } - - return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build(); - } - - // CSV format final Map reportParams = getReportParams(queryParams); + return switch (exportMode) { + case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); + case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); + case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); + default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue, + exportMode == DatatableExportTargetParameter.PRETTY_JSON); + }; + } + + private Response exportJSON(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) { + final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue, + reportParams, isSelfServiceUserReport); + + String json; + final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams); + final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams); + if (genericResultSetIsPassed) { + if (genericResultSet) { + json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); + } else { + json = this.genericDataService.generateJsonFromGenericResultsetData(result); + } + } else { + json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); + } + + return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build(); + } + + private Response exportS3(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + throw new UnsupportedOperationException("S3 export not supported for datatables"); + } + + private Response exportPDF(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + + final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams, + isSelfServiceUserReport); + + final File file = new File(pdfFileName); + + final ResponseBuilder response = Response.ok(file); + response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\""); + response.header("content-Type", "application/pdf"); + + return response.build(); + } + + private Response exportCSV(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams, isSelfServiceUserReport); return Response.ok().entity(result).type("text/csv") .header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build(); + } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java index 6d3ec207e..a1958c7bd 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java @@ -22,11 +22,12 @@ import com.lowagie.text.Document; import com.lowagie.text.PageSize; import com.lowagie.text.pdf.PdfPTable; import com.lowagie.text.pdf.PdfWriter; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; @@ -39,6 +40,8 @@ import java.util.Set; import javax.ws.rs.core.StreamingOutput; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.JdbcSupport; @@ -79,75 +82,27 @@ public class ReadReportingServiceImpl implements ReadReportingService { final boolean isSelfServiceUserReport) { return out -> { try { - final GenericResultsetData result = retrieveGenericResultset(name, type, queryParams, isSelfServiceUserReport); - final StringBuilder sb = generateCsvFileBuffer(result); - - final InputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8)); - - final byte[] outputByte = new byte[4096]; - Integer readLen = in.read(outputByte, 0, 4096); - - while (readLen != -1) { - out.write(outputByte, 0, readLen); - readLen = in.read(outputByte, 0, 4096); - } - // in.close(); - // out.flush(); - // out.close(); + generateCsvFileBuffer(result, out); } catch (final Exception e) { throw new PlatformDataIntegrityException("error.msg.exception.error", e.getMessage(), e); } }; } - private StringBuilder generateCsvFileBuffer(final GenericResultsetData result) { - final StringBuilder writer = new StringBuilder(); - - final List columnHeaders = result.getColumnHeaders(); - log.debug("NO. of Columns: {}", columnHeaders.size()); - final Integer chSize = columnHeaders.size(); - for (int i = 0; i < chSize; i++) { - writer.append('"' + columnHeaders.get(i).getColumnName() + '"'); - if (i < (chSize - 1)) { - writer.append(","); + private void generateCsvFileBuffer(final GenericResultsetData result, OutputStream out) throws IOException { + try (CSVPrinter printer = new CSVPrinter(new OutputStreamWriter(out, StandardCharsets.UTF_8), CSVFormat.EXCEL)) { + final List columnHeaders = result.getColumnHeaders(); + final List data = result.getData(); + final List header = new ArrayList<>(); + for (final ResultsetColumnHeaderData columnHeader : columnHeaders) { + header.add(columnHeader.getColumnName()); + } + printer.printRecord(header); + for (final ResultsetRowData row : data) { + printer.printRecord(row.getRow()); } } - writer.append('\n'); - - final List data = result.getData(); - List row; - Integer rSize; - // String currCol; - String currColType; - String currVal; - final String doubleQuote = "\""; - final String twoDoubleQuotes = doubleQuote + doubleQuote; - log.debug("NO. of Rows: {}", data.size()); - for (ResultsetRowData element : data) { - row = element.getRow(); - rSize = row.size(); - for (int j = 0; j < rSize; j++) { - // currCol = columnHeaders.get(j).getColumnName(); - currColType = columnHeaders.get(j).getColumnType(); - currVal = (String) row.get(j); - if (currVal != null) { - if (currColType.equals("DECIMAL") || currColType.equals("DOUBLE") || currColType.equals("BIGINT") - || currColType.equals("SMALLINT") || currColType.equals("INT")) { - writer.append(currVal); - } else { - writer.append('"' + this.genericDataService.replace(currVal, doubleQuote, twoDoubleQuotes) + '"'); - } - - } - if (j < (rSize - 1)) { - writer.append(","); - } - } - writer.append('\n'); - } - - return writer; } @Override