FINERACT-1707 - Export refactor

[x] Refactor Datatable export
[x] Fix CSV exceptions

buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle

@@ -60,6 +60,7 @@ dependencyManagement {
         dependency ('org.mnode.ical4j:ical4j:3.2.8') {
             exclude 'com.sun.mail:javax.mail'
         }
+        dependency 'org.apache.commons:commons-csv:1.9.0'
         dependency 'org.quartz-scheduler:quartz:2.3.2'
         dependency 'software.amazon.awssdk:bom:2.15.0'
         dependency 'org.ehcache:ehcache:3.10.8'

fineract-provider/dependencies.gradle

@@ -121,6 +121,7 @@ dependencies {
         exclude group: 'com.zaxxer', module: 'HikariCP-java7'
     }
     implementation platform('software.amazon.awssdk:bom')
+    implementation('org.apache.commons:commons-csv'){}
     implementation ('software.amazon.awssdk:s3') {
     }
     implementation ('software.amazon.awssdk:auth') {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java

@@ -100,24 +100,6 @@ public final class ApiParameterHelper {
         return locale;
     }
 
-    public static boolean exportCsv(final MultivaluedMap<String, String> queryParams) {
-        boolean exportCsv = false;
-        if (queryParams.getFirst("exportCSV") != null) {
-            final String exportCsvValue = queryParams.getFirst("exportCSV");
-            exportCsv = "true".equalsIgnoreCase(exportCsvValue);
-        }
-        return exportCsv;
-    }
-
-    public static boolean exportPdf(final MultivaluedMap<String, String> queryParams) {
-        boolean exportPDF = false;
-        if (queryParams.getFirst("exportPDF") != null) {
-            final String exportPdfValue = queryParams.getFirst("exportPDF");
-            exportPDF = "true".equalsIgnoreCase(exportPdfValue);
-        }
-        return exportPDF;
-    }
-
     public static boolean parameterType(final MultivaluedMap<String, String> queryParams) {
         boolean parameterType = false;
         if (queryParams.getFirst("parameterType") != null) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+public enum DatatableExportTargetParameter {
+
+    CSV("exportCSV"), PDF("exportPDF"), S3("exportS3"), JSON("exportJSON"), PRETTY_JSON("pretty");
+
+    private final String value;
+
+    DatatableExportTargetParameter(final String value) {
+        this.value = value;
+    }
+
+    public String getValue() {
+        return this.value;
+    }
+
+    public static DatatableExportTargetParameter checkTarget(final MultivaluedMap<String, String> queryParams) {
+        for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) {
+            String parameterName = parameter.getValue();
+            if (queryParams.getFirst(parameterName) != null) {
+                if ("true".equalsIgnoreCase(queryParams.getFirst(parameterName))) {
+                    return parameter;
+                }
+            }
+        }
+        return JSON;
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java

@@ -56,55 +56,67 @@ public class DatatableReportingProcessService implements ReportingProcessService
     public Response processRequest(String reportName, MultivaluedMap<String, String> queryParams) {
         boolean isSelfServiceUserReport = Boolean.parseBoolean(
                 queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0));
-        final boolean prettyPrint = ApiParameterHelper.prettyPrint(queryParams);
+
-        final boolean exportCsv = ApiParameterHelper.exportCsv(queryParams);
+        DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams);
-        final boolean exportPdf = ApiParameterHelper.exportPdf(queryParams);
         final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report";
-
-        // PDF format
-        if (exportPdf) {
-            final Map<String, String> reportParams = getReportParams(queryParams);
-            final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
-                    isSelfServiceUserReport);
-
-            final File file = new File(pdfFileName);
-
-            final ResponseBuilder response = Response.ok(file);
-            response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
-            response.header("content-Type", "application/pdf");
-
-            return response.build();
-        }
-
-        // JSON format
-        if (!exportCsv) {
-            final Map<String, String> reportParams = getReportParams(queryParams);
-
-            final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName,
-                    parameterTypeValue, reportParams, isSelfServiceUserReport);
-
-            String json;
-            final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
-            final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
-            if (genericResultSetIsPassed) {
-                if (genericResultSet) {
-                    json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
-                } else {
-                    json = this.genericDataService.generateJsonFromGenericResultsetData(result);
-                }
-            } else {
-                json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
-            }
-
-            return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
-        }
-
-        // CSV format
         final Map<String, String> reportParams = getReportParams(queryParams);
+        return switch (exportMode) {
+            case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+            case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+            case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+            default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue,
+                    exportMode == DatatableExportTargetParameter.PRETTY_JSON);
+        };
+    }
+
+    private Response exportJSON(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+            boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) {
+        final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue,
+                reportParams, isSelfServiceUserReport);
+
+        String json;
+        final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
+        final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
+        if (genericResultSetIsPassed) {
+            if (genericResultSet) {
+                json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+            } else {
+                json = this.genericDataService.generateJsonFromGenericResultsetData(result);
+            }
+        } else {
+            json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+        }
+
+        return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
+    }
+
+    private Response exportS3(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+            boolean isSelfServiceUserReport, String parameterTypeValue) {
+        throw new UnsupportedOperationException("S3 export not supported for datatables");
+    }
+
+    private Response exportPDF(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+            boolean isSelfServiceUserReport, String parameterTypeValue) {
+
+        final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
+                isSelfServiceUserReport);
+
+        final File file = new File(pdfFileName);
+
+        final ResponseBuilder response = Response.ok(file);
+        response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
+        response.header("content-Type", "application/pdf");
+
+        return response.build();
+    }
+
+    private Response exportCSV(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+            boolean isSelfServiceUserReport, String parameterTypeValue) {
         final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
                 isSelfServiceUserReport);
 
         return Response.ok().entity(result).type("text/csv")
                 .header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build();
+
     }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java

@@ -22,11 +22,12 @@ import com.lowagie.text.Document;
 import com.lowagie.text.PageSize;
 import com.lowagie.text.pdf.PdfPTable;
 import com.lowagie.text.pdf.PdfWriter;
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.InputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -39,6 +40,8 @@ import java.util.Set;
 import javax.ws.rs.core.StreamingOutput;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.JdbcSupport;
@@ -79,75 +82,27 @@ public class ReadReportingServiceImpl implements ReadReportingService {
             final boolean isSelfServiceUserReport) {
         return out -> {
             try {
-
                 final GenericResultsetData result = retrieveGenericResultset(name, type, queryParams, isSelfServiceUserReport);
-                final StringBuilder sb = generateCsvFileBuffer(result);
+                generateCsvFileBuffer(result, out);
-
-                final InputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
-
-                final byte[] outputByte = new byte[4096];
-                Integer readLen = in.read(outputByte, 0, 4096);
-
-                while (readLen != -1) {
-                    out.write(outputByte, 0, readLen);
-                    readLen = in.read(outputByte, 0, 4096);
-                }
-                // in.close();
-                // out.flush();
-                // out.close();
             } catch (final Exception e) {
                 throw new PlatformDataIntegrityException("error.msg.exception.error", e.getMessage(), e);
             }
         };
     }
 
-    private StringBuilder generateCsvFileBuffer(final GenericResultsetData result) {
+    private void generateCsvFileBuffer(final GenericResultsetData result, OutputStream out) throws IOException {
-        final StringBuilder writer = new StringBuilder();
+        try (CSVPrinter printer = new CSVPrinter(new OutputStreamWriter(out, StandardCharsets.UTF_8), CSVFormat.EXCEL)) {
-
+            final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
-        final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
+            final List<ResultsetRowData> data = result.getData();
-        log.debug("NO. of Columns: {}", columnHeaders.size());
+            final List<String> header = new ArrayList<>();
-        final Integer chSize = columnHeaders.size();
+            for (final ResultsetColumnHeaderData columnHeader : columnHeaders) {
-        for (int i = 0; i < chSize; i++) {
+                header.add(columnHeader.getColumnName());
-            writer.append('"' + columnHeaders.get(i).getColumnName() + '"');
+            }
-            if (i < (chSize - 1)) {
+            printer.printRecord(header);
-                writer.append(",");
+            for (final ResultsetRowData row : data) {
+                printer.printRecord(row.getRow());
             }
         }
-        writer.append('\n');
-
-        final List<ResultsetRowData> data = result.getData();
-        List<Object> row;
-        Integer rSize;
-        // String currCol;
-        String currColType;
-        String currVal;
-        final String doubleQuote = "\"";
-        final String twoDoubleQuotes = doubleQuote + doubleQuote;
-        log.debug("NO. of Rows: {}", data.size());
-        for (ResultsetRowData element : data) {
-            row = element.getRow();
-            rSize = row.size();
-            for (int j = 0; j < rSize; j++) {
-                // currCol = columnHeaders.get(j).getColumnName();
-                currColType = columnHeaders.get(j).getColumnType();
-                currVal = (String) row.get(j);
-                if (currVal != null) {
-                    if (currColType.equals("DECIMAL") || currColType.equals("DOUBLE") || currColType.equals("BIGINT")
-                            || currColType.equals("SMALLINT") || currColType.equals("INT")) {
-                        writer.append(currVal);
-                    } else {
-                        writer.append('"' + this.genericDataService.replace(currVal, doubleQuote, twoDoubleQuotes) + '"');
-                    }
-
-                }
-                if (j < (rSize - 1)) {
-                    writer.append(",");
-                }
-            }
-            writer.append('\n');
-        }
-
-        return writer;
     }
 
     @Override