CBSNexora/cbsnexora-backend
4
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

FINERACT-1707 - Export refactor

Browse Source 
[x] Refactor Datatable export
[x] Fix CSV exceptions
This commit is contained in:
Janos Haber
2023-01-19 15:45:48 +01:00
committed by Arnold Gálovics
parent 1b2c09eae5
commit e82c0cac52
6 changed files with 122 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
+1
View File
@@ -60,6 +60,7 @@ dependencyManagement {
dependency ('org.mnode.ical4j:ical4j:3.2.8') {
exclude 'com.sun.mail:javax.mail'
}
dependency 'org.apache.commons:commons-csv:1.9.0'
dependency 'org.quartz-scheduler:quartz:2.3.2'
dependency 'software.amazon.awssdk:bom:2.15.0'
dependency 'org.ehcache:ehcache:3.10.8'
fineract-provider/dependencies.gradle
+1
View File
@@ -121,6 +121,7 @@ dependencies {
exclude group: 'com.zaxxer', module: 'HikariCP-java7'
}
implementation platform('software.amazon.awssdk:bom')
implementation('org.apache.commons:commons-csv'){}
implementation ('software.amazon.awssdk:s3') {
}
implementation ('software.amazon.awssdk:auth') {
fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java
-18
View File
@@ -100,24 +100,6 @@ public final class ApiParameterHelper {
return locale;
}
public static boolean exportCsv(final MultivaluedMap<String, String> queryParams) {
boolean exportCsv = false;
if (queryParams.getFirst("exportCSV") != null) {
final String exportCsvValue = queryParams.getFirst("exportCSV");
exportCsv = "true".equalsIgnoreCase(exportCsvValue);
}
return exportCsv;
}
public static boolean exportPdf(final MultivaluedMap<String, String> queryParams) {
boolean exportPDF = false;
if (queryParams.getFirst("exportPDF") != null) {
final String exportPdfValue = queryParams.getFirst("exportPDF");
exportPDF = "true".equalsIgnoreCase(exportPdfValue);
}
return exportPDF;
}
public static boolean parameterType(final MultivaluedMap<String, String> queryParams) {
boolean parameterType = false;
if (queryParams.getFirst("parameterType") != null) {
fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
+48
View File
@@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.dataqueries.service;
import javax.ws.rs.core.MultivaluedMap;
public enum DatatableExportTargetParameter {
CSV("exportCSV"), PDF("exportPDF"), S3("exportS3"), JSON("exportJSON"), PRETTY_JSON("pretty");
private final String value;
DatatableExportTargetParameter(final String value) {
this.value = value;
}
public String getValue() {
return this.value;
}
public static DatatableExportTargetParameter checkTarget(final MultivaluedMap<String, String> queryParams) {
for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) {
String parameterName = parameter.getValue();
if (queryParams.getFirst(parameterName) != null) {
if ("true".equalsIgnoreCase(queryParams.getFirst(parameterName))) {
return parameter;
}
}
}
return JSON;
}
}
fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
+55 -43
View File
@@ -56,55 +56,67 @@ public class DatatableReportingProcessService implements ReportingProcessService
public Response processRequest(String reportName, MultivaluedMap<String, String> queryParams) {
boolean isSelfServiceUserReport = Boolean.parseBoolean(
queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0));
final boolean prettyPrint = ApiParameterHelper.prettyPrint(queryParams);
final boolean exportCsv = ApiParameterHelper.exportCsv(queryParams);
final boolean exportPdf = ApiParameterHelper.exportPdf(queryParams);
DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams);
final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report";
// PDF format
if (exportPdf) {
final Map<String, String> reportParams = getReportParams(queryParams);
final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
isSelfServiceUserReport);
final File file = new File(pdfFileName);
final ResponseBuilder response = Response.ok(file);
response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
response.header("content-Type", "application/pdf");
return response.build();
}
// JSON format
if (!exportCsv) {
final Map<String, String> reportParams = getReportParams(queryParams);
final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName,
parameterTypeValue, reportParams, isSelfServiceUserReport);
String json;
final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
if (genericResultSetIsPassed) {
if (genericResultSet) {
json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
} else {
json = this.genericDataService.generateJsonFromGenericResultsetData(result);
}
} else {
json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
}
return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
}
// CSV format
final Map<String, String> reportParams = getReportParams(queryParams);
return switch (exportMode) {
case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue,
exportMode == DatatableExportTargetParameter.PRETTY_JSON);
};
}
private Response exportJSON(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) {
final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue,
reportParams, isSelfServiceUserReport);
String json;
final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
if (genericResultSetIsPassed) {
if (genericResultSet) {
json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
} else {
json = this.genericDataService.generateJsonFromGenericResultsetData(result);
}
} else {
json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
}
return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
}
private Response exportS3(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
boolean isSelfServiceUserReport, String parameterTypeValue) {
throw new UnsupportedOperationException("S3 export not supported for datatables");
}
private Response exportPDF(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
boolean isSelfServiceUserReport, String parameterTypeValue) {
final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
isSelfServiceUserReport);
final File file = new File(pdfFileName);
final ResponseBuilder response = Response.ok(file);
response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
response.header("content-Type", "application/pdf");
return response.build();
}
private Response exportCSV(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
boolean isSelfServiceUserReport, String parameterTypeValue) {
final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
isSelfServiceUserReport);
return Response.ok().entity(result).type("text/csv")
.header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build();
}
}
fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
+17 -62
View File
@@ -22,11 +22,12 @@ import com.lowagie.text.Document;
import com.lowagie.text.PageSize;
import com.lowagie.text.pdf.PdfPTable;
import com.lowagie.text.pdf.PdfWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -39,6 +40,8 @@ import java.util.Set;
import javax.ws.rs.core.StreamingOutput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.JdbcSupport;
@@ -79,75 +82,27 @@ public class ReadReportingServiceImpl implements ReadReportingService {
final boolean isSelfServiceUserReport) {
return out -> {
try {
final GenericResultsetData result = retrieveGenericResultset(name, type, queryParams, isSelfServiceUserReport);
final StringBuilder sb = generateCsvFileBuffer(result);
final InputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
final byte[] outputByte = new byte[4096];
Integer readLen = in.read(outputByte, 0, 4096);
while (readLen != -1) {
out.write(outputByte, 0, readLen);
readLen = in.read(outputByte, 0, 4096);
}
// in.close();
// out.flush();
// out.close();
generateCsvFileBuffer(result, out);
} catch (final Exception e) {
throw new PlatformDataIntegrityException("error.msg.exception.error", e.getMessage(), e);
}
};
}
private StringBuilder generateCsvFileBuffer(final GenericResultsetData result) {
final StringBuilder writer = new StringBuilder();
final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
log.debug("NO. of Columns: {}", columnHeaders.size());
final Integer chSize = columnHeaders.size();
for (int i = 0; i < chSize; i++) {
writer.append('"' + columnHeaders.get(i).getColumnName() + '"');
if (i < (chSize - 1)) {
writer.append(",");
private void generateCsvFileBuffer(final GenericResultsetData result, OutputStream out) throws IOException {
try (CSVPrinter printer = new CSVPrinter(new OutputStreamWriter(out, StandardCharsets.UTF_8), CSVFormat.EXCEL)) {
final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
final List<ResultsetRowData> data = result.getData();
final List<String> header = new ArrayList<>();
for (final ResultsetColumnHeaderData columnHeader : columnHeaders) {
header.add(columnHeader.getColumnName());
}
printer.printRecord(header);
for (final ResultsetRowData row : data) {
printer.printRecord(row.getRow());
}
}
writer.append('\n');
final List<ResultsetRowData> data = result.getData();
List<Object> row;
Integer rSize;
// String currCol;
String currColType;
String currVal;
final String doubleQuote = "\"";
final String twoDoubleQuotes = doubleQuote + doubleQuote;
log.debug("NO. of Rows: {}", data.size());
for (ResultsetRowData element : data) {
row = element.getRow();
rSize = row.size();
for (int j = 0; j < rSize; j++) {
// currCol = columnHeaders.get(j).getColumnName();
currColType = columnHeaders.get(j).getColumnType();
currVal = (String) row.get(j);
if (currVal != null) {
if (currColType.equals("DECIMAL") || currColType.equals("DOUBLE") || currColType.equals("BIGINT")
|| currColType.equals("SMALLINT") || currColType.equals("INT")) {
writer.append(currVal);
} else {
writer.append('"' + this.genericDataService.replace(currVal, doubleQuote, twoDoubleQuotes) + '"');
}
}
if (j < (rSize - 1)) {
writer.append(",");
}
}
writer.append('\n');
}
return writer;
}
@Override