diff --git a/.github/workflows/build-mariadb.yml b/.github/workflows/build-mariadb.yml index e216973b7..eb5254e5f 100644 --- a/.github/workflows/build-mariadb.yml +++ b/.github/workflows/build-mariadb.yml @@ -52,12 +52,39 @@ jobs: run: | ./gradlew --no-daemon -q createDB -PdbName=fineract_tenants ./gradlew --no-daemon -q createDB -PdbName=fineract_default + - name: Start LocalStack + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + run: | + echo "Update python pyopenssl" + pip install --upgrade pyopenssl + echo "Install localstack" + pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal + docker pull localstack/localstack # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + echo "Create fineract S3 bucket" + awslocal s3api create-bucket --bucket fineract-reports + echo "LocalStack initialization complete" - name: Install additional software run: | sudo apt-get update sudo apt-get install ghostscript graphviz -y - name: Build & Test + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + FINERACT_REPORT_EXPORT_S3_ENABLED: true + FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports run: | ./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test ./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast diff --git a/.github/workflows/build-mysql.yml b/.github/workflows/build-mysql.yml index 4bb0d0635..ca7bf02d8 100644 --- a/.github/workflows/build-mysql.yml +++ b/.github/workflows/build-mysql.yml @@ -52,12 +52,39 @@ jobs: run: | ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_tenants ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_default + - name: Start LocalStack + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + run: | + echo "Update python pyopenssl" + pip install --upgrade pyopenssl + echo "Install localstack" + pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal + docker pull localstack/localstack # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + echo "Create fineract S3 bucket" + awslocal s3api create-bucket --bucket fineract-reports + echo "LocalStack initialization complete" - name: Install additional software run: | sudo apt-get update sudo apt-get install ghostscript graphviz -y - name: Build & Test + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + FINERACT_REPORT_EXPORT_S3_ENABLED: true + FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports run: | ./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=mysql ./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast -PdbType=mysql diff --git a/.github/workflows/build-postgresql.yml b/.github/workflows/build-postgresql.yml index 83382569b..3e861f7e6 100644 --- a/.github/workflows/build-postgresql.yml +++ b/.github/workflows/build-postgresql.yml @@ -53,12 +53,39 @@ jobs: run: | ./gradlew --no-daemon -q createPGDB -PdbName=fineract_tenants ./gradlew --no-daemon -q createPGDB -PdbName=fineract_default + - name: Start LocalStack + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + run: | + echo "Update python pyopenssl" + pip install --upgrade pyopenssl + echo "Install localstack" + pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal + docker pull localstack/localstack # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + echo "Create fineract S3 bucket" + awslocal s3api create-bucket --bucket fineract-reports + echo "LocalStack initialization complete" - name: Install additional software run: | sudo apt-get update sudo apt-get install ghostscript graphviz -y - name: Build & Test + env: + AWS_ENDPOINT_URL: http://localhost:4566 + AWS_ACCESS_KEY_ID: localstack + AWS_SECRET_ACCESS_KEY: localstack + AWS_REGION: us-east-1 + FINERACT_REPORT_EXPORT_S3_ENABLED: true + FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports run: | ./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=postgresql ./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast -PdbType=postgresql diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java index 168c1553e..a7b2cb580 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java @@ -134,4 +134,6 @@ public interface ConfigurationDomainService { boolean isCOBBulkEventEnabled(); Long retrieveExternalEventBatchSize(); + + String retrieveReportExportS3FolderName(); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java index c34ab9f9d..5088b3c38 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java @@ -50,6 +50,8 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService private static final String ENABLE_COB_BULK_EVENT = "enable-cob-bulk-event"; private static final String EXTERNAL_EVENT_BATCH_SIZE = "external-event-batch-size"; + private static final String REPORT_EXPORT_S3_FOLDER_NAME = "report-export-s3-folder-name"; + private final PermissionRepository permissionRepository; private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository; private final PlatformCacheRepository cacheTypeRepository; @@ -512,4 +514,9 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService return property.getValue(); } + @Override + public String retrieveReportExportS3FolderName() { + final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(REPORT_EXPORT_S3_FOLDER_NAME); + return property.getStringValue(); + } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java index 99b50c5fd..85fbef4b7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java @@ -48,6 +48,8 @@ public class FineractProperties { private FineractContentProperties content; + private FineractReportProperties report; + @Getter @Setter public static class FineractTenantProperties { @@ -201,4 +203,26 @@ public class FineractProperties { private String accessKey; private String secretKey; } + + @Getter + @Setter + public static class FineractReportProperties { + + private FineractExportProperties export; + } + + @Getter + @Setter + public static class FineractExportProperties { + + private FineractExportS3Properties s3; + } + + @Getter + @Setter + public static class FineractExportS3Properties { + + private String bucketName; + private Boolean enabled; + } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java new file mode 100644 index 000000000..725a99d82 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.core.service; + +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +public final class StreamUtil { + + private StreamUtil() {} + + public static Collector foldLeft(final B init, final BiFunction f) { + return Collectors.collectingAndThen(Collectors.reducing(Function.identity(), a -> b -> f.apply(b, a), Function::andThen), + endo -> endo.apply(init)); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java index 0718e8b14..ab859a25c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java @@ -20,6 +20,7 @@ package org.apache.fineract.infrastructure.dataqueries.api; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -39,6 +40,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.apache.fineract.infrastructure.core.api.ApiParameterHelper; import org.apache.fineract.infrastructure.core.exception.PlatformServiceUnavailableException; +import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider; import org.apache.fineract.infrastructure.report.service.ReportingProcessService; @@ -70,6 +72,29 @@ public class RunreportsApiResource { this.reportingProcessServiceProvider = reportingProcessServiceProvider; } + @GET + @Path("/availableExports/{reportName}") + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + @Operation(summary = "Return all available export types for the specific report", description = "Returns the list of all available export types.") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "", content = @Content(array = @ArraySchema(schema = @Schema(implementation = ReportExportType.class)))) }) + public Response retrieveAllAvailableExports(@PathParam("reportName") @Parameter(description = "reportName") final String reportName, + @Context final UriInfo uriInfo, + @DefaultValue("false") @QueryParam(IS_SELF_SERVICE_USER_REPORT_PARAMETER) @Parameter(description = IS_SELF_SERVICE_USER_REPORT_PARAMETER) final boolean isSelfServiceUserReport) { + MultivaluedMap queryParams = new MultivaluedStringMap(); + queryParams.putAll(uriInfo.getQueryParameters()); + + final boolean parameterType = ApiParameterHelper.parameterType(queryParams); + String reportType = readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType); + ReportingProcessService reportingProcessService = reportingProcessServiceProvider.findReportingProcessService(reportType); + if (reportingProcessService == null) { + throw new PlatformServiceUnavailableException("err.msg.report.service.implementation.missing", + ReportingProcessServiceProvider.SERVICE_MISSING + reportType, reportType); + } + return Response.ok().entity(reportingProcessService.getAvailableExportTargets()).build(); + } + @GET @Path("{reportName}") @Consumes({ MediaType.APPLICATION_JSON }) @@ -108,8 +133,8 @@ public class RunreportsApiResource { // Pass through isSelfServiceUserReport so that ReportingProcessService implementations can use it queryParams.putSingle(IS_SELF_SERVICE_USER_REPORT_PARAMETER, Boolean.toString(isSelfServiceUserReport)); - String reportType = this.readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType); - ReportingProcessService reportingProcessService = this.reportingProcessServiceProvider.findReportingProcessService(reportType); + String reportType = readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType); + ReportingProcessService reportingProcessService = reportingProcessServiceProvider.findReportingProcessService(reportType); if (reportingProcessService == null) { throw new PlatformServiceUnavailableException("err.msg.report.service.implementation.missing", ReportingProcessServiceProvider.SERVICE_MISSING + reportType, reportType); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java index f91ab04e0..712ef97bc 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java @@ -38,4 +38,5 @@ final class RunreportsApiResourceSwagger { public List columnHeaders; public List data; } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java new file mode 100644 index 000000000..10ea4daba --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.data; + +import java.io.Serializable; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor +public class ReportExportType implements Serializable { + + private final String key; + private final String queryParameter; +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java index e596b182e..4c5d22a80 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java @@ -34,7 +34,7 @@ public enum DatatableExportTargetParameter { return this.value; } - public static DatatableExportTargetParameter checkTarget(final MultivaluedMap queryParams) { + public static DatatableExportTargetParameter resolverExportTarget(final MultivaluedMap queryParams) { for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) { String parameterName = parameter.getValue(); if (queryParams.getFirst(parameterName) != null) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java index b5f2e584b..85176b567 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java @@ -18,105 +18,71 @@ */ package org.apache.fineract.infrastructure.dataqueries.service; -import java.io.File; +import java.util.Arrays; import java.util.List; import java.util.Map; -import javax.ws.rs.core.MediaType; +import java.util.Optional; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.ResponseBuilder; -import javax.ws.rs.core.StreamingOutput; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.core.api.ApiParameterHelper; -import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer; +import org.apache.fineract.infrastructure.core.service.StreamUtil; import org.apache.fineract.infrastructure.dataqueries.api.RunreportsApiResource; -import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData; -import org.apache.fineract.infrastructure.dataqueries.data.ReportData; +import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType; +import org.apache.fineract.infrastructure.dataqueries.service.export.DatatableReportExportService; +import org.apache.fineract.infrastructure.dataqueries.service.export.ResponseHolder; import org.apache.fineract.infrastructure.report.annotation.ReportService; import org.apache.fineract.infrastructure.report.service.ReportingProcessService; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Service @ReportService(type = { "Table", "Chart", "SMS" }) +@RequiredArgsConstructor +@Slf4j public class DatatableReportingProcessService implements ReportingProcessService { - private final ReadReportingService readExtraDataAndReportingService; - private final ToApiJsonSerializer toApiJsonSerializer; - private final GenericDataService genericDataService; - - @Autowired - public DatatableReportingProcessService(final ReadReportingService readExtraDataAndReportingService, - final GenericDataService genericDataService, final ToApiJsonSerializer toApiJsonSerializer) { - this.readExtraDataAndReportingService = readExtraDataAndReportingService; - this.toApiJsonSerializer = toApiJsonSerializer; - this.genericDataService = genericDataService; - } + private final List exportServices; @Override public Response processRequest(String reportName, MultivaluedMap queryParams) { boolean isSelfServiceUserReport = Boolean.parseBoolean( queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0)); - DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams); + DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.resolverExportTarget(queryParams); final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report"; final Map reportParams = getReportParams(queryParams); - return switch (exportMode) { - case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); - case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); - case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); - default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue, - exportMode == DatatableExportTargetParameter.PRETTY_JSON); - }; - } - - private Response exportJSON(String reportName, MultivaluedMap queryParams, Map reportParams, - boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) { - final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue, - reportParams, isSelfServiceUserReport); - - String json; - final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams); - final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams); - if (genericResultSetIsPassed) { - if (genericResultSet) { - json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); - } else { - json = this.genericDataService.generateJsonFromGenericResultsetData(result); - } - } else { - json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); + ResponseHolder response = findReportExportService(exportMode) // + .orElseThrow(() -> new IllegalArgumentException("Unsupported export target: " + exportMode)) // + .export(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue); + Response.ResponseBuilder builder = Response.status(response.status().getStatusCode()); + if (StringUtils.isNotBlank(response.contentType())) { + builder = builder.type(response.contentType()); } - - return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build(); + if (StringUtils.isNotBlank(response.fileName())) { + builder = builder.header("Content-Disposition", "attachment; filename=" + response.fileName()); + } + if (response.entity() != null) { + builder = builder.entity(response.entity()); + } + if (response.headers() != null && !response.headers().isEmpty()) { + builder = response.headers().stream().collect(StreamUtil.foldLeft(builder, (b, h) -> b.header(h.getKey(), h.getValue()))); + } + return builder.build(); } - private Response exportS3(String reportName, MultivaluedMap queryParams, Map reportParams, - boolean isSelfServiceUserReport, String parameterTypeValue) { - throw new UnsupportedOperationException("S3 export not supported for datatables"); + @Override + public List getAvailableExportTargets() { + return Arrays // + .stream(DatatableExportTargetParameter.values()) // + .filter(target -> findReportExportService(target).isPresent()) // + .map(target -> new ReportExportType(target.name(), target.getValue())) // + .toList(); } - private Response exportPDF(String reportName, MultivaluedMap queryParams, Map reportParams, - boolean isSelfServiceUserReport, String parameterTypeValue) { - - final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams, - isSelfServiceUserReport); - - final File file = new File(pdfFileName); - - final ResponseBuilder response = Response.ok(file); - response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\""); - response.header("content-Type", "application/pdf"); - - return response.build(); + private Optional findReportExportService(DatatableExportTargetParameter target) { + return exportServices.stream().filter(service -> service.supports(target)).findFirst(); } - private Response exportCSV(String reportName, MultivaluedMap queryParams, Map reportParams, - boolean isSelfServiceUserReport, String parameterTypeValue) { - final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams, - isSelfServiceUserReport); - - return Response.ok().entity(result).type("text/csv") - .header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build(); - - } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java new file mode 100644 index 000000000..bb5525eb8 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.util.Map; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; +import lombok.RequiredArgsConstructor; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter; +import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class CsvDatatableReportExportServiceImpl implements DatatableReportExportService { + + private final ReadReportingService readExtraDataAndReportingService; + + @Override + public ResponseHolder export(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams, + isSelfServiceUserReport); + return new ResponseHolder(Response.Status.OK).contentType("text/csv") + .addHeader("Content-Disposition", + "attachment;filename=" + DatatableExportUtil.generatePlainExportFileName(255, "csv", reportName, reportParams)) + .entity(result); + } + + @Override + public boolean supports(DatatableExportTargetParameter exportType) { + return exportType == DatatableExportTargetParameter.CSV; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java new file mode 100644 index 000000000..0a6b522c4 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.fineract.infrastructure.core.service.DateUtils; +import org.jetbrains.annotations.NotNull; + +public final class DatatableExportUtil { + + private DatatableExportUtil() {} + + public static String normalizeFolderName(String folderName) { + if (StringUtils.isBlank(folderName)) { + return ""; + } + String contentNormalizer = folderName.trim() // + .replaceAll("[^a-zA-Z0-9!\\-_.'()$/]", "_") // replace special characters + .replaceAll("/+", "/") // replace multiply / with a single / + .replaceAll("^[./]+", ""); // remove leading . and / + + return contentNormalizer.endsWith("/") ? contentNormalizer.substring(0, contentNormalizer.length() - 1) : contentNormalizer; + } + + public static String generatePlainExportFileName(int maxLength, String extension, String reportName, Map reportParams) { + exportBasicValidation(extension, reportName); + return generateReportFileName(maxLength, "", extension, reportName, reportParams); + } + + private static void exportBasicValidation(String extension, String reportName) { + if (StringUtils.isBlank(extension)) { + throw new IllegalArgumentException("The extension is required"); + } + if (StringUtils.isBlank(reportName)) { + throw new IllegalArgumentException("The report name is required"); + } + } + + public static String generateS3DatatableExportFileName(int maxLength, String folder, String extension, String reportName, + Map reportParams) { + exportBasicValidation(extension, reportName); + if (maxLength < 30) { + throw new IllegalArgumentException("The maximum length must be greater than 30"); + } + folder = normalizeFolderName(folder); + String reportFinalName = generateReportFileName(maxLength, folder, extension, reportName, reportParams); + if (StringUtils.isBlank(folder)) { + return reportFinalName; + } else { + return folder + "/" + reportFinalName; + } + } + + @NotNull + private static String generateReportFileName(int maxLength, String folder, String extension, String reportName, + Map reportParams) { + String extensionWithDot = extension.startsWith(".") ? extension : "." + extension; + String timestamp = "_" + DateUtils.getOffsetDateTimeOfTenant().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")); + + int reportMaximumFileName = maxLength - folder.length() - timestamp.length() - extensionWithDot.length() - 1; + if (reportMaximumFileName < 0) { + throw new IllegalArgumentException("The folder name is too long"); + } + String normalizedFileName = reportName.trim().replaceAll("[^a-zA-Z0-9!\\-_.'()$]", "_"); + if (reportParams != null) { + normalizedFileName += "(" + reportParams.entrySet().stream() + .map(entry -> extractReportParameterKey(entry.getKey()) + "_" + entry.getValue()).collect(Collectors.joining(";")) + + ")"; + } + String reportFinalName = normalizedFileName.substring(0, Math.min(normalizedFileName.length(), reportMaximumFileName)) + timestamp + + extensionWithDot; + return reportFinalName; + } + + private static String extractReportParameterKey(String key) { + return key.startsWith("${") && key.endsWith("}") ? key.substring(2, key.length() - 1) : key; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java new file mode 100644 index 000000000..4af32530a --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.util.Map; +import javax.ws.rs.core.MultivaluedMap; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter; + +public interface DatatableReportExportService { + + ResponseHolder export(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue); + + boolean supports(DatatableExportTargetParameter exportType); +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java new file mode 100644 index 000000000..40032c933 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Header { + + private String key; + private String value; +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java new file mode 100644 index 000000000..a5eb1a266 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.util.Map; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import lombok.RequiredArgsConstructor; +import org.apache.fineract.infrastructure.core.api.ApiParameterHelper; +import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer; +import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData; +import org.apache.fineract.infrastructure.dataqueries.data.ReportData; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter; +import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService; +import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class JsonDatatableReportExportService implements DatatableReportExportService { + + private final ReadReportingService readExtraDataAndReportingService; + private final ToApiJsonSerializer toApiJsonSerializer; + private final GenericDataService genericDataService; + + @Override + public ResponseHolder export(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + + final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue, + reportParams, isSelfServiceUserReport); + DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.resolverExportTarget(queryParams); + boolean prettyPrint = exportMode == DatatableExportTargetParameter.PRETTY_JSON; + String json; + final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams); + final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams); + if (genericResultSetIsPassed) { + if (genericResultSet) { + json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); + } else { + json = this.genericDataService.generateJsonFromGenericResultsetData(result); + } + } else { + json = this.toApiJsonSerializer.serializePretty(prettyPrint, result); + } + return new ResponseHolder(Response.Status.OK).entity(json).contentType(MediaType.APPLICATION_JSON); + + } + + @Override + public boolean supports(DatatableExportTargetParameter exportType) { + return exportType == DatatableExportTargetParameter.JSON || exportType == DatatableExportTargetParameter.PRETTY_JSON; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java new file mode 100644 index 000000000..d4e9a4a45 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.io.File; +import java.util.Map; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import lombok.RequiredArgsConstructor; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter; +import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class PdfDatatableReportExportService implements DatatableReportExportService { + + private final ReadReportingService readExtraDataAndReportingService; + + @Override + public ResponseHolder export(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams, + isSelfServiceUserReport); + + final File file = new File(pdfFileName); + + return new ResponseHolder(Response.Status.OK).contentType("application/pdf") + .addHeader("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"").entity(file); + } + + @Override + public boolean supports(DatatableExportTargetParameter exportType) { + return exportType == DatatableExportTargetParameter.PDF; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java new file mode 100644 index 000000000..5ed4a4796 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.util.ArrayList; +import java.util.List; +import javax.ws.rs.core.Response; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Accessors; + +@Data +@Accessors(fluent = true) +@RequiredArgsConstructor +public class ResponseHolder { + + private String contentType; + private String fileName; + private final Response.Status status; + + private Object entity; + + private List
headers = new ArrayList<>(); + + public ResponseHolder addHeader(String key, String value) { + headers.add(new Header(key, value)); + return this; + } + +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java new file mode 100644 index 000000000..3cc509334 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service.export; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; +import lombok.RequiredArgsConstructor; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.config.FineractProperties; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter; +import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; + +@RequiredArgsConstructor +public class S3DatatableReportExportServiceImpl implements DatatableReportExportService { + + public static final int AWS_S3_MAXIMUM_KEY_LENGTH = 1024; + private final ReadReportingService readExtraDataAndReportingService; + + private final ConfigurationDomainService configurationDomainService; + private final S3Client s3Client; + + private final FineractProperties properties; + + @Override + public ResponseHolder export(String reportName, MultivaluedMap queryParams, Map reportParams, + boolean isSelfServiceUserReport, String parameterTypeValue) { + try { + StreamingOutput output = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams, + isSelfServiceUserReport); + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + output.write(byteArrayOutputStream); + String folder = configurationDomainService.retrieveReportExportS3FolderName(); + String filePath = DatatableExportUtil.generateS3DatatableExportFileName(AWS_S3_MAXIMUM_KEY_LENGTH, folder, "csv", + reportName, reportParams); + s3Client.putObject( + builder -> builder.bucket(properties.getReport().getExport().getS3().getBucketName()).key(filePath).build(), + RequestBody.fromBytes(byteArrayOutputStream.toByteArray())); + return new ResponseHolder(Response.Status.NO_CONTENT); + } + } catch (IOException e) { + throw new IllegalStateException("Error while exporting to S3", e); + } + } + + @Override + public boolean supports(DatatableExportTargetParameter exportType) { + return DatatableExportTargetParameter.S3 == exportType; + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java index 406157f54..e4fd348af 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java @@ -19,16 +19,20 @@ package org.apache.fineract.infrastructure.report.service; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; +import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType; import org.apache.fineract.infrastructure.security.utils.SQLInjectionValidator; public interface ReportingProcessService { Response processRequest(String reportName, MultivaluedMap queryParams); + List getAvailableExportTargets(); + default Map getReportParams(final MultivaluedMap queryParams) { final Map reportParams = new HashMap<>(); final Set keys = queryParams.keySet(); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java new file mode 100644 index 000000000..1458c9391 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.s3; + +import java.util.List; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.config.FineractProperties; +import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; +import org.apache.fineract.infrastructure.dataqueries.service.export.S3DatatableReportExportServiceImpl; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +@Configuration +@Conditional(AmazonS3ConfigCondition.class) +public class AmazonS3Config { + + @Bean + public DefaultCredentialsProvider awsCredentialsProvider() { + return DefaultCredentialsProvider.create(); + } + + @Bean + public AwsRegionProvider awsRegionProvider() { + return DefaultAwsRegionProviderChain.builder().build(); + } + + @Bean("s3Client") + public S3Client s3Client(DefaultCredentialsProvider awsCredentialsProvider, AwsRegionProvider awsRegionProvider, + List customizers) { + S3ClientBuilder builder = S3Client.builder().credentialsProvider(awsCredentialsProvider).region(awsRegionProvider.getRegion()); + customizers.forEach(customizer -> customizer.customize(builder)); + return builder.build(); + } + + @Bean + @ConditionalOnBean(S3Client.class) + public S3DatatableReportExportServiceImpl s3DatatableReportExportServiceImpl(ReadReportingService reportServiceImpl, + ConfigurationDomainService configurationDomainService, S3Client s3Client, FineractProperties fineractProperties) { + return new S3DatatableReportExportServiceImpl(reportServiceImpl, configurationDomainService, s3Client, fineractProperties); + } + +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java new file mode 100644 index 000000000..932b1bc94 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.s3; + +import org.apache.commons.lang3.StringUtils; +import org.apache.fineract.infrastructure.core.condition.PropertiesCondition; +import org.apache.fineract.infrastructure.core.config.FineractProperties; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; + +public class AmazonS3ConfigCondition extends PropertiesCondition { + + @Override + protected boolean matches(FineractProperties properties) { + FineractProperties.FineractExportS3Properties s3ReportExportProperties = properties.getReport().getExport().getS3(); + return s3ReportExportProperties.getEnabled() && StringUtils.isNotBlank(s3ReportExportProperties.getBucketName()) + && isAwsCredentialValid(); + } + + private boolean isAwsCredentialValid() { + try { + DefaultCredentialsProvider.create().resolveCredentials(); + DefaultAwsRegionProviderChain.builder().build().getRegion(); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java new file mode 100644 index 000000000..6b764aa6a --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.s3; + +import java.net.URI; +import lombok.RequiredArgsConstructor; +import org.apache.poi.util.StringUtil; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +@Component +@RequiredArgsConstructor +@Profile("test") +public class LocalstackS3ClientCustomizer implements S3ClientCustomizer { + + private final Environment environment; + + @Override + public void customize(S3ClientBuilder builder) { + String env = environment.getProperty("AWS_ENDPOINT_URL", ""); + if (StringUtil.isNotBlank(env)) { + builder.endpointOverride(URI.create(env)).forcePathStyle(true); + } + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java new file mode 100644 index 000000000..14821aa23 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.s3; + +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +public interface S3ClientCustomizer { + + void customize(S3ClientBuilder builder); +} diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties index a322b44b4..c7f8078ab 100644 --- a/fineract-provider/src/main/resources/application.properties +++ b/fineract-provider/src/main/resources/application.properties @@ -86,6 +86,10 @@ fineract.content.s3.bucketName=${FINERACT_CONTENT_S3_BUCKET_NAME:} fineract.content.s3.accessKey=${FINERACT_CONTENT_S3_ACCESS_KEY:} fineract.content.s3.secretKey=${FINERACT_CONTENT_S3_SECRET_KEY:} + +fineract.report.export.s3.bucket=${FINERACT_REPORT_EXPORT_S3_BUCKET_NAME:} +fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false} + # Logging pattern for the console logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}} diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml index 21ea1ef16..7786690da 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml @@ -109,4 +109,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml new file mode 100644 index 000000000..1050bfc44 --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java new file mode 100644 index 000000000..1ff9c2944 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.service; + +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; +import org.apache.fineract.infrastructure.dataqueries.service.export.DatatableExportUtil; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +public class DatatableExportUtilTest { + + @Test + public void emptyFolderTest() { + Assert.assertEquals("", DatatableExportUtil.normalizeFolderName("")); + Assert.assertEquals("", DatatableExportUtil.normalizeFolderName("/")); + Assert.assertEquals("", DatatableExportUtil.normalizeFolderName(null)); + } + + @Test + public void specialCharacterFolderTest() { + Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("Á")); + Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("=")); + Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("\\")); + Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("@")); + } + + @Test + public void normalizedFolderNameTest() { + Assert.assertEquals("$", DatatableExportUtil.normalizeFolderName("$")); + Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("reports")); + Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("reports/")); + Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("/reports/")); + Assert.assertEquals("reports/content", DatatableExportUtil.normalizeFolderName("reports/content")); + Assert.assertEquals("reports/content", DatatableExportUtil.normalizeFolderName("reports/////content")); + } + + @Test + public void generateDatatableExportFileNameSuccessTest() { + String reportName = "reportName"; + Map reportParams = Collections.synchronizedSortedMap(new TreeMap<>(Map.of("param1", "value1", "param2", "value2"))); + String fileName = DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder", "csv", reportName, reportParams); + Assert.assertTrue(fileName.matches("folder/reportName\\(param1_value1;param2_value2\\)_\\d{14}.csv")); + } + + @Test + public void generateDatatableExportFileNameComplexTest() { + String reportName = "reportName"; + Map reportParams = Collections.synchronizedSortedMap(new TreeMap<>(Map.of("param1", "value1", "param2", "value2"))); + Assert.assertTrue(DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name///", "csv", reportName, reportParams) + .matches("folder/name/reportName\\(param1_value1;param2_value2\\)_\\d{14}.csv")); + IllegalArgumentException folderTooLongException = Assert.assertThrows(IllegalArgumentException.class, () -> { + DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", "csv", reportName, reportParams); + }); + Assert.assertEquals("The folder name is too long", folderTooLongException.getMessage()); + + IllegalArgumentException maximumLengthException = Assert.assertThrows(IllegalArgumentException.class, () -> { + DatatableExportUtil.generateS3DatatableExportFileName(29, "folder///name/", "csv", reportName, reportParams); + }); + Assert.assertEquals("The maximum length must be greater than 30", maximumLengthException.getMessage()); + + IllegalArgumentException extensionRequired = Assert.assertThrows(IllegalArgumentException.class, () -> { + DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", null, reportName, reportParams); + }); + Assert.assertEquals("The extension is required", extensionRequired.getMessage()); + + IllegalArgumentException reportNameRequired = Assert.assertThrows(IllegalArgumentException.class, () -> { + DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", "csv", null, reportParams); + }); + Assert.assertEquals("The report name is required", reportNameRequired.getMessage()); + + Assert.assertTrue(DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name/", ".csv", reportName, null) + .matches("folder/name/reportName_\\d{14}.csv")); + + Assert.assertTrue( + DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name/", "csv", "report/with/slash", reportParams) + .matches("folder/name/report_with_slash\\(param1_value1;param2_value2\\)_\\d{14}.csv")); + } + +} diff --git a/fineract-provider/src/test/resources/application-test.properties b/fineract-provider/src/test/resources/application-test.properties index 492a2aa65..3e2f22432 100644 --- a/fineract-provider/src/test/resources/application-test.properties +++ b/fineract-provider/src/test/resources/application-test.properties @@ -70,6 +70,8 @@ fineract.content.s3.enabled=false fineract.content.s3.bucketName= fineract.content.s3.accessKey= fineract.content.s3.secretKey= +fineract.report.export.s3.bucket=${FINERACT_REPORT_EXPORT_S3_BUCKET_NAME:} +fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false} management.health.jms.enabled=false diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java new file mode 100644 index 000000000..47e3d6649 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.integrationtests; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.TYPE, ElementType.METHOD }) +@EnabledIfEnvironmentVariable(named = "GITHUB_ACTIONS", matches = "true") +public @interface CIOnly {} diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java new file mode 100644 index 000000000..3de4ddbc3 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.integrationtests.client; + +import java.io.IOException; +import java.util.Map; +import okhttp3.MediaType; +import okhttp3.ResponseBody; +import org.apache.fineract.integrationtests.CIOnly; +import org.apache.fineract.integrationtests.common.Utils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import retrofit2.Response; + +/** + * Integration Test for /runreports/ API. + * + * @author Michael Vorburger.ch + */ +public class ReportExportTest extends IntegrationTest { + + @BeforeEach + public void setup() { + Utils.initializeRESTAssured(); + } + + @Test + void runClientListingTableReportCSV() throws IOException { + Response result = okR( + fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false)); + assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv")); + assertThat(result.body().string()).contains("Office/Branch"); + } + + @Test + @CIOnly + void runClientListingTableReportS3() throws IOException { + Response result = okR( + fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportS3", "true"), false)); + assertThat(result.code()).isEqualTo(204); + } + +} diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java index 72bfcef1d..7c538c4f1 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java @@ -27,6 +27,7 @@ import org.apache.fineract.integrationtests.common.Utils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import retrofit2.Response; /** * Integration Test for /runreports/ API. @@ -51,6 +52,14 @@ public class ReportsTest extends IntegrationTest { .getColumnName()).isEqualTo("Office/Branch"); } + @Test + void runClientListingTableReportCSV() throws IOException { + Response result = okR( + fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false)); + assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv")); + assertThat(result.body().string()).contains("Office/Branch"); + } + @Test // see FINERACT-1306 void runReportCategory() throws IOException { // Using raw OkHttp instead of Retrofit API here, because /runreports/reportCategoryList returns JSON Array - diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java index a92e8246b..6ba54e4c7 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java @@ -96,9 +96,9 @@ public class GlobalConfigurationHelper { // If any other column is modified by the integration test suite in // the future, it needs to be reset here. final Integer configDefaultId = (Integer) defaultGlobalConfiguration.get("id"); - final Integer configDefaultValue = (Integer) defaultGlobalConfiguration.get("value"); + final String configDefaultValue = String.valueOf(defaultGlobalConfiguration.get("value")); - updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue.toString()); + updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue); updateEnabledFlagForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), (Boolean) defaultGlobalConfiguration.get("enabled")); changedNo++; @@ -119,9 +119,9 @@ public class GlobalConfigurationHelper { ArrayList expectedGlobalConfigurations = getAllDefaultGlobalConfigurations(); ArrayList actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec); - // There are currently 48 global configurations. - Assertions.assertEquals(48, expectedGlobalConfigurations.size()); - Assertions.assertEquals(48, actualGlobalConfigurations.size()); + // There are currently 49 global configurations. + Assertions.assertEquals(49, expectedGlobalConfigurations.size()); + Assertions.assertEquals(49, actualGlobalConfigurations.size()); for (int i = 0; i < expectedGlobalConfigurations.size(); i++) { @@ -540,6 +540,14 @@ public class GlobalConfigurationHelper { externalEventBatchSize.put("enabled", false); externalEventBatchSize.put("trapDoor", false); defaults.add(externalEventBatchSize); + + HashMap reportExportS3FolderName = new HashMap<>(); + reportExportS3FolderName.put("id", 54); + reportExportS3FolderName.put("name", "report-export-s3-folder-name"); + reportExportS3FolderName.put("value", 0); + reportExportS3FolderName.put("enabled", false); + reportExportS3FolderName.put("trapDoor", false); + defaults.add(reportExportS3FolderName); return defaults; }