FINERACT-1794: Address improvement in file upload APIs

This commit is contained in:
Aleksandar Vidakovic
2022-11-23 20:28:50 +01:00
committed by Aleksandar Vidakovic
parent c1df5e73a2
commit 5338263bd8
22 changed files with 532 additions and 102 deletions
@@ -66,6 +66,38 @@ dependencyManagement {
dependency 'com.github.spullara.mustache.java:compiler:0.9.10' dependency 'com.github.spullara.mustache.java:compiler:0.9.10'
dependency 'com.jayway.jsonpath:json-path:2.7.0' dependency 'com.jayway.jsonpath:json-path:2.7.0'
dependency 'org.apache.tika:tika-core:2.6.0' dependency 'org.apache.tika:tika-core:2.6.0'
dependency ('org.apache.tika:tika-parser-microsoft-module:2.6.0') {
exclude 'org.bouncycastle:bcprov-jdk15on'
exclude 'org.bouncycastle:bcmail-jdk15on'
exclude 'commons-logging:commons-logging'
exclude 'org.apache.logging.log4j:log4j-api'
exclude 'org.slf4j:slf4j-api'
exclude 'commons-io:commons-io'
exclude 'commons-codec:commons-codec'
exclude 'org.apache.commons:commons-compress'
exclude 'org.apache.commons:commons-lang3'
exclude 'org.apache.poi:poi'
exclude 'org.apache.poi:poi-scratchpad'
exclude 'org.glassfish.jaxb:jaxb-runtime'
exclude 'org.apache.commons:commons-compress'
exclude 'xml-apis:xml-apis'
}
dependency ('org.apache.tika:tika-parser-miscoffice-module:2.6.0') {
exclude 'org.bouncycastle:bcprov-jdk15on'
exclude 'org.bouncycastle:bcmail-jdk15on'
exclude 'commons-logging:commons-logging'
exclude 'org.apache.logging.log4j:log4j-api'
exclude 'org.slf4j:slf4j-api'
exclude 'commons-io:commons-io'
exclude 'commons-codec:commons-codec'
exclude 'org.apache.commons:commons-compress'
exclude 'org.apache.commons:commons-lang3'
exclude 'org.apache.poi:poi'
exclude 'org.apache.poi:poi-scratchpad'
exclude 'org.glassfish.jaxb:jaxb-runtime'
exclude 'org.apache.commons:commons-compress'
exclude 'xml-apis:xml-apis'
}
dependency 'org.apache.httpcomponents:httpclient:4.5.13' dependency 'org.apache.httpcomponents:httpclient:4.5.13'
dependency 'jakarta.management.j2ee:jakarta.management.j2ee-api:1.1.4' dependency 'jakarta.management.j2ee:jakarta.management.j2ee-api:1.1.4'
dependency 'jakarta.jms:jakarta.jms-api:2.0.3' dependency 'jakarta.jms:jakarta.jms-api:2.0.3'
+2
View File
@@ -63,6 +63,8 @@ dependencies {
'org.apache.poi:poi', 'org.apache.poi:poi',
'org.apache.poi:poi-ooxml', 'org.apache.poi:poi-ooxml',
'org.apache.tika:tika-core', 'org.apache.tika:tika-core',
'org.apache.tika:tika-parser-microsoft-module',
'org.apache.tika:tika-parser-miscoffice-module',
'org.liquibase:liquibase-core', 'org.liquibase:liquibase-core',
@@ -18,6 +18,7 @@
*/ */
package org.apache.fineract.infrastructure.bulkimport.service; package org.apache.fineract.infrastructure.bulkimport.service;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
@@ -87,14 +88,13 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
final String dateFormat) { final String dateFormat) {
try { try {
if (entity != null && inputStream != null && fileDetail != null && locale != null && dateFormat != null) { if (entity != null && inputStream != null && fileDetail != null && locale != null && dateFormat != null) {
final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.copy(inputStream, baos); IOUtils.copy(inputStream, baos);
final byte[] bytes = baos.toByteArray(); final byte[] bytes = baos.toByteArray();
InputStream clonedInputStream = new ByteArrayInputStream(bytes); InputStream clonedInputStream = new ByteArrayInputStream(bytes);
InputStream clonedInputStreamWorkbook = new ByteArrayInputStream(bytes); final BufferedInputStream bis = new BufferedInputStream(new ByteArrayInputStream(bytes));
final Tika tika = new Tika(); final Tika tika = new Tika();
final TikaInputStream tikaInputStream = TikaInputStream.get(clonedInputStream); final TikaInputStream tikaInputStream = TikaInputStream.get(bis);
final String fileType = tika.detect(tikaInputStream); final String fileType = tika.detect(tikaInputStream);
if (!fileType.contains("msoffice") && !fileType.contains("application/vnd.ms-excel")) { if (!fileType.contains("msoffice") && !fileType.contains("application/vnd.ms-excel")) {
// We had a problem where we tried to upload the downloaded // We had a problem where we tried to upload the downloaded
@@ -104,7 +104,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
"Uploaded file extension is not recognized."); "Uploaded file extension is not recognized.");
} }
Workbook workbook = new HSSFWorkbook(clonedInputStreamWorkbook); Workbook workbook = new HSSFWorkbook(clonedInputStream);
GlobalEntityType entityType = null; GlobalEntityType entityType = null;
int primaryColumn = 0; int primaryColumn = 0;
if (entity.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { if (entity.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) {
@@ -169,7 +169,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
throw new GeneralPlatformDomainRuleException("error.msg.unable.to.find.resource", "Unable to find requested resource"); throw new GeneralPlatformDomainRuleException("error.msg.unable.to.find.resource", "Unable to find requested resource");
} }
return publishEvent(primaryColumn, fileDetail, clonedInputStreamWorkbook, entityType, workbook, locale, dateFormat); return publishEvent(primaryColumn, fileDetail, bis, entityType, workbook, locale, dateFormat);
} }
throw new GeneralPlatformDomainRuleException("error.msg.null", "One or more of the given parameters not found"); throw new GeneralPlatformDomainRuleException("error.msg.null", "One or more of the given parameters not found");
} catch (IOException e) { } catch (IOException e) {
@@ -21,6 +21,7 @@ package org.apache.fineract.infrastructure.campaigns.jobs.executeemail;
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailCampaignRepository; import org.apache.fineract.infrastructure.campaigns.email.domain.EmailCampaignRepository;
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRepository; import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRepository;
import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService; import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator; import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator;
@@ -57,6 +58,9 @@ public class ExecuteEmailConfig {
@Autowired @Autowired
private ReportMailingJobValidator reportMailingJobValidator; private ReportMailingJobValidator reportMailingJobValidator;
@Autowired
private FineractProperties fineractProperties;
@Bean @Bean
protected Step executeEmailStep() { protected Step executeEmailStep() {
return steps.get(JobName.EXECUTE_EMAIL.name()).tasklet(executeEmailTasklet()).build(); return steps.get(JobName.EXECUTE_EMAIL.name()).tasklet(executeEmailTasklet()).build();
@@ -70,6 +74,6 @@ public class ExecuteEmailConfig {
@Bean @Bean
public ExecuteEmailTasklet executeEmailTasklet() { public ExecuteEmailTasklet executeEmailTasklet() {
return new ExecuteEmailTasklet(emailMessageRepository, emailCampaignRepository, loanRepository, savingsAccountRepository, return new ExecuteEmailTasklet(emailMessageRepository, emailCampaignRepository, loanRepository, savingsAccountRepository,
emailMessageJobEmailService, readReportingService, reportMailingJobValidator); emailMessageJobEmailService, readReportingService, reportMailingJobValidator, fineractProperties);
} }
} }
@@ -39,10 +39,10 @@ import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRep
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageStatusType; import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageStatusType;
import org.apache.fineract.infrastructure.campaigns.email.domain.ScheduledEmailAttachmentFileFormat; import org.apache.fineract.infrastructure.campaigns.email.domain.ScheduledEmailAttachmentFileFormat;
import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService; import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException; import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException;
import org.apache.fineract.infrastructure.dataqueries.domain.Report; import org.apache.fineract.infrastructure.dataqueries.domain.Report;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.reportmailingjob.helper.IPv4Helper; import org.apache.fineract.infrastructure.reportmailingjob.helper.IPv4Helper;
import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator; import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator;
import org.apache.fineract.portfolio.client.domain.Client; import org.apache.fineract.portfolio.client.domain.Client;
@@ -66,6 +66,7 @@ public class ExecuteEmailTasklet implements Tasklet {
private final EmailMessageJobEmailService emailMessageJobEmailService; private final EmailMessageJobEmailService emailMessageJobEmailService;
private final ReadReportingService readReportingService; private final ReadReportingService readReportingService;
private final ReportMailingJobValidator reportMailingJobValidator; private final ReportMailingJobValidator reportMailingJobValidator;
private final FineractProperties fineractProperties;
@Override @Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
@@ -205,7 +206,7 @@ public class ExecuteEmailTasklet implements Tasklet {
try { try {
final ByteArrayOutputStream byteArrayOutputStream = readReportingService.generatePentahoReportAsOutputStream(reportName, final ByteArrayOutputStream byteArrayOutputStream = readReportingService.generatePentahoReportAsOutputStream(reportName,
emailAttachmentFileFormat.getValue(), reportParams, null, emailCampaign.getApprovedBy(), errorLog); emailAttachmentFileFormat.getValue(), reportParams, null, emailCampaign.getApprovedBy(), errorLog);
final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
final String fileNameWithoutExtension = fileLocation + File.separator + reportName; final String fileNameWithoutExtension = fileLocation + File.separator + reportName;
if (!new File(fileLocation).isDirectory()) { if (!new File(fileLocation).isDirectory()) {
new File(fileLocation).mkdirs(); new File(fileLocation).mkdirs();
@@ -18,6 +18,7 @@
*/ */
package org.apache.fineract.infrastructure.campaigns.jobs.executereportmailingjobs; package org.apache.fineract.infrastructure.campaigns.jobs.executereportmailingjobs;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider; import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider;
@@ -54,6 +55,8 @@ public class ExecuteReportMailingJobsConfig {
private ReportMailingJobEmailService reportMailingJobEmailService; private ReportMailingJobEmailService reportMailingJobEmailService;
@Autowired @Autowired
private ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository; private ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository;
@Autowired
private FineractProperties fineractProperties;
@Bean @Bean
protected Step executeReportMailingJobsStep() { protected Step executeReportMailingJobsStep() {
@@ -69,6 +72,6 @@ public class ExecuteReportMailingJobsConfig {
@Bean @Bean
public ExecuteReportMailingJobsTasklet executeReportMailingJobsTasklet() { public ExecuteReportMailingJobsTasklet executeReportMailingJobsTasklet() {
return new ExecuteReportMailingJobsTasklet(reportMailingJobRepository, reportMailingJobValidator, readReportingService, return new ExecuteReportMailingJobsTasklet(reportMailingJobRepository, reportMailingJobValidator, readReportingService,
reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository); reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository, fineractProperties);
} }
} }
@@ -34,10 +34,10 @@ import javax.ws.rs.core.Response;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.core.service.DateUtils;
import org.apache.fineract.infrastructure.dataqueries.domain.Report; import org.apache.fineract.infrastructure.dataqueries.domain.Report;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider; import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider;
import org.apache.fineract.infrastructure.report.service.ReportingProcessService; import org.apache.fineract.infrastructure.report.service.ReportingProcessService;
import org.apache.fineract.infrastructure.reportmailingjob.data.ReportMailingJobEmailAttachmentFileFormat; import org.apache.fineract.infrastructure.reportmailingjob.data.ReportMailingJobEmailAttachmentFileFormat;
@@ -68,6 +68,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet {
private final ReportingProcessServiceProvider reportingProcessServiceProvider; private final ReportingProcessServiceProvider reportingProcessServiceProvider;
private final ReportMailingJobEmailService reportMailingJobEmailService; private final ReportMailingJobEmailService reportMailingJobEmailService;
private final ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository; private final ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository;
private final FineractProperties fineractProperties;
private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
@@ -133,7 +134,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet {
if (responseObject != null && responseObject.getClass().equals(ByteArrayOutputStream.class)) { if (responseObject != null && responseObject.getClass().equals(ByteArrayOutputStream.class)) {
final ByteArrayOutputStream byteArrayOutputStream = (ByteArrayOutputStream) responseObject; final ByteArrayOutputStream byteArrayOutputStream = (ByteArrayOutputStream) responseObject;
final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
final String fileNameWithoutExtension = fileLocation + File.separator + reportName; final String fileNameWithoutExtension = fileLocation + File.separator + reportName;
if (!new File(fileLocation).isDirectory()) { if (!new File(fileLocation).isDirectory()) {
@@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.config;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Slf4j
@Configuration
public class ContentS3Config {
@Bean
@ConditionalOnProperty("fineract.content.s3.enabled")
public AmazonS3 contentS3Client(FineractProperties fineractProperties) {
return AmazonS3ClientBuilder.standard()
.withCredentials(
new AWSStaticCredentialsProvider(new BasicAWSCredentials(fineractProperties.getContent().getS3().getAccessKey(),
fineractProperties.getContent().getS3().getSecretKey())))
.build();
}
}
@@ -45,6 +45,8 @@ public class FineractProperties {
private FineractEventsProperties events; private FineractEventsProperties events;
private FineractContentProperties content;
@Getter @Getter
@Setter @Setter
public static class FineractTenantProperties { public static class FineractTenantProperties {
@@ -155,4 +157,32 @@ public class FineractProperties {
private String eventQueueName; private String eventQueueName;
private String brokerUrl; private String brokerUrl;
} }
@Getter
@Setter
public static class FineractContentProperties {
private boolean regexWhitelistEnabled;
private List<String> regexWhitelist;
private boolean mimeWhitelistEnabled;
private List<String> mimeWhitelist;
private FineractContentFilesystemProperties filesystem;
private FineractContentS3Properties s3;
}
@Getter
@Setter
public static class FineractContentFilesystemProperties {
private String rootFolder;
}
@Getter
@Setter
public static class FineractContentS3Properties {
private String bucketName;
private String accessKey;
private String secretKey;
}
} }
@@ -40,6 +40,7 @@ import javax.ws.rs.core.StreamingOutput;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.JdbcSupport; import org.apache.fineract.infrastructure.core.domain.JdbcSupport;
import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException; import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException;
import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator; import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
@@ -50,7 +51,6 @@ import org.apache.fineract.infrastructure.dataqueries.data.ReportParameterJoinDa
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
import org.apache.fineract.infrastructure.dataqueries.exception.ReportNotFoundException; import org.apache.fineract.infrastructure.dataqueries.exception.ReportNotFoundException;
import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
import org.apache.fineract.infrastructure.security.service.SqlInjectionPreventerService; import org.apache.fineract.infrastructure.security.service.SqlInjectionPreventerService;
import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil; import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil;
@@ -72,6 +72,7 @@ public class ReadReportingServiceImpl implements ReadReportingService {
private final GenericDataService genericDataService; private final GenericDataService genericDataService;
private final SqlInjectionPreventerService sqlInjectionPreventerService; private final SqlInjectionPreventerService sqlInjectionPreventerService;
private final DatabaseSpecificSQLGenerator sqlGenerator; private final DatabaseSpecificSQLGenerator sqlGenerator;
private final FineractProperties fineractProperties;
@Override @Override
public StreamingOutput retrieveReportCSV(final String name, final String type, final Map<String, String> queryParams, public StreamingOutput retrieveReportCSV(final String name, final String type, final Map<String, String> queryParams,
@@ -240,7 +241,7 @@ public class ReadReportingServiceImpl implements ReadReportingService {
public String retrieveReportPDF(final String reportName, final String type, final Map<String, String> queryParams, public String retrieveReportPDF(final String reportName, final String type, final Map<String, String> queryParams,
final boolean isSelfServiceUserReport) { final boolean isSelfServiceUserReport) {
final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
if (!new File(fileLocation).isDirectory()) { if (!new File(fileLocation).isDirectory()) {
new File(fileLocation).mkdirs(); new File(fileLocation).mkdirs();
} }
@@ -36,6 +36,8 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam; import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import org.apache.fineract.infrastructure.core.data.UploadRequest; import org.apache.fineract.infrastructure.core.data.UploadRequest;
@@ -54,19 +56,16 @@ import org.apache.fineract.portfolio.client.data.ClientData;
import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam; import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Slf4j
@RequiredArgsConstructor
@Component @Component
@Scope("singleton") @Scope("singleton")
@Path("{entity}/{entityId}/images") @Path("{entity}/{entityId}/images")
public class ImagesApiResource { public class ImagesApiResource {
private static final Logger LOG = LoggerFactory.getLogger(ImagesApiResource.class);
private final PlatformSecurityContext context; private final PlatformSecurityContext context;
private final ImageReadPlatformService imageReadPlatformService; private final ImageReadPlatformService imageReadPlatformService;
private final ImageWritePlatformService imageWritePlatformService; private final ImageWritePlatformService imageWritePlatformService;
@@ -74,18 +73,6 @@ public class ImagesApiResource {
private final FileUploadValidator fileUploadValidator; private final FileUploadValidator fileUploadValidator;
private final ImageResizer imageResizer; private final ImageResizer imageResizer;
@Autowired
public ImagesApiResource(final PlatformSecurityContext context, final ImageReadPlatformService readPlatformService,
final ImageWritePlatformService imageWritePlatformService, final DefaultToApiJsonSerializer<ClientData> toApiJsonSerializer,
final FileUploadValidator fileUploadValidator, final ImageResizer imageResizer) {
this.context = context;
this.imageReadPlatformService = readPlatformService;
this.imageWritePlatformService = imageWritePlatformService;
this.toApiJsonSerializer = toApiJsonSerializer;
this.fileUploadValidator = fileUploadValidator;
this.imageResizer = imageResizer;
}
/** /**
* Upload images through multi-part form upload * Upload images through multi-part form upload
*/ */
@@ -172,7 +159,7 @@ public class ImagesApiResource {
final String clientImageAsBase64Text = imageDataURISuffix + Base64.getMimeEncoder().encodeToString(resizedImageBytes); final String clientImageAsBase64Text = imageDataURISuffix + Base64.getMimeEncoder().encodeToString(resizedImageBytes);
return Response.ok(clientImageAsBase64Text, MediaType.TEXT_PLAIN_TYPE).build(); return Response.ok(clientImageAsBase64Text, MediaType.TEXT_PLAIN_TYPE).build();
} else { } else {
LOG.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId, log.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId,
maxWidth, maxHeight); maxWidth, maxHeight);
return Response.serverError().build(); return Response.serverError().build();
} }
@@ -0,0 +1,28 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import java.io.BufferedInputStream;
public interface ContentPathSanitizer {
String sanitize(String path);
String sanitize(String path, BufferedInputStream is);
}
@@ -18,40 +18,28 @@
*/ */
package org.apache.fineract.infrastructure.documentmanagement.contentrepository; package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import java.util.List;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.fineract.infrastructure.configuration.data.S3CredentialsData;
import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
import org.apache.fineract.infrastructure.configuration.service.ExternalServicesPropertiesReadPlatformService;
import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType; import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class ContentRepositoryFactory { public class ContentRepositoryFactory {
private final ApplicationContext applicationContext; // TODO: all configuration should be really moved to application.properties
private final ExternalServicesPropertiesReadPlatformService externalServicesReadPlatformService; private final ConfigurationDomainService configurationService;
private final List<ContentRepository> contentRepositories;
public ContentRepository getRepository() { public ContentRepository getRepository() {
final ConfigurationDomainService configurationDomainServiceJpa = this.applicationContext.getBean("configurationDomainServiceJpa", if (configurationService.isAmazonS3Enabled()) {
ConfigurationDomainService.class); return getRepository(StorageType.S3);
if (configurationDomainServiceJpa.isAmazonS3Enabled()) {
return createS3DocumentStore();
} }
return new FileSystemContentRepository(); return getRepository(StorageType.FILE_SYSTEM);
} }
public ContentRepository getRepository(final StorageType documentStoreType) { public ContentRepository getRepository(StorageType storageType) {
if (documentStoreType == StorageType.FILE_SYSTEM) { return contentRepositories.stream().filter(cr -> cr.getStorageType().equals(storageType)).findFirst().orElseThrow();
return new FileSystemContentRepository();
}
return createS3DocumentStore();
}
private ContentRepository createS3DocumentStore() {
final S3CredentialsData s3CredentialsData = this.externalServicesReadPlatformService.getS3Credentials();
return new S3ContentRepository(s3CredentialsData.getBucketName(), s3CredentialsData.getSecretKey(),
s3CredentialsData.getAccessKey());
} }
} }
@@ -0,0 +1,138 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import java.io.BufferedInputStream;
import java.nio.file.Path;
import java.util.List;
import java.util.regex.Pattern;
import javax.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
import org.apache.tika.Tika;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.sax.BodyContentHandler;
import org.springframework.stereotype.Component;
@Slf4j
@RequiredArgsConstructor
@Component
public class FileSystemContentPathSanitizer implements ContentPathSanitizer {
private final FineractProperties fineractProperties;
private List<Pattern> regexWhitelist;
private static Pattern OVERWRITE_SIBLING_IMAGE = Pattern.compile(".*\\.\\./+[0-9]+/+.*");
@PostConstruct
public void init() {
regexWhitelist = fineractProperties.getContent().getRegexWhitelist().stream().map(Pattern::compile).toList();
}
@Override
public String sanitize(String path) {
return sanitize(path, null);
}
@Override
public String sanitize(String path, BufferedInputStream is) {
try {
if (OVERWRITE_SIBLING_IMAGE.matcher(path).matches()) {
throw new RuntimeException(String.format("Trying to overwrite another resource's image: %s", path));
}
String sanitizedPath = Path.of(path).normalize().toString();
String fileName = FilenameUtils.getName(sanitizedPath).toLowerCase();
if (log.isDebugEnabled()) {
log.debug("Path: {} -> {} ({})", path, sanitizedPath, fileName);
}
if (fineractProperties.getContent().isRegexWhitelistEnabled()) {
boolean matches = regexWhitelist.stream().anyMatch(p -> p.matcher(fileName).matches());
if (!matches) {
throw new RuntimeException(String.format("File name not allowed: %s", fileName));
}
}
if (is != null && fineractProperties.getContent().isMimeWhitelistEnabled()) {
Tika tika = new Tika();
String extensionMimeType = tika.detect(fileName);
if (StringUtils.isEmpty(extensionMimeType)) {
throw new RuntimeException(String.format("Could not detect mime type for filename %s!", fileName));
}
if (!fineractProperties.getContent().getMimeWhitelist().contains(extensionMimeType)) {
throw new RuntimeException(
String.format("Detected mime type %s for filename %s not allowed!", extensionMimeType, fileName));
}
String contentMimeType = detectContentMimeType(is);
if (StringUtils.isEmpty(contentMimeType)) {
throw new RuntimeException(String.format("Could not detect content mime type for %s!", fileName));
}
if (!fineractProperties.getContent().getMimeWhitelist().contains(contentMimeType)) {
throw new RuntimeException(
String.format("Detected content mime type %s for %s not allowed!", contentMimeType, fileName));
}
if (!contentMimeType.equalsIgnoreCase(extensionMimeType)) {
throw new RuntimeException(String.format("Detected filename (%s) and content (%s) mime type do not match!",
extensionMimeType, contentMimeType));
}
}
Path target = Path.of(sanitizedPath);
Path rootFolder = Path.of(fineractProperties.getContent().getFilesystem().getRootFolder(),
ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim());
if (!target.startsWith(rootFolder)) {
throw new RuntimeException(String.format("Path traversal attempt: %s (%s)", target, rootFolder));
}
return sanitizedPath;
} catch (Exception e) {
throw new ContentManagementException(path, e.getMessage(), e);
}
}
private String detectContentMimeType(BufferedInputStream bis) throws Exception {
TikaInputStream tis = TikaInputStream.get(bis);
AutoDetectParser parser = new AutoDetectParser();
// NOTE: turn off write limit with "-1"
BodyContentHandler handler = new BodyContentHandler(-1);
Metadata metadata = new Metadata();
parser.parse(tis, handler, metadata);
return metadata.get("Content-Type");
}
}
@@ -19,12 +19,16 @@
package org.apache.fineract.infrastructure.documentmanagement.contentrepository; package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import com.google.common.io.Files; import com.google.common.io.Files;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Base64; import java.util.Base64;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage; import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand; import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand;
@@ -33,14 +37,17 @@ import org.apache.fineract.infrastructure.documentmanagement.data.FileData;
import org.apache.fineract.infrastructure.documentmanagement.data.ImageData; import org.apache.fineract.infrastructure.documentmanagement.data.ImageData;
import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType; import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException; import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
import org.slf4j.Logger; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component;
@Slf4j
@RequiredArgsConstructor
@Component
@ConditionalOnProperty("fineract.content.filesystem.enabled")
public class FileSystemContentRepository implements ContentRepository { public class FileSystemContentRepository implements ContentRepository {
private static final Logger LOG = LoggerFactory.getLogger(FileSystemContentRepository.class); private final FileSystemContentPathSanitizer pathSanitizer;
private final FineractProperties fineractProperties;
public static final String FINERACT_BASE_DIR = System.getProperty("user.home") + File.separator + ".fineract";
@Override @Override
public String saveFile(final InputStream uploadedInputStream, final DocumentCommand documentCommand) { public String saveFile(final InputStream uploadedInputStream, final DocumentCommand documentCommand) {
@@ -50,16 +57,14 @@ public class FileSystemContentRepository implements ContentRepository {
final String fileLocation = generateFileParentDirectory(documentCommand.getParentEntityType(), documentCommand.getParentEntityId()) final String fileLocation = generateFileParentDirectory(documentCommand.getParentEntityType(), documentCommand.getParentEntityId())
+ File.separator + fileName; + File.separator + fileName;
writeFileToFileSystem(fileName, uploadedInputStream, fileLocation); return writeFileToFileSystem(fileName, uploadedInputStream, fileLocation);
return fileLocation;
} }
@Override @Override
public String saveImage(final InputStream uploadedInputStream, final Long resourceId, final String imageName, final Long fileSize) { public String saveImage(final InputStream uploadedInputStream, final Long resourceId, final String imageName, final Long fileSize) {
ContentRepositoryUtils.validateFileSizeWithinPermissibleRange(fileSize, imageName); ContentRepositoryUtils.validateFileSizeWithinPermissibleRange(fileSize, imageName);
final String fileLocation = generateClientImageParentDirectory(resourceId) + File.separator + imageName; final String fileLocation = generateClientImageParentDirectory(resourceId) + File.separator + imageName;
writeFileToFileSystem(imageName, uploadedInputStream, fileLocation); return writeFileToFileSystem(imageName, uploadedInputStream, fileLocation);
return fileLocation;
} }
@Override @Override
@@ -69,10 +74,9 @@ public class FileSystemContentRepository implements ContentRepository {
String base64EncodedImageString = base64EncodedImage.getBase64EncodedString(); String base64EncodedImageString = base64EncodedImage.getBase64EncodedString();
try { try {
final InputStream toUploadInputStream = new ByteArrayInputStream(Base64.getMimeDecoder().decode(base64EncodedImageString)); final InputStream toUploadInputStream = new ByteArrayInputStream(Base64.getMimeDecoder().decode(base64EncodedImageString));
writeFileToFileSystem(imageName, toUploadInputStream, fileLocation); return writeFileToFileSystem(imageName, toUploadInputStream, fileLocation);
return fileLocation;
} catch (IllegalArgumentException iae) { } catch (IllegalArgumentException iae) {
LOG.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae); log.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae);
throw iae; throw iae;
} }
} }
@@ -88,23 +92,29 @@ public class FileSystemContentRepository implements ContentRepository {
} }
private void deleteFileInternal(final String documentPath) { private void deleteFileInternal(final String documentPath) {
final File fileToBeDeleted = new File(documentPath); String sanitizedPath = pathSanitizer.sanitize(documentPath);
final File fileToBeDeleted = new File(sanitizedPath);
final boolean fileDeleted = fileToBeDeleted.delete(); final boolean fileDeleted = fileToBeDeleted.delete();
if (!fileDeleted) { if (!fileDeleted) {
// no need to throw an Error, what's a caller going to do about it, so simply log a warning // no need to throw an Error, what's a caller going to do about it, so simply log a warning
LOG.warn("Unable to delete file {}", documentPath); log.warn("Unable to delete file {}", documentPath);
} }
} }
@Override @Override
public FileData fetchFile(final DocumentData documentData) { public FileData fetchFile(final DocumentData documentData) {
final File file = new File(documentData.fileLocation()); String sanitizedPath = pathSanitizer.sanitize(documentData.fileLocation());
final File file = new File(sanitizedPath);
return new FileData(Files.asByteSource(file), documentData.fileName(), documentData.contentType()); return new FileData(Files.asByteSource(file), documentData.fileName(), documentData.contentType());
} }
@Override @Override
public FileData fetchImage(final ImageData imageData) { public FileData fetchImage(final ImageData imageData) {
final File file = new File(imageData.location()); String sanitizedPath = pathSanitizer.sanitize(imageData.location());
final File file = new File(sanitizedPath);
return new FileData(Files.asByteSource(file), imageData.getEntityDisplayName(), imageData.contentType().getValue()); return new FileData(Files.asByteSource(file), imageData.getEntityDisplayName(), imageData.contentType().getValue());
} }
@@ -117,16 +127,17 @@ public class FileSystemContentRepository implements ContentRepository {
* Generate the directory path for storing the new document * Generate the directory path for storing the new document
*/ */
private String generateFileParentDirectory(final String entityType, final Long entityId) { private String generateFileParentDirectory(final String entityType, final Long entityId) {
return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator
+ ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "documents" + File.separator + ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "documents" + File.separator
+ entityType + File.separator + entityId + File.separator + ContentRepositoryUtils.generateRandomString(); + entityType + File.separator + entityId + File.separator + ContentRepositoryUtils.generateRandomString();
} }
/** /**
* Generate directory path for storing new Image * Generate ContentRepositoryUtilsfineractProperties.getContentgetWhitelist()getBlacklist() path for storing new
* Image
*/ */
private String generateClientImageParentDirectory(final Long resourceId) { private String generateClientImageParentDirectory(final Long resourceId) {
return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator
+ ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "images" + File.separator + ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "images" + File.separator
+ "clients" + File.separator + resourceId; + "clients" + File.separator + resourceId;
} }
@@ -135,16 +146,18 @@ public class FileSystemContentRepository implements ContentRepository {
* Recursively create the directory if it does not exist. * Recursively create the directory if it does not exist.
*/ */
private void makeDirectories(final String uploadDocumentLocation) throws IOException { private void makeDirectories(final String uploadDocumentLocation) throws IOException {
Files.createParentDirs(new File(uploadDocumentLocation)); String sanitizedPath = pathSanitizer.sanitize(uploadDocumentLocation);
Files.createParentDirs(new File(sanitizedPath));
} }
private void writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) { private String writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) {
try { try (BufferedInputStream bis = new BufferedInputStream(uploadedInputStream)) {
makeDirectories(fileLocation); String sanitizedPath = pathSanitizer.sanitize(fileLocation, bis);
FileUtils.copyInputStreamToFile(uploadedInputStream, new File(fileLocation)); // NOSONAR makeDirectories(sanitizedPath);
FileUtils.copyInputStreamToFile(bis, new File(sanitizedPath)); // NOSONAR
return sanitizedPath;
} catch (final IOException ioException) { } catch (final IOException ioException) {
LOG.warn("writeFileToFileSystem() IOException (logged because cause is not propagated in ContentManagementException)", log.warn("Failed to write file!", ioException);
ioException);
throw new ContentManagementException(fileName, ioException.getMessage(), ioException); throw new ContentManagementException(fileName, ioException.getMessage(), ioException);
} }
} }
@@ -20,10 +20,7 @@ package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException; import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -35,7 +32,9 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Base64; import java.util.Base64;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage; import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage;
import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand; import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand;
import org.apache.fineract.infrastructure.documentmanagement.data.DocumentData; import org.apache.fineract.infrastructure.documentmanagement.data.DocumentData;
@@ -45,18 +44,17 @@ import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException; import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
import org.apache.fineract.infrastructure.documentmanagement.exception.DocumentNotFoundException; import org.apache.fineract.infrastructure.documentmanagement.exception.DocumentNotFoundException;
import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil; import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Slf4j @Slf4j
@RequiredArgsConstructor
@Component
@ConditionalOnProperty("fineract.content.s3.enabled")
public class S3ContentRepository implements ContentRepository { public class S3ContentRepository implements ContentRepository {
private final String s3BucketName;
private final AmazonS3 s3Client; private final AmazonS3 s3Client;
private final FineractProperties fineractProperties;
public S3ContentRepository(final String bucketName, final String secretKey, final String accessKey) {
this.s3BucketName = bucketName;
this.s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))).build();
}
@Override @Override
public String saveFile(final InputStream toUpload, final DocumentCommand documentCommand) { public String saveFile(final InputStream toUpload, final DocumentCommand documentCommand) {
@@ -142,7 +140,7 @@ public class S3ContentRepository implements ContentRepository {
private void deleteObject(final String location) { private void deleteObject(final String location) {
try { try {
this.s3Client.deleteObject(new DeleteObjectRequest(this.s3BucketName, location)); this.s3Client.deleteObject(new DeleteObjectRequest(fineractProperties.getContent().getS3().getBucketName(), location));
} catch (final AmazonServiceException ase) { } catch (final AmazonServiceException ase) {
throw new ContentManagementException(location, "message=" + ase.getMessage() + ", Error Type=" + ase.getErrorType(), ase); throw new ContentManagementException(location, "message=" + ase.getMessage() + ", Error Type=" + ase.getErrorType(), ase);
} catch (final AmazonClientException ace) { } catch (final AmazonClientException ace) {
@@ -156,7 +154,8 @@ public class S3ContentRepository implements ContentRepository {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Uploading a new object to S3 {}", LogParameterEscapeUtil.escapeLogParameter(s3UploadLocation)); log.debug("Uploading a new object to S3 {}", LogParameterEscapeUtil.escapeLogParameter(s3UploadLocation));
} }
this.s3Client.putObject(new PutObjectRequest(this.s3BucketName, s3UploadLocation, inputStream, new ObjectMetadata())); this.s3Client.putObject(new PutObjectRequest(fineractProperties.getContent().getS3().getBucketName(), s3UploadLocation,
inputStream, new ObjectMetadata()));
} catch (AmazonClientException ase) { } catch (AmazonClientException ase) {
throw new ContentManagementException(filename, ase.getMessage(), ase); throw new ContentManagementException(filename, ase.getMessage(), ase);
} }
@@ -164,8 +163,9 @@ public class S3ContentRepository implements ContentRepository {
private S3Object getObject(String key) { private S3Object getObject(String key) {
try { try {
log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}", this.s3BucketName, key); log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}",
return this.s3Client.getObject(new GetObjectRequest(this.s3BucketName, key)); fineractProperties.getContent().getS3().getBucketName(), key);
return this.s3Client.getObject(new GetObjectRequest(fineractProperties.getContent().getS3().getBucketName(), key));
} catch (AmazonClientException ase) { } catch (AmazonClientException ase) {
throw new ContentManagementException(key, ase.getMessage(), ase); throw new ContentManagementException(key, ase.getMessage(), ase);
} }
@@ -70,6 +70,17 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=${F
fineract.loan.transactionprocessor.rbi-india.enabled=${FINERACT_LOAN_TRANSACTIONPROCESSOR_RBI_INDIA_ENABLED:true} fineract.loan.transactionprocessor.rbi-india.enabled=${FINERACT_LOAN_TRANSACTIONPROCESSOR_RBI_INDIA_ENABLED:true}
fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACTIONPROCESSOR_ERROR_NOT_FOUND_FAIL:true} fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACTIONPROCESSOR_ERROR_NOT_FOUND_FAIL:true}
fineract.content.regex-whitelist-enabled=${FINERACT_CONTENT_REGEX_WHITELIST_ENABLED:true}
fineract.content.regex-whitelist=${FINERACT_CONTENT_REGEX_WHITELIST:.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png}
fineract.content.mime-whitelist-enabled=${FINERACT_CONTENT_MIME_WHITELIST_ENABLED:true}
fineract.content.mime-whitelist=${FINERACT_CONTENT_MIME_WHITELIST:application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png}
fineract.content.filesystem.enabled=${FINERACT_CONTENT_FILESYSTEM_ENABLED:true}
fineract.content.filesystem.rootFolder=${FINERACT_CONTENT_FILESYSTEM_ROOT_FOLDER:${user.home}/.fineract}
fineract.content.s3.enabled=${FINERACT_CONTENT_S3_ENABLED:false}
fineract.content.s3.bucketName=${FINERACT_CONTENT_S3_BUCKET_NAME:}
fineract.content.s3.accessKey=${FINERACT_CONTENT_S3_ACCESS_KEY:}
fineract.content.s3.secretKey=${FINERACT_CONTENT_S3_SECRET_KEY:}
# Logging pattern for the console # Logging pattern for the console
logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}} logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
@@ -60,6 +60,16 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=tru
fineract.loan.transactionprocessor.rbi-india.enabled=true fineract.loan.transactionprocessor.rbi-india.enabled=true
fineract.loan.transactionprocessor.error-not-found-fail=true fineract.loan.transactionprocessor.error-not-found-fail=true
fineract.content.regex-whitelist-enabled=true
fineract.content.regex-whitelist=.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png
fineract.content.mime-whitelist-enabled=true
fineract.content.mime-whitelist=application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png
fineract.content.filesystem.enabled=true
fineract.content.filesystem.rootFolder=${user.home}/.fineract
fineract.content.s3.enabled=false
fineract.content.s3.bucketName=
fineract.content.s3.accessKey=
fineract.content.s3.secretKey=
management.health.jms.enabled=false management.health.jms.enabled=false
@@ -18,10 +18,15 @@
*/ */
package org.apache.fineract.integrationtests.client; package org.apache.fineract.integrationtests.client;
import java.io.File; import static org.junit.jupiter.api.Assertions.assertThrows;
import java.io.IOException; import java.io.IOException;
import lombok.extern.slf4j.Slf4j;
import okhttp3.MediaType; import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.RequestBody;
import okhttp3.ResponseBody; import okhttp3.ResponseBody;
import org.apache.commons.io.IOUtils;
import org.apache.fineract.client.services.ImagesApi; import org.apache.fineract.client.services.ImagesApi;
import org.apache.fineract.client.util.Parts; import org.apache.fineract.client.util.Parts;
import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Order;
@@ -36,9 +41,10 @@ import retrofit2.http.Headers;
* *
* @author Michael Vorburger.ch * @author Michael Vorburger.ch
*/ */
@Slf4j
public class ImageTest extends IntegrationTest { public class ImageTest extends IntegrationTest {
final File testImage = new File(getClass().getResource("/michael.vorburger-crepes.jpg").getFile()); final MultipartBody.Part testPart = createPart("michael.vorburger-crepes.jpg", "michael.vorburger-crepes.jpg", "image/jpeg");
Long clientId = new ClientTest().getClientId(); Long clientId = new ClientTest().getClientId();
Long staffId = new StaffTest().getStaffId(); Long staffId = new StaffTest().getStaffId();
@@ -46,8 +52,8 @@ public class ImageTest extends IntegrationTest {
@Test @Test
@Order(1) @Order(1)
void create() { void create() {
ok(fineract().images.create("staff", staffId, Parts.fromFile(testImage))); ok(fineract().images.create("staff", staffId, testPart));
ok(fineract().images.create("clients", clientId, Parts.fromFile(testImage))); ok(fineract().images.create("clients", clientId, testPart));
} }
@Test @Test
@@ -82,8 +88,8 @@ public class ImageTest extends IntegrationTest {
Response<ResponseBody> r = okR(fineract().images.get("staff", staffId, 3505, 1972, "inline_octet")); Response<ResponseBody> r = okR(fineract().images.get("staff", staffId, 3505, 1972, "inline_octet"));
try (ResponseBody body = r.body()) { try (ResponseBody body = r.body()) {
assertThat(body.contentType()).isEqualTo(MediaType.get("image/jpeg")); assertThat(body.contentType()).isEqualTo(MediaType.get("image/jpeg"));
assertThat(body.bytes().length).isEqualTo(testImage.length()); assertThat(body.bytes().length).isEqualTo(testPart.body().contentLength());
assertThat(body.contentLength()).isEqualTo(testImage.length()); assertThat(body.contentLength()).isEqualTo(testPart.body().contentLength());
} }
var staff = ok(fineract().staff.retrieveOne8(staffId)); var staff = ok(fineract().staff.retrieveOne8(staffId));
@@ -96,8 +102,8 @@ public class ImageTest extends IntegrationTest {
void getOctetOutput() throws IOException { void getOctetOutput() throws IOException {
ResponseBody r = ok(fineract().images.get("staff", staffId, 3505, 1972, "octet")); ResponseBody r = ok(fineract().images.get("staff", staffId, 3505, 1972, "octet"));
assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg")); assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg"));
assertThat(r.bytes().length).isEqualTo(testImage.length()); assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength());
assertThat(r.contentLength()).isEqualTo(testImage.length()); assertThat(r.contentLength()).isEqualTo(testPart.body().contentLength());
} }
@Test @Test
@@ -121,13 +127,13 @@ public class ImageTest extends IntegrationTest {
void getBytes() throws IOException { void getBytes() throws IOException {
ResponseBody r = ok(fineract().createService(ImagesApiWithHeadersForTest.class).getBytes("staff", staffId, 3505, 1972, null)); ResponseBody r = ok(fineract().createService(ImagesApiWithHeadersForTest.class).getBytes("staff", staffId, 3505, 1972, null));
assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg")); assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg"));
assertThat(r.bytes().length).isEqualTo(testImage.length()); assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength());
} }
@Test @Test
@Order(50) @Order(50)
void update() { void update() {
ok(fineract().images.update("staff", staffId, Parts.fromFile(testImage))); ok(fineract().images.update("staff", staffId, testPart));
} }
@Test @Test
@@ -137,6 +143,116 @@ public class ImageTest extends IntegrationTest {
ok(fineract().images.delete("clients", clientId)); ok(fineract().images.delete("clients", clientId));
} }
@Test
@Order(100)
void pathTraversalJsp() {
final MultipartBody.Part part = createPart("image-text-wrong-content.jsp",
"../../../../../../../../../../tmp/image-text-wrong-content.jsp", "image/gif");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a file that doesn't match the indicated content type: {}", exception.getMessage());
}
@Test
@Order(101)
void gifWithPngExtension() {
final MultipartBody.Part part = createPart("image-gif-wrong-extension.png", "image-gif-wrong-extension.png", "image/png");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a gif by just renaming the file extension: {}", exception.getMessage());
}
@Test
@Order(102)
void gifImage() {
final MultipartBody.Part part = createPart("image-gif-correct-extension.gif", "image-gif-correct-extension.gif", "image/png");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a gif it is not whitelisted: {}", exception.getMessage());
}
@Test
@Order(103)
void pathTraversalJpg() {
final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg",
"../../../../../../../../../../tmp/michael.vorburger-crepes.jpg", "image/jpeg");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
}
@Test
@Order(104)
void pathTraversalWithAbsolutePathJpg() {
final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "../17/michael.vorburger-crepes.jpg", "image/jpeg");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
}
@Test
@Order(105)
void pathTraversalWithAbsolutePathJpg2() {
final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "..//17//michael.vorburger-crepes.jpg", "image/jpeg");
assertThat(part).isNotNull();
Exception exception = assertThrows(Exception.class, () -> {
ok(fineract().images.create("clients", clientId, part));
});
assertThat(exception).isNotNull();
log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
}
private MultipartBody.Part createPart(String fileResource, String fileName, String mediaType) {
try {
byte[] data = IOUtils.toByteArray(ImageTest.class.getClassLoader().getResourceAsStream(fileResource));
RequestBody rb = RequestBody.create(data, MediaType.get(mediaType));
return MultipartBody.Part.createFormData("file", fileName, rb);
} catch (Exception e) {
log.error("Error creating file part.", e);
}
return null;
}
interface ImagesApiWithHeadersForTest extends ImagesApi { interface ImagesApiWithHeadersForTest extends ImagesApi {
@Headers("Accept: text/plain") @Headers("Accept: text/plain")
Binary file not shown.

After

Width:  |  Height:  |  Size: 995 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 995 KiB

@@ -0,0 +1,21 @@
<%--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
--%>
<JSP>add JSP CODE here </jsp>