diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle index 04ed207cf..f91a1e7ad 100644 --- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle +++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle @@ -66,6 +66,38 @@ dependencyManagement { dependency 'com.github.spullara.mustache.java:compiler:0.9.10' dependency 'com.jayway.jsonpath:json-path:2.7.0' dependency 'org.apache.tika:tika-core:2.6.0' + dependency ('org.apache.tika:tika-parser-microsoft-module:2.6.0') { + exclude 'org.bouncycastle:bcprov-jdk15on' + exclude 'org.bouncycastle:bcmail-jdk15on' + exclude 'commons-logging:commons-logging' + exclude 'org.apache.logging.log4j:log4j-api' + exclude 'org.slf4j:slf4j-api' + exclude 'commons-io:commons-io' + exclude 'commons-codec:commons-codec' + exclude 'org.apache.commons:commons-compress' + exclude 'org.apache.commons:commons-lang3' + exclude 'org.apache.poi:poi' + exclude 'org.apache.poi:poi-scratchpad' + exclude 'org.glassfish.jaxb:jaxb-runtime' + exclude 'org.apache.commons:commons-compress' + exclude 'xml-apis:xml-apis' + } + dependency ('org.apache.tika:tika-parser-miscoffice-module:2.6.0') { + exclude 'org.bouncycastle:bcprov-jdk15on' + exclude 'org.bouncycastle:bcmail-jdk15on' + exclude 'commons-logging:commons-logging' + exclude 'org.apache.logging.log4j:log4j-api' + exclude 'org.slf4j:slf4j-api' + exclude 'commons-io:commons-io' + exclude 'commons-codec:commons-codec' + exclude 'org.apache.commons:commons-compress' + exclude 'org.apache.commons:commons-lang3' + exclude 'org.apache.poi:poi' + exclude 'org.apache.poi:poi-scratchpad' + exclude 'org.glassfish.jaxb:jaxb-runtime' + exclude 'org.apache.commons:commons-compress' + exclude 'xml-apis:xml-apis' + } dependency 'org.apache.httpcomponents:httpclient:4.5.13' dependency 'jakarta.management.j2ee:jakarta.management.j2ee-api:1.1.4' dependency 'jakarta.jms:jakarta.jms-api:2.0.3' diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle index e8526710e..05c0e899f 100644 --- a/fineract-provider/dependencies.gradle +++ b/fineract-provider/dependencies.gradle @@ -63,6 +63,8 @@ dependencies { 'org.apache.poi:poi', 'org.apache.poi:poi-ooxml', 'org.apache.tika:tika-core', + 'org.apache.tika:tika-parser-microsoft-module', + 'org.apache.tika:tika-parser-miscoffice-module', 'org.liquibase:liquibase-core', diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java index 393fccbf1..26af6968d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java @@ -18,6 +18,7 @@ */ package org.apache.fineract.infrastructure.bulkimport.service; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -87,14 +88,13 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService final String dateFormat) { try { if (entity != null && inputStream != null && fileDetail != null && locale != null && dateFormat != null) { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); IOUtils.copy(inputStream, baos); final byte[] bytes = baos.toByteArray(); InputStream clonedInputStream = new ByteArrayInputStream(bytes); - InputStream clonedInputStreamWorkbook = new ByteArrayInputStream(bytes); + final BufferedInputStream bis = new BufferedInputStream(new ByteArrayInputStream(bytes)); final Tika tika = new Tika(); - final TikaInputStream tikaInputStream = TikaInputStream.get(clonedInputStream); + final TikaInputStream tikaInputStream = TikaInputStream.get(bis); final String fileType = tika.detect(tikaInputStream); if (!fileType.contains("msoffice") && !fileType.contains("application/vnd.ms-excel")) { // We had a problem where we tried to upload the downloaded @@ -104,7 +104,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService "Uploaded file extension is not recognized."); } - Workbook workbook = new HSSFWorkbook(clonedInputStreamWorkbook); + Workbook workbook = new HSSFWorkbook(clonedInputStream); GlobalEntityType entityType = null; int primaryColumn = 0; if (entity.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { @@ -169,7 +169,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService throw new GeneralPlatformDomainRuleException("error.msg.unable.to.find.resource", "Unable to find requested resource"); } - return publishEvent(primaryColumn, fileDetail, clonedInputStreamWorkbook, entityType, workbook, locale, dateFormat); + return publishEvent(primaryColumn, fileDetail, bis, entityType, workbook, locale, dateFormat); } throw new GeneralPlatformDomainRuleException("error.msg.null", "One or more of the given parameters not found"); } catch (IOException e) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java index 7b0573eef..3ce76ffae 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java @@ -21,6 +21,7 @@ package org.apache.fineract.infrastructure.campaigns.jobs.executeemail; import org.apache.fineract.infrastructure.campaigns.email.domain.EmailCampaignRepository; import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRepository; import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator; @@ -57,6 +58,9 @@ public class ExecuteEmailConfig { @Autowired private ReportMailingJobValidator reportMailingJobValidator; + @Autowired + private FineractProperties fineractProperties; + @Bean protected Step executeEmailStep() { return steps.get(JobName.EXECUTE_EMAIL.name()).tasklet(executeEmailTasklet()).build(); @@ -70,6 +74,6 @@ public class ExecuteEmailConfig { @Bean public ExecuteEmailTasklet executeEmailTasklet() { return new ExecuteEmailTasklet(emailMessageRepository, emailCampaignRepository, loanRepository, savingsAccountRepository, - emailMessageJobEmailService, readReportingService, reportMailingJobValidator); + emailMessageJobEmailService, readReportingService, reportMailingJobValidator, fineractProperties); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java index 1b5b4e92a..9cdac9bd7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java @@ -39,10 +39,10 @@ import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRep import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageStatusType; import org.apache.fineract.infrastructure.campaigns.email.domain.ScheduledEmailAttachmentFileFormat; import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException; import org.apache.fineract.infrastructure.dataqueries.domain.Report; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; -import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository; import org.apache.fineract.infrastructure.reportmailingjob.helper.IPv4Helper; import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator; import org.apache.fineract.portfolio.client.domain.Client; @@ -66,6 +66,7 @@ public class ExecuteEmailTasklet implements Tasklet { private final EmailMessageJobEmailService emailMessageJobEmailService; private final ReadReportingService readReportingService; private final ReportMailingJobValidator reportMailingJobValidator; + private final FineractProperties fineractProperties; @Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { @@ -205,7 +206,7 @@ public class ExecuteEmailTasklet implements Tasklet { try { final ByteArrayOutputStream byteArrayOutputStream = readReportingService.generatePentahoReportAsOutputStream(reportName, emailAttachmentFileFormat.getValue(), reportParams, null, emailCampaign.getApprovedBy(), errorLog); - final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; + final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + ""; final String fileNameWithoutExtension = fileLocation + File.separator + reportName; if (!new File(fileLocation).isDirectory()) { new File(fileLocation).mkdirs(); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java index d2d734c52..9146a25b9 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java @@ -18,6 +18,7 @@ */ package org.apache.fineract.infrastructure.campaigns.jobs.executereportmailingjobs; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider; @@ -54,6 +55,8 @@ public class ExecuteReportMailingJobsConfig { private ReportMailingJobEmailService reportMailingJobEmailService; @Autowired private ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository; + @Autowired + private FineractProperties fineractProperties; @Bean protected Step executeReportMailingJobsStep() { @@ -69,6 +72,6 @@ public class ExecuteReportMailingJobsConfig { @Bean public ExecuteReportMailingJobsTasklet executeReportMailingJobsTasklet() { return new ExecuteReportMailingJobsTasklet(reportMailingJobRepository, reportMailingJobValidator, readReportingService, - reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository); + reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository, fineractProperties); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java index 584fb1033..68d85df01 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java @@ -34,10 +34,10 @@ import javax.ws.rs.core.Response; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.dataqueries.domain.Report; import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService; -import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository; import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider; import org.apache.fineract.infrastructure.report.service.ReportingProcessService; import org.apache.fineract.infrastructure.reportmailingjob.data.ReportMailingJobEmailAttachmentFileFormat; @@ -68,6 +68,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet { private final ReportingProcessServiceProvider reportingProcessServiceProvider; private final ReportMailingJobEmailService reportMailingJobEmailService; private final ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository; + private final FineractProperties fineractProperties; private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; @@ -133,7 +134,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet { if (responseObject != null && responseObject.getClass().equals(ByteArrayOutputStream.class)) { final ByteArrayOutputStream byteArrayOutputStream = (ByteArrayOutputStream) responseObject; - final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; + final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + ""; final String fileNameWithoutExtension = fileLocation + File.separator + reportName; if (!new File(fileLocation).isDirectory()) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java new file mode 100644 index 000000000..c4114d20c --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fineract.infrastructure.core.config; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration +public class ContentS3Config { + + @Bean + @ConditionalOnProperty("fineract.content.s3.enabled") + public AmazonS3 contentS3Client(FineractProperties fineractProperties) { + return AmazonS3ClientBuilder.standard() + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(fineractProperties.getContent().getS3().getAccessKey(), + fineractProperties.getContent().getS3().getSecretKey()))) + .build(); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java index d0b70e6eb..c10e3ddd4 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java @@ -45,6 +45,8 @@ public class FineractProperties { private FineractEventsProperties events; + private FineractContentProperties content; + @Getter @Setter public static class FineractTenantProperties { @@ -155,4 +157,32 @@ public class FineractProperties { private String eventQueueName; private String brokerUrl; } + + @Getter + @Setter + public static class FineractContentProperties { + + private boolean regexWhitelistEnabled; + private List regexWhitelist; + private boolean mimeWhitelistEnabled; + private List mimeWhitelist; + private FineractContentFilesystemProperties filesystem; + private FineractContentS3Properties s3; + } + + @Getter + @Setter + public static class FineractContentFilesystemProperties { + + private String rootFolder; + } + + @Getter + @Setter + public static class FineractContentS3Properties { + + private String bucketName; + private String accessKey; + private String secretKey; + } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java index 1acab7c17..6d3ec207e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java @@ -40,6 +40,7 @@ import javax.ws.rs.core.StreamingOutput; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.JdbcSupport; import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException; import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator; @@ -50,7 +51,6 @@ import org.apache.fineract.infrastructure.dataqueries.data.ReportParameterJoinDa import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData; import org.apache.fineract.infrastructure.dataqueries.exception.ReportNotFoundException; -import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.SqlInjectionPreventerService; import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil; @@ -72,6 +72,7 @@ public class ReadReportingServiceImpl implements ReadReportingService { private final GenericDataService genericDataService; private final SqlInjectionPreventerService sqlInjectionPreventerService; private final DatabaseSpecificSQLGenerator sqlGenerator; + private final FineractProperties fineractProperties; @Override public StreamingOutput retrieveReportCSV(final String name, final String type, final Map queryParams, @@ -240,7 +241,7 @@ public class ReadReportingServiceImpl implements ReadReportingService { public String retrieveReportPDF(final String reportName, final String type, final Map queryParams, final boolean isSelfServiceUserReport) { - final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + ""; + final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + ""; if (!new File(fileLocation).isDirectory()) { new File(fileLocation).mkdirs(); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java index fcf1208cc..bf010e84c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java @@ -36,6 +36,8 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; import org.apache.fineract.infrastructure.core.data.UploadRequest; @@ -54,19 +56,16 @@ import org.apache.fineract.portfolio.client.data.ClientData; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +@Slf4j +@RequiredArgsConstructor @Component @Scope("singleton") @Path("{entity}/{entityId}/images") public class ImagesApiResource { - private static final Logger LOG = LoggerFactory.getLogger(ImagesApiResource.class); - private final PlatformSecurityContext context; private final ImageReadPlatformService imageReadPlatformService; private final ImageWritePlatformService imageWritePlatformService; @@ -74,18 +73,6 @@ public class ImagesApiResource { private final FileUploadValidator fileUploadValidator; private final ImageResizer imageResizer; - @Autowired - public ImagesApiResource(final PlatformSecurityContext context, final ImageReadPlatformService readPlatformService, - final ImageWritePlatformService imageWritePlatformService, final DefaultToApiJsonSerializer toApiJsonSerializer, - final FileUploadValidator fileUploadValidator, final ImageResizer imageResizer) { - this.context = context; - this.imageReadPlatformService = readPlatformService; - this.imageWritePlatformService = imageWritePlatformService; - this.toApiJsonSerializer = toApiJsonSerializer; - this.fileUploadValidator = fileUploadValidator; - this.imageResizer = imageResizer; - } - /** * Upload images through multi-part form upload */ @@ -172,7 +159,7 @@ public class ImagesApiResource { final String clientImageAsBase64Text = imageDataURISuffix + Base64.getMimeEncoder().encodeToString(resizedImageBytes); return Response.ok(clientImageAsBase64Text, MediaType.TEXT_PLAIN_TYPE).build(); } else { - LOG.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId, + log.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId, maxWidth, maxHeight); return Response.serverError().build(); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java new file mode 100644 index 000000000..415532fcd --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.documentmanagement.contentrepository; + +import java.io.BufferedInputStream; + +public interface ContentPathSanitizer { + + String sanitize(String path); + + String sanitize(String path, BufferedInputStream is); +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java index bdfcb5c69..5c8fa8f90 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java @@ -18,40 +18,28 @@ */ package org.apache.fineract.infrastructure.documentmanagement.contentrepository; +import java.util.List; import lombok.RequiredArgsConstructor; -import org.apache.fineract.infrastructure.configuration.data.S3CredentialsData; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; -import org.apache.fineract.infrastructure.configuration.service.ExternalServicesPropertiesReadPlatformService; import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; @Component @RequiredArgsConstructor public class ContentRepositoryFactory { - private final ApplicationContext applicationContext; - private final ExternalServicesPropertiesReadPlatformService externalServicesReadPlatformService; + // TODO: all configuration should be really moved to application.properties + private final ConfigurationDomainService configurationService; + private final List contentRepositories; public ContentRepository getRepository() { - final ConfigurationDomainService configurationDomainServiceJpa = this.applicationContext.getBean("configurationDomainServiceJpa", - ConfigurationDomainService.class); - if (configurationDomainServiceJpa.isAmazonS3Enabled()) { - return createS3DocumentStore(); + if (configurationService.isAmazonS3Enabled()) { + return getRepository(StorageType.S3); } - return new FileSystemContentRepository(); + return getRepository(StorageType.FILE_SYSTEM); } - public ContentRepository getRepository(final StorageType documentStoreType) { - if (documentStoreType == StorageType.FILE_SYSTEM) { - return new FileSystemContentRepository(); - } - return createS3DocumentStore(); - } - - private ContentRepository createS3DocumentStore() { - final S3CredentialsData s3CredentialsData = this.externalServicesReadPlatformService.getS3Credentials(); - return new S3ContentRepository(s3CredentialsData.getBucketName(), s3CredentialsData.getSecretKey(), - s3CredentialsData.getAccessKey()); + public ContentRepository getRepository(StorageType storageType) { + return contentRepositories.stream().filter(cr -> cr.getStorageType().equals(storageType)).findFirst().orElseThrow(); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java new file mode 100644 index 000000000..b25fc97c4 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.documentmanagement.contentrepository; + +import java.io.BufferedInputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Pattern; +import javax.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.fineract.infrastructure.core.config.FineractProperties; +import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; +import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException; +import org.apache.tika.Tika; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.sax.BodyContentHandler; +import org.springframework.stereotype.Component; + +@Slf4j +@RequiredArgsConstructor +@Component +public class FileSystemContentPathSanitizer implements ContentPathSanitizer { + + private final FineractProperties fineractProperties; + + private List regexWhitelist; + + private static Pattern OVERWRITE_SIBLING_IMAGE = Pattern.compile(".*\\.\\./+[0-9]+/+.*"); + + @PostConstruct + public void init() { + regexWhitelist = fineractProperties.getContent().getRegexWhitelist().stream().map(Pattern::compile).toList(); + } + + @Override + public String sanitize(String path) { + return sanitize(path, null); + } + + @Override + public String sanitize(String path, BufferedInputStream is) { + try { + if (OVERWRITE_SIBLING_IMAGE.matcher(path).matches()) { + throw new RuntimeException(String.format("Trying to overwrite another resource's image: %s", path)); + } + + String sanitizedPath = Path.of(path).normalize().toString(); + + String fileName = FilenameUtils.getName(sanitizedPath).toLowerCase(); + + if (log.isDebugEnabled()) { + log.debug("Path: {} -> {} ({})", path, sanitizedPath, fileName); + } + + if (fineractProperties.getContent().isRegexWhitelistEnabled()) { + boolean matches = regexWhitelist.stream().anyMatch(p -> p.matcher(fileName).matches()); + + if (!matches) { + throw new RuntimeException(String.format("File name not allowed: %s", fileName)); + } + } + + if (is != null && fineractProperties.getContent().isMimeWhitelistEnabled()) { + Tika tika = new Tika(); + String extensionMimeType = tika.detect(fileName); + + if (StringUtils.isEmpty(extensionMimeType)) { + throw new RuntimeException(String.format("Could not detect mime type for filename %s!", fileName)); + } + + if (!fineractProperties.getContent().getMimeWhitelist().contains(extensionMimeType)) { + throw new RuntimeException( + String.format("Detected mime type %s for filename %s not allowed!", extensionMimeType, fileName)); + } + + String contentMimeType = detectContentMimeType(is); + + if (StringUtils.isEmpty(contentMimeType)) { + throw new RuntimeException(String.format("Could not detect content mime type for %s!", fileName)); + } + + if (!fineractProperties.getContent().getMimeWhitelist().contains(contentMimeType)) { + throw new RuntimeException( + String.format("Detected content mime type %s for %s not allowed!", contentMimeType, fileName)); + } + + if (!contentMimeType.equalsIgnoreCase(extensionMimeType)) { + throw new RuntimeException(String.format("Detected filename (%s) and content (%s) mime type do not match!", + extensionMimeType, contentMimeType)); + } + } + + Path target = Path.of(sanitizedPath); + Path rootFolder = Path.of(fineractProperties.getContent().getFilesystem().getRootFolder(), + ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim()); + + if (!target.startsWith(rootFolder)) { + throw new RuntimeException(String.format("Path traversal attempt: %s (%s)", target, rootFolder)); + } + + return sanitizedPath; + } catch (Exception e) { + throw new ContentManagementException(path, e.getMessage(), e); + } + } + + private String detectContentMimeType(BufferedInputStream bis) throws Exception { + TikaInputStream tis = TikaInputStream.get(bis); + AutoDetectParser parser = new AutoDetectParser(); + // NOTE: turn off write limit with "-1" + BodyContentHandler handler = new BodyContentHandler(-1); + Metadata metadata = new Metadata(); + parser.parse(tis, handler, metadata); + + return metadata.get("Content-Type"); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java index 19a83e1fe..1329f87ea 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java @@ -19,12 +19,16 @@ package org.apache.fineract.infrastructure.documentmanagement.contentrepository; import com.google.common.io.Files; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Base64; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand; @@ -33,14 +37,17 @@ import org.apache.fineract.infrastructure.documentmanagement.data.FileData; import org.apache.fineract.infrastructure.documentmanagement.data.ImageData; import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType; import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +@Slf4j +@RequiredArgsConstructor +@Component +@ConditionalOnProperty("fineract.content.filesystem.enabled") public class FileSystemContentRepository implements ContentRepository { - private static final Logger LOG = LoggerFactory.getLogger(FileSystemContentRepository.class); - - public static final String FINERACT_BASE_DIR = System.getProperty("user.home") + File.separator + ".fineract"; + private final FileSystemContentPathSanitizer pathSanitizer; + private final FineractProperties fineractProperties; @Override public String saveFile(final InputStream uploadedInputStream, final DocumentCommand documentCommand) { @@ -50,16 +57,14 @@ public class FileSystemContentRepository implements ContentRepository { final String fileLocation = generateFileParentDirectory(documentCommand.getParentEntityType(), documentCommand.getParentEntityId()) + File.separator + fileName; - writeFileToFileSystem(fileName, uploadedInputStream, fileLocation); - return fileLocation; + return writeFileToFileSystem(fileName, uploadedInputStream, fileLocation); } @Override public String saveImage(final InputStream uploadedInputStream, final Long resourceId, final String imageName, final Long fileSize) { ContentRepositoryUtils.validateFileSizeWithinPermissibleRange(fileSize, imageName); final String fileLocation = generateClientImageParentDirectory(resourceId) + File.separator + imageName; - writeFileToFileSystem(imageName, uploadedInputStream, fileLocation); - return fileLocation; + return writeFileToFileSystem(imageName, uploadedInputStream, fileLocation); } @Override @@ -69,10 +74,9 @@ public class FileSystemContentRepository implements ContentRepository { String base64EncodedImageString = base64EncodedImage.getBase64EncodedString(); try { final InputStream toUploadInputStream = new ByteArrayInputStream(Base64.getMimeDecoder().decode(base64EncodedImageString)); - writeFileToFileSystem(imageName, toUploadInputStream, fileLocation); - return fileLocation; + return writeFileToFileSystem(imageName, toUploadInputStream, fileLocation); } catch (IllegalArgumentException iae) { - LOG.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae); + log.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae); throw iae; } } @@ -88,23 +92,29 @@ public class FileSystemContentRepository implements ContentRepository { } private void deleteFileInternal(final String documentPath) { - final File fileToBeDeleted = new File(documentPath); + String sanitizedPath = pathSanitizer.sanitize(documentPath); + + final File fileToBeDeleted = new File(sanitizedPath); final boolean fileDeleted = fileToBeDeleted.delete(); if (!fileDeleted) { // no need to throw an Error, what's a caller going to do about it, so simply log a warning - LOG.warn("Unable to delete file {}", documentPath); + log.warn("Unable to delete file {}", documentPath); } } @Override public FileData fetchFile(final DocumentData documentData) { - final File file = new File(documentData.fileLocation()); + String sanitizedPath = pathSanitizer.sanitize(documentData.fileLocation()); + + final File file = new File(sanitizedPath); return new FileData(Files.asByteSource(file), documentData.fileName(), documentData.contentType()); } @Override public FileData fetchImage(final ImageData imageData) { - final File file = new File(imageData.location()); + String sanitizedPath = pathSanitizer.sanitize(imageData.location()); + + final File file = new File(sanitizedPath); return new FileData(Files.asByteSource(file), imageData.getEntityDisplayName(), imageData.contentType().getValue()); } @@ -117,16 +127,17 @@ public class FileSystemContentRepository implements ContentRepository { * Generate the directory path for storing the new document */ private String generateFileParentDirectory(final String entityType, final Long entityId) { - return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "documents" + File.separator + entityType + File.separator + entityId + File.separator + ContentRepositoryUtils.generateRandomString(); } /** - * Generate directory path for storing new Image + * Generate ContentRepositoryUtilsfineractProperties.getContentgetWhitelist()getBlacklist() path for storing new + * Image */ private String generateClientImageParentDirectory(final Long resourceId) { - return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "images" + File.separator + "clients" + File.separator + resourceId; } @@ -135,16 +146,18 @@ public class FileSystemContentRepository implements ContentRepository { * Recursively create the directory if it does not exist. */ private void makeDirectories(final String uploadDocumentLocation) throws IOException { - Files.createParentDirs(new File(uploadDocumentLocation)); + String sanitizedPath = pathSanitizer.sanitize(uploadDocumentLocation); + Files.createParentDirs(new File(sanitizedPath)); } - private void writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) { - try { - makeDirectories(fileLocation); - FileUtils.copyInputStreamToFile(uploadedInputStream, new File(fileLocation)); // NOSONAR + private String writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) { + try (BufferedInputStream bis = new BufferedInputStream(uploadedInputStream)) { + String sanitizedPath = pathSanitizer.sanitize(fileLocation, bis); + makeDirectories(sanitizedPath); + FileUtils.copyInputStreamToFile(bis, new File(sanitizedPath)); // NOSONAR + return sanitizedPath; } catch (final IOException ioException) { - LOG.warn("writeFileToFileSystem() IOException (logged because cause is not propagated in ContentManagementException)", - ioException); + log.warn("Failed to write file!", ioException); throw new ContentManagementException(fileName, ioException.getMessage(), ioException); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java index 38bfdc9b7..0a91f4ad9 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java @@ -20,10 +20,7 @@ package org.apache.fineract.infrastructure.documentmanagement.contentrepository; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -35,7 +32,9 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Base64; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage; import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand; import org.apache.fineract.infrastructure.documentmanagement.data.DocumentData; @@ -45,18 +44,17 @@ import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType; import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException; import org.apache.fineract.infrastructure.documentmanagement.exception.DocumentNotFoundException; import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; @Slf4j +@RequiredArgsConstructor +@Component +@ConditionalOnProperty("fineract.content.s3.enabled") public class S3ContentRepository implements ContentRepository { - private final String s3BucketName; private final AmazonS3 s3Client; - - public S3ContentRepository(final String bucketName, final String secretKey, final String accessKey) { - this.s3BucketName = bucketName; - this.s3Client = AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))).build(); - } + private final FineractProperties fineractProperties; @Override public String saveFile(final InputStream toUpload, final DocumentCommand documentCommand) { @@ -142,7 +140,7 @@ public class S3ContentRepository implements ContentRepository { private void deleteObject(final String location) { try { - this.s3Client.deleteObject(new DeleteObjectRequest(this.s3BucketName, location)); + this.s3Client.deleteObject(new DeleteObjectRequest(fineractProperties.getContent().getS3().getBucketName(), location)); } catch (final AmazonServiceException ase) { throw new ContentManagementException(location, "message=" + ase.getMessage() + ", Error Type=" + ase.getErrorType(), ase); } catch (final AmazonClientException ace) { @@ -156,7 +154,8 @@ public class S3ContentRepository implements ContentRepository { if (log.isDebugEnabled()) { log.debug("Uploading a new object to S3 {}", LogParameterEscapeUtil.escapeLogParameter(s3UploadLocation)); } - this.s3Client.putObject(new PutObjectRequest(this.s3BucketName, s3UploadLocation, inputStream, new ObjectMetadata())); + this.s3Client.putObject(new PutObjectRequest(fineractProperties.getContent().getS3().getBucketName(), s3UploadLocation, + inputStream, new ObjectMetadata())); } catch (AmazonClientException ase) { throw new ContentManagementException(filename, ase.getMessage(), ase); } @@ -164,8 +163,9 @@ public class S3ContentRepository implements ContentRepository { private S3Object getObject(String key) { try { - log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}", this.s3BucketName, key); - return this.s3Client.getObject(new GetObjectRequest(this.s3BucketName, key)); + log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}", + fineractProperties.getContent().getS3().getBucketName(), key); + return this.s3Client.getObject(new GetObjectRequest(fineractProperties.getContent().getS3().getBucketName(), key)); } catch (AmazonClientException ase) { throw new ContentManagementException(key, ase.getMessage(), ase); } diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties index bd1a93a11..db24d4f67 100644 --- a/fineract-provider/src/main/resources/application.properties +++ b/fineract-provider/src/main/resources/application.properties @@ -70,6 +70,17 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=${F fineract.loan.transactionprocessor.rbi-india.enabled=${FINERACT_LOAN_TRANSACTIONPROCESSOR_RBI_INDIA_ENABLED:true} fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACTIONPROCESSOR_ERROR_NOT_FOUND_FAIL:true} +fineract.content.regex-whitelist-enabled=${FINERACT_CONTENT_REGEX_WHITELIST_ENABLED:true} +fineract.content.regex-whitelist=${FINERACT_CONTENT_REGEX_WHITELIST:.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png} +fineract.content.mime-whitelist-enabled=${FINERACT_CONTENT_MIME_WHITELIST_ENABLED:true} +fineract.content.mime-whitelist=${FINERACT_CONTENT_MIME_WHITELIST:application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png} +fineract.content.filesystem.enabled=${FINERACT_CONTENT_FILESYSTEM_ENABLED:true} +fineract.content.filesystem.rootFolder=${FINERACT_CONTENT_FILESYSTEM_ROOT_FOLDER:${user.home}/.fineract} +fineract.content.s3.enabled=${FINERACT_CONTENT_S3_ENABLED:false} +fineract.content.s3.bucketName=${FINERACT_CONTENT_S3_BUCKET_NAME:} +fineract.content.s3.accessKey=${FINERACT_CONTENT_S3_ACCESS_KEY:} +fineract.content.s3.secretKey=${FINERACT_CONTENT_S3_SECRET_KEY:} + # Logging pattern for the console logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}} diff --git a/fineract-provider/src/test/resources/application-test.properties b/fineract-provider/src/test/resources/application-test.properties index d19755693..492a2aa65 100644 --- a/fineract-provider/src/test/resources/application-test.properties +++ b/fineract-provider/src/test/resources/application-test.properties @@ -60,6 +60,16 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=tru fineract.loan.transactionprocessor.rbi-india.enabled=true fineract.loan.transactionprocessor.error-not-found-fail=true +fineract.content.regex-whitelist-enabled=true +fineract.content.regex-whitelist=.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png +fineract.content.mime-whitelist-enabled=true +fineract.content.mime-whitelist=application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png +fineract.content.filesystem.enabled=true +fineract.content.filesystem.rootFolder=${user.home}/.fineract +fineract.content.s3.enabled=false +fineract.content.s3.bucketName= +fineract.content.s3.accessKey= +fineract.content.s3.secretKey= management.health.jms.enabled=false diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java index 8bf466bbb..0bad4fb36 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java @@ -18,10 +18,15 @@ */ package org.apache.fineract.integrationtests.client; -import java.io.File; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.IOException; +import lombok.extern.slf4j.Slf4j; import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.RequestBody; import okhttp3.ResponseBody; +import org.apache.commons.io.IOUtils; import org.apache.fineract.client.services.ImagesApi; import org.apache.fineract.client.util.Parts; import org.junit.jupiter.api.Order; @@ -36,9 +41,10 @@ import retrofit2.http.Headers; * * @author Michael Vorburger.ch */ +@Slf4j public class ImageTest extends IntegrationTest { - final File testImage = new File(getClass().getResource("/michael.vorburger-crepes.jpg").getFile()); + final MultipartBody.Part testPart = createPart("michael.vorburger-crepes.jpg", "michael.vorburger-crepes.jpg", "image/jpeg"); Long clientId = new ClientTest().getClientId(); Long staffId = new StaffTest().getStaffId(); @@ -46,8 +52,8 @@ public class ImageTest extends IntegrationTest { @Test @Order(1) void create() { - ok(fineract().images.create("staff", staffId, Parts.fromFile(testImage))); - ok(fineract().images.create("clients", clientId, Parts.fromFile(testImage))); + ok(fineract().images.create("staff", staffId, testPart)); + ok(fineract().images.create("clients", clientId, testPart)); } @Test @@ -82,8 +88,8 @@ public class ImageTest extends IntegrationTest { Response r = okR(fineract().images.get("staff", staffId, 3505, 1972, "inline_octet")); try (ResponseBody body = r.body()) { assertThat(body.contentType()).isEqualTo(MediaType.get("image/jpeg")); - assertThat(body.bytes().length).isEqualTo(testImage.length()); - assertThat(body.contentLength()).isEqualTo(testImage.length()); + assertThat(body.bytes().length).isEqualTo(testPart.body().contentLength()); + assertThat(body.contentLength()).isEqualTo(testPart.body().contentLength()); } var staff = ok(fineract().staff.retrieveOne8(staffId)); @@ -96,8 +102,8 @@ public class ImageTest extends IntegrationTest { void getOctetOutput() throws IOException { ResponseBody r = ok(fineract().images.get("staff", staffId, 3505, 1972, "octet")); assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg")); - assertThat(r.bytes().length).isEqualTo(testImage.length()); - assertThat(r.contentLength()).isEqualTo(testImage.length()); + assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength()); + assertThat(r.contentLength()).isEqualTo(testPart.body().contentLength()); } @Test @@ -121,13 +127,13 @@ public class ImageTest extends IntegrationTest { void getBytes() throws IOException { ResponseBody r = ok(fineract().createService(ImagesApiWithHeadersForTest.class).getBytes("staff", staffId, 3505, 1972, null)); assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg")); - assertThat(r.bytes().length).isEqualTo(testImage.length()); + assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength()); } @Test @Order(50) void update() { - ok(fineract().images.update("staff", staffId, Parts.fromFile(testImage))); + ok(fineract().images.update("staff", staffId, testPart)); } @Test @@ -137,6 +143,116 @@ public class ImageTest extends IntegrationTest { ok(fineract().images.delete("clients", clientId)); } + @Test + @Order(100) + void pathTraversalJsp() { + final MultipartBody.Part part = createPart("image-text-wrong-content.jsp", + "../../../../../../../../../../tmp/image-text-wrong-content.jsp", "image/gif"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a file that doesn't match the indicated content type: {}", exception.getMessage()); + } + + @Test + @Order(101) + void gifWithPngExtension() { + final MultipartBody.Part part = createPart("image-gif-wrong-extension.png", "image-gif-wrong-extension.png", "image/png"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a gif by just renaming the file extension: {}", exception.getMessage()); + } + + @Test + @Order(102) + void gifImage() { + final MultipartBody.Part part = createPart("image-gif-correct-extension.gif", "image-gif-correct-extension.gif", "image/png"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a gif it is not whitelisted: {}", exception.getMessage()); + } + + @Test + @Order(103) + void pathTraversalJpg() { + final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", + "../../../../../../../../../../tmp/michael.vorburger-crepes.jpg", "image/jpeg"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage()); + } + + @Test + @Order(104) + void pathTraversalWithAbsolutePathJpg() { + final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "../17/michael.vorburger-crepes.jpg", "image/jpeg"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage()); + } + + @Test + @Order(105) + void pathTraversalWithAbsolutePathJpg2() { + final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "..//17//michael.vorburger-crepes.jpg", "image/jpeg"); + + assertThat(part).isNotNull(); + + Exception exception = assertThrows(Exception.class, () -> { + ok(fineract().images.create("clients", clientId, part)); + }); + + assertThat(exception).isNotNull(); + + log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage()); + } + + private MultipartBody.Part createPart(String fileResource, String fileName, String mediaType) { + try { + byte[] data = IOUtils.toByteArray(ImageTest.class.getClassLoader().getResourceAsStream(fileResource)); + RequestBody rb = RequestBody.create(data, MediaType.get(mediaType)); + return MultipartBody.Part.createFormData("file", fileName, rb); + } catch (Exception e) { + log.error("Error creating file part.", e); + } + + return null; + } + interface ImagesApiWithHeadersForTest extends ImagesApi { @Headers("Accept: text/plain") diff --git a/integration-tests/src/test/resources/image-gif-correct-extension.gif b/integration-tests/src/test/resources/image-gif-correct-extension.gif new file mode 100644 index 000000000..89b951662 Binary files /dev/null and b/integration-tests/src/test/resources/image-gif-correct-extension.gif differ diff --git a/integration-tests/src/test/resources/image-gif-wrong-extension.png b/integration-tests/src/test/resources/image-gif-wrong-extension.png new file mode 100644 index 000000000..89b951662 Binary files /dev/null and b/integration-tests/src/test/resources/image-gif-wrong-extension.png differ diff --git a/integration-tests/src/test/resources/image-text-wrong-content.jsp b/integration-tests/src/test/resources/image-text-wrong-content.jsp new file mode 100644 index 000000000..6e660673c --- /dev/null +++ b/integration-tests/src/test/resources/image-text-wrong-content.jsp @@ -0,0 +1,21 @@ +<%-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--%> +add JSP CODE here \ No newline at end of file