From 1a0d81dbb043163db3043d7f7544479daf60e5f5 Mon Sep 17 00:00:00 2001 From: Janos Haber Date: Tue, 17 Jan 2023 14:50:37 +0100 Subject: [PATCH] FINERACT-1707 - S3 upgrade [x] Upgrade S3 dependencies and libraries without code modifications --- .../org.apache.fineract.dependencies.gradle | 2 +- fineract-provider/dependencies.gradle | 7 ++- .../core/config/ContentS3Config.java | 15 +++--- .../S3ContentRepository.java | 52 ++++++++++--------- .../exception/DocumentNotFoundException.java | 4 +- ...cheduleRequestReadPlatformServiceImpl.java | 4 +- .../savings/domain/SavingsAccount.java | 1 - 7 files changed, 44 insertions(+), 41 deletions(-) diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle index 6637b44e7..a58531cff 100644 --- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle +++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle @@ -61,7 +61,7 @@ dependencyManagement { exclude 'com.sun.mail:javax.mail' } dependency 'org.quartz-scheduler:quartz:2.3.2' - dependency 'com.amazonaws:aws-java-sdk-s3:1.12.385' + dependency 'software.amazon.awssdk:bom:2.15.0' dependency 'org.ehcache:ehcache:3.10.8' dependency 'com.github.spullara.mustache.java:compiler:0.9.10' dependency 'com.jayway.jsonpath:json-path:2.7.0' diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle index 05c0e899f..c5d85003a 100644 --- a/fineract-provider/dependencies.gradle +++ b/fineract-provider/dependencies.gradle @@ -120,9 +120,12 @@ dependencies { implementation ('org.quartz-scheduler:quartz') { exclude group: 'com.zaxxer', module: 'HikariCP-java7' } - implementation ('com.amazonaws:aws-java-sdk-s3') { - exclude group: 'commons-logging' + implementation platform('software.amazon.awssdk:bom') + implementation ('software.amazon.awssdk:s3') { } + implementation ('software.amazon.awssdk:auth') { + } + implementation ('org.mnode.ical4j:ical4j') { exclude group: 'commons-logging' exclude group: 'javax.activation' diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java index c4114d20c..924e0dba5 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java @@ -19,14 +19,13 @@ package org.apache.fineract.infrastructure.core.config; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.s3.S3Client; @Slf4j @Configuration @@ -34,11 +33,9 @@ public class ContentS3Config { @Bean @ConditionalOnProperty("fineract.content.s3.enabled") - public AmazonS3 contentS3Client(FineractProperties fineractProperties) { - return AmazonS3ClientBuilder.standard() - .withCredentials( - new AWSStaticCredentialsProvider(new BasicAWSCredentials(fineractProperties.getContent().getS3().getAccessKey(), - fineractProperties.getContent().getS3().getSecretKey()))) + public S3Client contentS3Client(FineractProperties fineractProperties) { + return S3Client.builder().credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials + .create(fineractProperties.getContent().getS3().getAccessKey(), fineractProperties.getContent().getS3().getSecretKey()))) .build(); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java index 0a91f4ad9..b736b9d82 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java @@ -18,14 +18,6 @@ */ package org.apache.fineract.infrastructure.documentmanagement.contentrepository; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; import com.google.common.io.ByteSource; import java.io.ByteArrayInputStream; import java.io.File; @@ -34,6 +26,7 @@ import java.io.InputStream; import java.util.Base64; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage; import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand; @@ -46,6 +39,14 @@ import org.apache.fineract.infrastructure.documentmanagement.exception.DocumentN import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; @Slf4j @RequiredArgsConstructor @@ -53,7 +54,7 @@ import org.springframework.stereotype.Component; @ConditionalOnProperty("fineract.content.s3.enabled") public class S3ContentRepository implements ContentRepository { - private final AmazonS3 s3Client; + private final S3Client s3Client; private final FineractProperties fineractProperties; @Override @@ -106,8 +107,8 @@ public class S3ContentRepository implements ContentRepository { @Override public InputStream openStream() throws IOException { - final S3Object s3object = getObject(documentData.fileLocation()); - return s3object.getObjectContent(); + return s3Client.getObject(GetObjectRequest.builder().bucket(fineractProperties.getContent().getS3().getBucketName()) + .key(documentData.fileLocation()).build(), ResponseTransformer.toBytes()).asInputStream(); } }, documentData.fileName(), documentData.contentType()); } @@ -118,8 +119,8 @@ public class S3ContentRepository implements ContentRepository { @Override public InputStream openStream() throws IOException { - final S3Object s3object = getObject(imageData.location()); - return s3object.getObjectContent(); + return s3Client.getObject(GetObjectRequest.builder().bucket(fineractProperties.getContent().getS3().getBucketName()) + .key(imageData.location()).build(), ResponseTransformer.toBytes()).asInputStream(); } }, imageData.getEntityDisplayName(), imageData.contentType().getValue()); } @@ -140,33 +141,36 @@ public class S3ContentRepository implements ContentRepository { private void deleteObject(final String location) { try { - this.s3Client.deleteObject(new DeleteObjectRequest(fineractProperties.getContent().getS3().getBucketName(), location)); - } catch (final AmazonServiceException ase) { - throw new ContentManagementException(location, "message=" + ase.getMessage() + ", Error Type=" + ase.getErrorType(), ase); - } catch (final AmazonClientException ace) { + this.s3Client.deleteObject(builder -> builder.bucket(fineractProperties.getContent().getS3().getBucketName()).key(location)); + } catch (final AwsServiceException ase) { + throw new ContentManagementException(location, + "message=" + ase.getMessage() + ", Error Code=" + ase.awsErrorDetails().errorCode(), ase); + } catch (final SdkException ace) { throw new ContentManagementException(location, ace.getMessage(), ace); } } - private void putObject(final String filename, final InputStream inputStream, final String s3UploadLocation) + public void putObject(final String filename, final InputStream inputStream, final String s3UploadLocation) throws ContentManagementException { try { if (log.isDebugEnabled()) { log.debug("Uploading a new object to S3 {}", LogParameterEscapeUtil.escapeLogParameter(s3UploadLocation)); } - this.s3Client.putObject(new PutObjectRequest(fineractProperties.getContent().getS3().getBucketName(), s3UploadLocation, - inputStream, new ObjectMetadata())); - } catch (AmazonClientException ase) { + this.s3Client.putObject( + builder -> builder.bucket(fineractProperties.getContent().getS3().getBucketName()).key(s3UploadLocation), + RequestBody.fromBytes(IOUtils.toByteArray(inputStream))); + } catch (AwsServiceException | IOException ase) { throw new ContentManagementException(filename, ase.getMessage(), ase); } } - private S3Object getObject(String key) { + public ResponseBytes getObject(String key) { try { log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}", fineractProperties.getContent().getS3().getBucketName(), key); - return this.s3Client.getObject(new GetObjectRequest(fineractProperties.getContent().getS3().getBucketName(), key)); - } catch (AmazonClientException ase) { + return this.s3Client.getObject(builder -> builder.bucket(fineractProperties.getContent().getS3().getBucketName()).key(key), + ResponseTransformer.toBytes()); + } catch (SdkException ase) { throw new ContentManagementException(key, ase.getMessage(), ase); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/exception/DocumentNotFoundException.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/exception/DocumentNotFoundException.java index 6e02ecce9..cbe3ced4c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/exception/DocumentNotFoundException.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/exception/DocumentNotFoundException.java @@ -18,9 +18,9 @@ */ package org.apache.fineract.infrastructure.documentmanagement.exception; -import com.amazonaws.AmazonClientException; import org.apache.fineract.infrastructure.core.exception.AbstractPlatformResourceNotFoundException; import org.springframework.dao.EmptyResultDataAccessException; +import software.amazon.awssdk.core.exception.SdkException; public class DocumentNotFoundException extends AbstractPlatformResourceNotFoundException { @@ -34,7 +34,7 @@ public class DocumentNotFoundException extends AbstractPlatformResourceNotFoundE "Document with identifier " + id + " does not exist for the " + entityType + " with Identifier " + entityId, id, e); } - public DocumentNotFoundException(String entityType, Long entityId, Long id, AmazonClientException ace) { + public DocumentNotFoundException(String entityType, Long entityId, Long id, SdkException ace) { super("error.msg.document.id.invalid", "Document with identifier " + id + " does not exist for the " + entityType + " with Identifier " + entityId, id, ace); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/rescheduleloan/service/LoanRescheduleRequestReadPlatformServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/rescheduleloan/service/LoanRescheduleRequestReadPlatformServiceImpl.java index d4aedbe7f..c9fb65db4 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/rescheduleloan/service/LoanRescheduleRequestReadPlatformServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/rescheduleloan/service/LoanRescheduleRequestReadPlatformServiceImpl.java @@ -18,7 +18,6 @@ */ package org.apache.fineract.portfolio.loanaccount.rescheduleloan.service; -import com.amazonaws.util.StringUtils; import java.math.BigDecimal; import java.sql.ResultSet; import java.sql.SQLException; @@ -26,6 +25,7 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.codes.data.CodeValueData; import org.apache.fineract.infrastructure.codes.service.CodeValueReadPlatformService; import org.apache.fineract.infrastructure.core.domain.JdbcSupport; @@ -274,7 +274,7 @@ public class LoanRescheduleRequestReadPlatformServiceImpl implements LoanResched public List retrieveAllRescheduleRequests(String command) { LoanRescheduleRequestRowMapperForBulkApproval loanRescheduleRequestRowMapperForBulkApproval = new LoanRescheduleRequestRowMapperForBulkApproval(); String sql = "select " + loanRescheduleRequestRowMapperForBulkApproval.schema(); - if (!StringUtils.isNullOrEmpty(command) && !command.equalsIgnoreCase(RescheduleLoansApiConstants.allCommandParamName)) { + if (!StringUtils.isEmpty(command) && !command.equalsIgnoreCase(RescheduleLoansApiConstants.allCommandParamName)) { sql = sql + " where lrr.status_enum = ? "; Integer statusParam = 100; if (command.equalsIgnoreCase(RescheduleLoansApiConstants.approveCommandParamName)) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/domain/SavingsAccount.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/domain/SavingsAccount.java index a65f9b602..e5425c3a9 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/domain/SavingsAccount.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/domain/SavingsAccount.java @@ -2510,7 +2510,6 @@ public class SavingsAccount extends AbstractPersistableCustom { // final List transactionsSortedByDate = retrieveListOfTransactions(); Money runningBalance = this.summary.getAccountBalance(getCurrency()); Money minRequiredBalance = minRequiredBalanceDerived(getCurrency()); - org.joda.time.LocalDate lastSavingsDate = null; final BigDecimal withdrawalFee = null; // check last txn date