From 479a52013d9dfbe75ff56904f669a5a6ef13ed3d Mon Sep 17 00:00:00 2001 From: Peter Bagrij Date: Thu, 29 Jun 2023 16:44:23 +0200 Subject: [PATCH] FINERACT-1678 - COB Partitioning tasks creates fully balanced id ranges where the number of the loans between the min max are matching with the defined pageSize. --- .../fineract/cob/data/LoanCOBParameter.java | 2 + .../fineract/cob/data/LoanCOBPartition.java | 32 +++ .../loanaccount/domain/LoanRepository.java | 13 - .../cob/api/InternalCOBApiResource.java | 80 ++++++ .../cob/loan/LoanCOBManagerConfiguration.java | 24 +- .../fineract/cob/loan/LoanCOBPartitioner.java | 60 ++-- .../cob/loan/LoanIdParameterTasklet.java | 58 ---- ...RetrieveAllNonClosedLoanIdServiceImpl.java | 31 +- .../cob/loan/RetrieveLoanIdConfiguration.java | 6 +- .../cob/loan/RetrieveLoanIdService.java | 3 +- .../LoanCOBPartitionerStepDefinitions.java | 136 --------- .../cob/loan/LoanCOBPartitionerTest.java | 135 +++++++++ ...ieveAllNonClosedLoanIdServiceImplTest.java | 93 ++++++ .../cob/loan/cob.loan.partitioner.feature | 32 --- .../src/test/resources/logback.xml | 1 + .../cob/CobPartitioningTest.java | 268 ++++++++++++++++++ .../common/loans/CobHelper.java | 38 +++ 17 files changed, 729 insertions(+), 283 deletions(-) create mode 100644 fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBPartition.java create mode 100644 fineract-provider/src/main/java/org/apache/fineract/cob/api/InternalCOBApiResource.java delete mode 100644 fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanIdParameterTasklet.java delete mode 100644 fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerStepDefinitions.java create mode 100644 fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerTest.java create mode 100644 fineract-provider/src/test/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImplTest.java delete mode 100644 fineract-provider/src/test/resources/features/cob/loan/cob.loan.partitioner.feature create mode 100644 integration-tests/src/test/java/org/apache/fineract/integrationtests/cob/CobPartitioningTest.java create mode 100644 integration-tests/src/test/java/org/apache/fineract/integrationtests/common/loans/CobHelper.java diff --git a/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBParameter.java b/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBParameter.java index b6e2b5ed9..36f5d22a0 100644 --- a/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBParameter.java +++ b/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBParameter.java @@ -20,6 +20,7 @@ package org.apache.fineract.cob.data; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; @@ -27,6 +28,7 @@ import lombok.NoArgsConstructor; @Getter @NoArgsConstructor @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) +@EqualsAndHashCode public class LoanCOBParameter { private Long minLoanId; diff --git a/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBPartition.java b/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBPartition.java new file mode 100644 index 000000000..9039c1ab6 --- /dev/null +++ b/fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBPartition.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.data; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class LoanCOBPartition { + + private Long minId; + private Long maxId; + private Long pageNo; + private Long count; +} diff --git a/fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java b/fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java index 70081a7b7..fce514133 100644 --- a/fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java +++ b/fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java @@ -22,7 +22,6 @@ import java.time.LocalDate; import java.util.Collection; import java.util.List; import java.util.Optional; -import org.apache.fineract.cob.data.LoanCOBParameter; import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo; import org.apache.fineract.cob.data.LoanIdAndExternalIdAndStatus; import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; @@ -77,12 +76,6 @@ public interface LoanRepository extends JpaRepository, JpaSpecificat String FIND_ALL_NON_CLOSED = "select loan.id from Loan loan where loan.loanStatus in (100,200,300,303,304)"; - String FIND_MIN_AND_MAX_NON_CLOSED_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE = "select new org.apache.fineract.cob.data.LoanCOBParameter(min(loan.id), max(loan.id)) from Loan loan where loan.loanStatus in (100,200,300,303,304) " - + "and (:businessDate = loan.lastClosedBusinessDate or loan.lastClosedBusinessDate is NULL)"; - - String FIND_MIN_AND_MAX_NON_CLOSED_AND_NON_NULL_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE = "select new org.apache.fineract.cob.data.LoanCOBParameter(min(loan.id), max(loan.id)) from Loan loan where loan.loanStatus in (100,200,300,303,304) " - + "and :businessDate = loan.lastClosedBusinessDate"; - String FIND_NON_CLOSED_LOAN_THAT_BELONGS_TO_CLIENT = "select loan from Loan loan where loan.id = :loanId and loan.loanStatus = 300 and loan.client.id = :clientId"; String FIND_BY_ACCOUNT_NUMBER = "select loan from Loan loan where loan.accountNumber = :accountNumber"; @@ -211,12 +204,6 @@ public interface LoanRepository extends JpaRepository, JpaSpecificat @Query(FIND_ID_BY_EXTERNAL_ID) Long findIdByExternalId(@Param("externalId") ExternalId externalId); - @Query(FIND_MIN_AND_MAX_NON_CLOSED_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE) - LoanCOBParameter findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDate(@Param("businessDate") LocalDate businessDate); - - @Query(FIND_MIN_AND_MAX_NON_CLOSED_AND_NON_NULL_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE) - LoanCOBParameter findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDateNotNull(@Param("businessDate") LocalDate businessDate); - @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_BY_LOAN_IDS) List findAllNonClosedLoansBehindByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate, @Param("loanIds") List loanIds); diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/api/InternalCOBApiResource.java b/fineract-provider/src/main/java/org/apache/fineract/cob/api/InternalCOBApiResource.java new file mode 100644 index 000000000..3a4165617 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/api/InternalCOBApiResource.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.api; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.UriInfo; +import java.time.LocalDate; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.cob.data.LoanCOBPartition; +import org.apache.fineract.cob.loan.LoanCOBConstant; +import org.apache.fineract.cob.loan.RetrieveLoanIdService; +import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.core.api.ApiRequestParameterHelper; +import org.apache.fineract.infrastructure.core.serialization.ApiRequestJsonSerializationSettings; +import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer; +import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +@Profile("test") +@Component +@Path("/v1/internal/cob") +@RequiredArgsConstructor +@Slf4j +public class InternalCOBApiResource implements InitializingBean { + + private final RetrieveLoanIdService retrieveLoanIdService; + private final ApiRequestParameterHelper apiRequestParameterHelper; + private final ToApiJsonSerializer toApiJsonSerializerForList; + + @Override + public void afterPropertiesSet() throws Exception { + log.warn("------------------------------------------------------------"); + log.warn(" "); + log.warn("DO NOT USE THIS IN PRODUCTION!"); + log.warn("Internal client services mode is enabled"); + log.warn("DO NOT USE THIS IN PRODUCTION!"); + log.warn(" "); + log.warn("------------------------------------------------------------"); + } + + @GET + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + @Path("partitions/{partitionSize}") + public String getCobPartitions(@Context final UriInfo uriInfo, @PathParam("partitionSize") int partitionSize) { + LocalDate businessDate = ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.BUSINESS_DATE); + log.info("RetrieveLoanCOBPartitions is called with partitionSize {} for {}", partitionSize, businessDate); + List loanCOBPartitions = retrieveLoanIdService.retrieveLoanCOBPartitions(LoanCOBConstant.NUMBER_OF_DAYS_BEHIND, + businessDate, false, partitionSize); + final ApiRequestJsonSerializationSettings settings = this.apiRequestParameterHelper.process(uriInfo.getQueryParameters()); + return toApiJsonSerializerForList.serialize(settings, loanCOBPartitions); + } + +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java index f42625a16..de25ef946 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java @@ -20,7 +20,6 @@ package org.apache.fineract.cob.loan; import org.apache.fineract.cob.COBBusinessStepService; import org.apache.fineract.cob.common.CustomJobParameterResolver; -import org.apache.fineract.cob.data.LoanCOBParameter; import org.apache.fineract.cob.listener.COBExecutionListenerRunner; import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService; import org.apache.fineract.infrastructure.jobs.service.JobName; @@ -38,7 +37,6 @@ import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.integration.config.annotation.EnableBatchIntegration; import org.springframework.batch.integration.partition.RemotePartitioningManagerStepBuilderFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -78,19 +76,15 @@ public class LoanCOBManagerConfiguration { @Bean @JobScope - public LoanCOBPartitioner partitioner(@Value("#{jobExecutionContext['loanCobParameter']}") LoanCOBParameter loanCOBParameter) { - return new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, loanCOBParameter); + public LoanCOBPartitioner partitioner() { + return new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, + LoanCOBConstant.NUMBER_OF_DAYS_BEHIND); } @Bean public Step loanCOBStep() { return stepBuilderFactory.get(LoanCOBConstant.LOAN_COB_PARTITIONER_STEP) - .partitioner(LoanCOBConstant.LOAN_COB_WORKER_STEP, partitioner(null)).outputChannel(outboundRequests).build(); - } - - @Bean - public Step loanIdParameterStep() { - return new StepBuilder("Set loan ID parameter - Step", jobRepository).tasklet(loanIdParameterTasklet(), transactionManager).build(); + .partitioner(LoanCOBConstant.LOAN_COB_WORKER_STEP, partitioner()).outputChannel(outboundRequests).build(); } @Bean @@ -105,12 +99,6 @@ public class LoanCOBManagerConfiguration { .build(); } - @Bean - @JobScope - public LoanIdParameterTasklet loanIdParameterTasklet() { - return new LoanIdParameterTasklet(retrieveLoanIdService, customJobParameterResolver); - } - @Bean @JobScope public ResolveLoanCOBCustomJobParametersTasklet resolveCustomJobParametersTasklet() { @@ -128,7 +116,7 @@ public class LoanCOBManagerConfiguration { return new JobBuilder(JobName.LOAN_COB.name(), jobRepository) // .listener(new COBExecutionListenerRunner(applicationContext, JobName.LOAN_COB.name())) // .start(resolveCustomJobParametersStep()) // - .next(loanIdParameterStep()).next(loanCOBStep()).next(stayedLockedStep()) // + .next(loanCOBStep()).next(stayedLockedStep()) // .incrementer(new RunIdIncrementer()) // .build(); } @@ -136,7 +124,7 @@ public class LoanCOBManagerConfiguration { @Bean public ExecutionContextPromotionListener customJobParametersPromotionListener() { ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener(); - listener.setKeys(new String[] { LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME }); + listener.setKeys(new String[] { LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME, LoanCOBConstant.IS_CATCH_UP_PARAMETER_NAME }); return listener; } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBPartitioner.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBPartitioner.java index cc8da931a..e01ffdc8e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBPartitioner.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBPartitioner.java @@ -18,18 +18,19 @@ */ package org.apache.fineract.cob.loan; -import com.google.common.collect.Lists; -import java.util.HashMap; +import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.stream.LongStream; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.fineract.cob.COBBusinessStepService; import org.apache.fineract.cob.data.BusinessStepNameAndOrder; import org.apache.fineract.cob.data.LoanCOBParameter; +import org.apache.fineract.cob.data.LoanCOBPartition; import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.springbatch.PropertyService; import org.jetbrains.annotations.NotNull; @@ -40,6 +41,8 @@ import org.springframework.batch.core.launch.JobOperator; import org.springframework.batch.core.launch.NoSuchJobExecutionException; import org.springframework.batch.core.partition.support.Partitioner; import org.springframework.batch.item.ExecutionContext; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.util.StopWatch; @Slf4j @RequiredArgsConstructor @@ -49,10 +52,18 @@ public class LoanCOBPartitioner implements Partitioner { private final PropertyService propertyService; private final COBBusinessStepService cobBusinessStepService; + private final RetrieveLoanIdService retrieveLoanIdService; private final JobOperator jobOperator; private final JobExplorer jobExplorer; - private final LoanCOBParameter minAndMaxLoanId; + private final Long numberOfDays; + + @Value("#{jobExecutionContext['BusinessDate']}") + @Setter + private LocalDate businessDate; + @Value("#{jobExecutionContext['IS_CATCH_UP']}") + @Setter + private Boolean isCatchUp; @NotNull @Override @@ -64,31 +75,38 @@ public class LoanCOBPartitioner implements Partitioner { } private Map getPartitions(int partitionSize, Set cobBusinessSteps) { - Map partitions = new HashMap<>(); if (cobBusinessSteps.isEmpty()) { stopJobExecution(); return Map.of(); } - if (!Objects.isNull(minAndMaxLoanId)) { - List loanIdsInRange = LongStream.rangeClosed(minAndMaxLoanId.getMinLoanId(), minAndMaxLoanId.getMaxLoanId()).boxed() - .toList(); - List> loanIdPartitions = Lists.partition(loanIdsInRange, partitionSize); - for (int i = 0; i < loanIdPartitions.size(); i++) { - createNewPartition(partitions, i + 1, cobBusinessSteps, loanIdPartitions.get(i)); - } - } else { - createNewPartition(partitions, 1, cobBusinessSteps, List.of(0L)); + StopWatch sw = new StopWatch(); + sw.start(); + List loanCOBPartitions = new ArrayList<>(retrieveLoanIdService.retrieveLoanCOBPartitions(numberOfDays, + businessDate, isCatchUp != null ? isCatchUp : false, partitionSize)); + sw.stop(); + // if there is no loan to be closed, we still would like to create at least one partition + + if (loanCOBPartitions.size() == 0) { + loanCOBPartitions.add(new LoanCOBPartition(0L, 0L, 1L, 0L)); } - return partitions; + log.info( + "LoanCOBPartitioner found {} loans to be processed as part of COB. {} partitions were created using partition size {}. RetrieveLoanCOBPartitions was executed in {} ms.", + getLoanCount(loanCOBPartitions), loanCOBPartitions.size(), partitionSize, sw.getTotalTimeMillis()); + return loanCOBPartitions.stream() + .collect(Collectors.toMap(l -> PARTITION_PREFIX + l.getPageNo(), l -> createNewPartition(cobBusinessSteps, l))); } - private void createNewPartition(Map partitions, int partitionIndex, - Set cobBusinessSteps, List loanIds) { + private long getLoanCount(List loanCOBPartitions) { + return loanCOBPartitions.stream().map(LoanCOBPartition::getCount).reduce(0L, Long::sum); + } + + private ExecutionContext createNewPartition(Set cobBusinessSteps, LoanCOBPartition loanCOBPartition) { ExecutionContext executionContext = new ExecutionContext(); executionContext.put(LoanCOBConstant.BUSINESS_STEPS, cobBusinessSteps); - executionContext.put(LoanCOBConstant.LOAN_COB_PARAMETER, new LoanCOBParameter(loanIds.get(0), loanIds.get(loanIds.size() - 1))); - executionContext.put("partition", PARTITION_PREFIX + partitionIndex); - partitions.put(PARTITION_PREFIX + partitionIndex, executionContext); + executionContext.put(LoanCOBConstant.LOAN_COB_PARAMETER, + new LoanCOBParameter(loanCOBPartition.getMinId(), loanCOBPartition.getMaxId())); + executionContext.put("partition", PARTITION_PREFIX + loanCOBPartition.getPageNo()); + return executionContext; } private void stopJobExecution() { diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanIdParameterTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanIdParameterTasklet.java deleted file mode 100644 index 781bb051a..000000000 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanIdParameterTasklet.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.fineract.cob.loan; - -import java.time.LocalDate; -import java.util.Objects; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.fineract.cob.common.CustomJobParameterResolver; -import org.apache.fineract.cob.data.LoanCOBParameter; -import org.springframework.batch.core.StepContribution; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.batch.repeat.RepeatStatus; - -@Slf4j -@RequiredArgsConstructor -public class LoanIdParameterTasklet implements Tasklet { - - private final RetrieveLoanIdService retrieveLoanIdService; - private final CustomJobParameterResolver customJobParameterResolver; - - @Override - public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { - String businessDateParameter = (String) contribution.getStepExecution().getJobExecution().getExecutionContext() - .get(LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME); - LocalDate businessDate = LocalDate.parse(Objects.requireNonNull(businessDateParameter)); - LoanCOBParameter minAndMaxLoanId = retrieveLoanIdService.retrieveMinAndMaxLoanIdsNDaysBehind(LoanCOBConstant.NUMBER_OF_DAYS_BEHIND, - businessDate, customJobParameterResolver.getCustomJobParameterById(chunkContext.getStepContext().getStepExecution(), - LoanCOBConstant.IS_CATCH_UP_PARAMETER_NAME).map(Boolean::parseBoolean).orElse(false)); - if (Objects.isNull(minAndMaxLoanId) - || (Objects.isNull(minAndMaxLoanId.getMinLoanId()) && Objects.isNull(minAndMaxLoanId.getMaxLoanId()))) { - contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.LOAN_COB_PARAMETER, - new LoanCOBParameter(0L, 0L)); - return RepeatStatus.FINISHED; - } - - contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.LOAN_COB_PARAMETER, minAndMaxLoanId); - - return RepeatStatus.FINISHED; - } -} diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImpl.java index ecc1f6ebd..d4553d3de 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImpl.java @@ -18,28 +18,53 @@ */ package org.apache.fineract.cob.loan; +import java.sql.ResultSet; +import java.sql.SQLException; import java.time.LocalDate; import java.util.List; import lombok.RequiredArgsConstructor; import org.apache.fineract.cob.data.LoanCOBParameter; +import org.apache.fineract.cob.data.LoanCOBPartition; import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo; import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @RequiredArgsConstructor public class RetrieveAllNonClosedLoanIdServiceImpl implements RetrieveLoanIdService { private final LoanRepository loanRepository; + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Override - public LoanCOBParameter retrieveMinAndMaxLoanIdsNDaysBehind(Long numberOfDays, LocalDate businessDate, boolean isCatchUp) { + public List retrieveLoanCOBPartitions(Long numberOfDays, LocalDate businessDate, boolean isCatchUp, + int partitionSize) { + StringBuilder sql = new StringBuilder(); + sql.append("select min(id) as min, max(id) as max, page, count(id) as count from "); + sql.append(" (select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from "); + sql.append(" (select id from m_loan where loan_status_id in (:statusIds) and "); if (isCatchUp) { - return loanRepository.findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDateNotNull(businessDate.minusDays(numberOfDays)); + sql.append("last_closed_business_date = :businessDate "); } else { - return loanRepository.findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDate(businessDate.minusDays(numberOfDays)); + sql.append("(last_closed_business_date = :businessDate or last_closed_business_date is null) "); } + sql.append("order by id) t) t2 "); + sql.append("group by page "); + sql.append("order by page"); + + MapSqlParameterSource parameters = new MapSqlParameterSource(); + parameters.addValue("pageSize", partitionSize); + parameters.addValue("statusIds", List.of(100, 200, 300, 303, 304)); + parameters.addValue("businessDate", businessDate.minusDays(numberOfDays)); + return namedParameterJdbcTemplate.query(sql.toString(), parameters, RetrieveAllNonClosedLoanIdServiceImpl::mapRow); + } + + private static LoanCOBPartition mapRow(ResultSet rs, int rowNum) throws SQLException { + return new LoanCOBPartition(rs.getLong("min"), rs.getLong("max"), rs.getLong("page"), rs.getLong("count")); } @Override diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdConfiguration.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdConfiguration.java index d878d0de0..43aa60e6e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdConfiguration.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdConfiguration.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @Configuration public class RetrieveLoanIdConfiguration { @@ -30,9 +31,12 @@ public class RetrieveLoanIdConfiguration { @Autowired private LoanRepository loanRepository; + @Autowired + private NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Bean @ConditionalOnMissingBean public RetrieveLoanIdService retrieveLoanIdService() { - return new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository); + return new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository, namedParameterJdbcTemplate); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdService.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdService.java index 681e0ab26..4eef38322 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdService.java @@ -21,13 +21,14 @@ package org.apache.fineract.cob.loan; import java.time.LocalDate; import java.util.List; import org.apache.fineract.cob.data.LoanCOBParameter; +import org.apache.fineract.cob.data.LoanCOBPartition; import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo; import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; import org.springframework.data.repository.query.Param; public interface RetrieveLoanIdService { - LoanCOBParameter retrieveMinAndMaxLoanIdsNDaysBehind(Long numberOfDays, LocalDate businessDate, boolean isCatchUp); + List retrieveLoanCOBPartitions(Long numberOfDays, LocalDate businessDate, boolean isCatchUp, int partitionSize); List retrieveLoanIdsBehindDateOrNull(LocalDate businessDate, List loanIds); diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerStepDefinitions.java b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerStepDefinitions.java deleted file mode 100644 index 91b5fbb0f..000000000 --- a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerStepDefinitions.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.fineract.cob.loan; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -import com.google.gson.Gson; -import io.cucumber.java8.En; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import org.apache.fineract.cob.COBBusinessStepService; -import org.apache.fineract.cob.data.BusinessStepNameAndOrder; -import org.apache.fineract.cob.data.LoanCOBParameter; -import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper; -import org.apache.fineract.infrastructure.jobs.service.JobName; -import org.apache.fineract.infrastructure.springbatch.PropertyService; -import org.mockito.Mockito; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.explore.JobExplorer; -import org.springframework.batch.core.launch.JobOperator; -import org.springframework.batch.item.ExecutionContext; - -public class LoanCOBPartitionerStepDefinitions implements En { - - PropertyService propertyService = mock(PropertyService.class); - COBBusinessStepService cobBusinessStepService = mock(COBBusinessStepService.class); - JobOperator jobOperator = mock(JobOperator.class); - JobExplorer jobExplorer = mock(JobExplorer.class); - private final Gson gson = GoogleGsonSerializerHelper.createSimpleGson(); - - LoanCOBParameter loanIds; - private LoanCOBPartitioner loanCOBPartitioner; - - private Set cobBusinessSteps = new HashSet<>(); - - private Map resultItem; - private String action; - - public LoanCOBPartitionerStepDefinitions() { - Given("/^The LoanCOBPartitioner.partition method with action (.*)$/", (String action) -> { - - this.action = action; - lenient().when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(2); - if ("empty steps".equals(action)) { - lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) - .thenReturn(Collections.emptySet()); - lenient().when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(new JobExecution(3L))); - lenient().when(jobOperator.stop(3L)).thenReturn(Boolean.TRUE); - loanIds = null; - } else if ("empty loanIds".equals(action)) { - cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L)); - lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) - .thenReturn(cobBusinessSteps); - loanIds = null; - } else if ("good".equals(action)) { - cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L)); - lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) - .thenReturn(cobBusinessSteps); - loanIds = new LoanCOBParameter(1L, 3L); - } - loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, loanIds); - }); - - When("LoanCOBPartitioner.partition method executed", () -> { - resultItem = this.loanCOBPartitioner.partition(2); - }); - - Then("LoanCOBPartitioner.partition result should match", () -> { - if ("empty steps".equals(action)) { - verify(jobOperator, Mockito.times(1)).stop(3L); - assertTrue(resultItem.isEmpty()); - } else if ("good".equals(action)) { - verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong()); - assertEquals(2, resultItem.size()); - assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1")); - Set businessSteps = (Set) resultItem - .get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(), - businessSteps.stream().findFirst().get().getStepOrder()); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(), - businessSteps.stream().findFirst().get().getStepName()); - LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1") - .get(LoanCOBConstant.LOAN_COB_PARAMETER); - assertEquals(1, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId()); - assertEquals(1L, loanCOBParameter.getMinLoanId()); - assertEquals(2L, loanCOBParameter.getMaxLoanId()); - assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "2")); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(), - businessSteps.stream().findFirst().get().getStepOrder()); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(), - businessSteps.stream().findFirst().get().getStepName()); - loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "2") - .get(LoanCOBConstant.LOAN_COB_PARAMETER); - assertEquals(0, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId()); - assertEquals(3L, loanCOBParameter.getMinLoanId()); - assertEquals(3L, loanCOBParameter.getMaxLoanId()); - } else if ("empty loanIds".equals(action)) { - verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong()); - assertEquals(1, resultItem.size()); - assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1")); - Set businessSteps = (Set) resultItem - .get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(), - businessSteps.stream().findFirst().get().getStepOrder()); - assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(), - businessSteps.stream().findFirst().get().getStepName()); - LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1") - .get(LoanCOBConstant.LOAN_COB_PARAMETER); - assertEquals(0L, loanCOBParameter.getMinLoanId()); - assertEquals(0L, loanCOBParameter.getMaxLoanId()); - } - }); - } -} diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerTest.java b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerTest.java new file mode 100644 index 000000000..0ec011905 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.fineract.cob.COBBusinessStepService; +import org.apache.fineract.cob.data.BusinessStepNameAndOrder; +import org.apache.fineract.cob.data.LoanCOBParameter; +import org.apache.fineract.cob.data.LoanCOBPartition; +import org.apache.fineract.infrastructure.jobs.service.JobName; +import org.apache.fineract.infrastructure.springbatch.PropertyService; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.launch.JobExecutionNotRunningException; +import org.springframework.batch.core.launch.JobOperator; +import org.springframework.batch.core.launch.NoSuchJobExecutionException; +import org.springframework.batch.item.ExecutionContext; + +@ExtendWith(MockitoExtension.class) +class LoanCOBPartitionerTest { + + private static final Set BUSINESS_STEP_SET = Set.of(new BusinessStepNameAndOrder("Business step", 1L)); + private static final LocalDate BUSINESS_DATE = LocalDate.parse("2023-06-28"); + @Mock + private PropertyService propertyService; + @Mock + private COBBusinessStepService cobBusinessStepService; + @Mock + private RetrieveLoanIdService retrieveLoanIdService; + @Mock + private JobOperator jobOperator; + @Mock + private JobExplorer jobExplorer; + + @Test + public void testLoanCOBPartitioner() { + //given + when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5); + when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) + .thenReturn(BUSINESS_STEP_SET); + when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5)) + .thenReturn(List.of(new LoanCOBPartition(1L,10L, 1L, 5L), new LoanCOBPartition(11L,20L, 2L, 4L))); + LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L); + loanCOBPartitioner.setBusinessDate(BUSINESS_DATE); + loanCOBPartitioner.setIsCatchUp(false); + + //when + Map partitions = loanCOBPartitioner.partition(1); + + //then + Assertions.assertEquals(2, partitions.size()); + validatePartitions(partitions, 1, 1, 10); + validatePartitions(partitions, 2, 11, 20); + } + + @Test + public void testLoanCOBPartitionerEmptyBusinessSteps() throws NoSuchJobExecutionException, JobExecutionNotRunningException { + //given + when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5); + when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) + .thenReturn(Set.of()); + JobExecution jobExecution = Mockito.mock(JobExecution.class); + when(jobExecution.getId()).thenReturn(123L); + when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(jobExecution)); + LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L); + loanCOBPartitioner.setBusinessDate(BUSINESS_DATE); + loanCOBPartitioner.setIsCatchUp(false); + + //when + Map partitions = loanCOBPartitioner.partition(1); + + //then + Assertions.assertEquals(0, partitions.size()); + verify(jobExplorer, times(1)).findRunningJobExecutions(JobName.LOAN_COB.name()); + verify(jobOperator, times(1)).stop(123L); + } + + @Test + public void testLoanCOBPartitionerNoLoansFound() { + //given + when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5); + when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME)) + .thenReturn(BUSINESS_STEP_SET); + when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5)) + .thenReturn(List.of()); + LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L); + loanCOBPartitioner.setBusinessDate(BUSINESS_DATE); + loanCOBPartitioner.setBusinessDate(BUSINESS_DATE); + loanCOBPartitioner.setIsCatchUp(false); + + //when + Map partitions = loanCOBPartitioner.partition(1); + + //then + Assertions.assertEquals(1, partitions.size()); + validatePartitions(partitions, 1, 0, 0); + } + + private void validatePartitions(Map partitions, int index, long min, long max) { + Assertions.assertEquals(BUSINESS_STEP_SET, + partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.BUSINESS_STEPS)); + Assertions.assertEquals(new LoanCOBParameter(min, max), + partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.LOAN_COB_PARAMETER)); + Assertions.assertEquals("partition_" + index, partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get("partition")); + } +} diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImplTest.java b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImplTest.java new file mode 100644 index 000000000..668ebffa4 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImplTest.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.cob.loan; + +import static org.mockito.Mockito.times; + +import java.time.LocalDate; +import java.util.List; +import org.apache.fineract.cob.data.LoanCOBPartition; +import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; + +@ExtendWith(MockitoExtension.class) +public class RetrieveAllNonClosedLoanIdServiceImplTest { + + @Mock + private LoanRepository loanRepository; + @Mock + private NamedParameterJdbcTemplate namedParameterJdbcTemplate; + @Captor + private ArgumentCaptor sqlCaptor; + @Captor + private ArgumentCaptor paramsCaptor; + @Captor + private ArgumentCaptor> rowMapper; + + @Test + public void testRetrieveLoanCOBPartitionsNoCatchup() { + String expectedSQL = """ + select min(id) as min, max(id) as max, page, count(id) as count from + (select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from + (select id from m_loan where loan_status_id in (:statusIds) and (last_closed_business_date = :businessDate or last_closed_business_date is null) order by id) t) t2 + group by page + order by page + """; + testRetrieveLoanCOBPartitions(expectedSQL, false); + } + + @Test + public void testRetrieveLoanCOBPartitionsCatchup() { + String expectedSQL = """ + select min(id) as min, max(id) as max, page, count(id) as count from + (select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from + (select id from m_loan where loan_status_id in (:statusIds) and last_closed_business_date = :businessDate order by id) t) t2 + group by page + order by page + """; + testRetrieveLoanCOBPartitions(expectedSQL, true); + } + + private void testRetrieveLoanCOBPartitions(String expectedSQL, boolean isCatchup) { + RetrieveAllNonClosedLoanIdServiceImpl service = new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository, + namedParameterJdbcTemplate); + LocalDate businessDate = LocalDate.parse("2023-06-28"); + service.retrieveLoanCOBPartitions(1L, businessDate, isCatchup, 5); + Mockito.verify(namedParameterJdbcTemplate, times(1)).query(sqlCaptor.capture(), paramsCaptor.capture(), rowMapper.capture()); + Assertions.assertEquals(normalize(expectedSQL), normalize(sqlCaptor.getValue())); + Assertions.assertEquals(5, paramsCaptor.getValue().getValue("pageSize")); + Assertions.assertEquals(List.of(100, 200, 300, 303, 304), paramsCaptor.getValue().getValue("statusIds")); + Assertions.assertEquals(LocalDate.parse("2023-06-27"), paramsCaptor.getValue().getValue("businessDate")); + + } + + private String normalize(String str) { + return str.replaceAll(" +", " ").replaceAll("\r?\n", ""); + } +} diff --git a/fineract-provider/src/test/resources/features/cob/loan/cob.loan.partitioner.feature b/fineract-provider/src/test/resources/features/cob/loan/cob.loan.partitioner.feature deleted file mode 100644 index 16edca086..000000000 --- a/fineract-provider/src/test/resources/features/cob/loan/cob.loan.partitioner.feature +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -Feature: COB Loan partitioner Step - - @cob - Scenario Outline: LoanCOBPartitioner - run test - Given The LoanCOBPartitioner.partition method with action - When LoanCOBPartitioner.partition method executed - Then LoanCOBPartitioner.partition result should match - - Examples: - |action| - |empty steps| - |good| - |empty loanIds| diff --git a/fineract-provider/src/test/resources/logback.xml b/fineract-provider/src/test/resources/logback.xml index 9f704a984..ec106e90b 100644 --- a/fineract-provider/src/test/resources/logback.xml +++ b/fineract-provider/src/test/resources/logback.xml @@ -20,6 +20,7 @@ --> + false diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/cob/CobPartitioningTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/cob/CobPartitioningTest.java new file mode 100644 index 000000000..37fa51003 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/cob/CobPartitioningTest.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.integrationtests.cob; + +import static org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType.BUSINESS_DATE; + +import io.restassured.builder.RequestSpecBuilder; +import io.restassured.builder.ResponseSpecBuilder; +import io.restassured.http.ContentType; +import io.restassured.path.json.JsonPath; +import io.restassured.specification.RequestSpecification; +import io.restassured.specification.ResponseSpecification; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.accounting.common.AccountingConstants; +import org.apache.fineract.client.models.GetFinancialActivityAccountsResponse; +import org.apache.fineract.client.models.PostFinancialActivityAccountsRequest; +import org.apache.fineract.integrationtests.common.BusinessDateHelper; +import org.apache.fineract.integrationtests.common.BusinessStepHelper; +import org.apache.fineract.integrationtests.common.ClientHelper; +import org.apache.fineract.integrationtests.common.CollateralManagementHelper; +import org.apache.fineract.integrationtests.common.ExternalAssetOwnerHelper; +import org.apache.fineract.integrationtests.common.GlobalConfigurationHelper; +import org.apache.fineract.integrationtests.common.SchedulerJobHelper; +import org.apache.fineract.integrationtests.common.Utils; +import org.apache.fineract.integrationtests.common.accounting.Account; +import org.apache.fineract.integrationtests.common.accounting.AccountHelper; +import org.apache.fineract.integrationtests.common.accounting.FinancialActivityAccountHelper; +import org.apache.fineract.integrationtests.common.charges.ChargesHelper; +import org.apache.fineract.integrationtests.common.loans.CobHelper; +import org.apache.fineract.integrationtests.common.loans.LoanApplicationTestBuilder; +import org.apache.fineract.integrationtests.common.loans.LoanProductTestBuilder; +import org.apache.fineract.integrationtests.common.loans.LoanStatusChecker; +import org.apache.fineract.integrationtests.common.loans.LoanTestLifecycleExtension; +import org.apache.fineract.integrationtests.common.loans.LoanTransactionHelper; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@SuppressWarnings("rawtypes") +@Slf4j +@ExtendWith(LoanTestLifecycleExtension.class) +public class CobPartitioningTest { + + public static final int N = 10; + private static ResponseSpecification RESPONSE_SPEC; + private static RequestSpecification REQUEST_SPEC; + private static Account ASSET_ACCOUNT; + private static Account FEE_PENALTY_ACCOUNT; + private static Account TRANSFER_ACCOUNT; + private static Account EXPENSE_ACCOUNT; + private static Account INCOME_ACCOUNT; + private static Account OVERPAYMENT_ACCOUNT; + private static FinancialActivityAccountHelper FINANCIAL_ACTIVITY_ACCOUNT_HELPER; + private static ExternalAssetOwnerHelper EXTERNAL_ASSET_OWNER_HELPER; + private static LoanTransactionHelper LOAN_TRANSACTION_HELPER; + private static SchedulerJobHelper SCHEDULER_JOB_HELPER; + private static LocalDate TODAYS_DATE; + + @BeforeAll + public static void setupInvestorBusinessStep() { + Utils.initializeRESTAssured(); + REQUEST_SPEC = new RequestSpecBuilder().setContentType(ContentType.JSON).build(); + REQUEST_SPEC.header("Authorization", "Basic " + Utils.loginIntoServerAndGetBase64EncodedAuthenticationKey()); + RESPONSE_SPEC = new ResponseSpecBuilder().expectStatusCode(200).build(); + AccountHelper accountHelper = new AccountHelper(REQUEST_SPEC, RESPONSE_SPEC); + EXTERNAL_ASSET_OWNER_HELPER = new ExternalAssetOwnerHelper(); + SCHEDULER_JOB_HELPER = new SchedulerJobHelper(REQUEST_SPEC); + FINANCIAL_ACTIVITY_ACCOUNT_HELPER = new FinancialActivityAccountHelper(REQUEST_SPEC); + LOAN_TRANSACTION_HELPER = new LoanTransactionHelper(REQUEST_SPEC, RESPONSE_SPEC); + + TODAYS_DATE = Utils.getLocalDateOfTenant(); + new BusinessStepHelper().updateSteps("LOAN_CLOSE_OF_BUSINESS", "APPLY_CHARGE_TO_OVERDUE_LOANS", "LOAN_DELINQUENCY_CLASSIFICATION", + "CHECK_LOAN_REPAYMENT_DUE", "CHECK_LOAN_REPAYMENT_OVERDUE", "UPDATE_LOAN_ARREARS_AGING", "ADD_PERIODIC_ACCRUAL_ENTRIES", + "EXTERNAL_ASSET_OWNER_TRANSFER"); + + ASSET_ACCOUNT = accountHelper.createAssetAccount(); + FEE_PENALTY_ACCOUNT = accountHelper.createAssetAccount(); + TRANSFER_ACCOUNT = accountHelper.createAssetAccount(); + EXPENSE_ACCOUNT = accountHelper.createExpenseAccount(); + INCOME_ACCOUNT = accountHelper.createIncomeAccount(); + OVERPAYMENT_ACCOUNT = accountHelper.createLiabilityAccount(); + + setProperFinancialActivity(TRANSFER_ACCOUNT); + } + + private static void setProperFinancialActivity(Account transferAccount) { + List financialMappings = FINANCIAL_ACTIVITY_ACCOUNT_HELPER.getAllFinancialActivityAccounts(); + financialMappings.forEach(mapping -> FINANCIAL_ACTIVITY_ACCOUNT_HELPER.deleteFinancialActivityAccount(mapping.getId())); + FINANCIAL_ACTIVITY_ACCOUNT_HELPER.createFinancialActivityAccount(new PostFinancialActivityAccountsRequest() + .financialActivityId((long) AccountingConstants.FinancialActivity.ASSET_TRANSFER.getValue()) + .glAccountId((long) transferAccount.getAccountID())); + } + + @Test + public void testLoanCOBPartitioningQuery() throws InterruptedException { + try { + ExecutorService executorService = Executors.newFixedThreadPool(10); + GlobalConfigurationHelper.manageConfigurations(REQUEST_SPEC, RESPONSE_SPEC, + GlobalConfigurationHelper.ENABLE_AUTOGENERATED_EXTERNAL_ID, true); + setInitialBusinessDate("2020-03-02"); + + List loanIds = new CopyOnWriteArrayList<>(); + + // Let's create 1, 2, ..., N-1, N loans + final CountDownLatch createLatch = new CountDownLatch(N); + for (int i = 0; i < N; i++) { + Future unused = executorService.submit(() -> { + Integer clientID = createClient(); + Integer loanID = createLoanForClient(clientID); + loanIds.add(loanID); + createLatch.countDown(); + }); + } + createLatch.await(); + + // Force close loans 3, 4, ... , N-3, N-2 + Collections.sort(loanIds); + final CountDownLatch closeLatch = new CountDownLatch(N - 4); + for (int i = 2; i < N - 2; i++) { + final int idx = i; + Future unused = executorService.submit(() -> { + LOAN_TRANSACTION_HELPER.forecloseLoan("02 March 2020", loanIds.get(idx)); + closeLatch.countDown(); + }); + } + closeLatch.await(); + + // Let's retrieve the partitions + List> cobPartitions = CobHelper.getCobPartitions(REQUEST_SPEC, RESPONSE_SPEC, 3, ""); + log.info("\nLoans created: {},\nRetrieved partitions: {}", loanIds, cobPartitions); + Assertions.assertEquals(2, cobPartitions.size()); + + Assertions.assertEquals(0, cobPartitions.get(0).get("pageNo")); + Assertions.assertEquals(loanIds.get(0), cobPartitions.get(0).get("minId")); + Assertions.assertEquals(loanIds.get(8), cobPartitions.get(0).get("maxId")); + Assertions.assertEquals(3, cobPartitions.get(0).get("count")); + + Assertions.assertEquals(1, cobPartitions.get(1).get("pageNo")); + Assertions.assertEquals(loanIds.get(9), cobPartitions.get(1).get("minId")); + Assertions.assertEquals(loanIds.get(9), cobPartitions.get(1).get("maxId")); + Assertions.assertEquals(1, cobPartitions.get(1).get("count")); + + executorService.shutdown(); + } finally { + cleanUpAndRestoreBusinessDate(); + } + } + + private void setInitialBusinessDate(String date) { + GlobalConfigurationHelper.updateIsBusinessDateEnabled(REQUEST_SPEC, RESPONSE_SPEC, Boolean.TRUE); + BusinessDateHelper.updateBusinessDate(REQUEST_SPEC, RESPONSE_SPEC, BUSINESS_DATE, LocalDate.parse(date)); + GlobalConfigurationHelper.updateValueForGlobalConfiguration(REQUEST_SPEC, RESPONSE_SPEC, "10", "0"); + } + + private void cleanUpAndRestoreBusinessDate() { + REQUEST_SPEC = new RequestSpecBuilder().setContentType(ContentType.JSON).build(); + REQUEST_SPEC.header("Authorization", "Basic " + Utils.loginIntoServerAndGetBase64EncodedAuthenticationKey()); + REQUEST_SPEC.header("Fineract-Platform-TenantId", "default"); + RESPONSE_SPEC = new ResponseSpecBuilder().expectStatusCode(200).build(); + BusinessDateHelper.updateBusinessDate(REQUEST_SPEC, RESPONSE_SPEC, BUSINESS_DATE, TODAYS_DATE); + GlobalConfigurationHelper.updateIsBusinessDateEnabled(REQUEST_SPEC, RESPONSE_SPEC, Boolean.FALSE); + GlobalConfigurationHelper.manageConfigurations(REQUEST_SPEC, RESPONSE_SPEC, + GlobalConfigurationHelper.ENABLE_AUTOGENERATED_EXTERNAL_ID, false); + } + + @NotNull + private Integer createClient() { + final Integer clientID = ClientHelper.createClient(REQUEST_SPEC, RESPONSE_SPEC); + Assertions.assertNotNull(clientID); + return clientID; + } + + @NotNull + private Integer createLoanForClient(Integer clientID) { + Integer overdueFeeChargeId = ChargesHelper.createCharges(REQUEST_SPEC, RESPONSE_SPEC, + ChargesHelper.getLoanOverdueFeeJSONWithCalculationTypePercentage("1")); + Assertions.assertNotNull(overdueFeeChargeId); + + Integer loanProductID = createLoanProduct(overdueFeeChargeId.toString()); + Assertions.assertNotNull(loanProductID); + HashMap loanStatusHashMap; + + Integer loanID = applyForLoanApplication(clientID.toString(), loanProductID.toString(), "10 January 2020"); + + Assertions.assertNotNull(loanID); + + loanStatusHashMap = LoanStatusChecker.getStatusOfLoan(REQUEST_SPEC, RESPONSE_SPEC, loanID); + LoanStatusChecker.verifyLoanIsPending(loanStatusHashMap); + + loanStatusHashMap = LOAN_TRANSACTION_HELPER.approveLoan("01 March 2020", loanID); + LoanStatusChecker.verifyLoanIsApproved(loanStatusHashMap); + + String loanDetails = LOAN_TRANSACTION_HELPER.getLoanDetails(REQUEST_SPEC, RESPONSE_SPEC, loanID); + loanStatusHashMap = LOAN_TRANSACTION_HELPER.disburseLoanWithNetDisbursalAmount("02 March 2020", loanID, + JsonPath.from(loanDetails).get("netDisbursalAmount").toString()); + LoanStatusChecker.verifyLoanIsActive(loanStatusHashMap); + return loanID; + } + + private Integer createLoanProduct(final String chargeId) { + + final String loanProductJSON = new LoanProductTestBuilder().withPrincipal("15,000.00").withNumberOfRepayments("4") + .withRepaymentAfterEvery("1").withRepaymentTypeAsMonth().withinterestRatePerPeriod("1") + .withAccountingRulePeriodicAccrual(new Account[] { ASSET_ACCOUNT, EXPENSE_ACCOUNT, INCOME_ACCOUNT, OVERPAYMENT_ACCOUNT }) + .withInterestRateFrequencyTypeAsMonths().withAmortizationTypeAsEqualInstallments().withInterestTypeAsDecliningBalance() + .withFeeAndPenaltyAssetAccount(FEE_PENALTY_ACCOUNT).build(chargeId); + return LOAN_TRANSACTION_HELPER.getLoanProductId(loanProductJSON); + } + + private Integer applyForLoanApplication(final String clientID, final String loanProductID, final String date) { + List collaterals = new ArrayList<>(); + Integer collateralId = CollateralManagementHelper.createCollateralProduct(REQUEST_SPEC, RESPONSE_SPEC); + Assertions.assertNotNull(collateralId); + Integer clientCollateralId = CollateralManagementHelper.createClientCollateral(REQUEST_SPEC, RESPONSE_SPEC, clientID, collateralId); + Assertions.assertNotNull(clientCollateralId); + addCollaterals(collaterals, clientCollateralId, BigDecimal.valueOf(1)); + + String loanApplicationJSON = new LoanApplicationTestBuilder().withPrincipal("15,000.00").withLoanTermFrequency("4") + .withLoanTermFrequencyAsMonths().withNumberOfRepayments("4").withRepaymentEveryAfter("1") + .withRepaymentFrequencyTypeAsMonths().withInterestRatePerPeriod("2").withAmortizationTypeAsEqualInstallments() + .withInterestTypeAsDecliningBalance().withInterestCalculationPeriodTypeSameAsRepaymentPeriod() + .withExpectedDisbursementDate(date).withSubmittedOnDate(date).withCollaterals(collaterals) + .build(clientID, loanProductID, null); + return LOAN_TRANSACTION_HELPER.getLoanId(loanApplicationJSON); + } + + private void addCollaterals(List collaterals, Integer collateralId, BigDecimal quantity) { + collaterals.add(collaterals(collateralId, quantity)); + } + + private HashMap collaterals(Integer collateralId, BigDecimal quantity) { + HashMap collateral = new HashMap<>(2); + collateral.put("clientCollateralId", collateralId.toString()); + collateral.put("quantity", quantity.toString()); + return collateral; + } + +} diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/loans/CobHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/loans/CobHelper.java new file mode 100644 index 000000000..6b4188a36 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/loans/CobHelper.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.integrationtests.common.loans; + +import io.restassured.specification.RequestSpecification; +import io.restassured.specification.ResponseSpecification; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.integrationtests.client.IntegrationTest; +import org.apache.fineract.integrationtests.common.Utils; + +@Slf4j +public class CobHelper extends IntegrationTest { + + public static List> getCobPartitions(final RequestSpecification requestSpec, + final ResponseSpecification responseSpec, int partitionSize, final String jsonReturn) { + final String GET_LOAN_URL = "/fineract-provider/api/v1/internal/cob/partitions/" + partitionSize + "?" + Utils.TENANT_IDENTIFIER; + log.info("---------------------------------GET COB PARTITIONS---------------------------------------------"); + return Utils.performServerGet(requestSpec, responseSpec, GET_LOAN_URL, jsonReturn); + } +}