From 071cdae69bf67e64e28b54dc866167335add182e Mon Sep 17 00:00:00 2001 From: taskain7 Date: Mon, 13 Mar 2023 08:40:04 +0100 Subject: [PATCH] [FINERACT-1724] making partition for big parameter lists --- .../cob/loan/ApplyLoanLockTasklet.java | 8 +++-- .../cob/loan/FetchAndLockLoanTasklet.java | 10 +++++- .../cob/loan/LoanCOBManagerConfiguration.java | 5 ++- .../cob/loan/LoanCOBWorkerConfiguration.java | 6 +++- .../InlineLoanCOBExecutorServiceImpl.java | 9 +++++- .../core/config/FineractProperties.java | 10 ++++++ .../jobs/filter/LoanCOBApiFilter.java | 12 +++++-- ...ountTransfersWritePlatformServiceImpl.java | 12 +++++-- .../loanaccount/domain/LoanRepository.java | 4 --- .../domain/LoanRepositoryWrapper.java | 18 ++++++----- .../src/main/resources/application.properties | 2 ++ .../ApplyLoanLockTaskletStepDefinitions.java | 9 +++++- .../loan/FetchAndLockLoanStepDefinitions.java | 14 +++++++- .../InlineLoanCOBExecutorServiceImplTest.java | 32 +++++++++++++++++++ .../jobs/filter/LoanCOBApiFilterTest.java | 31 ++++++++++++++---- 15 files changed, 150 insertions(+), 32 deletions(-) diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java index fbfb39a47..35183807c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java @@ -18,6 +18,7 @@ */ package org.apache.fineract.cob.loan; +import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.fineract.cob.domain.LoanAccountLock; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.domain.LockOwner; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.jetbrains.annotations.NotNull; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; @@ -40,13 +42,15 @@ import org.springframework.batch.repeat.RepeatStatus; public class ApplyLoanLockTasklet implements Tasklet { private final LoanAccountLockRepository accountLockRepository; + private final FineractProperties fineractProperties; @Override public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { ExecutionContext executionContext = contribution.getStepExecution().getExecutionContext(); List loanIds = (List) executionContext.get(LoanCOBConstant.LOAN_IDS); - - List accountLocks = accountLockRepository.findAllByLoanIdIn(loanIds); + List> loanIdPartitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit()); + List accountLocks = new ArrayList<>(); + loanIdPartitions.forEach(loanIdPartition -> accountLocks.addAll(accountLockRepository.findAllByLoanIdIn(loanIdPartition))); Map alreadySoftLockedAccountsMap = accountLocks.stream() .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())) diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java index c864ba396..f11c8576b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java @@ -18,6 +18,7 @@ */ package org.apache.fineract.cob.loan; +import com.google.common.collect.Lists; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; @@ -27,6 +28,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.fineract.cob.domain.LoanAccountLock; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.domain.LockOwner; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.jetbrains.annotations.NotNull; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; @@ -43,6 +45,8 @@ public class FetchAndLockLoanTasklet implements Tasklet { private final RetrieveLoanIdService retrieveLoanIdService; + private final FineractProperties fineractProperties; + @Override public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception { String businessDateParameter = (String) contribution.getStepExecution().getJobExecution().getExecutionContext() @@ -54,7 +58,11 @@ public class FetchAndLockLoanTasklet implements Tasklet { } List remainingIds = new ArrayList<>(allNonClosedLoanIds); - List loanAccountLocks = loanAccountLockRepository.findAllByLoanIdIn(remainingIds); + List> remainingIdPartitions = Lists.partition(remainingIds, + fineractProperties.getQuery().getInClauseParameterSizeLimit()); + List loanAccountLocks = new ArrayList<>(); + remainingIdPartitions.forEach( + remainingIdPartition -> loanAccountLocks.addAll(loanAccountLockRepository.findAllByLoanIdIn(remainingIdPartition))); List alreadySoftLockedAccounts = loanAccountLocks.stream() .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList(); diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java index 4fa8ca2e3..eb8b7695f 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java @@ -23,6 +23,7 @@ import org.apache.fineract.cob.COBBusinessStepService; import org.apache.fineract.cob.common.CustomJobParameterResolver; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.listener.COBExecutionListenerRunner; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService; import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.springbatch.PropertyService; @@ -79,6 +80,8 @@ public class LoanCOBManagerConfiguration { private CustomJobParameterResolver customJobParameterResolver; @Autowired private LoanRepository loanRepository; + @Autowired + private FineractProperties fineractProperties; @Bean @JobScope @@ -111,7 +114,7 @@ public class LoanCOBManagerConfiguration { @Bean @JobScope public FetchAndLockLoanTasklet fetchAndLockLoanTasklet() { - return new FetchAndLockLoanTasklet(accountLockRepository, retrieveLoanIdService); + return new FetchAndLockLoanTasklet(accountLockRepository, retrieveLoanIdService, fineractProperties); } @Bean diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java index 0f6bccbb7..00512e67b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java @@ -23,6 +23,7 @@ import org.apache.fineract.cob.common.InitialisationTasklet; import org.apache.fineract.cob.common.ResetContextTasklet; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.jobs.service.JobName; import org.apache.fineract.infrastructure.springbatch.PropertyService; import org.apache.fineract.portfolio.loanaccount.domain.Loan; @@ -67,6 +68,9 @@ public class LoanCOBWorkerConfiguration { @Autowired private TransactionTemplate transactionTemplate; + @Autowired + private FineractProperties fineractProperties; + @Bean(name = LoanCOBConstant.LOAN_COB_WORKER_STEP) public Step loanCOBWorkerStep() { return stepBuilderFactory.get("Loan COB worker - Step").inputChannel(inboundRequests).flow(flow()).build(); @@ -118,7 +122,7 @@ public class LoanCOBWorkerConfiguration { @Bean public ApplyLoanLockTasklet applyLock() { - return new ApplyLoanLockTasklet(accountLockRepository); + return new ApplyLoanLockTasklet(accountLockRepository, fineractProperties); } @Bean diff --git a/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java index 12e31614e..838f06cb3 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java @@ -20,6 +20,7 @@ package org.apache.fineract.cob.service; import static org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW; +import com.google.common.collect.Lists; import com.google.gson.Gson; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -40,6 +41,7 @@ import org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledExcept import org.apache.fineract.cob.loan.LoanCOBConstant; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; import org.apache.fineract.infrastructure.core.api.JsonCommand; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder; import org.apache.fineract.infrastructure.core.exception.PlatformInternalServerException; @@ -85,6 +87,7 @@ public class InlineLoanCOBExecutorServiceImpl implements InlineExecutorService getLoansToBeProcessed(List loanIds, LocalDate cobBusinessDate) { - return loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate, loanIds); + List loanIdAndLastClosedBusinessDates = new ArrayList<>(); + List> partitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit()); + partitions.forEach(partition -> loanIdAndLastClosedBusinessDates + .addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate, partition))); + return loanIdAndLastClosedBusinessDates; } private List getLoanAccountLocks(List loanIds) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java index 2e5fc9d2a..a28aa4ee8 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java @@ -55,6 +55,8 @@ public class FineractProperties { private FineractTemplateProperties template; private FineractJpaProperties jpa; + private FineractQueryProperties query; + @Getter @Setter public static class FineractTenantProperties { @@ -255,4 +257,12 @@ public class FineractProperties { private boolean statementLoggingEnabled; } + + @Getter + @Setter + public static class FineractQueryProperties { + + private int inClauseParameterSizeLimit; + } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java index d8e9a6773..8583f6dec 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java @@ -18,8 +18,10 @@ */ package org.apache.fineract.infrastructure.jobs.filter; +import com.google.common.collect.Lists; import java.io.IOException; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.function.Predicate; @@ -35,9 +37,11 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.batch.domain.BatchRequest; import org.apache.fineract.batch.domain.BatchResponse; +import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; import org.apache.fineract.cob.service.InlineLoanCOBExecutorServiceImpl; import org.apache.fineract.cob.service.LoanAccountLockService; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.data.ApiGlobalErrorResponse; import org.apache.fineract.infrastructure.core.domain.ExternalId; import org.apache.fineract.infrastructure.core.filters.BatchFilter; @@ -64,6 +68,7 @@ public class LoanCOBApiFilter extends OncePerRequestFilter implements BatchFilte private final PlatformSecurityContext context; private final InlineLoanCOBExecutorServiceImpl inlineLoanCOBExecutorService; private final LoanRepository loanRepository; + private final FineractProperties fineractProperties; private final LoanRescheduleRequestRepository loanRescheduleRequestRepository; @@ -141,8 +146,11 @@ public class LoanCOBApiFilter extends OncePerRequestFilter implements BatchFilte } private boolean isLoanBehind(List loanIds) { - return CollectionUtils.isNotEmpty(loanRepository - .findAllNonClosedLoansBehindByLoanIds(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE), loanIds)); + List loanIdAndLastClosedBusinessDates = new ArrayList<>(); + List> partitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit()); + partitions.forEach(partition -> loanIdAndLastClosedBusinessDates.addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( + ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE), partition))); + return CollectionUtils.isNotEmpty(loanIdAndLastClosedBusinessDates); } private List calculateRelevantLoanIds(String pathInfo) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java index e32452f58..ffd011ff0 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java @@ -25,15 +25,18 @@ import static org.apache.fineract.portfolio.account.AccountDetailConstants.toAcc import static org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferAmountParamName; import static org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferDateParamName; +import com.google.common.collect.Lists; import java.math.BigDecimal; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Locale; import lombok.RequiredArgsConstructor; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.core.api.JsonCommand; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder; import org.apache.fineract.infrastructure.core.domain.ExternalId; @@ -86,6 +89,7 @@ public class AccountTransfersWritePlatformServiceImpl implements AccountTransfer private final GSIMRepositoy gsimRepository; private final ConfigurationDomainService configurationDomainService; private final ExternalIdFactory externalIdFactory; + private final FineractProperties fineractProperties; @Transactional @Override @@ -223,11 +227,13 @@ public class AccountTransfersWritePlatformServiceImpl implements AccountTransfer @Transactional public void reverseTransfersWithFromAccountTransactions(final Collection fromTransactionIds, final PortfolioAccountType accountTypeId) { - List acccountTransfers = null; + List acccountTransfers = new ArrayList<>(); if (accountTypeId.isLoanAccount()) { - acccountTransfers = this.accountTransferRepository.findByFromLoanTransactions(fromTransactionIds); + List> partitions = Lists.partition(fromTransactionIds.stream().toList(), + fineractProperties.getQuery().getInClauseParameterSizeLimit()); + partitions.forEach(partition -> acccountTransfers.addAll(this.accountTransferRepository.findByFromLoanTransactions(partition))); } - if (acccountTransfers != null && acccountTransfers.size() > 0) { + if (acccountTransfers.size() > 0) { undoTransactions(acccountTransfers); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java index 08e3cd0ef..2926fba7c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java @@ -192,10 +192,6 @@ public interface LoanRepository extends JpaRepository, JpaSpecificat @Query(FIND_ALL_NON_CLOSED_LOANS_BY_LAST_CLOSED_BUSINESS_DATE) List findAllNonClosedLoanIdsByLastClosedBusinessDate(@Param("businessDate") LocalDate businessDate); - @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_BY_LOAN_IDS) - List findAllNonClosedLoansBehindByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate, - @Param("loanIds") List loanIds); - @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_OR_NULL_BY_LOAN_IDS) List findAllNonClosedLoansBehindOrNullByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate, @Param("loanIds") List loanIds); diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java index f204786f2..07b479af5 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java @@ -18,14 +18,16 @@ */ package org.apache.fineract.portfolio.loanaccount.domain; +import com.google.common.collect.Lists; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import lombok.RequiredArgsConstructor; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.ExternalId; import org.apache.fineract.portfolio.loanaccount.exception.LoanNotFoundException; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -36,14 +38,11 @@ import org.springframework.transaction.annotation.Transactional; *

*/ @Service +@RequiredArgsConstructor public class LoanRepositoryWrapper { private final LoanRepository repository; - - @Autowired - public LoanRepositoryWrapper(final LoanRepository repository) { - this.repository = repository; - } + private final FineractProperties fineractProperties; @Transactional(readOnly = true) public Loan findOneWithNotFoundDetection(final Long id) { @@ -181,8 +180,11 @@ public class LoanRepositoryWrapper { // Looks like we need complete Data public List findByIdsAndLoanStatusAndLoanType(@Param("ids") Collection ids, @Param("loanStatuses") Collection loanStatuses, @Param("loanTypes") Collection loanTypes) { - List loans = this.repository.findByIdsAndLoanStatusAndLoanType(ids, loanStatuses, loanTypes); - if (loans != null && loans.size() > 0) { + List loans = new ArrayList<>(); + List> partitions = Lists.partition(ids.stream().toList(), fineractProperties.getQuery().getInClauseParameterSizeLimit()); + partitions + .forEach(partition -> loans.addAll(this.repository.findByIdsAndLoanStatusAndLoanType(partition, loanStatuses, loanTypes))); + if (loans.size() > 0) { for (Loan loan : loans) { loan.initializeLazyCollections(); } diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties index f1b158ea3..ec67e4051 100644 --- a/fineract-provider/src/main/resources/application.properties +++ b/fineract-provider/src/main/resources/application.properties @@ -42,6 +42,8 @@ fineract.mode.write-enabled=${FINERACT_MODE_WRITE_ENABLED:true} fineract.mode.batch-worker-enabled=${FINERACT_MODE_BATCH_WORKER_ENABLED:true} fineract.mode.batch-manager-enabled=${FINERACT_MODE_BATCH_MANAGER_ENABLED:true} +fineract.query.in-clause-parameter-size-limit=${FINERACT_QUERY_PARAMETER_SIZE:1000} + fineract.correlation.enabled=${FINERACT_LOGGING_HTTP_CORRELATION_ID_ENABLED:false} fineract.correlation.header-name=${FINERACT_LOGGING_HTTP_CORRELATION_ID_HEADER_NAME:X-Correlation-ID} diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java index fe8030eb9..78b014cff 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java +++ b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/ApplyLoanLockTaskletStepDefinitions.java @@ -29,6 +29,7 @@ import java.util.List; import org.apache.fineract.cob.domain.LoanAccountLock; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.domain.LockOwner; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.mockito.ArgumentCaptor; @@ -42,7 +43,9 @@ public class ApplyLoanLockTaskletStepDefinitions implements En { ArgumentCaptor valueCaptor = ArgumentCaptor.forClass(LoanAccountLock.class); private LoanAccountLockRepository accountLockRepository = mock(LoanAccountLockRepository.class); - private ApplyLoanLockTasklet applyLoanLockTasklet = new ApplyLoanLockTasklet(accountLockRepository); + private FineractProperties fineractProperties = mock(FineractProperties.class); + private FineractProperties.FineractQueryProperties fineractQueryProperties = mock(FineractProperties.FineractQueryProperties.class); + private ApplyLoanLockTasklet applyLoanLockTasklet = new ApplyLoanLockTasklet(accountLockRepository, fineractProperties); private RepeatStatus resultItem; private StepContribution stepContribution; @@ -56,12 +59,16 @@ public class ApplyLoanLockTaskletStepDefinitions implements En { this.stepContribution = new StepContribution(stepExecution); if ("error".equals(action)) { + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(this.accountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenThrow(new RuntimeException("fail")); } else { LoanAccountLock lock1 = new LoanAccountLock(1L, LockOwner.LOAN_COB_CHUNK_PROCESSING); LoanAccountLock lock2 = new LoanAccountLock(2L, LockOwner.LOAN_COB_PARTITIONING); LoanAccountLock lock3 = new LoanAccountLock(3L, LockOwner.LOAN_INLINE_COB_PROCESSING); List accountLocks = List.of(lock1, lock2, lock3); + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(this.accountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenReturn(accountLocks); } diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java index 2a4a16f7b..9f893edff 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java +++ b/fineract-provider/src/test/java/org/apache/fineract/cob/loan/FetchAndLockLoanStepDefinitions.java @@ -35,6 +35,7 @@ import org.apache.fineract.cob.domain.LoanAccountLock; import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.domain.LockOwner; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.mockito.Mockito; @@ -48,6 +49,9 @@ public class FetchAndLockLoanStepDefinitions implements En { private final LoanAccountLockRepository loanAccountLockRepository = mock(LoanAccountLockRepository.class); private final RetrieveLoanIdService retrieveLoanIdService = mock(RetrieveLoanIdService.class); + private final FineractProperties fineractProperties = mock(FineractProperties.class); + private final FineractProperties.FineractQueryProperties fineractQueryProperties = mock( + FineractProperties.FineractQueryProperties.class); StepContribution contribution; private FetchAndLockLoanTasklet fetchAndLockLoanTasklet; private String action; @@ -65,17 +69,25 @@ public class FetchAndLockLoanStepDefinitions implements En { lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(), any())).thenReturn(Collections.emptyList()); } else if ("good".equals(action)) { lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(), any())).thenReturn(List.of(1L, 2L, 3L)); + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList())).thenReturn(Collections.emptyList()); } else if ("soft lock".equals(action)) { lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(), any())).thenReturn(List.of(1L, 2L, 3L)); + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList())) .thenReturn(List.of(new LoanAccountLock(1L, LockOwner.LOAN_COB_PARTITIONING))); } else if ("inline cob".equals(action)) { lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(), any())).thenReturn(List.of(1L, 2L, 3L)); + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList())) .thenReturn(List.of(new LoanAccountLock(2L, LockOwner.LOAN_INLINE_COB_PROCESSING))); } else if ("chunk processing".equals(action)) { lenient().when(retrieveLoanIdService.retrieveLoanIdsNDaysBehind(anyLong(), any())).thenReturn(List.of(1L, 2L, 3L)); + lenient().when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + lenient().when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); lenient().when(loanAccountLockRepository.findAllByLoanIdIn(Mockito.anyList())) .thenReturn(List.of(new LoanAccountLock(3L, LockOwner.LOAN_COB_CHUNK_PROCESSING))); } @@ -85,7 +97,7 @@ public class FetchAndLockLoanStepDefinitions implements En { contribution = new StepContribution(stepExecution); contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME, LocalDate.now(ZoneId.systemDefault()).toString()); - fetchAndLockLoanTasklet = new FetchAndLockLoanTasklet(loanAccountLockRepository, retrieveLoanIdService); + fetchAndLockLoanTasklet = new FetchAndLockLoanTasklet(loanAccountLockRepository, retrieveLoanIdService, fineractProperties); }); When("FetchAndLockLoanTasklet.execute method executed", () -> { diff --git a/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java b/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java index 407dcfe3d..791b561c4 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImplTest.java @@ -23,6 +23,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.lang.reflect.InvocationTargetException; @@ -35,6 +37,7 @@ import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; import org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledException; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; import org.apache.fineract.infrastructure.core.api.JsonCommand; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository; @@ -59,6 +62,10 @@ class InlineLoanCOBExecutorServiceImplTest { private InlineLoanCOBExecutionDataParser dataParser; @Mock private LoanRepository loanRepository; + @Mock + private FineractProperties fineractProperties; + @Mock + private FineractProperties.FineractQueryProperties fineractQueryProperties; @Test void shouldExceptionThrownIfLoanIsAlreadyLocked() { @@ -72,10 +79,35 @@ class InlineLoanCOBExecutorServiceImplTest { ThreadLocalContextUtil.setBusinessDates(businessDates); when(transactionTemplate.execute(any())).thenThrow(new LoanAccountLockCannotBeOverruledException("")); + when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + when(dataParser.parseExecution(any())).thenReturn(List.of(1L)); + when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(65000); when(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(any(), anyList())).thenReturn(List.of(loan)); assertThrows(LoanAccountLockCannotBeOverruledException.class, () -> testObj.executeInlineJob(command, "INLINE_LOAN_COB")); } + @Test + void shouldListBePartitioned() { + JsonCommand command = mock(JsonCommand.class); + LoanIdAndLastClosedBusinessDate loan1 = mock(LoanIdAndLastClosedBusinessDate.class); + LoanIdAndLastClosedBusinessDate loan2 = mock(LoanIdAndLastClosedBusinessDate.class); + LoanIdAndLastClosedBusinessDate loan3 = mock(LoanIdAndLastClosedBusinessDate.class); + ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null)); + HashMap businessDates = new HashMap<>(); + LocalDate businessDate = LocalDate.now(ZoneId.systemDefault()); + businessDates.put(BusinessDateType.BUSINESS_DATE, businessDate); + businessDates.put(BusinessDateType.COB_DATE, businessDate.minusDays(1)); + ThreadLocalContextUtil.setBusinessDates(businessDates); + + when(transactionTemplate.execute(any())).thenThrow(new LoanAccountLockCannotBeOverruledException("")); + when(fineractProperties.getQuery()).thenReturn(fineractQueryProperties); + when(dataParser.parseExecution(any())).thenReturn(List.of(1L, 2L, 3L)); + when(fineractQueryProperties.getInClauseParameterSizeLimit()).thenReturn(2); + when(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(any(), anyList())).thenReturn(List.of(loan1, loan2, loan3)); + assertThrows(LoanAccountLockCannotBeOverruledException.class, () -> testObj.executeInlineJob(command, "INLINE_LOAN_COB")); + verify(loanRepository, times(2)).findAllNonClosedLoansBehindOrNullByLoanIds(any(), anyList()); + } + @Test void shouldOldestCloseBusinessDateReturnWithCorrectDate() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java index b787abffb..5e67a4c39 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilterTest.java @@ -42,6 +42,7 @@ import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate; import org.apache.fineract.cob.service.InlineLoanCOBExecutorServiceImpl; import org.apache.fineract.cob.service.LoanAccountLockService; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; @@ -80,6 +81,10 @@ class LoanCOBApiFilterTest { private InlineLoanCOBExecutorServiceImpl inlineLoanCOBExecutorService; @Mock private LoanRepository loanRepository; + @Mock + private FineractProperties fineractProperties; + @Mock + private FineractProperties.FineractQueryProperties fineractQueryProperties; @Mock private LoanRescheduleRequestRepository loanRescheduleRequestRepository; @@ -145,7 +150,9 @@ class LoanCOBApiFilterTest { given(request.getPathInfo()).willReturn("/loans/invalid2LoanId/charges"); given(request.getMethod()).willReturn(HTTPMethods.POST.value()); given(context.authenticatedUser()).willReturn(appUser); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.emptyList()); @@ -187,7 +194,9 @@ class LoanCOBApiFilterTest { given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false); given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false); given(context.authenticatedUser()).willReturn(appUser); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.emptyList()); @@ -214,7 +223,9 @@ class LoanCOBApiFilterTest { given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false); given(context.authenticatedUser()).willReturn(appUser); given(loanRepository.findIdByExternalId(any())).willReturn(2L); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.emptyList()); @@ -239,11 +250,13 @@ class LoanCOBApiFilterTest { given(request.getMethod()).willReturn(HTTPMethods.POST.value()); given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false); given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false); + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); LoanRescheduleRequest rescheduleRequest = mock(LoanRescheduleRequest.class); given(loanRescheduleRequestRepository.getLoanIdByRescheduleRequestId(resourceId)).willReturn(Optional.of(2L)); given(context.authenticatedUser()).willReturn(appUser); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.emptyList()); @@ -288,8 +301,10 @@ class LoanCOBApiFilterTest { given(request.getPathInfo()).willReturn("/loans/2?command=approve"); given(request.getMethod()).willReturn(HTTPMethods.POST.value()); given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false); - given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(true); + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.singletonList(result)); @@ -320,7 +335,9 @@ class LoanCOBApiFilterTest { given(request.getMethod()).willReturn(HTTPMethods.POST.value()); given(loanAccountLockService.isLoanHardLocked(2L)).willReturn(false); given(loanAccountLockService.isLoanSoftLocked(2L)).willReturn(false); - given(loanRepository.findAllNonClosedLoansBehindByLoanIds( + given(fineractProperties.getQuery()).willReturn(fineractQueryProperties); + given(fineractQueryProperties.getInClauseParameterSizeLimit()).willReturn(65000); + given(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds( eq(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE)), anyList())) .willReturn(Collections.emptyList());