[FINERACT-1724] making partition for big parameter lists

fineract-provider/src/main/java/org/apache/fineract/cob/loan/ApplyLoanLockTasklet.java

@@ -18,6 +18,7 @@
  */
 package org.apache.fineract.cob.loan;
 
+import com.google.common.collect.Lists;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -28,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.cob.domain.LoanAccountLock;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.jetbrains.annotations.NotNull;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
@@ -40,13 +42,15 @@ import org.springframework.batch.repeat.RepeatStatus;
 public class ApplyLoanLockTasklet implements Tasklet {
 
     private final LoanAccountLockRepository accountLockRepository;
+    private final FineractProperties fineractProperties;
 
     @Override
     public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception {
         ExecutionContext executionContext = contribution.getStepExecution().getExecutionContext();
         List<Long> loanIds = (List<Long>) executionContext.get(LoanCOBConstant.LOAN_IDS);
-
-        List<LoanAccountLock> accountLocks = accountLockRepository.findAllByLoanIdIn(loanIds);
+        List<List<Long>> loanIdPartitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit());
+        List<LoanAccountLock> accountLocks = new ArrayList<>();
+        loanIdPartitions.forEach(loanIdPartition -> accountLocks.addAll(accountLockRepository.findAllByLoanIdIn(loanIdPartition)));
 
         Map<Long, LoanAccountLock> alreadySoftLockedAccountsMap = accountLocks.stream()
                 .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner()))

fineract-provider/src/main/java/org/apache/fineract/cob/loan/FetchAndLockLoanTasklet.java

@@ -18,6 +18,7 @@
  */
 package org.apache.fineract.cob.loan;
 
+import com.google.common.collect.Lists;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
@@ -27,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.cob.domain.LoanAccountLock;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.jetbrains.annotations.NotNull;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
@@ -43,6 +45,8 @@ public class FetchAndLockLoanTasklet implements Tasklet {
 
     private final RetrieveLoanIdService retrieveLoanIdService;
 
+    private final FineractProperties fineractProperties;
+
     @Override
     public RepeatStatus execute(@NotNull StepContribution contribution, @NotNull ChunkContext chunkContext) throws Exception {
         String businessDateParameter = (String) contribution.getStepExecution().getJobExecution().getExecutionContext()
@@ -54,7 +58,11 @@ public class FetchAndLockLoanTasklet implements Tasklet {
         }
         List<Long> remainingIds = new ArrayList<>(allNonClosedLoanIds);
 
-        List<LoanAccountLock> loanAccountLocks = loanAccountLockRepository.findAllByLoanIdIn(remainingIds);
+        List<List<Long>> remainingIdPartitions = Lists.partition(remainingIds,
+                fineractProperties.getQuery().getInClauseParameterSizeLimit());
+        List<LoanAccountLock> loanAccountLocks = new ArrayList<>();
+        remainingIdPartitions.forEach(
+                remainingIdPartition -> loanAccountLocks.addAll(loanAccountLockRepository.findAllByLoanIdIn(remainingIdPartition)));
 
         List<Long> alreadySoftLockedAccounts = loanAccountLocks.stream()
                 .filter(e -> LockOwner.LOAN_COB_PARTITIONING.equals(e.getLockOwner())).map(LoanAccountLock::getLoanId).toList();

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java

@@ -23,6 +23,7 @@ import org.apache.fineract.cob.COBBusinessStepService;
 import org.apache.fineract.cob.common.CustomJobParameterResolver;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.listener.COBExecutionListenerRunner;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.infrastructure.jobs.service.JobName;
 import org.apache.fineract.infrastructure.springbatch.PropertyService;
@@ -79,6 +80,8 @@ public class LoanCOBManagerConfiguration {
     private CustomJobParameterResolver customJobParameterResolver;
     @Autowired
     private LoanRepository loanRepository;
+    @Autowired
+    private FineractProperties fineractProperties;
 
     @Bean
     @JobScope
@@ -111,7 +114,7 @@ public class LoanCOBManagerConfiguration {
     @Bean
     @JobScope
     public FetchAndLockLoanTasklet fetchAndLockLoanTasklet() {
-        return new FetchAndLockLoanTasklet(accountLockRepository, retrieveLoanIdService);
+        return new FetchAndLockLoanTasklet(accountLockRepository, retrieveLoanIdService, fineractProperties);
     }
 
     @Bean

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java

@@ -23,6 +23,7 @@ import org.apache.fineract.cob.common.InitialisationTasklet;
 import org.apache.fineract.cob.common.ResetContextTasklet;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.jobs.service.JobName;
 import org.apache.fineract.infrastructure.springbatch.PropertyService;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
@@ -67,6 +68,9 @@ public class LoanCOBWorkerConfiguration {
     @Autowired
     private TransactionTemplate transactionTemplate;
 
+    @Autowired
+    private FineractProperties fineractProperties;
+
     @Bean(name = LoanCOBConstant.LOAN_COB_WORKER_STEP)
     public Step loanCOBWorkerStep() {
         return stepBuilderFactory.get("Loan COB worker - Step").inputChannel(inboundRequests).flow(flow()).build();
@@ -118,7 +122,7 @@ public class LoanCOBWorkerConfiguration {
 
     @Bean
     public ApplyLoanLockTasklet applyLock() {
-        return new ApplyLoanLockTasklet(accountLockRepository);
+        return new ApplyLoanLockTasklet(accountLockRepository, fineractProperties);
     }
 
     @Bean

fineract-provider/src/main/java/org/apache/fineract/cob/service/InlineLoanCOBExecutorServiceImpl.java

@@ -20,6 +20,7 @@ package org.apache.fineract.cob.service;
 
 import static org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW;
 
+import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
@@ -40,6 +41,7 @@ import org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledExcept
 import org.apache.fineract.cob.loan.LoanCOBConstant;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
 import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
 import org.apache.fineract.infrastructure.core.exception.PlatformInternalServerException;
@@ -85,6 +87,7 @@ public class InlineLoanCOBExecutorServiceImpl implements InlineExecutorService<L
     private final CustomJobParameterRepository customJobParameterRepository;
     private final PlatformSecurityContext context;
     private final LoanRepository loanRepository;
+    private final FineractProperties fineractProperties;
 
     private final Gson gson = GoogleGsonSerializerHelper.createSimpleGson();
 
@@ -155,7 +158,11 @@ public class InlineLoanCOBExecutorServiceImpl implements InlineExecutorService<L
     }
 
     private List<LoanIdAndLastClosedBusinessDate> getLoansToBeProcessed(List<Long> loanIds, LocalDate cobBusinessDate) {
-        return loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate, loanIds);
+        List<LoanIdAndLastClosedBusinessDate> loanIdAndLastClosedBusinessDates = new ArrayList<>();
+        List<List<Long>> partitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit());
+        partitions.forEach(partition -> loanIdAndLastClosedBusinessDates
+                .addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(cobBusinessDate, partition)));
+        return loanIdAndLastClosedBusinessDates;
     }
 
     private List<LoanAccountLock> getLoanAccountLocks(List<Long> loanIds) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java

@@ -55,6 +55,8 @@ public class FineractProperties {
     private FineractTemplateProperties template;
     private FineractJpaProperties jpa;
 
+    private FineractQueryProperties query;
+
     @Getter
     @Setter
     public static class FineractTenantProperties {
@@ -255,4 +257,12 @@ public class FineractProperties {
 
         private boolean statementLoggingEnabled;
     }
+
+    @Getter
+    @Setter
+    public static class FineractQueryProperties {
+
+        private int inClauseParameterSizeLimit;
+    }
+
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/filter/LoanCOBApiFilter.java

@@ -18,8 +18,10 @@
  */
 package org.apache.fineract.infrastructure.jobs.filter;
 
+import com.google.common.collect.Lists;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Predicate;
@@ -35,9 +37,11 @@ import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.fineract.batch.domain.BatchRequest;
 import org.apache.fineract.batch.domain.BatchResponse;
+import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
 import org.apache.fineract.cob.service.InlineLoanCOBExecutorServiceImpl;
 import org.apache.fineract.cob.service.LoanAccountLockService;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.data.ApiGlobalErrorResponse;
 import org.apache.fineract.infrastructure.core.domain.ExternalId;
 import org.apache.fineract.infrastructure.core.filters.BatchFilter;
@@ -64,6 +68,7 @@ public class LoanCOBApiFilter extends OncePerRequestFilter implements BatchFilte
     private final PlatformSecurityContext context;
     private final InlineLoanCOBExecutorServiceImpl inlineLoanCOBExecutorService;
     private final LoanRepository loanRepository;
+    private final FineractProperties fineractProperties;
 
     private final LoanRescheduleRequestRepository loanRescheduleRequestRepository;
 
@@ -141,8 +146,11 @@ public class LoanCOBApiFilter extends OncePerRequestFilter implements BatchFilte
     }
 
     private boolean isLoanBehind(List<Long> loanIds) {
-        return CollectionUtils.isNotEmpty(loanRepository
-                .findAllNonClosedLoansBehindByLoanIds(ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE), loanIds));
+        List<LoanIdAndLastClosedBusinessDate> loanIdAndLastClosedBusinessDates = new ArrayList<>();
+        List<List<Long>> partitions = Lists.partition(loanIds, fineractProperties.getQuery().getInClauseParameterSizeLimit());
+        partitions.forEach(partition -> loanIdAndLastClosedBusinessDates.addAll(loanRepository.findAllNonClosedLoansBehindOrNullByLoanIds(
+                ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.COB_DATE), partition)));
+        return CollectionUtils.isNotEmpty(loanIdAndLastClosedBusinessDates);
     }
 
     private List<Long> calculateRelevantLoanIds(String pathInfo) {

fineract-provider/src/main/java/org/apache/fineract/portfolio/account/service/AccountTransfersWritePlatformServiceImpl.java

@@ -25,15 +25,18 @@ import static org.apache.fineract.portfolio.account.AccountDetailConstants.toAcc
 import static org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferAmountParamName;
 import static org.apache.fineract.portfolio.account.api.AccountTransfersApiConstants.transferDateParamName;
 
+import com.google.common.collect.Lists;
 import java.math.BigDecimal;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import lombok.RequiredArgsConstructor;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
 import org.apache.fineract.infrastructure.core.domain.ExternalId;
@@ -86,6 +89,7 @@ public class AccountTransfersWritePlatformServiceImpl implements AccountTransfer
     private final GSIMRepositoy gsimRepository;
     private final ConfigurationDomainService configurationDomainService;
     private final ExternalIdFactory externalIdFactory;
+    private final FineractProperties fineractProperties;
 
     @Transactional
     @Override
@@ -223,11 +227,13 @@ public class AccountTransfersWritePlatformServiceImpl implements AccountTransfer
     @Transactional
     public void reverseTransfersWithFromAccountTransactions(final Collection<Long> fromTransactionIds,
             final PortfolioAccountType accountTypeId) {
-        List<AccountTransferTransaction> acccountTransfers = null;
+        List<AccountTransferTransaction> acccountTransfers = new ArrayList<>();
         if (accountTypeId.isLoanAccount()) {
-            acccountTransfers = this.accountTransferRepository.findByFromLoanTransactions(fromTransactionIds);
+            List<List<Long>> partitions = Lists.partition(fromTransactionIds.stream().toList(),
+                    fineractProperties.getQuery().getInClauseParameterSizeLimit());
+            partitions.forEach(partition -> acccountTransfers.addAll(this.accountTransferRepository.findByFromLoanTransactions(partition)));
         }
-        if (acccountTransfers != null && acccountTransfers.size() > 0) {
+        if (acccountTransfers.size() > 0) {
             undoTransactions(acccountTransfers);
         }
 

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java

@@ -192,10 +192,6 @@ public interface LoanRepository extends JpaRepository<Loan, Long>, JpaSpecificat
     @Query(FIND_ALL_NON_CLOSED_LOANS_BY_LAST_CLOSED_BUSINESS_DATE)
     List<Long> findAllNonClosedLoanIdsByLastClosedBusinessDate(@Param("businessDate") LocalDate businessDate);
 
-    @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_BY_LOAN_IDS)
-    List<LoanIdAndLastClosedBusinessDate> findAllNonClosedLoansBehindByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate,
-            @Param("loanIds") List<Long> loanIds);
-
     @Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_OR_NULL_BY_LOAN_IDS)
     List<LoanIdAndLastClosedBusinessDate> findAllNonClosedLoansBehindOrNullByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate,
             @Param("loanIds") List<Long> loanIds);

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java

@@ -18,14 +18,16 @@
  */
 package org.apache.fineract.portfolio.loanaccount.domain;
 
+import com.google.common.collect.Lists;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.ExternalId;
 import org.apache.fineract.portfolio.loanaccount.exception.LoanNotFoundException;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -36,14 +38,11 @@ import org.springframework.transaction.annotation.Transactional;
  * </p>
  */
 @Service
+@RequiredArgsConstructor
 public class LoanRepositoryWrapper {
 
     private final LoanRepository repository;
-
-    @Autowired
-    public LoanRepositoryWrapper(final LoanRepository repository) {
-        this.repository = repository;
-    }
+    private final FineractProperties fineractProperties;
 
     @Transactional(readOnly = true)
     public Loan findOneWithNotFoundDetection(final Long id) {
@@ -181,8 +180,11 @@ public class LoanRepositoryWrapper {
     // Looks like we need complete Data
     public List<Loan> findByIdsAndLoanStatusAndLoanType(@Param("ids") Collection<Long> ids,
             @Param("loanStatuses") Collection<Integer> loanStatuses, @Param("loanTypes") Collection<Integer> loanTypes) {
-        List<Loan> loans = this.repository.findByIdsAndLoanStatusAndLoanType(ids, loanStatuses, loanTypes);
-        if (loans != null && loans.size() > 0) {
+        List<Loan> loans = new ArrayList<>();
+        List<List<Long>> partitions = Lists.partition(ids.stream().toList(), fineractProperties.getQuery().getInClauseParameterSizeLimit());
+        partitions
+                .forEach(partition -> loans.addAll(this.repository.findByIdsAndLoanStatusAndLoanType(partition, loanStatuses, loanTypes)));
+        if (loans.size() > 0) {
             for (Loan loan : loans) {
                 loan.initializeLazyCollections();
             }

fineract-provider/src/main/resources/application.properties

@@ -42,6 +42,8 @@ fineract.mode.write-enabled=${FINERACT_MODE_WRITE_ENABLED:true}
 fineract.mode.batch-worker-enabled=${FINERACT_MODE_BATCH_WORKER_ENABLED:true}
 fineract.mode.batch-manager-enabled=${FINERACT_MODE_BATCH_MANAGER_ENABLED:true}
 
+fineract.query.in-clause-parameter-size-limit=${FINERACT_QUERY_PARAMETER_SIZE:1000}
+
 fineract.correlation.enabled=${FINERACT_LOGGING_HTTP_CORRELATION_ID_ENABLED:false}
 fineract.correlation.header-name=${FINERACT_LOGGING_HTTP_CORRELATION_ID_HEADER_NAME:X-Correlation-ID}