FINERACT-1678

- ThreadPoolTaskExecutor task executor is added to COB workers which allows concurrent processing of chunks.

fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanItemReaderTest.java

@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.cob.loan;
+
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+import org.apache.fineract.cob.common.CustomJobParameterResolver;
+import org.apache.fineract.cob.data.LoanCOBParameter;
+import org.apache.fineract.cob.domain.LoanAccountLock;
+import org.apache.fineract.cob.domain.LockOwner;
+import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.portfolio.loanaccount.domain.Loan;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.item.ExecutionContext;
+
+@ExtendWith(MockitoExtension.class)
+class LoanItemReaderTest {
+
+    @Mock
+    private LoanRepository loanRepository;
+
+    @Mock
+    private RetrieveLoanIdService retrieveLoanIdService;
+
+    @Mock
+    private CustomJobParameterResolver customJobParameterResolver;
+
+    @Mock
+    private LoanLockingService loanLockingService;
+
+    @Mock
+    private StepExecution stepExecution;
+
+    @Mock
+    private ExecutionContext executionContext;
+
+    @Mock
+    private Loan loan;
+
+    @Test
+    public void testLoanItemReaderSimple() throws Exception {
+        // given
+        ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "test", "test", "UTC", null));
+        LoanItemReader loanItemReader = new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver,
+                loanLockingService);
+        when(stepExecution.getExecutionContext()).thenReturn(executionContext);
+        LoanCOBParameter loanCOBParameter = new LoanCOBParameter(1L, 5L);
+        when(executionContext.get(LoanCOBConstant.LOAN_COB_PARAMETER)).thenReturn(loanCOBParameter);
+        when(retrieveLoanIdService.retrieveAllNonClosedLoansByLastClosedBusinessDateAndMinAndMaxLoanId(loanCOBParameter, false))
+                .thenReturn(new ArrayList<>(List.of(1L, 2L, 3L, 4L, 5L)));
+        List<LoanAccountLock> accountLocks = List.of(1L, 2L, 3L, 4L, 5L).stream()
+                .map(l -> new LoanAccountLock(l, LockOwner.LOAN_COB_CHUNK_PROCESSING, LocalDate.of(2023, 7, 25))).toList();
+        when(loanLockingService.findAllByLoanIdInAndLockOwner(List.of(1L, 2L, 3L, 4L, 5L), LockOwner.LOAN_COB_CHUNK_PROCESSING))
+                .thenReturn(accountLocks);
+        when(loanRepository.findById(anyLong())).thenReturn(Optional.of(loan));
+
+        // when + then
+        loanItemReader.beforeStep(stepExecution);
+        for (long i = 1; i <= 5; i++) {
+            Loan myLoan = loanItemReader.read();
+            Assertions.assertEquals(loan, myLoan);
+            verify(loanRepository, times(1)).findById(i);
+        }
+
+        Mockito.verifyNoMoreInteractions(loanRepository);
+    }
+
+    @Test
+    public void testLoanItemReaderMultiThreadRead() throws Exception {
+        // given
+        ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "test", "test", "UTC", null));
+        LoanItemReader loanItemReader = new LoanItemReader(loanRepository, retrieveLoanIdService, customJobParameterResolver,
+                loanLockingService);
+        when(stepExecution.getExecutionContext()).thenReturn(executionContext);
+        LoanCOBParameter loanCOBParameter = new LoanCOBParameter(1L, 100L);
+        when(executionContext.get(LoanCOBConstant.LOAN_COB_PARAMETER)).thenReturn(loanCOBParameter);
+        when(retrieveLoanIdService.retrieveAllNonClosedLoansByLastClosedBusinessDateAndMinAndMaxLoanId(loanCOBParameter, false))
+                .thenReturn(new ArrayList<>(IntStream.rangeClosed(1, 100).boxed().map(Long::valueOf).toList()));
+        List<LoanAccountLock> accountLocks = IntStream.rangeClosed(1, 100).boxed().map(Long::valueOf)
+                .map(l -> new LoanAccountLock(l, LockOwner.LOAN_COB_CHUNK_PROCESSING, LocalDate.of(2023, 7, 25))).toList();
+        when(loanLockingService.findAllByLoanIdInAndLockOwner(IntStream.rangeClosed(1, 100).boxed().map(Long::valueOf).toList(),
+                LockOwner.LOAN_COB_CHUNK_PROCESSING)).thenReturn(accountLocks);
+        when(loanRepository.findById(anyLong())).thenReturn(Optional.of(loan));
+
+        // when + then
+        loanItemReader.beforeStep(stepExecution);
+        ExecutorService executorService = Executors.newFixedThreadPool(10);
+        for (int i = 1; i <= 100; i++) {
+            Future<?> notUsed = executorService.submit(() -> {
+                try {
+                    Loan myLoan = loanItemReader.read();
+                    Assertions.assertEquals(loan, myLoan);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+        executorService.shutdown();
+        boolean b = executorService.awaitTermination(5L, TimeUnit.SECONDS);
+        Assertions.assertTrue(b, "Executor did not terminate successfully");
+
+        // verify that this was called 100times, and for each loan it was called exactly once
+        for (long i = 1; i <= 100; i++) {
+            verify(loanRepository, times(1)).findById(i);
+        }
+
+        Mockito.verifyNoMoreInteractions(loanRepository);
+    }
+
+}

fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/config/FineractPartitionJobConfigValidationConditionTest.java

@@ -65,16 +65,6 @@ class FineractPartitionJobConfigValidationConditionTest {
         }
     }
 
-    @Test
-    public void testApplicationStartup_ShouldApplicationStartupFails_WhenPartitionSizeInvalid() {
-        try (MockedStatic<ExplicitConfigurationPropertiesFactory> propertyFactory = Mockito
-                .mockStatic(ExplicitConfigurationPropertiesFactory.class)) {
-            propertyFactory.when(() -> ExplicitConfigurationPropertiesFactory.getProperty(context, "fineract.partitioned-job",
-                    FineractProperties.FineractPartitionedJob.class)).thenReturn(getInvalidPartitionSize());
-            assertTrue(testObj.matches(context, metadata));
-        }
-    }
-
     @Test
     public void testApplicationStartup_ShouldApplicationStart_WhenConfigValid() {
         try (MockedStatic<ExplicitConfigurationPropertiesFactory> propertyFactory = Mockito
@@ -92,20 +82,9 @@ class FineractPartitionJobConfigValidationConditionTest {
         partitionedJobProperty.setJobName("LOAN_COB");
         partitionedJobProperty.setPartitionSize(100);
         partitionedJobProperty.setChunkSize(10);
-        partitionedJobProperty.setThreadCount(10);
-        jobProperties.add(partitionedJobProperty);
-        partitionedJob.setPartitionedJobProperties(jobProperties);
-        return partitionedJob;
-    }
-
-    private FineractProperties.FineractPartitionedJob getInvalidPartitionSize() {
-        FineractProperties.FineractPartitionedJob partitionedJob = new FineractProperties.FineractPartitionedJob();
-        List<FineractProperties.PartitionedJobProperty> jobProperties = new ArrayList<>();
-        FineractProperties.PartitionedJobProperty partitionedJobProperty = new FineractProperties.PartitionedJobProperty();
-        partitionedJobProperty.setJobName("LOAN_COB");
-        partitionedJobProperty.setPartitionSize(99);
-        partitionedJobProperty.setChunkSize(10);
-        partitionedJobProperty.setThreadCount(10);
+        partitionedJobProperty.setThreadPoolCorePoolSize(10);
+        partitionedJobProperty.setThreadPoolMaxPoolSize(20);
+        partitionedJobProperty.setThreadPoolQueueCapacity(10);
         jobProperties.add(partitionedJobProperty);
         partitionedJob.setPartitionedJobProperties(jobProperties);
         return partitionedJob;
@@ -118,7 +97,9 @@ class FineractPartitionJobConfigValidationConditionTest {
         partitionedJobProperty.setJobName("LOAN_COB");
         partitionedJobProperty.setPartitionSize(0);
         partitionedJobProperty.setChunkSize(1);
-        partitionedJobProperty.setThreadCount(1);
+        partitionedJobProperty.setThreadPoolCorePoolSize(1);
+        partitionedJobProperty.setThreadPoolMaxPoolSize(1);
+        partitionedJobProperty.setThreadPoolQueueCapacity(1);
         jobProperties.add(partitionedJobProperty);
         partitionedJob.setPartitionedJobProperties(jobProperties);
         return partitionedJob;

fineract-provider/src/test/resources/application-test.properties

@@ -42,7 +42,9 @@ fineract.mode.batch-enabled=true
 fineract.partitioned-job.partitioned-job-properties[0].job-name=LOAN_COB
 fineract.partitioned-job.partitioned-job-properties[0].chunk-size=100
 fineract.partitioned-job.partitioned-job-properties[0].partition-size=100
-fineract.partitioned-job.partitioned-job-properties[0].thread-count=1
+fineract.partitioned-job.partitioned-job-properties[0].thread-pool-core-pool-size=1
+fineract.partitioned-job.partitioned-job-properties[0].thread-pool-max-pool-size=1
+fineract.partitioned-job.partitioned-job-properties[0].thread-pool-queue-capacity=1
 fineract.partitioned-job.partitioned-job-properties[0].retry-limit=5
 
 fineract.remote-job-message-handler.spring-events.enabled=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_SPRING_EVENTS_ENABLED:true}