FINERACT-1678

- COB Partitioning tasks creates fully balanced id ranges where the number of the loans between the min max are matching with the defined pageSize.
This commit is contained in:
Peter Bagrij
2023-07-03 10:32:46 +02:00
committed by Arnold Gálovics
parent 1d8ee9acce
commit 479a52013d
17 changed files with 729 additions and 283 deletions
@@ -1,136 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import com.google.gson.Gson;
import io.cucumber.java8.En;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.mockito.Mockito;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.item.ExecutionContext;
public class LoanCOBPartitionerStepDefinitions implements En {
PropertyService propertyService = mock(PropertyService.class);
COBBusinessStepService cobBusinessStepService = mock(COBBusinessStepService.class);
JobOperator jobOperator = mock(JobOperator.class);
JobExplorer jobExplorer = mock(JobExplorer.class);
private final Gson gson = GoogleGsonSerializerHelper.createSimpleGson();
LoanCOBParameter loanIds;
private LoanCOBPartitioner loanCOBPartitioner;
private Set<BusinessStepNameAndOrder> cobBusinessSteps = new HashSet<>();
private Map<String, ExecutionContext> resultItem;
private String action;
public LoanCOBPartitionerStepDefinitions() {
Given("/^The LoanCOBPartitioner.partition method with action (.*)$/", (String action) -> {
this.action = action;
lenient().when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(2);
if ("empty steps".equals(action)) {
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(Collections.emptySet());
lenient().when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(new JobExecution(3L)));
lenient().when(jobOperator.stop(3L)).thenReturn(Boolean.TRUE);
loanIds = null;
} else if ("empty loanIds".equals(action)) {
cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L));
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(cobBusinessSteps);
loanIds = null;
} else if ("good".equals(action)) {
cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L));
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(cobBusinessSteps);
loanIds = new LoanCOBParameter(1L, 3L);
}
loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, loanIds);
});
When("LoanCOBPartitioner.partition method executed", () -> {
resultItem = this.loanCOBPartitioner.partition(2);
});
Then("LoanCOBPartitioner.partition result should match", () -> {
if ("empty steps".equals(action)) {
verify(jobOperator, Mockito.times(1)).stop(3L);
assertTrue(resultItem.isEmpty());
} else if ("good".equals(action)) {
verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong());
assertEquals(2, resultItem.size());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1"));
Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) resultItem
.get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS);
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(1, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId());
assertEquals(1L, loanCOBParameter.getMinLoanId());
assertEquals(2L, loanCOBParameter.getMaxLoanId());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "2"));
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "2")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(0, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId());
assertEquals(3L, loanCOBParameter.getMinLoanId());
assertEquals(3L, loanCOBParameter.getMaxLoanId());
} else if ("empty loanIds".equals(action)) {
verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong());
assertEquals(1, resultItem.size());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1"));
Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) resultItem
.get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS);
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(0L, loanCOBParameter.getMinLoanId());
assertEquals(0L, loanCOBParameter.getMaxLoanId());
}
});
}
}
@@ -0,0 +1,135 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobExecutionNotRunningException;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
import org.springframework.batch.item.ExecutionContext;
@ExtendWith(MockitoExtension.class)
class LoanCOBPartitionerTest {
private static final Set<BusinessStepNameAndOrder> BUSINESS_STEP_SET = Set.of(new BusinessStepNameAndOrder("Business step", 1L));
private static final LocalDate BUSINESS_DATE = LocalDate.parse("2023-06-28");
@Mock
private PropertyService propertyService;
@Mock
private COBBusinessStepService cobBusinessStepService;
@Mock
private RetrieveLoanIdService retrieveLoanIdService;
@Mock
private JobOperator jobOperator;
@Mock
private JobExplorer jobExplorer;
@Test
public void testLoanCOBPartitioner() {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(BUSINESS_STEP_SET);
when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5))
.thenReturn(List.of(new LoanCOBPartition(1L,10L, 1L, 5L), new LoanCOBPartition(11L,20L, 2L, 4L)));
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(2, partitions.size());
validatePartitions(partitions, 1, 1, 10);
validatePartitions(partitions, 2, 11, 20);
}
@Test
public void testLoanCOBPartitionerEmptyBusinessSteps() throws NoSuchJobExecutionException, JobExecutionNotRunningException {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(Set.of());
JobExecution jobExecution = Mockito.mock(JobExecution.class);
when(jobExecution.getId()).thenReturn(123L);
when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(jobExecution));
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(0, partitions.size());
verify(jobExplorer, times(1)).findRunningJobExecutions(JobName.LOAN_COB.name());
verify(jobOperator, times(1)).stop(123L);
}
@Test
public void testLoanCOBPartitionerNoLoansFound() {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(BUSINESS_STEP_SET);
when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5))
.thenReturn(List.of());
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(1, partitions.size());
validatePartitions(partitions, 1, 0, 0);
}
private void validatePartitions(Map<String, ExecutionContext> partitions, int index, long min, long max) {
Assertions.assertEquals(BUSINESS_STEP_SET,
partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.BUSINESS_STEPS));
Assertions.assertEquals(new LoanCOBParameter(min, max),
partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.LOAN_COB_PARAMETER));
Assertions.assertEquals("partition_" + index, partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get("partition"));
}
}
@@ -0,0 +1,93 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.mockito.Mockito.times;
import java.time.LocalDate;
import java.util.List;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
@ExtendWith(MockitoExtension.class)
public class RetrieveAllNonClosedLoanIdServiceImplTest {
@Mock
private LoanRepository loanRepository;
@Mock
private NamedParameterJdbcTemplate namedParameterJdbcTemplate;
@Captor
private ArgumentCaptor<String> sqlCaptor;
@Captor
private ArgumentCaptor<SqlParameterSource> paramsCaptor;
@Captor
private ArgumentCaptor<RowMapper<LoanCOBPartition>> rowMapper;
@Test
public void testRetrieveLoanCOBPartitionsNoCatchup() {
String expectedSQL = """
select min(id) as min, max(id) as max, page, count(id) as count from
(select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from
(select id from m_loan where loan_status_id in (:statusIds) and (last_closed_business_date = :businessDate or last_closed_business_date is null) order by id) t) t2
group by page
order by page
""";
testRetrieveLoanCOBPartitions(expectedSQL, false);
}
@Test
public void testRetrieveLoanCOBPartitionsCatchup() {
String expectedSQL = """
select min(id) as min, max(id) as max, page, count(id) as count from
(select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from
(select id from m_loan where loan_status_id in (:statusIds) and last_closed_business_date = :businessDate order by id) t) t2
group by page
order by page
""";
testRetrieveLoanCOBPartitions(expectedSQL, true);
}
private void testRetrieveLoanCOBPartitions(String expectedSQL, boolean isCatchup) {
RetrieveAllNonClosedLoanIdServiceImpl service = new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository,
namedParameterJdbcTemplate);
LocalDate businessDate = LocalDate.parse("2023-06-28");
service.retrieveLoanCOBPartitions(1L, businessDate, isCatchup, 5);
Mockito.verify(namedParameterJdbcTemplate, times(1)).query(sqlCaptor.capture(), paramsCaptor.capture(), rowMapper.capture());
Assertions.assertEquals(normalize(expectedSQL), normalize(sqlCaptor.getValue()));
Assertions.assertEquals(5, paramsCaptor.getValue().getValue("pageSize"));
Assertions.assertEquals(List.of(100, 200, 300, 303, 304), paramsCaptor.getValue().getValue("statusIds"));
Assertions.assertEquals(LocalDate.parse("2023-06-27"), paramsCaptor.getValue().getValue("businessDate"));
}
private String normalize(String str) {
return str.replaceAll(" +", " ").replaceAll("\r?\n", "");
}
}
@@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
Feature: COB Loan partitioner Step
@cob
Scenario Outline: LoanCOBPartitioner - run test
Given The LoanCOBPartitioner.partition method with action <action>
When LoanCOBPartitioner.partition method executed
Then LoanCOBPartitioner.partition result should match
Examples:
|action|
|empty steps|
|good|
|empty loanIds|
@@ -20,6 +20,7 @@
-->
<configuration scan="false">
<statusListener class="ch.qos.logback.core.status.NopStatusListener" /> <!-- this hides the warning related to jansi in windows -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>false</resetJUL>
</contextListener>