CBSNexora/cbsnexora-backend
4
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

FINERACT-1678

Browse Source 
- COB Partitioning tasks creates fully balanced id ranges where the number of the loans between the min max are matching with the defined pageSize.
This commit is contained in:
Peter Bagrij
2023-07-03 10:32:46 +02:00
committed by Arnold Gálovics
parent 1d8ee9acce
commit 479a52013d
17 changed files with 729 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBParameter.java
+2
View File
@@ -20,6 +20,7 @@ package org.apache.fineract.cob.data;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -27,6 +28,7 @@ import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
@EqualsAndHashCode
public class LoanCOBParameter {
private Long minLoanId;
fineract-loan/src/main/java/org/apache/fineract/cob/data/LoanCOBPartition.java
+32
View File
@@ -0,0 +1,32 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.data;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class LoanCOBPartition {
private Long minId;
private Long maxId;
private Long pageNo;
private Long count;
}
fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java
-13
View File
@@ -22,7 +22,6 @@ import java.time.LocalDate;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo;
import org.apache.fineract.cob.data.LoanIdAndExternalIdAndStatus;
import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
@@ -77,12 +76,6 @@ public interface LoanRepository extends JpaRepository<Loan, Long>, JpaSpecificat
String FIND_ALL_NON_CLOSED = "select loan.id from Loan loan where loan.loanStatus in (100,200,300,303,304)";
String FIND_MIN_AND_MAX_NON_CLOSED_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE = "select new org.apache.fineract.cob.data.LoanCOBParameter(min(loan.id), max(loan.id)) from Loan loan where loan.loanStatus in (100,200,300,303,304) "
+ "and (:businessDate = loan.lastClosedBusinessDate or loan.lastClosedBusinessDate is NULL)";
String FIND_MIN_AND_MAX_NON_CLOSED_AND_NON_NULL_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE = "select new org.apache.fineract.cob.data.LoanCOBParameter(min(loan.id), max(loan.id)) from Loan loan where loan.loanStatus in (100,200,300,303,304) "
+ "and :businessDate = loan.lastClosedBusinessDate";
String FIND_NON_CLOSED_LOAN_THAT_BELONGS_TO_CLIENT = "select loan from Loan loan where loan.id = :loanId and loan.loanStatus = 300 and loan.client.id = :clientId";
String FIND_BY_ACCOUNT_NUMBER = "select loan from Loan loan where loan.accountNumber = :accountNumber";
@@ -211,12 +204,6 @@ public interface LoanRepository extends JpaRepository<Loan, Long>, JpaSpecificat
@Query(FIND_ID_BY_EXTERNAL_ID)
Long findIdByExternalId(@Param("externalId") ExternalId externalId);
@Query(FIND_MIN_AND_MAX_NON_CLOSED_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE)
LoanCOBParameter findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDate(@Param("businessDate") LocalDate businessDate);
@Query(FIND_MIN_AND_MAX_NON_CLOSED_AND_NON_NULL_LOAN_IDS_BY_LAST_CLOSED_BUSINESS_DATE)
LoanCOBParameter findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDateNotNull(@Param("businessDate") LocalDate businessDate);
@Query(FIND_ALL_NON_CLOSED_LOANS_BEHIND_BY_LOAN_IDS)
List<LoanIdAndLastClosedBusinessDate> findAllNonClosedLoansBehindByLoanIds(@Param("cobBusinessDate") LocalDate cobBusinessDate,
@Param("loanIds") List<Long> loanIds);
fineract-provider/src/main/java/org/apache/fineract/cob/api/InternalCOBApiResource.java
+80
View File
@@ -0,0 +1,80 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.api;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.UriInfo;
import java.time.LocalDate;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.cob.loan.LoanCOBConstant;
import org.apache.fineract.cob.loan.RetrieveLoanIdService;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
import org.apache.fineract.infrastructure.core.api.ApiRequestParameterHelper;
import org.apache.fineract.infrastructure.core.serialization.ApiRequestJsonSerializationSettings;
import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
@Profile("test")
@Component
@Path("/v1/internal/cob")
@RequiredArgsConstructor
@Slf4j
public class InternalCOBApiResource implements InitializingBean {
private final RetrieveLoanIdService retrieveLoanIdService;
private final ApiRequestParameterHelper apiRequestParameterHelper;
private final ToApiJsonSerializer<List> toApiJsonSerializerForList;
@Override
public void afterPropertiesSet() throws Exception {
log.warn("------------------------------------------------------------");
log.warn(" ");
log.warn("DO NOT USE THIS IN PRODUCTION!");
log.warn("Internal client services mode is enabled");
log.warn("DO NOT USE THIS IN PRODUCTION!");
log.warn(" ");
log.warn("------------------------------------------------------------");
}
@GET
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
@Path("partitions/{partitionSize}")
public String getCobPartitions(@Context final UriInfo uriInfo, @PathParam("partitionSize") int partitionSize) {
LocalDate businessDate = ThreadLocalContextUtil.getBusinessDateByType(BusinessDateType.BUSINESS_DATE);
log.info("RetrieveLoanCOBPartitions is called with partitionSize {} for {}", partitionSize, businessDate);
List<LoanCOBPartition> loanCOBPartitions = retrieveLoanIdService.retrieveLoanCOBPartitions(LoanCOBConstant.NUMBER_OF_DAYS_BEHIND,
businessDate, false, partitionSize);
final ApiRequestJsonSerializationSettings settings = this.apiRequestParameterHelper.process(uriInfo.getQueryParameters());
return toApiJsonSerializerForList.serialize(settings, loanCOBPartitions);
}
}
fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBManagerConfiguration.java
+6 -18
View File
@@ -20,7 +20,6 @@ package org.apache.fineract.cob.loan;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.common.CustomJobParameterResolver;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.listener.COBExecutionListenerRunner;
import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import org.apache.fineract.infrastructure.jobs.service.JobName;
@@ -38,7 +37,6 @@ import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.integration.config.annotation.EnableBatchIntegration;
import org.springframework.batch.integration.partition.RemotePartitioningManagerStepBuilderFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
@@ -78,19 +76,15 @@ public class LoanCOBManagerConfiguration {
@Bean
@JobScope
public LoanCOBPartitioner partitioner(@Value("#{jobExecutionContext['loanCobParameter']}") LoanCOBParameter loanCOBParameter) {
return new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, loanCOBParameter);
public LoanCOBPartitioner partitioner() {
return new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer,
LoanCOBConstant.NUMBER_OF_DAYS_BEHIND);
}
@Bean
public Step loanCOBStep() {
return stepBuilderFactory.get(LoanCOBConstant.LOAN_COB_PARTITIONER_STEP)
.partitioner(LoanCOBConstant.LOAN_COB_WORKER_STEP, partitioner(null)).outputChannel(outboundRequests).build();
}
@Bean
public Step loanIdParameterStep() {
return new StepBuilder("Set loan ID parameter - Step", jobRepository).tasklet(loanIdParameterTasklet(), transactionManager).build();
.partitioner(LoanCOBConstant.LOAN_COB_WORKER_STEP, partitioner()).outputChannel(outboundRequests).build();
}
@Bean
@@ -105,12 +99,6 @@ public class LoanCOBManagerConfiguration {
.build();
}
@Bean
@JobScope
public LoanIdParameterTasklet loanIdParameterTasklet() {
return new LoanIdParameterTasklet(retrieveLoanIdService, customJobParameterResolver);
}
@Bean
@JobScope
public ResolveLoanCOBCustomJobParametersTasklet resolveCustomJobParametersTasklet() {
@@ -128,7 +116,7 @@ public class LoanCOBManagerConfiguration {
return new JobBuilder(JobName.LOAN_COB.name(), jobRepository) //
.listener(new COBExecutionListenerRunner(applicationContext, JobName.LOAN_COB.name())) //
.start(resolveCustomJobParametersStep()) //
.next(loanIdParameterStep()).next(loanCOBStep()).next(stayedLockedStep()) //
.next(loanCOBStep()).next(stayedLockedStep()) //
.incrementer(new RunIdIncrementer()) //
.build();
}
@@ -136,7 +124,7 @@ public class LoanCOBManagerConfiguration {
@Bean
public ExecutionContextPromotionListener customJobParametersPromotionListener() {
ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener();
listener.setKeys(new String[] { LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME });
listener.setKeys(new String[] { LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME, LoanCOBConstant.IS_CATCH_UP_PARAMETER_NAME });
return listener;
}
}
fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBPartitioner.java
+39 -21
View File
@@ -18,18 +18,19 @@
*/
package org.apache.fineract.cob.loan;
import com.google.common.collect.Lists;
import java.util.HashMap;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.LongStream;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.jetbrains.annotations.NotNull;
@@ -40,6 +41,8 @@ import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.util.StopWatch;
@Slf4j
@RequiredArgsConstructor
@@ -49,10 +52,18 @@ public class LoanCOBPartitioner implements Partitioner {
private final PropertyService propertyService;
private final COBBusinessStepService cobBusinessStepService;
private final RetrieveLoanIdService retrieveLoanIdService;
private final JobOperator jobOperator;
private final JobExplorer jobExplorer;
private final LoanCOBParameter minAndMaxLoanId;
private final Long numberOfDays;
@Value("#{jobExecutionContext['BusinessDate']}")
@Setter
private LocalDate businessDate;
@Value("#{jobExecutionContext['IS_CATCH_UP']}")
@Setter
private Boolean isCatchUp;
@NotNull
@Override
@@ -64,31 +75,38 @@ public class LoanCOBPartitioner implements Partitioner {
}
private Map<String, ExecutionContext> getPartitions(int partitionSize, Set<BusinessStepNameAndOrder> cobBusinessSteps) {
Map<String, ExecutionContext> partitions = new HashMap<>();
if (cobBusinessSteps.isEmpty()) {
stopJobExecution();
return Map.of();
}
if (!Objects.isNull(minAndMaxLoanId)) {
List<Long> loanIdsInRange = LongStream.rangeClosed(minAndMaxLoanId.getMinLoanId(), minAndMaxLoanId.getMaxLoanId()).boxed()
.toList();
List<List<Long>> loanIdPartitions = Lists.partition(loanIdsInRange, partitionSize);
for (int i = 0; i < loanIdPartitions.size(); i++) {
createNewPartition(partitions, i + 1, cobBusinessSteps, loanIdPartitions.get(i));
}
} else {
createNewPartition(partitions, 1, cobBusinessSteps, List.of(0L));
StopWatch sw = new StopWatch();
sw.start();
List<LoanCOBPartition> loanCOBPartitions = new ArrayList<>(retrieveLoanIdService.retrieveLoanCOBPartitions(numberOfDays,
businessDate, isCatchUp != null ? isCatchUp : false, partitionSize));
sw.stop();
// if there is no loan to be closed, we still would like to create at least one partition
if (loanCOBPartitions.size() == 0) {
loanCOBPartitions.add(new LoanCOBPartition(0L, 0L, 1L, 0L));
}
return partitions;
log.info(
"LoanCOBPartitioner found {} loans to be processed as part of COB. {} partitions were created using partition size {}. RetrieveLoanCOBPartitions was executed in {} ms.",
getLoanCount(loanCOBPartitions), loanCOBPartitions.size(), partitionSize, sw.getTotalTimeMillis());
return loanCOBPartitions.stream()
.collect(Collectors.toMap(l -> PARTITION_PREFIX + l.getPageNo(), l -> createNewPartition(cobBusinessSteps, l)));
}
private void createNewPartition(Map<String, ExecutionContext> partitions, int partitionIndex,
Set<BusinessStepNameAndOrder> cobBusinessSteps, List<Long> loanIds) {
private long getLoanCount(List<LoanCOBPartition> loanCOBPartitions) {
return loanCOBPartitions.stream().map(LoanCOBPartition::getCount).reduce(0L, Long::sum);
}
private ExecutionContext createNewPartition(Set<BusinessStepNameAndOrder> cobBusinessSteps, LoanCOBPartition loanCOBPartition) {
ExecutionContext executionContext = new ExecutionContext();
executionContext.put(LoanCOBConstant.BUSINESS_STEPS, cobBusinessSteps);
executionContext.put(LoanCOBConstant.LOAN_COB_PARAMETER, new LoanCOBParameter(loanIds.get(0), loanIds.get(loanIds.size() - 1)));
executionContext.put("partition", PARTITION_PREFIX + partitionIndex);
partitions.put(PARTITION_PREFIX + partitionIndex, executionContext);
executionContext.put(LoanCOBConstant.LOAN_COB_PARAMETER,
new LoanCOBParameter(loanCOBPartition.getMinId(), loanCOBPartition.getMaxId()));
executionContext.put("partition", PARTITION_PREFIX + loanCOBPartition.getPageNo());
return executionContext;
}
private void stopJobExecution() {
fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanIdParameterTasklet.java
-58
View File
@@ -1,58 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import java.time.LocalDate;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.cob.common.CustomJobParameterResolver;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
@Slf4j
@RequiredArgsConstructor
public class LoanIdParameterTasklet implements Tasklet {
private final RetrieveLoanIdService retrieveLoanIdService;
private final CustomJobParameterResolver customJobParameterResolver;
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
String businessDateParameter = (String) contribution.getStepExecution().getJobExecution().getExecutionContext()
.get(LoanCOBConstant.BUSINESS_DATE_PARAMETER_NAME);
LocalDate businessDate = LocalDate.parse(Objects.requireNonNull(businessDateParameter));
LoanCOBParameter minAndMaxLoanId = retrieveLoanIdService.retrieveMinAndMaxLoanIdsNDaysBehind(LoanCOBConstant.NUMBER_OF_DAYS_BEHIND,
businessDate, customJobParameterResolver.getCustomJobParameterById(chunkContext.getStepContext().getStepExecution(),
LoanCOBConstant.IS_CATCH_UP_PARAMETER_NAME).map(Boolean::parseBoolean).orElse(false));
if (Objects.isNull(minAndMaxLoanId)
|| (Objects.isNull(minAndMaxLoanId.getMinLoanId()) && Objects.isNull(minAndMaxLoanId.getMaxLoanId()))) {
contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.LOAN_COB_PARAMETER,
new LoanCOBParameter(0L, 0L));
return RepeatStatus.FINISHED;
}
contribution.getStepExecution().getJobExecution().getExecutionContext().put(LoanCOBConstant.LOAN_COB_PARAMETER, minAndMaxLoanId);
return RepeatStatus.FINISHED;
}
}
fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImpl.java
+28 -3
View File
@@ -18,28 +18,53 @@
*/
package org.apache.fineract.cob.loan;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo;
import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
@RequiredArgsConstructor
public class RetrieveAllNonClosedLoanIdServiceImpl implements RetrieveLoanIdService {
private final LoanRepository loanRepository;
private final NamedParameterJdbcTemplate namedParameterJdbcTemplate;
@Override
public LoanCOBParameter retrieveMinAndMaxLoanIdsNDaysBehind(Long numberOfDays, LocalDate businessDate, boolean isCatchUp) {
public List<LoanCOBPartition> retrieveLoanCOBPartitions(Long numberOfDays, LocalDate businessDate, boolean isCatchUp,
int partitionSize) {
StringBuilder sql = new StringBuilder();
sql.append("select min(id) as min, max(id) as max, page, count(id) as count from ");
sql.append(" (select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from ");
sql.append(" (select id from m_loan where loan_status_id in (:statusIds) and ");
if (isCatchUp) {
return loanRepository.findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDateNotNull(businessDate.minusDays(numberOfDays));
sql.append("last_closed_business_date = :businessDate ");
} else {
return loanRepository.findMinAndMaxNonClosedLoanIdsByLastClosedBusinessDate(businessDate.minusDays(numberOfDays));
sql.append("(last_closed_business_date = :businessDate or last_closed_business_date is null) ");
}
sql.append("order by id) t) t2 ");
sql.append("group by page ");
sql.append("order by page");
MapSqlParameterSource parameters = new MapSqlParameterSource();
parameters.addValue("pageSize", partitionSize);
parameters.addValue("statusIds", List.of(100, 200, 300, 303, 304));
parameters.addValue("businessDate", businessDate.minusDays(numberOfDays));
return namedParameterJdbcTemplate.query(sql.toString(), parameters, RetrieveAllNonClosedLoanIdServiceImpl::mapRow);
}
private static LoanCOBPartition mapRow(ResultSet rs, int rowNum) throws SQLException {
return new LoanCOBPartition(rs.getLong("min"), rs.getLong("max"), rs.getLong("page"), rs.getLong("count"));
}
@Override
fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdConfiguration.java
+5 -1
View File
@@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
@Configuration
public class RetrieveLoanIdConfiguration {
@@ -30,9 +31,12 @@ public class RetrieveLoanIdConfiguration {
@Autowired
private LoanRepository loanRepository;
@Autowired
private NamedParameterJdbcTemplate namedParameterJdbcTemplate;
@Bean
@ConditionalOnMissingBean
public RetrieveLoanIdService retrieveLoanIdService() {
return new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository);
return new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository, namedParameterJdbcTemplate);
}
}
fineract-provider/src/main/java/org/apache/fineract/cob/loan/RetrieveLoanIdService.java
+2 -1
View File
@@ -21,13 +21,14 @@ package org.apache.fineract.cob.loan;
import java.time.LocalDate;
import java.util.List;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.cob.data.LoanIdAndExternalIdAndAccountNo;
import org.apache.fineract.cob.data.LoanIdAndLastClosedBusinessDate;
import org.springframework.data.repository.query.Param;
public interface RetrieveLoanIdService {
LoanCOBParameter retrieveMinAndMaxLoanIdsNDaysBehind(Long numberOfDays, LocalDate businessDate, boolean isCatchUp);
List<LoanCOBPartition> retrieveLoanCOBPartitions(Long numberOfDays, LocalDate businessDate, boolean isCatchUp, int partitionSize);
List<LoanIdAndLastClosedBusinessDate> retrieveLoanIdsBehindDateOrNull(LocalDate businessDate, List<Long> loanIds);
fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerStepDefinitions.java
-136
View File
@@ -1,136 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import com.google.gson.Gson;
import io.cucumber.java8.En;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.mockito.Mockito;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.item.ExecutionContext;
public class LoanCOBPartitionerStepDefinitions implements En {
PropertyService propertyService = mock(PropertyService.class);
COBBusinessStepService cobBusinessStepService = mock(COBBusinessStepService.class);
JobOperator jobOperator = mock(JobOperator.class);
JobExplorer jobExplorer = mock(JobExplorer.class);
private final Gson gson = GoogleGsonSerializerHelper.createSimpleGson();
LoanCOBParameter loanIds;
private LoanCOBPartitioner loanCOBPartitioner;
private Set<BusinessStepNameAndOrder> cobBusinessSteps = new HashSet<>();
private Map<String, ExecutionContext> resultItem;
private String action;
public LoanCOBPartitionerStepDefinitions() {
Given("/^The LoanCOBPartitioner.partition method with action (.*)$/", (String action) -> {
this.action = action;
lenient().when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(2);
if ("empty steps".equals(action)) {
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(Collections.emptySet());
lenient().when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(new JobExecution(3L)));
lenient().when(jobOperator.stop(3L)).thenReturn(Boolean.TRUE);
loanIds = null;
} else if ("empty loanIds".equals(action)) {
cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L));
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(cobBusinessSteps);
loanIds = null;
} else if ("good".equals(action)) {
cobBusinessSteps.add(new BusinessStepNameAndOrder("Business step", 1L));
lenient().when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(cobBusinessSteps);
loanIds = new LoanCOBParameter(1L, 3L);
}
loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, jobOperator, jobExplorer, loanIds);
});
When("LoanCOBPartitioner.partition method executed", () -> {
resultItem = this.loanCOBPartitioner.partition(2);
});
Then("LoanCOBPartitioner.partition result should match", () -> {
if ("empty steps".equals(action)) {
verify(jobOperator, Mockito.times(1)).stop(3L);
assertTrue(resultItem.isEmpty());
} else if ("good".equals(action)) {
verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong());
assertEquals(2, resultItem.size());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1"));
Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) resultItem
.get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS);
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(1, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId());
assertEquals(1L, loanCOBParameter.getMinLoanId());
assertEquals(2L, loanCOBParameter.getMaxLoanId());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "2"));
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "2")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(0, loanCOBParameter.getMaxLoanId() - loanCOBParameter.getMinLoanId());
assertEquals(3L, loanCOBParameter.getMinLoanId());
assertEquals(3L, loanCOBParameter.getMaxLoanId());
} else if ("empty loanIds".equals(action)) {
verify(jobOperator, Mockito.times(0)).stop(Mockito.anyLong());
assertEquals(1, resultItem.size());
assertTrue(resultItem.containsKey(LoanCOBPartitioner.PARTITION_PREFIX + "1"));
Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) resultItem
.get(LoanCOBPartitioner.PARTITION_PREFIX + "1").get(LoanCOBConstant.BUSINESS_STEPS);
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepOrder(),
businessSteps.stream().findFirst().get().getStepOrder());
assertEquals(cobBusinessSteps.stream().findFirst().get().getStepName(),
businessSteps.stream().findFirst().get().getStepName());
LoanCOBParameter loanCOBParameter = (LoanCOBParameter) resultItem.get(LoanCOBPartitioner.PARTITION_PREFIX + "1")
.get(LoanCOBConstant.LOAN_COB_PARAMETER);
assertEquals(0L, loanCOBParameter.getMinLoanId());
assertEquals(0L, loanCOBParameter.getMaxLoanId());
}
});
}
}
fineract-provider/src/test/java/org/apache/fineract/cob/loan/LoanCOBPartitionerTest.java
+135
View File
@@ -0,0 +1,135 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.fineract.cob.COBBusinessStepService;
import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
import org.apache.fineract.cob.data.LoanCOBParameter;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.springbatch.PropertyService;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobExecutionNotRunningException;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
import org.springframework.batch.item.ExecutionContext;
@ExtendWith(MockitoExtension.class)
class LoanCOBPartitionerTest {
private static final Set<BusinessStepNameAndOrder> BUSINESS_STEP_SET = Set.of(new BusinessStepNameAndOrder("Business step", 1L));
private static final LocalDate BUSINESS_DATE = LocalDate.parse("2023-06-28");
@Mock
private PropertyService propertyService;
@Mock
private COBBusinessStepService cobBusinessStepService;
@Mock
private RetrieveLoanIdService retrieveLoanIdService;
@Mock
private JobOperator jobOperator;
@Mock
private JobExplorer jobExplorer;
@Test
public void testLoanCOBPartitioner() {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(BUSINESS_STEP_SET);
when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5))
.thenReturn(List.of(new LoanCOBPartition(1L,10L, 1L, 5L), new LoanCOBPartition(11L,20L, 2L, 4L)));
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(2, partitions.size());
validatePartitions(partitions, 1, 1, 10);
validatePartitions(partitions, 2, 11, 20);
}
@Test
public void testLoanCOBPartitionerEmptyBusinessSteps() throws NoSuchJobExecutionException, JobExecutionNotRunningException {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(Set.of());
JobExecution jobExecution = Mockito.mock(JobExecution.class);
when(jobExecution.getId()).thenReturn(123L);
when(jobExplorer.findRunningJobExecutions(JobName.LOAN_COB.name())).thenReturn(Set.of(jobExecution));
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(0, partitions.size());
verify(jobExplorer, times(1)).findRunningJobExecutions(JobName.LOAN_COB.name());
verify(jobOperator, times(1)).stop(123L);
}
@Test
public void testLoanCOBPartitionerNoLoansFound() {
//given
when(propertyService.getPartitionSize(LoanCOBConstant.JOB_NAME)).thenReturn(5);
when(cobBusinessStepService.getCOBBusinessSteps(LoanCOBBusinessStep.class, LoanCOBConstant.LOAN_COB_JOB_NAME))
.thenReturn(BUSINESS_STEP_SET);
when(retrieveLoanIdService.retrieveLoanCOBPartitions(1L, BUSINESS_DATE, false, 5))
.thenReturn(List.of());
LoanCOBPartitioner loanCOBPartitioner = new LoanCOBPartitioner(propertyService, cobBusinessStepService, retrieveLoanIdService, jobOperator, jobExplorer, 1L);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setBusinessDate(BUSINESS_DATE);
loanCOBPartitioner.setIsCatchUp(false);
//when
Map<String, ExecutionContext> partitions = loanCOBPartitioner.partition(1);
//then
Assertions.assertEquals(1, partitions.size());
validatePartitions(partitions, 1, 0, 0);
}
private void validatePartitions(Map<String, ExecutionContext> partitions, int index, long min, long max) {
Assertions.assertEquals(BUSINESS_STEP_SET,
partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.BUSINESS_STEPS));
Assertions.assertEquals(new LoanCOBParameter(min, max),
partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get(LoanCOBConstant.LOAN_COB_PARAMETER));
Assertions.assertEquals("partition_" + index, partitions.get(LoanCOBPartitioner.PARTITION_PREFIX + index).get("partition"));
}
}
fineract-provider/src/test/java/org/apache/fineract/cob/loan/RetrieveAllNonClosedLoanIdServiceImplTest.java
+93
View File
@@ -0,0 +1,93 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.cob.loan;
import static org.mockito.Mockito.times;
import java.time.LocalDate;
import java.util.List;
import org.apache.fineract.cob.data.LoanCOBPartition;
import org.apache.fineract.portfolio.loanaccount.domain.LoanRepository;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
@ExtendWith(MockitoExtension.class)
public class RetrieveAllNonClosedLoanIdServiceImplTest {
@Mock
private LoanRepository loanRepository;
@Mock
private NamedParameterJdbcTemplate namedParameterJdbcTemplate;
@Captor
private ArgumentCaptor<String> sqlCaptor;
@Captor
private ArgumentCaptor<SqlParameterSource> paramsCaptor;
@Captor
private ArgumentCaptor<RowMapper<LoanCOBPartition>> rowMapper;
@Test
public void testRetrieveLoanCOBPartitionsNoCatchup() {
String expectedSQL = """
select min(id) as min, max(id) as max, page, count(id) as count from
(select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from
(select id from m_loan where loan_status_id in (:statusIds) and (last_closed_business_date = :businessDate or last_closed_business_date is null) order by id) t) t2
group by page
order by page
""";
testRetrieveLoanCOBPartitions(expectedSQL, false);
}
@Test
public void testRetrieveLoanCOBPartitionsCatchup() {
String expectedSQL = """
select min(id) as min, max(id) as max, page, count(id) as count from
(select floor(((row_number() over(order by id))-1) / :pageSize) as page, t.* from
(select id from m_loan where loan_status_id in (:statusIds) and last_closed_business_date = :businessDate order by id) t) t2
group by page
order by page
""";
testRetrieveLoanCOBPartitions(expectedSQL, true);
}
private void testRetrieveLoanCOBPartitions(String expectedSQL, boolean isCatchup) {
RetrieveAllNonClosedLoanIdServiceImpl service = new RetrieveAllNonClosedLoanIdServiceImpl(loanRepository,
namedParameterJdbcTemplate);
LocalDate businessDate = LocalDate.parse("2023-06-28");
service.retrieveLoanCOBPartitions(1L, businessDate, isCatchup, 5);
Mockito.verify(namedParameterJdbcTemplate, times(1)).query(sqlCaptor.capture(), paramsCaptor.capture(), rowMapper.capture());
Assertions.assertEquals(normalize(expectedSQL), normalize(sqlCaptor.getValue()));
Assertions.assertEquals(5, paramsCaptor.getValue().getValue("pageSize"));
Assertions.assertEquals(List.of(100, 200, 300, 303, 304), paramsCaptor.getValue().getValue("statusIds"));
Assertions.assertEquals(LocalDate.parse("2023-06-27"), paramsCaptor.getValue().getValue("businessDate"));
}
private String normalize(String str) {
return str.replaceAll(" +", " ").replaceAll("\r?\n", "");
}
}
fineract-provider/src/test/resources/features/cob/loan/cob.loan.partitioner.feature
-32
View File
@@ -1,32 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
Feature: COB Loan partitioner Step
@cob
Scenario Outline: LoanCOBPartitioner - run test
Given The LoanCOBPartitioner.partition method with action <action>
When LoanCOBPartitioner.partition method executed
Then LoanCOBPartitioner.partition result should match
Examples:
|action|
|empty steps|
|good|
|empty loanIds|
fineract-provider/src/test/resources/logback.xml
+1
View File
@@ -20,6 +20,7 @@
-->
<configuration scan="false">
<statusListener class="ch.qos.logback.core.status.NopStatusListener" /> <!-- this hides the warning related to jansi in windows -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>false</resetJUL>
</contextListener>
integration-tests/src/test/java/org/apache/fineract/integrationtests/cob/CobPartitioningTest.java
+268
View File
@@ -0,0 +1,268 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.integrationtests.cob;
import static org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType.BUSINESS_DATE;
import io.restassured.builder.RequestSpecBuilder;
import io.restassured.builder.ResponseSpecBuilder;
import io.restassured.http.ContentType;
import io.restassured.path.json.JsonPath;
import io.restassured.specification.RequestSpecification;
import io.restassured.specification.ResponseSpecification;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.accounting.common.AccountingConstants;
import org.apache.fineract.client.models.GetFinancialActivityAccountsResponse;
import org.apache.fineract.client.models.PostFinancialActivityAccountsRequest;
import org.apache.fineract.integrationtests.common.BusinessDateHelper;
import org.apache.fineract.integrationtests.common.BusinessStepHelper;
import org.apache.fineract.integrationtests.common.ClientHelper;
import org.apache.fineract.integrationtests.common.CollateralManagementHelper;
import org.apache.fineract.integrationtests.common.ExternalAssetOwnerHelper;
import org.apache.fineract.integrationtests.common.GlobalConfigurationHelper;
import org.apache.fineract.integrationtests.common.SchedulerJobHelper;
import org.apache.fineract.integrationtests.common.Utils;
import org.apache.fineract.integrationtests.common.accounting.Account;
import org.apache.fineract.integrationtests.common.accounting.AccountHelper;
import org.apache.fineract.integrationtests.common.accounting.FinancialActivityAccountHelper;
import org.apache.fineract.integrationtests.common.charges.ChargesHelper;
import org.apache.fineract.integrationtests.common.loans.CobHelper;
import org.apache.fineract.integrationtests.common.loans.LoanApplicationTestBuilder;
import org.apache.fineract.integrationtests.common.loans.LoanProductTestBuilder;
import org.apache.fineract.integrationtests.common.loans.LoanStatusChecker;
import org.apache.fineract.integrationtests.common.loans.LoanTestLifecycleExtension;
import org.apache.fineract.integrationtests.common.loans.LoanTransactionHelper;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@SuppressWarnings("rawtypes")
@Slf4j
@ExtendWith(LoanTestLifecycleExtension.class)
public class CobPartitioningTest {
public static final int N = 10;
private static ResponseSpecification RESPONSE_SPEC;
private static RequestSpecification REQUEST_SPEC;
private static Account ASSET_ACCOUNT;
private static Account FEE_PENALTY_ACCOUNT;
private static Account TRANSFER_ACCOUNT;
private static Account EXPENSE_ACCOUNT;
private static Account INCOME_ACCOUNT;
private static Account OVERPAYMENT_ACCOUNT;
private static FinancialActivityAccountHelper FINANCIAL_ACTIVITY_ACCOUNT_HELPER;
private static ExternalAssetOwnerHelper EXTERNAL_ASSET_OWNER_HELPER;
private static LoanTransactionHelper LOAN_TRANSACTION_HELPER;
private static SchedulerJobHelper SCHEDULER_JOB_HELPER;
private static LocalDate TODAYS_DATE;
@BeforeAll
public static void setupInvestorBusinessStep() {
Utils.initializeRESTAssured();
REQUEST_SPEC = new RequestSpecBuilder().setContentType(ContentType.JSON).build();
REQUEST_SPEC.header("Authorization", "Basic " + Utils.loginIntoServerAndGetBase64EncodedAuthenticationKey());
RESPONSE_SPEC = new ResponseSpecBuilder().expectStatusCode(200).build();
AccountHelper accountHelper = new AccountHelper(REQUEST_SPEC, RESPONSE_SPEC);
EXTERNAL_ASSET_OWNER_HELPER = new ExternalAssetOwnerHelper();
SCHEDULER_JOB_HELPER = new SchedulerJobHelper(REQUEST_SPEC);
FINANCIAL_ACTIVITY_ACCOUNT_HELPER = new FinancialActivityAccountHelper(REQUEST_SPEC);
LOAN_TRANSACTION_HELPER = new LoanTransactionHelper(REQUEST_SPEC, RESPONSE_SPEC);
TODAYS_DATE = Utils.getLocalDateOfTenant();
new BusinessStepHelper().updateSteps("LOAN_CLOSE_OF_BUSINESS", "APPLY_CHARGE_TO_OVERDUE_LOANS", "LOAN_DELINQUENCY_CLASSIFICATION",
"CHECK_LOAN_REPAYMENT_DUE", "CHECK_LOAN_REPAYMENT_OVERDUE", "UPDATE_LOAN_ARREARS_AGING", "ADD_PERIODIC_ACCRUAL_ENTRIES",
"EXTERNAL_ASSET_OWNER_TRANSFER");
ASSET_ACCOUNT = accountHelper.createAssetAccount();
FEE_PENALTY_ACCOUNT = accountHelper.createAssetAccount();
TRANSFER_ACCOUNT = accountHelper.createAssetAccount();
EXPENSE_ACCOUNT = accountHelper.createExpenseAccount();
INCOME_ACCOUNT = accountHelper.createIncomeAccount();
OVERPAYMENT_ACCOUNT = accountHelper.createLiabilityAccount();
setProperFinancialActivity(TRANSFER_ACCOUNT);
}
private static void setProperFinancialActivity(Account transferAccount) {
List<GetFinancialActivityAccountsResponse> financialMappings = FINANCIAL_ACTIVITY_ACCOUNT_HELPER.getAllFinancialActivityAccounts();
financialMappings.forEach(mapping -> FINANCIAL_ACTIVITY_ACCOUNT_HELPER.deleteFinancialActivityAccount(mapping.getId()));
FINANCIAL_ACTIVITY_ACCOUNT_HELPER.createFinancialActivityAccount(new PostFinancialActivityAccountsRequest()
.financialActivityId((long) AccountingConstants.FinancialActivity.ASSET_TRANSFER.getValue())
.glAccountId((long) transferAccount.getAccountID()));
}
@Test
public void testLoanCOBPartitioningQuery() throws InterruptedException {
try {
ExecutorService executorService = Executors.newFixedThreadPool(10);
GlobalConfigurationHelper.manageConfigurations(REQUEST_SPEC, RESPONSE_SPEC,
GlobalConfigurationHelper.ENABLE_AUTOGENERATED_EXTERNAL_ID, true);
setInitialBusinessDate("2020-03-02");
List<Integer> loanIds = new CopyOnWriteArrayList<>();
// Let's create 1, 2, ..., N-1, N loans
final CountDownLatch createLatch = new CountDownLatch(N);
for (int i = 0; i < N; i++) {
Future<?> unused = executorService.submit(() -> {
Integer clientID = createClient();
Integer loanID = createLoanForClient(clientID);
loanIds.add(loanID);
createLatch.countDown();
});
}
createLatch.await();
// Force close loans 3, 4, ... , N-3, N-2
Collections.sort(loanIds);
final CountDownLatch closeLatch = new CountDownLatch(N - 4);
for (int i = 2; i < N - 2; i++) {
final int idx = i;
Future<?> unused = executorService.submit(() -> {
LOAN_TRANSACTION_HELPER.forecloseLoan("02 March 2020", loanIds.get(idx));
closeLatch.countDown();
});
}
closeLatch.await();
// Let's retrieve the partitions
List<Map<String, Object>> cobPartitions = CobHelper.getCobPartitions(REQUEST_SPEC, RESPONSE_SPEC, 3, "");
log.info("\nLoans created: {},\nRetrieved partitions: {}", loanIds, cobPartitions);
Assertions.assertEquals(2, cobPartitions.size());
Assertions.assertEquals(0, cobPartitions.get(0).get("pageNo"));
Assertions.assertEquals(loanIds.get(0), cobPartitions.get(0).get("minId"));
Assertions.assertEquals(loanIds.get(8), cobPartitions.get(0).get("maxId"));
Assertions.assertEquals(3, cobPartitions.get(0).get("count"));
Assertions.assertEquals(1, cobPartitions.get(1).get("pageNo"));
Assertions.assertEquals(loanIds.get(9), cobPartitions.get(1).get("minId"));
Assertions.assertEquals(loanIds.get(9), cobPartitions.get(1).get("maxId"));
Assertions.assertEquals(1, cobPartitions.get(1).get("count"));
executorService.shutdown();
} finally {
cleanUpAndRestoreBusinessDate();
}
}
private void setInitialBusinessDate(String date) {
GlobalConfigurationHelper.updateIsBusinessDateEnabled(REQUEST_SPEC, RESPONSE_SPEC, Boolean.TRUE);
BusinessDateHelper.updateBusinessDate(REQUEST_SPEC, RESPONSE_SPEC, BUSINESS_DATE, LocalDate.parse(date));
GlobalConfigurationHelper.updateValueForGlobalConfiguration(REQUEST_SPEC, RESPONSE_SPEC, "10", "0");
}
private void cleanUpAndRestoreBusinessDate() {
REQUEST_SPEC = new RequestSpecBuilder().setContentType(ContentType.JSON).build();
REQUEST_SPEC.header("Authorization", "Basic " + Utils.loginIntoServerAndGetBase64EncodedAuthenticationKey());
REQUEST_SPEC.header("Fineract-Platform-TenantId", "default");
RESPONSE_SPEC = new ResponseSpecBuilder().expectStatusCode(200).build();
BusinessDateHelper.updateBusinessDate(REQUEST_SPEC, RESPONSE_SPEC, BUSINESS_DATE, TODAYS_DATE);
GlobalConfigurationHelper.updateIsBusinessDateEnabled(REQUEST_SPEC, RESPONSE_SPEC, Boolean.FALSE);
GlobalConfigurationHelper.manageConfigurations(REQUEST_SPEC, RESPONSE_SPEC,
GlobalConfigurationHelper.ENABLE_AUTOGENERATED_EXTERNAL_ID, false);
}
@NotNull
private Integer createClient() {
final Integer clientID = ClientHelper.createClient(REQUEST_SPEC, RESPONSE_SPEC);
Assertions.assertNotNull(clientID);
return clientID;
}
@NotNull
private Integer createLoanForClient(Integer clientID) {
Integer overdueFeeChargeId = ChargesHelper.createCharges(REQUEST_SPEC, RESPONSE_SPEC,
ChargesHelper.getLoanOverdueFeeJSONWithCalculationTypePercentage("1"));
Assertions.assertNotNull(overdueFeeChargeId);
Integer loanProductID = createLoanProduct(overdueFeeChargeId.toString());
Assertions.assertNotNull(loanProductID);
HashMap loanStatusHashMap;
Integer loanID = applyForLoanApplication(clientID.toString(), loanProductID.toString(), "10 January 2020");
Assertions.assertNotNull(loanID);
loanStatusHashMap = LoanStatusChecker.getStatusOfLoan(REQUEST_SPEC, RESPONSE_SPEC, loanID);
LoanStatusChecker.verifyLoanIsPending(loanStatusHashMap);
loanStatusHashMap = LOAN_TRANSACTION_HELPER.approveLoan("01 March 2020", loanID);
LoanStatusChecker.verifyLoanIsApproved(loanStatusHashMap);
String loanDetails = LOAN_TRANSACTION_HELPER.getLoanDetails(REQUEST_SPEC, RESPONSE_SPEC, loanID);
loanStatusHashMap = LOAN_TRANSACTION_HELPER.disburseLoanWithNetDisbursalAmount("02 March 2020", loanID,
JsonPath.from(loanDetails).get("netDisbursalAmount").toString());
LoanStatusChecker.verifyLoanIsActive(loanStatusHashMap);
return loanID;
}
private Integer createLoanProduct(final String chargeId) {
final String loanProductJSON = new LoanProductTestBuilder().withPrincipal("15,000.00").withNumberOfRepayments("4")
.withRepaymentAfterEvery("1").withRepaymentTypeAsMonth().withinterestRatePerPeriod("1")
.withAccountingRulePeriodicAccrual(new Account[] { ASSET_ACCOUNT, EXPENSE_ACCOUNT, INCOME_ACCOUNT, OVERPAYMENT_ACCOUNT })
.withInterestRateFrequencyTypeAsMonths().withAmortizationTypeAsEqualInstallments().withInterestTypeAsDecliningBalance()
.withFeeAndPenaltyAssetAccount(FEE_PENALTY_ACCOUNT).build(chargeId);
return LOAN_TRANSACTION_HELPER.getLoanProductId(loanProductJSON);
}
private Integer applyForLoanApplication(final String clientID, final String loanProductID, final String date) {
List<HashMap> collaterals = new ArrayList<>();
Integer collateralId = CollateralManagementHelper.createCollateralProduct(REQUEST_SPEC, RESPONSE_SPEC);
Assertions.assertNotNull(collateralId);
Integer clientCollateralId = CollateralManagementHelper.createClientCollateral(REQUEST_SPEC, RESPONSE_SPEC, clientID, collateralId);
Assertions.assertNotNull(clientCollateralId);
addCollaterals(collaterals, clientCollateralId, BigDecimal.valueOf(1));
String loanApplicationJSON = new LoanApplicationTestBuilder().withPrincipal("15,000.00").withLoanTermFrequency("4")
.withLoanTermFrequencyAsMonths().withNumberOfRepayments("4").withRepaymentEveryAfter("1")
.withRepaymentFrequencyTypeAsMonths().withInterestRatePerPeriod("2").withAmortizationTypeAsEqualInstallments()
.withInterestTypeAsDecliningBalance().withInterestCalculationPeriodTypeSameAsRepaymentPeriod()
.withExpectedDisbursementDate(date).withSubmittedOnDate(date).withCollaterals(collaterals)
.build(clientID, loanProductID, null);
return LOAN_TRANSACTION_HELPER.getLoanId(loanApplicationJSON);
}
private void addCollaterals(List<HashMap> collaterals, Integer collateralId, BigDecimal quantity) {
collaterals.add(collaterals(collateralId, quantity));
}
private HashMap<String, String> collaterals(Integer collateralId, BigDecimal quantity) {
HashMap<String, String> collateral = new HashMap<>(2);
collateral.put("clientCollateralId", collateralId.toString());
collateral.put("quantity", quantity.toString());
return collateral;
}
}
integration-tests/src/test/java/org/apache/fineract/integrationtests/common/loans/CobHelper.java
+38
View File
@@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.integrationtests.common.loans;
import io.restassured.specification.RequestSpecification;
import io.restassured.specification.ResponseSpecification;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.integrationtests.client.IntegrationTest;
import org.apache.fineract.integrationtests.common.Utils;
@Slf4j
public class CobHelper extends IntegrationTest {
public static List<Map<String, Object>> getCobPartitions(final RequestSpecification requestSpec,
final ResponseSpecification responseSpec, int partitionSize, final String jsonReturn) {
final String GET_LOAN_URL = "/fineract-provider/api/v1/internal/cob/partitions/" + partitionSize + "?" + Utils.TENANT_IDENTIFIER;
log.info("---------------------------------GET COB PARTITIONS---------------------------------------------");
return Utils.performServerGet(requestSpec, responseSpec, GET_LOAN_URL, jsonReturn);
}
}