From 0ae8445f07c0ca196827cf363bcbbd75bdda8d37 Mon Sep 17 00:00:00 2001 From: Ruchi Dhamankar Date: Mon, 10 Oct 2022 13:19:22 +0530 Subject: [PATCH] FINERACT-1694-Regular-Event-Purging --- .../domain/ConfigurationDomainService.java | 2 + .../domain/ConfigurationDomainServiceJpa.java | 8 ++ .../jobs/PurgeExternalEventsConfig.java | 50 ++++++++++ .../jobs/PurgeExternalEventsTasklet.java | 54 +++++++++++ .../repository/ExternalEventRepository.java | 7 ++ .../infrastructure/jobs/service/JobName.java | 3 +- .../db/changelog/tenant/changelog-tenant.xml | 1 + .../0053_add_external_events_purge_job.xml | 59 +++++++++++ .../jobs/PurgeExternalEventsTaskletTest.java | 97 +++++++++++++++++++ .../ClientLoanIntegrationTest.java | 2 +- .../common/GlobalConfigurationHelper.java | 12 ++- 11 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsConfig.java create mode 100644 fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTasklet.java create mode 100644 fineract-provider/src/main/resources/db/changelog/tenant/parts/0053_add_external_events_purge_job.xml create mode 100644 fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTaskletTest.java diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java index f87ec0f32..9d7d23d2a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java @@ -118,4 +118,6 @@ public interface ConfigurationDomainService { boolean isCOBDateAdjustmentEnabled(); boolean isReversalTransactionAllowed(); + + Long retrieveExternalEventsPurgeDaysCriteria(); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java index 4657bab32..e684eb80a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java @@ -40,6 +40,7 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService public static final String ENABLE_BUSINESS_DATE = "enable_business_date"; public static final String ENABLE_AUTOMATIC_COB_DATE_ADJUSTMENT = "enable_automatic_cob_date_adjustment"; + public static final String EXTERNAL_EVENTS_PURGE_DAYS = "purge-external-events-older-than-days"; private final PermissionRepository permissionRepository; private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository; private final PlatformCacheRepository cacheTypeRepository; @@ -451,4 +452,11 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(propertyName); return property.isEnabled(); } + + @Override + public Long retrieveExternalEventsPurgeDaysCriteria() { + final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(EXTERNAL_EVENTS_PURGE_DAYS); + return property.getValue(); + + } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsConfig.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsConfig.java new file mode 100644 index 000000000..c5735456a --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsConfig.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.event.external.jobs; + +import org.apache.fineract.infrastructure.jobs.service.JobName; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PurgeExternalEventsConfig { + + @Autowired + private JobBuilderFactory jobs; + @Autowired + private StepBuilderFactory steps; + @Autowired + private PurgeExternalEventsTasklet tasklet; + + @Bean + protected Step purgeExternalEventsStep() { + return steps.get(JobName.PURGE_EXTERNAL_EVENTS.name()).tasklet(tasklet).build(); + } + + @Bean + public Job purgeExternalEventsJob() { + return jobs.get(JobName.PURGE_EXTERNAL_EVENTS.name()).start(purgeExternalEventsStep()).incrementer(new RunIdIncrementer()).build(); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTasklet.java new file mode 100644 index 000000000..895cda3a2 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTasklet.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.event.external.jobs; + +import java.time.LocalDate; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.service.DateUtils; +import org.apache.fineract.infrastructure.event.external.repository.ExternalEventRepository; +import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEventStatus; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +@Slf4j +@AllArgsConstructor +@Component +public class PurgeExternalEventsTasklet implements Tasklet { + + private final ExternalEventRepository repository; + private final ConfigurationDomainService configurationDomainService; + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + try { + Long numberOfDaysForPurgeCriteria = configurationDomainService.retrieveExternalEventsPurgeDaysCriteria(); + LocalDate dateForPurgeCriteria = DateUtils.getBusinessLocalDate().minusDays(numberOfDaysForPurgeCriteria); + repository.deleteOlderEventsWithSentStatus(ExternalEventStatus.SENT, dateForPurgeCriteria); + } catch (Exception e) { + log.error("Error occurred while purging external events: ", e); + } + return RepeatStatus.FINISHED; + } + +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/repository/ExternalEventRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/repository/ExternalEventRepository.java index 8282bc854..4a04bfa84 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/repository/ExternalEventRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/repository/ExternalEventRepository.java @@ -18,13 +18,20 @@ */ package org.apache.fineract.infrastructure.event.external.repository; +import java.time.LocalDate; import java.util.List; import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEvent; import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEventStatus; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; public interface ExternalEventRepository extends JpaRepository { List findByStatusOrderById(ExternalEventStatus status, Pageable batchSize); + + @Modifying(flushAutomatically = true) + @Query("delete from ExternalEvent e where e.status = :status and e.businessDate <= :dateForPurgeCriteria") + void deleteOlderEventsWithSentStatus(ExternalEventStatus status, LocalDate dateForPurgeCriteria); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java index 152fe5cad..9b1a32308 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java @@ -57,7 +57,8 @@ public enum JobName { INCREASE_COB_DATE_BY_1_DAY("Increase COB Date by 1 day"), // LOAN_COB("Loan COB"), // LOAN_DELINQUENCY_CLASSIFICATION("Loan Delinquency Classification"), // - SEND_ASYNCHRONOUS_EVENTS("Send Asynchronous Events"); + SEND_ASYNCHRONOUS_EVENTS("Send Asynchronous Events"), // + PURGE_EXTERNAL_EVENTS("Purge External Events"); private final String name; diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml index ba4e3bfc4..933c2b523 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml @@ -72,4 +72,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0053_add_external_events_purge_job.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0053_add_external_events_purge_job.xml new file mode 100644 index 000000000..57d771010 --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0053_add_external_events_purge_job.xml @@ -0,0 +1,59 @@ + + + + + + SELECT SETVAL('c_configuration_id_seq', COALESCE(MAX(id), 0)+1, false ) FROM c_configuration; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTaskletTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTaskletTest.java new file mode 100644 index 000000000..3aa5e61e9 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTaskletTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.event.external.jobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; +import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; +import org.apache.fineract.infrastructure.core.service.DateUtils; +import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; +import org.apache.fineract.infrastructure.event.external.repository.ExternalEventRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.repeat.RepeatStatus; + +@ExtendWith(MockitoExtension.class) +public class PurgeExternalEventsTaskletTest { + + @Mock + private ExternalEventRepository repository; + @Mock + private ConfigurationDomainService configurationDomainService; + @Mock + private StepContribution stepContribution; + @Mock + private ChunkContext chunkContext; + private RepeatStatus resultStatus; + private PurgeExternalEventsTasklet underTest; + + @BeforeEach + public void setUp() { + ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null)); + ThreadLocalContextUtil + .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault())))); + underTest = new PurgeExternalEventsTasklet(repository, configurationDomainService); + } + + @Test + public void givenEventsForPurgeWhenTaskExecutionThenEventsPurgeForDaysCriteria() { + // given + ArgumentCaptor dateCriteriaCaptor = ArgumentCaptor.forClass(LocalDate.class); + when(configurationDomainService.retrieveExternalEventsPurgeDaysCriteria()).thenReturn(2L); + // when + resultStatus = underTest.execute(stepContribution, chunkContext); + // then + verify(repository, times(1)).deleteOlderEventsWithSentStatus(Mockito.any(), Mockito.any()); + verify(repository).deleteOlderEventsWithSentStatus(Mockito.any(), dateCriteriaCaptor.capture()); + LocalDate expectedDateForPurgeCriteriaTest = DateUtils.getBusinessLocalDate().minusDays(2); + LocalDate actualDateForPurgeCriteria = dateCriteriaCaptor.getValue(); + assertEquals(expectedDateForPurgeCriteriaTest, actualDateForPurgeCriteria); + assertEquals(RepeatStatus.FINISHED, resultStatus); + } + + @Test + public void givenEventsForPurgeWhenExceptionOccursThenJobExecutionFinishesSuccessfully() { + // given + when(configurationDomainService.retrieveExternalEventsPurgeDaysCriteria()).thenReturn(2L); + doThrow(new RuntimeException("Test Exception")).when(repository).deleteOlderEventsWithSentStatus(Mockito.any(), Mockito.any()); + // when + resultStatus = underTest.execute(stepContribution, chunkContext); + // then + assertEquals(RepeatStatus.FINISHED, resultStatus); + } + +} diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/ClientLoanIntegrationTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/ClientLoanIntegrationTest.java index 69515a928..d068f74ab 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/ClientLoanIntegrationTest.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/ClientLoanIntegrationTest.java @@ -4783,7 +4783,7 @@ public class ClientLoanIntegrationTest { .withInterestCalculationPeriodTypeAsDays() // .withExpectedDisbursementDate(disbursementDate) // .withSubmittedOnDate(disbursementDate) // - .withwithRepaymentStrategy(repaymentStrategy).withRepaymentFrequencyTypeAsMonths()// + .withRepaymentStrategy(repaymentStrategy).withRepaymentFrequencyTypeAsMonths()// .withFirstRepaymentDate(firstRepaymentDate).withCollaterals(collaterals) .build(clientID.toString(), loanProductID.toString(), null); return this.loanTransactionHelper.getLoanId(loanApplicationJSON); diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java index 231168ce9..6154631b9 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java @@ -99,8 +99,8 @@ public class GlobalConfigurationHelper { ArrayList actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec); // There are currently 37 global configurations. - Assertions.assertEquals(41, expectedGlobalConfigurations.size()); - Assertions.assertEquals(41, actualGlobalConfigurations.size()); + Assertions.assertEquals(42, expectedGlobalConfigurations.size()); + Assertions.assertEquals(42, actualGlobalConfigurations.size()); for (int i = 0; i < expectedGlobalConfigurations.size(); i++) { @@ -464,6 +464,14 @@ public class GlobalConfigurationHelper { isReversalTransactionAllowed.put("trapDoor", false); defaults.add(isReversalTransactionAllowed); + HashMap purgeExternalEventsOlderThanDaysDefault = new HashMap<>(); + purgeExternalEventsOlderThanDaysDefault.put("id", 47); + purgeExternalEventsOlderThanDaysDefault.put("name", "purge-external-events-older-than-days"); + purgeExternalEventsOlderThanDaysDefault.put("value", 30); + purgeExternalEventsOlderThanDaysDefault.put("enabled", false); + purgeExternalEventsOlderThanDaysDefault.put("trapDoor", false); + defaults.add(purgeExternalEventsOlderThanDaysDefault); + return defaults; }