FINERACT-1694-Regular-Event-Purging

fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java

@@ -118,4 +118,6 @@ public interface ConfigurationDomainService {
     boolean isCOBDateAdjustmentEnabled();
 
     boolean isReversalTransactionAllowed();
+
+    Long retrieveExternalEventsPurgeDaysCriteria();
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java

@@ -40,6 +40,7 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
 
     public static final String ENABLE_BUSINESS_DATE = "enable_business_date";
     public static final String ENABLE_AUTOMATIC_COB_DATE_ADJUSTMENT = "enable_automatic_cob_date_adjustment";
+    public static final String EXTERNAL_EVENTS_PURGE_DAYS = "purge-external-events-older-than-days";
     private final PermissionRepository permissionRepository;
     private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository;
     private final PlatformCacheRepository cacheTypeRepository;
@@ -451,4 +452,11 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
         final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(propertyName);
         return property.isEnabled();
     }
+
+    @Override
+    public Long retrieveExternalEventsPurgeDaysCriteria() {
+        final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(EXTERNAL_EVENTS_PURGE_DAYS);
+        return property.getValue();
+
+    }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsConfig.java

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.jobs;
+
+import org.apache.fineract.infrastructure.jobs.service.JobName;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.core.launch.support.RunIdIncrementer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class PurgeExternalEventsConfig {
+
+    @Autowired
+    private JobBuilderFactory jobs;
+    @Autowired
+    private StepBuilderFactory steps;
+    @Autowired
+    private PurgeExternalEventsTasklet tasklet;
+
+    @Bean
+    protected Step purgeExternalEventsStep() {
+        return steps.get(JobName.PURGE_EXTERNAL_EVENTS.name()).tasklet(tasklet).build();
+    }
+
+    @Bean
+    public Job purgeExternalEventsJob() {
+        return jobs.get(JobName.PURGE_EXTERNAL_EVENTS.name()).start(purgeExternalEventsStep()).incrementer(new RunIdIncrementer()).build();
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTasklet.java

@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.jobs;
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.service.DateUtils;
+import org.apache.fineract.infrastructure.event.external.repository.ExternalEventRepository;
+import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEventStatus;
+import org.springframework.batch.core.StepContribution;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.core.step.tasklet.Tasklet;
+import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@AllArgsConstructor
+@Component
+public class PurgeExternalEventsTasklet implements Tasklet {
+
+    private final ExternalEventRepository repository;
+    private final ConfigurationDomainService configurationDomainService;
+
+    @Override
+    public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) {
+        try {
+            Long numberOfDaysForPurgeCriteria = configurationDomainService.retrieveExternalEventsPurgeDaysCriteria();
+            LocalDate dateForPurgeCriteria = DateUtils.getBusinessLocalDate().minusDays(numberOfDaysForPurgeCriteria);
+            repository.deleteOlderEventsWithSentStatus(ExternalEventStatus.SENT, dateForPurgeCriteria);
+        } catch (Exception e) {
+            log.error("Error occurred while purging external events: ", e);
+        }
+        return RepeatStatus.FINISHED;
+    }
+
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/repository/ExternalEventRepository.java

@@ -18,13 +18,20 @@
  */
 package org.apache.fineract.infrastructure.event.external.repository;
 
+import java.time.LocalDate;
 import java.util.List;
 import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEvent;
 import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEventStatus;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 
 public interface ExternalEventRepository extends JpaRepository<ExternalEvent, Long> {
 
     List<ExternalEvent> findByStatusOrderById(ExternalEventStatus status, Pageable batchSize);
+
+    @Modifying(flushAutomatically = true)
+    @Query("delete from ExternalEvent e where e.status = :status and e.businessDate <= :dateForPurgeCriteria")
+    void deleteOlderEventsWithSentStatus(ExternalEventStatus status, LocalDate dateForPurgeCriteria);
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java

@@ -57,7 +57,8 @@ public enum JobName {
     INCREASE_COB_DATE_BY_1_DAY("Increase COB Date by 1 day"), //
     LOAN_COB("Loan COB"), //
     LOAN_DELINQUENCY_CLASSIFICATION("Loan Delinquency Classification"), //
-    SEND_ASYNCHRONOUS_EVENTS("Send Asynchronous Events");
+    SEND_ASYNCHRONOUS_EVENTS("Send Asynchronous Events"), //
+    PURGE_EXTERNAL_EVENTS("Purge External Events");
 
     private final String name;
 

fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml

@@ -72,4 +72,5 @@
     <include file="parts/0050_add_reverse_flag_disbursement_details.xml" relativeToChangelogFile="true"/>
     <include file="parts/0051_external_event_table_category_info.xml" relativeToChangelogFile="true"/>
     <include file="parts/0052_loan_transaction_chargeback.xml" relativeToChangelogFile="true"/>
+    <include file="parts/0053_add_external_events_purge_job.xml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant/parts/0053_add_external_events_purge_job.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.3.xsd">
+    <changeSet author="fineract" id="1" context="postgresql">
+        <sql>
+            SELECT SETVAL('c_configuration_id_seq', COALESCE(MAX(id), 0)+1, false ) FROM c_configuration;
+        </sql>
+    </changeSet>
+    <changeSet author="fineract" id="2">
+        <insert tableName="c_configuration">
+            <column name="name" value="purge-external-events-older-than-days"/>
+            <column name="value" valueNumeric="30"/>
+            <column name="date_value"/>
+            <column name="string_value"/>
+            <column name="enabled" valueBoolean="false"/>
+            <column name="is_trap_door" valueBoolean="false"/>
+            <column name="description" value="Number of days criteria to purge old external events sent to message channel"/>
+        </insert>
+        <insert tableName="job">
+            <column name="name" value="Purge External Events"/>
+            <column name="display_name" value="Purge External Events"/>
+            <column name="cron_expression" value="0 1 0 1/1 * ? *"/>
+            <column name="create_time" valueDate="${current_datetime}"/>
+            <column name="task_priority" valueNumeric="5"/>
+            <column name="group_name"/>
+            <column name="previous_run_start_time"/>
+            <column name="job_key" value="Purge External Events _ DEFAULT"/>
+            <column name="initializing_errorlog"/>
+            <column name="is_active" valueBoolean="false"/>
+            <column name="currently_running" valueBoolean="false"/>
+            <column name="updates_allowed" valueBoolean="true"/>
+            <column name="scheduler_group" valueNumeric="0"/>
+            <column name="is_misfired" valueBoolean="false"/>
+            <column name="node_id" valueNumeric="1"/>
+            <column name="is_mismatched_job" valueBoolean="true"/>
+        </insert>
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/PurgeExternalEventsTaskletTest.java

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.jobs;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
+import org.apache.fineract.infrastructure.core.service.DateUtils;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.event.external.repository.ExternalEventRepository;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.batch.core.StepContribution;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.repeat.RepeatStatus;
+
+@ExtendWith(MockitoExtension.class)
+public class PurgeExternalEventsTaskletTest {
+
+    @Mock
+    private ExternalEventRepository repository;
+    @Mock
+    private ConfigurationDomainService configurationDomainService;
+    @Mock
+    private StepContribution stepContribution;
+    @Mock
+    private ChunkContext chunkContext;
+    private RepeatStatus resultStatus;
+    private PurgeExternalEventsTasklet underTest;
+
+    @BeforeEach
+    public void setUp() {
+        ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null));
+        ThreadLocalContextUtil
+                .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault()))));
+        underTest = new PurgeExternalEventsTasklet(repository, configurationDomainService);
+    }
+
+    @Test
+    public void givenEventsForPurgeWhenTaskExecutionThenEventsPurgeForDaysCriteria() {
+        // given
+        ArgumentCaptor<LocalDate> dateCriteriaCaptor = ArgumentCaptor.forClass(LocalDate.class);
+        when(configurationDomainService.retrieveExternalEventsPurgeDaysCriteria()).thenReturn(2L);
+        // when
+        resultStatus = underTest.execute(stepContribution, chunkContext);
+        // then
+        verify(repository, times(1)).deleteOlderEventsWithSentStatus(Mockito.any(), Mockito.any());
+        verify(repository).deleteOlderEventsWithSentStatus(Mockito.any(), dateCriteriaCaptor.capture());
+        LocalDate expectedDateForPurgeCriteriaTest = DateUtils.getBusinessLocalDate().minusDays(2);
+        LocalDate actualDateForPurgeCriteria = dateCriteriaCaptor.getValue();
+        assertEquals(expectedDateForPurgeCriteriaTest, actualDateForPurgeCriteria);
+        assertEquals(RepeatStatus.FINISHED, resultStatus);
+    }
+
+    @Test
+    public void givenEventsForPurgeWhenExceptionOccursThenJobExecutionFinishesSuccessfully() {
+        // given
+        when(configurationDomainService.retrieveExternalEventsPurgeDaysCriteria()).thenReturn(2L);
+        doThrow(new RuntimeException("Test Exception")).when(repository).deleteOlderEventsWithSentStatus(Mockito.any(), Mockito.any());
+        // when
+        resultStatus = underTest.execute(stepContribution, chunkContext);
+        // then
+        assertEquals(RepeatStatus.FINISHED, resultStatus);
+    }
+
+}

integration-tests/src/test/java/org/apache/fineract/integrationtests/ClientLoanIntegrationTest.java

@@ -4783,7 +4783,7 @@ public class ClientLoanIntegrationTest {
                 .withInterestCalculationPeriodTypeAsDays() //
                 .withExpectedDisbursementDate(disbursementDate) //
                 .withSubmittedOnDate(disbursementDate) //
-                .withwithRepaymentStrategy(repaymentStrategy).withRepaymentFrequencyTypeAsMonths()//
+                .withRepaymentStrategy(repaymentStrategy).withRepaymentFrequencyTypeAsMonths()//
                 .withFirstRepaymentDate(firstRepaymentDate).withCollaterals(collaterals)
                 .build(clientID.toString(), loanProductID.toString(), null);
         return this.loanTransactionHelper.getLoanId(loanApplicationJSON);

integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java

@@ -99,8 +99,8 @@ public class GlobalConfigurationHelper {
         ArrayList<HashMap> actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec);
 
         // There are currently 37 global configurations.
-        Assertions.assertEquals(41, expectedGlobalConfigurations.size());
+        Assertions.assertEquals(42, expectedGlobalConfigurations.size());
-        Assertions.assertEquals(41, actualGlobalConfigurations.size());
+        Assertions.assertEquals(42, actualGlobalConfigurations.size());
 
         for (int i = 0; i < expectedGlobalConfigurations.size(); i++) {
 
@@ -464,6 +464,14 @@ public class GlobalConfigurationHelper {
         isReversalTransactionAllowed.put("trapDoor", false);
         defaults.add(isReversalTransactionAllowed);
 
+        HashMap<String, Object> purgeExternalEventsOlderThanDaysDefault = new HashMap<>();
+        purgeExternalEventsOlderThanDaysDefault.put("id", 47);
+        purgeExternalEventsOlderThanDaysDefault.put("name", "purge-external-events-older-than-days");
+        purgeExternalEventsOlderThanDaysDefault.put("value", 30);
+        purgeExternalEventsOlderThanDaysDefault.put("enabled", false);
+        purgeExternalEventsOlderThanDaysDefault.put("trapDoor", false);
+        defaults.add(purgeExternalEventsOlderThanDaysDefault);
+
         return defaults;
     }