From 8c7e417c5de080f610c497bf885e8b09e55a2689 Mon Sep 17 00:00:00 2001 From: Janos Haber Date: Mon, 28 Nov 2022 16:37:41 +0100 Subject: [PATCH] FINERACT-1744 - [x] Purge Processed Commands Jobs - [x] Refactor CommandProcessingResultType to CommandProcessingStatus --- .../domain/CommandSourceRepository.java | 7 ++ .../jobs/PurgeProcessedCommandsConfig.java | 52 ++++++++++ .../jobs/PurgeProcessedCommandsTasklet.java | 54 +++++++++++ .../domain/ConfigurationDomainService.java | 2 + .../domain/ConfigurationDomainServiceJpa.java | 9 ++ .../infrastructure/jobs/service/JobName.java | 3 +- .../infrastructure/jobs/service/StepName.java | 23 +++++ .../db/changelog/tenant/changelog-tenant.xml | 1 + .../0075_add_processed_commands_purge_job.xml | 59 +++++++++++ .../PurgeProcessedCommandsTaskletTest.java | 97 +++++++++++++++++++ .../common/GlobalConfigurationHelper.java | 13 ++- 11 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsConfig.java create mode 100644 fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTasklet.java create mode 100644 fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/StepName.java create mode 100644 fineract-provider/src/main/resources/db/changelog/tenant/parts/0075_add_processed_commands_purge_job.xml create mode 100644 fineract-provider/src/test/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTaskletTest.java diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/domain/CommandSourceRepository.java b/fineract-provider/src/main/java/org/apache/fineract/commands/domain/CommandSourceRepository.java index 1977c6ede..13b6affbb 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/commands/domain/CommandSourceRepository.java +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/domain/CommandSourceRepository.java @@ -18,11 +18,18 @@ */ package org.apache.fineract.commands.domain; +import java.time.LocalDate; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; public interface CommandSourceRepository extends JpaRepository, JpaSpecificationExecutor { CommandSource findByActionNameAndEntityNameAndIdempotencyKey(String actionName, String entityName, String idempotencyKey); + @Modifying(flushAutomatically = true) + @Query("delete from CommandSource c where c.status = :status and c.madeOnDate is not null and c.madeOnDate <= :dateForPurgeCriteria") + void deleteOlderEventsWithStatus(CommandProcessingResultType status, LocalDate dateForPurgeCriteria); + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsConfig.java b/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsConfig.java new file mode 100644 index 000000000..90e389ffb --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsConfig.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.commands.jobs; + +import org.apache.fineract.infrastructure.jobs.service.JobName; +import org.apache.fineract.infrastructure.jobs.service.StepName; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PurgeProcessedCommandsConfig { + + @Autowired + private JobBuilderFactory jobs; + @Autowired + private StepBuilderFactory steps; + @Autowired + private PurgeProcessedCommandsTasklet tasklet; + + @Bean + protected Step purgeProcessedCommandsStep() { + return steps.get(StepName.PURGE_PROCESSED_COMMANDS_STEP.name()).tasklet(tasklet).build(); + } + + @Bean + public Job purgeProcessedCommandsJob() { + return jobs.get(JobName.PURGE_PROCESSED_COMMANDS.name()).start(purgeProcessedCommandsStep()).incrementer(new RunIdIncrementer()) + .build(); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTasklet.java new file mode 100644 index 000000000..92cfac681 --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTasklet.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.commands.jobs; + +import java.time.LocalDate; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.fineract.commands.domain.CommandProcessingResultType; +import org.apache.fineract.commands.domain.CommandSourceRepository; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.service.DateUtils; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +@Slf4j +@AllArgsConstructor +@Component +public class PurgeProcessedCommandsTasklet implements Tasklet { + + private final CommandSourceRepository repository; + private final ConfigurationDomainService configurationDomainService; + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { + try { + Long numberOfDaysForPurgeCriteria = configurationDomainService.retrieveProcessedCommandsPurgeDaysCriteria(); + LocalDate dateForPurgeCriteria = DateUtils.getBusinessLocalDate().minusDays(numberOfDaysForPurgeCriteria); + repository.deleteOlderEventsWithStatus(CommandProcessingResultType.PROCESSED, dateForPurgeCriteria); + } catch (Exception e) { + log.error("Error occurred while purging processed commands: ", e); + } + return RepeatStatus.FINISHED; + } + +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java index b025e0813..e3ac140e4 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java @@ -121,6 +121,8 @@ public interface ConfigurationDomainService { Long retrieveExternalEventsPurgeDaysCriteria(); + Long retrieveProcessedCommandsPurgeDaysCriteria(); + Long retrieveRepaymentDueDays(); Long retrieveRepaymentOverdueDays(); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java index be0e45c88..36fe9e933 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java @@ -41,6 +41,8 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService public static final String ENABLE_BUSINESS_DATE = "enable_business_date"; public static final String ENABLE_AUTOMATIC_COB_DATE_ADJUSTMENT = "enable_automatic_cob_date_adjustment"; public static final String EXTERNAL_EVENTS_PURGE_DAYS = "purge-external-events-older-than-days"; + + public static final String PROCESSED_COMMANDS_PURGE_DAYS = "purge-processed-commands-older-than-days"; private static final String DAYS_BEFORE_REPAYMENT_IS_DUE = "days-before-repayment-is-due"; private static final String DAYS_AFTER_REPAYMENT_IS_OVERDUE = "days-after-repayment-is-overdue"; private static final String ENABLE_EXTERNAL_ID_AUTO_GENERATION = "enable-auto-generated-external-id"; @@ -463,6 +465,13 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService } + @Override + public Long retrieveProcessedCommandsPurgeDaysCriteria() { + final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(PROCESSED_COMMANDS_PURGE_DAYS); + return property.getValue(); + + } + @Override public Long retrieveRepaymentDueDays() { final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(DAYS_BEFORE_REPAYMENT_IS_DUE); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java index 1858733e4..2f9e5fb44 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobName.java @@ -55,7 +55,8 @@ public enum JobName { LOAN_COB("Loan COB"), // LOAN_DELINQUENCY_CLASSIFICATION("Loan Delinquency Classification"), // SEND_ASYNCHRONOUS_EVENTS("Send Asynchronous Events"), // - PURGE_EXTERNAL_EVENTS("Purge External Events"); + PURGE_EXTERNAL_EVENTS("Purge External Events"), // + PURGE_PROCESSED_COMMANDS("Purge Processed Commands"); private final String name; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/StepName.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/StepName.java new file mode 100644 index 000000000..2dbac082c --- /dev/null +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/StepName.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.infrastructure.jobs.service; + +public enum StepName { + PURGE_PROCESSED_COMMANDS_STEP; +} diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml index f9185c1d3..ea5dc17f8 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml @@ -94,4 +94,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0075_add_processed_commands_purge_job.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0075_add_processed_commands_purge_job.xml new file mode 100644 index 000000000..35ae9d3ab --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0075_add_processed_commands_purge_job.xml @@ -0,0 +1,59 @@ + + + + + + SELECT SETVAL('c_configuration_id_seq', COALESCE(MAX(id), 0)+1, false ) FROM c_configuration; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTaskletTest.java b/fineract-provider/src/test/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTaskletTest.java new file mode 100644 index 000000000..050438443 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/commands/jobs/PurgeProcessedCommandsTaskletTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.commands.jobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; +import org.apache.fineract.commands.domain.CommandSourceRepository; +import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; +import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; +import org.apache.fineract.infrastructure.core.service.DateUtils; +import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.repeat.RepeatStatus; + +@ExtendWith(MockitoExtension.class) +public class PurgeProcessedCommandsTaskletTest { + + @Mock + private CommandSourceRepository repository; + @Mock + private ConfigurationDomainService configurationDomainService; + @Mock + private StepContribution stepContribution; + @Mock + private ChunkContext chunkContext; + private RepeatStatus resultStatus; + private PurgeProcessedCommandsTasklet underTest; + + @BeforeEach + public void setUp() { + ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null)); + ThreadLocalContextUtil + .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault())))); + underTest = new PurgeProcessedCommandsTasklet(repository, configurationDomainService); + } + + @Test + public void givenEventsForPurgeWhenTaskExecutionThenEventsPurgeForDaysCriteria() { + // given + ArgumentCaptor dateCriteriaCaptor = ArgumentCaptor.forClass(LocalDate.class); + when(configurationDomainService.retrieveProcessedCommandsPurgeDaysCriteria()).thenReturn(2L); + // when + resultStatus = underTest.execute(stepContribution, chunkContext); + // then + verify(repository, times(1)).deleteOlderEventsWithStatus(Mockito.any(), Mockito.any()); + verify(repository).deleteOlderEventsWithStatus(Mockito.any(), dateCriteriaCaptor.capture()); + LocalDate expectedDateForPurgeCriteriaTest = DateUtils.getBusinessLocalDate().minusDays(2); + LocalDate actualDateForPurgeCriteria = dateCriteriaCaptor.getValue(); + assertEquals(expectedDateForPurgeCriteriaTest, actualDateForPurgeCriteria); + assertEquals(RepeatStatus.FINISHED, resultStatus); + } + + @Test + public void givenEventsForPurgeWhenExceptionOccursThenJobExecutionFinishesSuccessfully() { + // given + when(configurationDomainService.retrieveProcessedCommandsPurgeDaysCriteria()).thenReturn(2L); + doThrow(new RuntimeException("Test Exception")).when(repository).deleteOlderEventsWithStatus(Mockito.any(), Mockito.any()); + // when + resultStatus = underTest.execute(stepContribution, chunkContext); + // then + assertEquals(RepeatStatus.FINISHED, resultStatus); + } + +} diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java index 4b3866ea7..8cb12c05b 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java @@ -98,9 +98,9 @@ public class GlobalConfigurationHelper { ArrayList expectedGlobalConfigurations = getAllDefaultGlobalConfigurations(); ArrayList actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec); - // There are currently 45 global configurations. - Assertions.assertEquals(45, expectedGlobalConfigurations.size()); - Assertions.assertEquals(45, actualGlobalConfigurations.size()); + // There are currently 46 global configurations. + Assertions.assertEquals(46, expectedGlobalConfigurations.size()); + Assertions.assertEquals(46, actualGlobalConfigurations.size()); for (int i = 0; i < expectedGlobalConfigurations.size(); i++) { @@ -496,6 +496,13 @@ public class GlobalConfigurationHelper { isAutomaticExternalIdGenerationEnabled.put("trapDoor", false); defaults.add(isAutomaticExternalIdGenerationEnabled); + HashMap purgeProcessCommandDaysDefault = new HashMap<>(); + purgeProcessCommandDaysDefault.put("id", 51); + purgeProcessCommandDaysDefault.put("name", "purge-processed-commands-older-than-days"); + purgeProcessCommandDaysDefault.put("value", 0); + purgeProcessCommandDaysDefault.put("enabled", false); + purgeProcessCommandDaysDefault.put("trapDoor", false); + defaults.add(purgeProcessCommandDaysDefault); return defaults; }