From e4106bd11dcb6fe96090f7fed2390a6a01639cd2 Mon Sep 17 00:00:00 2001 From: Ruchi Dhamankar Date: Tue, 17 Jan 2023 16:52:54 +0530 Subject: [PATCH] FINERACT-1694-External-event-producer-batch-size-configuration --- .../domain/ConfigurationDomainService.java | 1 + .../domain/ConfigurationDomainServiceJpa.java | 7 ++++ .../core/config/FineractProperties.java | 1 - .../jobs/SendAsynchronousEventsTasklet.java | 5 ++- .../src/main/resources/application.properties | 1 - .../db/changelog/tenant/changelog-tenant.xml | 1 + ...onfiguration_event_producer_batch_size.xml | 41 +++++++++++++++++++ .../SendAsynchronousEventsTaskletTest.java | 26 ++++++++++-- .../common/GlobalConfigurationHelper.java | 14 +++++-- 9 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 fineract-provider/src/main/resources/db/changelog/tenant/parts/0081_add_configuration_event_producer_batch_size.xml diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java index fa0b7939d..168c1553e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java @@ -133,4 +133,5 @@ public interface ConfigurationDomainService { boolean isCOBBulkEventEnabled(); + Long retrieveExternalEventBatchSize(); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java index 1e3ce5288..c34ab9f9d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java @@ -48,6 +48,7 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService private static final String ENABLE_EXTERNAL_ID_AUTO_GENERATION = "enable-auto-generated-external-id"; private static final String ENABLE_ADDRESS = "Enable-Address"; private static final String ENABLE_COB_BULK_EVENT = "enable-cob-bulk-event"; + private static final String EXTERNAL_EVENT_BATCH_SIZE = "external-event-batch-size"; private final PermissionRepository permissionRepository; private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository; @@ -505,4 +506,10 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService return property.isEnabled(); } + @Override + public Long retrieveExternalEventBatchSize() { + final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(EXTERNAL_EVENT_BATCH_SIZE); + return property.getValue(); + } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java index 36c016da2..b28722c43 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java @@ -152,7 +152,6 @@ public class FineractProperties { @Setter public static class FineractExternalEventsProducerProperties { - private int readBatchSize; private FineractExternalEventsProducerJmsProperties jms; } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java index 4209f0eb7..289f20591 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java @@ -23,6 +23,7 @@ import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.fineract.avro.MessageV1; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.event.external.producer.ExternalEventProducer; @@ -49,6 +50,7 @@ public class SendAsynchronousEventsTasklet implements Tasklet { private final ExternalEventProducer eventProducer; private final MessageFactory messageFactory; private final ByteBufferConverter byteBufferConverter; + private final ConfigurationDomainService configurationDomainService; @Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) { @@ -85,7 +87,8 @@ public class SendAsynchronousEventsTasklet implements Tasklet { } private int getBatchSize() { - return fineractProperties.getEvents().getExternal().getProducer().getReadBatchSize(); + Long externalEventBatchSize = configurationDomainService.retrieveExternalEventBatchSize(); + return externalEventBatchSize.intValue(); } } diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties index 7ecc24686..385400e3c 100644 --- a/fineract-provider/src/main/resources/application.properties +++ b/fineract-provider/src/main/resources/application.properties @@ -56,7 +56,6 @@ fineract.remote-job-message-handler.jms.broker-username=${FINERACT_REMOTE_JOB_ME fineract.remote-job-message-handler.jms.broker-password=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_JMS_BROKER_PASSWORD:} fineract.events.external.enabled=${FINERACT_EXTERNAL_EVENTS_ENABLED:false} -fineract.events.external.producer.read-batch-size=${FINERACT_EXTERNAL_EVENTS_PRODUCER_READ_BATCH_SIZE:1000} fineract.events.external.producer.jms.enabled=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_ENABLED:false} fineract.events.external.producer.jms.event-queue-name=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_QUEUE_NAME:} fineract.events.external.producer.jms.event-topic-name=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_TOPIC_NAME:} diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml index 685cd5e0a..fecc2da04 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml @@ -100,4 +100,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0081_add_configuration_event_producer_batch_size.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0081_add_configuration_event_producer_batch_size.xml new file mode 100644 index 000000000..52e5cb5b2 --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0081_add_configuration_event_producer_batch_size.xml @@ -0,0 +1,41 @@ + + + + + + SELECT SETVAL('c_configuration_id_seq', COALESCE(MAX(id), 0)+1, false ) FROM c_configuration; + + + + + + + + + + + + + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTaskletTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTaskletTest.java index 42eb13510..0fdfca0fd 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTaskletTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTaskletTest.java @@ -29,12 +29,14 @@ import static org.mockito.Mockito.when; import java.nio.ByteBuffer; import java.time.LocalDate; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.fineract.avro.MessageV1; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; +import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.core.config.FineractProperties; import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; @@ -55,6 +57,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.data.domain.Pageable; @ExtendWith(MockitoExtension.class) class SendAsynchronousEventsTaskletTest { @@ -73,6 +76,8 @@ class SendAsynchronousEventsTaskletTest { private ChunkContext chunkContext; @Mock private ByteBufferConverter byteBufferConverter; + @Mock + private ConfigurationDomainService configurationDomainService; private SendAsynchronousEventsTasklet underTest; private RepeatStatus resultStatus; @@ -81,22 +86,23 @@ class SendAsynchronousEventsTaskletTest { ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null)); ThreadLocalContextUtil .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault())))); - configureExternalEventsProducerReadBatchSizeProperty(1000); - underTest = new SendAsynchronousEventsTasklet(fineractProperties, repository, eventProducer, messageFactory, byteBufferConverter); + configureExternalEventsProducerReadBatchSizeProperty(); + underTest = new SendAsynchronousEventsTasklet(fineractProperties, repository, eventProducer, messageFactory, byteBufferConverter, + configurationDomainService); } - private void configureExternalEventsProducerReadBatchSizeProperty(int readBatchSize) { + private void configureExternalEventsProducerReadBatchSizeProperty() { FineractProperties.FineractEventsProperties eventsProperties = new FineractProperties.FineractEventsProperties(); FineractProperties.FineractExternalEventsProperties externalProperties = new FineractProperties.FineractExternalEventsProperties(); FineractProperties.FineractExternalEventsProducerProperties externalEventsProducerProperties = new FineractProperties.FineractExternalEventsProducerProperties(); FineractProperties.FineractExternalEventsProducerJmsProperties externalEventsProducerJMSProperties = new FineractProperties.FineractExternalEventsProducerJmsProperties(); externalEventsProducerJMSProperties.setEnabled(true); externalProperties.setEnabled(true); - externalEventsProducerProperties.setReadBatchSize(readBatchSize); externalEventsProducerProperties.setJms(externalEventsProducerJMSProperties); externalProperties.setProducer(externalEventsProducerProperties); eventsProperties.setExternal(externalProperties); when(fineractProperties.getEvents()).thenReturn(eventsProperties); + when(configurationDomainService.retrieveExternalEventBatchSize()).thenReturn(10L); } @Test @@ -159,4 +165,16 @@ class SendAsynchronousEventsTaskletTest { assertThat(externalEvent.getStatus()).isEqualTo(ExternalEventStatus.SENT); assertEquals(RepeatStatus.FINISHED, resultStatus); } + + @Test + public void givenEventBatchSizeIsConfiguredAs10WhenTaskExecutionThenEventReadPageSizeIsCorrect() { + ArgumentCaptor externalEventPageSizeArgumentCaptor = ArgumentCaptor.forClass(Pageable.class); + List events = new ArrayList<>(); + when(repository.findByStatusOrderById(Mockito.any(), Mockito.any())).thenReturn(events); + // when + resultStatus = this.underTest.execute(stepContribution, chunkContext); + // then + verify(repository).findByStatusOrderById(Mockito.any(), externalEventPageSizeArgumentCaptor.capture()); + assertThat(externalEventPageSizeArgumentCaptor.getValue().getPageSize()).isEqualTo(10); + } } diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java index 33cf0277e..a92e8246b 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java @@ -119,9 +119,9 @@ public class GlobalConfigurationHelper { ArrayList expectedGlobalConfigurations = getAllDefaultGlobalConfigurations(); ArrayList actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec); - // There are currently 47 global configurations. - Assertions.assertEquals(47, expectedGlobalConfigurations.size()); - Assertions.assertEquals(47, actualGlobalConfigurations.size()); + // There are currently 48 global configurations. + Assertions.assertEquals(48, expectedGlobalConfigurations.size()); + Assertions.assertEquals(48, actualGlobalConfigurations.size()); for (int i = 0; i < expectedGlobalConfigurations.size(); i++) { @@ -532,6 +532,14 @@ public class GlobalConfigurationHelper { isCOBBulkEventEnabled.put("enabled", false); isCOBBulkEventEnabled.put("trapDoor", false); defaults.add(isCOBBulkEventEnabled); + + HashMap externalEventBatchSize = new HashMap<>(); + externalEventBatchSize.put("id", 53); + externalEventBatchSize.put("name", "external-event-batch-size"); + externalEventBatchSize.put("value", 1000); + externalEventBatchSize.put("enabled", false); + externalEventBatchSize.put("trapDoor", false); + defaults.add(externalEventBatchSize); return defaults; }