FINERACT-1694-External-event-producer-batch-size-configuration

fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java

@@ -133,4 +133,5 @@ public interface ConfigurationDomainService {
 
     boolean isCOBBulkEventEnabled();
 
+    Long retrieveExternalEventBatchSize();
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java

@@ -48,6 +48,7 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
     private static final String ENABLE_EXTERNAL_ID_AUTO_GENERATION = "enable-auto-generated-external-id";
     private static final String ENABLE_ADDRESS = "Enable-Address";
     private static final String ENABLE_COB_BULK_EVENT = "enable-cob-bulk-event";
+    private static final String EXTERNAL_EVENT_BATCH_SIZE = "external-event-batch-size";
 
     private final PermissionRepository permissionRepository;
     private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository;
@@ -505,4 +506,10 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
         return property.isEnabled();
     }
 
+    @Override
+    public Long retrieveExternalEventBatchSize() {
+        final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(EXTERNAL_EVENT_BATCH_SIZE);
+        return property.getValue();
+    }
+
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java

@@ -152,7 +152,6 @@ public class FineractProperties {
     @Setter
     public static class FineractExternalEventsProducerProperties {
 
-        private int readBatchSize;
         private FineractExternalEventsProducerJmsProperties jms;
     }
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java

@@ -23,6 +23,7 @@ import java.util.List;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.avro.MessageV1;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.service.DateUtils;
 import org.apache.fineract.infrastructure.event.external.producer.ExternalEventProducer;
@@ -49,6 +50,7 @@ public class SendAsynchronousEventsTasklet implements Tasklet {
     private final ExternalEventProducer eventProducer;
     private final MessageFactory messageFactory;
     private final ByteBufferConverter byteBufferConverter;
+    private final ConfigurationDomainService configurationDomainService;
 
     @Override
     public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) {
@@ -85,7 +87,8 @@ public class SendAsynchronousEventsTasklet implements Tasklet {
     }
 
     private int getBatchSize() {
-        return fineractProperties.getEvents().getExternal().getProducer().getReadBatchSize();
+        Long externalEventBatchSize = configurationDomainService.retrieveExternalEventBatchSize();
+        return externalEventBatchSize.intValue();
     }
 
 }

fineract-provider/src/main/resources/application.properties

@@ -56,7 +56,6 @@ fineract.remote-job-message-handler.jms.broker-username=${FINERACT_REMOTE_JOB_ME
 fineract.remote-job-message-handler.jms.broker-password=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_JMS_BROKER_PASSWORD:}
 
 fineract.events.external.enabled=${FINERACT_EXTERNAL_EVENTS_ENABLED:false}
-fineract.events.external.producer.read-batch-size=${FINERACT_EXTERNAL_EVENTS_PRODUCER_READ_BATCH_SIZE:1000}
 fineract.events.external.producer.jms.enabled=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_ENABLED:false}
 fineract.events.external.producer.jms.event-queue-name=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_QUEUE_NAME:}
 fineract.events.external.producer.jms.event-topic-name=${FINERACT_EXTERNAL_EVENTS_PRODUCER_JMS_TOPIC_NAME:}

fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml

@@ -100,4 +100,5 @@
     <include file="parts/0078_add_configuration_cob_bulk_event.xml" relativeToChangelogFile="true" />
     <include file="parts/0079_add_charge_off_details_to_loan.xml" relativeToChangelogFile="true" />
     <include file="parts/0080_add_external_event_configuration_for_stayed_locked_loans_business_event.xml" relativeToChangelogFile="true" />
+    <include file="parts/0081_add_configuration_event_producer_batch_size.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant/parts/0081_add_configuration_event_producer_batch_size.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.3.xsd">
+    <changeSet author="fineract" id="1" context="postgresql">
+        <sql>
+            SELECT SETVAL('c_configuration_id_seq', COALESCE(MAX(id), 0)+1, false ) FROM c_configuration;
+        </sql>
+    </changeSet>
+    <changeSet author="fineract" id="2">
+        <insert tableName="c_configuration">
+            <column name="name" value="external-event-batch-size"/>
+            <column name="value" valueNumeric="1000"/>
+            <column name="date_value"/>
+            <column name="string_value"/>
+            <column name="enabled" valueBoolean="false"/>
+            <column name="is_trap_door" valueBoolean="false"/>
+            <column name="description" value="External event producer batch size"/>
+        </insert>
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/test/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTaskletTest.java

@@ -29,12 +29,14 @@ import static org.mockito.Mockito.when;
 import java.nio.ByteBuffer;
 import java.time.LocalDate;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.fineract.avro.MessageV1;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
@@ -55,6 +57,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.data.domain.Pageable;
 
 @ExtendWith(MockitoExtension.class)
 class SendAsynchronousEventsTaskletTest {
@@ -73,6 +76,8 @@ class SendAsynchronousEventsTaskletTest {
     private ChunkContext chunkContext;
     @Mock
     private ByteBufferConverter byteBufferConverter;
+    @Mock
+    private ConfigurationDomainService configurationDomainService;
     private SendAsynchronousEventsTasklet underTest;
     private RepeatStatus resultStatus;
 
@@ -81,22 +86,23 @@ class SendAsynchronousEventsTaskletTest {
         ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null));
         ThreadLocalContextUtil
                 .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault()))));
-        configureExternalEventsProducerReadBatchSizeProperty(1000);
+        configureExternalEventsProducerReadBatchSizeProperty();
-        underTest = new SendAsynchronousEventsTasklet(fineractProperties, repository, eventProducer, messageFactory, byteBufferConverter);
+        underTest = new SendAsynchronousEventsTasklet(fineractProperties, repository, eventProducer, messageFactory, byteBufferConverter,
+                configurationDomainService);
     }
 
-    private void configureExternalEventsProducerReadBatchSizeProperty(int readBatchSize) {
+    private void configureExternalEventsProducerReadBatchSizeProperty() {
         FineractProperties.FineractEventsProperties eventsProperties = new FineractProperties.FineractEventsProperties();
         FineractProperties.FineractExternalEventsProperties externalProperties = new FineractProperties.FineractExternalEventsProperties();
         FineractProperties.FineractExternalEventsProducerProperties externalEventsProducerProperties = new FineractProperties.FineractExternalEventsProducerProperties();
         FineractProperties.FineractExternalEventsProducerJmsProperties externalEventsProducerJMSProperties = new FineractProperties.FineractExternalEventsProducerJmsProperties();
         externalEventsProducerJMSProperties.setEnabled(true);
         externalProperties.setEnabled(true);
-        externalEventsProducerProperties.setReadBatchSize(readBatchSize);
         externalEventsProducerProperties.setJms(externalEventsProducerJMSProperties);
         externalProperties.setProducer(externalEventsProducerProperties);
         eventsProperties.setExternal(externalProperties);
         when(fineractProperties.getEvents()).thenReturn(eventsProperties);
+        when(configurationDomainService.retrieveExternalEventBatchSize()).thenReturn(10L);
     }
 
     @Test
@@ -159,4 +165,16 @@ class SendAsynchronousEventsTaskletTest {
         assertThat(externalEvent.getStatus()).isEqualTo(ExternalEventStatus.SENT);
         assertEquals(RepeatStatus.FINISHED, resultStatus);
     }
+
+    @Test
+    public void givenEventBatchSizeIsConfiguredAs10WhenTaskExecutionThenEventReadPageSizeIsCorrect() {
+        ArgumentCaptor<Pageable> externalEventPageSizeArgumentCaptor = ArgumentCaptor.forClass(Pageable.class);
+        List<ExternalEvent> events = new ArrayList<>();
+        when(repository.findByStatusOrderById(Mockito.any(), Mockito.any())).thenReturn(events);
+        // when
+        resultStatus = this.underTest.execute(stepContribution, chunkContext);
+        // then
+        verify(repository).findByStatusOrderById(Mockito.any(), externalEventPageSizeArgumentCaptor.capture());
+        assertThat(externalEventPageSizeArgumentCaptor.getValue().getPageSize()).isEqualTo(10);
+    }
 }

integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java

@@ -119,9 +119,9 @@ public class GlobalConfigurationHelper {
         ArrayList<HashMap> expectedGlobalConfigurations = getAllDefaultGlobalConfigurations();
         ArrayList<HashMap> actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec);
 
-        // There are currently 47 global configurations.
+        // There are currently 48 global configurations.
-        Assertions.assertEquals(47, expectedGlobalConfigurations.size());
+        Assertions.assertEquals(48, expectedGlobalConfigurations.size());
-        Assertions.assertEquals(47, actualGlobalConfigurations.size());
+        Assertions.assertEquals(48, actualGlobalConfigurations.size());
 
         for (int i = 0; i < expectedGlobalConfigurations.size(); i++) {
 
@@ -532,6 +532,14 @@ public class GlobalConfigurationHelper {
         isCOBBulkEventEnabled.put("enabled", false);
         isCOBBulkEventEnabled.put("trapDoor", false);
         defaults.add(isCOBBulkEventEnabled);
+
+        HashMap<String, Object> externalEventBatchSize = new HashMap<>();
+        externalEventBatchSize.put("id", 53);
+        externalEventBatchSize.put("name", "external-event-batch-size");
+        externalEventBatchSize.put("value", 1000);
+        externalEventBatchSize.put("enabled", false);
+        externalEventBatchSize.put("trapDoor", false);
+        defaults.add(externalEventBatchSize);
         return defaults;
     }