FINERACT-2081: External event sending make mark as sent async

2024-11-20 20:17:01 +01:00
parent bb18aadb74
commit 425085f4eb
7 changed files with 141 additions and 9 deletions
@@ -288,6 +288,9 @@ public class FineractProperties {
        private boolean enabled;
        private FineractExternalEventsProducerProperties producer;
        private int partitionSize;
+        private int threadPoolCorePoolSize;
+        private int threadPoolMaxPoolSize;
+        private int threadPoolQueueCapacity;
    }

    @Getter
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.config;
+
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+@Configuration
+@RequiredArgsConstructor
+public class EventTaskExecutorConfig {
+
+    private final FineractProperties fineractProperties;
+
+    @Bean(TaskExecutorConstant.EVENT_MARKS_AS_SENT_EXECUTOR_BEAN_NAME)
+    public ThreadPoolTaskExecutor sendAsynchronousEventsThreadPool() {
+        ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
+        threadPoolTaskExecutor.setCorePoolSize(fineractProperties.getEvents().getExternal().getThreadPoolCorePoolSize());
+        threadPoolTaskExecutor.setMaxPoolSize(fineractProperties.getEvents().getExternal().getThreadPoolMaxPoolSize());
+        threadPoolTaskExecutor.setQueueCapacity(fineractProperties.getEvents().getExternal().getThreadPoolQueueCapacity());
+        threadPoolTaskExecutor.setThreadNamePrefix("external-events-");
+        threadPoolTaskExecutor.initialize();
+
+        return threadPoolTaskExecutor;
+    }
+}
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.config;
+
+public final class TaskExecutorConstant {
+
+    private TaskExecutorConstant() {}
+
+    public static final String EVENT_MARKS_AS_SENT_EXECUTOR_BEAN_NAME = "eventMarksAsSentExecutor";
+}
@@ -29,12 +29,17 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.avro.MessageV1;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.domain.FineractContext;
 import org.apache.fineract.infrastructure.core.service.DateUtils;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.event.external.config.TaskExecutorConstant;
 import org.apache.fineract.infrastructure.event.external.producer.ExternalEventProducer;
 import org.apache.fineract.infrastructure.event.external.repository.ExternalEventRepository;
 import org.apache.fineract.infrastructure.event.external.repository.domain.ExternalEventStatus;
@@ -45,9 +50,12 @@ import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.core.step.tasklet.Tasklet;
 import org.springframework.batch.repeat.RepeatStatus;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.support.TransactionTemplate;

@Slf4j
@RequiredArgsConstructor
@@ -60,6 +68,9 @@ public class SendAsynchronousEventsTasklet implements Tasklet {
    private final MessageFactory messageFactory;
    private final ByteBufferConverter byteBufferConverter;
    private final ConfigurationDomainService configurationDomainService;
+    private final TransactionTemplate transactionTemplate;
+    @Qualifier(TaskExecutorConstant.EVENT_MARKS_AS_SENT_EXECUTOR_BEAN_NAME)
+    private final ThreadPoolTaskExecutor threadPoolTaskExecutor;

    @Override
    public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) {
@@ -104,14 +115,31 @@ public class SendAsynchronousEventsTasklet implements Tasklet {
        // Partitioning dataset to avoid exception: PreparedStatement can have at most 65,535 parameters
        final int partitionSize = fineractProperties.getEvents().getExternal().getPartitionSize();
        List<List<Long>> partitions = Lists.partition(eventIds, partitionSize);
+        List<Future<?>> tasks = new ArrayList<>();
+        final FineractContext context = ThreadLocalContextUtil.getContext();
        partitions //
                .forEach(partitionedEventIds -> {
-                    measure(() -> {
-                        repository.markEventsSent(partitionedEventIds, sentAt);
-                    }, timeTaken -> {
-                        log.debug("Took {}ms to update {} events", timeTaken.toMillis(), partitionedEventIds.size());
-                    });
+                    tasks.add(threadPoolTaskExecutor.submit(() -> {
+                        ThreadLocalContextUtil.init(context);
+                        transactionTemplate.execute((status) -> {
+                            measure(() -> {
+                                repository.markEventsSent(partitionedEventIds, sentAt);
+                            }, timeTaken -> {
+                                log.debug("Took {}ms to update {} events", timeTaken.toMillis(), partitionedEventIds.size());
+                            });
+                            return null;
+                        });
+                    }));
                });
+        for (Future<?> task : tasks) {
+            try {
+                task.get();
+            } catch (InterruptedException e) {
+                log.error("Interrupted while marking events as sent", e);
+            } catch (ExecutionException e) {
+                log.error("Exception while marking events as sent", e);
+            }
+        }
    }

    private Map<Long, List<byte[]>> generatePartitions(List<ExternalEventView> queuedEvents) {