FINERACT-1724: Sampling enhancements

custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java

@@ -26,7 +26,8 @@ import org.apache.fineract.cob.domain.BatchBusinessStepRepository;
 import org.apache.fineract.cob.service.ReloaderService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingConfiguration;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.portfolio.loanaccount.domain.LoanAccountDomainService;
 import org.springframework.beans.factory.ListableBeanFactory;
@@ -50,15 +51,14 @@ public class TestDefaultConfiguration {
     @Bean
     public COBBusinessStepService cobBusinessStepService(BatchBusinessStepRepository batchBusinessStepRepository,
             ApplicationContext context, ListableBeanFactory beanFactory, BusinessEventNotifierService businessEventNotifierService,
-            ConfigurationDomainService configurationDomainService, ReloaderService reloaderService,
-            SamplingServiceFactory samplingServiceFactory) {
+            ConfigurationDomainService configurationDomainService, ReloaderService reloaderService) {
         return new COBBusinessStepServiceImpl(batchBusinessStepRepository, context, beanFactory, businessEventNotifierService,
-                configurationDomainService, reloaderService, samplingServiceFactory);
+                configurationDomainService, reloaderService);
     }
 
     @Bean
-    public SamplingServiceFactory samplingServiceFactory(FineractProperties fineractProperties) {
-        return new SamplingServiceFactory(fineractProperties);
+    public SamplingServiceFactory samplingServiceFactory(SamplingConfiguration samplingConfiguration) {
+        return new SamplingServiceFactory(samplingConfiguration);
     }
 
     @Bean

fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java

@@ -35,11 +35,8 @@ import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
 import org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingService;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.jetbrains.annotations.NotNull;
-import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.ListableBeanFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.stereotype.Service;
@@ -47,7 +44,7 @@ import org.springframework.stereotype.Service;
 @Service
 @Slf4j
 @RequiredArgsConstructor
-public class COBBusinessStepServiceImpl implements COBBusinessStepService, InitializingBean {
+public class COBBusinessStepServiceImpl implements COBBusinessStepService {
 
     private final BatchBusinessStepRepository batchBusinessStepRepository;
     private final ApplicationContext applicationContext;
@@ -57,15 +54,6 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService, Initi
 
     private final ReloaderService reloaderService;
 
-    private final SamplingServiceFactory samplingServiceFactory;
-
-    private SamplingService samplingService;
-
-    @Override
-    public void afterPropertiesSet() throws Exception {
-        this.samplingService = samplingServiceFactory.forClass(COBBusinessStepServiceImpl.class);
-    }
-
     @SuppressWarnings({ "unchecked" })
     @Override
     public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom> S run(TreeMap<Long, String> executionMap, S item) {
@@ -83,8 +71,8 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService, Initi
                 try {
                     ThreadLocalContextUtil.setActionContext(ActionContext.COB);
                     COBBusinessStep<S> businessStepBean = (COBBusinessStep<S>) applicationContext.getBean(businessStep);
-                    item = reloadAndSample(item);
-                    item = executeAndSample(item, businessStepBean);
+                    item = reloaderService.reload(item);
+                    item = businessStepBean.execute(item);
                 } catch (Exception e) {
                     throw new BusinessStepException("Error happened during business step execution", e);
                 } finally {
@@ -104,16 +92,6 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService, Initi
         return item;
     }
 
-    private <S extends AbstractPersistableCustom> S reloadAndSample(S item) {
-        String key = reloaderService.getClass().getSimpleName();
-        return samplingService.sample(key, () -> reloaderService.reload(item));
-    }
-
-    private <S extends AbstractPersistableCustom> S executeAndSample(S item, COBBusinessStep<S> businessStepBean) {
-        String key = businessStepBean.getClass().getSimpleName();
-        return samplingService.sample(key, () -> businessStepBean.execute(item));
-    }
-
     @NotNull
     @Override
     public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom> Set<BusinessStepNameAndOrder> getCOBBusinessSteps(

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java

@@ -19,15 +19,11 @@
 package org.apache.fineract.cob.loan;
 
 import org.apache.fineract.cob.COBBusinessStepService;
-import org.apache.fineract.cob.COBBusinessStepServiceImpl;
 import org.apache.fineract.cob.common.InitialisationTasklet;
 import org.apache.fineract.cob.common.ResetContextTasklet;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingDataPrinter;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingStepExecutionListener;
 import org.apache.fineract.infrastructure.jobs.service.JobName;
 import org.apache.fineract.infrastructure.springbatch.PropertyService;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
@@ -71,10 +67,6 @@ public class LoanCOBWorkerConfiguration {
     private AppUserRepositoryWrapper userRepository;
     @Autowired
     private TransactionTemplate transactionTemplate;
-    @Autowired
-    private SamplingServiceFactory samplingServiceFactory;
-    @Autowired
-    private SamplingDataPrinter samplingDataPrinter;
 
     @Autowired
     private FineractProperties fineractProperties;
@@ -104,8 +96,7 @@ public class LoanCOBWorkerConfiguration {
         return localStepBuilderFactory.get("Loan Business - Step:" + partitionName)
                 .<Loan, Loan>chunk(propertyService.getChunkSize(JobName.LOAN_COB.name())).reader(cobWorkerItemReader())
                 .processor(cobWorkerItemProcessor()).writer(cobWorkerItemWriter()).faultTolerant().skip(Exception.class)
-                .skipLimit(propertyService.getChunkSize(JobName.LOAN_COB.name()) + 1).listener(loanItemListener())
-                .listener(samplingStepExecutionListener()).build();
+                .skipLimit(propertyService.getChunkSize(JobName.LOAN_COB.name()) + 1).listener(loanItemListener()).build();
     }
 
     @Bean
@@ -159,11 +150,4 @@ public class LoanCOBWorkerConfiguration {
         repositoryItemWriter.setRepository(loanRepository);
         return repositoryItemWriter;
     }
-
-    @Bean
-    public SamplingStepExecutionListener samplingStepExecutionListener() {
-        SamplingStepExecutionListener listener = new SamplingStepExecutionListener(samplingServiceFactory, samplingDataPrinter);
-        listener.setSampledClasses(COBBusinessStepServiceImpl.class);
-        return listener;
-    }
 }

fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java

@@ -18,7 +18,7 @@
  */
 package org.apache.fineract.cob.loan;
 
-import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.diagnostics.performance.MeasuringUtil.measure;
 
 import java.util.Optional;
 import lombok.RequiredArgsConstructor;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java

@@ -336,5 +336,6 @@ public class FineractProperties {
         private boolean enabled;
         private int samplingRate;
         private String sampledClasses;
+        private int resetPeriodSec;
     }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/MeasuringUtil.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance;
+package org.apache.fineract.infrastructure.core.diagnostics.performance;
 
 import java.time.Duration;
 import java.util.function.BiConsumer;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/AbstractSamplingService.java

@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
-import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.diagnostics.performance.MeasuringUtil.measure;
 
 import java.time.Duration;
 import java.util.Map;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/InMemorySamplingService.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
 import java.time.Duration;
 import java.util.List;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/NoopSamplingService.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
 import static java.util.Collections.emptyMap;
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/SamplingConfiguration.java

@@ -16,52 +16,59 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
-import java.util.Arrays;
-import java.util.Map;
+import com.google.common.base.Splitter;
+import java.time.Duration;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.stereotype.Component;
 
-@Component
 @RequiredArgsConstructor
-public class SamplingServiceFactory implements InitializingBean {
+@Component
+@Slf4j
+public class SamplingConfiguration implements InitializingBean {
 
-    private final Map<Class<?>, SamplingService> services = new ConcurrentHashMap<>();
-    private final Set<String> classesToSample = ConcurrentHashMap.newKeySet();
+    private final Set<Class<?>> classesToSample = ConcurrentHashMap.newKeySet();
 
     private final FineractProperties properties;
 
     @Override
     public void afterPropertiesSet() throws Exception {
+        if (!properties.getSampling().isEnabled()) {
+            return;
+        }
         String sampledClasses = properties.getSampling().getSampledClasses();
-        String[] fqdns = sampledClasses.split(",");
-        Arrays.stream(fqdns).map(String::trim).forEach(classesToSample::add);
-    }
-
-    public SamplingService forClass(Class<?> contextClass) {
-        return services.computeIfAbsent(contextClass, (cc) -> {
-            if (isSamplingEnabled() && isSamplingConfiguredForClass(contextClass)) {
-                return new InMemorySamplingService(getSamplingRate());
-            } else {
-                return new NoopSamplingService();
+        Iterable<String> fqdns = Splitter.on(',').split(sampledClasses);
+        for (String fqdn : fqdns) {
+            String trimmedFqdn = fqdn.trim();
+            try {
+                Class<?> aClass = Class.forName(trimmedFqdn);
+                classesToSample.add(aClass);
+            } catch (Exception e) {
+                // ignore exception and proceed with other classes to sample
+                log.info("Cannot load class for sampling [{}]", trimmedFqdn);
             }
-        });
+        }
     }
 
-    private boolean isSamplingConfiguredForClass(Class<?> contextClass) {
-        return classesToSample.contains(contextClass.getName());
+    public boolean isSamplingConfiguredForClass(Class<?> contextClass) {
+        return classesToSample.stream().anyMatch(c -> c.isAssignableFrom(contextClass));
     }
 
-    private boolean isSamplingEnabled() {
+    public boolean isSamplingEnabled() {
         return properties.getSampling().isEnabled();
     }
 
-    private int getSamplingRate() {
+    public int getSamplingRate() {
         return properties.getSampling().getSamplingRate();
     }
+
+    public Duration getResetPeriod() {
+        return Duration.ofSeconds(properties.getSampling().getResetPeriodSec());
+    }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/SamplingData.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
 import java.time.Duration;
 import java.util.List;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/SamplingService.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
 
 import java.util.function.Supplier;
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/core/SamplingServiceFactory.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class SamplingServiceFactory {
+
+    private final Map<Class<?>, SamplingService> services = new ConcurrentHashMap<>();
+    private final SamplingConfiguration samplingConfiguration;
+
+    public SamplingService forClass(Class<?> contextClass) {
+        return services.computeIfAbsent(contextClass, (cc) -> {
+            if (samplingConfiguration.isSamplingEnabled() && samplingConfiguration.isSamplingConfiguredForClass(contextClass)) {
+                return new InMemorySamplingService(samplingConfiguration.getSamplingRate());
+            } else {
+                return new NoopSamplingService();
+            }
+        });
+    }
+
+    public void doWithAll(Consumer<Map<Class<?>, SamplingService>> c) {
+        c.accept(Map.copyOf(services));
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/output/SamplingDataPrinter.java

@@ -16,48 +16,71 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.output;
 
+import static java.lang.System.lineSeparator;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.math3.stat.StatUtils;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingData;
 import org.springframework.stereotype.Component;
 
 @Component
 @Slf4j
 public class SamplingDataPrinter {
 
-    public void printForClass(Class<?> clazz, SamplingData samplingData) {
+    @SuppressFBWarnings({ "SLF4J_FORMAT_SHOULD_BE_CONST" })
+    public void print(Map<Class<?>, SamplingData> data) {
+        if (log.isInfoEnabled()) {
+            if (MapUtils.isNotEmpty(data)) {
+                String logMsg = data.entrySet() //
+                        .stream() //
+                        .map(e -> getFormattedSamplingData(e.getKey(), e.getValue())) //
+                        .filter(Objects::nonNull).collect(Collectors.joining(lineSeparator())); //
+                if (StringUtils.isNotBlank(logMsg)) {
+                    log.info(logMsg);
+                }
+            }
+        }
+    }
+
+    private String getFormattedSamplingData(Class<?> clazz, SamplingData samplingData) {
         if (clazz != null && samplingData != null) {
             Map<String, List<Duration>> timings = samplingData.getTimings();
             if (!timings.isEmpty()) {
-                log.info("""
+                return """
 
-                        Sampling data for {}
+                        Sampling data for %s
                         -------------
-                        {}
-                        """, clazz.getName(), getTimingsLog(timings));
+                        %s
+                        """.formatted(clazz.getName(), getTimingsLog(timings));
             }
         }
+        return null;
     }
 
     private String getTimingsLog(Map<String, List<Duration>> timings) {
         return timings.entrySet().stream() //
                 .map(e -> getSingleTimingLog(e.getKey(), e.getValue())) //
-                .collect(Collectors.joining(System.lineSeparator())); //
+                .collect(Collectors.joining(lineSeparator())); //
     }
 
     private String getSingleTimingLog(String key, List<Duration> durations) {
         double[] millis = durations.stream().mapToLong(Duration::toMillis).asDoubleStream().toArray();
-        double highest = StatUtils.max(millis);
+        double percentile99 = StatUtils.percentile(millis, 99);
         double average = Arrays.stream(millis).average().orElse(Double.NaN);
         double median = StatUtils.percentile(millis, 50);
         double lowest = StatUtils.min(millis);
-        return "%s with %d data points -> highest: %.0fms, average: %.0fms, median: %.0fms, lowest: %.0fms".formatted(key, millis.length,
-                highest, average, median, lowest);
+        return "%s with %d data points -> 99th percentile: %.0fms, average: %.0fms, median: %.0fms, lowest: %.0fms".formatted(key,
+                millis.length, percentile99, average, median, lowest);
     }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/output/SamplingScheduler.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.output;
+
+import static java.util.stream.Collectors.toMap;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingConfiguration;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingData;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingService;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingServiceFactory;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.support.SamplingEnabledCondition;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.support.PeriodicTrigger;
+import org.springframework.stereotype.Component;
+
+@RequiredArgsConstructor
+@Component
+@Conditional(SamplingEnabledCondition.class)
+@Slf4j
+public class SamplingScheduler implements InitializingBean {
+
+    private final SamplingServiceFactory samplingServiceFactory;
+    private final SamplingDataPrinter printer;
+    private final TaskScheduler taskScheduler;
+    private final SamplingConfiguration samplingConfiguration;
+
+    @SuppressWarnings({ "FutureReturnValueIgnored" })
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        long resetPeriodInSec = samplingConfiguration.getResetPeriod().toSeconds();
+        if (resetPeriodInSec < 10) {
+            log.warn("Reset period for sampling cannot be smaller than 10 seconds, setting back the minimum 10");
+            resetPeriodInSec = 10;
+        }
+        PeriodicTrigger trigger = new PeriodicTrigger(resetPeriodInSec, TimeUnit.SECONDS);
+        trigger.setInitialDelay(resetPeriodInSec);
+        taskScheduler.schedule(this::printAndResetPeriodically, trigger);
+    }
+
+    private void printAndResetPeriodically() {
+        samplingServiceFactory.doWithAll((serviceMap) -> {
+            Map<Class<?>, SamplingData> samplingDataMap = serviceMap.entrySet() //
+                    .stream() //
+                    .collect(toMap(Map.Entry::getKey, e -> e.getValue().getSamplingData())); //
+            printer.print(samplingDataMap);
+            serviceMap.values().forEach(SamplingService::reset);
+        });
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/support/SamplingBeanPostProcessor.java

@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.support;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingConfiguration;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingService;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingServiceFactory;
+import org.springframework.aop.framework.AopProxyUtils;
+import org.springframework.aop.framework.ProxyFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.stereotype.Component;
+
+@RequiredArgsConstructor
+@Component
+@Slf4j
+public class SamplingBeanPostProcessor implements BeanPostProcessor {
+
+    private SamplingConfiguration samplingConfiguration;
+    private SamplingServiceFactory samplingServiceFactory;
+
+    @Override
+    public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
+        if (isSamplingEnabled()) {
+            Class<?> beanClass = AopProxyUtils.ultimateTargetClass(bean);
+            boolean isSamplingConfiguredForBean = samplingConfiguration.isSamplingConfiguredForClass(beanClass);
+            if (isSamplingConfiguredForBean) {
+                log.info("Sampling is enabled for {}", beanClass);
+                SamplingService samplingService = samplingServiceFactory.forClass(beanClass);
+                ProxyFactory proxyFactory = new ProxyFactory();
+                proxyFactory.setTarget(bean);
+                proxyFactory.addAdvice(new SamplingMethodInterceptor(samplingService));
+                return proxyFactory.getProxy();
+            }
+        }
+
+        return bean;
+    }
+
+    private boolean isSamplingEnabled() {
+        return samplingConfiguration != null && samplingConfiguration.isSamplingEnabled();
+    }
+
+    @Autowired
+    public void setSamplingServiceFactory(SamplingServiceFactory samplingServiceFactory) {
+        this.samplingServiceFactory = samplingServiceFactory;
+    }
+
+    @Autowired
+    public void setSamplingConfiguration(SamplingConfiguration samplingConfiguration) {
+        this.samplingConfiguration = samplingConfiguration;
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/support/SamplingEnabledCondition.java

@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.support;
+
+import org.apache.fineract.infrastructure.core.condition.PropertiesCondition;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+
+public class SamplingEnabledCondition extends PropertiesCondition {
+
+    @Override
+    protected boolean matches(FineractProperties properties) {
+        return properties.getSampling().isEnabled();
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/diagnostics/performance/sampling/support/SamplingMethodInterceptor.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.support;
+
+import java.util.StringJoiner;
+import lombok.RequiredArgsConstructor;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.apache.fineract.infrastructure.core.diagnostics.performance.sampling.core.SamplingService;
+
+@RequiredArgsConstructor
+public class SamplingMethodInterceptor implements MethodInterceptor {
+
+    private final SamplingService samplingService;
+
+    @Override
+    public Object invoke(MethodInvocation invocation) throws Throwable {
+        Object aThis = invocation.getThis();
+        if (aThis != null) {
+            String samplingKey = invocation.getMethod().getName() + getFormattedArguments(invocation);
+            return samplingService.sample(samplingKey, () -> {
+                try {
+                    return invocation.proceed();
+                } catch (Throwable e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        } else {
+            return invocation.proceed();
+        }
+    }
+
+    private String getFormattedArguments(MethodInvocation invocation) {
+        Class<?>[] parameterTypes = invocation.getMethod().getParameterTypes();
+        if (parameterTypes.length == 0) {
+            return "";
+        } else {
+            StringJoiner stringJoiner = new StringJoiner(",");
+            for (Class<?> parameterType : parameterTypes) {
+                stringJoiner.add(parameterType.getSimpleName());
+            }
+            return "(" + stringJoiner + ")";
+        }
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingStepExecutionListener.java

@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.fineract.infrastructure.core.service.performance.sampling;
-
-import static java.util.function.Function.identity;
-import static java.util.stream.Collectors.toMap;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import lombok.RequiredArgsConstructor;
-import org.springframework.batch.core.ExitStatus;
-import org.springframework.batch.core.StepExecution;
-import org.springframework.batch.core.StepExecutionListener;
-
-@RequiredArgsConstructor
-public class SamplingStepExecutionListener implements StepExecutionListener {
-
-    private final SamplingServiceFactory samplingServiceFactory;
-    private final SamplingDataPrinter samplingDataPrinter;
-    private final List<Class<?>> sampledClasses = new ArrayList<>();
-
-    @Override
-    public void beforeStep(StepExecution stepExecution) {
-
-    }
-
-    @Override
-    public ExitStatus afterStep(StepExecution stepExecution) {
-        sampledClasses.stream().collect(toMap(identity(), contextClass -> samplingServiceFactory.forClass(contextClass).getSamplingData()))
-                .forEach(samplingDataPrinter::printForClass);
-        sampledClasses.stream().map(samplingServiceFactory::forClass).forEach(SamplingService::reset);
-        return null;
-    }
-
-    public void setSampledClasses(Class<?>... classes) {
-        sampledClasses.addAll(Arrays.asList(classes));
-    }
-}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java

@@ -20,7 +20,7 @@ package org.apache.fineract.infrastructure.event.external.jobs;
 
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toMap;
-import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.diagnostics.performance.MeasuringUtil.measure;
 
 import com.google.common.collect.Lists;
 import java.io.IOException;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java

@@ -18,7 +18,7 @@
  */
 package org.apache.fineract.infrastructure.event.external.producer.jms;
 
-import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.diagnostics.performance.MeasuringUtil.measure;
 
 import java.util.ArrayList;
 import java.util.Collection;

fineract-provider/src/main/resources/application.properties

@@ -109,6 +109,7 @@ fineract.logging.json.enabled=${FINERACT_LOGGING_JSON_ENABLED:false}
 fineract.sampling.enabled=${FINERACT_SAMPLING_ENABLED:false}
 fineract.sampling.samplingRate=${FINERACT_SAMPLING_RATE:1000}
 fineract.sampling.sampledClasses=${FINERACT_SAMPLED_CLASSES:}
+fineract.sampling.resetPeriodSec=${FINERACT_SAMPLING_RESET_PERIOD_IN_SEC:60}
 
 # Logging pattern for the console
 logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
@@ -203,6 +204,7 @@ springdoc.show-actuator=${SPRINGDOC_SHOW_ACTUATOR:false}
 spring.web.resources.static-locations=classpath:/static/
 
 spring.main.allow-bean-definition-overriding=true
+spring.task.scheduling.pool.size=4
 spring.batch.initialize-schema=NEVER
 # Disabling Spring Batch jobs on startup
 spring.batch.job.enabled=false

fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java

@@ -40,11 +40,9 @@ import org.apache.fineract.cob.exceptions.BusinessStepException;
 import org.apache.fineract.cob.loan.LoanCOBBusinessStep;
 import org.apache.fineract.cob.service.ReloaderService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
-import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.AbstractAuditableCustom;
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.mix.data.MixTaxonomyData;
 import org.mockito.Mockito;
@@ -54,8 +52,6 @@ import org.springframework.context.ApplicationContext;
 
 public class COBBusinessStepServiceStepDefinitions implements En {
 
-    private SamplingServiceFactory samplingServiceFactory;
-
     private ApplicationContext applicationContext = mock(ApplicationContext.class);
     private ListableBeanFactory beanFactory = mock(ListableBeanFactory.class);
     private BatchBusinessStepRepository batchBusinessStepRepository = mock(BatchBusinessStepRepository.class);
@@ -82,17 +78,8 @@ public class COBBusinessStepServiceStepDefinitions implements En {
     private Set<BusinessStepNameAndOrder> resultSet;
 
     public COBBusinessStepServiceStepDefinitions() throws Exception {
-        FineractProperties.FineractSamplingProperties sampling = new FineractProperties.FineractSamplingProperties();
-        sampling.setEnabled(false);
-        sampling.setSampledClasses("");
-        FineractProperties fineractProperties = new FineractProperties();
-        fineractProperties.setSampling(sampling);
-        samplingServiceFactory = new SamplingServiceFactory(fineractProperties);
-        samplingServiceFactory.afterPropertiesSet();
-
         businessStepService = new COBBusinessStepServiceImpl(batchBusinessStepRepository, applicationContext, beanFactory,
-                businessEventNotifierService, configurationDomainService, reloaderService, samplingServiceFactory);
-        businessStepService.afterPropertiesSet();
+                businessEventNotifierService, configurationDomainService, reloaderService);
 
         Given("/^The COBBusinessStepService.run method with executeMap (.*)$/", (String executionMap) -> {
             if ("null".equals(executionMap)) {

fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java

@@ -41,8 +41,6 @@ import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.NoopSamplingService;
-import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
 import org.junit.jupiter.api.AfterEach;
@@ -68,8 +66,6 @@ public class COBBulkEventConfigurationTest {
     private BusinessEventNotifierService businessEventNotifierService;
     @Mock
     private ConfigurationDomainService configurationDomainService;
-    @Mock
-    private SamplingServiceFactory samplingServiceFactory;
     @InjectMocks
     private COBBusinessStepServiceImpl underTest;
 
@@ -83,8 +79,6 @@ public class COBBulkEventConfigurationTest {
         ThreadLocalContextUtil
                 .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault()))));
         when(reloaderService.reload(any())).thenAnswer(invocation -> invocation.getArgument(0));
-        when(samplingServiceFactory.forClass(any())).thenReturn(new NoopSamplingService());
-        underTest.afterPropertiesSet();
     }
 
     @AfterEach