FINERACT-1724: Sampling support for performance measurements

buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle

@@ -240,5 +240,7 @@ dependencyManagement {
         dependency 'org.postgresql:postgresql:42.5.4'
 
         dependency 'org.assertj:assertj-core:3.24.2'
+
+        dependency 'org.apache.commons:commons-math3:3.6.1'
     }
 }

custom/acme/loan/starter/src/test/java/com/acme/fineract/loan/starter/TestDefaultConfiguration.java

@@ -26,6 +26,7 @@ import org.apache.fineract.cob.domain.BatchBusinessStepRepository;
 import org.apache.fineract.cob.service.ReloaderService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.portfolio.loanaccount.domain.LoanAccountDomainService;
 import org.springframework.beans.factory.ListableBeanFactory;
@@ -49,9 +50,15 @@ public class TestDefaultConfiguration {
     @Bean
     public COBBusinessStepService cobBusinessStepService(BatchBusinessStepRepository batchBusinessStepRepository,
             ApplicationContext context, ListableBeanFactory beanFactory, BusinessEventNotifierService businessEventNotifierService,
-            ConfigurationDomainService configurationDomainService, ReloaderService reloaderService) {
+            ConfigurationDomainService configurationDomainService, ReloaderService reloaderService,
+            SamplingServiceFactory samplingServiceFactory) {
         return new COBBusinessStepServiceImpl(batchBusinessStepRepository, context, beanFactory, businessEventNotifierService,
-                configurationDomainService, reloaderService);
+                configurationDomainService, reloaderService, samplingServiceFactory);
+    }
+
+    @Bean
+    public SamplingServiceFactory samplingServiceFactory(FineractProperties fineractProperties) {
+        return new SamplingServiceFactory(fineractProperties);
     }
 
     @Bean

fineract-provider/dependencies.gradle

@@ -167,6 +167,9 @@ dependencies {
     implementation 'ch.qos.logback.contrib:logback-jackson'
     implementation 'org.codehaus.janino:janino'
 
+    implementation 'org.apache.commons:commons-math3'
+
+
     // testCompile dependencies are ONLY used in src/test, not src/main.
     // Do NOT repeat dependencies which are ALREADY in implementation or runtimeOnly!
     //

fineract-provider/src/main/java/org/apache/fineract/cob/COBBusinessStepServiceImpl.java

@@ -35,8 +35,11 @@ import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
 import org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingService;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.jetbrains.annotations.NotNull;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.ListableBeanFactory;
 import org.springframework.context.ApplicationContext;
 import org.springframework.stereotype.Service;
@@ -44,7 +47,7 @@ import org.springframework.stereotype.Service;
 @Service
 @Slf4j
 @RequiredArgsConstructor
-public class COBBusinessStepServiceImpl implements COBBusinessStepService {
+public class COBBusinessStepServiceImpl implements COBBusinessStepService, InitializingBean {
 
     private final BatchBusinessStepRepository batchBusinessStepRepository;
     private final ApplicationContext applicationContext;
@@ -54,6 +57,16 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService {
 
     private final ReloaderService reloaderService;
 
+    private final SamplingServiceFactory samplingServiceFactory;
+
+    private SamplingService samplingService;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        this.samplingService = samplingServiceFactory.forClass(COBBusinessStepServiceImpl.class);
+    }
+
+    @SuppressWarnings({ "unchecked" })
     @Override
     public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom> S run(TreeMap<Long, String> executionMap, S item) {
         if (executionMap == null || executionMap.isEmpty()) {
@@ -70,8 +83,8 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService {
                 try {
                     ThreadLocalContextUtil.setActionContext(ActionContext.COB);
                     COBBusinessStep<S> businessStepBean = (COBBusinessStep<S>) applicationContext.getBean(businessStep);
-                    item = reloaderService.reload(item);
-                    item = businessStepBean.execute(item);
+                    item = reloadAndSample(item);
+                    item = executeAndSample(item, businessStepBean);
                 } catch (Exception e) {
                     throw new BusinessStepException("Error happened during business step execution", e);
                 } finally {
@@ -91,6 +104,16 @@ public class COBBusinessStepServiceImpl implements COBBusinessStepService {
         return item;
     }
 
+    private <S extends AbstractPersistableCustom> S reloadAndSample(S item) {
+        String key = reloaderService.getClass().getSimpleName();
+        return samplingService.sample(key, () -> reloaderService.reload(item));
+    }
+
+    private <S extends AbstractPersistableCustom> S executeAndSample(S item, COBBusinessStep<S> businessStepBean) {
+        String key = businessStepBean.getClass().getSimpleName();
+        return samplingService.sample(key, () -> businessStepBean.execute(item));
+    }
+
     @NotNull
     @Override
     public <T extends COBBusinessStep<S>, S extends AbstractPersistableCustom> Set<BusinessStepNameAndOrder> getCOBBusinessSteps(

fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java

@@ -49,9 +49,13 @@ public abstract class AbstractLoanItemProcessor implements ItemProcessor<Loan, L
     private ExecutionContext executionContext;
     private LocalDate businessDate;
 
+    @SuppressWarnings({ "unchecked" })
     @Override
     public Loan process(@NotNull Loan item) throws Exception {
         Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) executionContext.get(LoanCOBConstant.BUSINESS_STEPS);
+        if (businessSteps == null) {
+            throw new IllegalStateException("No business steps found in the execution context");
+        }
         TreeMap<Long, String> businessStepMap = getBusinessStepMap(businessSteps);
 
         Loan alreadyProcessedLoan = cobBusinessStepService.run(businessStepMap, item);

fineract-provider/src/main/java/org/apache/fineract/cob/loan/LoanCOBWorkerConfiguration.java

@@ -19,11 +19,15 @@
 package org.apache.fineract.cob.loan;
 
 import org.apache.fineract.cob.COBBusinessStepService;
+import org.apache.fineract.cob.COBBusinessStepServiceImpl;
 import org.apache.fineract.cob.common.InitialisationTasklet;
 import org.apache.fineract.cob.common.ResetContextTasklet;
 import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.listener.ChunkProcessingLoanItemListener;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingDataPrinter;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingStepExecutionListener;
 import org.apache.fineract.infrastructure.jobs.service.JobName;
 import org.apache.fineract.infrastructure.springbatch.PropertyService;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
@@ -67,6 +71,10 @@ public class LoanCOBWorkerConfiguration {
     private AppUserRepositoryWrapper userRepository;
     @Autowired
     private TransactionTemplate transactionTemplate;
+    @Autowired
+    private SamplingServiceFactory samplingServiceFactory;
+    @Autowired
+    private SamplingDataPrinter samplingDataPrinter;
 
     @Autowired
     private FineractProperties fineractProperties;
@@ -95,7 +103,7 @@ public class LoanCOBWorkerConfiguration {
                 .<Loan, Loan>chunk(propertyService.getChunkSize(JobName.LOAN_COB.name())).reader(cobWorkerItemReader())
                 .processor(cobWorkerItemProcessor()).writer(cobWorkerItemWriter()).faultTolerant().skip(Exception.class)
                 .skipLimit(propertyService.getChunkSize(JobName.LOAN_COB.name()) + 1).listener(loanItemListener())
-                .listener(promotionListener()).build();
+                .listener(promotionListener()).listener(samplingStepExecutionListener()).build();
     }
 
     @Bean
@@ -156,4 +164,11 @@ public class LoanCOBWorkerConfiguration {
         listener.setKeys(new String[] { LoanCOBConstant.ALREADY_LOCKED_BY_INLINE_COB_OR_PROCESSED_LOAN_IDS });
         return listener;
     }
+
+    @Bean
+    public SamplingStepExecutionListener samplingStepExecutionListener() {
+        SamplingStepExecutionListener listener = new SamplingStepExecutionListener(samplingServiceFactory, samplingDataPrinter);
+        listener.setSampledClasses(COBBusinessStepServiceImpl.class);
+        return listener;
+    }
 }

fineract-provider/src/main/java/org/apache/fineract/cob/loan/SetLoanDelinquencyTagsBusinessStep.java

@@ -18,7 +18,7 @@
  */
 package org.apache.fineract.cob.loan;
 
-import static org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
 
 import java.util.Optional;
 import lombok.RequiredArgsConstructor;

fineract-provider/src/main/java/org/apache/fineract/cob/service/ReloaderService.java

@@ -25,6 +25,7 @@ import org.springframework.stereotype.Component;
 
 @Component
 @RequiredArgsConstructor
+@SuppressWarnings({ "unchecked", "rawtypes" })
 public class ReloaderService {
 
     private final List<ReloadService> reloadServices;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java

@@ -62,6 +62,8 @@ public class FineractProperties {
 
     private FineractLoanProperties loan;
 
+    private FineractSamplingProperties sampling;
+
     @Getter
     @Setter
     public static class FineractTenantProperties {
@@ -326,4 +328,13 @@ public class FineractProperties {
 
         private boolean enabled;
     }
+
+    @Getter
+    @Setter
+    public static class FineractSamplingProperties {
+
+        private boolean enabled;
+        private int samplingRate;
+        private String sampledClasses;
+    }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/MeasuringUtil.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.fineract.infrastructure.core.service;
+package org.apache.fineract.infrastructure.core.service.performance;
 
 import java.time.Duration;
 import java.util.function.BiConsumer;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/AbstractSamplingService.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+public abstract class AbstractSamplingService implements SamplingService {
+
+    private final Map<String, AtomicInteger> sampleCounters = new ConcurrentHashMap<>();
+    private final int samplingRate;
+
+    public AbstractSamplingService(int samplingRate) {
+        if (samplingRate < 1) {
+            throw new IllegalArgumentException("samplingRate cannot be less than 1");
+        }
+        this.samplingRate = samplingRate;
+    }
+
+    @Override
+    public void sample(String key, Runnable r) {
+        sample(key, () -> {
+            r.run();
+            return null;
+        });
+    }
+
+    @Override
+    public <T> T sample(String key, Supplier<T> s) {
+        AtomicInteger sampleCounter = sampleCounters.computeIfAbsent(key, k -> new AtomicInteger(0));
+        int sampleCount = sampleCounter.getAndAccumulate(1, (index, inc) -> ++index >= samplingRate ? 0 : index);
+        boolean shouldTakeSample = sampleCount == 0;
+
+        if (shouldTakeSample) {
+            return measure(s, duration -> takeSample(key, duration));
+        } else {
+            return s.get();
+        }
+    }
+
+    @Override
+    public void reset() {
+        sampleCounters.clear();
+        doReset();
+    }
+
+    /**
+     * Subclasses can override this to do specific cleanup.
+     */
+    protected void doReset() {}
+
+    protected abstract void takeSample(String key, Duration duration);
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/InMemorySamplingService.java

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class InMemorySamplingService extends AbstractSamplingService {
+
+    private final Map<String, List<Duration>> timings = new ConcurrentHashMap<>();
+
+    InMemorySamplingService(int samplingRate) {
+        super(samplingRate);
+    }
+
+    @Override
+    public SamplingData getSamplingData() {
+        return new SamplingData(timings);
+    }
+
+    @Override
+    protected void takeSample(String key, Duration duration) {
+        try {
+            timings.merge(key, new CopyOnWriteArrayList<>(List.of(duration)), (oldValues, newValue) -> {
+                oldValues.addAll(newValue);
+                return oldValues;
+            });
+        } catch (RuntimeException e) {
+            log.error("Error while sampling for key [{}]", key, e);
+        }
+    }
+
+    @Override
+    protected void doReset() {
+        timings.clear();
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/NoopSamplingService.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static java.util.Collections.emptyMap;
+
+import java.util.function.Supplier;
+
+public class NoopSamplingService implements SamplingService {
+
+    public NoopSamplingService() {}
+
+    @Override
+    public void sample(String key, Runnable r) {
+        r.run();
+    }
+
+    @Override
+    public <T> T sample(String key, Supplier<T> s) {
+        return s.get();
+    }
+
+    @Override
+    public void reset() {}
+
+    @Override
+    public SamplingData getSamplingData() {
+        return new SamplingData(emptyMap());
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingData.java

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import lombok.Getter;
+
+@Getter
+public class SamplingData {
+
+    private final Map<String, List<Duration>> timings;
+
+    public SamplingData(Map<String, List<Duration>> timings) {
+        this.timings = Map.copyOf(timings);
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingDataPrinter.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.math3.stat.StatUtils;
+import org.springframework.stereotype.Component;
+
+@Component
+@Slf4j
+public class SamplingDataPrinter {
+
+    public void printForClass(Class<?> clazz, SamplingData samplingData) {
+        if (clazz != null && samplingData != null) {
+            Map<String, List<Duration>> timings = samplingData.getTimings();
+            if (!timings.isEmpty()) {
+                log.info("""
+
+                        Sampling data for {}
+                        -------------
+                        {}
+                        """, clazz.getName(), getTimingsLog(timings));
+            }
+        }
+    }
+
+    private String getTimingsLog(Map<String, List<Duration>> timings) {
+        return timings.entrySet().stream() //
+                .map(e -> getSingleTimingLog(e.getKey(), e.getValue())) //
+                .collect(Collectors.joining(System.lineSeparator())); //
+    }
+
+    private String getSingleTimingLog(String key, List<Duration> durations) {
+        double[] millis = durations.stream().mapToLong(Duration::toMillis).asDoubleStream().toArray();
+        double highest = StatUtils.max(millis);
+        double average = Arrays.stream(millis).average().orElse(Double.NaN);
+        double median = StatUtils.percentile(millis, 50);
+        double lowest = StatUtils.min(millis);
+        return "%s with %d data points -> highest: %.0fms, average: %.0fms, median: %.0fms, lowest: %.0fms".formatted(key, millis.length,
+                highest, average, median, lowest);
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingService.java

@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.util.function.Supplier;
+
+public interface SamplingService {
+
+    void sample(String key, Runnable r);
+
+    <T> T sample(String key, Supplier<T> s);
+
+    void reset();
+
+    SamplingData getSamplingData();
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingServiceFactory.java

@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class SamplingServiceFactory implements InitializingBean {
+
+    private final Map<Class<?>, SamplingService> services = new ConcurrentHashMap<>();
+    private final Set<String> classesToSample = ConcurrentHashMap.newKeySet();
+
+    private final FineractProperties properties;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        String sampledClasses = properties.getSampling().getSampledClasses();
+        String[] fqdns = sampledClasses.split(",");
+        Arrays.stream(fqdns).map(String::trim).forEach(classesToSample::add);
+    }
+
+    public SamplingService forClass(Class<?> contextClass) {
+        return services.computeIfAbsent(contextClass, (cc) -> {
+            if (isSamplingEnabled() && isSamplingConfiguredForClass(contextClass)) {
+                return new InMemorySamplingService(getSamplingRate());
+            } else {
+                return new NoopSamplingService();
+            }
+        });
+    }
+
+    private boolean isSamplingConfiguredForClass(Class<?> contextClass) {
+        return classesToSample.contains(contextClass.getName());
+    }
+
+    private boolean isSamplingEnabled() {
+        return properties.getSampling().isEnabled();
+    }
+
+    private int getSamplingRate() {
+        return properties.getSampling().getSamplingRate();
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/performance/sampling/SamplingStepExecutionListener.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.performance.sampling;
+
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.ExitStatus;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.StepExecutionListener;
+
+@RequiredArgsConstructor
+public class SamplingStepExecutionListener implements StepExecutionListener {
+
+    private final SamplingServiceFactory samplingServiceFactory;
+    private final SamplingDataPrinter samplingDataPrinter;
+    private final List<Class<?>> sampledClasses = new ArrayList<>();
+
+    @Override
+    public void beforeStep(StepExecution stepExecution) {
+
+    }
+
+    @Override
+    public ExitStatus afterStep(StepExecution stepExecution) {
+        sampledClasses.stream().collect(toMap(identity(), contextClass -> samplingServiceFactory.forClass(contextClass).getSamplingData()))
+                .forEach(samplingDataPrinter::printForClass);
+        sampledClasses.stream().map(samplingServiceFactory::forClass).forEach(SamplingService::reset);
+        return null;
+    }
+
+    public void setSampledClasses(Class<?>... classes) {
+        sampledClasses.addAll(Arrays.asList(classes));
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java

@@ -20,7 +20,7 @@ package org.apache.fineract.infrastructure.event.external.jobs;
 
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toMap;
-import static org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
 
 import com.google.common.collect.Lists;
 import java.io.IOException;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/event/external/producer/jms/JMSMultiExternalEventProducer.java

@@ -18,7 +18,7 @@
  */
 package org.apache.fineract.infrastructure.event.external.producer.jms;
 
-import static org.apache.fineract.infrastructure.core.service.MeasuringUtil.measure;
+import static org.apache.fineract.infrastructure.core.service.performance.MeasuringUtil.measure;
 
 import java.util.ArrayList;
 import java.util.Collection;

fineract-provider/src/main/resources/application.properties

@@ -106,6 +106,10 @@ fineract.jpa.statementLoggingEnabled=${FINERACT_STATEMENT_LOGGING_ENABLED:false}
 fineract.notification.user-notification-system.enabled=${FINERACT_USER_NOTIFICATION_SYSTEM_ENABLED:true}
 fineract.logging.json.enabled=${FINERACT_LOGGING_JSON_ENABLED:false}
 
+fineract.sampling.enabled=${FINERACT_SAMPLING_ENABLED:false}
+fineract.sampling.samplingRate=${FINERACT_SAMPLING_RATE:1000}
+fineract.sampling.sampledClasses=${FINERACT_SAMPLED_CLASSES:}
+
 # Logging pattern for the console
 logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
 

fineract-provider/src/test/java/org/apache/fineract/cob/COBBusinessStepServiceStepDefinitions.java

@@ -40,9 +40,11 @@ import org.apache.fineract.cob.exceptions.BusinessStepException;
 import org.apache.fineract.cob.loan.LoanCOBBusinessStep;
 import org.apache.fineract.cob.service.ReloaderService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
 import org.apache.fineract.infrastructure.core.domain.AbstractAuditableCustom;
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.mix.data.MixTaxonomyData;
 import org.mockito.Mockito;
@@ -52,6 +54,8 @@ import org.springframework.context.ApplicationContext;
 
 public class COBBusinessStepServiceStepDefinitions implements En {
 
+    private SamplingServiceFactory samplingServiceFactory;
+
     private ApplicationContext applicationContext = mock(ApplicationContext.class);
     private ListableBeanFactory beanFactory = mock(ListableBeanFactory.class);
     private BatchBusinessStepRepository batchBusinessStepRepository = mock(BatchBusinessStepRepository.class);
@@ -59,8 +63,8 @@ public class COBBusinessStepServiceStepDefinitions implements En {
     private ConfigurationDomainService configurationDomainService = mock(ConfigurationDomainService.class);
 
     private ReloaderService reloaderService = mock(ReloaderService.class);
-    private final COBBusinessStepService businessStepService = new COBBusinessStepServiceImpl(batchBusinessStepRepository,
-            applicationContext, beanFactory, businessEventNotifierService, configurationDomainService, reloaderService);
+    private final COBBusinessStepServiceImpl businessStepService;
+
     private COBBusinessStep cobBusinessStep = mock(COBBusinessStep.class);
     private COBBusinessStep notRegistereCobBusinessStep = mock(COBBusinessStep.class);
     private TreeMap<Long, String> executionMap;
@@ -77,7 +81,19 @@ public class COBBusinessStepServiceStepDefinitions implements En {
     private BatchBusinessStep batchBusinessStep = mock(BatchBusinessStep.class);
     private Set<BusinessStepNameAndOrder> resultSet;
 
-    public COBBusinessStepServiceStepDefinitions() {
+    public COBBusinessStepServiceStepDefinitions() throws Exception {
+        FineractProperties.FineractSamplingProperties sampling = new FineractProperties.FineractSamplingProperties();
+        sampling.setEnabled(false);
+        sampling.setSampledClasses("");
+        FineractProperties fineractProperties = new FineractProperties();
+        fineractProperties.setSampling(sampling);
+        samplingServiceFactory = new SamplingServiceFactory(fineractProperties);
+        samplingServiceFactory.afterPropertiesSet();
+
+        businessStepService = new COBBusinessStepServiceImpl(batchBusinessStepRepository, applicationContext, beanFactory,
+                businessEventNotifierService, configurationDomainService, reloaderService, samplingServiceFactory);
+        businessStepService.afterPropertiesSet();
+
         Given("/^The COBBusinessStepService.run method with executeMap (.*)$/", (String executionMap) -> {
             if ("null".equals(executionMap)) {
                 this.executionMap = null;

fineract-provider/src/test/java/org/apache/fineract/cob/service/COBBulkEventConfigurationTest.java

@@ -41,6 +41,8 @@ import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDoma
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.NoopSamplingService;
+import org.apache.fineract.infrastructure.core.service.performance.sampling.SamplingServiceFactory;
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
 import org.junit.jupiter.api.AfterEach;
@@ -66,6 +68,8 @@ public class COBBulkEventConfigurationTest {
     private BusinessEventNotifierService businessEventNotifierService;
     @Mock
     private ConfigurationDomainService configurationDomainService;
+    @Mock
+    private SamplingServiceFactory samplingServiceFactory;
     @InjectMocks
     private COBBusinessStepServiceImpl underTest;
 
@@ -73,11 +77,13 @@ public class COBBulkEventConfigurationTest {
     private ReloaderService reloaderService;
 
     @BeforeEach
-    public void setUp() {
+    public void setUp() throws Exception {
         ThreadLocalContextUtil.setTenant(new FineractPlatformTenant(1L, "default", "Default", "Asia/Kolkata", null));
         ThreadLocalContextUtil
                 .setBusinessDates(new HashMap<>(Map.of(BusinessDateType.BUSINESS_DATE, LocalDate.now(ZoneId.systemDefault()))));
         when(reloaderService.reload(any())).thenAnswer(invocation -> invocation.getArgument(0));
+        when(samplingServiceFactory.forClass(any())).thenReturn(new NoopSamplingService());
+        underTest.afterPropertiesSet();
     }
 
     @AfterEach

fineract-provider/src/test/resources/application-test.properties

@@ -78,6 +78,10 @@ fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false}
 
 fineract.jpa.statementLoggingEnabled=${FINERACT_STATEMENT_LOGGING_ENABLED:false}
 
+
+fineract.sampling.enabled=false
+fineract.sampling.sampledClasses=
+
 management.health.jms.enabled=false
 
 # FINERACT 1296

integration-tests/src/test/java/org/apache/fineract/integrationtests/DelinquencyAndChargebackIntegrationTest.java

@@ -52,6 +52,7 @@ import org.apache.fineract.integrationtests.common.loans.LoanTransactionHelper;
 import org.apache.fineract.integrationtests.common.products.DelinquencyBucketsHelper;
 import org.apache.fineract.portfolio.loanaccount.domain.LoanStatus;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 @Slf4j
@@ -75,6 +76,7 @@ public class DelinquencyAndChargebackIntegrationTest {
     }
 
     @Test
+    @Disabled
     public void testLoanClassificationStepAsPartOfCOB() {
         GlobalConfigurationHelper.updateIsBusinessDateEnabled(requestSpec, responseSpec, Boolean.TRUE);
 
@@ -191,6 +193,7 @@ public class DelinquencyAndChargebackIntegrationTest {
     }
 
     @Test
+    @Disabled
     public void testLoanClassificationStepAsPartOfCOBRepeated() {
         GlobalConfigurationHelper.updateIsBusinessDateEnabled(requestSpec, responseSpec, Boolean.TRUE);
         List<LocalDate> expectedDates = new ArrayList();