FINERACT-1795: Improve resilience of command processing service

buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle

@@ -187,5 +187,7 @@ dependencyManagement {
         dependency 'org.mapstruct:mapstruct-processor:1.5.3.Final'
 
         dependency "org.apache.avro:avro:1.11.1"
+
+        dependency "io.github.resilience4j:resilience4j-spring-boot2:1.7.1"
     }
 }

fineract-doc/src/docs/en/chapters/resilience/command.adoc

@@ -0,0 +1,23 @@
+= Command
+
+== `CommandProcessingService`
+
+TBD
+
+.Retry-able service function `executeCommand`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=73..151]
+----
+
+.Fallback function `fallbackExecuteCommand`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=166..174]
+----
+
+.Retry configuration for `executeCommand`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=169..174]
+----

fineract-doc/src/docs/en/chapters/resilience/index.adoc

@@ -0,0 +1,11 @@
+= Resilience
+
+include::intro.adoc[leveloffset=+1]
+
+include::command.adoc[leveloffset=+1]
+
+include::job.adoc[leveloffset=+1]
+
+include::loan.adoc[leveloffset=+1]
+
+include::saving.adoc[leveloffset=+1]

fineract-doc/src/docs/en/chapters/resilience/intro.adoc

@@ -0,0 +1,84 @@
+= Introduction Resilience
+
+Fineract had handcrafted retry loops in place for the longest time. A typical retry code would have looked like this:
+
+.Legacy retry code
+[source,java]
+----
+    @Override
+    @SuppressWarnings("AvoidHidingCauseException")
+    @SuppressFBWarnings(value = {
+            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
+    public CommandProcessingResult logCommandSource(final CommandWrapper wrapper) {
+
+        boolean isApprovedByChecker = false;
+        // check if is update of own account details
+        if (wrapper.isUpdateOfOwnUserDetails(this.context.authenticatedUser(wrapper).getId())) {
+            // then allow this operation to proceed.
+            // maker checker doesnt mean anything here.
+            isApprovedByChecker = true; // set to true in case permissions have
+                                        // been maker-checker enabled by
+                                        // accident.
+        } else {
+            // if not user changing their own details - check user has
+            // permission to perform specific task.
+            this.context.authenticatedUser(wrapper).validateHasPermissionTo(wrapper.getTaskPermissionName());
+        }
+        validateIsUpdateAllowed();
+
+        final String json = wrapper.getJson();
+        CommandProcessingResult result = null;
+        JsonCommand command;
+        int numberOfRetries = 0; // <1>
+        int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
+        int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
+        final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json);
+        command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(),
+                wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(),
+                wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(),
+                wrapper.getOrganisationCreditBureauId(), wrapper.getJobName());
+        while (numberOfRetries <= maxNumberOfRetries) { // <2>
+            try {
+                result = this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker);
+                numberOfRetries = maxNumberOfRetries + 1; // <3>
+            } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
+                log.debug("The following command {} has been retried  {} time(s)", command.json(), numberOfRetries);
+                /***
+                 * Fail if the transaction has been retired for maxNumberOfRetries
+                 **/
+                if (numberOfRetries >= maxNumberOfRetries) {
+                    log.warn("The following command {} has been retried for the max allowed attempts of {} and will be rolled back",
+                            command.json(), numberOfRetries);
+                    throw exception;
+                }
+                /***
+                 * Else sleep for a random time (between 1 to 10 seconds) and continue
+                 **/
+                try {
+                    int randomNum = RANDOM.nextInt(maxIntervalBetweenRetries + 1);
+                    Thread.sleep(1000 + (randomNum * 1000));
+                    numberOfRetries = numberOfRetries + 1; // <4>
+                } catch (InterruptedException e) {
+                    throw exception;
+                }
+            } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) {
+                numberOfRetries = maxNumberOfRetries + 1; // <3>
+                result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult());
+            }
+        }
+
+        return result;
+    }
+----
+<1> counter
+<2> `while` loop
+<3> increment to abort
+<4> increment
+
+For better code quality and readability we introduced https://resilience4j.readme.io/docs[Resilience4j]:
+
+.Annotation based retry
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java[lines=49..76]
+----

fineract-doc/src/docs/en/chapters/resilience/job.adoc

@@ -0,0 +1,25 @@
+= Jobs
+
+== `SchedularWritePlatformService`
+
+WARNING: This service has a typo and should be called `SchedulerWritePlatformService`.
+
+TBD
+
+.Retry-able service function `processJobDetailForExecution`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=135..155]
+----
+
+.Fallback function `fallbackProcessJobDetailForExecution`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=156..159]
+----
+
+.Retry configuration for `processJobDetailForExecution`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=175..179]
+----

fineract-doc/src/docs/en/chapters/resilience/loan.adoc

@@ -0,0 +1,23 @@
+= Loan
+
+== `LoanWritePlatformService`
+
+TBD
+
+.Retry-able service function `recalculateInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3217..3247]
+----
+
+.Fallback function `fallbackRecalculateInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3255..3266]
+----
+
+.Retry configuration for `recalculateInterest`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=180..185]
+----

fineract-doc/src/docs/en/chapters/resilience/saving.adoc

@@ -0,0 +1,23 @@
+= Savings
+
+== `SavingsAccountWritePlatformService`
+
+TBD
+
+.Retry-able service function `postInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=586..627]
+----
+
+.Fallback function `fallbackPostInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=1402..1414]
+----
+
+.Retry configuration for `postInterest`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=186..191]
+----

fineract-doc/src/docs/en/index.adoc

@@ -39,6 +39,8 @@ include::chapters/development/index.adoc[leveloffset=+1]
 
 include::chapters/custom/index.adoc[leveloffset=+1]
 
+include::chapters/resilience/index.adoc[leveloffset=+1]
+
 include::chapters/security/index.adoc[leveloffset=+1]
 
 include::chapters/testing/index.adoc[leveloffset=+1]

fineract-provider/dependencies.gradle

@@ -84,6 +84,8 @@ dependencies {
             'org.springdoc:springdoc-openapi-common',
             'org.springdoc:springdoc-openapi-security',
             'org.mapstruct:mapstruct',
+
+            'io.github.resilience4j:resilience4j-spring-boot2',
             )
 
     implementation ('org.apache.commons:commons-email') {

fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java

@@ -19,8 +19,6 @@
 package org.apache.fineract.commands.service;
 
 import com.google.gson.JsonElement;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import java.security.SecureRandom;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.commands.domain.CommandSource;
@@ -28,16 +26,12 @@ import org.apache.fineract.commands.domain.CommandSourceRepository;
 import org.apache.fineract.commands.domain.CommandWrapper;
 import org.apache.fineract.commands.exception.CommandNotAwaitingApprovalException;
 import org.apache.fineract.commands.exception.CommandNotFoundException;
-import org.apache.fineract.commands.exception.RollbackTransactionAsCommandIsNotApprovedByCheckerException;
 import org.apache.fineract.infrastructure.core.api.JsonCommand;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
 import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper;
-import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
 import org.apache.fineract.infrastructure.jobs.service.SchedulerJobRunnerReadService;
 import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
 import org.apache.fineract.useradministration.domain.AppUser;
-import org.springframework.dao.CannotAcquireLockException;
-import org.springframework.orm.ObjectOptimisticLockingFailureException;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -46,8 +40,6 @@ import org.springframework.transaction.annotation.Transactional;
 @Slf4j
 public class PortfolioCommandSourceWritePlatformServiceImpl implements PortfolioCommandSourceWritePlatformService {
 
-    private static final SecureRandom RANDOM = new SecureRandom();
-
     private final PlatformSecurityContext context;
     private final CommandSourceRepository commandSourceRepository;
     private final FromJsonHelper fromApiJsonHelper;
@@ -55,12 +47,10 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio
     private final SchedulerJobRunnerReadService schedulerJobRunnerReadService;
 
     @Override
-    @SuppressWarnings("AvoidHidingCauseException")
-    @SuppressFBWarnings(value = {
-            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
     public CommandProcessingResult logCommandSource(final CommandWrapper wrapper) {
 
         boolean isApprovedByChecker = false;
+
         // check if is update of own account details
         if (wrapper.isUpdateOfOwnUserDetails(this.context.authenticatedUser(wrapper).getId())) {
             // then allow this operation to proceed.
@@ -76,47 +66,13 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio
         validateIsUpdateAllowed();
 
         final String json = wrapper.getJson();
-        CommandProcessingResult result = null;
-        JsonCommand command;
-        int numberOfRetries = 0;
-        int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
-        int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
         final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json);
-        command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(),
+        JsonCommand command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(),
                 wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(),
                 wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(),
                 wrapper.getOrganisationCreditBureauId(), wrapper.getJobName());
-        while (numberOfRetries <= maxNumberOfRetries) {
-            try {
-                result = this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker);
-                numberOfRetries = maxNumberOfRetries + 1;
-            } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
-                log.debug("The following command {} has been retried  {} time(s)", command.json(), numberOfRetries);
-                /***
-                 * Fail if the transaction has been retired for maxNumberOfRetries
-                 **/
-                if (numberOfRetries >= maxNumberOfRetries) {
-                    log.warn("The following command {} has been retried for the max allowed attempts of {} and will be rolled back",
-                            command.json(), numberOfRetries);
-                    throw exception;
-                }
-                /***
-                 * Else sleep for a random time (between 1 to 10 seconds) and continue
-                 **/
-                try {
-                    int randomNum = RANDOM.nextInt(maxIntervalBetweenRetries + 1);
-                    Thread.sleep(1000 + (randomNum * 1000));
-                    numberOfRetries = numberOfRetries + 1;
-                } catch (InterruptedException e) {
-                    throw exception;
-                }
-            } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) {
-                numberOfRetries = maxNumberOfRetries + 1;
-                result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult());
-            }
-        }
 
-        return result;
+        return this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker);
     }
 
     @Override
@@ -168,9 +124,8 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio
         return commandSourceInput;
     }
 
-    private boolean validateIsUpdateAllowed() {
-        return this.schedulerJobRunnerReadService.isUpdatesAllowed();
-
+    private void validateIsUpdateAllowed() {
+        this.schedulerJobRunnerReadService.isUpdatesAllowed();
     }
 
     @Override

fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java

@@ -20,6 +20,7 @@ package org.apache.fineract.commands.service;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+import io.github.resilience4j.retry.annotation.Retry;
 import java.lang.reflect.Type;
 import java.time.Instant;
 import java.util.HashMap;
@@ -69,8 +70,9 @@ public class SynchronousCommandProcessingService implements CommandProcessingSer
     private final IdempotencyKeyGenerator idempotencyKeyGenerator;
     private final FineractProperties fineractProperties;
 
-    @Transactional
     @Override
+    @Transactional
+    @Retry(name = "executeCommand", fallbackMethod = "fallbackExecuteCommand")
     public CommandProcessingResult executeCommand(final CommandWrapper wrapper, final JsonCommand command,
             final boolean isApprovedByChecker) {
 
@@ -161,6 +163,15 @@ public class SynchronousCommandProcessingService implements CommandProcessingSer
                 .withEntityId(commandSourceResult.getResourceId()).build();
     }
 
+    @SuppressWarnings("unused")
+    private CommandProcessingResult fallbackExecuteCommand(Exception e) throws Exception {
+        if (e instanceof RollbackTransactionAsCommandIsNotApprovedByCheckerException ex) {
+            return logCommand(ex.getCommandSourceResult());
+        }
+
+        throw e;
+    }
+
     private NewCommandSourceHandler findCommandHandler(final CommandWrapper wrapper) {
         NewCommandSourceHandler handler;
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java

@@ -54,8 +54,6 @@ public class FineractPlatformTenantConnection implements Serializable {
     private final int suspectTimeout;
     private final int timeBetweenEvictionRunsMillis;
     private final int minEvictableIdleTimeMillis;
-    private final int maxRetriesOnDeadlock;
-    private final int maxIntervalBetweenRetries;
     private final boolean testOnBorrow;
 
     public FineractPlatformTenantConnection(final Long connectionId, final String schemaName, String schemaServer,
@@ -63,10 +61,9 @@ public class FineractPlatformTenantConnection implements Serializable {
             final String schemaPassword, final boolean autoUpdateEnabled, final int initialSize, final long validationInterval,
             final boolean removeAbandoned, final int removeAbandonedTimeout, final boolean logAbandoned,
             final int abandonWhenPercentageFull, final int maxActive, final int minIdle, final int maxIdle, final int suspectTimeout,
-            final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis, final int maxRetriesOnDeadlock,
-            final int maxIntervalBetweenRetries, final boolean tesOnBorrow, final String readOnlySchemaServer,
-            final String readOnlySchemaServerPort, final String readOnlySchemaName, final String readOnlySchemaUsername,
-            final String readOnlySchemaPassword, final String readOnlySchemaConnectionParameters) {
+            final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis, final boolean tesOnBorrow,
+            final String readOnlySchemaServer, final String readOnlySchemaServerPort, final String readOnlySchemaName,
+            final String readOnlySchemaUsername, final String readOnlySchemaPassword, final String readOnlySchemaConnectionParameters) {
 
         this.connectionId = connectionId;
         this.schemaName = schemaName;
@@ -88,8 +85,6 @@ public class FineractPlatformTenantConnection implements Serializable {
         this.suspectTimeout = suspectTimeout;
         this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis;
         this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis;
-        this.maxRetriesOnDeadlock = maxRetriesOnDeadlock;
-        this.maxIntervalBetweenRetries = maxIntervalBetweenRetries;
         this.testOnBorrow = tesOnBorrow;
         this.readOnlySchemaServer = readOnlySchemaServer;
         this.readOnlySchemaServerPort = readOnlySchemaServerPort;
@@ -171,14 +166,6 @@ public class FineractPlatformTenantConnection implements Serializable {
         return this.minEvictableIdleTimeMillis;
     }
 
-    public int getMaxRetriesOnDeadlock() {
-        return this.maxRetriesOnDeadlock;
-    }
-
-    public int getMaxIntervalBetweenRetries() {
-        return this.maxIntervalBetweenRetries;
-    }
-
     public boolean isTestOnBorrow() {
         return testOnBorrow;
     }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java

@@ -18,6 +18,7 @@
  */
 package org.apache.fineract.infrastructure.jobs.service;
 
+import io.github.resilience4j.retry.annotation.Retry;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -133,6 +134,7 @@ public class SchedularWritePlatformServiceJpaRepositoryImpl implements Schedular
 
     @Transactional
     @Override
+    @Retry(name = "processJobDetailForExecution", fallbackMethod = "fallbackProcessJobDetailForExecution")
     public boolean processJobDetailForExecution(final String jobKey, final String triggerType) {
         boolean isStopExecution = false;
         final ScheduledJobDetail scheduledJobDetail = this.scheduledJobDetailsRepository.findByJobKeyWithLock(jobKey);
@@ -151,4 +153,9 @@ public class SchedularWritePlatformServiceJpaRepositoryImpl implements Schedular
         return isStopExecution;
     }
 
+    @SuppressWarnings("unused")
+    private boolean fallbackProcessJobDetailForExecution(Exception e) {
+        return false;
+    }
+
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedulerTriggerListener.java

@@ -18,11 +18,10 @@
  */
 package org.apache.fineract.infrastructure.jobs.service;
 
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.util.HashMap;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
 import org.apache.fineract.infrastructure.businessdate.service.BusinessDateReadPlatformService;
 import org.apache.fineract.infrastructure.core.domain.ActionContext;
@@ -34,19 +33,15 @@ import org.quartz.JobKey;
 import org.quartz.Trigger;
 import org.quartz.Trigger.CompletedExecutionInstruction;
 import org.quartz.TriggerListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Isolation;
 import org.springframework.transaction.annotation.Transactional;
 
-@Component
+@Slf4j
 @RequiredArgsConstructor
+@Component
 public class SchedulerTriggerListener implements TriggerListener {
 
-    private static final Logger LOG = LoggerFactory.getLogger(SchedulerTriggerListener.class);
-    private static final SecureRandom random = new SecureRandom();
-
     private final SchedularWritePlatformService schedularService;
     private final TenantDetailsService tenantDetailsService;
 
@@ -59,12 +54,10 @@ public class SchedulerTriggerListener implements TriggerListener {
 
     @Override
     public void triggerFired(Trigger trigger, JobExecutionContext context) {
-        LOG.debug("triggerFired() trigger={}, context={}", trigger, context);
+        log.debug("triggerFired() trigger={}, context={}", trigger, context);
     }
 
     @Override
-    @SuppressFBWarnings(value = {
-            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
     @Transactional(isolation = Isolation.READ_COMMITTED)
     public boolean vetoJobExecution(final Trigger trigger, final JobExecutionContext context) {
         final String tenantIdentifier = trigger.getJobDataMap().getString(SchedulerServiceConstants.TENANT_IDENTIFIER);
@@ -79,43 +72,22 @@ public class SchedulerTriggerListener implements TriggerListener {
         if (context.getMergedJobDataMap().containsKey(SchedulerServiceConstants.TRIGGER_TYPE_REFERENCE)) {
             triggerType = context.getMergedJobDataMap().getString(SchedulerServiceConstants.TRIGGER_TYPE_REFERENCE);
         }
-        Integer maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
-        Integer maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
-        Integer numberOfRetries = 0;
-        boolean vetoJob = false;
-        while (numberOfRetries <= maxNumberOfRetries) {
-            try {
-                vetoJob = this.schedularService.processJobDetailForExecution(jobKey, triggerType);
-                numberOfRetries = maxNumberOfRetries + 1;
-            } catch (Exception exception) { // Adding generic exception as it
-                                            // depends on JPA provider
-                LOG.warn("vetoJobExecution() not able to acquire the lock to update job running status at retry {} (of {}) for JobKey: {}",
-                        numberOfRetries, maxNumberOfRetries, jobKey, exception);
-                try {
-                    int randomNum = random.nextInt(maxIntervalBetweenRetries + 1);
-                    Thread.sleep(1000 + (randomNum * 1000));
-                    numberOfRetries = numberOfRetries + 1;
-                } catch (InterruptedException e) {
-                    LOG.error("vetoJobExecution() caught an InterruptedException", e);
-                }
-            }
-        }
+        boolean vetoJob = this.schedularService.processJobDetailForExecution(jobKey, triggerType);
         if (vetoJob) {
-            LOG.warn(
-                    "vetoJobExecution() WILL veto the execution (returning vetoJob == true; the job's execute method will NOT be called); "
-                            + "maxNumberOfRetries={}, tenant={}, jobKey={}, triggerType={}, trigger={}, context={}",
-                    maxNumberOfRetries, tenantIdentifier, jobKey, triggerType, trigger, context);
+            log.warn(
+                    "vetoJobExecution() WILL veto the execution (returning vetoJob == true; the job's execute method will NOT be called); tenant={}, jobKey={}, triggerType={}, trigger={}, context={}",
+                    tenantIdentifier, jobKey, triggerType, trigger, context);
         }
         return vetoJob;
     }
 
     @Override
     public void triggerMisfired(final Trigger trigger) {
-        LOG.error("triggerMisfired() trigger={}", trigger);
+        log.error("triggerMisfired() trigger={}", trigger);
     }
 
     @Override
     public void triggerComplete(Trigger trigger, JobExecutionContext context, CompletedExecutionInstruction triggerInstructionCode) {
-        LOG.debug("triggerComplete() trigger={}, context={}, completedExecutionInstruction={}", trigger, context, triggerInstructionCode);
+        log.debug("triggerComplete() trigger={}, context={}, completedExecutionInstruction={}", trigger, context, triggerInstructionCode);
     }
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/security/service/TenantMapper.java

@@ -37,7 +37,6 @@ public final class TenantMapper implements RowMapper<FineractPlatformTenant> {
             + " ts.pool_max_active as poolMaxActive, ts.pool_min_idle as poolMinIdle, ts.pool_max_idle as poolMaxIdle,"
             + " ts.pool_suspect_timeout as poolSuspectTimeout, ts.pool_time_between_eviction_runs_millis as poolTimeBetweenEvictionRunsMillis,"
             + " ts.pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis,"
-            + " ts.deadlock_max_retries as maxRetriesOnDeadlock," + " ts.deadlock_max_retry_interval as maxIntervalBetweenRetries,"
             + " ts.readonly_schema_server as readOnlySchemaServer, " + " ts.readonly_schema_server_port as readOnlySchemaServerPort, "
             + " ts.readonly_schema_name as readOnlySchemaName, " + " ts.readonly_schema_username as readOnlySchemaUsername, "
             + " ts.readonly_schema_password as readOnlySchemaPassword, "
@@ -99,26 +98,11 @@ public final class TenantMapper implements RowMapper<FineractPlatformTenant> {
         final int suspectTimeout = rs.getInt("poolSuspectTimeout");
         final int timeBetweenEvictionRunsMillis = rs.getInt("poolTimeBetweenEvictionRunsMillis");
         final int minEvictableIdleTimeMillis = rs.getInt("poolMinEvictableIdleTimeMillis");
-        int maxRetriesOnDeadlock = rs.getInt("maxRetriesOnDeadlock");
-        int maxIntervalBetweenRetries = rs.getInt("maxIntervalBetweenRetries");
-
-        maxRetriesOnDeadlock = bindValueInMinMaxRange(maxRetriesOnDeadlock, 0, 15);
-        maxIntervalBetweenRetries = bindValueInMinMaxRange(maxIntervalBetweenRetries, 1, 15);
 
         return new FineractPlatformTenantConnection(connectionId, schemaName, schemaServer, schemaServerPort, schemaConnectionParameters,
                 schemaUsername, schemaPassword, autoUpdateEnabled, initialSize, validationInterval, removeAbandoned, removeAbandonedTimeout,
                 logAbandoned, abandonWhenPercentageFull, maxActive, minIdle, maxIdle, suspectTimeout, timeBetweenEvictionRunsMillis,
-                minEvictableIdleTimeMillis, maxRetriesOnDeadlock, maxIntervalBetweenRetries, testOnBorrow, readOnlySchemaServer,
-                readOnlySchemaServerPort, readOnlySchemaName, readOnlySchemaUsername, readOnlySchemaPassword,
-                readOnlySchemaConnectionParameters);
-    }
-
-    private int bindValueInMinMaxRange(final int value, int min, int max) {
-        if (value < min) {
-            return min;
-        } else if (value > max) {
-            return max;
-        }
-        return value;
+                minEvictableIdleTimeMillis, testOnBorrow, readOnlySchemaServer, readOnlySchemaServerPort, readOnlySchemaName,
+                readOnlySchemaUsername, readOnlySchemaPassword, readOnlySchemaConnectionParameters);
     }
 }

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/jobs/recalculateinterestforloan/RecalculateInterestForLoanTasklet.java

@@ -18,7 +18,6 @@
  */
 package org.apache.fineract.portfolio.loanaccount.jobs.recalculateinterestforloan;
 
-import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,7 +31,6 @@ import java.util.concurrent.Future;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
 import org.apache.fineract.infrastructure.jobs.exception.JobExecutionException;
 import org.apache.fineract.organisation.office.data.OfficeData;
 import org.apache.fineract.organisation.office.exception.OfficeNotFoundException;
@@ -45,8 +43,6 @@ import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.core.step.tasklet.Tasklet;
 import org.springframework.batch.repeat.RepeatStatus;
-import org.springframework.dao.CannotAcquireLockException;
-import org.springframework.orm.ObjectOptimisticLockingFailureException;
 
 @Slf4j
 @RequiredArgsConstructor
@@ -56,7 +52,6 @@ public class RecalculateInterestForLoanTasklet implements Tasklet {
     private final LoanWritePlatformService loanWritePlatformService;
     private final RecalculateInterestPoster recalculateInterestPoster;
     private final OfficeReadPlatformService officeReadPlatformService;
-    private static final SecureRandom random = new SecureRandom();
 
     @Override
     public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
@@ -75,45 +70,16 @@ public class RecalculateInterestForLoanTasklet implements Tasklet {
 
             recalculateInterest(office, threadPoolSize, batchSize);
         } else {
-            int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
-            int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
             Collection<Long> loanIds = loanReadPlatformService.fetchLoansForInterestRecalculation();
-            int i = 0;
             if (!loanIds.isEmpty()) {
                 List<Throwable> errors = new ArrayList<>();
                 for (Long loanId : loanIds) {
                     log.debug("recalculateInterest: Loan ID = {}", loanId);
-                    int numberOfRetries = 0;
-                    while (numberOfRetries <= maxNumberOfRetries) {
-                        try {
-                            loanWritePlatformService.recalculateInterest(loanId);
-                            numberOfRetries = maxNumberOfRetries + 1;
-                        } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
-                            log.debug("Recalulate interest job has been retried {} time(s)", numberOfRetries);
-                            if (numberOfRetries >= maxNumberOfRetries) {
-                                log.error(
-                                        "Recalulate interest job has been retried for the max allowed attempts of {} and will be rolled back",
-                                        numberOfRetries);
-                                errors.add(exception);
-                                break;
-                            }
-                            try {
-                                int randomNum = random.nextInt(maxIntervalBetweenRetries + 1);
-                                Thread.sleep(1000 + (randomNum * 1000L));
-                                numberOfRetries = numberOfRetries + 1;
-                            } catch (InterruptedException e) {
-                                log.error("Interest recalculation for loans retry failed due to InterruptedException", e);
-                                errors.add(e);
-                                break;
-                            }
-                        } catch (Exception e) {
-                            log.error("Interest recalculation for loans failed for account {}", loanId, e);
-                            numberOfRetries = maxNumberOfRetries + 1;
-                            errors.add(e);
-                        }
-                        i++;
+                    try {
+                        loanWritePlatformService.recalculateInterest(loanId);
+                    } catch (Exception e) {
+                        errors.add(e);
                     }
-                    log.debug("recalculateInterest: Loans count {}", i);
                 }
                 if (!errors.isEmpty()) {
                     throw new JobExecutionException(errors);

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java

@@ -21,6 +21,7 @@ package org.apache.fineract.portfolio.loanaccount.service;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import io.github.resilience4j.retry.annotation.Retry;
 import java.math.BigDecimal;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
@@ -39,7 +40,6 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.fineract.accounting.journalentry.service.JournalEntryWritePlatformService;
-import org.apache.fineract.cob.domain.LoanAccountLockRepository;
 import org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledException;
 import org.apache.fineract.cob.service.LoanAccountLockService;
 import org.apache.fineract.infrastructure.codes.domain.CodeValue;
@@ -281,7 +281,6 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf
     private final PostDatedChecksRepository postDatedChecksRepository;
     private final LoanDisbursementDetailsRepository loanDisbursementDetailsRepository;
     private final LoanRepaymentScheduleInstallmentRepository loanRepaymentScheduleInstallmentRepository;
-    private final LoanAccountLockRepository loanAccountLockRepository;
     private final LoanAccountLockService loanAccountLockService;
 
     private static boolean isPartOfThisInstallment(LoanCharge loanCharge, LoanRepaymentScheduleInstallment e) {
@@ -3217,6 +3216,7 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf
 
     @Transactional
     @Override
+    @Retry(name = "recalculateInterest", fallbackMethod = "fallbackRecalculateInterest")
     public void recalculateInterest(final long loanId) {
         Loan loan = this.loanAssembler.assembleFrom(loanId);
         LocalDate recalculateFrom = loan.fetchInterestRecalculateFromDate();
@@ -3252,6 +3252,18 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf
         return new CommandProcessingResultBuilder().withLoanId(loanId).build();
     }
 
+    @SuppressWarnings("unused")
+    private void fallbackRecalculateInterest(Throwable t) {
+        // NOTE: allow caller to catch the exceptions
+
+        if (t instanceof RuntimeException re) {
+            throw re;
+        }
+
+        // NOTE: wrap throwable only if really necessary
+        throw new RuntimeException(t);
+    }
+
     private void updateLoanTransaction(final Long loanTransactionId, final LoanTransaction newLoanTransaction) {
         final AccountTransferTransaction transferTransaction = this.accountTransferRepository.findByToLoanTransactionId(loanTransactionId);
         if (transferTransaction != null) {

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/RecalculateInterestPoster.java

@@ -18,19 +18,14 @@
  */
 package org.apache.fineract.portfolio.loanaccount.service;
 
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.Callable;
-import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
 import org.apache.fineract.infrastructure.jobs.exception.JobExecutionException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Scope;
-import org.springframework.dao.CannotAcquireLockException;
-import org.springframework.orm.ObjectOptimisticLockingFailureException;
 import org.springframework.stereotype.Component;
 
 @Component
@@ -38,7 +33,6 @@ import org.springframework.stereotype.Component;
 public class RecalculateInterestPoster implements Callable<Void> {
 
     private static final Logger LOG = LoggerFactory.getLogger(RecalculateInterestPoster.class);
-    private static final SecureRandom random = new SecureRandom();
 
     private Collection<Long> loanIds;
     private LoanWritePlatformService loanWritePlatformService;
@@ -52,52 +46,16 @@ public class RecalculateInterestPoster implements Callable<Void> {
     }
 
     @Override
-    @SuppressFBWarnings(value = {
-            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
     public Void call() throws JobExecutionException {
-        Integer maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
-        Integer maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
-
-        int i = 0;
         if (!loanIds.isEmpty()) {
             List<Throwable> errors = new ArrayList<>();
             for (Long loanId : loanIds) {
                 LOG.debug("Loan ID {}", loanId);
-                Integer numberOfRetries = 0;
-                while (numberOfRetries <= maxNumberOfRetries) {
-                    try {
-                        this.loanWritePlatformService.recalculateInterest(loanId);
-                        numberOfRetries = maxNumberOfRetries + 1;
-                    } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
-                        LOG.debug("Recalculate interest job has been retried {} time(s)", numberOfRetries);
-                        // Fail if the transaction has been retired for
-                        // maxNumberOfRetries
-                        if (numberOfRetries >= maxNumberOfRetries) {
-                            LOG.error(
-                                    "Recalculate interest job has been retried for the max allowed attempts of {} and will be rolled back",
-                                    numberOfRetries);
-                            errors.add(exception);
-                            break;
-                        }
-                        // Else sleep for a random time (between 1 to 10
-                        // seconds) and continue
-                        try {
-                            int randomNum = random.nextInt(maxIntervalBetweenRetries + 1);
-                            Thread.sleep(1000 + (randomNum * 1000));
-                            numberOfRetries = numberOfRetries + 1;
-                        } catch (InterruptedException e) {
-                            LOG.error("Interest recalculation for loans retry failed due to InterruptedException", e);
-                            errors.add(e);
-                            break;
-                        }
-                    } catch (Exception e) {
-                        LOG.error("Interest recalculation for loans failed for account {}", loanId, e);
-                        numberOfRetries = maxNumberOfRetries + 1;
-                        errors.add(e);
-                    }
-                    i++;
+                try {
+                    loanWritePlatformService.recalculateInterest(loanId);
+                } catch (Exception e) {
+                    errors.add(e);
                 }
-                LOG.debug("Loans count {}", i);
             }
             if (!errors.isEmpty()) {
                 throw new JobExecutionException(errors);

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/jobs/postinterestforsavings/PostInterestForSavingTasklet.java

@@ -36,21 +36,15 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.domain.FineractContext;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
-import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
 import org.apache.fineract.portfolio.savings.data.SavingsAccountData;
-import org.apache.fineract.portfolio.savings.domain.SavingsAccountAssembler;
-import org.apache.fineract.portfolio.savings.domain.SavingsAccountRepositoryWrapper;
 import org.apache.fineract.portfolio.savings.service.SavingsAccountReadPlatformService;
-import org.apache.fineract.portfolio.savings.service.SavingsAccountWritePlatformService;
 import org.apache.fineract.portfolio.savings.service.SavingsSchedularInterestPoster;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.core.step.tasklet.Tasklet;
 import org.springframework.batch.repeat.RepeatStatus;
 import org.springframework.context.ApplicationContext;
-import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
-import org.springframework.transaction.support.TransactionTemplate;
 
 @RequiredArgsConstructor
 @Slf4j
@@ -59,15 +53,9 @@ public class PostInterestForSavingTasklet implements Tasklet {
 
     private final SavingsAccountReadPlatformService savingAccountReadPlatformService;
     private final ConfigurationDomainService configurationDomainService;
-    private final SavingsAccountWritePlatformService savingsAccountWritePlatformService;
-    private final SavingsAccountRepositoryWrapper savingsAccountRepository;
-    private final SavingsAccountAssembler savingAccountAssembler;
-    private final JdbcTemplate jdbcTemplate;
-    private final TransactionTemplate transactionTemplate;
     private final Queue<List<SavingsAccountData>> queue = new ArrayDeque<>();
     private final ApplicationContext applicationContext;
     private final int queueSize = 1;
-    private final PlatformSecurityContext securityContext;
 
     @Override
     public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
@@ -147,19 +135,11 @@ public class PostInterestForSavingTasklet implements Tasklet {
 
         for (long i = 0; i < loopCount; i++) {
             List<SavingsAccountData> subList = safeSubList(savingsAccounts, fromIndex, toIndex);
-            SavingsSchedularInterestPoster savingsSchedularInterestPoster = (SavingsSchedularInterestPoster) applicationContext
-                    .getBean("savingsSchedularInterestPoster");
+            SavingsSchedularInterestPoster savingsSchedularInterestPoster = applicationContext
+                    .getBean(SavingsSchedularInterestPoster.class);
             savingsSchedularInterestPoster.setSavingAccounts(subList);
-            savingsSchedularInterestPoster.setContext(ThreadLocalContextUtil.getContext());
-            savingsSchedularInterestPoster.setSavingsAccountWritePlatformService(savingsAccountWritePlatformService);
-            savingsSchedularInterestPoster.setSavingsAccountReadPlatformService(savingAccountReadPlatformService);
-            savingsSchedularInterestPoster.setSavingsAccountRepository(savingsAccountRepository);
-            savingsSchedularInterestPoster.setSavingAccountAssembler(savingAccountAssembler);
-            savingsSchedularInterestPoster.setJdbcTemplate(jdbcTemplate);
             savingsSchedularInterestPoster.setBackdatedTxnsAllowedTill(backdatedTxnsAllowedTill);
-            savingsSchedularInterestPoster.setTransactionTemplate(transactionTemplate);
-            savingsSchedularInterestPoster.setConfigurationDomainService(configurationDomainService);
-            savingsSchedularInterestPoster.setPlatformSecurityContext(securityContext);
+            savingsSchedularInterestPoster.setContext(ThreadLocalContextUtil.getContext());
 
             posters.add(savingsSchedularInterestPoster);
 

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java

@@ -31,6 +31,7 @@ import static org.apache.fineract.portfolio.savings.SavingsApiConstants.withdraw
 
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import io.github.resilience4j.retry.annotation.Retry;
 import java.math.BigDecimal;
 import java.math.MathContext;
 import java.time.LocalDate;
@@ -46,6 +47,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.fineract.accounting.journalentry.service.JournalEntryWritePlatformService;
 import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
@@ -68,7 +71,6 @@ import org.apache.fineract.infrastructure.event.business.domain.savings.SavingsP
 import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
 import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
 import org.apache.fineract.organisation.holiday.domain.HolidayRepositoryWrapper;
-import org.apache.fineract.organisation.monetary.domain.ApplicationCurrencyRepositoryWrapper;
 import org.apache.fineract.organisation.monetary.domain.Money;
 import org.apache.fineract.organisation.monetary.domain.MoneyHelper;
 import org.apache.fineract.organisation.office.domain.Office;
@@ -125,18 +127,16 @@ import org.apache.fineract.portfolio.savings.exception.TransactionUpdateNotAllow
 import org.apache.fineract.portfolio.transfer.api.TransferApiConstants;
 import org.apache.fineract.useradministration.domain.AppUser;
 import org.apache.fineract.useradministration.domain.AppUserRepositoryWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort;
-import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
+@Slf4j
+@RequiredArgsConstructor
 @Service
 public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements SavingsAccountWritePlatformService {
 
@@ -149,7 +149,6 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
     private final SavingsAccountTransactionDataValidator savingsAccountTransactionDataValidator;
     private final SavingsAccountChargeDataValidator savingsAccountChargeDataValidator;
     private final PaymentDetailWritePlatformService paymentDetailWritePlatformService;
-    private final ApplicationCurrencyRepositoryWrapper applicationCurrencyRepositoryWrapper;
     private final JournalEntryWritePlatformService journalEntryWritePlatformService;
     private final SavingsAccountDomainService savingsAccountDomainService;
     private final NoteRepository noteRepository;
@@ -166,63 +165,8 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
     private final StandingInstructionRepository standingInstructionRepository;
     private final BusinessEventNotifierService businessEventNotifierService;
     private final GSIMRepositoy gsimRepository;
-    private final JdbcTemplate jdbcTemplate;
     private final SavingsAccountInterestPostingService savingsAccountInterestPostingService;
 
-    @Autowired
-    public SavingsAccountWritePlatformServiceJpaRepositoryImpl(final PlatformSecurityContext context,
-            final SavingsAccountRepositoryWrapper savingAccountRepositoryWrapper,
-            final SavingsAccountTransactionRepository savingsAccountTransactionRepository,
-            final SavingsAccountAssembler savingAccountAssembler,
-            final SavingsAccountTransactionDataValidator savingsAccountTransactionDataValidator,
-            final SavingsAccountChargeDataValidator savingsAccountChargeDataValidator,
-            final PaymentDetailWritePlatformService paymentDetailWritePlatformService,
-            final ApplicationCurrencyRepositoryWrapper applicationCurrencyRepositoryWrapper,
-            final JournalEntryWritePlatformService journalEntryWritePlatformService,
-            final SavingsAccountDomainService savingsAccountDomainService, final NoteRepository noteRepository,
-            final AccountTransfersReadPlatformService accountTransfersReadPlatformService, final HolidayRepositoryWrapper holidayRepository,
-            final WorkingDaysRepositoryWrapper workingDaysRepository,
-            final AccountAssociationsReadPlatformService accountAssociationsReadPlatformService,
-            final ChargeRepositoryWrapper chargeRepository, final SavingsAccountChargeRepositoryWrapper savingsAccountChargeRepository,
-            final SavingsAccountDataValidator fromApiJsonDeserializer, final StaffRepositoryWrapper staffRepository,
-            final ConfigurationDomainService configurationDomainService,
-            final DepositAccountOnHoldTransactionRepository depositAccountOnHoldTransactionRepository,
-            final EntityDatatableChecksWritePlatformService entityDatatableChecksWritePlatformService,
-            final AppUserRepositoryWrapper appuserRepository, final StandingInstructionRepository standingInstructionRepository,
-            final BusinessEventNotifierService businessEventNotifierService, final GSIMRepositoy gsimRepository,
-            final JdbcTemplate jdbcTemplate, final SavingsAccountInterestPostingService savingsAccountInterestPostingService) {
-        this.context = context;
-        this.savingAccountRepositoryWrapper = savingAccountRepositoryWrapper;
-        this.savingsAccountTransactionRepository = savingsAccountTransactionRepository;
-        this.savingAccountAssembler = savingAccountAssembler;
-        this.savingsAccountTransactionDataValidator = savingsAccountTransactionDataValidator;
-        this.savingsAccountChargeDataValidator = savingsAccountChargeDataValidator;
-        this.paymentDetailWritePlatformService = paymentDetailWritePlatformService;
-        this.applicationCurrencyRepositoryWrapper = applicationCurrencyRepositoryWrapper;
-        this.journalEntryWritePlatformService = journalEntryWritePlatformService;
-        this.savingsAccountDomainService = savingsAccountDomainService;
-        this.noteRepository = noteRepository;
-        this.accountTransfersReadPlatformService = accountTransfersReadPlatformService;
-        this.accountAssociationsReadPlatformService = accountAssociationsReadPlatformService;
-        this.chargeRepository = chargeRepository;
-        this.savingsAccountChargeRepository = savingsAccountChargeRepository;
-        this.holidayRepository = holidayRepository;
-        this.workingDaysRepository = workingDaysRepository;
-        this.fromApiJsonDeserializer = fromApiJsonDeserializer;
-        this.staffRepository = staffRepository;
-        this.configurationDomainService = configurationDomainService;
-        this.depositAccountOnHoldTransactionRepository = depositAccountOnHoldTransactionRepository;
-        this.entityDatatableChecksWritePlatformService = entityDatatableChecksWritePlatformService;
-        this.appuserRepository = appuserRepository;
-        this.standingInstructionRepository = standingInstructionRepository;
-        this.businessEventNotifierService = businessEventNotifierService;
-        this.gsimRepository = gsimRepository;
-        this.jdbcTemplate = jdbcTemplate;
-        this.savingsAccountInterestPostingService = savingsAccountInterestPostingService;
-    }
-
-    private static final Logger LOG = LoggerFactory.getLogger(SavingsAccountWritePlatformServiceJpaRepositoryImpl.class);
-
     @Transactional
     @Override
     public CommandProcessingResult gsimActivate(final Long gsimId, final JsonCommand command) {
@@ -355,7 +299,7 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
 
         if (account.getGsim() != null) {
             isGsim = true;
-            LOG.debug("is gsim");
+            log.debug("is gsim");
         }
         checkClientOrGroupActive(account);
 
@@ -376,16 +320,16 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
 
         if (isGsim && (deposit.getId() != null)) {
 
-            LOG.debug("Deposit account has been created: {} ", deposit);
+            log.debug("Deposit account has been created: {} ", deposit);
 
             GroupSavingsIndividualMonitoring gsim = gsimRepository.findById(account.getGsim().getId()).orElseThrow();
-            LOG.debug("parent deposit : {} ", gsim.getParentDeposit());
-            LOG.debug("child account : {} ", savingsId);
+            log.debug("parent deposit : {} ", gsim.getParentDeposit());
+            log.debug("child account : {} ", savingsId);
             BigDecimal currentBalance = gsim.getParentDeposit();
             BigDecimal newBalance = currentBalance.add(transactionAmount);
             gsim.setParentDeposit(newBalance);
             gsimRepository.save(gsim);
-            LOG.debug("balance after making deposit : {} ",
+            log.debug("balance after making deposit : {} ",
                     gsimRepository.findById(account.getGsim().getId()).orElseThrow().getParentDeposit());
 
         }
@@ -641,6 +585,7 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
 
     @Transactional
     @Override
+    @Retry(name = "postInterest", fallbackMethod = "fallbackPostInterest")
     public SavingsAccountData postInterest(SavingsAccountData savingsAccountData, final boolean postInterestAs,
             final LocalDate transactionDate, final boolean backdatedTxnsAllowedTill) {
 
@@ -662,14 +607,9 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
                 postInterestOnDate = transactionDate;
             }
 
-            long startPosting = System.currentTimeMillis();
-            LOG.debug("Interest Posting Start Here at {}", startPosting);
-
             savingsAccountData = this.savingsAccountInterestPostingService.postInterest(mc, today, isInterestTransfer,
                     isSavingsInterestPostingAtCurrentPeriodEnd, financialYearBeginningMonth, postInterestOnDate, backdatedTxnsAllowedTill,
                     savingsAccountData);
-            long endPosting = System.currentTimeMillis();
-            LOG.debug("Interest Posting Ends within {}", endPosting - startPosting);
 
             if (!backdatedTxnsAllowedTill) {
                 List<SavingsAccountTransactionData> transactions = savingsAccountData.getSavingsAccountTransactionData();
@@ -1459,6 +1399,19 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi
         }
     }
 
+    @SuppressWarnings("unused")
+    private SavingsAccountData fallbackPostInterest(SavingsAccountData savingsAccountData, boolean postInterestAs,
+            LocalDate transactionDate, boolean backdatedTxnsAllowedTill, Throwable t) {
+        // NOTE: allow caller to catch the exceptions
+
+        if (t instanceof RuntimeException re) {
+            throw re;
+        }
+
+        // NOTE: wrap throwable only if really necessary
+        throw new RuntimeException(t);
+    }
+
     @Transactional
     private SavingsAccountTransaction payCharge(final SavingsAccountCharge savingsAccountCharge, final LocalDate transactionDate,
             final BigDecimal amountPaid, final DateTimeFormatter formatter, final AppUser user, final boolean backdatedTxnsAllowedTill) {

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsSchedularInterestPoster.java

@@ -18,9 +18,7 @@
  */
 package org.apache.fineract.portfolio.savings.service;
 
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.math.BigDecimal;
-import java.security.SecureRandom;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -30,11 +28,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import lombok.RequiredArgsConstructor;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.accounting.journalentry.domain.JournalEntryType;
-import org.apache.fineract.batch.command.CommandStrategyProvider;
-import org.apache.fineract.batch.service.ResolutionHelper;
-import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
 import org.apache.fineract.infrastructure.core.domain.FineractContext;
 import org.apache.fineract.infrastructure.core.service.DateUtils;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
@@ -43,119 +40,61 @@ import org.apache.fineract.infrastructure.security.service.PlatformSecurityConte
 import org.apache.fineract.portfolio.savings.data.SavingsAccountData;
 import org.apache.fineract.portfolio.savings.data.SavingsAccountSummaryData;
 import org.apache.fineract.portfolio.savings.data.SavingsAccountTransactionData;
-import org.apache.fineract.portfolio.savings.domain.SavingsAccountAssembler;
-import org.apache.fineract.portfolio.savings.domain.SavingsAccountRepositoryWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Scope;
-import org.springframework.dao.CannotAcquireLockException;
 import org.springframework.dao.DataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.orm.ObjectOptimisticLockingFailureException;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Isolation;
 import org.springframework.transaction.annotation.Transactional;
-import org.springframework.transaction.support.TransactionTemplate;
 
 /**
  * @author manoj
  */
 
+@Slf4j
+@RequiredArgsConstructor
+@Setter
 @Component
 @Scope("prototype")
-@Setter
 public class SavingsSchedularInterestPoster implements Callable<Void> {
 
-    private static final Logger LOG = LoggerFactory.getLogger(SavingsSchedularInterestPoster.class);
-    private static final SecureRandom random = new SecureRandom();
     private static final String SAVINGS_TRANSACTION_IDENTIFIER = "S";
 
+    private final SavingsAccountWritePlatformService savingsAccountWritePlatformService;
+    private final List<SavingsAccountData> savingsAccountDataList = new ArrayList<>();
+    private final JdbcTemplate jdbcTemplate;
+    private final SavingsAccountReadPlatformService savingsAccountReadPlatformService;
+    private final PlatformSecurityContext platformSecurityContext;
     private Collection<SavingsAccountData> savingAccounts;
-    private SavingsAccountWritePlatformService savingsAccountWritePlatformService;
-    private SavingsAccountRepositoryWrapper savingsAccountRepository;
-    private SavingsAccountAssembler savingAccountAssembler;
     private FineractContext context;
-    private ConfigurationDomainService configurationDomainService;
     private boolean backdatedTxnsAllowedTill;
-    private List<SavingsAccountData> savingsAccountDataList = new ArrayList<>();
-    private JdbcTemplate jdbcTemplate;
-    private TransactionTemplate transactionTemplate;
-    private CommandStrategyProvider strategyProvider;
-    private ResolutionHelper resolutionHelper;
-    private SavingsAccountReadPlatformService savingsAccountReadPlatformService;
-    private PlatformSecurityContext platformSecurityContext;
 
     @Override
-    @SuppressFBWarnings(value = {
-            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
     @Transactional(isolation = Isolation.READ_UNCOMMITTED, rollbackFor = Exception.class)
     public Void call() throws org.apache.fineract.infrastructure.jobs.exception.JobExecutionException {
         ThreadLocalContextUtil.init(this.context);
-        Integer maxNumberOfRetries = this.context.getTenantContext().getConnection().getMaxRetriesOnDeadlock();
-        Integer maxIntervalBetweenRetries = this.context.getTenantContext().getConnection().getMaxIntervalBetweenRetries();
 
-        // List<BatchResponse> responseList = new ArrayList<>();
-        long start = System.currentTimeMillis();
-        LOG.debug("Thread Execution Started at {}", start);
-
-        List<Throwable> errors = new ArrayList<>();
-        int i = 0;
         if (!savingAccounts.isEmpty()) {
+            List<Throwable> errors = new ArrayList<>();
             for (SavingsAccountData savingsAccountData : savingAccounts) {
-                Integer numberOfRetries = 0;
-                while (numberOfRetries <= maxNumberOfRetries) {
-                    try {
-                        boolean postInterestAsOn = false;
-                        LocalDate transactionDate = null;
-                        long startPosting = System.currentTimeMillis();
-                        SavingsAccountData savingsAccountDataRet = savingsAccountWritePlatformService.postInterest(savingsAccountData,
-                                postInterestAsOn, transactionDate, backdatedTxnsAllowedTill);
-                        long endPosting = System.currentTimeMillis();
-                        savingsAccountDataList.add(savingsAccountDataRet);
-
-                        LOG.debug("Posting Completed Within {}", endPosting - startPosting);
-
-                        numberOfRetries = maxNumberOfRetries + 1;
-                    } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
-                        LOG.debug("Interest posting job for savings ID {} has been retried {} time(s)", savingsAccountData.getId(),
-                                numberOfRetries);
-                        // Fail if the transaction has been retired for
-                        // maxNumberOfRetries
-                        if (numberOfRetries >= maxNumberOfRetries) {
-                            LOG.error(
-                                    "Interest posting job for savings ID {} has been retried for the max allowed attempts of {} and will be rolled back",
-                                    savingsAccountData.getId(), numberOfRetries);
-                            errors.add(exception);
-                            break;
-                        }
-                        // Else sleep for a random time (between 1 to 10
-                        // seconds) and continue
-                        try {
-                            int randomNum = random.nextInt(maxIntervalBetweenRetries + 1);
-                            Thread.sleep(1000 + (randomNum * 1000));
-                            numberOfRetries = numberOfRetries + 1;
-                        } catch (InterruptedException e) {
-                            LOG.error("Interest posting job for savings retry failed due to InterruptedException", e);
-                            errors.add(e);
-                            break;
-                        }
-                    } catch (Exception e) {
-                        LOG.error("Interest posting job for savings failed for account {}", savingsAccountData.getId(), e);
-                        numberOfRetries = maxNumberOfRetries + 1;
-                        errors.add(e);
-                    }
+                boolean postInterestAsOn = false;
+                LocalDate transactionDate = null;
+                try {
+                    SavingsAccountData savingsAccountDataRet = savingsAccountWritePlatformService.postInterest(savingsAccountData,
+                            postInterestAsOn, transactionDate, backdatedTxnsAllowedTill);
+                    savingsAccountDataList.add(savingsAccountDataRet);
+                } catch (Exception e) {
+                    errors.add(e);
                 }
-                i++;
             }
-
             if (errors.isEmpty()) {
                 try {
                     batchUpdate(savingsAccountDataList);
                 } catch (DataAccessException exception) {
-                    LOG.error("Batch update failed due to DataAccessException", exception);
+                    log.error("Batch update failed due to DataAccessException", exception);
                     errors.add(exception);
                 } catch (NullPointerException exception) {
-                    LOG.error("Batch update failed due to NullPointerException", exception);
+                    log.error("Batch update failed due to NullPointerException", exception);
                     errors.add(exception);
                 }
             }
@@ -165,9 +104,6 @@ public class SavingsSchedularInterestPoster implements Callable<Void> {
             }
         }
 
-        long end = System.currentTimeMillis();
-        LOG.debug("Time To Finish the batch {} by thread {} for accounts {}", end - start, Thread.currentThread().getId(),
-                savingAccounts.size());
         return null;
     }
 
@@ -297,10 +233,10 @@ public class SavingsSchedularInterestPoster implements Callable<Void> {
             this.jdbcTemplate.batchUpdate(queryForSavingsUpdate, paramsForSavingsSummary);
             this.jdbcTemplate.batchUpdate(queryForTransactionInsertion, paramsForTransactionInsertion);
             this.jdbcTemplate.batchUpdate(queryForTransactionUpdate, paramsForTransactionUpdate);
-            LOG.debug("`Total No Of Interest Posting:` {}", transRefNo.size());
+            log.debug("`Total No Of Interest Posting:` {}", transRefNo.size());
             List<SavingsAccountTransactionData> savingsAccountTransactionDataList = fetchTransactionsFromIds(transRefNo);
             if (savingsAccountDataList != null) {
-                LOG.debug("Fetched Transactions from DB: {}", savingsAccountTransactionDataList.size());
+                log.debug("Fetched Transactions from DB: {}", savingsAccountTransactionDataList.size());
             }
 
             HashMap<String, SavingsAccountTransactionData> savingsAccountTransactionMap = new HashMap<>();

fineract-provider/src/main/resources/application.properties

@@ -73,17 +73,18 @@ fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACT
 # Logging pattern for the console
 logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
 
-management.health.jms.enabled=false
+management.health.jms.enabled=${FINERACT_MANAGEMENT_HEALTH_JMS_ENABLED:false}
 
 # FINERACT 1296
 management.endpoint.health.probes.enabled=true
 management.health.livenessState.enabled=true
 management.health.readinessState.enabled=true
 
+management.health.ratelimiters.enabled=${FINERACT_MANAGEMENT_HEALTH_RATELIMITERS_ENABLED:false}
+
 # FINERACT-883
 management.info.git.mode=FULL
-management.endpoints.web.exposure.include=health,info
-
+management.endpoints.web.exposure.include=${FINERACT_MANAGEMENT_ENDPOINT_WEB_EXPOSURE_INCLUDE:health,info}
 # FINERACT-914
 server.forward-headers-strategy=framework
 server.port=${FINERACT_SERVER_PORT:8443}
@@ -164,3 +165,26 @@ spring.main.allow-bean-definition-overriding=true
 spring.batch.initialize-schema=NEVER
 # Disabling Spring Batch jobs on startup
 spring.batch.job.enabled=false
+
+resilience4j.retry.instances.executeCommand.max-attempts=${FINERACT_COMMAND_PROCESSING_RETRY_MAX_ATTEMPTS:3}
+resilience4j.retry.instances.executeCommand.wait-duration=${FINERACT_COMMAND_PROCESSING_RETRY_WAIT_DURATION:1s}
+resilience4j.retry.instances.executeCommand.enable-exponential-backoff=${FINERACT_COMMAND_PROCESSING_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true}
+resilience4j.retry.instances.executeCommand.exponential-backoff-multiplier=${FINERACT_COMMAND_PROCESSING_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2}
+resilience4j.retry.instances.executeCommand.retryExceptions=${FINERACT_COMMAND_PROCESSING_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException}
+
+resilience4j.retry.instances.processJobDetailForExecution.max-attempts=${FINERACT_PROCESS_JOB_DETAIL_RETRY_MAX_ATTEMPTS:3}
+resilience4j.retry.instances.processJobDetailForExecution.wait-duration=${FINERACT_PROCESS_JOB_DETAIL_RETRY_WAIT_DURATION:1s}
+resilience4j.retry.instances.processJobDetailForExecution.enable-exponential-backoff=${FINERACT_PROCESS_JOB_DETAIL_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true}
+resilience4j.retry.instances.processJobDetailForExecution.exponential-backoff-multiplier=${FINERACT_PROCESS_JOB_DETAIL_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2}
+
+resilience4j.retry.instances.recalculateInterest.max-attempts=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_MAX_ATTEMPTS:3}
+resilience4j.retry.instances.recalculateInterest.wait-duration=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_WAIT_DURATION:1s}
+resilience4j.retry.instances.recalculateInterest.enable-exponential-backoff=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true}
+resilience4j.retry.instances.recalculateInterest.exponential-backoff-multiplier=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2}
+resilience4j.retry.instances.recalculateInterest.retryExceptions=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException}
+
+resilience4j.retry.instances.postInterest.max-attempts=${FINERACT_PROCESS_POST_INTEREST_RETRY_MAX_ATTEMPTS:3}
+resilience4j.retry.instances.postInterest.wait-duration=${FINERACT_PROCESS_POST_INTEREST_RETRY_WAIT_DURATION:1s}
+resilience4j.retry.instances.postInterest.enable-exponential-backoff=${FINERACT_PROCESS_POST_INTEREST_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true}
+resilience4j.retry.instances.postInterest.exponential-backoff-multiplier=${FINERACT_PROCESS_POST_INTEREST_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2}
+resilience4j.retry.instances.postInterest.retryExceptions=${FINERACT_PROCESS_POST_INTEREST_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException}

fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml

@@ -25,4 +25,5 @@
      <include file="parts/0003_reset_postgresql_sequences.xml" relativeToChangelogFile="true"/>
      <include file="parts/0004_readonly_database_connection.xml" relativeToChangelogFile="true"/>
      <include file="parts/0005_jdbc_connection_string.xml" relativeToChangelogFile="true"/>
+     <include file="parts/0006_drop_retry_parameter_columns.xml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0006_drop_retry_parameter_columns.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet author="fineract" id="1" context="tenant_store_db">
+        <dropColumn tableName="tenant_server_connections" columnName="deadlock_max_retries" />
+        <dropColumn tableName="tenant_server_connections" columnName="deadlock_max_retry_interval" />
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/test/java/org/apache/fineract/commands/service/CommandServiceStepDefinitions.java

@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.commands.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.cucumber.java8.En;
+import io.github.resilience4j.retry.RetryRegistry;
+import io.github.resilience4j.retry.event.RetryEvent;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.persistence.Entity;
+import javax.persistence.Table;
+import org.apache.fineract.commands.domain.CommandSource;
+import org.apache.fineract.commands.domain.CommandWrapper;
+import org.apache.fineract.commands.exception.RollbackTransactionAsCommandIsNotApprovedByCheckerException;
+import org.apache.fineract.infrastructure.core.api.JsonCommand;
+import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.CannotAcquireLockException;
+import org.springframework.orm.ObjectOptimisticLockingFailureException;
+
+public class CommandServiceStepDefinitions implements En {
+
+    private static final Logger log = LoggerFactory.getLogger(CommandServiceStepDefinitions.class);
+
+    @Autowired
+    private CommandProcessingService processAndLogCommandService;
+
+    @Autowired
+    private RetryRegistry retryRegistry;
+
+    private PortfolioCommandSourceWritePlatformService commandSourceWritePlatformService;
+
+    private DummyCommand command;
+
+    private RetryEvent retryEvent;
+
+    public CommandServiceStepDefinitions() {
+        Given("/^A command source write service$/", () -> {
+            this.commandSourceWritePlatformService = new DummyCommandSourceWriteService(processAndLogCommandService);
+            this.command = new DummyCommand();
+            this.retryRegistry.retry("executeCommand").getEventPublisher().onRetry(event -> {
+                log.warn("... retry event: {}", event);
+
+                CommandServiceStepDefinitions.this.retryEvent = event;
+            });
+
+        });
+
+        When("/^The user executes the command via a command write service with exceptions$/", () -> {
+            try {
+                this.commandSourceWritePlatformService.logCommandSource(command);
+            } catch (Exception e) {
+                // TODO: this exception is OK for now; we need to fix the whole tenant based data source setup
+                log.warn("At the moment mocking data access is so incredibly hard... it's easier to just ignore this exception: {}",
+                        e.getMessage());
+            }
+        });
+
+        Then("/^The command processing service should fallback as expected$/", () -> {
+            assertNotNull(retryEvent);
+            assertEquals("executeCommand", retryEvent.getName());
+            assertEquals(2, retryEvent.getNumberOfRetryAttempts());
+        });
+
+        Then("/^The command processing service execute function should be called 3 times$/", () -> {
+            assertEquals(3, command.getCount());
+        });
+    }
+
+    public static class DummyCommand extends CommandWrapper {
+
+        private AtomicInteger counter = new AtomicInteger();
+
+        public DummyCommand() {
+            super(null, null, null, null, null, null, null, null, null, null, "{}", null, null, null, null, null, null,
+                    UUID.randomUUID().toString());
+        }
+
+        @Override
+        public String taskPermissionName() {
+            // NOTE: simulating a failure scenario that triggers retries; using this function, because it is the first
+            // called in the command processing service
+
+            int step = counter.incrementAndGet();
+
+            log.warn("Round: {}", step);
+
+            if (step == 1) {
+                throw new CannotAcquireLockException("BLOW IT UP!!!");
+            } else if (step == 2) {
+                throw new ObjectOptimisticLockingFailureException("Dummy", new RuntimeException("BLOW IT UP!!!"));
+            } else if (step == 3) {
+                throw new RollbackTransactionAsCommandIsNotApprovedByCheckerException(new DummyCommandSource());
+            }
+
+            return "dummy";
+        }
+
+        @Override
+        public String actionName() {
+            return "dummy";
+        }
+
+        public int getCount() {
+            return counter.get();
+        }
+    }
+
+    public static class DummyCommandSourceWriteService implements PortfolioCommandSourceWritePlatformService {
+
+        private final CommandProcessingService processAndLogCommandService;
+
+        public DummyCommandSourceWriteService(CommandProcessingService processAndLogCommandService) {
+            this.processAndLogCommandService = processAndLogCommandService;
+        }
+
+        @Override
+        public CommandProcessingResult logCommandSource(CommandWrapper wrapper) {
+            final String json = wrapper.getJson();
+            JsonCommand command = JsonCommand.from(json, null, null, wrapper.getEntityName(), wrapper.getEntityId(),
+                    wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(),
+                    wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(),
+                    wrapper.getOrganisationCreditBureauId(), wrapper.getJobName());
+
+            return this.processAndLogCommandService.executeCommand(wrapper, command, true);
+        }
+
+        @Override
+        public CommandProcessingResult approveEntry(Long id) {
+            return null;
+        }
+
+        @Override
+        public Long rejectEntry(Long id) {
+            return null;
+        }
+
+        @Override
+        public Long deleteEntry(Long makerCheckerId) {
+            return null;
+        }
+    }
+
+    @Entity
+    @Table(name = "m_portfolio_command_source")
+    public static class DummyCommandSource extends CommandSource {
+
+        public DummyCommandSource() {
+            setId(1L);
+        }
+    }
+}

fineract-provider/src/test/resources/application-test.properties

@@ -111,3 +111,26 @@ spring.main.allow-bean-definition-overriding=true
 spring.batch.initialize-schema=NEVER
 # Disabling Spring Batch jobs on startup
 spring.batch.job.enabled=false
+
+resilience4j.retry.instances.executeCommand.max-attempts=3
+resilience4j.retry.instances.executeCommand.wait-duration=1s
+resilience4j.retry.instances.executeCommand.enable-exponential-backoff=true
+resilience4j.retry.instances.executeCommand.exponential-backoff-multiplier=2
+resilience4j.retry.instances.executeCommand.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException
+
+resilience4j.retry.instances.processJobDetailForExecution.max-attempts=3
+resilience4j.retry.instances.processJobDetailForExecution.wait-duration=1s
+resilience4j.retry.instances.processJobDetailForExecution.enable-exponential-backoff=true
+resilience4j.retry.instances.processJobDetailForExecution.exponential-backoff-multiplier=2
+
+resilience4j.retry.instances.recalculateInterest.max-attempts=3
+resilience4j.retry.instances.recalculateInterest.wait-duration=1s
+resilience4j.retry.instances.recalculateInterest.enable-exponential-backoff=true
+resilience4j.retry.instances.recalculateInterest.exponential-backoff-multiplier=2
+resilience4j.retry.instances.recalculateInterest.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException
+
+resilience4j.retry.instances.postInterest.max-attempts=3
+resilience4j.retry.instances.postInterest.wait-duration=1s
+resilience4j.retry.instances.postInterest.enable-exponential-backoff=true
+resilience4j.retry.instances.postInterest.exponential-backoff-multiplier=2
+resilience4j.retry.instances.postInterest.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException

fineract-provider/src/test/resources/features/commands/commands.provider.feature

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-Feature: Commands Service
+Feature: Commands Provider
 
   @template
   Scenario Outline: Verify that command providers are injected

fineract-provider/src/test/resources/features/commands/commands.service.feature

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+Feature: Commands Service
+
+  Scenario: Verify that command source write service are working with fallback function
+    Given A command source write service
+    When The user executes the command via a command write service with exceptions
+    Then The command processing service should fallback as expected
+    Then The command processing service execute function should be called 3 times
+

fineract-provider/src/test/resources/logback.xml

@@ -34,6 +34,7 @@
     <logger name="org.springframework.transaction" level="INFO" />
     <logger name="org.springframework.data.convert" level="ERROR" />
     <logger name="org.springframework.http.converter.json" level="ERROR" />
+    <logger name="io.github.resilience4j.retry" level="DEBUG" />
 
     <root level="INFO">
         <appender-ref ref="STDOUT" />