FINERACT-1795: Improve resilience of command processing service

fineract-doc/src/docs/en/chapters/resilience/command.adoc

@@ -0,0 +1,23 @@
+= Command
+
+== `CommandProcessingService`
+
+TBD
+
+.Retry-able service function `executeCommand`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=73..151]
+----
+
+.Fallback function `fallbackExecuteCommand`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=166..174]
+----
+
+.Retry configuration for `executeCommand`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=169..174]
+----

fineract-doc/src/docs/en/chapters/resilience/index.adoc

@@ -0,0 +1,11 @@
+= Resilience
+
+include::intro.adoc[leveloffset=+1]
+
+include::command.adoc[leveloffset=+1]
+
+include::job.adoc[leveloffset=+1]
+
+include::loan.adoc[leveloffset=+1]
+
+include::saving.adoc[leveloffset=+1]

fineract-doc/src/docs/en/chapters/resilience/intro.adoc

@@ -0,0 +1,84 @@
+= Introduction Resilience
+
+Fineract had handcrafted retry loops in place for the longest time. A typical retry code would have looked like this:
+
+.Legacy retry code
+[source,java]
+----
+    @Override
+    @SuppressWarnings("AvoidHidingCauseException")
+    @SuppressFBWarnings(value = {
+            "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once")
+    public CommandProcessingResult logCommandSource(final CommandWrapper wrapper) {
+
+        boolean isApprovedByChecker = false;
+        // check if is update of own account details
+        if (wrapper.isUpdateOfOwnUserDetails(this.context.authenticatedUser(wrapper).getId())) {
+            // then allow this operation to proceed.
+            // maker checker doesnt mean anything here.
+            isApprovedByChecker = true; // set to true in case permissions have
+                                        // been maker-checker enabled by
+                                        // accident.
+        } else {
+            // if not user changing their own details - check user has
+            // permission to perform specific task.
+            this.context.authenticatedUser(wrapper).validateHasPermissionTo(wrapper.getTaskPermissionName());
+        }
+        validateIsUpdateAllowed();
+
+        final String json = wrapper.getJson();
+        CommandProcessingResult result = null;
+        JsonCommand command;
+        int numberOfRetries = 0; // <1>
+        int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock();
+        int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries();
+        final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json);
+        command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(),
+                wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(),
+                wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(),
+                wrapper.getOrganisationCreditBureauId(), wrapper.getJobName());
+        while (numberOfRetries <= maxNumberOfRetries) { // <2>
+            try {
+                result = this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker);
+                numberOfRetries = maxNumberOfRetries + 1; // <3>
+            } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
+                log.debug("The following command {} has been retried  {} time(s)", command.json(), numberOfRetries);
+                /***
+                 * Fail if the transaction has been retired for maxNumberOfRetries
+                 **/
+                if (numberOfRetries >= maxNumberOfRetries) {
+                    log.warn("The following command {} has been retried for the max allowed attempts of {} and will be rolled back",
+                            command.json(), numberOfRetries);
+                    throw exception;
+                }
+                /***
+                 * Else sleep for a random time (between 1 to 10 seconds) and continue
+                 **/
+                try {
+                    int randomNum = RANDOM.nextInt(maxIntervalBetweenRetries + 1);
+                    Thread.sleep(1000 + (randomNum * 1000));
+                    numberOfRetries = numberOfRetries + 1; // <4>
+                } catch (InterruptedException e) {
+                    throw exception;
+                }
+            } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) {
+                numberOfRetries = maxNumberOfRetries + 1; // <3>
+                result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult());
+            }
+        }
+
+        return result;
+    }
+----
+<1> counter
+<2> `while` loop
+<3> increment to abort
+<4> increment
+
+For better code quality and readability we introduced https://resilience4j.readme.io/docs[Resilience4j]:
+
+.Annotation based retry
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java[lines=49..76]
+----

fineract-doc/src/docs/en/chapters/resilience/job.adoc

@@ -0,0 +1,25 @@
+= Jobs
+
+== `SchedularWritePlatformService`
+
+WARNING: This service has a typo and should be called `SchedulerWritePlatformService`.
+
+TBD
+
+.Retry-able service function `processJobDetailForExecution`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=135..155]
+----
+
+.Fallback function `fallbackProcessJobDetailForExecution`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=156..159]
+----
+
+.Retry configuration for `processJobDetailForExecution`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=175..179]
+----

fineract-doc/src/docs/en/chapters/resilience/loan.adoc

@@ -0,0 +1,23 @@
+= Loan
+
+== `LoanWritePlatformService`
+
+TBD
+
+.Retry-able service function `recalculateInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3217..3247]
+----
+
+.Fallback function `fallbackRecalculateInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3255..3266]
+----
+
+.Retry configuration for `recalculateInterest`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=180..185]
+----

fineract-doc/src/docs/en/chapters/resilience/saving.adoc

@@ -0,0 +1,23 @@
+= Savings
+
+== `SavingsAccountWritePlatformService`
+
+TBD
+
+.Retry-able service function `postInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=586..627]
+----
+
+.Fallback function `fallbackPostInterest`
+[source,java]
+----
+include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=1402..1414]
+----
+
+.Retry configuration for `postInterest`
+[source,properties]
+----
+include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=186..191]
+----

fineract-doc/src/docs/en/index.adoc

@@ -39,6 +39,8 @@ include::chapters/development/index.adoc[leveloffset=+1]
 
 include::chapters/custom/index.adoc[leveloffset=+1]
 
+include::chapters/resilience/index.adoc[leveloffset=+1]
+
 include::chapters/security/index.adoc[leveloffset=+1]
 
 include::chapters/testing/index.adoc[leveloffset=+1]