diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle index 9b74095b5..e77484796 100644 --- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle +++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle @@ -187,5 +187,7 @@ dependencyManagement { dependency 'org.mapstruct:mapstruct-processor:1.5.3.Final' dependency "org.apache.avro:avro:1.11.1" + + dependency "io.github.resilience4j:resilience4j-spring-boot2:1.7.1" } } diff --git a/fineract-doc/src/docs/en/chapters/resilience/command.adoc b/fineract-doc/src/docs/en/chapters/resilience/command.adoc new file mode 100644 index 000000000..c1fd0784f --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/command.adoc @@ -0,0 +1,23 @@ += Command + +== `CommandProcessingService` + +TBD + +.Retry-able service function `executeCommand` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=73..151] +---- + +.Fallback function `fallbackExecuteCommand` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java[lines=166..174] +---- + +.Retry configuration for `executeCommand` +[source,properties] +---- +include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=169..174] +---- diff --git a/fineract-doc/src/docs/en/chapters/resilience/index.adoc b/fineract-doc/src/docs/en/chapters/resilience/index.adoc new file mode 100644 index 000000000..200fd7909 --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/index.adoc @@ -0,0 +1,11 @@ += Resilience + +include::intro.adoc[leveloffset=+1] + +include::command.adoc[leveloffset=+1] + +include::job.adoc[leveloffset=+1] + +include::loan.adoc[leveloffset=+1] + +include::saving.adoc[leveloffset=+1] diff --git a/fineract-doc/src/docs/en/chapters/resilience/intro.adoc b/fineract-doc/src/docs/en/chapters/resilience/intro.adoc new file mode 100644 index 000000000..6efa6e850 --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/intro.adoc @@ -0,0 +1,84 @@ += Introduction Resilience + +Fineract had handcrafted retry loops in place for the longest time. A typical retry code would have looked like this: + +.Legacy retry code +[source,java] +---- + @Override + @SuppressWarnings("AvoidHidingCauseException") + @SuppressFBWarnings(value = { + "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once") + public CommandProcessingResult logCommandSource(final CommandWrapper wrapper) { + + boolean isApprovedByChecker = false; + // check if is update of own account details + if (wrapper.isUpdateOfOwnUserDetails(this.context.authenticatedUser(wrapper).getId())) { + // then allow this operation to proceed. + // maker checker doesnt mean anything here. + isApprovedByChecker = true; // set to true in case permissions have + // been maker-checker enabled by + // accident. + } else { + // if not user changing their own details - check user has + // permission to perform specific task. + this.context.authenticatedUser(wrapper).validateHasPermissionTo(wrapper.getTaskPermissionName()); + } + validateIsUpdateAllowed(); + + final String json = wrapper.getJson(); + CommandProcessingResult result = null; + JsonCommand command; + int numberOfRetries = 0; // <1> + int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock(); + int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries(); + final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json); + command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(), + wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(), + wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(), + wrapper.getOrganisationCreditBureauId(), wrapper.getJobName()); + while (numberOfRetries <= maxNumberOfRetries) { // <2> + try { + result = this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker); + numberOfRetries = maxNumberOfRetries + 1; // <3> + } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) { + log.debug("The following command {} has been retried {} time(s)", command.json(), numberOfRetries); + /*** + * Fail if the transaction has been retired for maxNumberOfRetries + **/ + if (numberOfRetries >= maxNumberOfRetries) { + log.warn("The following command {} has been retried for the max allowed attempts of {} and will be rolled back", + command.json(), numberOfRetries); + throw exception; + } + /*** + * Else sleep for a random time (between 1 to 10 seconds) and continue + **/ + try { + int randomNum = RANDOM.nextInt(maxIntervalBetweenRetries + 1); + Thread.sleep(1000 + (randomNum * 1000)); + numberOfRetries = numberOfRetries + 1; // <4> + } catch (InterruptedException e) { + throw exception; + } + } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) { + numberOfRetries = maxNumberOfRetries + 1; // <3> + result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult()); + } + } + + return result; + } +---- +<1> counter +<2> `while` loop +<3> increment to abort +<4> increment + +For better code quality and readability we introduced https://resilience4j.readme.io/docs[Resilience4j]: + +.Annotation based retry +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java[lines=49..76] +---- diff --git a/fineract-doc/src/docs/en/chapters/resilience/job.adoc b/fineract-doc/src/docs/en/chapters/resilience/job.adoc new file mode 100644 index 000000000..74c8975eb --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/job.adoc @@ -0,0 +1,25 @@ += Jobs + +== `SchedularWritePlatformService` + +WARNING: This service has a typo and should be called `SchedulerWritePlatformService`. + +TBD + +.Retry-able service function `processJobDetailForExecution` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=135..155] +---- + +.Fallback function `fallbackProcessJobDetailForExecution` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java[lines=156..159] +---- + +.Retry configuration for `processJobDetailForExecution` +[source,properties] +---- +include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=175..179] +---- diff --git a/fineract-doc/src/docs/en/chapters/resilience/loan.adoc b/fineract-doc/src/docs/en/chapters/resilience/loan.adoc new file mode 100644 index 000000000..bbb3709d6 --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/loan.adoc @@ -0,0 +1,23 @@ += Loan + +== `LoanWritePlatformService` + +TBD + +.Retry-able service function `recalculateInterest` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3217..3247] +---- + +.Fallback function `fallbackRecalculateInterest` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java[lines=3255..3266] +---- + +.Retry configuration for `recalculateInterest` +[source,properties] +---- +include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=180..185] +---- diff --git a/fineract-doc/src/docs/en/chapters/resilience/saving.adoc b/fineract-doc/src/docs/en/chapters/resilience/saving.adoc new file mode 100644 index 000000000..0b1e7e845 --- /dev/null +++ b/fineract-doc/src/docs/en/chapters/resilience/saving.adoc @@ -0,0 +1,23 @@ += Savings + +== `SavingsAccountWritePlatformService` + +TBD + +.Retry-able service function `postInterest` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=586..627] +---- + +.Fallback function `fallbackPostInterest` +[source,java] +---- +include::{rootdir}/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java[lines=1402..1414] +---- + +.Retry configuration for `postInterest` +[source,properties] +---- +include::{rootdir}/fineract-provider/src/main/resources/application.properties[lines=186..191] +---- diff --git a/fineract-doc/src/docs/en/index.adoc b/fineract-doc/src/docs/en/index.adoc index b31430c7d..2c038fe33 100644 --- a/fineract-doc/src/docs/en/index.adoc +++ b/fineract-doc/src/docs/en/index.adoc @@ -39,6 +39,8 @@ include::chapters/development/index.adoc[leveloffset=+1] include::chapters/custom/index.adoc[leveloffset=+1] +include::chapters/resilience/index.adoc[leveloffset=+1] + include::chapters/security/index.adoc[leveloffset=+1] include::chapters/testing/index.adoc[leveloffset=+1] diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle index 07952b14e..e8526710e 100644 --- a/fineract-provider/dependencies.gradle +++ b/fineract-provider/dependencies.gradle @@ -84,6 +84,8 @@ dependencies { 'org.springdoc:springdoc-openapi-common', 'org.springdoc:springdoc-openapi-security', 'org.mapstruct:mapstruct', + + 'io.github.resilience4j:resilience4j-spring-boot2', ) implementation ('org.apache.commons:commons-email') { diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java index c8ebab8c2..188c64c6e 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java @@ -19,8 +19,6 @@ package org.apache.fineract.commands.service; import com.google.gson.JsonElement; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.security.SecureRandom; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.fineract.commands.domain.CommandSource; @@ -28,16 +26,12 @@ import org.apache.fineract.commands.domain.CommandSourceRepository; import org.apache.fineract.commands.domain.CommandWrapper; import org.apache.fineract.commands.exception.CommandNotAwaitingApprovalException; import org.apache.fineract.commands.exception.CommandNotFoundException; -import org.apache.fineract.commands.exception.RollbackTransactionAsCommandIsNotApprovedByCheckerException; import org.apache.fineract.infrastructure.core.api.JsonCommand; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper; -import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.jobs.service.SchedulerJobRunnerReadService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.useradministration.domain.AppUser; -import org.springframework.dao.CannotAcquireLockException; -import org.springframework.orm.ObjectOptimisticLockingFailureException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -46,8 +40,6 @@ import org.springframework.transaction.annotation.Transactional; @Slf4j public class PortfolioCommandSourceWritePlatformServiceImpl implements PortfolioCommandSourceWritePlatformService { - private static final SecureRandom RANDOM = new SecureRandom(); - private final PlatformSecurityContext context; private final CommandSourceRepository commandSourceRepository; private final FromJsonHelper fromApiJsonHelper; @@ -55,12 +47,10 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio private final SchedulerJobRunnerReadService schedulerJobRunnerReadService; @Override - @SuppressWarnings("AvoidHidingCauseException") - @SuppressFBWarnings(value = { - "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once") public CommandProcessingResult logCommandSource(final CommandWrapper wrapper) { boolean isApprovedByChecker = false; + // check if is update of own account details if (wrapper.isUpdateOfOwnUserDetails(this.context.authenticatedUser(wrapper).getId())) { // then allow this operation to proceed. @@ -76,47 +66,13 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio validateIsUpdateAllowed(); final String json = wrapper.getJson(); - CommandProcessingResult result = null; - JsonCommand command; - int numberOfRetries = 0; - int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock(); - int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries(); final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json); - command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(), + JsonCommand command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(), wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(), wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(), wrapper.getOrganisationCreditBureauId(), wrapper.getJobName()); - while (numberOfRetries <= maxNumberOfRetries) { - try { - result = this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker); - numberOfRetries = maxNumberOfRetries + 1; - } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) { - log.debug("The following command {} has been retried {} time(s)", command.json(), numberOfRetries); - /*** - * Fail if the transaction has been retired for maxNumberOfRetries - **/ - if (numberOfRetries >= maxNumberOfRetries) { - log.warn("The following command {} has been retried for the max allowed attempts of {} and will be rolled back", - command.json(), numberOfRetries); - throw exception; - } - /*** - * Else sleep for a random time (between 1 to 10 seconds) and continue - **/ - try { - int randomNum = RANDOM.nextInt(maxIntervalBetweenRetries + 1); - Thread.sleep(1000 + (randomNum * 1000)); - numberOfRetries = numberOfRetries + 1; - } catch (InterruptedException e) { - throw exception; - } - } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) { - numberOfRetries = maxNumberOfRetries + 1; - result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult()); - } - } - return result; + return this.processAndLogCommandService.executeCommand(wrapper, command, isApprovedByChecker); } @Override @@ -168,9 +124,8 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio return commandSourceInput; } - private boolean validateIsUpdateAllowed() { - return this.schedulerJobRunnerReadService.isUpdatesAllowed(); - + private void validateIsUpdateAllowed() { + this.schedulerJobRunnerReadService.isUpdatesAllowed(); } @Override diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java b/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java index 43ae13cb4..a5ea977f3 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java @@ -20,6 +20,7 @@ package org.apache.fineract.commands.service; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import io.github.resilience4j.retry.annotation.Retry; import java.lang.reflect.Type; import java.time.Instant; import java.util.HashMap; @@ -69,8 +70,9 @@ public class SynchronousCommandProcessingService implements CommandProcessingSer private final IdempotencyKeyGenerator idempotencyKeyGenerator; private final FineractProperties fineractProperties; - @Transactional @Override + @Transactional + @Retry(name = "executeCommand", fallbackMethod = "fallbackExecuteCommand") public CommandProcessingResult executeCommand(final CommandWrapper wrapper, final JsonCommand command, final boolean isApprovedByChecker) { @@ -161,6 +163,15 @@ public class SynchronousCommandProcessingService implements CommandProcessingSer .withEntityId(commandSourceResult.getResourceId()).build(); } + @SuppressWarnings("unused") + private CommandProcessingResult fallbackExecuteCommand(Exception e) throws Exception { + if (e instanceof RollbackTransactionAsCommandIsNotApprovedByCheckerException ex) { + return logCommand(ex.getCommandSourceResult()); + } + + throw e; + } + private NewCommandSourceHandler findCommandHandler(final CommandWrapper wrapper) { NewCommandSourceHandler handler; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java index 1a5cbaec6..6f2dee679 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java @@ -54,8 +54,6 @@ public class FineractPlatformTenantConnection implements Serializable { private final int suspectTimeout; private final int timeBetweenEvictionRunsMillis; private final int minEvictableIdleTimeMillis; - private final int maxRetriesOnDeadlock; - private final int maxIntervalBetweenRetries; private final boolean testOnBorrow; public FineractPlatformTenantConnection(final Long connectionId, final String schemaName, String schemaServer, @@ -63,10 +61,9 @@ public class FineractPlatformTenantConnection implements Serializable { final String schemaPassword, final boolean autoUpdateEnabled, final int initialSize, final long validationInterval, final boolean removeAbandoned, final int removeAbandonedTimeout, final boolean logAbandoned, final int abandonWhenPercentageFull, final int maxActive, final int minIdle, final int maxIdle, final int suspectTimeout, - final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis, final int maxRetriesOnDeadlock, - final int maxIntervalBetweenRetries, final boolean tesOnBorrow, final String readOnlySchemaServer, - final String readOnlySchemaServerPort, final String readOnlySchemaName, final String readOnlySchemaUsername, - final String readOnlySchemaPassword, final String readOnlySchemaConnectionParameters) { + final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis, final boolean tesOnBorrow, + final String readOnlySchemaServer, final String readOnlySchemaServerPort, final String readOnlySchemaName, + final String readOnlySchemaUsername, final String readOnlySchemaPassword, final String readOnlySchemaConnectionParameters) { this.connectionId = connectionId; this.schemaName = schemaName; @@ -88,8 +85,6 @@ public class FineractPlatformTenantConnection implements Serializable { this.suspectTimeout = suspectTimeout; this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; - this.maxRetriesOnDeadlock = maxRetriesOnDeadlock; - this.maxIntervalBetweenRetries = maxIntervalBetweenRetries; this.testOnBorrow = tesOnBorrow; this.readOnlySchemaServer = readOnlySchemaServer; this.readOnlySchemaServerPort = readOnlySchemaServerPort; @@ -171,14 +166,6 @@ public class FineractPlatformTenantConnection implements Serializable { return this.minEvictableIdleTimeMillis; } - public int getMaxRetriesOnDeadlock() { - return this.maxRetriesOnDeadlock; - } - - public int getMaxIntervalBetweenRetries() { - return this.maxIntervalBetweenRetries; - } - public boolean isTestOnBorrow() { return testOnBorrow; } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java index b9c81fdc4..f2645cd7c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedularWritePlatformServiceJpaRepositoryImpl.java @@ -18,6 +18,7 @@ */ package org.apache.fineract.infrastructure.jobs.service; +import io.github.resilience4j.retry.annotation.Retry; import java.util.Date; import java.util.List; import java.util.Map; @@ -133,6 +134,7 @@ public class SchedularWritePlatformServiceJpaRepositoryImpl implements Schedular @Transactional @Override + @Retry(name = "processJobDetailForExecution", fallbackMethod = "fallbackProcessJobDetailForExecution") public boolean processJobDetailForExecution(final String jobKey, final String triggerType) { boolean isStopExecution = false; final ScheduledJobDetail scheduledJobDetail = this.scheduledJobDetailsRepository.findByJobKeyWithLock(jobKey); @@ -151,4 +153,9 @@ public class SchedularWritePlatformServiceJpaRepositoryImpl implements Schedular return isStopExecution; } + @SuppressWarnings("unused") + private boolean fallbackProcessJobDetailForExecution(Exception e) { + return false; + } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedulerTriggerListener.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedulerTriggerListener.java index a48910de9..55b2db779 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedulerTriggerListener.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/SchedulerTriggerListener.java @@ -18,11 +18,10 @@ */ package org.apache.fineract.infrastructure.jobs.service; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.security.SecureRandom; import java.time.LocalDate; import java.util.HashMap; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType; import org.apache.fineract.infrastructure.businessdate.service.BusinessDateReadPlatformService; import org.apache.fineract.infrastructure.core.domain.ActionContext; @@ -34,19 +33,15 @@ import org.quartz.JobKey; import org.quartz.Trigger; import org.quartz.Trigger.CompletedExecutionInstruction; import org.quartz.TriggerListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; -@Component +@Slf4j @RequiredArgsConstructor +@Component public class SchedulerTriggerListener implements TriggerListener { - private static final Logger LOG = LoggerFactory.getLogger(SchedulerTriggerListener.class); - private static final SecureRandom random = new SecureRandom(); - private final SchedularWritePlatformService schedularService; private final TenantDetailsService tenantDetailsService; @@ -59,12 +54,10 @@ public class SchedulerTriggerListener implements TriggerListener { @Override public void triggerFired(Trigger trigger, JobExecutionContext context) { - LOG.debug("triggerFired() trigger={}, context={}", trigger, context); + log.debug("triggerFired() trigger={}, context={}", trigger, context); } @Override - @SuppressFBWarnings(value = { - "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once") @Transactional(isolation = Isolation.READ_COMMITTED) public boolean vetoJobExecution(final Trigger trigger, final JobExecutionContext context) { final String tenantIdentifier = trigger.getJobDataMap().getString(SchedulerServiceConstants.TENANT_IDENTIFIER); @@ -79,43 +72,22 @@ public class SchedulerTriggerListener implements TriggerListener { if (context.getMergedJobDataMap().containsKey(SchedulerServiceConstants.TRIGGER_TYPE_REFERENCE)) { triggerType = context.getMergedJobDataMap().getString(SchedulerServiceConstants.TRIGGER_TYPE_REFERENCE); } - Integer maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock(); - Integer maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries(); - Integer numberOfRetries = 0; - boolean vetoJob = false; - while (numberOfRetries <= maxNumberOfRetries) { - try { - vetoJob = this.schedularService.processJobDetailForExecution(jobKey, triggerType); - numberOfRetries = maxNumberOfRetries + 1; - } catch (Exception exception) { // Adding generic exception as it - // depends on JPA provider - LOG.warn("vetoJobExecution() not able to acquire the lock to update job running status at retry {} (of {}) for JobKey: {}", - numberOfRetries, maxNumberOfRetries, jobKey, exception); - try { - int randomNum = random.nextInt(maxIntervalBetweenRetries + 1); - Thread.sleep(1000 + (randomNum * 1000)); - numberOfRetries = numberOfRetries + 1; - } catch (InterruptedException e) { - LOG.error("vetoJobExecution() caught an InterruptedException", e); - } - } - } + boolean vetoJob = this.schedularService.processJobDetailForExecution(jobKey, triggerType); if (vetoJob) { - LOG.warn( - "vetoJobExecution() WILL veto the execution (returning vetoJob == true; the job's execute method will NOT be called); " - + "maxNumberOfRetries={}, tenant={}, jobKey={}, triggerType={}, trigger={}, context={}", - maxNumberOfRetries, tenantIdentifier, jobKey, triggerType, trigger, context); + log.warn( + "vetoJobExecution() WILL veto the execution (returning vetoJob == true; the job's execute method will NOT be called); tenant={}, jobKey={}, triggerType={}, trigger={}, context={}", + tenantIdentifier, jobKey, triggerType, trigger, context); } return vetoJob; } @Override public void triggerMisfired(final Trigger trigger) { - LOG.error("triggerMisfired() trigger={}", trigger); + log.error("triggerMisfired() trigger={}", trigger); } @Override public void triggerComplete(Trigger trigger, JobExecutionContext context, CompletedExecutionInstruction triggerInstructionCode) { - LOG.debug("triggerComplete() trigger={}, context={}, completedExecutionInstruction={}", trigger, context, triggerInstructionCode); + log.debug("triggerComplete() trigger={}, context={}, completedExecutionInstruction={}", trigger, context, triggerInstructionCode); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/security/service/TenantMapper.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/security/service/TenantMapper.java index 02ade1944..7a4f57a6a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/security/service/TenantMapper.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/security/service/TenantMapper.java @@ -37,7 +37,6 @@ public final class TenantMapper implements RowMapper { + " ts.pool_max_active as poolMaxActive, ts.pool_min_idle as poolMinIdle, ts.pool_max_idle as poolMaxIdle," + " ts.pool_suspect_timeout as poolSuspectTimeout, ts.pool_time_between_eviction_runs_millis as poolTimeBetweenEvictionRunsMillis," + " ts.pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis," - + " ts.deadlock_max_retries as maxRetriesOnDeadlock," + " ts.deadlock_max_retry_interval as maxIntervalBetweenRetries," + " ts.readonly_schema_server as readOnlySchemaServer, " + " ts.readonly_schema_server_port as readOnlySchemaServerPort, " + " ts.readonly_schema_name as readOnlySchemaName, " + " ts.readonly_schema_username as readOnlySchemaUsername, " + " ts.readonly_schema_password as readOnlySchemaPassword, " @@ -99,26 +98,11 @@ public final class TenantMapper implements RowMapper { final int suspectTimeout = rs.getInt("poolSuspectTimeout"); final int timeBetweenEvictionRunsMillis = rs.getInt("poolTimeBetweenEvictionRunsMillis"); final int minEvictableIdleTimeMillis = rs.getInt("poolMinEvictableIdleTimeMillis"); - int maxRetriesOnDeadlock = rs.getInt("maxRetriesOnDeadlock"); - int maxIntervalBetweenRetries = rs.getInt("maxIntervalBetweenRetries"); - - maxRetriesOnDeadlock = bindValueInMinMaxRange(maxRetriesOnDeadlock, 0, 15); - maxIntervalBetweenRetries = bindValueInMinMaxRange(maxIntervalBetweenRetries, 1, 15); return new FineractPlatformTenantConnection(connectionId, schemaName, schemaServer, schemaServerPort, schemaConnectionParameters, schemaUsername, schemaPassword, autoUpdateEnabled, initialSize, validationInterval, removeAbandoned, removeAbandonedTimeout, logAbandoned, abandonWhenPercentageFull, maxActive, minIdle, maxIdle, suspectTimeout, timeBetweenEvictionRunsMillis, - minEvictableIdleTimeMillis, maxRetriesOnDeadlock, maxIntervalBetweenRetries, testOnBorrow, readOnlySchemaServer, - readOnlySchemaServerPort, readOnlySchemaName, readOnlySchemaUsername, readOnlySchemaPassword, - readOnlySchemaConnectionParameters); - } - - private int bindValueInMinMaxRange(final int value, int min, int max) { - if (value < min) { - return min; - } else if (value > max) { - return max; - } - return value; + minEvictableIdleTimeMillis, testOnBorrow, readOnlySchemaServer, readOnlySchemaServerPort, readOnlySchemaName, + readOnlySchemaUsername, readOnlySchemaPassword, readOnlySchemaConnectionParameters); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/jobs/recalculateinterestforloan/RecalculateInterestForLoanTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/jobs/recalculateinterestforloan/RecalculateInterestForLoanTasklet.java index 922d4e986..7d70c478b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/jobs/recalculateinterestforloan/RecalculateInterestForLoanTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/jobs/recalculateinterestforloan/RecalculateInterestForLoanTasklet.java @@ -18,7 +18,6 @@ */ package org.apache.fineract.portfolio.loanaccount.jobs.recalculateinterestforloan; -import java.security.SecureRandom; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,7 +31,6 @@ import java.util.concurrent.Future; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; -import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.jobs.exception.JobExecutionException; import org.apache.fineract.organisation.office.data.OfficeData; import org.apache.fineract.organisation.office.exception.OfficeNotFoundException; @@ -45,8 +43,6 @@ import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.core.step.tasklet.Tasklet; import org.springframework.batch.repeat.RepeatStatus; -import org.springframework.dao.CannotAcquireLockException; -import org.springframework.orm.ObjectOptimisticLockingFailureException; @Slf4j @RequiredArgsConstructor @@ -56,7 +52,6 @@ public class RecalculateInterestForLoanTasklet implements Tasklet { private final LoanWritePlatformService loanWritePlatformService; private final RecalculateInterestPoster recalculateInterestPoster; private final OfficeReadPlatformService officeReadPlatformService; - private static final SecureRandom random = new SecureRandom(); @Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { @@ -75,45 +70,16 @@ public class RecalculateInterestForLoanTasklet implements Tasklet { recalculateInterest(office, threadPoolSize, batchSize); } else { - int maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock(); - int maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries(); Collection loanIds = loanReadPlatformService.fetchLoansForInterestRecalculation(); - int i = 0; if (!loanIds.isEmpty()) { List errors = new ArrayList<>(); for (Long loanId : loanIds) { log.debug("recalculateInterest: Loan ID = {}", loanId); - int numberOfRetries = 0; - while (numberOfRetries <= maxNumberOfRetries) { - try { - loanWritePlatformService.recalculateInterest(loanId); - numberOfRetries = maxNumberOfRetries + 1; - } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) { - log.debug("Recalulate interest job has been retried {} time(s)", numberOfRetries); - if (numberOfRetries >= maxNumberOfRetries) { - log.error( - "Recalulate interest job has been retried for the max allowed attempts of {} and will be rolled back", - numberOfRetries); - errors.add(exception); - break; - } - try { - int randomNum = random.nextInt(maxIntervalBetweenRetries + 1); - Thread.sleep(1000 + (randomNum * 1000L)); - numberOfRetries = numberOfRetries + 1; - } catch (InterruptedException e) { - log.error("Interest recalculation for loans retry failed due to InterruptedException", e); - errors.add(e); - break; - } - } catch (Exception e) { - log.error("Interest recalculation for loans failed for account {}", loanId, e); - numberOfRetries = maxNumberOfRetries + 1; - errors.add(e); - } - i++; + try { + loanWritePlatformService.recalculateInterest(loanId); + } catch (Exception e) { + errors.add(e); } - log.debug("recalculateInterest: Loans count {}", i); } if (!errors.isEmpty()) { throw new JobExecutionException(errors); diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java index 608353ff2..15050010c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanWritePlatformServiceJpaRepositoryImpl.java @@ -21,6 +21,7 @@ package org.apache.fineract.portfolio.loanaccount.service; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import io.github.resilience4j.retry.annotation.Retry; import java.math.BigDecimal; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -39,7 +40,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.accounting.journalentry.service.JournalEntryWritePlatformService; -import org.apache.fineract.cob.domain.LoanAccountLockRepository; import org.apache.fineract.cob.exceptions.LoanAccountLockCannotBeOverruledException; import org.apache.fineract.cob.service.LoanAccountLockService; import org.apache.fineract.infrastructure.codes.domain.CodeValue; @@ -281,7 +281,6 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf private final PostDatedChecksRepository postDatedChecksRepository; private final LoanDisbursementDetailsRepository loanDisbursementDetailsRepository; private final LoanRepaymentScheduleInstallmentRepository loanRepaymentScheduleInstallmentRepository; - private final LoanAccountLockRepository loanAccountLockRepository; private final LoanAccountLockService loanAccountLockService; private static boolean isPartOfThisInstallment(LoanCharge loanCharge, LoanRepaymentScheduleInstallment e) { @@ -3217,6 +3216,7 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf @Transactional @Override + @Retry(name = "recalculateInterest", fallbackMethod = "fallbackRecalculateInterest") public void recalculateInterest(final long loanId) { Loan loan = this.loanAssembler.assembleFrom(loanId); LocalDate recalculateFrom = loan.fetchInterestRecalculateFromDate(); @@ -3252,6 +3252,18 @@ public class LoanWritePlatformServiceJpaRepositoryImpl implements LoanWritePlatf return new CommandProcessingResultBuilder().withLoanId(loanId).build(); } + @SuppressWarnings("unused") + private void fallbackRecalculateInterest(Throwable t) { + // NOTE: allow caller to catch the exceptions + + if (t instanceof RuntimeException re) { + throw re; + } + + // NOTE: wrap throwable only if really necessary + throw new RuntimeException(t); + } + private void updateLoanTransaction(final Long loanTransactionId, final LoanTransaction newLoanTransaction) { final AccountTransferTransaction transferTransaction = this.accountTransferRepository.findByToLoanTransactionId(loanTransactionId); if (transferTransaction != null) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/RecalculateInterestPoster.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/RecalculateInterestPoster.java index 1c1cb0e12..6d0846fef 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/RecalculateInterestPoster.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/RecalculateInterestPoster.java @@ -18,19 +18,14 @@ */ package org.apache.fineract.portfolio.loanaccount.service; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.security.SecureRandom; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; -import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; import org.apache.fineract.infrastructure.jobs.exception.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; -import org.springframework.dao.CannotAcquireLockException; -import org.springframework.orm.ObjectOptimisticLockingFailureException; import org.springframework.stereotype.Component; @Component @@ -38,7 +33,6 @@ import org.springframework.stereotype.Component; public class RecalculateInterestPoster implements Callable { private static final Logger LOG = LoggerFactory.getLogger(RecalculateInterestPoster.class); - private static final SecureRandom random = new SecureRandom(); private Collection loanIds; private LoanWritePlatformService loanWritePlatformService; @@ -52,52 +46,16 @@ public class RecalculateInterestPoster implements Callable { } @Override - @SuppressFBWarnings(value = { - "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once") public Void call() throws JobExecutionException { - Integer maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxRetriesOnDeadlock(); - Integer maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getConnection().getMaxIntervalBetweenRetries(); - - int i = 0; if (!loanIds.isEmpty()) { List errors = new ArrayList<>(); for (Long loanId : loanIds) { LOG.debug("Loan ID {}", loanId); - Integer numberOfRetries = 0; - while (numberOfRetries <= maxNumberOfRetries) { - try { - this.loanWritePlatformService.recalculateInterest(loanId); - numberOfRetries = maxNumberOfRetries + 1; - } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) { - LOG.debug("Recalculate interest job has been retried {} time(s)", numberOfRetries); - // Fail if the transaction has been retired for - // maxNumberOfRetries - if (numberOfRetries >= maxNumberOfRetries) { - LOG.error( - "Recalculate interest job has been retried for the max allowed attempts of {} and will be rolled back", - numberOfRetries); - errors.add(exception); - break; - } - // Else sleep for a random time (between 1 to 10 - // seconds) and continue - try { - int randomNum = random.nextInt(maxIntervalBetweenRetries + 1); - Thread.sleep(1000 + (randomNum * 1000)); - numberOfRetries = numberOfRetries + 1; - } catch (InterruptedException e) { - LOG.error("Interest recalculation for loans retry failed due to InterruptedException", e); - errors.add(e); - break; - } - } catch (Exception e) { - LOG.error("Interest recalculation for loans failed for account {}", loanId, e); - numberOfRetries = maxNumberOfRetries + 1; - errors.add(e); - } - i++; + try { + loanWritePlatformService.recalculateInterest(loanId); + } catch (Exception e) { + errors.add(e); } - LOG.debug("Loans count {}", i); } if (!errors.isEmpty()) { throw new JobExecutionException(errors); diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/jobs/postinterestforsavings/PostInterestForSavingTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/jobs/postinterestforsavings/PostInterestForSavingTasklet.java index 88211beaf..209667244 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/jobs/postinterestforsavings/PostInterestForSavingTasklet.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/jobs/postinterestforsavings/PostInterestForSavingTasklet.java @@ -36,21 +36,15 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.core.domain.FineractContext; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; -import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.portfolio.savings.data.SavingsAccountData; -import org.apache.fineract.portfolio.savings.domain.SavingsAccountAssembler; -import org.apache.fineract.portfolio.savings.domain.SavingsAccountRepositoryWrapper; import org.apache.fineract.portfolio.savings.service.SavingsAccountReadPlatformService; -import org.apache.fineract.portfolio.savings.service.SavingsAccountWritePlatformService; import org.apache.fineract.portfolio.savings.service.SavingsSchedularInterestPoster; import org.springframework.batch.core.StepContribution; import org.springframework.batch.core.scope.context.ChunkContext; import org.springframework.batch.core.step.tasklet.Tasklet; import org.springframework.batch.repeat.RepeatStatus; import org.springframework.context.ApplicationContext; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import org.springframework.transaction.support.TransactionTemplate; @RequiredArgsConstructor @Slf4j @@ -59,15 +53,9 @@ public class PostInterestForSavingTasklet implements Tasklet { private final SavingsAccountReadPlatformService savingAccountReadPlatformService; private final ConfigurationDomainService configurationDomainService; - private final SavingsAccountWritePlatformService savingsAccountWritePlatformService; - private final SavingsAccountRepositoryWrapper savingsAccountRepository; - private final SavingsAccountAssembler savingAccountAssembler; - private final JdbcTemplate jdbcTemplate; - private final TransactionTemplate transactionTemplate; private final Queue> queue = new ArrayDeque<>(); private final ApplicationContext applicationContext; private final int queueSize = 1; - private final PlatformSecurityContext securityContext; @Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { @@ -147,19 +135,11 @@ public class PostInterestForSavingTasklet implements Tasklet { for (long i = 0; i < loopCount; i++) { List subList = safeSubList(savingsAccounts, fromIndex, toIndex); - SavingsSchedularInterestPoster savingsSchedularInterestPoster = (SavingsSchedularInterestPoster) applicationContext - .getBean("savingsSchedularInterestPoster"); + SavingsSchedularInterestPoster savingsSchedularInterestPoster = applicationContext + .getBean(SavingsSchedularInterestPoster.class); savingsSchedularInterestPoster.setSavingAccounts(subList); - savingsSchedularInterestPoster.setContext(ThreadLocalContextUtil.getContext()); - savingsSchedularInterestPoster.setSavingsAccountWritePlatformService(savingsAccountWritePlatformService); - savingsSchedularInterestPoster.setSavingsAccountReadPlatformService(savingAccountReadPlatformService); - savingsSchedularInterestPoster.setSavingsAccountRepository(savingsAccountRepository); - savingsSchedularInterestPoster.setSavingAccountAssembler(savingAccountAssembler); - savingsSchedularInterestPoster.setJdbcTemplate(jdbcTemplate); savingsSchedularInterestPoster.setBackdatedTxnsAllowedTill(backdatedTxnsAllowedTill); - savingsSchedularInterestPoster.setTransactionTemplate(transactionTemplate); - savingsSchedularInterestPoster.setConfigurationDomainService(configurationDomainService); - savingsSchedularInterestPoster.setPlatformSecurityContext(securityContext); + savingsSchedularInterestPoster.setContext(ThreadLocalContextUtil.getContext()); posters.add(savingsSchedularInterestPoster); diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java index b3e9d4983..fb9d2d05d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsAccountWritePlatformServiceJpaRepositoryImpl.java @@ -31,6 +31,7 @@ import static org.apache.fineract.portfolio.savings.SavingsApiConstants.withdraw import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import io.github.resilience4j.retry.annotation.Retry; import java.math.BigDecimal; import java.math.MathContext; import java.time.LocalDate; @@ -46,6 +47,8 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.accounting.journalentry.service.JournalEntryWritePlatformService; import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; @@ -68,7 +71,6 @@ import org.apache.fineract.infrastructure.event.business.domain.savings.SavingsP import org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.organisation.holiday.domain.HolidayRepositoryWrapper; -import org.apache.fineract.organisation.monetary.domain.ApplicationCurrencyRepositoryWrapper; import org.apache.fineract.organisation.monetary.domain.Money; import org.apache.fineract.organisation.monetary.domain.MoneyHelper; import org.apache.fineract.organisation.office.domain.Office; @@ -125,18 +127,16 @@ import org.apache.fineract.portfolio.savings.exception.TransactionUpdateNotAllow import org.apache.fineract.portfolio.transfer.api.TransferApiConstants; import org.apache.fineract.useradministration.domain.AppUser; import org.apache.fineract.useradministration.domain.AppUserRepositoryWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +@Slf4j +@RequiredArgsConstructor @Service public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements SavingsAccountWritePlatformService { @@ -149,7 +149,6 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi private final SavingsAccountTransactionDataValidator savingsAccountTransactionDataValidator; private final SavingsAccountChargeDataValidator savingsAccountChargeDataValidator; private final PaymentDetailWritePlatformService paymentDetailWritePlatformService; - private final ApplicationCurrencyRepositoryWrapper applicationCurrencyRepositoryWrapper; private final JournalEntryWritePlatformService journalEntryWritePlatformService; private final SavingsAccountDomainService savingsAccountDomainService; private final NoteRepository noteRepository; @@ -166,63 +165,8 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi private final StandingInstructionRepository standingInstructionRepository; private final BusinessEventNotifierService businessEventNotifierService; private final GSIMRepositoy gsimRepository; - private final JdbcTemplate jdbcTemplate; private final SavingsAccountInterestPostingService savingsAccountInterestPostingService; - @Autowired - public SavingsAccountWritePlatformServiceJpaRepositoryImpl(final PlatformSecurityContext context, - final SavingsAccountRepositoryWrapper savingAccountRepositoryWrapper, - final SavingsAccountTransactionRepository savingsAccountTransactionRepository, - final SavingsAccountAssembler savingAccountAssembler, - final SavingsAccountTransactionDataValidator savingsAccountTransactionDataValidator, - final SavingsAccountChargeDataValidator savingsAccountChargeDataValidator, - final PaymentDetailWritePlatformService paymentDetailWritePlatformService, - final ApplicationCurrencyRepositoryWrapper applicationCurrencyRepositoryWrapper, - final JournalEntryWritePlatformService journalEntryWritePlatformService, - final SavingsAccountDomainService savingsAccountDomainService, final NoteRepository noteRepository, - final AccountTransfersReadPlatformService accountTransfersReadPlatformService, final HolidayRepositoryWrapper holidayRepository, - final WorkingDaysRepositoryWrapper workingDaysRepository, - final AccountAssociationsReadPlatformService accountAssociationsReadPlatformService, - final ChargeRepositoryWrapper chargeRepository, final SavingsAccountChargeRepositoryWrapper savingsAccountChargeRepository, - final SavingsAccountDataValidator fromApiJsonDeserializer, final StaffRepositoryWrapper staffRepository, - final ConfigurationDomainService configurationDomainService, - final DepositAccountOnHoldTransactionRepository depositAccountOnHoldTransactionRepository, - final EntityDatatableChecksWritePlatformService entityDatatableChecksWritePlatformService, - final AppUserRepositoryWrapper appuserRepository, final StandingInstructionRepository standingInstructionRepository, - final BusinessEventNotifierService businessEventNotifierService, final GSIMRepositoy gsimRepository, - final JdbcTemplate jdbcTemplate, final SavingsAccountInterestPostingService savingsAccountInterestPostingService) { - this.context = context; - this.savingAccountRepositoryWrapper = savingAccountRepositoryWrapper; - this.savingsAccountTransactionRepository = savingsAccountTransactionRepository; - this.savingAccountAssembler = savingAccountAssembler; - this.savingsAccountTransactionDataValidator = savingsAccountTransactionDataValidator; - this.savingsAccountChargeDataValidator = savingsAccountChargeDataValidator; - this.paymentDetailWritePlatformService = paymentDetailWritePlatformService; - this.applicationCurrencyRepositoryWrapper = applicationCurrencyRepositoryWrapper; - this.journalEntryWritePlatformService = journalEntryWritePlatformService; - this.savingsAccountDomainService = savingsAccountDomainService; - this.noteRepository = noteRepository; - this.accountTransfersReadPlatformService = accountTransfersReadPlatformService; - this.accountAssociationsReadPlatformService = accountAssociationsReadPlatformService; - this.chargeRepository = chargeRepository; - this.savingsAccountChargeRepository = savingsAccountChargeRepository; - this.holidayRepository = holidayRepository; - this.workingDaysRepository = workingDaysRepository; - this.fromApiJsonDeserializer = fromApiJsonDeserializer; - this.staffRepository = staffRepository; - this.configurationDomainService = configurationDomainService; - this.depositAccountOnHoldTransactionRepository = depositAccountOnHoldTransactionRepository; - this.entityDatatableChecksWritePlatformService = entityDatatableChecksWritePlatformService; - this.appuserRepository = appuserRepository; - this.standingInstructionRepository = standingInstructionRepository; - this.businessEventNotifierService = businessEventNotifierService; - this.gsimRepository = gsimRepository; - this.jdbcTemplate = jdbcTemplate; - this.savingsAccountInterestPostingService = savingsAccountInterestPostingService; - } - - private static final Logger LOG = LoggerFactory.getLogger(SavingsAccountWritePlatformServiceJpaRepositoryImpl.class); - @Transactional @Override public CommandProcessingResult gsimActivate(final Long gsimId, final JsonCommand command) { @@ -355,7 +299,7 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi if (account.getGsim() != null) { isGsim = true; - LOG.debug("is gsim"); + log.debug("is gsim"); } checkClientOrGroupActive(account); @@ -376,16 +320,16 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi if (isGsim && (deposit.getId() != null)) { - LOG.debug("Deposit account has been created: {} ", deposit); + log.debug("Deposit account has been created: {} ", deposit); GroupSavingsIndividualMonitoring gsim = gsimRepository.findById(account.getGsim().getId()).orElseThrow(); - LOG.debug("parent deposit : {} ", gsim.getParentDeposit()); - LOG.debug("child account : {} ", savingsId); + log.debug("parent deposit : {} ", gsim.getParentDeposit()); + log.debug("child account : {} ", savingsId); BigDecimal currentBalance = gsim.getParentDeposit(); BigDecimal newBalance = currentBalance.add(transactionAmount); gsim.setParentDeposit(newBalance); gsimRepository.save(gsim); - LOG.debug("balance after making deposit : {} ", + log.debug("balance after making deposit : {} ", gsimRepository.findById(account.getGsim().getId()).orElseThrow().getParentDeposit()); } @@ -641,6 +585,7 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi @Transactional @Override + @Retry(name = "postInterest", fallbackMethod = "fallbackPostInterest") public SavingsAccountData postInterest(SavingsAccountData savingsAccountData, final boolean postInterestAs, final LocalDate transactionDate, final boolean backdatedTxnsAllowedTill) { @@ -662,14 +607,9 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi postInterestOnDate = transactionDate; } - long startPosting = System.currentTimeMillis(); - LOG.debug("Interest Posting Start Here at {}", startPosting); - savingsAccountData = this.savingsAccountInterestPostingService.postInterest(mc, today, isInterestTransfer, isSavingsInterestPostingAtCurrentPeriodEnd, financialYearBeginningMonth, postInterestOnDate, backdatedTxnsAllowedTill, savingsAccountData); - long endPosting = System.currentTimeMillis(); - LOG.debug("Interest Posting Ends within {}", endPosting - startPosting); if (!backdatedTxnsAllowedTill) { List transactions = savingsAccountData.getSavingsAccountTransactionData(); @@ -1459,6 +1399,19 @@ public class SavingsAccountWritePlatformServiceJpaRepositoryImpl implements Savi } } + @SuppressWarnings("unused") + private SavingsAccountData fallbackPostInterest(SavingsAccountData savingsAccountData, boolean postInterestAs, + LocalDate transactionDate, boolean backdatedTxnsAllowedTill, Throwable t) { + // NOTE: allow caller to catch the exceptions + + if (t instanceof RuntimeException re) { + throw re; + } + + // NOTE: wrap throwable only if really necessary + throw new RuntimeException(t); + } + @Transactional private SavingsAccountTransaction payCharge(final SavingsAccountCharge savingsAccountCharge, final LocalDate transactionDate, final BigDecimal amountPaid, final DateTimeFormatter formatter, final AppUser user, final boolean backdatedTxnsAllowedTill) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsSchedularInterestPoster.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsSchedularInterestPoster.java index b2fcc557a..d89814bbf 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsSchedularInterestPoster.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/SavingsSchedularInterestPoster.java @@ -18,9 +18,7 @@ */ package org.apache.fineract.portfolio.savings.service; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigDecimal; -import java.security.SecureRandom; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; @@ -30,11 +28,10 @@ import java.util.HashMap; import java.util.List; import java.util.UUID; import java.util.concurrent.Callable; +import lombok.RequiredArgsConstructor; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.fineract.accounting.journalentry.domain.JournalEntryType; -import org.apache.fineract.batch.command.CommandStrategyProvider; -import org.apache.fineract.batch.service.ResolutionHelper; -import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService; import org.apache.fineract.infrastructure.core.domain.FineractContext; import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil; @@ -43,119 +40,61 @@ import org.apache.fineract.infrastructure.security.service.PlatformSecurityConte import org.apache.fineract.portfolio.savings.data.SavingsAccountData; import org.apache.fineract.portfolio.savings.data.SavingsAccountSummaryData; import org.apache.fineract.portfolio.savings.data.SavingsAccountTransactionData; -import org.apache.fineract.portfolio.savings.domain.SavingsAccountAssembler; -import org.apache.fineract.portfolio.savings.domain.SavingsAccountRepositoryWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; -import org.springframework.dao.CannotAcquireLockException; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.orm.ObjectOptimisticLockingFailureException; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionTemplate; /** * @author manoj */ +@Slf4j +@RequiredArgsConstructor +@Setter @Component @Scope("prototype") -@Setter public class SavingsSchedularInterestPoster implements Callable { - private static final Logger LOG = LoggerFactory.getLogger(SavingsSchedularInterestPoster.class); - private static final SecureRandom random = new SecureRandom(); private static final String SAVINGS_TRANSACTION_IDENTIFIER = "S"; + private final SavingsAccountWritePlatformService savingsAccountWritePlatformService; + private final List savingsAccountDataList = new ArrayList<>(); + private final JdbcTemplate jdbcTemplate; + private final SavingsAccountReadPlatformService savingsAccountReadPlatformService; + private final PlatformSecurityContext platformSecurityContext; private Collection savingAccounts; - private SavingsAccountWritePlatformService savingsAccountWritePlatformService; - private SavingsAccountRepositoryWrapper savingsAccountRepository; - private SavingsAccountAssembler savingAccountAssembler; private FineractContext context; - private ConfigurationDomainService configurationDomainService; private boolean backdatedTxnsAllowedTill; - private List savingsAccountDataList = new ArrayList<>(); - private JdbcTemplate jdbcTemplate; - private TransactionTemplate transactionTemplate; - private CommandStrategyProvider strategyProvider; - private ResolutionHelper resolutionHelper; - private SavingsAccountReadPlatformService savingsAccountReadPlatformService; - private PlatformSecurityContext platformSecurityContext; @Override - @SuppressFBWarnings(value = { - "DMI_RANDOM_USED_ONLY_ONCE" }, justification = "False positive for random object created and used only once") @Transactional(isolation = Isolation.READ_UNCOMMITTED, rollbackFor = Exception.class) public Void call() throws org.apache.fineract.infrastructure.jobs.exception.JobExecutionException { ThreadLocalContextUtil.init(this.context); - Integer maxNumberOfRetries = this.context.getTenantContext().getConnection().getMaxRetriesOnDeadlock(); - Integer maxIntervalBetweenRetries = this.context.getTenantContext().getConnection().getMaxIntervalBetweenRetries(); - // List responseList = new ArrayList<>(); - long start = System.currentTimeMillis(); - LOG.debug("Thread Execution Started at {}", start); - - List errors = new ArrayList<>(); - int i = 0; if (!savingAccounts.isEmpty()) { + List errors = new ArrayList<>(); for (SavingsAccountData savingsAccountData : savingAccounts) { - Integer numberOfRetries = 0; - while (numberOfRetries <= maxNumberOfRetries) { - try { - boolean postInterestAsOn = false; - LocalDate transactionDate = null; - long startPosting = System.currentTimeMillis(); - SavingsAccountData savingsAccountDataRet = savingsAccountWritePlatformService.postInterest(savingsAccountData, - postInterestAsOn, transactionDate, backdatedTxnsAllowedTill); - long endPosting = System.currentTimeMillis(); - savingsAccountDataList.add(savingsAccountDataRet); - - LOG.debug("Posting Completed Within {}", endPosting - startPosting); - - numberOfRetries = maxNumberOfRetries + 1; - } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) { - LOG.debug("Interest posting job for savings ID {} has been retried {} time(s)", savingsAccountData.getId(), - numberOfRetries); - // Fail if the transaction has been retired for - // maxNumberOfRetries - if (numberOfRetries >= maxNumberOfRetries) { - LOG.error( - "Interest posting job for savings ID {} has been retried for the max allowed attempts of {} and will be rolled back", - savingsAccountData.getId(), numberOfRetries); - errors.add(exception); - break; - } - // Else sleep for a random time (between 1 to 10 - // seconds) and continue - try { - int randomNum = random.nextInt(maxIntervalBetweenRetries + 1); - Thread.sleep(1000 + (randomNum * 1000)); - numberOfRetries = numberOfRetries + 1; - } catch (InterruptedException e) { - LOG.error("Interest posting job for savings retry failed due to InterruptedException", e); - errors.add(e); - break; - } - } catch (Exception e) { - LOG.error("Interest posting job for savings failed for account {}", savingsAccountData.getId(), e); - numberOfRetries = maxNumberOfRetries + 1; - errors.add(e); - } + boolean postInterestAsOn = false; + LocalDate transactionDate = null; + try { + SavingsAccountData savingsAccountDataRet = savingsAccountWritePlatformService.postInterest(savingsAccountData, + postInterestAsOn, transactionDate, backdatedTxnsAllowedTill); + savingsAccountDataList.add(savingsAccountDataRet); + } catch (Exception e) { + errors.add(e); } - i++; } - if (errors.isEmpty()) { try { batchUpdate(savingsAccountDataList); } catch (DataAccessException exception) { - LOG.error("Batch update failed due to DataAccessException", exception); + log.error("Batch update failed due to DataAccessException", exception); errors.add(exception); } catch (NullPointerException exception) { - LOG.error("Batch update failed due to NullPointerException", exception); + log.error("Batch update failed due to NullPointerException", exception); errors.add(exception); } } @@ -165,9 +104,6 @@ public class SavingsSchedularInterestPoster implements Callable { } } - long end = System.currentTimeMillis(); - LOG.debug("Time To Finish the batch {} by thread {} for accounts {}", end - start, Thread.currentThread().getId(), - savingAccounts.size()); return null; } @@ -297,10 +233,10 @@ public class SavingsSchedularInterestPoster implements Callable { this.jdbcTemplate.batchUpdate(queryForSavingsUpdate, paramsForSavingsSummary); this.jdbcTemplate.batchUpdate(queryForTransactionInsertion, paramsForTransactionInsertion); this.jdbcTemplate.batchUpdate(queryForTransactionUpdate, paramsForTransactionUpdate); - LOG.debug("`Total No Of Interest Posting:` {}", transRefNo.size()); + log.debug("`Total No Of Interest Posting:` {}", transRefNo.size()); List savingsAccountTransactionDataList = fetchTransactionsFromIds(transRefNo); if (savingsAccountDataList != null) { - LOG.debug("Fetched Transactions from DB: {}", savingsAccountTransactionDataList.size()); + log.debug("Fetched Transactions from DB: {}", savingsAccountTransactionDataList.size()); } HashMap savingsAccountTransactionMap = new HashMap<>(); diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties index d04f98c19..bd1a93a11 100644 --- a/fineract-provider/src/main/resources/application.properties +++ b/fineract-provider/src/main/resources/application.properties @@ -73,17 +73,18 @@ fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACT # Logging pattern for the console logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}} -management.health.jms.enabled=false +management.health.jms.enabled=${FINERACT_MANAGEMENT_HEALTH_JMS_ENABLED:false} # FINERACT 1296 management.endpoint.health.probes.enabled=true management.health.livenessState.enabled=true management.health.readinessState.enabled=true +management.health.ratelimiters.enabled=${FINERACT_MANAGEMENT_HEALTH_RATELIMITERS_ENABLED:false} + # FINERACT-883 management.info.git.mode=FULL -management.endpoints.web.exposure.include=health,info - +management.endpoints.web.exposure.include=${FINERACT_MANAGEMENT_ENDPOINT_WEB_EXPOSURE_INCLUDE:health,info} # FINERACT-914 server.forward-headers-strategy=framework server.port=${FINERACT_SERVER_PORT:8443} @@ -164,3 +165,26 @@ spring.main.allow-bean-definition-overriding=true spring.batch.initialize-schema=NEVER # Disabling Spring Batch jobs on startup spring.batch.job.enabled=false + +resilience4j.retry.instances.executeCommand.max-attempts=${FINERACT_COMMAND_PROCESSING_RETRY_MAX_ATTEMPTS:3} +resilience4j.retry.instances.executeCommand.wait-duration=${FINERACT_COMMAND_PROCESSING_RETRY_WAIT_DURATION:1s} +resilience4j.retry.instances.executeCommand.enable-exponential-backoff=${FINERACT_COMMAND_PROCESSING_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true} +resilience4j.retry.instances.executeCommand.exponential-backoff-multiplier=${FINERACT_COMMAND_PROCESSING_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2} +resilience4j.retry.instances.executeCommand.retryExceptions=${FINERACT_COMMAND_PROCESSING_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException} + +resilience4j.retry.instances.processJobDetailForExecution.max-attempts=${FINERACT_PROCESS_JOB_DETAIL_RETRY_MAX_ATTEMPTS:3} +resilience4j.retry.instances.processJobDetailForExecution.wait-duration=${FINERACT_PROCESS_JOB_DETAIL_RETRY_WAIT_DURATION:1s} +resilience4j.retry.instances.processJobDetailForExecution.enable-exponential-backoff=${FINERACT_PROCESS_JOB_DETAIL_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true} +resilience4j.retry.instances.processJobDetailForExecution.exponential-backoff-multiplier=${FINERACT_PROCESS_JOB_DETAIL_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2} + +resilience4j.retry.instances.recalculateInterest.max-attempts=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_MAX_ATTEMPTS:3} +resilience4j.retry.instances.recalculateInterest.wait-duration=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_WAIT_DURATION:1s} +resilience4j.retry.instances.recalculateInterest.enable-exponential-backoff=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true} +resilience4j.retry.instances.recalculateInterest.exponential-backoff-multiplier=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2} +resilience4j.retry.instances.recalculateInterest.retryExceptions=${FINERACT_PROCESS_RECALCULATE_INTEREST_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException} + +resilience4j.retry.instances.postInterest.max-attempts=${FINERACT_PROCESS_POST_INTEREST_RETRY_MAX_ATTEMPTS:3} +resilience4j.retry.instances.postInterest.wait-duration=${FINERACT_PROCESS_POST_INTEREST_RETRY_WAIT_DURATION:1s} +resilience4j.retry.instances.postInterest.enable-exponential-backoff=${FINERACT_PROCESS_POST_INTEREST_RETRY_ENABLE_EXPONENTIAL_BACKOFF:true} +resilience4j.retry.instances.postInterest.exponential-backoff-multiplier=${FINERACT_PROCESS_POST_INTEREST_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER:2} +resilience4j.retry.instances.postInterest.retryExceptions=${FINERACT_PROCESS_POST_INTEREST_RETRY_EXCEPTIONS:org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException} diff --git a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml index 74dd115ee..402892a9a 100644 --- a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml +++ b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml @@ -25,4 +25,5 @@ + diff --git a/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0006_drop_retry_parameter_columns.xml b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0006_drop_retry_parameter_columns.xml new file mode 100644 index 000000000..d8376a59f --- /dev/null +++ b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0006_drop_retry_parameter_columns.xml @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/fineract-provider/src/test/java/org/apache/fineract/commands/service/CommandServiceStepDefinitions.java b/fineract-provider/src/test/java/org/apache/fineract/commands/service/CommandServiceStepDefinitions.java new file mode 100644 index 000000000..cd8914745 --- /dev/null +++ b/fineract-provider/src/test/java/org/apache/fineract/commands/service/CommandServiceStepDefinitions.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.fineract.commands.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.cucumber.java8.En; +import io.github.resilience4j.retry.RetryRegistry; +import io.github.resilience4j.retry.event.RetryEvent; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import javax.persistence.Entity; +import javax.persistence.Table; +import org.apache.fineract.commands.domain.CommandSource; +import org.apache.fineract.commands.domain.CommandWrapper; +import org.apache.fineract.commands.exception.RollbackTransactionAsCommandIsNotApprovedByCheckerException; +import org.apache.fineract.infrastructure.core.api.JsonCommand; +import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.CannotAcquireLockException; +import org.springframework.orm.ObjectOptimisticLockingFailureException; + +public class CommandServiceStepDefinitions implements En { + + private static final Logger log = LoggerFactory.getLogger(CommandServiceStepDefinitions.class); + + @Autowired + private CommandProcessingService processAndLogCommandService; + + @Autowired + private RetryRegistry retryRegistry; + + private PortfolioCommandSourceWritePlatformService commandSourceWritePlatformService; + + private DummyCommand command; + + private RetryEvent retryEvent; + + public CommandServiceStepDefinitions() { + Given("/^A command source write service$/", () -> { + this.commandSourceWritePlatformService = new DummyCommandSourceWriteService(processAndLogCommandService); + this.command = new DummyCommand(); + this.retryRegistry.retry("executeCommand").getEventPublisher().onRetry(event -> { + log.warn("... retry event: {}", event); + + CommandServiceStepDefinitions.this.retryEvent = event; + }); + + }); + + When("/^The user executes the command via a command write service with exceptions$/", () -> { + try { + this.commandSourceWritePlatformService.logCommandSource(command); + } catch (Exception e) { + // TODO: this exception is OK for now; we need to fix the whole tenant based data source setup + log.warn("At the moment mocking data access is so incredibly hard... it's easier to just ignore this exception: {}", + e.getMessage()); + } + }); + + Then("/^The command processing service should fallback as expected$/", () -> { + assertNotNull(retryEvent); + assertEquals("executeCommand", retryEvent.getName()); + assertEquals(2, retryEvent.getNumberOfRetryAttempts()); + }); + + Then("/^The command processing service execute function should be called 3 times$/", () -> { + assertEquals(3, command.getCount()); + }); + } + + public static class DummyCommand extends CommandWrapper { + + private AtomicInteger counter = new AtomicInteger(); + + public DummyCommand() { + super(null, null, null, null, null, null, null, null, null, null, "{}", null, null, null, null, null, null, + UUID.randomUUID().toString()); + } + + @Override + public String taskPermissionName() { + // NOTE: simulating a failure scenario that triggers retries; using this function, because it is the first + // called in the command processing service + + int step = counter.incrementAndGet(); + + log.warn("Round: {}", step); + + if (step == 1) { + throw new CannotAcquireLockException("BLOW IT UP!!!"); + } else if (step == 2) { + throw new ObjectOptimisticLockingFailureException("Dummy", new RuntimeException("BLOW IT UP!!!")); + } else if (step == 3) { + throw new RollbackTransactionAsCommandIsNotApprovedByCheckerException(new DummyCommandSource()); + } + + return "dummy"; + } + + @Override + public String actionName() { + return "dummy"; + } + + public int getCount() { + return counter.get(); + } + } + + public static class DummyCommandSourceWriteService implements PortfolioCommandSourceWritePlatformService { + + private final CommandProcessingService processAndLogCommandService; + + public DummyCommandSourceWriteService(CommandProcessingService processAndLogCommandService) { + this.processAndLogCommandService = processAndLogCommandService; + } + + @Override + public CommandProcessingResult logCommandSource(CommandWrapper wrapper) { + final String json = wrapper.getJson(); + JsonCommand command = JsonCommand.from(json, null, null, wrapper.getEntityName(), wrapper.getEntityId(), + wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(), + wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId(), wrapper.getCreditBureauId(), + wrapper.getOrganisationCreditBureauId(), wrapper.getJobName()); + + return this.processAndLogCommandService.executeCommand(wrapper, command, true); + } + + @Override + public CommandProcessingResult approveEntry(Long id) { + return null; + } + + @Override + public Long rejectEntry(Long id) { + return null; + } + + @Override + public Long deleteEntry(Long makerCheckerId) { + return null; + } + } + + @Entity + @Table(name = "m_portfolio_command_source") + public static class DummyCommandSource extends CommandSource { + + public DummyCommandSource() { + setId(1L); + } + } +} diff --git a/fineract-provider/src/test/resources/application-test.properties b/fineract-provider/src/test/resources/application-test.properties index 0247e9519..d19755693 100644 --- a/fineract-provider/src/test/resources/application-test.properties +++ b/fineract-provider/src/test/resources/application-test.properties @@ -111,3 +111,26 @@ spring.main.allow-bean-definition-overriding=true spring.batch.initialize-schema=NEVER # Disabling Spring Batch jobs on startup spring.batch.job.enabled=false + +resilience4j.retry.instances.executeCommand.max-attempts=3 +resilience4j.retry.instances.executeCommand.wait-duration=1s +resilience4j.retry.instances.executeCommand.enable-exponential-backoff=true +resilience4j.retry.instances.executeCommand.exponential-backoff-multiplier=2 +resilience4j.retry.instances.executeCommand.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException + +resilience4j.retry.instances.processJobDetailForExecution.max-attempts=3 +resilience4j.retry.instances.processJobDetailForExecution.wait-duration=1s +resilience4j.retry.instances.processJobDetailForExecution.enable-exponential-backoff=true +resilience4j.retry.instances.processJobDetailForExecution.exponential-backoff-multiplier=2 + +resilience4j.retry.instances.recalculateInterest.max-attempts=3 +resilience4j.retry.instances.recalculateInterest.wait-duration=1s +resilience4j.retry.instances.recalculateInterest.enable-exponential-backoff=true +resilience4j.retry.instances.recalculateInterest.exponential-backoff-multiplier=2 +resilience4j.retry.instances.recalculateInterest.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException + +resilience4j.retry.instances.postInterest.max-attempts=3 +resilience4j.retry.instances.postInterest.wait-duration=1s +resilience4j.retry.instances.postInterest.enable-exponential-backoff=true +resilience4j.retry.instances.postInterest.exponential-backoff-multiplier=2 +resilience4j.retry.instances.postInterest.retryExceptions=org.springframework.dao.CannotAcquireLockException,org.springframework.orm.ObjectOptimisticLockingFailureException diff --git a/fineract-provider/src/test/resources/features/commands/commands.provider.feature b/fineract-provider/src/test/resources/features/commands/commands.provider.feature index 34fb1518e..b5fea5b20 100644 --- a/fineract-provider/src/test/resources/features/commands/commands.provider.feature +++ b/fineract-provider/src/test/resources/features/commands/commands.provider.feature @@ -17,7 +17,7 @@ # under the License. # -Feature: Commands Service +Feature: Commands Provider @template Scenario Outline: Verify that command providers are injected diff --git a/fineract-provider/src/test/resources/features/commands/commands.service.feature b/fineract-provider/src/test/resources/features/commands/commands.service.feature new file mode 100644 index 000000000..be49ab8ab --- /dev/null +++ b/fineract-provider/src/test/resources/features/commands/commands.service.feature @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +Feature: Commands Service + + Scenario: Verify that command source write service are working with fallback function + Given A command source write service + When The user executes the command via a command write service with exceptions + Then The command processing service should fallback as expected + Then The command processing service execute function should be called 3 times + diff --git a/fineract-provider/src/test/resources/logback.xml b/fineract-provider/src/test/resources/logback.xml index 30cf96640..9f704a984 100644 --- a/fineract-provider/src/test/resources/logback.xml +++ b/fineract-provider/src/test/resources/logback.xml @@ -34,6 +34,7 @@ +