MIFOSX-1908

mifosng-provider/src/main/java/org/mifosplatform/commands/service/PortfolioCommandSourceWritePlatformServiceImpl.java

@@ -17,6 +17,7 @@ import org.mifosplatform.commands.exception.RollbackTransactionAsCommandIsNotApp
 import org.mifosplatform.infrastructure.core.api.JsonCommand;
 import org.mifosplatform.infrastructure.core.data.CommandProcessingResult;
 import org.mifosplatform.infrastructure.core.serialization.FromJsonHelper;
+import org.mifosplatform.infrastructure.core.service.ThreadLocalContextUtil;
 import org.mifosplatform.infrastructure.jobs.service.SchedulerJobRunnerReadService;
 import org.mifosplatform.infrastructure.security.service.PlatformSecurityContext;
 import org.mifosplatform.useradministration.domain.AppUser;
@@ -72,18 +73,19 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio
         final String json = wrapper.getJson();
         CommandProcessingResult result = null;
         JsonCommand command = null;
-        Integer numberOfRetries = 1;
-        Integer maxNumberOfRetries = 10;
+        Integer numberOfRetries = 0;
+        Integer maxNumberOfRetries = ThreadLocalContextUtil.getTenant().getMaxRetriesOnDeadlock();
+        Integer maxIntervalBetweenRetries = ThreadLocalContextUtil.getTenant().getMaxIntervalBetweenRetries();
         final JsonElement parsedCommand = this.fromApiJsonHelper.parse(json);
         command = JsonCommand.from(json, parsedCommand, this.fromApiJsonHelper, wrapper.getEntityName(), wrapper.getEntityId(),
                 wrapper.getSubentityId(), wrapper.getGroupId(), wrapper.getClientId(), wrapper.getLoanId(), wrapper.getSavingsId(),
                 wrapper.getTransactionId(), wrapper.getHref(), wrapper.getProductId());
-        while (numberOfRetries < maxNumberOfRetries) {
+        while (numberOfRetries <= maxNumberOfRetries) {
             try {
                 result = this.processAndLogCommandService.processAndLogCommand(wrapper, command, isApprovedByChecker);
-                numberOfRetries = maxNumberOfRetries;
+                numberOfRetries = maxNumberOfRetries + 1;
             } catch (CannotAcquireLockException | ObjectOptimisticLockingFailureException exception) {
-                logger.warn("The following command " + command.json() + " has been retried for the " + numberOfRetries + " time");
+                logger.info("The following command " + command.json() + " has been retried  " + numberOfRetries + " time(s)");
                 /***
                  * Fail if the transaction has been retired for
                  * maxNumberOfRetries
@@ -99,14 +101,14 @@ public class PortfolioCommandSourceWritePlatformServiceImpl implements Portfolio
                  **/
                 try {
                     Random random = new Random();
-                    int randomNum = random.nextInt(10) + 1;
+                    int randomNum = random.nextInt(maxIntervalBetweenRetries + 1);
                     Thread.sleep(1000 + (randomNum * 1000));
                     numberOfRetries = numberOfRetries + 1;
                 } catch (InterruptedException e) {
                     throw (exception);
                 }
             } catch (final RollbackTransactionAsCommandIsNotApprovedByCheckerException e) {
-
+                numberOfRetries = maxNumberOfRetries + 1;
                 result = this.processAndLogCommandService.logCommand(e.getCommandSourceResult());
             }
         }

mifosng-provider/src/main/java/org/mifosplatform/infrastructure/core/domain/MifosPlatformTenant.java

@@ -30,13 +30,16 @@ public class MifosPlatformTenant {
     private final int suspectTimeout;
     private final int timeBetweenEvictionRunsMillis;
     private final int minEvictableIdleTimeMillis;
+    private final int maxRetriesOnDeadlock;
+    private final int maxIntervalBetweenRetries;
 
     public MifosPlatformTenant(final Long id, final String tenantIdentifier, final String name, final String schemaName,
             final String schemaServer, final String schemaServerPort, final String schemaUsername, final String schemaPassword,
             final String timezoneId, final boolean autoUpdateEnabled, final int initialSize, final boolean testOnBorrow,
             final long validationInterval, final boolean removeAbandoned, final int removeAbandonedTimeout, final boolean logAbandoned,
             final int abandonWhenPercentageFull, final int maxActive, final int minIdle, final int maxIdle, final int suspectTimeout,
-            final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis) {
+            final int timeBetweenEvictionRunsMillis, final int minEvictableIdleTimeMillis, final int maxRetriesOnDeadlock,
+            final int maxIntervalBetweenRetries) {
         this.id = id;
         this.tenantIdentifier = tenantIdentifier;
         this.name = name;
@@ -60,6 +63,8 @@ public class MifosPlatformTenant {
         this.suspectTimeout = suspectTimeout;
         this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis;
         this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis;
+        this.maxIntervalBetweenRetries = maxIntervalBetweenRetries;
+        this.maxRetriesOnDeadlock = maxRetriesOnDeadlock;
     }
 
     public String databaseURL() {
@@ -152,4 +157,12 @@ public class MifosPlatformTenant {
         return this.suspectTimeout;
     }
 
+    public int getMaxRetriesOnDeadlock() {
+        return maxRetriesOnDeadlock;
+    }
+
+    public int getMaxIntervalBetweenRetries() {
+        return maxIntervalBetweenRetries;
+    }
+
 }

mifosng-provider/src/main/java/org/mifosplatform/infrastructure/security/service/BasicAuthTenantDetailsServiceJdbc.java

@@ -43,7 +43,9 @@ public class BasicAuthTenantDetailsServiceJdbc implements BasicAuthTenantDetails
                 .append(" pool_log_abandoned as logAbandoned, pool_abandon_when_percentage_full as abandonedWhenPercentageFull, pool_test_on_borrow as testOnBorrow,  ")//
                 .append(" pool_max_active as poolMaxActive, pool_min_idle as poolMinIdle, pool_max_idle as poolMaxIdle, ")//
                 .append(" pool_suspect_timeout as poolSuspectTimeout, pool_time_between_eviction_runs_millis as poolTimeBetweenEvictionRunsMillis, ")//
-                .append(" pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis ")//
+                .append(" pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis, ")//
+                .append(" deadlock_max_retries as maxRetriesOnDeadlock, ")//
+                .append(" deadlock_max_retry_interval as maxIntervalBetweenRetries ")//
                 .append(" from tenants t");//
 
         public String schema() {
@@ -76,11 +78,23 @@ public class BasicAuthTenantDetailsServiceJdbc implements BasicAuthTenantDetails
             final int suspectTimeout = rs.getInt("poolSuspectTimeout");
             final int timeBetweenEvictionRunsMillis = rs.getInt("poolTimeBetweenEvictionRunsMillis");
             final int minEvictableIdleTimeMillis = rs.getInt("poolMinEvictableIdleTimeMillis");
+            int maxRetriesOnDeadlock = rs.getInt("maxRetriesOnDeadlock");
+            int maxIntervalBetweenRetries = rs.getInt("maxIntervalBetweenRetries");
+
+            maxRetriesOnDeadlock = bindValueInMinMaxRange(maxRetriesOnDeadlock, 0, 15);
+            maxIntervalBetweenRetries = bindValueInMinMaxRange(maxIntervalBetweenRetries, 1, 15);
 
             return new MifosPlatformTenant(id, tenantIdentifier, name, schemaName, schemaServer, schemaServerPort, schemaUsername,
                     schemaPassword, timezoneId, autoUpdateEnabled, initialSize, testOnBorrow, validationInterval, removeAbandoned,
                     removeAbandonedTimeout, logAbandoned, abandonWhenPercentageFull, maxActive, minIdle, maxIdle, suspectTimeout,
-                    timeBetweenEvictionRunsMillis, minEvictableIdleTimeMillis);
+                    timeBetweenEvictionRunsMillis, minEvictableIdleTimeMillis, maxRetriesOnDeadlock, maxIntervalBetweenRetries);
+        }
+
+        private int bindValueInMinMaxRange(final int value, int min, int max) {
+            if (value < min) {
+                return min;
+            } else if (value > max) { return max; }
+            return value;
         }
     }
 

mifosng-provider/src/main/java/org/mifosplatform/infrastructure/security/service/JdbcTenantDetailsService.java

@@ -44,7 +44,9 @@ public class JdbcTenantDetailsService implements TenantDetailsService {
                 .append(" pool_log_abandoned as logAbandoned, pool_abandon_when_percentage_full as abandonedWhenPercentageFull, pool_test_on_borrow as testOnBorrow,  ")//
                 .append(" pool_max_active as poolMaxActive, pool_min_idle as poolMinIdle, pool_max_idle as poolMaxIdle, ")//
                 .append(" pool_suspect_timeout as poolSuspectTimeout, pool_time_between_eviction_runs_millis as poolTimeBetweenEvictionRunsMillis, ")//
-                .append(" pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis ")//
+                .append(" pool_min_evictable_idle_time_millis as poolMinEvictableIdleTimeMillis, ")//
+                .append(" deadlock_max_retries as maxRetriesOnDeadlock, ")//
+                .append(" deadlock_max_retry_interval as maxIntervalBetweenRetries ")//
                 .append(" from tenants t");//
 
         public String schema() {
@@ -77,11 +79,23 @@ public class JdbcTenantDetailsService implements TenantDetailsService {
             final int suspectTimeout = rs.getInt("poolSuspectTimeout");
             final int timeBetweenEvictionRunsMillis = rs.getInt("poolTimeBetweenEvictionRunsMillis");
             final int minEvictableIdleTimeMillis = rs.getInt("poolMinEvictableIdleTimeMillis");
+            int maxRetriesOnDeadlock = rs.getInt("maxRetriesOnDeadlock");
+            int maxIntervalBetweenRetries = rs.getInt("maxIntervalBetweenRetries");
+
+            maxRetriesOnDeadlock = bindValueInMinMaxRange(maxRetriesOnDeadlock, 0, 15);
+            maxIntervalBetweenRetries = bindValueInMinMaxRange(maxIntervalBetweenRetries, 1, 15);
 
             return new MifosPlatformTenant(id, tenantIdentifier, name, schemaName, schemaServer, schemaServerPort, schemaUsername,
                     schemaPassword, timezoneId, autoUpdateEnabled, initialSize, testOnBorrow, validationInterval, removeAbandoned,
                     removeAbandonedTimeout, logAbandoned, abandonWhenPercentageFull, maxActive, minIdle, maxIdle, suspectTimeout,
-                    timeBetweenEvictionRunsMillis, minEvictableIdleTimeMillis);
+                    timeBetweenEvictionRunsMillis, minEvictableIdleTimeMillis, maxRetriesOnDeadlock, maxIntervalBetweenRetries);
+        }
+
+        private int bindValueInMinMaxRange(final int value, int min, int max) {
+            if (value < min) {
+                return min;
+            } else if (value > max) { return max; }
+            return value;
         }
     }
 

mifosng-provider/src/main/resources/sql/migrations/list_db/V3__deadlock-retry-properties.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `tenants`
+  ADD COLUMN `deadlock_max_retries` int(5) DEFAULT 0,
+  ADD COLUMN `deadlock_max_retry_interval` int(5) DEFAULT 1;