FINERACT-2304: Handle Stuck CommandProcessing Retries

fineract-core/src/main/java/org/apache/fineract/commands/configuration/RetryConfigurationAssembler.java

@@ -26,6 +26,7 @@ import java.util.Arrays;
 import lombok.AllArgsConstructor;
 import org.apache.fineract.batch.service.BatchExecutionException;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.domain.FineractRequestContextHolder;
 import org.springframework.stereotype.Service;
 
 @AllArgsConstructor
@@ -34,19 +35,32 @@ public class RetryConfigurationAssembler {
 
     public static final String EXECUTE_COMMAND = "executeCommand";
     public static final String BATCH_RETRY = "batchRetry";
+    private static final String LAST_EXECUTION_EXCEPTION_KEY = "LAST_EXECUTION_EXCEPTION";
     private final RetryRegistry registry;
     private final FineractProperties fineractProperties;
+    private final FineractRequestContextHolder fineractRequestContextHolder;
 
     private boolean isAssignableFrom(Object e, Class<? extends Throwable>[] exceptionList) {
         return Arrays.stream(exceptionList).anyMatch(re -> re.isAssignableFrom(e.getClass()));
     }
 
+    private void setLastException(Object ex) {
+        fineractRequestContextHolder.setAttribute(LAST_EXECUTION_EXCEPTION_KEY, ex.getClass());
+    }
+
+    public Class<?> getLastException() {
+        return (Class<?>) fineractRequestContextHolder.getAttribute(RetryConfigurationAssembler.LAST_EXECUTION_EXCEPTION_KEY);
+    }
+
     public Retry getRetryConfigurationForExecuteCommand() {
         Class<? extends Throwable>[] exceptionList = fineractProperties.getRetry().getInstances().getExecuteCommand().getRetryExceptions();
         RetryConfig.Builder configBuilder = buildCommonExecuteCommandConfiguration();
 
         if (exceptionList != null) {
-            configBuilder.retryOnException(e -> isAssignableFrom(e, exceptionList));
+            configBuilder.retryOnException(ex -> {
+                setLastException(ex);
+                return isAssignableFrom(ex, exceptionList);
+            });
         }
 
         RetryConfig config = configBuilder.build();
@@ -60,8 +74,10 @@ public class RetryConfigurationAssembler {
         if (exceptionList != null) {
             configBuilder.retryOnException(ex -> {
                 if (ex instanceof BatchExecutionException e) {
+                    setLastException(e.getCause().getClass());
                     return isAssignableFrom(e.getCause(), exceptionList);
                 } else {
+                    setLastException(ex);
                     return isAssignableFrom(ex, exceptionList);
                 }
             });

fineract-core/src/main/java/org/apache/fineract/commands/service/SynchronousCommandProcessingService.java

@@ -188,7 +188,13 @@ public class SynchronousCommandProcessingService implements CommandProcessingSer
         }
         CommandProcessingResultType status = CommandProcessingResultType.fromInt(command.getStatus());
         switch (status) {
-            case UNDER_PROCESSING -> throw new IdempotentCommandProcessUnderProcessingException(wrapper, idempotencyKey);
+            case UNDER_PROCESSING -> {
+                Class<?> lastExecutionExceptionClass = retryConfigurationAssembler.getLastException();
+                if (lastExecutionExceptionClass == null
+                        || IdempotentCommandProcessUnderProcessingException.class.isAssignableFrom(lastExecutionExceptionClass)) {
+                    throw new IdempotentCommandProcessUnderProcessingException(wrapper, idempotencyKey);
+                }
+            }
             case PROCESSED -> throw new IdempotentCommandProcessSucceedException(wrapper, idempotencyKey, command);
             case ERROR -> {
                 if (!retry) {

fineract-core/src/test/java/org/apache/fineract/batch/service/BatchApiServiceImplTest.java

@@ -42,6 +42,7 @@ import org.apache.fineract.batch.domain.BatchResponse;
 import org.apache.fineract.batch.exception.ErrorInfo;
 import org.apache.fineract.commands.configuration.RetryConfigurationAssembler;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.domain.FineractRequestContextHolder;
 import org.apache.fineract.infrastructure.core.exception.ErrorHandler;
 import org.apache.fineract.infrastructure.core.filters.BatchRequestPreprocessor;
 import org.apache.fineract.infrastructure.core.persistence.ExtendedJpaTransactionManager;
@@ -53,6 +54,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
+import org.mockito.Spy;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.transaction.support.DefaultTransactionStatus;
 
@@ -78,6 +80,9 @@ class BatchApiServiceImplTest {
     @Mock
     private FineractProperties fineractProperties;
 
+    @Spy
+    private FineractRequestContextHolder fineractRequestContextHolder;
+
     @InjectMocks
     private RetryConfigurationAssembler retryConfigurationAssembler;