FINERACT-2326: Enhanced loan retrieval API to avoid some of the N+1 queries when loading summaries

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/exception/ErrorHandler.java

@@ -177,6 +177,7 @@ public final class ErrorHandler {
             msg = defaultMsg == null ? cause.getMessage() : defaultMsg;
             if (nre instanceof NonTransientDataAccessException) {
                 msgCode = msgCode == null ? codePfx + ".data.integrity.issue" : msgCode;
+                log.warn("Handled exception is", nre);
                 return new PlatformDataIntegrityException(msgCode, msg, param, args);
             } else if (cause instanceof OptimisticLockException) {
                 return (RuntimeException) cause;

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseSpecificSQLGenerator.java

@@ -22,28 +22,29 @@ import static java.lang.String.format;
 
 import jakarta.validation.constraints.NotNull;
 import java.math.BigInteger;
+import java.sql.SQLException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import lombok.RequiredArgsConstructor;
 import org.apache.fineract.infrastructure.core.service.DateUtils;
 import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
 import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Sort;
+import org.springframework.jdbc.datasource.DataSourceUtils;
 import org.springframework.jdbc.support.GeneratedKeyHolder;
 import org.springframework.stereotype.Component;
 
 @Component
+@RequiredArgsConstructor
 public class DatabaseSpecificSQLGenerator {
 
     private final DatabaseTypeResolver databaseTypeResolver;
+    private final RoutingDataSource dataSource;
     public static final String SELECT_CLAUSE = "SELECT %s";
-
-    @Autowired
-    public DatabaseSpecificSQLGenerator(DatabaseTypeResolver databaseTypeResolver) {
-        this.databaseTypeResolver = databaseTypeResolver;
-    }
+    public static final int IN_CLAUSE_MAX_PARAMS = 10_000;
 
     public DatabaseType getDialect() {
         return databaseTypeResolver.databaseType();
@@ -298,4 +299,67 @@ public class DatabaseSpecificSQLGenerator {
 
     }
 
+    /**
+     * Builds an SQL fragment for filtering a column by a list of IDs in a dialect-specific way.
+     * <p>
+     * For PostgreSQL:
+     * <ul>
+     * <li>Returns a fragment using {@code = ANY (?)}, where the single {@code ?} is bound to a SQL array.</li>
+     * <li>This avoids the PostgreSQL limit of 65,535 bind parameters, since all IDs are passed as one array
+     * parameter.</li>
+     * </ul>
+     * For MySQL:
+     * <ul>
+     * <li>Returns a fragment using {@code IN (?, ?, ...)}, expanding placeholders to match the number of IDs.</li>
+     * <li>MySQL does not support array parameters, so each ID must be bound as an individual parameter.</li>
+     * </ul>
+     *
+     * @param column
+     *            the name of the column to filter on (e.g. {@code "id"})
+     * @param ids
+     *            the list of IDs to include in the condition; must not be empty
+     * @return an SQL fragment representing the {@code IN} condition, ready to be appended to a query
+     */
+    public String in(String column, List<Long> ids) {
+        return switch (getDialect()) {
+            case POSTGRESQL -> column + " = ANY (?)";
+            case MYSQL -> {
+                String inSql = String.join(",", Collections.nCopies(ids.size(), "?"));
+                yield column + " IN (" + inSql + ")";
+            }
+        };
+    }
+
+    /**
+     * Provides the bind parameter values corresponding to the SQL fragment generated by {@link #in(String, List)}.
+     * <p>
+     * For PostgreSQL:
+     * <ul>
+     * <li>Returns a single-element array containing a {@link java.sql.Array} of type {@code bigint[]}.</li>
+     * <li>This array should be bound to the single {@code ?} placeholder in the {@code = ANY (?)} fragment.</li>
+     * </ul>
+     * For MySQL:
+     * <ul>
+     * <li>Returns an {@code Object[]} of the individual ID values, one per placeholder.</li>
+     * <li>This matches the expanded {@code IN (?, ?, ...)} fragment produced by {@link #in(String, List)}.</li>
+     * </ul>
+     *
+     * @param ids
+     *            the list of IDs to be bound; must not be empty
+     * @return an array of parameter values to bind in the same order as the placeholders
+     * @throws RuntimeException
+     *             if PostgreSQL array creation fails due to a {@link java.sql.SQLException}
+     */
+    public Object[] inParametersFor(List<Long> ids) {
+        return switch (getDialect()) {
+            case POSTGRESQL -> {
+                try {
+                    yield new Object[] { DataSourceUtils.getConnection(dataSource).createArrayOf("bigint", ids.toArray(new Long[0])) };
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            case MYSQL -> ids.toArray();
+        };
+    }
 }

fineract-core/src/main/java/org/apache/fineract/util/StreamUtil.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+public final class StreamUtil {
+
+    private StreamUtil() {}
+
+    public static <A, B> Collector<A, ?, B> foldLeft(final B init, final BiFunction<? super B, ? super A, ? extends B> f) {
+        return Collectors.collectingAndThen(Collectors.reducing(Function.<B>identity(), a -> b -> f.apply(b, a), Function::andThen),
+                endo -> endo.apply(init));
+    }
+
+    /**
+     * Collector that merges a stream of maps (with list values) into a single map.
+     * <p>
+     * If the same key appears in multiple maps, the lists are concatenated.
+     *
+     * Example:
+     *
+     * <pre>{@code
+     *
+     * Map<Long, List<Foo>> merged = streamOfMaps.collect(StreamUtils.mergeMapsOfLists());
+     * }</pre>
+     *
+     * @param <K>
+     *            the type of map keys
+     * @param <V>
+     *            the type of elements in the value lists
+     * @return a collector producing a merged map with concatenated list values
+     */
+    public static <K, V> Collector<Map<K, List<V>>, ?, Map<K, List<V>>> mergeMapsOfLists() {
+        return Collectors.collectingAndThen(Collectors.flatMapping((Map<K, List<V>> m) -> m.entrySet().stream(),
+                Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (list1, list2) -> {
+                    List<V> merged = new ArrayList<>(list1);
+                    merged.addAll(list2);
+                    return merged;
+                })), HashMap::new // ensures the result is mutable
+        );
+    }
+}

fineract-core/src/test/java/org/apache/fineract/infrastructure/core/service/database/DatabaseSpecificSQLGeneratorTest.java

@@ -25,7 +25,9 @@ import org.mockito.Mockito;
 public class DatabaseSpecificSQLGeneratorTest {
 
     private final DatabaseTypeResolver databaseTypeResolver = Mockito.mock(DatabaseTypeResolver.class);
-    private final DatabaseSpecificSQLGenerator databaseSpecificSQLGenerator = new DatabaseSpecificSQLGenerator(databaseTypeResolver);
+    private final RoutingDataSource dataSource = Mockito.mock(RoutingDataSource.class);
+    private final DatabaseSpecificSQLGenerator databaseSpecificSQLGenerator = new DatabaseSpecificSQLGenerator(databaseTypeResolver,
+            dataSource);
 
     @Test
     public void testCountQueryResultOnEmptyString() {