FINERACT-2202: Bulk import fix

fineract-core/src/main/java/org/apache/fineract/portfolio/savings/DepositsApiConstants.java

@@ -181,6 +181,9 @@ public final class DepositsApiConstants {
     public static final String amountOutstandingParamName = "amountOutstanding";
     public static final String amountOrPercentageParamName = "amountOrPercentage";
     public static final String amountParamName = "amount";
+    public static final String isManualTransaction = "isManualTransaction";
+    public static final String lienTransaction = "lienTransaction";
+    public static final String chargesPaidByData = "chargesPaidByData";
     public static final String amountPaidParamName = "amountPaid";
     public static final String chargeOptionsParamName = "chargeOptions";
     public static final String chargePaymentModeParamName = "chargePaymentMode";

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java

@@ -53,7 +53,8 @@ public final class ImportHandlerUtils {
         Integer noOfEntries = 0;
         // getLastRowNum and getPhysicalNumberOfRows showing false values
         // sometimes
-        while (true) {
+        int maxRows = sheet.getLastRowNum();
+        while (noOfEntries < maxRows) {
             Row row = sheet.getRow(noOfEntries + 1);
             if (row == null) {
                 break;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/chartofaccounts/ChartOfAccountsImportHandler.java

@@ -129,7 +129,8 @@ public class ChartOfAccountsImportHandler implements ImportHandler {
         String glCode = ImportHandlerUtils.readAsString(ChartOfAcountsConstants.GL_CODE_COL, row);
         Long tagId = null;
         CodeValueData tagIdCodeValueData = null;
-        if (ImportHandlerUtils.readAsString(ChartOfAcountsConstants.TAG_ID_COL, row) != null) {
+        if (ImportHandlerUtils.readAsString(ChartOfAcountsConstants.TAG_ID_COL, row) != null
+                && !ImportHandlerUtils.readAsString(ChartOfAcountsConstants.TAG_ID_COL, row).equals("0")) {
             tagId = Long.parseLong(Objects.requireNonNull(ImportHandlerUtils.readAsString(ChartOfAcountsConstants.TAG_ID_COL, row)));
             tagIdCodeValueData = new CodeValueData().setId(tagId);
         }
@@ -147,7 +148,7 @@ public class ChartOfAccountsImportHandler implements ImportHandler {
         GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
         gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataIdSerializer());
         gsonBuilder.registerTypeAdapter(CodeValueData.class, new CodeValueDataIdSerializer());
-        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
         gsonBuilder.registerTypeAdapter(CurrencyData.class, new CurrencyDateCodeSerializer());
         int successCount = 0;
         int errorCount = 0;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/fixeddeposits/FixedDepositImportHandler.java

@@ -345,7 +345,7 @@ public class FixedDepositImportHandler implements ImportHandler {
             final String dateFormat) {
         if (closedOnDates.get(i) != null) {
             GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, closedOnDates.get(i).getLocale()));
             String payload = gsonBuilder.create().toJson(closedOnDates.get(i));
             final CommandWrapper commandRequest = new CommandWrapperBuilder() //
                     .closeFixedDepositAccount(savingsId)//
@@ -358,7 +358,7 @@ public class FixedDepositImportHandler implements ImportHandler {
 
     private CommandProcessingResult importSavings(List<FixedDepositAccountData> savings, final int i, final String dateFormat) {
         GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, savings.get(i).getLocale()));
         gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataIdSerializer());
         JsonObject savingsJsonob = gsonBuilder.create().toJsonTree(savings.get(i)).getAsJsonObject();
         savingsJsonob.remove("withdrawalFeeForTransfers");
@@ -383,7 +383,7 @@ public class FixedDepositImportHandler implements ImportHandler {
             final String dateFormat) {
         if (approvalDates.get(i) != null) {
             GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, approvalDates.get(i).getLocale()));
             String payload = gsonBuilder.create().toJson(approvalDates.get(i));
             final CommandWrapper commandRequest = new CommandWrapperBuilder() //
                     .approveFixedDepositAccountApplication(savingsId)//
@@ -398,7 +398,7 @@ public class FixedDepositImportHandler implements ImportHandler {
             final String dateFormat) {
         if (activationDates.get(i) != null) {
             GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+            gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, activationDates.get(i).getLocale()));
             String payload = gsonBuilder.create().toJson(activationDates.get(i));
             final CommandWrapper commandRequest = new CommandWrapperBuilder() //
                     .fixedDepositAccountActivation(savingsId)//

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/fixeddeposits/FixedDepositTransactionImportHandler.java

@@ -66,7 +66,7 @@ public class FixedDepositTransactionImportHandler implements ImportHandler {
     @Override
     public Count process(final Workbook workbook, final String locale, final String dateFormat) {
         List<SavingsAccountTransactionData> savingsTransactions = readExcelFile(workbook, locale, dateFormat);
-        return importEntity(workbook, savingsTransactions, dateFormat);
+        return importEntity(workbook, savingsTransactions, dateFormat, locale);
     }
 
     public List<SavingsAccountTransactionData> readExcelFile(final Workbook workbook, final String locale, final String dateFormat) {
@@ -116,13 +116,13 @@ public class FixedDepositTransactionImportHandler implements ImportHandler {
     }
 
     public Count importEntity(final Workbook workbook, final List<SavingsAccountTransactionData> savingsTransactions,
-            final String dateFormat) {
+            final String dateFormat, final String locale) {
         Sheet savingsTransactionSheet = workbook.getSheet(TemplatePopulateImportConstants.FIXED_DEPOSIT_TRANSACTION_SHEET_NAME);
         int successCount = 0;
         int errorCount = 0;
         String errorMessage = "";
         GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
         gsonBuilder.registerTypeAdapter(SavingsAccountTransactionEnumData.class, new SavingsAccountTransactionEnumValueSerialiser());
 
         for (SavingsAccountTransactionData transaction : savingsTransactions) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/journalentry/JournalEntriesImportHandler.java

@@ -63,7 +63,7 @@ public class JournalEntriesImportHandler implements ImportHandler {
     public Count process(final Workbook workbook, final String locale, final String dateFormat) {
 
         List<JournalEntryData> glTransactions = readExcelFile(workbook, locale, dateFormat);
-        return importEntity(workbook, glTransactions, dateFormat);
+        return importEntity(workbook, glTransactions, dateFormat, locale);
     }
 
     private List<JournalEntryData> readExcelFile(final Workbook workbook, final String locale, final String dateFormat) {
@@ -185,13 +185,14 @@ public class JournalEntriesImportHandler implements ImportHandler {
 
     }
 
-    private Count importEntity(final Workbook workbook, final List<JournalEntryData> glTransactions, String dateFormat) {
+    private Count importEntity(final Workbook workbook, final List<JournalEntryData> glTransactions, String dateFormat,
+            final String locale) {
         Sheet addJournalEntriesSheet = workbook.getSheet(TemplatePopulateImportConstants.JOURNAL_ENTRY_SHEET_NAME);
         int successCount = 0;
         int errorCount = 0;
         String errorMessage = "";
         GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
         gsonBuilder.registerTypeAdapter(CurrencyData.class, new CurrencyDateCodeSerializer());
 
         for (JournalEntryData transaction : glTransactions) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/recurringdeposit/RecurringDepositTransactionImportHandler.java

@@ -63,7 +63,7 @@ public class RecurringDepositTransactionImportHandler implements ImportHandler {
     @Override
     public Count process(final Workbook workbook, final String locale, final String dateFormat) {
         List<SavingsAccountTransactionData> savingsTransactions = readExcelFile(workbook, locale, dateFormat);
-        return importEntity(workbook, savingsTransactions, dateFormat);
+        return importEntity(workbook, savingsTransactions, dateFormat, locale);
     }
 
     public List<SavingsAccountTransactionData> readExcelFile(final Workbook workbook, final String locale, final String dateFormat) {
@@ -112,13 +112,13 @@ public class RecurringDepositTransactionImportHandler implements ImportHandler {
     }
 
     public Count importEntity(final Workbook workbook, final List<SavingsAccountTransactionData> savingsTransactions,
-            final String dateFormat) {
+            final String dateFormat, final String locale) {
         Sheet savingsTransactionSheet = workbook.getSheet(TemplatePopulateImportConstants.SAVINGS_TRANSACTION_SHEET_NAME);
         int successCount = 0;
         int errorCount = 0;
         String errorMessage = "";
         GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
-        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
+        gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
         gsonBuilder.registerTypeAdapter(SavingsAccountTransactionEnumData.class, new SavingsAccountTransactionEnumValueSerialiser());
 
         for (SavingsAccountTransactionData transaction : savingsTransactions) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/chartofaccounts/ChartOfAccountsWorkbook.java

@@ -189,10 +189,6 @@ public class ChartOfAccountsWorkbook extends AbstractWorkbookPopulator {
                 writeFormula(ChartOfAcountsConstants.TAG_ID_COL, row,
                         "IF(ISERROR(VLOOKUP($H" + (rowNo + 1) + ",$V$2:$W$" + (glAccounts.size() + 1) + ",2,FALSE))," + "\"\",(VLOOKUP($H"
                                 + (rowNo + 1) + ",$V$2:$W$" + (glAccounts.size() + 1) + ",2,FALSE)))");
-                // auto populate office id for bulk import of opening balance
-                writeFormula(ChartOfAcountsConstants.OFFICE_COL_ID, row,
-                        "IF(ISERROR(VLOOKUP($K" + (rowNo + 1) + ",$X$2:$Y$" + (offices.size() + 1) + ",2,FALSE)),\"\",(VLOOKUP($K"
-                                + (rowNo + 1) + ",$X$2:$Y$" + (offices.size() + 1) + ",2,FALSE)))");
             }
         } catch (Exception e) {
             LOG.error("Problem occurred in setDefaults function", e);
@@ -300,12 +296,6 @@ public class ChartOfAccountsWorkbook extends AbstractWorkbookPopulator {
         writeString(ChartOfAcountsConstants.TAG_COL, rowHeader, "Tag");
         writeString(ChartOfAcountsConstants.TAG_ID_COL, rowHeader, "Tag Id");
         writeString(ChartOfAcountsConstants.DESCRIPTION_COL, rowHeader, "Description *");
-        // adding data for opening balance bulk import
-        writeString(ChartOfAcountsConstants.OFFICE_COL, rowHeader, "Parent Office for Opening Balance");
-        writeString(ChartOfAcountsConstants.OFFICE_COL_ID, rowHeader, "Parent Office Code Opening Balance");
-        writeString(ChartOfAcountsConstants.CURRENCY_CODE, rowHeader, "Currency Code");
-        writeString(ChartOfAcountsConstants.DEBIT_AMOUNT, rowHeader, "Debit Amount");
-        writeString(ChartOfAcountsConstants.CREDIT_AMOUNT, rowHeader, "Credit Amount");
 
         writeString(ChartOfAcountsConstants.LOOKUP_ACCOUNT_TYPE_COL, rowHeader, "Lookup Account type");
         writeString(ChartOfAcountsConstants.LOOKUP_TAG_COL, rowHeader, "Lookup Tag");

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/fixeddeposits/FixedDepositTransactionWorkbookPopulator.java

@@ -112,8 +112,8 @@ public class FixedDepositTransactionWorkbookPopulator extends AbstractWorkbookPo
                 .createExplicitListConstraint(new String[] { "Withdrawal", "Deposit" });
         DataValidationConstraint paymentTypeConstraint = validationHelper.createFormulaListConstraint("PaymentTypes");
         DataValidationConstraint transactionDateConstraint = validationHelper.createDateConstraint(
-                DataValidationConstraint.OperatorType.BETWEEN, "=VLOOKUP($C1,$Q$2:$T$" + (savingsAccounts.size() + 1) + ",4,FALSE)",
+                DataValidationConstraint.OperatorType.BETWEEN,
-                "=TODAY()", dateFormat);
+                "=DATEVALUE(VLOOKUP($C1,$Q$2:$T$" + (savingsAccounts.size() + 1) + ",4,FALSE))", "=TODAY()", dateFormat);
 
         DataValidation officeValidation = validationHelper.createValidation(officeNameConstraint, officeNameRange);
         DataValidation clientValidation = validationHelper.createValidation(clientNameConstraint, clientNameRange);

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/fixeddeposits/FixedDepositWorkbookPopulator.java

@@ -115,7 +115,7 @@ public class FixedDepositWorkbookPopulator extends AbstractWorkbookPopulator {
                 .createFormulaListConstraint("INDIRECT(CONCATENATE(\"Staff_\",$A1))");
         DataValidationConstraint submittedDateConstraint = validationHelper.createDateConstraint(
                 DataValidationConstraint.OperatorType.BETWEEN,
-                "=VLOOKUP($B1,$AF$2:$AG$" + (clientSheetPopulator.getClientsSize() + 1) + ",2,FALSE)", "=TODAY()", dateFormat);
+                "=DATEVALUE(VLOOKUP($B1,$AF$2:$AG$" + (clientSheetPopulator.getClientsSize() + 1) + ",2,FALSE))", "=TODAY()", dateFormat);
         DataValidationConstraint approvalDateConstraint = validationHelper
                 .createDateConstraint(DataValidationConstraint.OperatorType.BETWEEN, "=$E1", "=TODAY()", dateFormat);
         DataValidationConstraint activationDateConstraint = validationHelper
@@ -341,7 +341,7 @@ public class FixedDepositWorkbookPopulator extends AbstractWorkbookPopulator {
         writeString(FixedDepositConstants.INTEREST_CALCULATION_DAYS_IN_YEAR_COL, rowHeader, "# Days in Year*");
         writeString(FixedDepositConstants.LOCKIN_PERIOD_COL, rowHeader, "Locked In For");
         writeString(FixedDepositConstants.DEPOSIT_AMOUNT_COL, rowHeader, "Deposit Amount");
-        writeString(FixedDepositConstants.DEPOSIT_PERIOD_COL, rowHeader, "Deposit Period");
+        writeString(FixedDepositConstants.DEPOSIT_PERIOD_COL, rowHeader, "Deposit Period*");
         writeString(FixedDepositConstants.EXTERNAL_ID_COL, rowHeader, "External Id");
 
         writeString(FixedDepositConstants.CHARGE_ID_1, rowHeader, "Charge Id");

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/loanrepayment/LoanRepaymentWorkbookPopulator.java

@@ -25,6 +25,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import org.apache.fineract.infrastructure.bulkimport.constants.LoanRepaymentConstants;
 import org.apache.fineract.infrastructure.bulkimport.constants.TemplatePopulateImportConstants;
 import org.apache.fineract.infrastructure.bulkimport.populator.AbstractWorkbookPopulator;
@@ -185,12 +186,12 @@ public class LoanRepaymentWorkbookPopulator extends AbstractWorkbookPopulator {
         String clientName = "";
         String clientId = "";
         for (int i = 0; i < allloans.size(); i++) {
-            if (!clientName.equals(allloans.get(i).getClientName())) {
+            if (!Objects.equals(clientName, allloans.get(i).getClientName())) {
                 endIndex = i + 1;
                 clientNameToBeginEndIndexes.put(clientName, new Integer[] { startIndex, endIndex });
                 startIndex = i + 2;
                 clientName = allloans.get(i).getClientName();
-                clientId = allloans.get(i).getClientId().toString();
+                clientId = String.valueOf(allloans.get(i).getClientId());
                 if (!clientsWithActiveLoans.contains(clientName)) {
                     clientsWithActiveLoans.add(clientName);
                     clientIdsWithActiveLoans.add(clientId);

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/recurringdeposit/RecurringDepositTransactionWorkbookPopulator.java

@@ -113,8 +113,8 @@ public class RecurringDepositTransactionWorkbookPopulator extends AbstractWorkbo
                 .createExplicitListConstraint(new String[] { "Withdrawal", "Deposit" });
         DataValidationConstraint paymentTypeConstraint = validationHelper.createFormulaListConstraint("PaymentTypes");
         DataValidationConstraint transactionDateConstraint = validationHelper.createDateConstraint(
-                DataValidationConstraint.OperatorType.BETWEEN, "=VLOOKUP($C1,$Q$2:$T$" + (savingsAccounts.size() + 1) + ",4,FALSE)",
+                DataValidationConstraint.OperatorType.BETWEEN,
-                "=TODAY()", dateFormat);
+                "=DATEVALUE(VLOOKUP($C1,$Q$2:$T$" + (savingsAccounts.size() + 1) + ",4,FALSE))", "=TODAY()", dateFormat);
 
         DataValidation officeValidation = validationHelper.createValidation(officeNameConstraint, officeNameRange);
         DataValidation clientValidation = validationHelper.createValidation(clientNameConstraint, clientNameRange);

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/data/DepositAccountTransactionDataValidator.java

@@ -61,10 +61,12 @@ public class DepositAccountTransactionDataValidator {
 
     private final FromJsonHelper fromApiJsonHelper;
 
-    private static final Set<String> DEPOSIT_ACCOUNT_TRANSACTION_REQUEST_DATA_PARAMETERS = new HashSet<>(
+    private static final Set<String> DEPOSIT_ACCOUNT_TRANSACTION_REQUEST_DATA_PARAMETERS = new HashSet<>(Arrays.asList(
-            Arrays.asList(DepositsApiConstants.localeParamName, DepositsApiConstants.dateFormatParamName, transactionDateParamName,
+            DepositsApiConstants.localeParamName, DepositsApiConstants.dateFormatParamName, transactionDateParamName,
-                    transactionAmountParamName, paymentTypeIdParamName, transactionAccountNumberParamName, checkNumberParamName,
+            transactionAmountParamName, paymentTypeIdParamName, transactionAccountNumberParamName, checkNumberParamName,
-                    routingCodeParamName, receiptNumberParamName, bankNumberParamName));
+            routingCodeParamName, receiptNumberParamName, bankNumberParamName, DepositsApiConstants.amountParamName,
+            DepositsApiConstants.accountIdParamName, DepositsApiConstants.dateParamName, DepositsApiConstants.submittedOnDateParamName,
+            DepositsApiConstants.lienTransaction, DepositsApiConstants.isManualTransaction, DepositsApiConstants.chargesPaidByData));
 
     private static final Set<String> DEPOSIT_ACCOUNT_RECOMMENDED_DEPOSIT_AMOUNT_UPDATE_REQUEST_DATA_PARAMETERS = new HashSet<>(
             Arrays.asList(DepositsApiConstants.localeParamName, DepositsApiConstants.dateFormatParamName,

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/DepositAccountWritePlatformServiceJpaRepositoryImpl.java

@@ -349,7 +349,25 @@ public class DepositAccountWritePlatformServiceJpaRepositoryImpl implements Depo
     @Override
     public CommandProcessingResult depositToFDAccount(final Long savingsId, @SuppressWarnings("unused") final JsonCommand command) {
         // this.context.authenticatedUser();
-        throw new DepositAccountTransactionNotAllowedException(savingsId, "deposit", DepositAccountType.FIXED_DEPOSIT);
+        this.depositAccountTransactionDataValidator.validate(command, DepositAccountType.FIXED_DEPOSIT);
+
+        final FixedDepositAccount account = (FixedDepositAccount) this.depositAccountAssembler.assembleFrom(savingsId,
+                DepositAccountType.FIXED_DEPOSIT);
+        checkClientOrGroupActive(account);
+
+        final Locale locale = command.extractLocale();
+        final DateTimeFormatter fmt = DateTimeFormatter.ofPattern(command.dateFormat()).withLocale(locale);
+
+        final LocalDate transactionDate = command.localDateValueOfParameterNamed("transactionDate");
+        final BigDecimal transactionAmount = command.bigDecimalValueOfParameterNamed("transactionAmount");
+
+        final Map<String, Object> changes = new LinkedHashMap<>();
+        final PaymentDetail paymentDetail = this.paymentDetailWritePlatformService.createAndPersistPaymentDetail(command, changes);
+        final SavingsAccountTransaction deposit = this.depositAccountDomainService.handleFDDeposit(account, fmt, transactionDate,
+                transactionAmount, paymentDetail);
+
+        return new CommandProcessingResultBuilder().withEntityId(deposit.getId()).withOfficeId(account.officeId())
+                .withClientId(account.clientId()).withGroupId(account.groupId()).withSavingsId(savingsId).with(changes).build();
 
     }