From 055fd58ea1d6cfb4ba8da7b037a32fa09f71a1ae Mon Sep 17 00:00:00 2001 From: danielales2541 Date: Thu, 3 Apr 2025 21:55:12 +0200 Subject: [PATCH] FINERACT-2202: Bulk import fix --- .../portfolio/calendar/data/CalendarData.java | 14 ++++++++ .../importhandler/ImportHandlerUtils.java | 13 +++++-- .../client/ClientEntityImportHandler.java | 6 ++-- .../group/GroupImportHandler.java | 12 +++++-- .../importhandler/helper/DateSerializer.java | 16 ++++++++- .../importhandler/loan/LoanImportHandler.java | 8 ++--- .../office/OfficeImportHandler.java | 6 ++-- .../populator/AbstractWorkbookPopulator.java | 1 - .../centers/CentersWorkbookPopulator.java | 8 ++--- .../group/GroupsWorkbookPopulator.java | 35 +++++++++++++++++-- .../GroupingTypesDataValidator.java | 1 - 11 files changed, 95 insertions(+), 25 deletions(-) diff --git a/fineract-core/src/main/java/org/apache/fineract/portfolio/calendar/data/CalendarData.java b/fineract-core/src/main/java/org/apache/fineract/portfolio/calendar/data/CalendarData.java index 04d16e0ae..476a9a309 100644 --- a/fineract-core/src/main/java/org/apache/fineract/portfolio/calendar/data/CalendarData.java +++ b/fineract-core/src/main/java/org/apache/fineract/portfolio/calendar/data/CalendarData.java @@ -198,6 +198,20 @@ public final class CalendarData implements Serializable { this.duration = null; } + public CalendarData(String title, String description, LocalDate startDate, boolean repeating, EnumOptionData frequency, + Integer interval, EnumOptionData repeatsOnDay, String dateFormat, String locale, String typeId) { + this.title = title; + this.description = description; + this.startDate = startDate; + this.repeating = repeating; + this.frequency = frequency; + this.interval = interval; + this.repeatsOnDay = repeatsOnDay; + this.dateFormat = dateFormat; + this.locale = locale; + this.typeId = typeId; + } + public void setCenterId(String centerId) { this.centerId = centerId; } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index 44aee4146..24e2c3620 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -53,10 +53,17 @@ public final class ImportHandlerUtils { Integer noOfEntries = 0; // getLastRowNum and getPhysicalNumberOfRows showing false values // sometimes - while (sheet.getRow(noOfEntries + 1) != null && sheet.getRow(noOfEntries + 1).getCell(primaryColumn) != null) { + while (true) { + Row row = sheet.getRow(noOfEntries + 1); + if (row == null) { + break; + } + Cell cell = row.getCell(primaryColumn); + if (cell == null || cell.getCellType() == CellType.BLANK) { + break; + } noOfEntries++; } - return noOfEntries; } @@ -107,7 +114,7 @@ public final class ImportHandlerUtils { String res = trimEmptyDecimalPortion(value.getStringValue()); - if (!StringUtils.isNotEmpty(res)) { + if (StringUtils.isNotEmpty(res)) { return res.trim(); } } catch (Exception e) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index 104892d03..66f73d581 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -63,7 +63,7 @@ public class ClientEntityImportHandler implements ImportHandler { public Count process(final Workbook workbook, final String locale, final String dateFormat) { List clients = readExcelFile(workbook, locale, dateFormat); - return importEntity(workbook, clients, dateFormat); + return importEntity(workbook, clients, dateFormat, locale); } private List readExcelFile(final Workbook workbook, final String locale, final String dateFormat) { @@ -195,7 +195,7 @@ public class ClientEntityImportHandler implements ImportHandler { locale, dateFormat); } - private Count importEntity(final Workbook workbook, final List clients, final String dateFormat) { + private Count importEntity(final Workbook workbook, final List clients, final String dateFormat, final String locale) { Sheet clientSheet = workbook.getSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME); int successCount = 0; @@ -203,7 +203,7 @@ public class ClientEntityImportHandler implements ImportHandler { String errorMessage; GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale)); for (ClientData client : clients) { try { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/group/GroupImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/group/GroupImportHandler.java index 68558fa79..117ac604b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/group/GroupImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/group/GroupImportHandler.java @@ -199,6 +199,7 @@ public class GroupImportHandler implements ImportHandler { statusCell.setCellValue(TemplatePopulateImportConstants.STATUS_CELL_IMPORTED); statusCell.setCellStyle(ImportHandlerUtils.getCellStyle(workbook, IndexedColors.LIGHT_GREEN)); + successCount++; } catch (RuntimeException ex) { errorCount++; LOG.error("Problem occurred in importEntity function", ex); @@ -240,10 +241,15 @@ public class GroupImportHandler implements ImportHandler { CalendarData calendarData = meetings.get(rowIndex); calendarData.setTitle("group_" + result.getGroupId().toString() + "_CollectionMeeting"); GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, meetings.get(rowIndex).getLocale())); gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataValueSerializer()); - String payload = gsonBuilder.create().toJson(calendarData); + CalendarData modifiedCalendarData = new CalendarData(calendarData.getTitle(), calendarData.getDescription(), + calendarData.getStartDate(), calendarData.isRepeating(), calendarData.getFrequency(), calendarData.getInterval(), + calendarData.getRepeatsOnDay(), calendarData.getDateFormat(), calendarData.getLocale(), calendarData.getTypeId()); + + String payload = gsonBuilder.create().toJson(modifiedCalendarData); + CommandWrapper commandWrapper = new CommandWrapper(result.getOfficeId(), result.getGroupId(), result.getClientId(), result.getLoanId(), result.getSavingsId(), null, null, null, null, null, payload, result.getTransactionId(), result.getProductId(), null, null, null, null, idempotencyKeyGenerator.create(), null); @@ -257,7 +263,7 @@ public class GroupImportHandler implements ImportHandler { private CommandProcessingResult importGroup(final List groups, final int rowIndex, final String dateFormat) { GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, groups.get(rowIndex).getLocale())); Type clientCollectionType = new TypeToken>() { }.getType(); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/helper/DateSerializer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/helper/DateSerializer.java index 4fc46c707..85f0147c8 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/helper/DateSerializer.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/helper/DateSerializer.java @@ -25,10 +25,18 @@ import com.google.gson.JsonSerializer; import java.lang.reflect.Type; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Locale; +import org.apache.commons.lang3.StringUtils; public class DateSerializer implements JsonSerializer { private final String dateFormat; + private String localeCode; + + public DateSerializer(String dateFormat, String localeCode) { + this.dateFormat = dateFormat; + this.localeCode = localeCode; + } public DateSerializer(String dateFormat) { this.dateFormat = dateFormat; @@ -36,6 +44,12 @@ public class DateSerializer implements JsonSerializer { @Override public JsonElement serialize(LocalDate src, Type typeOfSrc, JsonSerializationContext context) { - return new JsonPrimitive(src.format(DateTimeFormatter.ofPattern(dateFormat))); + DateTimeFormatter formatter; + if (StringUtils.isNotEmpty(localeCode)) { + formatter = DateTimeFormatter.ofPattern(dateFormat, new Locale(localeCode)); + } else { + formatter = DateTimeFormatter.ofPattern(dateFormat); + } + return new JsonPrimitive(src.format(formatter)); } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/loan/LoanImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/loan/LoanImportHandler.java index 119dc2a5b..b1bc4315f 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/loan/LoanImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/loan/LoanImportHandler.java @@ -462,7 +462,7 @@ public class LoanImportHandler implements ImportHandler { private Integer importLoanRepayment(final List loanRepayments, final CommandProcessingResult result, final int rowIndex, final String dateFormat) { GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, loanRepayments.get(rowIndex).getLocale())); JsonObject loanRepaymentJsonob = gsonBuilder.create().toJsonTree(loanRepayments.get(rowIndex)).getAsJsonObject(); loanRepaymentJsonob.remove("manuallyReversed"); loanRepaymentJsonob.remove("numberOfRepayments"); @@ -483,7 +483,7 @@ public class LoanImportHandler implements ImportHandler { DisbursementData disbusalData = disbursalDates.get(rowIndex); String linkAccountId = disbusalData.getLinkAccountId(); GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, approvalDates.get(rowIndex).getLocale())); if (linkAccountId != null && !EMPTY_STR.equals(linkAccountId)) { String payload = gsonBuilder.create().toJson(disbusalData); final CommandWrapper commandRequest = new CommandWrapperBuilder() // @@ -508,7 +508,7 @@ public class LoanImportHandler implements ImportHandler { final String dateFormat) { if (approvalDates.get(rowIndex) != null) { GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, approvalDates.get(rowIndex).getLocale())); String payload = gsonBuilder.create().toJson(approvalDates.get(rowIndex)); final CommandWrapper commandRequest = new CommandWrapperBuilder() // .approveLoanApplication(result.getLoanId()) // @@ -522,7 +522,7 @@ public class LoanImportHandler implements ImportHandler { private CommandProcessingResult importLoan(final List loans, final int rowIndex, final String dateFormat) { GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, loans.get(rowIndex).getLocale())); gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataValueSerializer()); JsonObject loanJsonOb = gsonBuilder.create().toJsonTree(loans.get(rowIndex)).getAsJsonObject(); loanJsonOb.remove("isLoanProductLinkedToFloatingRate"); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/office/OfficeImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/office/OfficeImportHandler.java index 49d007fe6..1f0c3838d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/office/OfficeImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/office/OfficeImportHandler.java @@ -58,7 +58,7 @@ public class OfficeImportHandler implements ImportHandler { @Override public Count process(final Workbook workbook, final String locale, final String dateFormat) { List offices = readExcelFile(workbook, locale, dateFormat); - return importEntity(workbook, offices, dateFormat); + return importEntity(workbook, offices, dateFormat, locale); } private List readExcelFile(final Workbook workbook, final String locale, final String dateFormat) { @@ -85,10 +85,10 @@ public class OfficeImportHandler implements ImportHandler { return office; } - private Count importEntity(final Workbook workbook, final List offices, final String dateFormat) { + private Count importEntity(final Workbook workbook, final List offices, final String dateFormat, final String locale) { Sheet officeSheet = workbook.getSheet(TemplatePopulateImportConstants.OFFICE_SHEET_NAME); GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder(); - gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat)); + gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale)); int successCount = 0; int errorCount = 0; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/AbstractWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/AbstractWorkbookPopulator.java index f47d29b5f..8204feaad 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/AbstractWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/AbstractWorkbookPopulator.java @@ -103,7 +103,6 @@ public abstract class AbstractWorkbookPopulator implements WorkbookPopulator { writeString(officeNameCol, row, office.getName().trim().replaceAll("[ )(]", "_")); writeDate(activationDateCol, row, "" + office.getOpeningDate().getDayOfMonth() + "/" + office.getOpeningDate().getMonthValue() + "/" + office.getOpeningDate().getYear(), dateCellStyle, dateFormat); - } } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/centers/CentersWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/centers/CentersWorkbookPopulator.java index 307aa9822..22be56719 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/centers/CentersWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/centers/CentersWorkbookPopulator.java @@ -95,10 +95,10 @@ public class CentersWorkbookPopulator extends AbstractWorkbookPopulator { writeString(CenterConstants.ACTIVATION_DATE_COL, rowHeader, "Activation Date*"); writeString(CenterConstants.SUBMITTED_ON_DATE_COL, rowHeader, "Submitted On Date"); writeString(CenterConstants.MEETING_START_DATE_COL, rowHeader, "Meeting Start Date* (On or After)"); - writeString(CenterConstants.IS_REPEATING_COL, rowHeader, "Repeat*"); - writeString(CenterConstants.FREQUENCY_COL, rowHeader, "Frequency*"); - writeString(CenterConstants.INTERVAL_COL, rowHeader, "Interval*"); - writeString(CenterConstants.REPEATS_ON_DAY_COL, rowHeader, "Repeats On*"); + writeString(CenterConstants.IS_REPEATING_COL, rowHeader, "Repeat"); + writeString(CenterConstants.FREQUENCY_COL, rowHeader, "Frequency"); + writeString(CenterConstants.INTERVAL_COL, rowHeader, "Interval"); + writeString(CenterConstants.REPEATS_ON_DAY_COL, rowHeader, "Repeats On"); writeString(CenterConstants.GROUP_NAMES_STARTING_COL, rowHeader, "Group Names* (Enter in consecutive cells horizontally)"); writeString(CenterConstants.LOOKUP_OFFICE_NAME_COL, rowHeader, "Office Name"); writeString(CenterConstants.LOOKUP_OFFICE_OPENING_DATE_COL, rowHeader, "Opening Date"); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/group/GroupsWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/group/GroupsWorkbookPopulator.java index 6748adbe9..38e9be457 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/group/GroupsWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/group/GroupsWorkbookPopulator.java @@ -30,6 +30,9 @@ import org.apache.fineract.organisation.office.data.OfficeData; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.CreationHelper; import org.apache.poi.ss.usermodel.DataValidation; import org.apache.poi.ss.usermodel.DataValidationConstraint; import org.apache.poi.ss.usermodel.DataValidationHelper; @@ -54,6 +57,14 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator { this.clientSheetPopulator = clientSheetPopulator; } + private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellStyle cellStyle) { + Cell cell = row.getCell(columnIndex); + if (cell == null) { + cell = row.createCell(columnIndex); + } + cell.setCellStyle(cellStyle); + } + @Override public void populate(Workbook workbook, String dateFormat) { Sheet groupSheet = workbook.createSheet(TemplatePopulateImportConstants.GROUP_SHEET_NAME); @@ -63,10 +74,29 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator { clientSheetPopulator.populate(workbook, dateFormat); setLayout(groupSheet); setLookupTable(groupSheet, dateFormat); + setFormatStyle(workbook, groupSheet); setRules(groupSheet, dateFormat); } + private void setFormatStyle(Workbook workbook, Sheet worksheet) { + CellStyle dateCellStyle = workbook.createCellStyle(); + CreationHelper createHelper = workbook.getCreationHelper(); + dateCellStyle.setDataFormat(createHelper.createDataFormat().getFormat("yyyy-MM-dd")); + + for (int rowIndex = 1; rowIndex < SpreadsheetVersion.EXCEL97.getMaxRows(); rowIndex++) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + + setFormatActivationAndSubmittedDate(row, GroupConstants.ACTIVATION_DATE_COL, dateCellStyle); + setFormatActivationAndSubmittedDate(row, GroupConstants.SUBMITTED_ON_DATE_COL, dateCellStyle); + setFormatActivationAndSubmittedDate(row, GroupConstants.MEETING_START_DATE_COL, dateCellStyle); + + } + } + private void setLayout(Sheet worksheet) { Row rowHeader = worksheet.createRow(TemplatePopulateImportConstants.ROWHEADER_INDEX); rowHeader.setHeight(TemplatePopulateImportConstants.ROW_HEADER_HEIGHT); @@ -172,8 +202,9 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator { .createFormulaListConstraint("INDIRECT(CONCATENATE(\"Staff_\",$B1))"); DataValidationConstraint booleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); DataValidationConstraint activationDateConstraint = validationHelper.createDateConstraint( - DataValidationConstraint.OperatorType.BETWEEN, "=VLOOKUP($B1,$IR$2:$IS" + (offices.size() + 1) + ",2,FALSE)", "=TODAY()", - dateFormat); + DataValidationConstraint.OperatorType.BETWEEN, "=DATEVALUE(VLOOKUP($B1,$IR$2:$IS" + (offices.size() + 1) + ",2,FALSE))", + "=TODAY()", dateFormat); + DataValidationConstraint submittedOnDateConstraint = validationHelper .createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=$G1", null, dateFormat); DataValidationConstraint meetingStartDateConstraint = validationHelper diff --git a/fineract-provider/src/main/java/org/apache/fineract/portfolio/group/serialization/GroupingTypesDataValidator.java b/fineract-provider/src/main/java/org/apache/fineract/portfolio/group/serialization/GroupingTypesDataValidator.java index ea6f0fcad..25470d90c 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/portfolio/group/serialization/GroupingTypesDataValidator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/portfolio/group/serialization/GroupingTypesDataValidator.java @@ -341,7 +341,6 @@ public final class GroupingTypesDataValidator { public void validateForUpdateGroup(final JsonCommand command, final Long groupId) { final String json = command.json(); - if (StringUtils.isBlank(json)) { throw new InvalidJsonException(); }