CBSNexora/cbsnexora-backend
4
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

FINERACT-2202: Bulk import fix

Browse Source
This commit is contained in:
danielales2541
2025-04-10 15:20:51 +02:00
committed by Adam Saghy
parent 872f97b6b6
commit 055fd58ea1
11 changed files with 95 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fineract-core/src/main/java/org/apache/fineract/portfolio/calendar/data/CalendarData.java
+14
View File
@@ -198,6 +198,20 @@ public final class CalendarData implements Serializable {
this.duration = null;
}
public CalendarData(String title, String description, LocalDate startDate, boolean repeating, EnumOptionData frequency,
Integer interval, EnumOptionData repeatsOnDay, String dateFormat, String locale, String typeId) {
this.title = title;
this.description = description;
this.startDate = startDate;
this.repeating = repeating;
this.frequency = frequency;
this.interval = interval;
this.repeatsOnDay = repeatsOnDay;
this.dateFormat = dateFormat;
this.locale = locale;
this.typeId = typeId;
}
public void setCenterId(String centerId) {
this.centerId = centerId;
}
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java
+10 -3
View File
@@ -53,10 +53,17 @@ public final class ImportHandlerUtils {
Integer noOfEntries = 0;
// getLastRowNum and getPhysicalNumberOfRows showing false values
// sometimes
while (sheet.getRow(noOfEntries + 1) != null && sheet.getRow(noOfEntries + 1).getCell(primaryColumn) != null) {
while (true) {
Row row = sheet.getRow(noOfEntries + 1);
if (row == null) {
break;
}
Cell cell = row.getCell(primaryColumn);
if (cell == null || cell.getCellType() == CellType.BLANK) {
break;
}
noOfEntries++;
}
return noOfEntries;
}
@@ -107,7 +114,7 @@ public final class ImportHandlerUtils {
String res = trimEmptyDecimalPortion(value.getStringValue());
if (!StringUtils.isNotEmpty(res)) {
if (StringUtils.isNotEmpty(res)) {
return res.trim();
}
} catch (Exception e) {
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java
+3 -3
View File
@@ -63,7 +63,7 @@ public class ClientEntityImportHandler implements ImportHandler {
public Count process(final Workbook workbook, final String locale, final String dateFormat) {
List<ClientData> clients = readExcelFile(workbook, locale, dateFormat);
return importEntity(workbook, clients, dateFormat);
return importEntity(workbook, clients, dateFormat, locale);
}
private List<ClientData> readExcelFile(final Workbook workbook, final String locale, final String dateFormat) {
@@ -195,7 +195,7 @@ public class ClientEntityImportHandler implements ImportHandler {
locale, dateFormat);
}
private Count importEntity(final Workbook workbook, final List<ClientData> clients, final String dateFormat) {
private Count importEntity(final Workbook workbook, final List<ClientData> clients, final String dateFormat, final String locale) {
Sheet clientSheet = workbook.getSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME);
int successCount = 0;
@@ -203,7 +203,7 @@ public class ClientEntityImportHandler implements ImportHandler {
String errorMessage;
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
for (ClientData client : clients) {
try {
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/group/GroupImportHandler.java
+9 -3
View File
@@ -199,6 +199,7 @@ public class GroupImportHandler implements ImportHandler {
statusCell.setCellValue(TemplatePopulateImportConstants.STATUS_CELL_IMPORTED);
statusCell.setCellStyle(ImportHandlerUtils.getCellStyle(workbook, IndexedColors.LIGHT_GREEN));
successCount++;
} catch (RuntimeException ex) {
errorCount++;
LOG.error("Problem occurred in importEntity function", ex);
@@ -240,10 +241,15 @@ public class GroupImportHandler implements ImportHandler {
CalendarData calendarData = meetings.get(rowIndex);
calendarData.setTitle("group_" + result.getGroupId().toString() + "_CollectionMeeting");
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, meetings.get(rowIndex).getLocale()));
gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataValueSerializer());
String payload = gsonBuilder.create().toJson(calendarData);
CalendarData modifiedCalendarData = new CalendarData(calendarData.getTitle(), calendarData.getDescription(),
calendarData.getStartDate(), calendarData.isRepeating(), calendarData.getFrequency(), calendarData.getInterval(),
calendarData.getRepeatsOnDay(), calendarData.getDateFormat(), calendarData.getLocale(), calendarData.getTypeId());
String payload = gsonBuilder.create().toJson(modifiedCalendarData);
CommandWrapper commandWrapper = new CommandWrapper(result.getOfficeId(), result.getGroupId(), result.getClientId(),
result.getLoanId(), result.getSavingsId(), null, null, null, null, null, payload, result.getTransactionId(),
result.getProductId(), null, null, null, null, idempotencyKeyGenerator.create(), null);
@@ -257,7 +263,7 @@ public class GroupImportHandler implements ImportHandler {
private CommandProcessingResult importGroup(final List<GroupGeneralData> groups, final int rowIndex, final String dateFormat) {
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, groups.get(rowIndex).getLocale()));
Type clientCollectionType = new TypeToken<Collection<ClientData>>() {
}.getType();
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/helper/DateSerializer.java
+15 -1
View File
@@ -25,10 +25,18 @@ import com.google.gson.JsonSerializer;
import java.lang.reflect.Type;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import org.apache.commons.lang3.StringUtils;
public class DateSerializer implements JsonSerializer<LocalDate> {
private final String dateFormat;
private String localeCode;
public DateSerializer(String dateFormat, String localeCode) {
this.dateFormat = dateFormat;
this.localeCode = localeCode;
}
public DateSerializer(String dateFormat) {
this.dateFormat = dateFormat;
@@ -36,6 +44,12 @@ public class DateSerializer implements JsonSerializer<LocalDate> {
@Override
public JsonElement serialize(LocalDate src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(src.format(DateTimeFormatter.ofPattern(dateFormat)));
DateTimeFormatter formatter;
if (StringUtils.isNotEmpty(localeCode)) {
formatter = DateTimeFormatter.ofPattern(dateFormat, new Locale(localeCode));
} else {
formatter = DateTimeFormatter.ofPattern(dateFormat);
}
return new JsonPrimitive(src.format(formatter));
}
}
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/loan/LoanImportHandler.java
+4 -4
View File
@@ -462,7 +462,7 @@ public class LoanImportHandler implements ImportHandler {
private Integer importLoanRepayment(final List<LoanTransactionData> loanRepayments, final CommandProcessingResult result,
final int rowIndex, final String dateFormat) {
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, loanRepayments.get(rowIndex).getLocale()));
JsonObject loanRepaymentJsonob = gsonBuilder.create().toJsonTree(loanRepayments.get(rowIndex)).getAsJsonObject();
loanRepaymentJsonob.remove("manuallyReversed");
loanRepaymentJsonob.remove("numberOfRepayments");
@@ -483,7 +483,7 @@ public class LoanImportHandler implements ImportHandler {
DisbursementData disbusalData = disbursalDates.get(rowIndex);
String linkAccountId = disbusalData.getLinkAccountId();
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, approvalDates.get(rowIndex).getLocale()));
if (linkAccountId != null && !EMPTY_STR.equals(linkAccountId)) {
String payload = gsonBuilder.create().toJson(disbusalData);
final CommandWrapper commandRequest = new CommandWrapperBuilder() //
@@ -508,7 +508,7 @@ public class LoanImportHandler implements ImportHandler {
final String dateFormat) {
if (approvalDates.get(rowIndex) != null) {
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, approvalDates.get(rowIndex).getLocale()));
String payload = gsonBuilder.create().toJson(approvalDates.get(rowIndex));
final CommandWrapper commandRequest = new CommandWrapperBuilder() //
.approveLoanApplication(result.getLoanId()) //
@@ -522,7 +522,7 @@ public class LoanImportHandler implements ImportHandler {
private CommandProcessingResult importLoan(final List<LoanAccountData> loans, final int rowIndex, final String dateFormat) {
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, loans.get(rowIndex).getLocale()));
gsonBuilder.registerTypeAdapter(EnumOptionData.class, new EnumOptionDataValueSerializer());
JsonObject loanJsonOb = gsonBuilder.create().toJsonTree(loans.get(rowIndex)).getAsJsonObject();
loanJsonOb.remove("isLoanProductLinkedToFloatingRate");
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/office/OfficeImportHandler.java
+3 -3
View File
@@ -58,7 +58,7 @@ public class OfficeImportHandler implements ImportHandler {
@Override
public Count process(final Workbook workbook, final String locale, final String dateFormat) {
List<OfficeData> offices = readExcelFile(workbook, locale, dateFormat);
return importEntity(workbook, offices, dateFormat);
return importEntity(workbook, offices, dateFormat, locale);
}
private List<OfficeData> readExcelFile(final Workbook workbook, final String locale, final String dateFormat) {
@@ -85,10 +85,10 @@ public class OfficeImportHandler implements ImportHandler {
return office;
}
private Count importEntity(final Workbook workbook, final List<OfficeData> offices, final String dateFormat) {
private Count importEntity(final Workbook workbook, final List<OfficeData> offices, final String dateFormat, final String locale) {
Sheet officeSheet = workbook.getSheet(TemplatePopulateImportConstants.OFFICE_SHEET_NAME);
GsonBuilder gsonBuilder = GoogleGsonSerializerHelper.createGsonBuilder();
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat));
gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale));
int successCount = 0;
int errorCount = 0;
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/AbstractWorkbookPopulator.java
-1
View File
@@ -103,7 +103,6 @@ public abstract class AbstractWorkbookPopulator implements WorkbookPopulator {
writeString(officeNameCol, row, office.getName().trim().replaceAll("[ )(]", "_"));
writeDate(activationDateCol, row, "" + office.getOpeningDate().getDayOfMonth() + "/"
+ office.getOpeningDate().getMonthValue() + "/" + office.getOpeningDate().getYear(), dateCellStyle, dateFormat);
}
}
}
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/centers/CentersWorkbookPopulator.java
+4 -4
View File
@@ -95,10 +95,10 @@ public class CentersWorkbookPopulator extends AbstractWorkbookPopulator {
writeString(CenterConstants.ACTIVATION_DATE_COL, rowHeader, "Activation Date*");
writeString(CenterConstants.SUBMITTED_ON_DATE_COL, rowHeader, "Submitted On Date");
writeString(CenterConstants.MEETING_START_DATE_COL, rowHeader, "Meeting Start Date* (On or After)");
writeString(CenterConstants.IS_REPEATING_COL, rowHeader, "Repeat*");
writeString(CenterConstants.FREQUENCY_COL, rowHeader, "Frequency*");
writeString(CenterConstants.INTERVAL_COL, rowHeader, "Interval*");
writeString(CenterConstants.REPEATS_ON_DAY_COL, rowHeader, "Repeats On*");
writeString(CenterConstants.IS_REPEATING_COL, rowHeader, "Repeat");
writeString(CenterConstants.FREQUENCY_COL, rowHeader, "Frequency");
writeString(CenterConstants.INTERVAL_COL, rowHeader, "Interval");
writeString(CenterConstants.REPEATS_ON_DAY_COL, rowHeader, "Repeats On");
writeString(CenterConstants.GROUP_NAMES_STARTING_COL, rowHeader, "Group Names* (Enter in consecutive cells horizontally)");
writeString(CenterConstants.LOOKUP_OFFICE_NAME_COL, rowHeader, "Office Name");
writeString(CenterConstants.LOOKUP_OFFICE_OPENING_DATE_COL, rowHeader, "Opening Date");
fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/group/GroupsWorkbookPopulator.java
+33 -2
View File
@@ -30,6 +30,9 @@ import org.apache.fineract.organisation.office.data.OfficeData;
import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.ss.SpreadsheetVersion;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.CreationHelper;
import org.apache.poi.ss.usermodel.DataValidation;
import org.apache.poi.ss.usermodel.DataValidationConstraint;
import org.apache.poi.ss.usermodel.DataValidationHelper;
@@ -54,6 +57,14 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator {
this.clientSheetPopulator = clientSheetPopulator;
}
private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellStyle cellStyle) {
Cell cell = row.getCell(columnIndex);
if (cell == null) {
cell = row.createCell(columnIndex);
}
cell.setCellStyle(cellStyle);
}
@Override
public void populate(Workbook workbook, String dateFormat) {
Sheet groupSheet = workbook.createSheet(TemplatePopulateImportConstants.GROUP_SHEET_NAME);
@@ -63,10 +74,29 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator {
clientSheetPopulator.populate(workbook, dateFormat);
setLayout(groupSheet);
setLookupTable(groupSheet, dateFormat);
setFormatStyle(workbook, groupSheet);
setRules(groupSheet, dateFormat);
}
private void setFormatStyle(Workbook workbook, Sheet worksheet) {
CellStyle dateCellStyle = workbook.createCellStyle();
CreationHelper createHelper = workbook.getCreationHelper();
dateCellStyle.setDataFormat(createHelper.createDataFormat().getFormat("yyyy-MM-dd"));
for (int rowIndex = 1; rowIndex < SpreadsheetVersion.EXCEL97.getMaxRows(); rowIndex++) {
Row row = worksheet.getRow(rowIndex);
if (row == null) {
row = worksheet.createRow(rowIndex);
}
setFormatActivationAndSubmittedDate(row, GroupConstants.ACTIVATION_DATE_COL, dateCellStyle);
setFormatActivationAndSubmittedDate(row, GroupConstants.SUBMITTED_ON_DATE_COL, dateCellStyle);
setFormatActivationAndSubmittedDate(row, GroupConstants.MEETING_START_DATE_COL, dateCellStyle);
}
}
private void setLayout(Sheet worksheet) {
Row rowHeader = worksheet.createRow(TemplatePopulateImportConstants.ROWHEADER_INDEX);
rowHeader.setHeight(TemplatePopulateImportConstants.ROW_HEADER_HEIGHT);
@@ -172,8 +202,9 @@ public class GroupsWorkbookPopulator extends AbstractWorkbookPopulator {
.createFormulaListConstraint("INDIRECT(CONCATENATE(\"Staff_\",$B1))");
DataValidationConstraint booleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" });
DataValidationConstraint activationDateConstraint = validationHelper.createDateConstraint(
DataValidationConstraint.OperatorType.BETWEEN, "=VLOOKUP($B1,$IR$2:$IS" + (offices.size() + 1) + ",2,FALSE)", "=TODAY()",
dateFormat);
DataValidationConstraint.OperatorType.BETWEEN, "=DATEVALUE(VLOOKUP($B1,$IR$2:$IS" + (offices.size() + 1) + ",2,FALSE))",
"=TODAY()", dateFormat);
DataValidationConstraint submittedOnDateConstraint = validationHelper
.createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=$G1", null, dateFormat);
DataValidationConstraint meetingStartDateConstraint = validationHelper
fineract-provider/src/main/java/org/apache/fineract/portfolio/group/serialization/GroupingTypesDataValidator.java
-1
View File
@@ -341,7 +341,6 @@ public final class GroupingTypesDataValidator {
public void validateForUpdateGroup(final JsonCommand command, final Long groupId) {
final String json = command.json();
if (StringUtils.isBlank(json)) {
throw new InvalidJsonException();
}