diff --git a/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecord.java b/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecord.java index ff4fec7..6718f89 100644 --- a/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecord.java +++ b/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecord.java @@ -18,7 +18,11 @@ import java.time.LocalDateTime; @Entity @Table(name = "loan_record", indexes = { @Index(name = "ix_loan_enterprise", columnList = "enterpriseId"), - @Index(name = "ix_loan_account_ref", columnList = "accountRef", unique = true), + // One row per loan per extract date, not one row per loan. DPD history is + // the training label, and a unique index on accountRef alone would let the + // book hold only its current state — every monthly extract overwriting the + // evidence the model is meant to learn from. + @Index(name = "ix_loan_account_asof", columnList = "accountRef,asOfDate", unique = true), }) @Getter @Setter @@ -32,7 +36,7 @@ public class LoanRecord { private Long id; /** Opaque loan reference from the lender's extract — not an account number. */ - @Column(nullable = false, unique = true, length = 64) + @Column(nullable = false, length = 64) private String accountRef; @Column(nullable = false, length = 24) diff --git a/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecordRepository.java b/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecordRepository.java index 44d7cf8..9ef47db 100644 --- a/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecordRepository.java +++ b/src/main/java/com/lincfusion/udyamlinc/enterprise/LoanRecordRepository.java @@ -15,5 +15,8 @@ public interface LoanRecordRepository extends JpaRepository { Optional findByAccountRef(String accountRef); + /** One row per loan per extract date — the key an upload upserts on. */ + Optional findByAccountRefAndAsOfDate(String accountRef, LocalDate asOfDate); + long countByEnterpriseId(String enterpriseId); } diff --git a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java index 5618e74..257480f 100644 --- a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java +++ b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java @@ -17,6 +17,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.stream.Collectors; /** * Turns signals into indicators. The only writer of the feature store, and the @@ -358,11 +359,28 @@ public class FeatureEngineeringService { // ── institutional derivations ─────────────────────────── - private Computed emiDelay(String enterpriseId, LocalDate asOfDate) { - List loans = loanRepository - .findByEnterpriseIdAndAsOfDateLessThanEqual(enterpriseId, asOfDate).stream() - .filter(l -> !Boolean.TRUE.equals(l.getClosed())) + /** + * The state of each open loan as of the scoring date — the newest extract per + * {@code accountRef}, never the union of every extract. + * + *

Once the book holds monthly snapshots, taking the worst DPD across all + * rows would report a loan that was sixty days down in March as sixty days + * down today. The history is there to train on, not to be summarised as the + * present. + */ + private List currentLoans(String enterpriseId, LocalDate asOfDate) { + return loanRepository.findByEnterpriseIdAndAsOfDateLessThanEqual(enterpriseId, asOfDate).stream() + .collect(Collectors.toMap( + LoanRecord::getAccountRef, + loan -> loan, + (a, b) -> a.getAsOfDate().isAfter(b.getAsOfDate()) ? a : b)) + .values().stream() + .filter(loan -> !Boolean.TRUE.equals(loan.getClosed())) .toList(); + } + + private Computed emiDelay(String enterpriseId, LocalDate asOfDate) { + List loans = currentLoans(enterpriseId, asOfDate); if (loans.isEmpty()) return Computed.missing(FeatureCatalog.NO_CBS); LocalDate newest = loans.stream().map(LoanRecord::getAsOfDate).max(LocalDate::compareTo).orElse(asOfDate); @@ -392,10 +410,7 @@ public class FeatureEngineeringService { * record of its trade, not an institutional estimate of it. */ private Computed emiToInflow(String enterpriseId, LocalDate asOfDate) { - List loans = loanRepository - .findByEnterpriseIdAndAsOfDateLessThanEqual(enterpriseId, asOfDate).stream() - .filter(l -> !Boolean.TRUE.equals(l.getClosed())) - .toList(); + List loans = currentLoans(enterpriseId, asOfDate); if (loans.isEmpty()) return Computed.missing(FeatureCatalog.NO_CBS); BigDecimal instalment = loans.stream() diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/LoanBookEntryService.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/LoanBookEntryService.java new file mode 100644 index 0000000..33a8569 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/LoanBookEntryService.java @@ -0,0 +1,332 @@ +package com.lincfusion.udyamlinc.manualentry; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.lincfusion.udyamlinc.enterprise.EnterpriseRepository; +import com.lincfusion.udyamlinc.enterprise.LoanRecord; +import com.lincfusion.udyamlinc.enterprise.LoanRecordRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; + +/** + * The institutional side of the record, uploaded rather than extracted. + * + *

A nightly CBS or LMS feed is the eventual answer; until a lender builds one, + * this is how a real loan book reaches the system. It matters more than the price + * feeds: {@code emi_delay} and {@code emi_to_inflow} are computed from it, and + * DPD history is the training label — without a real one, the model cannot + * be trained at all. + * + *

The no-PII promise is enforced here, at the parse boundary. The + * {@code enterprise} table deliberately holds no name, phone, Aadhaar or account + * number, and a bank's own export is full of exactly those columns. Every column + * outside the template is dropped before anything is persisted, and a row whose + * headers carry an identifying field is rejected with the offending column named + * — silently ignoring it would let a bank believe it had sent something we had + * promised to keep, and quietly not kept. + * + *

{@code as_of_date} is the extract date and the effective date. One row per + * loan per extract, so uploading January through June builds the DPD series the + * model needs rather than six times overwriting the present. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class LoanBookEntryService { + + public static final String ENTRY_TYPE = "LOAN_RECORD"; + + private static final List DATE_FORMATS = List.of( + DateTimeFormatter.ISO_LOCAL_DATE, + DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("d/M/yyyy"), + DateTimeFormatter.ofPattern("dd-MM-yyyy")); + + /** + * Built here rather than injected: Boot does not reliably expose an + * {@code ObjectMapper} bean, and this one only round-trips a flat map of the + * uploaded row, so it wants none of the application's serialisation config. + */ + private static final ObjectMapper JSON = new ObjectMapper(); + + /** + * Columns a lender's export commonly carries that this system must never + * store. Matching is on the normalised header, so {@code Customer Name}, + * {@code customer_name} and {@code CUSTOMERNAME} are all caught. + */ + private static final Set FORBIDDEN_COLUMNS = Set.of( + "name", "customername", "borrowername", "applicantname", "fathername", + "phone", "mobile", "mobileno", "contact", "email", + "aadhaar", "aadhar", "uid", "pan", "voterid", + "accountnumber", "accountno", "acno", "bankaccount", "ifsc", + "address", "addressline1", "dob", "dateofbirth"); + + private final TabularParser parser; + private final ManualEntryBatchRepository batchRepository; + private final ManualEntryLineRepository lineRepository; + private final LoanRecordRepository loanRepository; + private final EnterpriseRepository enterpriseRepository; + + // ── upload → preview ──────────────────────────────────── + + @Transactional + public ManualEntryPreview upload(MultipartFile file, String username) throws IOException { + TabularParser.Sheet sheet = parser.parse(file); + + ManualEntryBatch batch = batchRepository.save(ManualEntryBatch.builder() + .entryType(ENTRY_TYPE) + .filename(file.getOriginalFilename()) + .uploadedBy(username) + .status("PREVIEW") + .rowsParsed(sheet.rows().size()) + .build()); + + List forbidden = forbiddenHeaders(sheet.headers()); + Set knownEnterprises = new HashSet<>(); + enterpriseRepository.findAll().forEach(e -> knownEnterprises.add(e.getEnterpriseId().toUpperCase(Locale.ROOT))); + + Set seenInFile = new HashSet<>(); + List lines = new ArrayList<>(); + + int rowNumber = 1; + for (Map row : sheet.rows()) { + lines.add(validate(row, ++rowNumber, batch.getId(), knownEnterprises, seenInFile, forbidden)); + } + lineRepository.saveAll(lines); + + int accepted = (int) lines.stream().filter(ManualEntryLine::isAccepted).count(); + int overwriting = (int) lines.stream().filter(l -> l.isAccepted() && l.isOverwriting()).count(); + batch.setRowsAccepted(accepted); + batch.setRowsRejected(lines.size() - accepted); + batch.setRowsOverwriting(overwriting); + if (!forbidden.isEmpty()) { + batch.setNote("Identifying columns present and not stored: " + String.join(", ", forbidden)); + } + batchRepository.save(batch); + + if (!forbidden.isEmpty()) { + log.warn("Loan upload {} carried identifying columns {} — every row rejected", + batch.getId(), forbidden); + } + return new ManualEntryPreview(batch, lines); + } + + /** Header names in the file that this system has promised never to hold. */ + private List forbiddenHeaders(List headers) { + return headers.stream() + .filter(Objects::nonNull) + .filter(header -> FORBIDDEN_COLUMNS.contains( + header.replace("_", "").replace(" ", "").replace("-", "").toLowerCase(Locale.ROOT))) + .toList(); + } + + // ── validation ────────────────────────────────────────── + + private ManualEntryLine validate(Map row, int rowNumber, Long batchId, + Set knownEnterprises, Set seenInFile, + List forbidden) { + + String enterpriseRef = TabularParser.get(row, "enterprise_ref", "enterprise_id", "enterprise"); + String accountRef = TabularParser.get(row, "loan_account_ref", "account_ref", "loan_ref"); + String rawAsOf = TabularParser.get(row, "as_of_date", "extract_date", "effective_date"); + String rawDpd = TabularParser.get(row, "dpd", "days_past_due"); + + ManualEntryLine line = ManualEntryLine.builder() + .batchId(batchId) + .rowNumber(rowNumber) + .rawDistrict(enterpriseRef) + .rawCommodity(accountRef) + .rawEffectiveDate(rawAsOf) + .rawModalPrice(rawDpd) + .accepted(false) + .rawRow(writeRawRow(row)) + .build(); + + // Refusing the whole file rather than quietly dropping the column: the + // uploader must learn that this data is not wanted, before they build a + // nightly job that keeps sending it. + if (!forbidden.isEmpty()) { + return reject(line, "IDENTIFYING_COLUMN", + "remove these columns and re-upload: " + String.join(", ", forbidden)); + } + + if (enterpriseRef == null || accountRef == null || rawAsOf == null) { + return reject(line, "MISSING_REQUIRED", + "enterprise_ref, loan_account_ref and as_of_date are all required"); + } + if (!knownEnterprises.contains(enterpriseRef.trim().toUpperCase(Locale.ROOT))) { + return reject(line, "UNKNOWN_ENTERPRISE", "no enterprise with id '" + enterpriseRef + "'"); + } + + LocalDate asOf = parseDate(rawAsOf); + if (asOf == null) return reject(line, "BAD_DATE", "could not read '" + rawAsOf + "' as a date"); + if (asOf.isAfter(LocalDate.now())) { + return reject(line, "FUTURE_DATE", asOf + " is in the future"); + } + + Integer dpd = parseInt(rawDpd); + if (rawDpd != null && dpd == null) { + return reject(line, "BAD_NUMBER", "could not read '" + rawDpd + "' as a whole number of days"); + } + if (dpd != null && dpd < 0) { + return reject(line, "BAD_NUMBER", "dpd cannot be negative"); + } + + String fileKey = accountRef.trim().toUpperCase(Locale.ROOT) + ":" + asOf; + if (!seenInFile.add(fileKey)) { + return reject(line, "DUPLICATE_IN_FILE", + "another row already covers loan " + accountRef + " as at " + asOf); + } + + line.setEffectiveDate(asOf); + line.setModalPrice(dpd == null ? null : BigDecimal.valueOf(dpd)); + line.setMinPrice(parseNumber(TabularParser.get(row, "outstanding", "outstanding_amount"))); + line.setMaxPrice(parseNumber(TabularParser.get(row, "emi_amount", "instalment_amount", "emi"))); + line.setAccepted(true); + + loanRepository.findByAccountRefAndAsOfDate(accountRef.trim(), asOf).ifPresent(existing -> { + line.setOverwriting(true); + line.setPriorModalPrice(existing.getDpd() == null ? null : BigDecimal.valueOf(existing.getDpd())); + }); + + return line; + } + + private ManualEntryLine reject(ManualEntryLine line, String reason, String detail) { + line.setAccepted(false); + line.setRejectReason(reason); + line.setRejectDetail(detail); + return line; + } + + // ── commit ────────────────────────────────────────────── + + /** + * Re-reads the file's own values from the staged lines and writes one + * {@code loan_record} per (loan, extract date). + */ + @Transactional + public ManualEntryPreview commit(Long batchId, String username) { + ManualEntryBatch batch = batchRepository.findById(batchId) + .orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId)); + if (!"PREVIEW".equals(batch.getStatus())) { + throw new IllegalStateException("Batch " + batchId + " is " + batch.getStatus() + + ", and only a PREVIEW batch can be committed"); + } + + List accepted = lineRepository.findByBatchIdAndAcceptedOrderByRowNumber(batchId, true); + LocalDateTime observedAt = LocalDateTime.now(); + + for (ManualEntryLine line : accepted) { + Map row = readRawRow(line); + String accountRef = line.getRawCommodity().trim(); + + LoanRecord loan = loanRepository + .findByAccountRefAndAsOfDate(accountRef, line.getEffectiveDate()) + .orElseGet(() -> LoanRecord.builder() + .accountRef(accountRef) + .asOfDate(line.getEffectiveDate()) + .build()); + + loan.setEnterpriseId(line.getRawDistrict().trim().toUpperCase(Locale.ROOT)); + loan.setProduct(TabularParser.get(row, "product")); + loan.setSanctionedAmount(parseNumber(TabularParser.get(row, "sanctioned_amount", "sanctioned"))); + loan.setOutstandingAmount(line.getMinPrice()); + loan.setInstalmentAmount(line.getMaxPrice()); + loan.setNextDueDate(parseDate(TabularParser.get(row, "due_date", "next_due_date"))); + loan.setLastPaidDate(parseDate(TabularParser.get(row, "last_paid_date"))); + loan.setDpd(line.getModalPrice() == null ? null : line.getModalPrice().intValue()); + loan.setBucket(bucketOf(loan.getDpd(), TabularParser.get(row, "bucket"))); + loan.setClosed(parseBoolean(TabularParser.get(row, "closed"))); + // As with market signals, when we learned it is ours to stamp. + loan.setObservedAt(observedAt); + loanRepository.save(loan); + } + + batch.setStatus("COMMITTED"); + batch.setCommittedAt(observedAt); + batch.setCommittedBy(username); + batchRepository.save(batch); + + log.info("Loan batch {} committed by {} — {} loan records written", batchId, username, accepted.size()); + return new ManualEntryPreview(batch, lineRepository.findByBatchIdOrderByRowNumber(batchId)); + } + + /** Trusts the file's bucket when it supplies one, else derives it from DPD. */ + private String bucketOf(Integer dpd, String supplied) { + if (supplied != null && !supplied.isBlank()) return supplied.trim(); + if (dpd == null) return null; + if (dpd == 0) return "0"; + if (dpd <= 30) return "1-30"; + if (dpd <= 60) return "31-60"; + if (dpd <= 90) return "61-90"; + return "90+"; + } + + // ── helpers ───────────────────────────────────────────── + + @SuppressWarnings("unchecked") + private Map readRawRow(ManualEntryLine line) { + if (line.getRawRow() == null || line.getRawRow().isBlank()) return Map.of(); + try { + return JSON.readValue(line.getRawRow(), Map.class); + } catch (Exception e) { + log.warn("Could not read stored row for line {}: {}", line.getId(), e.toString()); + return Map.of(); + } + } + + private String writeRawRow(Map row) { + try { + return JSON.writeValueAsString(row); + } catch (Exception e) { + log.warn("Could not store raw row: {}", e.toString()); + return null; + } + } + + private LocalDate parseDate(String raw) { + if (raw == null || raw.isBlank()) return null; + for (DateTimeFormatter format : DATE_FORMATS) { + try { + return LocalDate.parse(raw.trim(), format); + } catch (Exception ignored) { + // try the next shape + } + } + return null; + } + + private BigDecimal parseNumber(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return new BigDecimal(raw.replace(",", "").replace("₹", "").trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private Integer parseInt(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return Integer.valueOf(raw.replace(",", "").trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private Boolean parseBoolean(String raw) { + if (raw == null || raw.isBlank()) return Boolean.FALSE; + String value = raw.trim().toLowerCase(Locale.ROOT); + return value.equals("true") || value.equals("y") || value.equals("yes") || value.equals("1"); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java index 12ad7a8..a0c63ef 100644 --- a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java @@ -29,6 +29,7 @@ import java.util.Map; public class ManualEntryController { private final ManualMarketEntryService marketEntryService; + private final LoanBookEntryService loanBookEntryService; private final ManualEntryBatchRepository batchRepository; private final ManualEntryLineRepository lineRepository; @@ -52,16 +53,49 @@ public class ManualEntryController { .body(csv.getBytes(StandardCharsets.UTF_8)); } + /** + * The loan-book template. It carries no borrower identity by design — see + * {@link LoanBookEntryService}, which rejects a file whose headers include + * one rather than dropping the column quietly. + */ + @GetMapping(value = "/template/loans", produces = "text/csv") + public ResponseEntity loanTemplate() { + String csv = """ + enterprise_ref,loan_account_ref,product,sanctioned_amount,outstanding,emi_amount,due_date,last_paid_date,dpd,bucket,closed,as_of_date + MSME-4417,MSME-4417-LN1,TERM_LOAN,250000,180000,8500,2026-08-15,2026-06-28,39,31-60,false,2026-07-31 + MSME-2861,MSME-2861-LN1,WORKING_CAPITAL,150000,96000,6200,2026-08-10,2026-07-29,2,0,false,2026-07-31 + """; + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"udyamlinc-loanbook-template.csv\"") + .contentType(MediaType.parseMediaType("text/csv")) + .body(csv.getBytes(StandardCharsets.UTF_8)); + } + // ── upload → preview → commit ─────────────────────────── - /** Stages the file and reports what it would do. Writes no market signal. */ + /** + * Stages the file and reports what it would do. Writes nothing. + * + *

{@code type} selects the template — {@code MARKET_SIGNAL} or + * {@code LOAN_RECORD}. Both go through the same batch, preview and audit + * machinery; only the validation and the destination table differ. + */ @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ApiResponse> upload(@RequestParam("file") MultipartFile file, - Authentication authentication) throws IOException { + public ApiResponse> upload( + @RequestParam("file") MultipartFile file, + @RequestParam(value = "type", defaultValue = ManualMarketEntryService.ENTRY_TYPE) String type, + Authentication authentication) throws IOException { + if (file == null || file.isEmpty()) { return ApiResponse.fail("No file uploaded"); } - ManualEntryPreview preview = marketEntryService.upload(file, username(authentication)); + String entryType = type.trim().toUpperCase(); + ManualEntryPreview preview = switch (entryType) { + case LoanBookEntryService.ENTRY_TYPE -> loanBookEntryService.upload(file, username(authentication)); + case ManualMarketEntryService.ENTRY_TYPE -> marketEntryService.upload(file, username(authentication)); + default -> throw new IllegalArgumentException( + "Unknown upload type '" + type + "'; expected MARKET_SIGNAL or LOAN_RECORD"); + }; return ApiResponse.ok( "%d parsed · %d accepted · %d rejected · %d would overwrite".formatted( preview.lines().size(), preview.accepted().size(), @@ -69,11 +103,20 @@ public class ManualEntryController { describe(preview)); } + /** The batch remembers its own type, so commit does not need to be told again. */ @PostMapping("/{batchId}/commit") public ApiResponse> commit(@PathVariable Long batchId, Authentication authentication) { - ManualEntryPreview committed = marketEntryService.commit(batchId, username(authentication)); + ManualEntryBatch staged = batchRepository.findById(batchId) + .orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId)); + + ManualEntryPreview committed = LoanBookEntryService.ENTRY_TYPE.equals(staged.getEntryType()) + ? loanBookEntryService.commit(batchId, username(authentication)) + : marketEntryService.commit(batchId, username(authentication)); + + String what = LoanBookEntryService.ENTRY_TYPE.equals(staged.getEntryType()) + ? "loan records" : "market signals"; return ApiResponse.ok( - "%d market signals written".formatted(committed.accepted().size()), + "%d %s written".formatted(committed.accepted().size(), what), describe(committed)); } diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java index 18c4168..a8cb833 100644 --- a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java @@ -52,6 +52,14 @@ public class ManualEntryLine { @Column(length = 32) private String rawModalPrice; + /** + * The whole row as supplied, as JSON. Commit reads its columns from here + * rather than from a re-parse of a file nobody kept, and an auditor asking + * what the bank actually sent gets the row rather than our reading of it. + */ + @Column(columnDefinition = "text") + private String rawRow; + // ── as resolved ───────────────────────────────────────── private Long districtId;