feat(connector): core banking / LMS seam, one real adapter and four dialects
The loan book is the training label and the one input no public feed
supplies. Integrating with a lender's core is not a technical unknown —
it is an onboarding process per bank, with a schema agreed in a room.
What can be built before that room is the seam, and this is it.
CoreBankingConnector fetches and returns rows; it never writes. Mapped
rows go to LoanBookEntryService.stage — the same validator an uploaded
spreadsheet goes through, landing in the same manual_entry_batch with
the same preview. A parallel validator for the automated path would be
the obvious shape and would eventually enforce the no-PII rule
differently.
FileDropConnector is real: it reads the newest matching file from a
directory a bank delivers to, and leaves it there so a failed run is
repeatable. Finacle, FLEXCUBE, BaNCS and an in-house REST LMS are
simulated, each emitting the same portfolio in its own column dialect —
DPD_DAYS, OVERDUE_DAYS, DAYS_OVERDUE, daysPastDue — which is what makes
the case for a mapping layer rather than asserting it.
The mapping is {canonical: vendor}, keyed by canonical name, so an
unmapped vendor column has no route through and is dropped at the
boundary. CUST_NAME and MOBILE_NO stay out structurally rather than by
anyone remembering. Auto-commit is off on every profile: a sync stages
and stops, keeping the rule that nothing is written until someone has
seen what will be. simulated is stamped per run, not read from the
profile, so switching a profile live cannot make old runs look real.
Fixes a real defect this surfaced: FORBIDDEN_COLUMNS matched on the
exact normalised name, so CUST_NAME — what Finacle actually calls it —
was missed, and a directly uploaded bank file would have kept the
borrower's name in manual_entry_line.raw_row. Matching now splits
separators and camelCase, catching borrowerMobile too, while
loan_account_ref still splits clean.
Verified against a running instance: Finacle 44/44 accepted with 4
columns dropped, the file drop 2 accepted and 1 UNKNOWN_ENTERPRISE
rejected from a real CSV, and both CUST_NAME and borrowerMobile refused
at mapping save.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
LincPay
@@ -0,0 +1,150 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.lincfusion.udyamlinc.enterprise.EnterpriseRepository;
|
||||
import com.lincfusion.udyamlinc.enterprise.LoanRecordRepository;
|
||||
import com.lincfusion.udyamlinc.institution.InstitutionRepository;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Every adapter the platform can run, and the dialect each one speaks.
|
||||
*
|
||||
* <p>The four simulated dialects are not decoration. Read the {@code columns}
|
||||
* maps side by side and the case for a mapping layer makes itself: the days-past
|
||||
* due field is {@code DPD_DAYS}, {@code OVERDUE_DAYS}, {@code DAYS_OVERDUE} and
|
||||
* {@code daysPastDue} depending on whose core it is, and a bank will not be
|
||||
* renaming it for us. Column names are drawn from the public documentation and
|
||||
* common extract conventions for each core; a given bank's actual export will
|
||||
* differ again, which is exactly why the mapping is per-profile data rather than
|
||||
* per-vendor code.
|
||||
*
|
||||
* <p>{@code extraColumns} on each dialect carries the fields a real export ships
|
||||
* that this system has promised never to store — the borrower's name, a mobile
|
||||
* number. They are emitted so that the mapping screen can show them arriving and
|
||||
* being dropped unmapped at the boundary. An extract that were already clean
|
||||
* would let a reviewer assume the promise is kept by luck.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ConnectorCatalog {
|
||||
|
||||
private final FileDropConnector fileDropConnector;
|
||||
private final EnterpriseRepository enterpriseRepository;
|
||||
private final LoanRecordRepository loanRepository;
|
||||
private final InstitutionRepository institutionRepository;
|
||||
|
||||
private final Map<String, CoreBankingConnector> byVendor = new LinkedHashMap<>();
|
||||
|
||||
@PostConstruct
|
||||
void register() {
|
||||
add(fileDropConnector);
|
||||
add(simulated(finacle()));
|
||||
add(simulated(flexcube()));
|
||||
add(simulated(bancs()));
|
||||
add(simulated(customApi()));
|
||||
}
|
||||
|
||||
private void add(CoreBankingConnector connector) {
|
||||
byVendor.put(connector.vendor(), connector);
|
||||
}
|
||||
|
||||
private SimulatedCoreConnector simulated(SimulatedCoreConnector.Dialect dialect) {
|
||||
return new SimulatedCoreConnector(dialect, enterpriseRepository, loanRepository, institutionRepository);
|
||||
}
|
||||
|
||||
public List<CoreBankingConnector> all() {
|
||||
return List.copyOf(byVendor.values());
|
||||
}
|
||||
|
||||
public Optional<CoreBankingConnector> find(String vendor) {
|
||||
return Optional.ofNullable(byVendor.get(vendor == null ? "" : vendor.trim().toUpperCase(Locale.ROOT)));
|
||||
}
|
||||
|
||||
// ── dialects ────────────────────────────────────────────
|
||||
|
||||
private SimulatedCoreConnector.Dialect finacle() {
|
||||
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
|
||||
columns.put("enterprise_ref", "ENTITY_REF");
|
||||
columns.put("loan_account_ref", "CUST_ACCT_ID");
|
||||
columns.put("product", "SCHM_CODE");
|
||||
columns.put("sanctioned_amount", "SANCT_AMT");
|
||||
columns.put("outstanding", "PRIN_OS");
|
||||
columns.put("emi_amount", "INSTL_AMT");
|
||||
columns.put("due_date", "NEXT_DUE_DT");
|
||||
columns.put("last_paid_date", "LAST_PAID_DT");
|
||||
columns.put("dpd", "DPD_DAYS");
|
||||
columns.put("bucket", "ASSET_CLASS");
|
||||
columns.put("closed", "ACCT_CLS_FLG");
|
||||
columns.put("as_of_date", "EXTRACT_DT");
|
||||
return new SimulatedCoreConnector.Dialect("FINACLE", "Infosys Finacle (simulated)", "FILE",
|
||||
"Finacle exports through a scheduled CRM/Reports extract. A bank enables it, "
|
||||
+ "agrees the column list, and the file lands on the drop path — at which point "
|
||||
+ "this profile switches to the file-drop adapter with the mapping already made.",
|
||||
columns, List.of("CUST_NAME", "MOBILE_NO", "BRANCH_CODE", "CCY"));
|
||||
}
|
||||
|
||||
private SimulatedCoreConnector.Dialect flexcube() {
|
||||
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
|
||||
columns.put("enterprise_ref", "CUSTOMER_REF");
|
||||
columns.put("loan_account_ref", "ACCOUNT_NUMBER");
|
||||
columns.put("product", "PRODUCT_CODE");
|
||||
columns.put("sanctioned_amount", "AMOUNT_FINANCED");
|
||||
columns.put("outstanding", "PRINCIPAL_OUTSTANDING");
|
||||
columns.put("emi_amount", "INSTALMENT_AMOUNT");
|
||||
columns.put("due_date", "NEXT_DUE_DATE");
|
||||
columns.put("last_paid_date", "LAST_PAYMENT_DATE");
|
||||
columns.put("dpd", "OVERDUE_DAYS");
|
||||
columns.put("bucket", "NPA_STAGE");
|
||||
columns.put("closed", "ACCOUNT_STATUS");
|
||||
columns.put("as_of_date", "VALUE_DATE");
|
||||
return new SimulatedCoreConnector.Dialect("FLEXCUBE", "Oracle FLEXCUBE (simulated)", "JDBC",
|
||||
"FLEXCUBE is usually read through a reporting schema or an extract view rather than "
|
||||
+ "the transaction tables. Needs a read-only account scoped to the loan module, "
|
||||
+ "and the bank's DBA in the room.",
|
||||
columns, List.of("CUSTOMER_NAME", "CONTACT_MOBILE", "BRANCH_CODE"));
|
||||
}
|
||||
|
||||
private SimulatedCoreConnector.Dialect bancs() {
|
||||
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
|
||||
columns.put("enterprise_ref", "PARTY_REF");
|
||||
columns.put("loan_account_ref", "ARRANGEMENT_ID");
|
||||
columns.put("product", "PRODUCT_GROUP");
|
||||
columns.put("sanctioned_amount", "LIMIT_AMT");
|
||||
columns.put("outstanding", "OUTSTANDING_BAL");
|
||||
columns.put("emi_amount", "EMI");
|
||||
columns.put("due_date", "DUE_DATE");
|
||||
columns.put("last_paid_date", "PAID_DATE");
|
||||
columns.put("dpd", "DAYS_OVERDUE");
|
||||
columns.put("bucket", "CLASSIFICATION");
|
||||
columns.put("closed", "STATUS");
|
||||
columns.put("as_of_date", "POSITION_DATE");
|
||||
return new SimulatedCoreConnector.Dialect("BANCS", "TCS BaNCS (simulated)", "FILE",
|
||||
"BaNCS deployments here are typically cooperative-bank hosted. A nightly position "
|
||||
+ "file is the realistic ask; a live API rarely is in year one.",
|
||||
columns, List.of("PARTY_NAME", "MOBILE", "BRANCH"));
|
||||
}
|
||||
|
||||
private SimulatedCoreConnector.Dialect customApi() {
|
||||
LinkedHashMap<String, String> columns = new LinkedHashMap<>();
|
||||
columns.put("enterprise_ref", "borrowerRef");
|
||||
columns.put("loan_account_ref", "loanId");
|
||||
columns.put("product", "productCode");
|
||||
columns.put("sanctioned_amount", "sanctionedAmount");
|
||||
columns.put("outstanding", "outstandingPrincipal");
|
||||
columns.put("emi_amount", "instalment");
|
||||
columns.put("due_date", "nextDueDate");
|
||||
columns.put("last_paid_date", "lastRepaymentDate");
|
||||
columns.put("dpd", "daysPastDue");
|
||||
columns.put("bucket", "assetClassification");
|
||||
columns.put("closed", "isClosed");
|
||||
columns.put("as_of_date", "asOfDate");
|
||||
return new SimulatedCoreConnector.Dialect("CUSTOM_API", "In-house LMS over REST (simulated)", "REST",
|
||||
"For a lender whose loan management system is their own. They expose one paged "
|
||||
+ "read-only endpoint returning these fields; the mapping absorbs whatever they "
|
||||
+ "happen to call them.",
|
||||
columns, List.of("borrowerName", "borrowerMobile", "branchCode"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.lincfusion.udyamlinc.common.ApiResponse;
|
||||
import com.lincfusion.udyamlinc.manualentry.LoanBookEntryService;
|
||||
import com.lincfusion.udyamlinc.manualentry.ManualEntryPreview;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.format.annotation.DateTimeFormat;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* The core-banking integration surface.
|
||||
*
|
||||
* <p>Mounted under {@code /admin}, so the existing split holds without a new
|
||||
* rule: any bank user may read the catalogue, the profiles and the run history;
|
||||
* changing a mapping or triggering a sync needs a role carrying CONFIG:MANAGE.
|
||||
* Reading who is connected is exactly the sort of thing a field officer should be
|
||||
* able to check before trusting a DPD number.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/admin/connectors")
|
||||
@RequiredArgsConstructor
|
||||
public class ConnectorController {
|
||||
|
||||
private final ConnectorCatalog catalog;
|
||||
private final ConnectorProfileRepository profileRepository;
|
||||
private final ConnectorSyncService syncService;
|
||||
private final FieldMapper fieldMapper;
|
||||
|
||||
// ── catalogue ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Every adapter the platform can run, with its dialect and what it would take
|
||||
* to make it live. The console renders {@code simulated} as a badge.
|
||||
*/
|
||||
@GetMapping("/catalog")
|
||||
public ApiResponse<List<Map<String, Object>>> catalog() {
|
||||
List<Map<String, Object>> out = new ArrayList<>();
|
||||
for (CoreBankingConnector connector : catalog.all()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("vendor", connector.vendor());
|
||||
row.put("displayName", connector.displayName());
|
||||
row.put("transport", connector.transport());
|
||||
row.put("simulated", connector.simulated());
|
||||
row.put("nativeColumns", connector.nativeColumns());
|
||||
row.put("activationNote", connector.activationNote());
|
||||
row.put("suggestedMapping", suggestedMapping(connector));
|
||||
out.add(row);
|
||||
}
|
||||
return ApiResponse.ok(out);
|
||||
}
|
||||
|
||||
/** The canonical template a mapping targets, so the editor need not hardcode it. */
|
||||
@GetMapping("/canonical-columns")
|
||||
public ApiResponse<Map<String, Object>> canonicalColumns() {
|
||||
return ApiResponse.ok(Map.of(
|
||||
"columns", LoanBookEntryService.CANONICAL_COLUMNS,
|
||||
"required", LoanBookEntryService.REQUIRED_COLUMNS,
|
||||
"neverStored", LoanBookEntryService.FORBIDDEN_COLUMNS.stream().sorted().toList()));
|
||||
}
|
||||
|
||||
/**
|
||||
* A vendor's own columns, in canonical order — the draft a bank starts from.
|
||||
* Empty for the file drop, which cannot know until a file arrives.
|
||||
*/
|
||||
private Map<String, String> suggestedMapping(CoreBankingConnector connector) {
|
||||
if (!(connector instanceof SimulatedCoreConnector simulated)) {
|
||||
Map<String, String> identity = new LinkedHashMap<>();
|
||||
LoanBookEntryService.CANONICAL_COLUMNS.forEach(c -> identity.put(c, c));
|
||||
return identity;
|
||||
}
|
||||
return simulated.dialectColumns();
|
||||
}
|
||||
|
||||
// ── profiles ────────────────────────────────────────────
|
||||
|
||||
@GetMapping("/profiles")
|
||||
public ApiResponse<List<Map<String, Object>>> profiles(@RequestParam(required = false) String institutionCode) {
|
||||
List<ConnectorProfile> profiles = institutionCode == null
|
||||
? profileRepository.findAllByOrderByDisplayNameAsc()
|
||||
: profileRepository.findByInstitutionCodeOrderByDisplayNameAsc(institutionCode);
|
||||
return ApiResponse.ok(profiles.stream().map(this::describe).toList());
|
||||
}
|
||||
|
||||
@GetMapping("/profiles/{code}")
|
||||
public ApiResponse<Map<String, Object>> profile(@PathVariable String code) {
|
||||
return ApiResponse.ok(describe(profileRepository.findByCode(code)
|
||||
.orElseThrow(() -> new IllegalArgumentException("No connector profile '" + code + "'"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a profile's mapping and settings.
|
||||
*
|
||||
* <p>The mapping is validated before it is stored rather than at the next
|
||||
* nightly run — see {@link FieldMapper#validate}. A mapping that would reject
|
||||
* every row is refused here, where someone is looking at the screen.
|
||||
*/
|
||||
@PutMapping("/profiles/{code}")
|
||||
public ApiResponse<Map<String, Object>> save(@PathVariable String code,
|
||||
@RequestBody SaveProfileRequest request) {
|
||||
ConnectorProfile profile = profileRepository.findByCode(code)
|
||||
.orElseThrow(() -> new IllegalArgumentException("No connector profile '" + code + "'"));
|
||||
|
||||
if (request.mapping() != null) {
|
||||
CoreBankingConnector connector = catalog.find(profile.getVendor()).orElse(null);
|
||||
Collection<String> available = connector == null ? List.of() : connector.nativeColumns();
|
||||
FieldMapper.Validation validation = fieldMapper.validate(request.mapping(), available);
|
||||
if (!validation.valid()) {
|
||||
return ApiResponse.fail("Mapping not saved — " + String.join("; ", validation.problems()));
|
||||
}
|
||||
profile.setMappingJson(fieldMapper.write(request.mapping()));
|
||||
}
|
||||
if (request.displayName() != null) profile.setDisplayName(request.displayName());
|
||||
if (request.endpoint() != null) profile.setEndpoint(request.endpoint());
|
||||
if (request.scheduleCron() != null) profile.setScheduleCron(request.scheduleCron());
|
||||
if (request.autoCommit() != null) profile.setAutoCommit(request.autoCommit());
|
||||
if (request.active() != null) profile.setActive(request.active());
|
||||
|
||||
return ApiResponse.ok("Profile saved", describe(profileRepository.save(profile)));
|
||||
}
|
||||
|
||||
/** Creates a profile for a bank being onboarded. */
|
||||
@PostMapping("/profiles")
|
||||
public ApiResponse<Map<String, Object>> create(@RequestBody CreateProfileRequest request) {
|
||||
if (request.code() == null || request.code().isBlank()) {
|
||||
return ApiResponse.fail("A profile needs a code");
|
||||
}
|
||||
if (profileRepository.findByCode(request.code()).isPresent()) {
|
||||
return ApiResponse.fail("A profile called '" + request.code() + "' already exists");
|
||||
}
|
||||
if (catalog.find(request.vendor()).isEmpty()) {
|
||||
return ApiResponse.fail("No adapter for vendor '" + request.vendor() + "'");
|
||||
}
|
||||
|
||||
CoreBankingConnector connector = catalog.find(request.vendor()).orElseThrow();
|
||||
ConnectorProfile profile = ConnectorProfile.builder()
|
||||
.code(request.code().trim().toUpperCase(Locale.ROOT))
|
||||
.displayName(request.displayName() == null ? connector.displayName() : request.displayName())
|
||||
.vendor(connector.vendor())
|
||||
.institutionCode(request.institutionCode())
|
||||
.endpoint(request.endpoint())
|
||||
.scheduleCron(request.scheduleCron() == null ? "0 30 1 * * *" : request.scheduleCron())
|
||||
.mappingJson(fieldMapper.write(suggestedMapping(connector)))
|
||||
.autoCommit(false)
|
||||
.active(true)
|
||||
.build();
|
||||
return ApiResponse.ok("Profile created", describe(profileRepository.save(profile)));
|
||||
}
|
||||
|
||||
// ── test, sync, history ─────────────────────────────────
|
||||
|
||||
/** Reaches the source and shows what came back. Stages nothing, writes nothing. */
|
||||
@PostMapping("/profiles/{code}/test")
|
||||
public ApiResponse<Map<String, Object>> test(
|
||||
@PathVariable String code,
|
||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate asOf) {
|
||||
Map<String, Object> result = syncService.test(code, asOf);
|
||||
return Boolean.TRUE.equals(result.get("reachable"))
|
||||
? ApiResponse.ok("Reached the source", result)
|
||||
: ApiResponse.ok("Could not reach the source", result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pulls the loan book and stages it. Writes only when the profile has
|
||||
* auto-commit on, which is off by default.
|
||||
*/
|
||||
@PostMapping("/profiles/{code}/sync")
|
||||
public ApiResponse<Map<String, Object>> sync(
|
||||
@PathVariable String code,
|
||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate asOf,
|
||||
Authentication authentication) {
|
||||
|
||||
ConnectorSyncService.SyncResult result =
|
||||
syncService.sync(code, asOf, "MANUAL", username(authentication));
|
||||
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("run", result.run());
|
||||
body.put("droppedColumns", result.droppedColumns());
|
||||
body.put("extractLabel", result.extractLabel());
|
||||
if (result.preview() != null) body.put("preview", describe(result.preview()));
|
||||
|
||||
return ApiResponse.ok(result.run().getMessage(), body);
|
||||
}
|
||||
|
||||
@GetMapping("/runs")
|
||||
public ApiResponse<List<ConnectorSyncRun>> runs(@RequestParam(required = false) String profileCode,
|
||||
@RequestParam(defaultValue = "30") int limit) {
|
||||
return ApiResponse.ok(syncService.runs(profileCode, limit));
|
||||
}
|
||||
|
||||
/** The staged batch behind a run — the same preview shape the upload screen renders. */
|
||||
@GetMapping("/runs/{runId}/preview")
|
||||
public ApiResponse<Map<String, Object>> runPreview(@PathVariable Long runId) {
|
||||
ConnectorSyncRun run = syncService.runs(null, 200).stream()
|
||||
.filter(r -> r.getId().equals(runId))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalArgumentException("No such run: " + runId));
|
||||
if (run.getBatchId() == null) {
|
||||
return ApiResponse.fail("Run " + runId + " staged nothing — " + run.getMessage());
|
||||
}
|
||||
return ApiResponse.ok(describe(syncService.preview(run.getBatchId())));
|
||||
}
|
||||
|
||||
// ── shaping ─────────────────────────────────────────────
|
||||
|
||||
private Map<String, Object> describe(ConnectorProfile profile) {
|
||||
CoreBankingConnector connector = catalog.find(profile.getVendor()).orElse(null);
|
||||
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("code", profile.getCode());
|
||||
row.put("displayName", profile.getDisplayName());
|
||||
row.put("vendor", profile.getVendor());
|
||||
row.put("vendorName", connector == null ? profile.getVendor() : connector.displayName());
|
||||
row.put("transport", connector == null ? null : connector.transport());
|
||||
row.put("simulated", connector != null && connector.simulated());
|
||||
row.put("activationNote", connector == null ? null : connector.activationNote());
|
||||
row.put("institutionCode", profile.getInstitutionCode());
|
||||
row.put("endpoint", profile.getEndpoint());
|
||||
row.put("scheduleCron", profile.getScheduleCron());
|
||||
row.put("autoCommit", profile.getAutoCommit());
|
||||
row.put("active", profile.getActive());
|
||||
row.put("lastSyncAt", profile.getLastSyncAt());
|
||||
row.put("lastStatus", profile.getLastStatus());
|
||||
row.put("mapping", fieldMapper.read(profile));
|
||||
row.put("sourceColumns", connector == null ? List.of() : connector.nativeColumns());
|
||||
return row;
|
||||
}
|
||||
|
||||
private Map<String, Object> describe(ManualEntryPreview preview) {
|
||||
return Map.of(
|
||||
"batch", preview.batch(),
|
||||
"accepted", preview.accepted(),
|
||||
"rejected", preview.rejected(),
|
||||
"overwriting", preview.overwriting(),
|
||||
"warnings", preview.warnings());
|
||||
}
|
||||
|
||||
private String username(Authentication authentication) {
|
||||
return authentication == null ? "unknown" : authentication.getName();
|
||||
}
|
||||
|
||||
// ── request bodies ──────────────────────────────────────
|
||||
|
||||
public record SaveProfileRequest(String displayName, String endpoint, String scheduleCron,
|
||||
Boolean autoCommit, Boolean active, Map<String, String> mapping) {
|
||||
}
|
||||
|
||||
public record CreateProfileRequest(String code, String displayName, String vendor,
|
||||
String institutionCode, String endpoint, String scheduleCron) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* One bank's configured link to its own core — which adapter, pointed where,
|
||||
* and whose column means what.
|
||||
*
|
||||
* <p><b>The mapping is the product.</b> The adapter for a given core is written
|
||||
* once; what changes bank to bank is that this one calls the days-past-due column
|
||||
* {@code DPD_DAYS} and that one calls it {@code OVERDUE_DAYS}, and that a
|
||||
* cooperative bank's export has no instalment column at all. Holding that as
|
||||
* per-bank data rather than per-bank code is what makes onboarding a new lender a
|
||||
* configuration exercise instead of a release.
|
||||
*
|
||||
* <p>{@code mappingJson} is a flat {@code {canonical_column: vendor_column}}
|
||||
* object. Direction matters: keyed by the canonical name, so a column the bank
|
||||
* does not have is simply absent, and a vendor column nobody mapped is dropped at
|
||||
* the boundary rather than carried into the store. That default — drop unless
|
||||
* mapped — is what keeps a bank's borrower names out of a system that promised
|
||||
* not to hold them.
|
||||
*
|
||||
* <p>No credential is stored here. A live deployment needs one, and where it
|
||||
* goes is a decision for the pilot; a field on this entity would be the wrong
|
||||
* answer written down too early.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "connector_profile", indexes = {
|
||||
@Index(name = "ix_connector_profile_code", columnList = "code", unique = true),
|
||||
@Index(name = "ix_connector_profile_institution", columnList = "institutionCode"),
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class ConnectorProfile {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
/** Unique per deployment — "MPGB-FINACLE". */
|
||||
@Column(nullable = false, unique = true, length = 40)
|
||||
private String code;
|
||||
|
||||
@Column(nullable = false, length = 120)
|
||||
private String displayName;
|
||||
|
||||
/** Which {@link CoreBankingConnector} runs this profile. */
|
||||
@Column(nullable = false, length = 32)
|
||||
private String vendor;
|
||||
|
||||
/** The partner bank this reads from. Matches {@code institution.code}. */
|
||||
@Column(length = 32)
|
||||
private String institutionCode;
|
||||
|
||||
/**
|
||||
* Where the extract is: a drop directory glob for FILE, a URL for REST.
|
||||
* Free text because the adapter is the only thing that interprets it.
|
||||
*/
|
||||
@Column(length = 300)
|
||||
private String endpoint;
|
||||
|
||||
/** {@code {canonical_column: vendor_column}}. Null means "use the canonical names". */
|
||||
@Column(length = 4000)
|
||||
private String mappingJson;
|
||||
|
||||
/**
|
||||
* The nightly window a real deployment would run in. Recorded even though
|
||||
* nothing schedules it yet — the value of showing it is that the bank sees
|
||||
* when we would be reading, and can say no.
|
||||
*/
|
||||
@Column(length = 40)
|
||||
private String scheduleCron;
|
||||
|
||||
/**
|
||||
* Whether a clean sync writes without a person pressing Commit.
|
||||
*
|
||||
* <p>Off by default, and deliberately so: the manual path's rule is that
|
||||
* nothing lands until someone has seen what will, and automating a feed is
|
||||
* not a reason to drop that — it is the reason to keep it.
|
||||
*/
|
||||
@Column(nullable = false)
|
||||
private Boolean autoCommit;
|
||||
|
||||
@Column(nullable = false)
|
||||
private Boolean active;
|
||||
|
||||
private LocalDateTime lastSyncAt;
|
||||
|
||||
/** SUCCESS | PARTIAL | FAILED | NEVER_RUN — denormalised for the console list. */
|
||||
@Column(length = 16)
|
||||
private String lastStatus;
|
||||
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@PrePersist
|
||||
void onCreate() {
|
||||
if (createdAt == null) createdAt = LocalDateTime.now();
|
||||
if (active == null) active = Boolean.TRUE;
|
||||
if (autoCommit == null) autoCommit = Boolean.FALSE;
|
||||
if (lastStatus == null) lastStatus = "NEVER_RUN";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface ConnectorProfileRepository extends JpaRepository<ConnectorProfile, Long> {
|
||||
|
||||
Optional<ConnectorProfile> findByCode(String code);
|
||||
|
||||
List<ConnectorProfile> findAllByOrderByDisplayNameAsc();
|
||||
|
||||
List<ConnectorProfile> findByInstitutionCodeOrderByDisplayNameAsc(String institutionCode);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* One configured link per partner bank, so the integration screen has something
|
||||
* to be about on a fresh database.
|
||||
*
|
||||
* <p>Guarded per profile code rather than by {@code count() > 0}. That pattern
|
||||
* has bitten this codebase twice: anything added to a seeder after a database was
|
||||
* first populated never reaches it, and the symptom is a screen that is empty for
|
||||
* everyone except whoever last dropped their schema.
|
||||
*
|
||||
* <p>The three lenders are on three different cores on purpose. It is also true
|
||||
* of the sector — a regional rural bank, a district cooperative bank and a small
|
||||
* finance bank will not be on one core — and a demo where every bank is on
|
||||
* Finacle would quietly suggest that the mapping layer is optional.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class ConnectorProfileSeeder {
|
||||
|
||||
private final ConnectorProfileRepository profileRepository;
|
||||
private final ConnectorCatalog catalog;
|
||||
private final FieldMapper fieldMapper;
|
||||
|
||||
private record Seed(String code, String displayName, String vendor, String institutionCode,
|
||||
String endpoint, String cron) {
|
||||
}
|
||||
|
||||
public void seed() {
|
||||
List<Seed> seeds = List.of(
|
||||
new Seed("MPGB-FINACLE", "MP Gramin Bank — Finacle nightly extract",
|
||||
"FINACLE", "MPGB", "loanbook-*.csv", "0 30 1 * * *"),
|
||||
new Seed("SEHDCCB-DROP", "Sehore DCCB — SFTP file drop",
|
||||
FileDropConnector.VENDOR, "SEHDCCB", "sehdccb-loanbook-*.{csv,xlsx}", "0 0 2 * * *"),
|
||||
new Seed("VSFB-LMS", "Vindhya SFB — in-house LMS over REST",
|
||||
"CUSTOM_API", "VSFB", "https://lms.vindhyasfb.example/api/loans", "0 15 1 * * *"));
|
||||
|
||||
int written = 0;
|
||||
for (Seed seed : seeds) {
|
||||
if (profileRepository.findByCode(seed.code()).isPresent()) continue;
|
||||
CoreBankingConnector connector = catalog.find(seed.vendor()).orElse(null);
|
||||
if (connector == null) continue;
|
||||
|
||||
profileRepository.save(ConnectorProfile.builder()
|
||||
.code(seed.code())
|
||||
.displayName(seed.displayName())
|
||||
.vendor(connector.vendor())
|
||||
.institutionCode(seed.institutionCode())
|
||||
.endpoint(seed.endpoint())
|
||||
.scheduleCron(seed.cron())
|
||||
.mappingJson(fieldMapper.write(mappingFor(connector)))
|
||||
// Off, on every seeded profile. A demo that writes to the loan
|
||||
// book the first time someone presses Sync would be a demo that
|
||||
// taught the wrong thing about how this is meant to run.
|
||||
.autoCommit(false)
|
||||
.active(true)
|
||||
.build());
|
||||
written++;
|
||||
}
|
||||
if (written > 0) log.info("Seeded {} core-banking connector profiles", written);
|
||||
}
|
||||
|
||||
private java.util.Map<String, String> mappingFor(CoreBankingConnector connector) {
|
||||
if (connector instanceof SimulatedCoreConnector simulated) return simulated.dialectColumns();
|
||||
// A file drop has no declared dialect — start from the canonical template,
|
||||
// which is what the bank is sent when it asks what to put in the file.
|
||||
java.util.LinkedHashMap<String, String> identity = new java.util.LinkedHashMap<>();
|
||||
com.lincfusion.udyamlinc.manualentry.LoanBookEntryService.CANONICAL_COLUMNS
|
||||
.forEach(c -> identity.put(c, c));
|
||||
return identity;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* One attempt to pull a loan book, kept whether it worked or not.
|
||||
*
|
||||
* <p>Modelled on {@code IngestionRun} for the same reason: a feed that fails
|
||||
* silently is worse than one that is absent, because the scores keep being
|
||||
* computed and nothing about them says the loan book stopped moving three weeks
|
||||
* ago. A failed run is a row here, and the console reads it.
|
||||
*
|
||||
* <p>{@code batchId} points at the {@code manual_entry_batch} the rows were
|
||||
* staged into, so a connector sync is inspectable through exactly the same
|
||||
* preview screen an upload is — accepted, rejected, and what it would overwrite.
|
||||
*
|
||||
* <p>{@code simulated} is stamped per run rather than read from the profile. A
|
||||
* profile that is switched from a simulated adapter to a live one must not
|
||||
* retroactively make its old runs look real.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "connector_sync_run", indexes = {
|
||||
@Index(name = "ix_sync_run_profile", columnList = "profileCode"),
|
||||
@Index(name = "ix_sync_run_started", columnList = "startedAt"),
|
||||
})
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class ConnectorSyncRun {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(nullable = false, length = 40)
|
||||
private String profileCode;
|
||||
|
||||
@Column(length = 32)
|
||||
private String vendor;
|
||||
|
||||
@Column(length = 32)
|
||||
private String institutionCode;
|
||||
|
||||
/** MANUAL | SCHEDULED — who asked for this run. */
|
||||
@Column(nullable = false, length = 16)
|
||||
private String trigger;
|
||||
|
||||
@Column(nullable = false, length = 64)
|
||||
private String triggeredBy;
|
||||
|
||||
/** SUCCESS | PARTIAL | FAILED. PARTIAL means rows were rejected but some landed. */
|
||||
@Column(nullable = false, length = 16)
|
||||
private String status;
|
||||
|
||||
/** True when the rows were generated by a stub adapter rather than read from a core. */
|
||||
@Column(nullable = false)
|
||||
private Boolean simulated;
|
||||
|
||||
/** The date the extract describes, not the date it was pulled. */
|
||||
private java.time.LocalDate asOfDate;
|
||||
|
||||
private Integer rowsFetched;
|
||||
private Integer rowsAccepted;
|
||||
private Integer rowsRejected;
|
||||
private Integer rowsOverwriting;
|
||||
|
||||
/** Vendor columns nobody mapped, and therefore never crossed the boundary. */
|
||||
private Integer columnsDropped;
|
||||
|
||||
/** The staged batch, so the run opens in the same preview an upload does. */
|
||||
private Long batchId;
|
||||
|
||||
/** Whether this run's rows were written, or are still sitting in preview. */
|
||||
@Column(nullable = false)
|
||||
private Boolean committed;
|
||||
|
||||
private Integer durationMs;
|
||||
|
||||
@Column(nullable = false)
|
||||
private LocalDateTime startedAt;
|
||||
|
||||
@Column(length = 600)
|
||||
private String message;
|
||||
|
||||
@PrePersist
|
||||
void onCreate() {
|
||||
if (startedAt == null) startedAt = LocalDateTime.now();
|
||||
if (committed == null) committed = Boolean.FALSE;
|
||||
if (simulated == null) simulated = Boolean.TRUE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ConnectorSyncRunRepository extends JpaRepository<ConnectorSyncRun, Long> {
|
||||
|
||||
List<ConnectorSyncRun> findAllByOrderByStartedAtDesc(Pageable pageable);
|
||||
|
||||
List<ConnectorSyncRun> findByProfileCodeOrderByStartedAtDesc(String profileCode, Pageable pageable);
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.lincfusion.udyamlinc.manualentry.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Runs one sync: fetch, map, validate, stage, and record what happened.
|
||||
*
|
||||
* <p><b>The validation is not this class's.</b> Mapped rows are handed to
|
||||
* {@code LoanBookEntryService.stage}, the same method a spreadsheet upload goes
|
||||
* through, and land in the same {@code manual_entry_batch} with the same preview.
|
||||
* Writing a second validator for the automated path would be the obvious shape
|
||||
* and the wrong one — the no-PII rule, the unknown-enterprise reject and the
|
||||
* point-in-time as-of date would then be enforced in one path and re-implemented,
|
||||
* eventually differently, in the other.
|
||||
*
|
||||
* <p>Auto-commit is off by default on every profile, so a sync stages and stops.
|
||||
* That preserves the manual path's rule — nothing is written until someone has
|
||||
* seen what will be — and a bank that wants the nightly feed unattended turns it
|
||||
* on per profile, knowingly.
|
||||
*
|
||||
* <p>A failed run is still a run. The {@link ConnectorSyncRun} row is written
|
||||
* whether the fetch threw, returned nothing, or worked, because a feed that
|
||||
* stopped three weeks ago must be visible as something other than silence.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class ConnectorSyncService {
|
||||
|
||||
private final ConnectorCatalog catalog;
|
||||
private final ConnectorProfileRepository profileRepository;
|
||||
private final ConnectorSyncRunRepository runRepository;
|
||||
private final FieldMapper fieldMapper;
|
||||
private final LoanBookEntryService loanBookEntryService;
|
||||
private final ManualEntryBatchRepository batchRepository;
|
||||
private final ManualEntryLineRepository lineRepository;
|
||||
|
||||
/** A sync and everything the console needs to explain it. */
|
||||
public record SyncResult(ConnectorSyncRun run, ManualEntryPreview preview,
|
||||
List<String> droppedColumns, String extractLabel) {
|
||||
}
|
||||
|
||||
// ── running a sync ──────────────────────────────────────
|
||||
|
||||
public SyncResult sync(String profileCode, LocalDate asOf, String trigger, String username) {
|
||||
ConnectorProfile profile = profileRepository.findByCode(profileCode)
|
||||
.orElseThrow(() -> new IllegalArgumentException("No connector profile '" + profileCode + "'"));
|
||||
CoreBankingConnector connector = catalog.find(profile.getVendor())
|
||||
.orElseThrow(() -> new IllegalStateException(
|
||||
"Profile " + profileCode + " names vendor '" + profile.getVendor()
|
||||
+ "', which no adapter implements"));
|
||||
|
||||
LocalDate effectiveAsOf = asOf == null ? LocalDate.now() : asOf;
|
||||
LocalDateTime startedAt = LocalDateTime.now();
|
||||
|
||||
ConnectorSyncRun run = ConnectorSyncRun.builder()
|
||||
.profileCode(profile.getCode())
|
||||
.vendor(profile.getVendor())
|
||||
.institutionCode(profile.getInstitutionCode())
|
||||
.trigger(trigger)
|
||||
.triggeredBy(username)
|
||||
.simulated(connector.simulated())
|
||||
.asOfDate(effectiveAsOf)
|
||||
.startedAt(startedAt)
|
||||
.committed(false)
|
||||
.build();
|
||||
|
||||
try {
|
||||
CoreBankingConnector.Extract extract = connector.fetch(profile, effectiveAsOf);
|
||||
return complete(profile, connector, run, extract, effectiveAsOf, startedAt, username);
|
||||
} catch (Exception e) {
|
||||
log.warn("Connector sync {} failed: {}", profileCode, e.toString());
|
||||
run.setStatus("FAILED");
|
||||
run.setRowsFetched(0);
|
||||
run.setDurationMs(elapsedMs(startedAt));
|
||||
run.setMessage(e.getMessage() == null ? e.toString() : e.getMessage());
|
||||
runRepository.save(run);
|
||||
stampProfile(profile, run);
|
||||
return new SyncResult(run, null, List.of(), null);
|
||||
}
|
||||
}
|
||||
|
||||
private SyncResult complete(ConnectorProfile profile, CoreBankingConnector connector, ConnectorSyncRun run,
|
||||
CoreBankingConnector.Extract extract, LocalDate asOf,
|
||||
LocalDateTime startedAt, String username) {
|
||||
|
||||
run.setRowsFetched(extract.rows().size());
|
||||
|
||||
if (extract.rows().isEmpty()) {
|
||||
// Nothing to read is a normal night, not a fault. Saying FAILED here
|
||||
// would train everyone to ignore the status.
|
||||
run.setStatus("SUCCESS");
|
||||
run.setRowsAccepted(0);
|
||||
run.setRowsRejected(0);
|
||||
run.setColumnsDropped(0);
|
||||
run.setDurationMs(elapsedMs(startedAt));
|
||||
run.setMessage("Nothing to read — " + extract.label());
|
||||
runRepository.save(run);
|
||||
stampProfile(profile, run);
|
||||
return new SyncResult(run, null, List.of(), extract.label());
|
||||
}
|
||||
|
||||
Map<String, String> mapping = fieldMapper.read(profile);
|
||||
FieldMapper.Mapped mapped = fieldMapper.apply(extract, mapping);
|
||||
|
||||
if (!mapped.missingRequired().isEmpty()) {
|
||||
run.setStatus("FAILED");
|
||||
run.setRowsAccepted(0);
|
||||
run.setRowsRejected(extract.rows().size());
|
||||
run.setColumnsDropped(mapped.droppedColumns().size());
|
||||
run.setDurationMs(elapsedMs(startedAt));
|
||||
run.setMessage("Mapping is incomplete — " + String.join(", ", mapped.missingRequired())
|
||||
+ " not mapped, so no row can be validated");
|
||||
runRepository.save(run);
|
||||
stampProfile(profile, run);
|
||||
return new SyncResult(run, null, mapped.droppedColumns(), extract.label());
|
||||
}
|
||||
|
||||
String filename = "%s · %s".formatted(profile.getCode(), extract.label());
|
||||
ManualEntryPreview preview = loanBookEntryService.stage(
|
||||
new TabularParser.Sheet(mapped.headers(), mapped.rows()),
|
||||
filename, username, profile.getCode());
|
||||
|
||||
boolean committed = false;
|
||||
if (Boolean.TRUE.equals(profile.getAutoCommit()) && !preview.accepted().isEmpty()) {
|
||||
preview = loanBookEntryService.commit(preview.batch().getId(), username);
|
||||
committed = true;
|
||||
}
|
||||
|
||||
run.setRowsAccepted(preview.accepted().size());
|
||||
run.setRowsRejected(preview.rejected().size());
|
||||
run.setRowsOverwriting(preview.overwriting().size());
|
||||
run.setColumnsDropped(mapped.droppedColumns().size());
|
||||
run.setBatchId(preview.batch().getId());
|
||||
run.setCommitted(committed);
|
||||
run.setDurationMs(elapsedMs(startedAt));
|
||||
run.setStatus(preview.rejected().isEmpty() ? "SUCCESS" : "PARTIAL");
|
||||
run.setMessage(describe(connector, preview, mapped, committed));
|
||||
runRepository.save(run);
|
||||
stampProfile(profile, run);
|
||||
|
||||
return new SyncResult(run, preview, mapped.droppedColumns(), extract.label());
|
||||
}
|
||||
|
||||
private String describe(CoreBankingConnector connector, ManualEntryPreview preview,
|
||||
FieldMapper.Mapped mapped, boolean committed) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(preview.accepted().size()).append(" accepted · ")
|
||||
.append(preview.rejected().size()).append(" rejected");
|
||||
if (!mapped.droppedColumns().isEmpty()) {
|
||||
sb.append(" · ").append(mapped.droppedColumns().size())
|
||||
.append(" unmapped columns dropped at the boundary");
|
||||
}
|
||||
sb.append(committed ? " · written" : " · staged, awaiting commit");
|
||||
if (connector.simulated()) {
|
||||
sb.append(" · SIMULATED extract, not a live core");
|
||||
}
|
||||
return sb.length() > 590 ? sb.substring(0, 590) : sb.toString();
|
||||
}
|
||||
|
||||
private void stampProfile(ConnectorProfile profile, ConnectorSyncRun run) {
|
||||
profile.setLastSyncAt(run.getStartedAt());
|
||||
profile.setLastStatus(run.getStatus());
|
||||
profileRepository.save(profile);
|
||||
}
|
||||
|
||||
private int elapsedMs(LocalDateTime startedAt) {
|
||||
return (int) Duration.between(startedAt, LocalDateTime.now()).toMillis();
|
||||
}
|
||||
|
||||
// ── connection test ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Fetches without staging anything, so a bank operations user can see the
|
||||
* bank's own columns and a sample row before drafting a mapping.
|
||||
*
|
||||
* <p>Separate from a sync on purpose: the first thing anyone does with a new
|
||||
* connector is press the button to see whether it can reach anything, and
|
||||
* that must not be a thing that writes.
|
||||
*/
|
||||
public Map<String, Object> test(String profileCode, LocalDate asOf) {
|
||||
ConnectorProfile profile = profileRepository.findByCode(profileCode)
|
||||
.orElseThrow(() -> new IllegalArgumentException("No connector profile '" + profileCode + "'"));
|
||||
CoreBankingConnector connector = catalog.find(profile.getVendor())
|
||||
.orElseThrow(() -> new IllegalStateException("No adapter for vendor " + profile.getVendor()));
|
||||
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("profileCode", profile.getCode());
|
||||
result.put("vendor", connector.vendor());
|
||||
result.put("simulated", connector.simulated());
|
||||
result.put("transport", connector.transport());
|
||||
|
||||
try {
|
||||
CoreBankingConnector.Extract extract =
|
||||
connector.fetch(profile, asOf == null ? LocalDate.now() : asOf);
|
||||
Map<String, String> mapping = fieldMapper.read(profile);
|
||||
FieldMapper.Mapped mapped = fieldMapper.apply(extract, mapping);
|
||||
|
||||
result.put("reachable", true);
|
||||
result.put("label", extract.label());
|
||||
result.put("rowsAvailable", extract.rows().size());
|
||||
result.put("sourceColumns", extract.headers().isEmpty()
|
||||
? sourceColumnsFrom(extract) : extract.headers());
|
||||
result.put("sampleRow", extract.rows().isEmpty() ? Map.of() : extract.rows().get(0));
|
||||
result.put("mappedSampleRow", mapped.rows().isEmpty() ? Map.of() : mapped.rows().get(0));
|
||||
result.put("droppedColumns", mapped.droppedColumns());
|
||||
result.put("missingRequired", mapped.missingRequired());
|
||||
} catch (Exception e) {
|
||||
result.put("reachable", false);
|
||||
result.put("message", e.getMessage() == null ? e.toString() : e.getMessage());
|
||||
result.put("sourceColumns", connector.nativeColumns());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** A file drop declares no columns up front — read them off the extract itself. */
|
||||
private List<String> sourceColumnsFrom(CoreBankingConnector.Extract extract) {
|
||||
return extract.rows().isEmpty() ? List.of() : List.copyOf(extract.rows().get(0).keySet());
|
||||
}
|
||||
|
||||
// ── history ─────────────────────────────────────────────
|
||||
|
||||
public List<ConnectorSyncRun> runs(String profileCode, int limit) {
|
||||
PageRequest page = PageRequest.of(0, Math.min(limit, 200));
|
||||
return profileCode == null
|
||||
? runRepository.findAllByOrderByStartedAtDesc(page)
|
||||
: runRepository.findByProfileCodeOrderByStartedAtDesc(profileCode, page);
|
||||
}
|
||||
|
||||
/** The staged batch behind a run, so the console can open the same preview an upload shows. */
|
||||
public ManualEntryPreview preview(Long batchId) {
|
||||
ManualEntryBatch batch = batchRepository.findById(batchId)
|
||||
.orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId));
|
||||
return new ManualEntryPreview(batch, lineRepository.findByBatchIdOrderByRowNumber(batchId));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* One way of getting a loan book out of a lender's core banking or loan
|
||||
* management system.
|
||||
*
|
||||
* <p><b>Why an interface and not an integration.</b> Every bank's core is
|
||||
* different — Finacle, Flexcube, BaNCS, a cooperative bank's in-house system —
|
||||
* and each names the same eleven fields differently, exposes them over a
|
||||
* different transport, and will only agree a schema during its own onboarding.
|
||||
* There is no single integration to build ahead of that conversation. What can
|
||||
* be built ahead of it is the shape of the conversation: a connector produces a
|
||||
* table of the bank's own columns, a per-bank mapping profile says which of its
|
||||
* columns is the DPD, and everything downstream is identical regardless of which
|
||||
* core it came from.
|
||||
*
|
||||
* <p><b>A connector never writes.</b> It fetches and returns rows. Mapping,
|
||||
* validation, staging and commit belong to {@link ConnectorSyncService}, which
|
||||
* hands them to the same {@code LoanBookEntryService} a manual upload goes
|
||||
* through. An adapter that could write would be an adapter that could bypass the
|
||||
* no-PII rule, and the point of routing both paths through one validator is that
|
||||
* neither can.
|
||||
*
|
||||
* <p>{@link #simulated()} is not a debug flag. It is rendered in the console and
|
||||
* stamped on every sync run, because a demo that cannot be told apart from a
|
||||
* live feed is a demo that will eventually be mistaken for one.
|
||||
*/
|
||||
public interface CoreBankingConnector {
|
||||
|
||||
/** Stable code — {@code FINACLE}, {@code FILE_DROP}. Matches a profile's vendor. */
|
||||
String vendor();
|
||||
|
||||
/** What a bank operations user would call it. */
|
||||
String displayName();
|
||||
|
||||
/** How the extract physically arrives: {@code FILE}, {@code REST}, {@code JDBC}. */
|
||||
String transport();
|
||||
|
||||
/**
|
||||
* True when the rows are generated rather than read from a lender's system.
|
||||
* Travels all the way to the console badge and the sync-run record.
|
||||
*/
|
||||
boolean simulated();
|
||||
|
||||
/**
|
||||
* The column names this core exports, for the mapping editor to offer.
|
||||
*
|
||||
* <p>Declared rather than discovered so a mapping can be drafted before the
|
||||
* bank has opened anything — which is the order these conversations actually
|
||||
* happen in.
|
||||
*/
|
||||
List<String> nativeColumns();
|
||||
|
||||
/**
|
||||
* One sentence on what a bank would have to do to make this real. Shown in
|
||||
* the console beside a simulated connector so the gap is stated by the
|
||||
* product rather than discovered in a meeting.
|
||||
*/
|
||||
String activationNote();
|
||||
|
||||
/** Reads the extract for {@code asOf}. Returns the bank's own column names, unmapped. */
|
||||
Extract fetch(ConnectorProfile profile, LocalDate asOf) throws Exception;
|
||||
|
||||
/**
|
||||
* What a connector returns: the vendor's headers and rows, plus a label
|
||||
* naming where it came from for the audit trail.
|
||||
*/
|
||||
record Extract(String label, List<String> headers, List<Map<String, String>> rows) {
|
||||
|
||||
public static Extract empty(String label) {
|
||||
return new Extract(label, List.of(), List.of());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lincfusion.udyamlinc.manualentry.LoanBookEntryService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Turns one bank's column names into the canonical loan-book template.
|
||||
*
|
||||
* <p>This is the whole answer to "every bank's core is different". The adapters
|
||||
* differ by transport; the banks differ by vocabulary; and everything downstream
|
||||
* of this class sees one template. Adding a lender is editing a JSON object,
|
||||
* which is a thing a bank operations user can do in a screen — not a release.
|
||||
*
|
||||
* <p><b>Unmapped columns are dropped, not carried.</b> The direction of the map
|
||||
* makes that structural rather than a rule someone has to remember: it is keyed
|
||||
* by canonical name, so a vendor column nobody asked for has no way through. That
|
||||
* is what keeps {@code CUST_NAME} and {@code MOBILE_NO} out of a system that
|
||||
* promised not to hold them, and it is why {@link #validate} refuses a mapping
|
||||
* that points a canonical column at an identifying one rather than accepting it
|
||||
* and failing later at the parse boundary.
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class FieldMapper {
|
||||
|
||||
/** As elsewhere in this codebase, constructed rather than injected — see the Boot 4 note. */
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
/** What a mapping did to one extract. */
|
||||
public record Mapped(List<String> headers, List<Map<String, String>> rows,
|
||||
List<String> droppedColumns, List<String> missingRequired) {
|
||||
}
|
||||
|
||||
/** Whether a mapping can be saved, and what is wrong with it if not. */
|
||||
public record Validation(boolean valid, List<String> problems, List<String> missingRequired,
|
||||
List<String> identifyingColumns) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a stored mapping. A null or unreadable one means "the extract already
|
||||
* uses the canonical names" — which is true of the loan-book template itself,
|
||||
* so a bank that adopts our template needs no mapping at all.
|
||||
*/
|
||||
public Map<String, String> read(ConnectorProfile profile) {
|
||||
String json = profile == null ? null : profile.getMappingJson();
|
||||
if (json == null || json.isBlank()) return identity();
|
||||
try {
|
||||
Map<?, ?> raw = JSON.readValue(json, Map.class);
|
||||
Map<String, String> mapping = new LinkedHashMap<>();
|
||||
raw.forEach((k, v) -> {
|
||||
if (k != null && v != null && !String.valueOf(v).isBlank()) {
|
||||
mapping.put(String.valueOf(k), String.valueOf(v));
|
||||
}
|
||||
});
|
||||
return mapping;
|
||||
} catch (Exception e) {
|
||||
log.warn("Connector {} has an unreadable mapping, falling back to canonical names: {}",
|
||||
profile.getCode(), e.toString());
|
||||
return identity();
|
||||
}
|
||||
}
|
||||
|
||||
public String write(Map<String, String> mapping) {
|
||||
try {
|
||||
return JSON.writeValueAsString(mapping == null ? Map.of() : mapping);
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException("Could not store that mapping: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> identity() {
|
||||
Map<String, String> mapping = new LinkedHashMap<>();
|
||||
LoanBookEntryService.CANONICAL_COLUMNS.forEach(c -> mapping.put(c, c));
|
||||
return mapping;
|
||||
}
|
||||
|
||||
// ── validation ──────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Checks a mapping before it is saved, against the columns the extract
|
||||
* actually carries when those are known.
|
||||
*
|
||||
* <p>Refusing early matters: the alternative is a mapping that saves cleanly,
|
||||
* runs nightly, and rejects every row for the next fortnight because the
|
||||
* as-of-date column was never pointed anywhere.
|
||||
*/
|
||||
public Validation validate(Map<String, String> mapping, Collection<String> availableColumns) {
|
||||
List<String> problems = new ArrayList<>();
|
||||
List<String> missing = new ArrayList<>();
|
||||
|
||||
for (String required : LoanBookEntryService.REQUIRED_COLUMNS) {
|
||||
String source = mapping.get(required);
|
||||
if (source == null || source.isBlank()) {
|
||||
missing.add(required);
|
||||
}
|
||||
}
|
||||
if (!missing.isEmpty()) {
|
||||
problems.add("not mapped, and every row will be rejected without them: " + String.join(", ", missing));
|
||||
}
|
||||
|
||||
for (Map.Entry<String, String> entry : mapping.entrySet()) {
|
||||
if (!LoanBookEntryService.CANONICAL_COLUMNS.contains(entry.getKey())) {
|
||||
problems.add("'" + entry.getKey() + "' is not a loan-book column");
|
||||
}
|
||||
if (availableColumns != null && !availableColumns.isEmpty()
|
||||
&& entry.getValue() != null && !entry.getValue().isBlank()
|
||||
&& availableColumns.stream().noneMatch(c -> c.equalsIgnoreCase(entry.getValue()))) {
|
||||
problems.add("the extract has no column called '" + entry.getValue() + "'");
|
||||
}
|
||||
}
|
||||
|
||||
// A mapping that renames CUST_NAME to 'product' would smuggle an
|
||||
// identifying field past a parse-time check that only looks at headers.
|
||||
List<String> identifying = LoanBookEntryService.forbiddenHeaders(
|
||||
mapping.values().stream().filter(Objects::nonNull).toList());
|
||||
if (!identifying.isEmpty()) {
|
||||
problems.add("these carry borrower identity and cannot be mapped in: "
|
||||
+ String.join(", ", identifying));
|
||||
}
|
||||
|
||||
return new Validation(problems.isEmpty(), problems, missing, identifying);
|
||||
}
|
||||
|
||||
// ── application ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Rewrites an extract into the canonical template.
|
||||
*
|
||||
* <p>Reads case-, space- and underscore-insensitively for the same reason
|
||||
* {@code TabularParser} does — a bank that ships {@code Next_Due_Dt} where its
|
||||
* documentation said {@code NEXT_DUE_DT} has not made a mistake worth failing
|
||||
* a nightly run over.
|
||||
*/
|
||||
public Mapped apply(CoreBankingConnector.Extract extract, Map<String, String> mapping) {
|
||||
Set<String> used = new HashSet<>();
|
||||
List<String> headers = new ArrayList<>();
|
||||
|
||||
for (String canonical : LoanBookEntryService.CANONICAL_COLUMNS) {
|
||||
String source = mapping.get(canonical);
|
||||
if (source != null && !source.isBlank()) {
|
||||
headers.add(canonical);
|
||||
used.add(normalise(source));
|
||||
}
|
||||
}
|
||||
|
||||
List<Map<String, String>> rows = new ArrayList<>();
|
||||
for (Map<String, String> row : extract.rows()) {
|
||||
Map<String, String> mapped = new LinkedHashMap<>();
|
||||
for (String canonical : headers) {
|
||||
mapped.put(canonical, valueOf(row, mapping.get(canonical)));
|
||||
}
|
||||
rows.add(mapped);
|
||||
}
|
||||
|
||||
List<String> dropped = extract.headers().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(h -> !used.contains(normalise(h)))
|
||||
.toList();
|
||||
|
||||
List<String> missingRequired = LoanBookEntryService.REQUIRED_COLUMNS.stream()
|
||||
.filter(c -> !headers.contains(c))
|
||||
.toList();
|
||||
|
||||
return new Mapped(headers, rows, dropped, missingRequired);
|
||||
}
|
||||
|
||||
private String valueOf(Map<String, String> row, String column) {
|
||||
if (column == null) return null;
|
||||
for (Map.Entry<String, String> entry : row.entrySet()) {
|
||||
if (entry.getKey() != null && normalise(entry.getKey()).equals(normalise(column))) {
|
||||
String value = entry.getValue();
|
||||
return value == null || value.isBlank() ? null : value.trim();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String normalise(String s) {
|
||||
return s.replace("_", "").replace(" ", "").replace("-", "").toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.lincfusion.udyamlinc.manualentry.TabularParser;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.*;
|
||||
import java.time.LocalDate;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* The one connector that actually reads a lender's data.
|
||||
*
|
||||
* <p>A bank drops its nightly extract on a path — an SFTP landing directory, a
|
||||
* mounted share — and this picks up the newest file matching the profile's
|
||||
* pattern. It is deliberately the least clever option available, and it is also
|
||||
* the one most likely to be what a district cooperative bank can actually do in
|
||||
* month one: no API to expose, no core vendor change request, no firewall
|
||||
* conversation beyond one they have already had for their regulatory returns.
|
||||
*
|
||||
* <p>SFTP polling itself is not implemented — this reads a local directory, on
|
||||
* the assumption that the drop is mounted or synced there by something that
|
||||
* already exists at the bank. That is a real deployment pattern rather than a
|
||||
* shortcut, but it is worth being plain that no credential is held and no remote
|
||||
* host is contacted by this class.
|
||||
*
|
||||
* <p>The file is left where it was found. Moving or deleting it would make a
|
||||
* failed sync unrepeatable, and the first thing anyone wants after a bad run is
|
||||
* to run it again against the same bytes.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class FileDropConnector implements CoreBankingConnector {
|
||||
|
||||
public static final String VENDOR = "FILE_DROP";
|
||||
|
||||
private final TabularParser parser;
|
||||
|
||||
@Value("${udyam.connector.file-drop.dir:./data/cbs-drop}")
|
||||
private String dropDir;
|
||||
|
||||
@Override
|
||||
public String vendor() {
|
||||
return VENDOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String displayName() {
|
||||
return "Nightly file drop (SFTP / mounted share)";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String transport() {
|
||||
return "FILE";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean simulated() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unknown until a file arrives — a drop carries whatever the bank's export
|
||||
* job produces. The mapping editor discovers these from the last extract
|
||||
* rather than being told them up front.
|
||||
*/
|
||||
@Override
|
||||
public List<String> nativeColumns() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String activationNote() {
|
||||
return "Point the profile at the directory the bank's extract lands in. "
|
||||
+ "No core-system change is needed — this reads the file the bank already produces.";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Extract fetch(ConnectorProfile profile, LocalDate asOf) throws Exception {
|
||||
Path dir = Paths.get(dropDir).toAbsolutePath().normalize();
|
||||
if (!Files.isDirectory(dir)) {
|
||||
throw new IllegalStateException("No drop directory at " + dir
|
||||
+ " — create it, or point udyam.connector.file-drop.dir somewhere else");
|
||||
}
|
||||
|
||||
String pattern = (profile.getEndpoint() == null || profile.getEndpoint().isBlank())
|
||||
? "*.{csv,xlsx}"
|
||||
: profile.getEndpoint().trim();
|
||||
PathMatcher matcher = dir.getFileSystem().getPathMatcher("glob:" + pattern);
|
||||
|
||||
Optional<Path> newest;
|
||||
try (var paths = Files.list(dir)) {
|
||||
newest = paths
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(p -> matcher.matches(p.getFileName()))
|
||||
.max(Comparator.comparingLong(p -> p.toFile().lastModified()));
|
||||
}
|
||||
|
||||
if (newest.isEmpty()) {
|
||||
// Not an error. A bank that has not dropped tonight's file yet is a
|
||||
// normal state, and it must read differently from a broken connector.
|
||||
return Extract.empty("no file matching '" + pattern + "' in " + dir);
|
||||
}
|
||||
|
||||
Path file = newest.get();
|
||||
try (InputStream in = Files.newInputStream(file)) {
|
||||
TabularParser.Sheet sheet = parser.parse(in, file.getFileName().toString());
|
||||
log.info("File-drop connector {} read {} rows from {}",
|
||||
profile.getCode(), sheet.rows().size(), file.getFileName());
|
||||
return new Extract(file.getFileName().toString(), sheet.headers(), sheet.rows());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package com.lincfusion.udyamlinc.connector;
|
||||
|
||||
import com.lincfusion.udyamlinc.enterprise.Enterprise;
|
||||
import com.lincfusion.udyamlinc.enterprise.EnterpriseRepository;
|
||||
import com.lincfusion.udyamlinc.enterprise.LoanRecord;
|
||||
import com.lincfusion.udyamlinc.enterprise.LoanRecordRepository;
|
||||
import com.lincfusion.udyamlinc.institution.InstitutionRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.time.LocalDate;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* A core banking system that does not exist, exporting in the dialect of one
|
||||
* that does.
|
||||
*
|
||||
* <p><b>What this is for, and what it is not.</b> Integrating with Finacle,
|
||||
* Flexcube or BaNCS is not a technical unknown — it is an onboarding process per
|
||||
* bank, with a schema agreed in a room. Until that room, the thing worth
|
||||
* demonstrating is not a fake feed but the <i>shape of the seam</i>: that each
|
||||
* core names the same eleven fields differently, that a mapping profile is what
|
||||
* reconciles them, and that nothing after the mapping cares which core it was.
|
||||
* Running the same portfolio out through four dialects makes that visible in a
|
||||
* way a paragraph of intent does not.
|
||||
*
|
||||
* <p>Every row it produces is stamped simulated, all the way to the console badge
|
||||
* and the sync-run record. That matters more than it sounds: the failure mode
|
||||
* here is not a wrong number, it is a demo mistaken for a pilot.
|
||||
*
|
||||
* <p>The rows are derived from the enterprises and loan records already in the
|
||||
* database rather than invented, so a mapped sync validates against real
|
||||
* enterprise ids and exercises the real reject paths. The drift applied to DPD is
|
||||
* seeded from the account reference, so the same extract is reproducible — a
|
||||
* connector whose output changed every call would make a preview meaningless.
|
||||
*/
|
||||
@RequiredArgsConstructor
|
||||
public class SimulatedCoreConnector implements CoreBankingConnector {
|
||||
|
||||
/**
|
||||
* A core's column names for the eleven fields that matter, keyed by canonical
|
||||
* name. This is the entire per-vendor difference, which is rather the point.
|
||||
*/
|
||||
public record Dialect(String vendor, String displayName, String transport, String activationNote,
|
||||
LinkedHashMap<String, String> columns, List<String> extraColumns) {
|
||||
}
|
||||
|
||||
private final Dialect dialect;
|
||||
private final EnterpriseRepository enterpriseRepository;
|
||||
private final LoanRecordRepository loanRepository;
|
||||
private final InstitutionRepository institutionRepository;
|
||||
|
||||
@Override
|
||||
public String vendor() {
|
||||
return dialect.vendor();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String displayName() {
|
||||
return dialect.displayName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String transport() {
|
||||
return dialect.transport();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean simulated() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> nativeColumns() {
|
||||
List<String> all = new ArrayList<>(dialect.columns().values());
|
||||
all.addAll(dialect.extraColumns());
|
||||
return all;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String activationNote() {
|
||||
return dialect.activationNote();
|
||||
}
|
||||
|
||||
/**
|
||||
* This core's {@code {canonical: vendor}} map — the draft mapping the console
|
||||
* offers when a bank on this core is first onboarded.
|
||||
*/
|
||||
public LinkedHashMap<String, String> dialectColumns() {
|
||||
return new LinkedHashMap<>(dialect.columns());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Extract fetch(ConnectorProfile profile, LocalDate asOf) {
|
||||
List<Enterprise> enterprises = enterprisesFor(profile);
|
||||
if (enterprises.isEmpty()) {
|
||||
return Extract.empty("no enterprises for " + profile.getInstitutionCode());
|
||||
}
|
||||
|
||||
List<Map<String, String>> rows = new ArrayList<>();
|
||||
for (Enterprise enterprise : enterprises) {
|
||||
rows.addAll(rowsFor(enterprise, asOf));
|
||||
}
|
||||
return new Extract(
|
||||
"%s simulated extract · %d accounts as at %s".formatted(dialect.vendor(), rows.size(), asOf),
|
||||
nativeColumns(), rows);
|
||||
}
|
||||
|
||||
/** The bank's own book, when the profile names one; otherwise everything. */
|
||||
private List<Enterprise> enterprisesFor(ConnectorProfile profile) {
|
||||
if (profile.getInstitutionCode() == null || profile.getInstitutionCode().isBlank()) {
|
||||
return enterpriseRepository.findAll();
|
||||
}
|
||||
return institutionRepository.findByCode(profile.getInstitutionCode())
|
||||
.map(institution -> enterpriseRepository.findAll().stream()
|
||||
.filter(e -> Objects.equals(e.getInstitutionId(), institution.getId()))
|
||||
.toList())
|
||||
.orElseGet(List::of);
|
||||
}
|
||||
|
||||
/**
|
||||
* One row per open loan, carried forward to {@code asOf}.
|
||||
*
|
||||
* <p>An enterprise with no loan record yet still produces a row, built from
|
||||
* the outstanding and instalment the enterprise itself carries. Skipping them
|
||||
* would make the simulated extract cover only the six enterprises that already
|
||||
* have a loan book, which is precisely the gap this feature exists to discuss.
|
||||
*/
|
||||
private List<Map<String, String>> rowsFor(Enterprise enterprise, LocalDate asOf) {
|
||||
List<LoanRecord> open = loanRepository.findByEnterpriseIdAndClosedFalse(enterprise.getEnterpriseId());
|
||||
if (open.isEmpty()) {
|
||||
return List.of(syntheticRow(enterprise, asOf));
|
||||
}
|
||||
|
||||
// Latest extract per account, carried forward to the requested date.
|
||||
Map<String, LoanRecord> latest = new LinkedHashMap<>();
|
||||
for (LoanRecord loan : open) {
|
||||
latest.merge(loan.getAccountRef(), loan,
|
||||
(a, b) -> a.getAsOfDate().isAfter(b.getAsOfDate()) ? a : b);
|
||||
}
|
||||
|
||||
List<Map<String, String>> rows = new ArrayList<>();
|
||||
for (LoanRecord loan : latest.values()) {
|
||||
rows.add(rowFrom(enterprise, loan, asOf));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private Map<String, String> syntheticRow(Enterprise enterprise, LocalDate asOf) {
|
||||
BigDecimal outstanding = enterprise.getLoanOutstanding() == null
|
||||
? BigDecimal.ZERO : enterprise.getLoanOutstanding();
|
||||
BigDecimal instalment = enterprise.getMonthlyInstalment() == null
|
||||
? BigDecimal.ZERO : enterprise.getMonthlyInstalment();
|
||||
int dpd = drift(enterprise.getEnterpriseId(), enterprise.getBand());
|
||||
|
||||
return emit(enterprise, enterprise.getEnterpriseId() + "-LN1", "TERM_LOAN",
|
||||
outstanding.multiply(BigDecimal.valueOf(1.25)).setScale(0, RoundingMode.HALF_UP),
|
||||
outstanding, instalment,
|
||||
asOf.plusDays(12), asOf.minusDays(Math.max(1, dpd)), dpd, asOf);
|
||||
}
|
||||
|
||||
private Map<String, String> rowFrom(Enterprise enterprise, LoanRecord loan, LocalDate asOf) {
|
||||
int base = loan.getDpd() == null ? 0 : loan.getDpd();
|
||||
int dpd = Math.max(0, base + drift(loan.getAccountRef(), enterprise.getBand()) - 2);
|
||||
|
||||
return emit(enterprise, loan.getAccountRef(), loan.getProduct(),
|
||||
loan.getSanctionedAmount(), loan.getOutstandingAmount(), loan.getInstalmentAmount(),
|
||||
loan.getNextDueDate() == null ? asOf.plusDays(10) : loan.getNextDueDate(),
|
||||
loan.getLastPaidDate() == null ? asOf.minusDays(Math.max(1, dpd)) : loan.getLastPaidDate(),
|
||||
dpd, asOf);
|
||||
}
|
||||
|
||||
/** Writes the values out under this core's own column names. */
|
||||
private Map<String, String> emit(Enterprise enterprise, String accountRef, String product,
|
||||
BigDecimal sanctioned, BigDecimal outstanding, BigDecimal instalment,
|
||||
LocalDate dueDate, LocalDate lastPaid, int dpd, LocalDate asOf) {
|
||||
|
||||
Map<String, String> canonical = new LinkedHashMap<>();
|
||||
canonical.put("enterprise_ref", enterprise.getEnterpriseId());
|
||||
canonical.put("loan_account_ref", accountRef);
|
||||
canonical.put("product", product == null ? "TERM_LOAN" : product);
|
||||
canonical.put("sanctioned_amount", plain(sanctioned));
|
||||
canonical.put("outstanding", plain(outstanding));
|
||||
canonical.put("emi_amount", plain(instalment));
|
||||
canonical.put("due_date", dueDate.toString());
|
||||
canonical.put("last_paid_date", lastPaid.toString());
|
||||
canonical.put("dpd", String.valueOf(dpd));
|
||||
canonical.put("bucket", bucket(dpd));
|
||||
canonical.put("closed", "false");
|
||||
canonical.put("as_of_date", asOf.toString());
|
||||
|
||||
Map<String, String> row = new LinkedHashMap<>();
|
||||
dialect.columns().forEach((canonicalName, vendorColumn) ->
|
||||
row.put(vendorColumn, canonical.get(canonicalName)));
|
||||
|
||||
// The columns a real export carries and this system will not store. Emitted
|
||||
// deliberately: the mapping screen's job is to show them being dropped at
|
||||
// the boundary, and a tidy extract would never exercise that.
|
||||
for (String extra : dialect.extraColumns()) {
|
||||
row.put(extra, extraValue(extra, enterprise, accountRef));
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
private String extraValue(String column, Enterprise enterprise, String accountRef) {
|
||||
String normalised = column.replace("_", "").toLowerCase(Locale.ROOT);
|
||||
if (normalised.contains("name")) return enterprise.getName();
|
||||
if (normalised.contains("mobile") || normalised.contains("phone")) return "9###" + hash(accountRef, 900000);
|
||||
if (normalised.contains("branch")) return "BR-" + (100 + hash(accountRef, 40));
|
||||
if (normalised.contains("ccy") || normalised.contains("currency")) return "INR";
|
||||
return String.valueOf(hash(accountRef, 9999));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic per-account drift, so the same pull twice gives the same rows.
|
||||
* Stressed books move more, which is what makes a simulated extract worth
|
||||
* looking at rather than a flat line.
|
||||
*/
|
||||
private int drift(String key, String band) {
|
||||
int spread = "RED".equals(band) ? 55 : "WATCH".equals(band) ? 18 : 4;
|
||||
return hash(key, spread + 1);
|
||||
}
|
||||
|
||||
private int hash(String key, int bound) {
|
||||
if (bound <= 0) return 0;
|
||||
int h = key == null ? 7 : key.hashCode();
|
||||
return Math.abs(h % bound);
|
||||
}
|
||||
|
||||
private String bucket(int dpd) {
|
||||
if (dpd == 0) return "0";
|
||||
if (dpd <= 30) return "1-30";
|
||||
if (dpd <= 60) return "31-60";
|
||||
if (dpd <= 90) return "61-90";
|
||||
return "90+";
|
||||
}
|
||||
|
||||
private String plain(BigDecimal value) {
|
||||
return value == null ? null : value.setScale(2, RoundingMode.HALF_UP).toPlainString();
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,22 @@ public class LoanBookEntryService {
|
||||
|
||||
public static final String ENTRY_TYPE = "LOAN_RECORD";
|
||||
|
||||
/** A batch that came from a person with a spreadsheet, rather than a connector. */
|
||||
public static final String SOURCE_UPLOAD = "UPLOAD";
|
||||
|
||||
/**
|
||||
* The canonical loan-book columns, in template order. A connector's mapping
|
||||
* profile maps a vendor's own column names onto these, and nothing outside
|
||||
* this list is carried across the boundary.
|
||||
*/
|
||||
public static final List<String> CANONICAL_COLUMNS = List.of(
|
||||
"enterprise_ref", "loan_account_ref", "product", "sanctioned_amount", "outstanding",
|
||||
"emi_amount", "due_date", "last_paid_date", "dpd", "bucket", "closed", "as_of_date");
|
||||
|
||||
/** The three a row cannot be validated without. */
|
||||
public static final List<String> REQUIRED_COLUMNS = List.of(
|
||||
"enterprise_ref", "loan_account_ref", "as_of_date");
|
||||
|
||||
private static final List<DateTimeFormatter> DATE_FORMATS = List.of(
|
||||
DateTimeFormatter.ISO_LOCAL_DATE,
|
||||
DateTimeFormatter.ofPattern("dd/MM/yyyy"),
|
||||
@@ -63,7 +79,7 @@ public class LoanBookEntryService {
|
||||
* store. Matching is on the normalised header, so {@code Customer Name},
|
||||
* {@code customer_name} and {@code CUSTOMERNAME} are all caught.
|
||||
*/
|
||||
private static final Set<String> FORBIDDEN_COLUMNS = Set.of(
|
||||
public static final Set<String> FORBIDDEN_COLUMNS = Set.of(
|
||||
"name", "customername", "borrowername", "applicantname", "fathername",
|
||||
"phone", "mobile", "mobileno", "contact", "email",
|
||||
"aadhaar", "aadhar", "uid", "pan", "voterid",
|
||||
@@ -80,13 +96,27 @@ public class LoanBookEntryService {
|
||||
|
||||
@Transactional
|
||||
public ManualEntryPreview upload(MultipartFile file, String username) throws IOException {
|
||||
TabularParser.Sheet sheet = parser.parse(file);
|
||||
return stage(parser.parse(file), file.getOriginalFilename(), username, SOURCE_UPLOAD);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates an already-parsed sheet and stages it, writing nothing.
|
||||
*
|
||||
* <p>Split out from {@link #upload} so a core-banking connector can reach the
|
||||
* same validation. That reuse is the point: a nightly CBS extract and a
|
||||
* hand-made spreadsheet must be judged by one set of rules, or the automated
|
||||
* path quietly becomes the one where the no-PII promise is not enforced.
|
||||
*
|
||||
* @param source {@code UPLOAD}, or the connector code the rows arrived through
|
||||
*/
|
||||
@Transactional
|
||||
public ManualEntryPreview stage(TabularParser.Sheet sheet, String filename, String username, String source) {
|
||||
ManualEntryBatch batch = batchRepository.save(ManualEntryBatch.builder()
|
||||
.entryType(ENTRY_TYPE)
|
||||
.filename(file.getOriginalFilename())
|
||||
.filename(filename)
|
||||
.uploadedBy(username)
|
||||
.status("PREVIEW")
|
||||
.source(source == null ? SOURCE_UPLOAD : source)
|
||||
.rowsParsed(sheet.rows().size())
|
||||
.build());
|
||||
|
||||
@@ -120,15 +150,45 @@ public class LoanBookEntryService {
|
||||
return new ManualEntryPreview(batch, lines);
|
||||
}
|
||||
|
||||
/** Header names in the file that this system has promised never to hold. */
|
||||
private List<String> forbiddenHeaders(List<String> headers) {
|
||||
/**
|
||||
* Header names in the file that this system has promised never to hold.
|
||||
*
|
||||
* <p>Static and public so a connector can run the same test against a vendor's
|
||||
* column list <i>before</i> a sync is configured, rather than discovering it on
|
||||
* the first nightly run.
|
||||
*/
|
||||
public static List<String> forbiddenHeaders(Collection<String> headers) {
|
||||
return headers.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(header -> FORBIDDEN_COLUMNS.contains(
|
||||
header.replace("_", "").replace(" ", "").replace("-", "").toLowerCase(Locale.ROOT)))
|
||||
.filter(LoanBookEntryService::isIdentifying)
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a column name carries borrower identity.
|
||||
*
|
||||
* <p>Matched on the whole normalised name <b>and</b> on each separator-delimited
|
||||
* part, because a core's real column is {@code CUST_NAME}, not
|
||||
* {@code customer_name} — an exact-match list quietly missed it, and the row
|
||||
* would then have been kept verbatim in {@code manual_entry_line.raw_row}.
|
||||
*
|
||||
* <p>Part-matching rather than substring-matching on purpose:
|
||||
* {@code loan_account_ref} splits to {@code loan/account/ref} and stays clean,
|
||||
* where a substring test would have to special-case it. camelCase counts as a
|
||||
* separator too — an in-house LMS ships {@code borrowerMobile}, which has no
|
||||
* underscore to split on and is no less a phone number for it.
|
||||
*/
|
||||
private static boolean isIdentifying(String header) {
|
||||
String normalised = header.replace("_", "").replace(" ", "").replace("-", "").toLowerCase(Locale.ROOT);
|
||||
if (FORBIDDEN_COLUMNS.contains(normalised)) return true;
|
||||
|
||||
String delimited = header.replaceAll("([a-z0-9])([A-Z])", "$1_$2").toLowerCase(Locale.ROOT);
|
||||
for (String part : delimited.split("[_\\s\\-.]+")) {
|
||||
if (!part.isBlank() && FORBIDDEN_COLUMNS.contains(part)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── validation ──────────────────────────────────────────
|
||||
|
||||
private ManualEntryLine validate(Map<String, String> row, int rowNumber, Long batchId,
|
||||
|
||||
@@ -38,6 +38,17 @@ public class ManualEntryBatch {
|
||||
@Column(length = 260)
|
||||
private String filename;
|
||||
|
||||
/**
|
||||
* {@code UPLOAD}, or the connector code the rows arrived through.
|
||||
*
|
||||
* <p>Kept on the batch rather than inferred from the filename, because once a
|
||||
* CBS connector runs nightly, "who put this number here" stops being answerable
|
||||
* by "someone uploaded a file" — and that question is the reason this table
|
||||
* exists.
|
||||
*/
|
||||
@Column(length = 40)
|
||||
private String source;
|
||||
|
||||
/** The username from the JWT. Never a display name — this is an audit field. */
|
||||
@Column(nullable = false, length = 64)
|
||||
private String uploadedBy;
|
||||
@@ -68,5 +79,6 @@ public class ManualEntryBatch {
|
||||
void onCreate() {
|
||||
if (uploadedAt == null) uploadedAt = LocalDateTime.now();
|
||||
if (status == null) status = "PREVIEW";
|
||||
if (source == null) source = "UPLOAD";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -34,14 +35,26 @@ public class TabularParser {
|
||||
}
|
||||
|
||||
public Sheet parse(MultipartFile file) throws IOException {
|
||||
String name = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase(Locale.ROOT);
|
||||
return name.endsWith(".csv") ? parseCsv(file) : parseWorkbook(file);
|
||||
return parse(file.getInputStream(), file.getOriginalFilename());
|
||||
}
|
||||
|
||||
/**
|
||||
* The same parse, for bytes that never came through a browser.
|
||||
*
|
||||
* <p>A core-banking extract arrives on a drop path rather than a form post,
|
||||
* and it must land in exactly the shape an upload does — the whole value of
|
||||
* the connector layer is that it feeds the validation the manual path already
|
||||
* has, rather than a second, laxer one beside it.
|
||||
*/
|
||||
public Sheet parse(InputStream stream, String filename) throws IOException {
|
||||
String name = filename == null ? "" : filename.toLowerCase(Locale.ROOT);
|
||||
return name.endsWith(".csv") ? parseCsv(stream) : parseWorkbook(stream);
|
||||
}
|
||||
|
||||
// ── xlsx ────────────────────────────────────────────────
|
||||
|
||||
private Sheet parseWorkbook(MultipartFile file) throws IOException {
|
||||
try (Workbook workbook = WorkbookFactory.create(file.getInputStream())) {
|
||||
private Sheet parseWorkbook(InputStream stream) throws IOException {
|
||||
try (Workbook workbook = WorkbookFactory.create(stream)) {
|
||||
org.apache.poi.ss.usermodel.Sheet sheet = workbook.getSheetAt(0);
|
||||
if (sheet == null) return new Sheet(List.of(), List.of());
|
||||
|
||||
@@ -102,9 +115,9 @@ public class TabularParser {
|
||||
|
||||
// ── csv ─────────────────────────────────────────────────
|
||||
|
||||
private Sheet parseCsv(MultipartFile file) throws IOException {
|
||||
private Sheet parseCsv(InputStream stream) throws IOException {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8))) {
|
||||
new InputStreamReader(stream, StandardCharsets.UTF_8))) {
|
||||
|
||||
String headerLine = reader.readLine();
|
||||
if (headerLine == null) return new Sheet(List.of(), List.of());
|
||||
|
||||
@@ -81,6 +81,7 @@ public class UdyamDataSeeder implements CommandLineRunner {
|
||||
private final UdyamUserRepository userRepository;
|
||||
private final PasswordEncoder passwordEncoder;
|
||||
private final InstitutionSeeder institutionSeeder;
|
||||
private final com.lincfusion.udyamlinc.connector.ConnectorProfileSeeder connectorProfileSeeder;
|
||||
|
||||
@Value("${udyam.seed.enabled:true}")
|
||||
private boolean seedEnabled;
|
||||
@@ -114,6 +115,9 @@ public class UdyamDataSeeder implements CommandLineRunner {
|
||||
// The institutional tier last: it assigns the portfolio the steps above
|
||||
// created, so it has to see a finished book.
|
||||
institutionSeeder.seed();
|
||||
// Connector profiles after the institutions they read from, since each
|
||||
// names a partner bank by code.
|
||||
connectorProfileSeeder.seed();
|
||||
log.info("UdyamLinc seed complete — {} enterprises on book", enterpriseRepository.count());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user