diff --git a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java index f81a388..5618e74 100644 --- a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java +++ b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java @@ -66,6 +66,10 @@ public class FeatureEngineeringService { @Value("${udyam.features.min-zscore-samples:6}") private int minZScoreSamples; + /** Whether {@code dscr_p10} may read forecast rows the seeder wrote. See {@code dscrP10}. */ + @Value("${udyam.forecast.trust-seeded:true}") + private boolean trustSeededForecasts; + // ── entry points ──────────────────────────────────────── /** One pass over the whole book. Consent-revoked enterprises are skipped, not computed. */ @@ -425,6 +429,15 @@ public class FeatureEngineeringService { */ private Computed dscrP10(String enterpriseId, LocalDate asOfDate) { List points = forecastPointRepository.findByEnterpriseIdOrderByHorizonAsc(enterpriseId); + + // Seeded rows carry no engine. Once a real forecast engine exists, a + // demo path and a computed one are indistinguishable downstream — the + // score, the band and the alert all look equally earned. Setting this + // false makes an enterprise nobody can forecast report NO_FORECAST + // instead of borrowing the seeder's optimism. + if (!trustSeededForecasts) { + points = points.stream().filter(point -> point.getEngine() != null).toList(); + } if (points.isEmpty()) return Computed.missing(FeatureCatalog.NO_FORECAST); Long latestRun = points.stream().map(ForecastPoint::getRunId) diff --git a/src/main/java/com/lincfusion/udyamlinc/risk/DeterministicForecastStrategy.java b/src/main/java/com/lincfusion/udyamlinc/risk/DeterministicForecastStrategy.java new file mode 100644 index 0000000..57a115a --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/risk/DeterministicForecastStrategy.java @@ -0,0 +1,226 @@ +package com.lincfusion.udyamlinc.risk; + +import com.lincfusion.udyamlinc.enterprise.Enterprise; +import com.lincfusion.udyamlinc.enterprise.LedgerEntry; +import com.lincfusion.udyamlinc.enterprise.LedgerEntryRepository; +import com.lincfusion.udyamlinc.features.FeatureCatalog; +import com.lincfusion.udyamlinc.features.FeatureValue; +import com.lincfusion.udyamlinc.master.Sector; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.YearMonth; +import java.util.*; + +/** + * A quantile path without a model. + * + *

This exists because {@code dscr_p10} — the feature TW-01 fires on, and so + * the origin of most alerts an officer ever sees — was reading forecast rows + * nothing had written since the seeder. A headline number computed from demo + * data is worse than no headline number, because it looks identical to a real + * one. + * + *

The method is deliberately simple and wholly explainable: + * + *

    + *
  1. Baseline. The median monthly net position from the ledger. Median + * rather than mean because a single festival month should not become the + * expectation.
    2. + *
  2. Drift. Margin pressure from the features — {@code margin_squeeze} + * first, else {@code feed_cost_change} inverted — applied per month and + * capped, so a sharp input-cost move bends the path without running away + * over six months.
    3. + *
  3. Spread. P10 and P90 widen with the square root of the horizon, + * because uncertainty compounds with time rather than linearly, and widen + * again for every input we did not have.
    4. + *
+ * + *

Not knowing makes the band wider, never the centre lower. Missing + * inputs must not quietly become a pessimistic P50 — that would manufacture + * breaches out of ignorance. They widen the spread, so P10 falls and confidence + * drops, which is the honest shape of "we know less about this one". + * + *

An enterprise with no ledger at all gets no forecast rather than a fabricated + * one. {@code dscr_p10} then resolves {@code NO_FORECAST}, visibly. + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class DeterministicForecastStrategy implements ForecastStrategy { + + private final LedgerEntryRepository ledgerRepository; + + @Value("${udyam.forecast.horizon-months:6}") + private int horizonMonths; + + /** Months of ledger read to establish the baseline. */ + @Value("${udyam.forecast.lookback-months:6}") + private int lookbackMonths; + + /** Fewest distinct months of ledger before a baseline means anything. */ + @Value("${udyam.forecast.min-months:2}") + private int minMonths; + + /** P10/P90 half-width at h=1, as a fraction of P50, when every input is present. */ + @Value("${udyam.forecast.base-spread:0.18}") + private double baseSpread; + + /** Added to the spread for each feature the path had to do without. */ + @Value("${udyam.forecast.missing-input-penalty:0.10}") + private double missingInputPenalty; + + /** Ceiling on the spread — beyond this a forecast says nothing worth storing. */ + @Value("${udyam.forecast.max-spread:0.75}") + private double maxSpread; + + /** Ceiling on cumulative margin drift, either direction. */ + @Value("${udyam.forecast.max-drift:0.30}") + private double maxDrift; + + @Override + public String engine() { + return "DETERMINISTIC"; + } + + @Override + public String version() { + return "deterministic-v1"; + } + + @Override + public List forecast(Enterprise enterprise, Sector sector, + Map features, LocalDate asOfDate) { + + List monthlyNet = monthlyNetPositions(enterprise.getEnterpriseId(), asOfDate); + if (monthlyNet.size() < minMonths) { + // No ledger, no forecast. A path invented from the loan record alone + // would be a restatement of the obligation dressed up as a prediction. + return List.of(); + } + + BigDecimal baseline = median(monthlyNet); + if (baseline == null || baseline.signum() <= 0) { + // A business whose median month is negative is already in distress and + // the ledger says so directly; projecting it forward adds nothing an + // officer cannot already see. + return List.of(); + } + + BigDecimal obligation = enterprise.getMonthlyInstalment() == null + ? BigDecimal.ZERO + : enterprise.getMonthlyInstalment(); + + MarginSignal margin = marginSignal(features); + double spread = Math.min(maxSpread, baseSpread + margin.missingInputs() * missingInputPenalty); + + List points = new ArrayList<>(horizonMonths); + YearMonth start = YearMonth.from(asOfDate); + + for (int h = 1; h <= horizonMonths; h++) { + // Drift compounds with the horizon but is capped, so a single sharp + // input-cost month does not extrapolate into a cliff at month six. + double drift = Math.max(-maxDrift, Math.min(maxDrift, margin.perMonthDrift() * h)); + BigDecimal p50 = baseline.multiply(BigDecimal.valueOf(1 + drift)) + .setScale(2, RoundingMode.HALF_UP); + + // Uncertainty grows with the square root of time, not linearly. + double horizonSpread = Math.min(maxSpread, spread * Math.sqrt(h)); + BigDecimal p10 = p50.multiply(BigDecimal.valueOf(1 - horizonSpread)).setScale(2, RoundingMode.HALF_UP); + BigDecimal p90 = p50.multiply(BigDecimal.valueOf(1 + horizonSpread)).setScale(2, RoundingMode.HALF_UP); + + points.add(ForecastPoint.builder() + .enterpriseId(enterprise.getEnterpriseId()) + .horizon(h) + .month(start.plusMonths(h).toString()) + .p10(p10) + .p50(p50) + .p90(p90) + .obligation(obligation) + .breach(obligation.signum() > 0 && p10.compareTo(obligation) < 0) + .engine(engine()) + .modelVersion(version()) + .build()); + } + return points; + } + + // ── baseline ──────────────────────────────────────────── + + /** + * Net position per calendar month from the ledger, most recent first. + * + *

Only entries dated on or before {@code asOfDate} count — a forecast + * recomputed for last month must not see money that moved since. + */ + private List monthlyNetPositions(String enterpriseId, LocalDate asOfDate) { + LocalDate from = asOfDate.minusMonths(lookbackMonths).withDayOfMonth(1); + List entries = ledgerRepository + .findByEnterpriseIdAndEntryDateGreaterThanEqualOrderByEntryDateDesc(enterpriseId, from); + + Map byMonth = new TreeMap<>(Comparator.reverseOrder()); + for (LedgerEntry entry : entries) { + if (entry.getEntryDate() == null || entry.getEntryDate().isAfter(asOfDate)) continue; + if (entry.getAmount() == null) continue; + // STOCK entries record inventory movement, not cash — including them + // would double-count the sale that follows. + String kind = entry.getKind(); + if (!"IN".equalsIgnoreCase(kind) && !"OUT".equalsIgnoreCase(kind)) continue; + + BigDecimal signed = "IN".equalsIgnoreCase(kind) ? entry.getAmount() : entry.getAmount().negate(); + byMonth.merge(YearMonth.from(entry.getEntryDate()), signed, BigDecimal::add); + } + + // The current month is partial by definition and would read as a collapse. + byMonth.remove(YearMonth.from(asOfDate)); + return new ArrayList<>(byMonth.values()); + } + + private BigDecimal median(List values) { + if (values.isEmpty()) return null; + List sorted = new ArrayList<>(values); + sorted.sort(Comparator.naturalOrder()); + int mid = sorted.size() / 2; + if (sorted.size() % 2 == 1) return sorted.get(mid); + return sorted.get(mid - 1).add(sorted.get(mid)) + .divide(BigDecimal.valueOf(2), 2, RoundingMode.HALF_UP); + } + + // ── drift ─────────────────────────────────────────────── + + /** The per-month drift the margin features imply, and how many were missing. */ + private record MarginSignal(double perMonthDrift, int missingInputs) { + } + + /** + * {@code margin_squeeze} is input change minus output change, so a positive + * value means costs are outrunning prices and cash should be expected to + * fall. Where it is absent, {@code feed_cost_change} alone carries the same + * sign with less of the picture — and counts as a missing input, which widens + * the band. + */ + private MarginSignal marginSignal(Map features) { + BigDecimal squeeze = valueOf(features, FeatureCatalog.MARGIN_SQUEEZE); + if (squeeze != null) { + return new MarginSignal(-squeeze.doubleValue(), 0); + } + BigDecimal feedCost = valueOf(features, FeatureCatalog.FEED_COST_CHANGE); + if (feedCost != null) { + return new MarginSignal(-feedCost.doubleValue(), 1); + } + // No margin view at all: a flat centre and the widest band this method + // will produce. Flat rather than falling, because inventing a decline we + // cannot evidence would manufacture breaches. + return new MarginSignal(0.0, 2); + } + + private BigDecimal valueOf(Map features, String code) { + FeatureValue value = features.get(code); + return value == null ? null : value.getValue(); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/risk/ForecastPoint.java b/src/main/java/com/lincfusion/udyamlinc/risk/ForecastPoint.java index ce158bb..ae493d4 100644 --- a/src/main/java/com/lincfusion/udyamlinc/risk/ForecastPoint.java +++ b/src/main/java/com/lincfusion/udyamlinc/risk/ForecastPoint.java @@ -55,4 +55,15 @@ public class ForecastPoint { /** True when p10 falls short of the obligation. Stored so queries stay simple. */ private Boolean breach; + + /** + * DETERMINISTIC | MODEL, and the version that produced it. Null on rows the + * seeder wrote, which is the point: a demo path and a computed one must never + * be indistinguishable once both can exist in the same table. + */ + @Column(length = 16) + private String engine; + + @Column(length = 32) + private String modelVersion; } diff --git a/src/main/java/com/lincfusion/udyamlinc/risk/ForecastStrategy.java b/src/main/java/com/lincfusion/udyamlinc/risk/ForecastStrategy.java new file mode 100644 index 0000000..4afefbc --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/risk/ForecastStrategy.java @@ -0,0 +1,43 @@ +package com.lincfusion.udyamlinc.risk; + +import com.lincfusion.udyamlinc.enterprise.Enterprise; +import com.lincfusion.udyamlinc.features.FeatureValue; +import com.lincfusion.udyamlinc.master.Sector; + +import java.time.LocalDate; +import java.util.List; +import java.util.Map; + +/** + * The seam the quantile model arrives through, deliberately shaped like + * {@link ScoringStrategy}. + * + *

The forecast and the score answer different questions and fail in different + * ways, so they get separate seams: "how much cash will there be, and how sure + * are we" is not "how stressed is this enterprise". Keeping them apart means the + * LightGBM quantile heads can replace the forecast without touching the risk + * engine, and vice versa. + * + *

Implementations read features, the ledger and the loan record. Like a + * scoring strategy they must not reach into the signal tables — a market price + * reaches a forecast as a feature or not at all. + */ +public interface ForecastStrategy { + + /** DETERMINISTIC | MODEL — stored on every point it produces. */ + String engine(); + + /** Recorded on each point so a historic breach names the engine that predicted it. */ + String version(); + + /** + * The quantile path for one enterprise, horizon 1…n months ahead. + * + *

Returning an empty list is legitimate and means "not enough to forecast + * from". It leaves {@code dscr_p10} null with {@code NO_FORECAST}, which is + * the honest answer and visibly different from a forecast that says the + * enterprise is fine. + */ + List forecast(Enterprise enterprise, Sector sector, + Map features, LocalDate asOfDate); +} diff --git a/src/main/java/com/lincfusion/udyamlinc/risk/RiskEngineService.java b/src/main/java/com/lincfusion/udyamlinc/risk/RiskEngineService.java index f5b4333..814103d 100644 --- a/src/main/java/com/lincfusion/udyamlinc/risk/RiskEngineService.java +++ b/src/main/java/com/lincfusion/udyamlinc/risk/RiskEngineService.java @@ -47,7 +47,9 @@ public class RiskEngineService { private final StressScoreRepository scoreRepository; private final ReasonCodeRepository reasonRepository; private final RiskAlertRepository alertRepository; + private final ForecastPointRepository forecastPointRepository; private final ScoringStrategy strategy; + private final ForecastStrategy forecastStrategy; @Value("${udyam.forecast.horizon-months:6}") private int horizon; @@ -61,8 +63,11 @@ public class RiskEngineService { StressScoreRepository scoreRepository, ReasonCodeRepository reasonRepository, RiskAlertRepository alertRepository, + ForecastPointRepository forecastPointRepository, List strategies, - @Value("${udyam.risk.engine:RULE}") String engineName) { + List forecastStrategies, + @Value("${udyam.risk.engine:RULE}") String engineName, + @Value("${udyam.forecast.engine:DETERMINISTIC}") String forecastEngineName) { this.enterpriseRepository = enterpriseRepository; this.sectorRepository = sectorRepository; @@ -73,13 +78,22 @@ public class RiskEngineService { this.scoreRepository = scoreRepository; this.reasonRepository = reasonRepository; this.alertRepository = alertRepository; + this.forecastPointRepository = forecastPointRepository; + this.forecastStrategy = forecastStrategies.stream() + .filter(s -> s.engine().equalsIgnoreCase(forecastEngineName)) + .findFirst() + .orElseThrow(() -> new IllegalStateException( + "No ForecastStrategy for udyam.forecast.engine=" + forecastEngineName + + "; available: " + forecastStrategies.stream().map(ForecastStrategy::engine).toList())); this.strategy = strategies.stream() .filter(s -> s.engine().equalsIgnoreCase(engineName)) .findFirst() .orElseThrow(() -> new IllegalStateException( "No ScoringStrategy for udyam.risk.engine=" + engineName + "; available: " + strategies.stream().map(ScoringStrategy::engine).toList())); - log.info("Risk engine bound to {} ({})", this.strategy.engine(), this.strategy.modelVersion()); + log.info("Risk engine bound to {} ({}), forecast to {} ({})", + this.strategy.engine(), this.strategy.modelVersion(), + this.forecastStrategy.engine(), this.forecastStrategy.version()); } // ── batch ─────────────────────────────────────────────── @@ -144,14 +158,21 @@ public class RiskEngineService { LocalDateTime observedAsOf = LocalDateTime.now(); FeatureSnapshot snapshot = featureService.computeFor(enterprise, asOfDate, observedAsOf, run.getId()); - Map features = new HashMap<>(); - featureValueRepository.findByEnterpriseIdAndValidForDateOrderByFeatureCode( - enterprise.getEnterpriseId(), asOfDate) - .forEach(value -> features.put(value.getFeatureCode(), value)); + Map features = loadFeatures(enterprise, asOfDate); Sector sector = sectorRepository.findById(enterprise.getSectorId()).orElse(null); String sectorCode = sector == null ? null : sector.getCode(); + // The forecast reads the margin features, and dscr_p10 reads the forecast. + // That circle is resolved by computing features twice rather than by + // letting either half read a previous run's answer: a path built on last + // month's margin, or a dscr_p10 describing a forecast this run replaced, + // would both be wrong in a way nothing downstream could detect. + if (generateForecast(enterprise, sector, features, asOfDate, run.getId())) { + snapshot = featureService.computeFor(enterprise, asOfDate, observedAsOf, run.getId()); + features = loadFeatures(enterprise, asOfDate); + } + ScoreOutcome scored = strategy.score(enterprise, sector, features); // Trip-wires read the same features, never the score. @@ -172,6 +193,33 @@ public class RiskEngineService { snapshot.getId(), scored, outcomes, fired, raised); } + private Map loadFeatures(Enterprise enterprise, LocalDate asOfDate) { + Map features = new HashMap<>(); + featureValueRepository.findByEnterpriseIdAndValidForDateOrderByFeatureCode( + enterprise.getEnterpriseId(), asOfDate) + .forEach(value -> features.put(value.getFeatureCode(), value)); + return features; + } + + /** + * Writes this run's quantile path, and reports whether it wrote one. + * + *

An empty path is a legitimate outcome — an enterprise with too little + * ledger to forecast from gets no rows, and {@code dscr_p10} stays null with + * {@code NO_FORECAST}. Returning false then skips the second feature pass, + * because there is nothing new for it to see. + */ + private boolean generateForecast(Enterprise enterprise, Sector sector, + Map features, + LocalDate asOfDate, Long runId) { + List points = forecastStrategy.forecast(enterprise, sector, features, asOfDate); + if (points.isEmpty()) return false; + + points.forEach(point -> point.setRunId(runId)); + forecastPointRepository.saveAll(points); + return true; + } + /** * A RED trip-wire escalates whatever the score said; a WATCH one only lifts a * STANDARD. The score is left untouched — the band is a decision, the score is diff --git a/src/main/java/com/lincfusion/udyamlinc/seed/UdyamDataSeeder.java b/src/main/java/com/lincfusion/udyamlinc/seed/UdyamDataSeeder.java index 4ca78a3..6e7c93a 100644 --- a/src/main/java/com/lincfusion/udyamlinc/seed/UdyamDataSeeder.java +++ b/src/main/java/com/lincfusion/udyamlinc/seed/UdyamDataSeeder.java @@ -49,6 +49,15 @@ public class UdyamDataSeeder implements CommandLineRunner { public static final String DEFAULT_ENTERPRISE_PIN = "1234"; private static final int TOTAL = 142; + + /** + * Days of ledger history per demo enterprise. Five months rather than one, + * because the deterministic forecast takes its baseline from the median of + * whole calendar months and discards the partial current one — a single month + * of entries leaves nothing to take a median of, and every enterprise reports + * NO_FORECAST. + */ + private static final int LEDGER_DAYS = 150; private static final Map TARGET = Map.of("RED", 9, "WATCH", 31, "STANDARD", 102); private static final String MODEL_VERSION = "rule-v0.9.3"; @@ -420,11 +429,20 @@ public class UdyamDataSeeder implements CommandLineRunner { * per-enterprise guard, so this is safe on every boot. */ private void backfillPortfolio() { - long before = loanRecordRepository.count(); - enterpriseRepository.findAll().forEach(this::seedLoans); - long added = loanRecordRepository.count() - before; - if (added > 0) { - log.info("Backfilled {} loan records onto an already-seeded portfolio", added); + long loansBefore = loanRecordRepository.count(); + long ledgerBefore = ledgerRepository.count(); + + List all = enterpriseRepository.findAll(); + all.forEach(this::seedLoans); + all.forEach(this::seedLedger); + + long loansAdded = loanRecordRepository.count() - loansBefore; + long ledgerAdded = ledgerRepository.count() - ledgerBefore; + if (loansAdded > 0) { + log.info("Backfilled {} loan records onto an already-seeded portfolio", loansAdded); + } + if (ledgerAdded > 0) { + log.info("Backfilled {} ledger entries onto an already-seeded portfolio", ledgerAdded); } } @@ -637,7 +655,7 @@ public class UdyamDataSeeder implements CommandLineRunner { String[] outCats = {"INPUT", "WAGES", "TRANSPORT", "EMI"}; List entries = new ArrayList<>(); - for (int d = 30; d >= 0; d--) { + for (int d = LEDGER_DAYS; d >= 0; d--) { int count = 1 + r.nextInt(3); for (int i = 0; i < count; i++) { boolean in = r.nextDouble() > 0.45; @@ -652,7 +670,13 @@ public class UdyamDataSeeder implements CommandLineRunner { .build()); } } - ledgerRepository.saveAll(entries); + // clientId is generated on the phone and is the identity an offline entry + // keeps through sync, so it is also the right idempotence key here: a + // re-run extends the history backwards without duplicating a single day. + List fresh = entries.stream() + .filter(entry -> ledgerRepository.findByClientId(entry.getClientId()).isEmpty()) + .toList(); + if (!fresh.isEmpty()) ledgerRepository.saveAll(fresh); } // ── helpers ───────────────────────────────────────────── diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 82ddd79..8937432 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -116,3 +116,24 @@ udyam.features.climate-window-weeks=8 udyam.features.min-zscore-samples=6 logging.level.com.lincfusion.udyamlinc=INFO + +# ── Forecast (ARCHITECTURE R1 seam) ───────────────────────── +# DETERMINISTIC until the LightGBM quantile heads exist. Both implement +# ForecastStrategy, so the swap is this one property. +udyam.forecast.engine=${UDYAM_FORECAST_ENGINE:DETERMINISTIC} +udyam.forecast.lookback-months=6 +udyam.forecast.min-months=2 +# P10/P90 half-width at h=1 when every input is present; each missing input adds +# to it, and it grows with the square root of the horizon. Missing inputs widen +# the band, never lower the centre — a pessimistic P50 built on ignorance would +# manufacture breaches. +udyam.forecast.base-spread=0.18 +udyam.forecast.missing-input-penalty=0.10 +udyam.forecast.max-spread=0.75 +udyam.forecast.max-drift=0.30 +# Whether dscr_p10 may fall back to the seeder's forecast rows, which carry no +# engine. True keeps the demo portfolio populated; false makes every enterprise +# the forecast cannot reach report NO_FORECAST, and TW-01 skip rather than fire +# on a number nothing computed. Set false before any run whose alerts are meant +# to be believed. +udyam.forecast.trust-seeded=${UDYAM_TRUST_SEEDED_FORECASTS:true}