feat: the UdyamLinc ML layer — scoring, explanation and quantile forecast

Brings the Python service under version control for the first time. It has
been running and serving the demo since 23 August; until now it existed only
on one developer's disk.

Contents:

  training/build_dataset.py   point-in-time join over the feature store, with
                              leakage assertions. dscr_p10 and dscr_p50 are
                              excluded deliberately — both are derived from
                              the forecast, which is derived from the
                              instalment, so the model would read its own
                              answer.
  training/train_stress.py    LightGBM vs XGBoost vs two single-feature
                              baselines, expanding-window validation over
                              as-of dates, publishes to the model_metric table
  training/train_forecast.py  three quantile heads (P10/P50/P90), horizon as a
                              feature rather than one model per horizon
  serving/explain.py          SHAP TreeExplainer folded into the five risk
                              families the console already speaks. The family
                              map mirrors RuleBasedScoringStrategy in the API
                              repo; where they disagree the rule engine wins.
  serving/app.py              /score, /score/batch, /forecast, /health, /model

The trained models, their metrics and the training dataset are committed
alongside the code. Without them the published AUC of 0.760 is a claim nobody
else can check; at ~2.2 MB together they are worth the space.

Measured, on 5,523 rows / 263 enterprises / 242 positives / 100% synthetic
labels: LightGBM AUC-ROC 0.7599, and the emi_to_inflow baseline 0.7685 on the
same folds. The baseline wins. That is recorded rather than tuned away — on
generated labels, beating it would mostly prove the model had learned the
generator.

The quantile forecast beats the deterministic strategy on all three quantiles
(pinball loss, 12-33% better) but flags every borrower as breaching, so the
deployment runs udyam.forecast.engine=DETERMINISTIC and this model is held
back pending calibration against real repayment outcomes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

serving/app.py

@@ -0,0 +1,289 @@
+"""
+The inference service Spring Boot calls.
+
+One network boundary, and it is deliberately narrow: the request carries the
+feature map and nothing else about the borrower. No name, no phone, no account
+reference — those do not exist in the platform's schema, and this service must
+never become the reason someone adds them.
+
+The response mirrors ``ScoreOutcome`` field for field, so the Java adapter is a
+mapping rather than a translation.
+"""
+
+from __future__ import annotations
+
+import pickle
+from pathlib import Path
+
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+from explain import Explainer, FAMILIES
+
+MODEL_DIR = Path(__file__).resolve().parent.parent / "models"
+MODEL_PATH = MODEL_DIR / "stress_model.pkl"
+
+app = FastAPI(title="UdyamLinc ML", version="1.0.0")
+
+_bundle: dict | None = None
+_explainer: Explainer | None = None
+
+
+def bundle() -> dict:
+    global _bundle, _explainer
+    if _bundle is None:
+        with open(MODEL_PATH, "rb") as fh:
+            _bundle = pickle.load(fh)
+        _explainer = Explainer(_bundle["model"], _bundle["features"])
+    return _bundle
+
+
+# --------------------------------------------------------------------------- #
+
+
+class ScoreRequest(BaseModel):
+    enterpriseId: str
+    sectorId: int | None = None
+    districtId: int | None = None
+    # Nulls are meaningful and must survive the wire: the engine has to be able to
+    # see what it did not get, exactly as ScoringStrategy requires of a Java engine.
+    features: dict[str, float | None] = Field(default_factory=dict)
+    dpdAtT: float | None = None
+    outstandingAtT: float | None = None
+    emiAtT: float | None = None
+
+
+class Driver(BaseModel):
+    family: str
+    weight: int
+    featureCode: str | None
+    value: float | None
+    onPrior: bool
+
+
+class ScoreResponse(BaseModel):
+    enterpriseId: str
+    score: int
+    band: str
+    confidence: int
+    probability: float
+    subScores: dict[str, int]
+    drivers: list[Driver]
+    familiesOnPrior: int
+    engine: str = "MODEL"
+    modelVersion: str
+
+
+class BatchRequest(BaseModel):
+    items: list[ScoreRequest]
+
+
+def _frame(req: ScoreRequest, features: list[str]) -> pd.DataFrame:
+    row: dict[str, float | None] = {}
+    for name in features:
+        if name.endswith("__missing"):
+            base = name.removesuffix("__missing")
+            row[name] = 1 if req.features.get(base) is None else 0
+        elif name == "dpd_at_t":
+            row[name] = req.dpdAtT
+        elif name == "outstanding_at_t":
+            row[name] = req.outstandingAtT
+        elif name == "emi_at_t":
+            row[name] = req.emiAtT
+        elif name == "sector_id":
+            row[name] = req.sectorId
+        elif name == "district_id":
+            row[name] = req.districtId
+        else:
+            row[name] = req.features.get(name)
+    # Everything float64: a dict of mixed None and float builds an object column,
+    # and LightGBM rejects object dtypes rather than coercing. None becomes NaN,
+    # which is exactly how the model was trained to read a missing feature.
+    return pd.DataFrame([row], columns=features).astype("float64")
+
+
+def _score_one(req: ScoreRequest) -> ScoreResponse:
+    b = bundle()
+    frame = _frame(req, b["features"])
+    probability = float(b["model"].predict_proba(frame)[:, 1][0])
+    score = int(round(probability * 100))
+
+    band = ("RED" if score >= b["red_threshold"]
+            else "WATCH" if score >= b["watch_threshold"]
+            else "STANDARD")
+
+    sub_scores, drivers = _explainer.explain(frame)  # type: ignore[union-attr]
+    on_prior = sum(1 for d in drivers if d["onPrior"])
+
+    # Confidence falls with the share of the catalogue that came back null. A
+    # prediction built on holes must report as one — the rule engine already does
+    # this through familiesOnPrior, and the two must not disagree.
+    supplied = sum(1 for v in req.features.values() if v is not None)
+    coverage = supplied / max(1, len(req.features))
+    confidence = int(round(max(20.0, min(95.0, 95 * coverage))))
+
+    return ScoreResponse(
+        enterpriseId=req.enterpriseId,
+        score=score,
+        band=band,
+        confidence=confidence,
+        probability=round(probability, 5),
+        subScores=sub_scores,
+        drivers=[Driver(**{k: d[k] for k in ("family", "weight", "featureCode", "value", "onPrior")})
+                 for d in drivers],
+        familiesOnPrior=on_prior,
+        modelVersion=f"{b['algorithm']}-stress-v1",
+    )
+
+
+# --------------------------------------------------------------------------- #
+
+
+@app.get("/health")
+def health() -> dict:
+    return {"status": "UP", "model": MODEL_PATH.exists()}
+
+
+@app.get("/model")
+def model_info() -> dict:
+    b = bundle()
+    return {
+        "algorithm": b["algorithm"],
+        "modelVersion": f"{b['algorithm']}-stress-v1",
+        "features": b["features"],
+        "families": FAMILIES,
+        "redThreshold": b["red_threshold"],
+        "watchThreshold": b["watch_threshold"],
+        "trainedOn": "synthetic DPD history — metrics are provisional",
+    }
+
+
+@app.post("/score", response_model=ScoreResponse)
+def score(req: ScoreRequest) -> ScoreResponse:
+    return _score_one(req)
+
+
+@app.post("/score/batch", response_model=list[ScoreResponse])
+def score_batch(req: BatchRequest) -> list[ScoreResponse]:
+    """One call for a nightly run over the whole book, rather than 263."""
+    return [_score_one(item) for item in req.items]
+
+
+# --------------------------------------------------------------------------- #
+# Forecast — quantile heads                                                    #
+# --------------------------------------------------------------------------- #
+
+FORECAST_PATH = MODEL_DIR / "forecast_model.pkl"
+
+_forecast: dict | None = None
+
+
+def forecast_bundle() -> dict | None:
+    """None when no forecast model has been trained — the caller falls back."""
+    global _forecast
+    if _forecast is None and FORECAST_PATH.exists():
+        with open(FORECAST_PATH, "rb") as fh:
+            _forecast = pickle.load(fh)
+    return _forecast
+
+
+class ForecastRequest(BaseModel):
+    enterpriseId: str
+    sectorId: int | None = None
+    districtId: int | None = None
+    features: dict[str, float | None] = Field(default_factory=dict)
+    # Trailing ledger statistics, computed by the caller. The service stays
+    # stateless: it holds no database credentials and cannot read a borrower's
+    # ledger even if asked to. Java already reads the ledger for the
+    # deterministic path, so this is the same query it was already making.
+    trailMean: float | None = None
+    trailSd: float | None = None
+    trailMin: float | None = None
+    trailInflow: float | None = None
+    trailMonths: float | None = None
+    emi: float | None = None
+    horizons: int = 6
+
+
+class ForecastPoint(BaseModel):
+    horizon: int
+    p10: float
+    p50: float
+    p90: float
+
+
+class ForecastResponse(BaseModel):
+    enterpriseId: str
+    engine: str = "MODEL"
+    modelVersion: str
+    points: list[ForecastPoint]
+
+
+@app.post("/forecast", response_model=ForecastResponse)
+def forecast(req: ForecastRequest) -> ForecastResponse:
+    b = forecast_bundle()
+    if b is None:
+        raise HTTPException(status_code=503, detail="No forecast model trained")
+
+    # Three months of ledger is the floor the deterministic strategy uses too. An
+    # enterprise with less gets no forecast rather than a fabricated one.
+    if (req.trailMonths or 0) < 3:
+        return ForecastResponse(
+            enterpriseId=req.enterpriseId, modelVersion=b["version"], points=[]
+        )
+
+    rows = []
+    for h in range(1, min(req.horizons, b["horizons"]) + 1):
+        row: dict[str, float | None] = {}
+        for name in b["features"]:
+            if name.endswith("__missing"):
+                row[name] = 1 if req.features.get(name.removesuffix("__missing")) is None else 0
+            elif name == "horizon":
+                row[name] = h
+            elif name == "trail_mean":
+                row[name] = req.trailMean
+            elif name == "trail_sd":
+                row[name] = req.trailSd
+            elif name == "trail_min":
+                row[name] = req.trailMin
+            elif name == "trail_inflow":
+                row[name] = req.trailInflow
+            elif name == "trail_months":
+                row[name] = req.trailMonths
+            elif name == "emi":
+                row[name] = req.emi
+            elif name == "sector_id":
+                row[name] = req.sectorId
+            elif name == "district_id":
+                row[name] = req.districtId
+            else:
+                row[name] = req.features.get(name)
+        rows.append(row)
+
+    frame = pd.DataFrame(rows, columns=b["features"]).astype("float64")
+
+    preds = {q: b["models"][q].predict(frame) for q in ("p10", "p50", "p90")}
+
+    points = []
+    for i, h in enumerate(range(1, len(rows) + 1)):
+        # Quantile regression does not guarantee ordering — sort per row. A
+        # crossed band is worse than no band, and it is the kind of thing that
+        # renders as a chart nobody can read rather than as an error.
+        lo, mid, hi = sorted(float(preds[q][i]) for q in ("p10", "p50", "p90"))
+
+        # Missing inputs widen the band and never lower its centre. The rule the
+        # deterministic strategy already honoured: a pessimistic P50 built on
+        # ignorance manufactures breaches out of not knowing.
+        absent = sum(1 for v in req.features.values() if v is None)
+        if absent:
+            widen = 1 + 0.06 * absent
+            half_lo = (mid - lo) * widen
+            half_hi = (hi - mid) * widen
+            lo, hi = mid - half_lo, mid + half_hi
+
+        points.append(ForecastPoint(horizon=h, p10=round(lo, 2), p50=round(mid, 2), p90=round(hi, 2)))
+
+    return ForecastResponse(
+        enterpriseId=req.enterpriseId, modelVersion=b["version"], points=points
+    )

serving/explain.py

@@ -0,0 +1,111 @@
+"""
+SHAP contributions, folded into the five families the console already speaks.
+
+The console, the reason sentences and the Hindi PWA are all written against
+``liquidity | behavioural | trendVolatility | obligationLoad | externalShock``.
+A model that explained itself in thirteen feature names would need every one of
+those surfaces rewritten, and would hand a field officer a vocabulary they have
+never used. So the SHAP values are aggregated to the families instead, and the
+feature that contributed most inside each family is named alongside it.
+
+The family map mirrors ``RuleBasedScoringStrategy`` exactly. Where the two
+disagree, the rule engine is right and this file is wrong — an officer must not
+see the same enterprise explained two different ways depending on which engine
+happened to score it.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import shap
+
+LIQUIDITY = "liquidity"
+BEHAVIOURAL = "behavioural"
+TREND_VOLATILITY = "trendVolatility"
+OBLIGATION_LOAD = "obligationLoad"
+EXTERNAL_SHOCK = "externalShock"
+
+FAMILIES = [LIQUIDITY, BEHAVIOURAL, TREND_VOLATILITY, OBLIGATION_LOAD, EXTERNAL_SHOCK]
+
+FAMILY_OF: dict[str, str] = {
+    # liquidity — is there cash in the business
+    "cash_flow_change": LIQUIDITY,
+    "inventory_days": LIQUIDITY,
+    "upi_inflow_change": LIQUIDITY,
+    # behavioural — how the borrower has been acting
+    "emi_delay": BEHAVIOURAL,
+    "days_since_last_entry": BEHAVIOURAL,
+    "dpd_at_t": BEHAVIOURAL,
+    # trend / volatility — how unstable the market around them is
+    "price_z_score": TREND_VOLATILITY,
+    # obligation load — how heavy the instalment is against what comes in
+    "emi_to_inflow": OBLIGATION_LOAD,
+    "emi_at_t": OBLIGATION_LOAD,
+    "outstanding_at_t": OBLIGATION_LOAD,
+    # external shock — what the weather and the mandi are doing to them
+    "margin_squeeze": EXTERNAL_SHOCK,
+    "feed_cost_change": EXTERNAL_SHOCK,
+    "output_price_change": EXTERNAL_SHOCK,
+    "rainfall_anomaly": EXTERNAL_SHOCK,
+    "heat_anomaly": EXTERNAL_SHOCK,
+}
+
+
+class Explainer:
+    """Wraps a TreeExplainer and does the family arithmetic."""
+
+    def __init__(self, model, features: list[str]):
+        self.features = features
+        self._shap = shap.TreeExplainer(model)
+
+    def explain(self, row: pd.DataFrame) -> tuple[dict[str, int], list[dict]]:
+        """
+        :return: (sub-scores 0–100 per family, drivers sorted by contribution)
+
+        Only contributions that push the score **up** are reported as drivers.
+        A negative SHAP value means "this is one of the reasons the score is not
+        worse", which is true and useless to an officer deciding whether to visit.
+        """
+        values = self._shap.shap_values(row[self.features])
+        if isinstance(values, list):  # older binary API returns one array per class
+            values = values[1]
+        contributions = np.asarray(values).reshape(-1)
+
+        by_family: dict[str, float] = {f: 0.0 for f in FAMILIES}
+        best_feature: dict[str, tuple[str, float]] = {}
+
+        for name, value in zip(self.features, contributions):
+            base = name.removesuffix("__missing")
+            family = FAMILY_OF.get(base)
+            if family is None:
+                continue
+            by_family[family] += float(value)
+            held = best_feature.get(family)
+            if held is None or abs(value) > abs(held[1]):
+                best_feature[family] = (base, float(value))
+
+        positive = {f: max(0.0, v) for f, v in by_family.items()}
+        total = sum(positive.values())
+
+        # No family pushed the score up — a genuinely calm enterprise. Spread the
+        # weight evenly rather than emitting five zeros, which the console would
+        # render as an empty explanation for a score that still exists.
+        if total <= 0:
+            sub_scores = {f: 20 for f in FAMILIES}
+        else:
+            sub_scores = {f: int(round(100 * v / total)) for f, v in positive.items()}
+
+        drivers = []
+        for family in FAMILIES:
+            feature, raw = best_feature.get(family, (None, 0.0))
+            drivers.append({
+                "family": family,
+                "weight": sub_scores[family],
+                "featureCode": feature,
+                "value": float(row[feature].iloc[0]) if feature in row and pd.notna(row[feature].iloc[0]) else None,
+                "onPrior": bool(feature is None or (feature in row and pd.isna(row[feature].iloc[0]))),
+                "shap": round(raw, 5),
+            })
+        drivers.sort(key=lambda d: d["weight"], reverse=True)
+        return sub_scores, drivers