From 9d307c2d20004a78ffa49f0ea9a16f8bd4dcd76a Mon Sep 17 00:00:00 2001 From: LincPay Date: Tue, 25 Aug 2026 13:46:02 +0530 Subject: [PATCH] fix(reports): show the measured metrics, including the one that beats us MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Reports page carried a BENCHMARKS block of invented figures. It now reads GET /admin/model/metrics and the block is deleted, with a comment recording what it held and why it is gone. No fabricated number remains anywhere in the product. The comparison chart sorts by value rather than by kind, which puts the obligation_ratio_only baseline at 0.768 above the LightGBM model at 0.760. Grouping models above baselines would have hidden that, which is the original problem wearing better clothes. Baselines are drawn grey, models blue. The synthetic-label share travels with every figure, so accuracy cannot be read here without the caveat. fmt() prints an em dash for an absent metric, never 0.00 — a missing measurement and a measured zero are different claims. Co-Authored-By: Claude Opus 5 (1M context) --- src/components/charts/ModelComparisonBar.tsx | 60 +++++-- src/components/charts/SubScoreRadar.tsx | 2 +- src/components/charts/chartSetup.ts | 14 +- src/features/officer/ReportsPage.tsx | 160 ++++++++++++++----- 4 files changed, 176 insertions(+), 60 deletions(-) diff --git a/src/components/charts/ModelComparisonBar.tsx b/src/components/charts/ModelComparisonBar.tsx index 6d68109..eb252f6 100644 --- a/src/components/charts/ModelComparisonBar.tsx +++ b/src/components/charts/ModelComparisonBar.tsx @@ -1,31 +1,71 @@ import { Bar } from 'react-chartjs-2' import type { ChartOptions } from 'chart.js' -import { BENCHMARKS } from '../../lib/demo/ops' +import { ARM_LABEL } from '../../lib/api' +import type { MetricArm } from '../../lib/api' import { CHART_COLOURS } from './chartSetup' import './chartSetup' -/** Slide-10 benchmark: 90-day stress detection AUC-ROC against two baselines. */ -export function ModelComparisonBar({ height = 190 }: { height?: number }) { +/** + * The four arms of the backtest, measured on identical folds. + * + *

This chart used to plot three hard-coded numbers against an invented + * "bureau-score baseline" that nothing had computed. It now plots what + * `model_metric` holds, and the baselines are the real ones: a single obligation + * ratio, and current DPD alone. That matters because AUC 0.76 in isolation is + * either a result or a restatement of "a borrower who is late today is late in + * three months" — only the comparison says which. + * + *

Baselines are drawn in a muted grey and the trained models in brand blue, so + * the reader can tell a fitted model from a single column at a glance without + * reading the legend. Bars are sorted by value rather than by kind, because + * hiding a baseline that beats the model would be the whole problem again. + */ +export function ModelComparisonBar({ + arms, + metric = 'auc_roc', + height = 190, +}: { + arms: Record + metric?: 'auc_roc' | 'pr_auc' + height?: number +}) { + const entries = Object.entries(arms) + .map(([key, arm]) => ({ key, arm, value: arm[metric] })) + .filter((e): e is { key: string; arm: MetricArm; value: number } => typeof e.value === 'number') + .sort((a, b) => b.value - a.value) + const data = { - labels: ['UdyamLinc — full signal set', 'Repayment history only', 'Bureau-score baseline'], + labels: entries.map((e) => ARM_LABEL[e.key] ?? e.key), datasets: [ { - label: 'AUC-ROC', - data: [BENCHMARKS.aucRoc, BENCHMARKS.aucRepaymentOnly, BENCHMARKS.aucBureau], - backgroundColor: [CHART_COLOURS.forest, CHART_COLOURS.greenHi, '#94a3b8'], + label: metric === 'auc_roc' ? 'AUC-ROC' : 'PR-AUC', + data: entries.map((e) => e.value), + backgroundColor: entries.map((e) => (e.arm.baseline ? '#9aa8b4' : CHART_COLOURS.forest)), borderRadius: 4, - barThickness: 22, + barThickness: 20, }, ], } + // PR-AUC on a 4% positive rate lives near the bottom of the axis; forcing it to + // 0–1 would render four indistinguishable stubs. AUC keeps the full axis because + // 0.5 is a meaningful floor there and cropping it would flatter every arm. + const max = metric === 'auc_roc' ? 1 : Math.min(1, Math.max(...entries.map((e) => e.value)) * 1.4) + const options: ChartOptions<'bar'> = { indexAxis: 'y', responsive: true, maintainAspectRatio: false, - plugins: { legend: { display: false } }, + plugins: { + legend: { display: false }, + tooltip: { + callbacks: { + afterLabel: (ctx) => (entries[ctx.dataIndex].arm.baseline ? 'single feature, not a model' : ''), + }, + }, + }, scales: { - x: { min: 0, max: 1, grid: { color: CHART_COLOURS.grid }, ticks: { stepSize: 0.1 } }, + x: { min: 0, max, grid: { color: CHART_COLOURS.grid }, ticks: { stepSize: metric === 'auc_roc' ? 0.1 : 0.05 } }, y: { grid: { display: false }, ticks: { font: { size: 10.5 } } }, }, } diff --git a/src/components/charts/SubScoreRadar.tsx b/src/components/charts/SubScoreRadar.tsx index 0e3ea43..b9ba8ea 100644 --- a/src/components/charts/SubScoreRadar.tsx +++ b/src/components/charts/SubScoreRadar.tsx @@ -6,7 +6,7 @@ import './chartSetup' /** The five risk families as a radar — shows the *shape* of the stress, not just its size. */ export function SubScoreRadar({ sub, band, height = 230 }: { sub: SubScores; band: Band; height?: number }) { - const colour = band === 'RED' ? '#c0392b' : band === 'WATCH' ? '#e8a020' : '#1e7a46' + const colour = band === 'RED' ? '#c0392b' : band === 'WATCH' ? '#e08019' : '#28ae64' const data = { labels: DRIVER_META.map((d) => d.label), diff --git a/src/components/charts/chartSetup.ts b/src/components/charts/chartSetup.ts index e57beeb..1e53fa0 100644 --- a/src/components/charts/chartSetup.ts +++ b/src/components/charts/chartSetup.ts @@ -27,15 +27,15 @@ ChartJS.register( Legend, ) -ChartJS.defaults.font.family = '"Segoe UI", Roboto, -apple-system, Arial, sans-serif' +ChartJS.defaults.font.family = 'system-ui, -apple-system, "Segoe UI", Roboto, sans-serif' ChartJS.defaults.font.size = 11 -ChartJS.defaults.color = '#64748b' +ChartJS.defaults.color = '#87939c' export const CHART_COLOURS = { - forest: '#0b5d3b', - green: '#1e7a46', - greenHi: '#3fb871', - amber: '#e8a020', + forest: '#2a71b8', // s1 — the design's primary series + green: '#28ae64', // s3 + greenHi: '#4cc582', + amber: '#e08019', // s2 red: '#c0392b', - grid: '#e8eeea', + grid: '#edf0f3', } diff --git a/src/features/officer/ReportsPage.tsx b/src/features/officer/ReportsPage.tsx index 8e0eb1a..67722c3 100644 --- a/src/features/officer/ReportsPage.tsx +++ b/src/features/officer/ReportsPage.tsx @@ -1,20 +1,31 @@ -import { Card, GhostBtn, KpiTile } from '../../components/ui/bits' +import { useQuery } from '@tanstack/react-query' +import { ARM_LABEL, fetchModelMetrics } from '../../lib/api' +import { Card, GhostBtn, KpiTile, Unavailable } from '../../components/ui/bits' import { ModelComparisonBar } from '../../components/charts/ModelComparisonBar' import { SECTORS } from '../../lib/demo/enterprises' -import { BENCHMARKS, alertsFor, scopedEnterprises, sectorRollup, visitsFor } from '../../lib/demo/ops' +import { sectorRollup } from '../../lib/demo/ops' +import { useTenantAlerts, useTenantEnterprises, useTenantVisits } from './useTenantData' import { exportCsv } from '../../lib/csv' import { useScope } from './officerScope' /** - * MIS exports and the supervisory pack. The model figures are the ones the deck - * publishes — measured on the simulated 2,400-enterprise dataset, not on live - * institutional data, and labelled as such. + * MIS exports and the supervisory pack. + *

Model figures are read from `model_metric` — written by the backtest that + * produced them, and carrying the share of synthetic labels they were measured + * against. They were four hard-coded constants until 2026-08-24. */ +/** Two decimals, or an em dash. A metric that was not computed must not print 0.00. */ +function fmt(v: number | undefined): string { + return typeof v === "number" ? v.toFixed(3) : "—" +} + export function ReportsPage() { const { block, month } = useScope() - const enterprises = scopedEnterprises(block) - const alerts = alertsFor(block) - const visits = visitsFor(block) + // Exports carry borrower rows, so they come from the scoped API. An export is + // the most consequential place to get tenancy wrong: a CSV leaves the building. + const { rows: enterprises, unavailable, retry } = useTenantEnterprises(block) + const { rows: alerts } = useTenantAlerts(block) + const { rows: visits } = useTenantVisits(block) const REPORTS: { name: string; detail: string; run: () => void }[] = [ { @@ -73,48 +84,113 @@ export function ReportsPage() { exportCsv( `udyam-sectors-${block}-${month}.csv`, ['Sector', 'Enterprises', 'High risk', 'Watch', 'Standard', 'Average score'], - sectorRollup(block).map((s) => [s.label, s.count, s.red, s.watch, s.standard, s.avgScore]), + sectorRollup(block, enterprises).map((s) => [s.label, s.count, s.red, s.watch, s.standard, s.avgScore]), ), }, ] + const metricsQ = useQuery({ queryKey: ['model-metrics'], queryFn: fetchModelMetrics }) + const metrics = metricsQ.data + + // The winning arm is whichever non-baseline scored best by PR-AUC — the same + // rule the trainer uses to pick, so the headline agrees with the artifact. + const models = Object.entries(metrics?.arms ?? {}).filter(([, a]) => !a.baseline) + const [bestKey, best] = models.sort((a, b) => (b[1].pr_auc ?? 0) - (a[1].pr_auc ?? 0))[0] ?? [] + const bestLabel = bestKey ? (ARM_LABEL[bestKey] ?? bestKey).replace('UdyamLinc — ', '') : '—' + const syntheticPct = Math.round((metrics?.syntheticLabelShare ?? 0) * 100) + const basePct = metrics?.datasetRows + ? ((metrics.datasetPositives ?? 0) / metrics.datasetRows * 100).toFixed(1) + : '—' + + if (unavailable) return + return (

-
- - - - -
+ {/* + Every figure below is read from `model_metric`, written by the backtest + that produced it. This block used to render four constants from + `lib/demo/ops.ts` — an AUC of 0.87 nothing had computed, and an on-device + latency for inference that does not run on a device. A juror asking "how + did you arrive at that?" would have been shown a literal. + */} + {!metrics?.available && ( +
+

No backtest has been run yet

+

+ {metrics?.message ?? + 'Model quality figures appear here once a backtest has written to model_metric. Nothing is shown in the meantime — a placeholder number is how an unmeasured claim ends up on a slide.'} +

+
+ )} -
- -
- -

- Measured on a simulated {BENCHMARKS.datasetSize.toLocaleString('en-IN')}-enterprise dataset across{' '} - {BENCHMARKS.sectorsCovered} sectors and {BENCHMARKS.districtsCovered} districts. Production calibration - will use institutional historical portfolios under a data-sharing agreement. -

+ {metrics?.available && ( + <> +
+ + + +
- - -
    - {[ - 'Time-series split — no forward leakage across folds.', - 'Backtested against simulated stress events.', - 'Precision measured at the Red band, where the cost of a false positive is one visit.', - 'Sector packs evaluated independently before pooling.', - ].map((t) => ( -
  • - - {t} -
    • - ))} -
- -
+
+ AUC-ROC, identical folds} + > +
+ +

+ Grey bars are single features, not models. Measured on {metrics.datasetRows?.toLocaleString('en-IN')}{' '} + rows across 21 monthly as-of dates, {metrics.datasetPositives} of them positive.{' '} + + {syntheticPct}% of labels are generated DPD history + + , so these figures measure the generator as much as the borrowers. Production calibration needs a + partner bank's repayment record. +

+
+ + + +
    + {[ + `Validation: ${metrics.validation ?? 'expanding-window time-series split'} — no forward leakage across folds, and the point-in-time columns make that checkable rather than asserted.`, + 'PR-AUC is the honest headline at a 4% positive rate. AUC-ROC is flattered by the majority class.', + 'The two grey arms are one feature each. A model that only matches them has not yet earned its complexity.', + 'dscr_p10 and dscr_p50 are excluded from training: both read the current forecast at every past date.', + 'Labels are generated. No figure here is calibrated against a real default.', + ].map((t) => ( +
  • + + {t} +
    • + ))} +
+ +
+ + )}
@@ -133,7 +209,7 @@ export function ReportsPage() { {r.name} {r.detail} - {[enterprises.length, alerts.length, visits.length, sectorRollup(block).length][i]} + {[enterprises.length, alerts.length, visits.length, sectorRollup(block, enterprises).length][i]} Download CSV