diff --git a/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiAdminController.java b/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiAdminController.java new file mode 100644 index 0000000..7cf81fc --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiAdminController.java @@ -0,0 +1,49 @@ +package com.lincfusion.udyamlinc.enterprise; + +import com.lincfusion.udyamlinc.common.ApiResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; + +/** + * The simulated-UPI surface. + * + *

Mounted under {@code /admin} so the existing rule applies — any officer may + * read the coverage, only a supervisor may generate. Kept separate from the + * Account Aggregator integration it stands in for, so that when consent lands + * the real ingestion is a new endpoint beside this one rather than a change + * inside it. + */ +@RestController +@RequestMapping("/admin/upi") +@RequiredArgsConstructor +public class UpiAdminController { + + private final UpiSimulationService simulationService; + + /** + * Generate 26 weeks of weekly aggregates for every consented enterprise that + * has none. Rows carry {@code source = SIMULATED} and are never written over + * a real Account Aggregator row. + */ + @PostMapping("/simulate") + public ApiResponse simulate( + @RequestParam(defaultValue = "false") boolean overwrite) { + UpiSimulationService.SimulationResult result = simulationService.simulate(overwrite); + return ApiResponse.ok(result.message(), result); + } + + /** + * How much of the UPI table is simulated. The console reads this to badge the + * behavioural family honestly — a screen showing an inflow trend without + * saying where it came from is the failure this whole marker exists to + * prevent. + */ + @GetMapping("/coverage") + public ApiResponse coverage() { + UpiSimulationService.Coverage c = simulationService.coverage(); + String message = c.consented() + ? "%d rows · %d from Account Aggregator · %d simulated".formatted(c.total(), c.accountAggregator(), c.simulated()) + : "%d rows, all simulated — no Account Aggregator consent exists yet".formatted(c.total()); + return ApiResponse.ok(message, c); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiSimulationService.java b/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiSimulationService.java new file mode 100644 index 0000000..525cdfb --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/enterprise/UpiSimulationService.java @@ -0,0 +1,249 @@ +package com.lincfusion.udyamlinc.enterprise; + +import com.lincfusion.udyamlinc.ingestion.Weeks; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Objects; +import java.util.Random; + +/** + * Weekly UPI aggregates for the demo portfolio, written with + * {@code source = SIMULATED} so nothing can mistake them for consented data. + * + *

Account Aggregator consent is not built — FIU registration under Sahamati + * is a relationship with a long lead time, not a sprint — so + * {@code upi_inflow_change} is null with {@code NO_CONSENT} for every enterprise + * and TW-02 has never been able to fire. That leaves a whole + * behavioural family dark in a console whose job is to show behaviour changing. + * + *

The rule this follows is the one {@code LoanRecord.source} already + * establishes: simulated data is fine in a prototype provided the real seam + * exists beside it and the two can never be confused. So these rows go into the + * same {@link UpiSummary} table an Account Aggregator would write to, through + * the same feature derivation, and carry a marker that survives in the database + * forever. When AA consent lands, real rows arrive alongside these and + * {@link #coverage()} reports the split. + * + *

What is deliberately not simulated

+ * + *

Only enterprises with {@code consentActive} get rows. An enterprise that + * has not consented stays null with {@code NO_CONSENT}, because the point of + * that reason code is to distinguish "we may not look" from "we looked and found + * nothing" — and a simulator that ignored consent would erase the distinction + * the schema was built to hold. + * + *

Individual transactions are never generated, only weekly counts and totals. + * A consent artefact permits the aggregate; simulating more than the real thing + * would be allowed to collect would make this a worse rehearsal, not a better + * one. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class UpiSimulationService { + + /** The marker that keeps simulated inflow distinguishable forever. */ + public static final String SOURCE = "SIMULATED"; + + /** + * Twenty-six weeks. {@code upi_inflow_change} compares an eight-week window + * against the eight before it and returns INSUFFICIENT_HISTORY below sixteen, + * so anything less than that is a generator that produces no feature. The + * extra ten weeks give the trailing query something to be point-in-time + * correct about. + */ + private static final int WEEKS = 26; + + private final EnterpriseRepository enterpriseRepository; + private final UpiSummaryRepository upiRepository; + private final LedgerEntryRepository ledgerRepository; + + public record SimulationResult(int enterprisesWritten, int weeksWritten, int skippedNoConsent, + int skippedExisting, String message) { + } + + public record Coverage(long total, long simulated, long accountAggregator, long other, + boolean consented) { + } + + /** + * @param overwrite replace existing SIMULATED rows. Rows from any other + * source are never touched — once a real Account Aggregator + * feed exists for an enterprise, the simulator must not be + * able to overwrite it, whatever it is asked to do. + */ + @Transactional + public SimulationResult simulate(boolean overwrite) { + LocalDate thisWeek = Weeks.startOf(LocalDate.now()); + LocalDateTime observedAt = LocalDateTime.now(); + + int enterprises = 0, weeks = 0, noConsent = 0, existing = 0; + + for (Enterprise e : enterpriseRepository.findAll()) { + if (!Boolean.TRUE.equals(e.getConsentActive())) { + noConsent++; + continue; + } + + List already = upiRepository + .findTrailing(e.getEnterpriseId(), thisWeek, observedAt); + boolean hasReal = already.stream().anyMatch(u -> !SOURCE.equals(u.getSource())); + if (hasReal) { + existing++; + continue; + } + if (!already.isEmpty() && !overwrite) { + existing++; + continue; + } + + int written = writeSeries(e, thisWeek, observedAt); + if (written > 0) { + enterprises++; + weeks += written; + } + } + + String message = "%d enterprises · %d weeks · %d skipped for no consent · %d already covered" + .formatted(enterprises, weeks, noConsent, existing); + log.info("UPI simulation — {}", message); + return new SimulationResult(enterprises, weeks, noConsent, existing, message); + } + + /** + * One enterprise's series. + * + *

The level is anchored to the enterprise's own ledger inflow rather than + * to a constant, so UPI and the ledger tell the same story about the size of + * the business — an enterprise whose books show ₹40,000 a week and whose UPI + * shows ₹4,00,000 would make {@code emi_to_inflow} and + * {@code upi_inflow_change} disagree about the same firm. + * + *

The trend is what carries the signal, and it tracks the band: + * a RED enterprise's digital receipts are falling, which is the observation + * TW-02 exists to catch. Deterministic off the enterprise id, like every + * other seeded figure here, so a demo shows the same series twice. + */ + private int writeSeries(Enterprise e, LocalDate thisWeek, LocalDateTime observedAt) { + BigDecimal weeklyBase = weeklyLedgerInflow(e); + if (weeklyBase.signum() <= 0) return 0; + + Random r = new Random((e.getEnterpriseId() + ":upi").hashCode()); + + // Total drift across the whole 26-week window. + double drift = switch (Objects.toString(e.getBand(), "STANDARD")) { + case "RED" -> -0.42 - r.nextDouble() * 0.16; + case "WATCH" -> -0.12 - r.nextDouble() * 0.10; + default -> 0.02 + r.nextDouble() * 0.10; + }; + + String consentRef = "SIM-CONSENT-" + e.getEnterpriseId(); + int written = 0; + + for (int i = 0; i < WEEKS; i++) { + LocalDate weekStart = thisWeek.minusWeeks(i); + // i = 0 is the newest week, so progress runs backwards: the oldest + // week sits at the start of the drift and the newest at the end. + double progress = (double) (WEEKS - 1 - i) / (WEEKS - 1); + // Deterioration starts partway in and accelerates, rather than + // sloping evenly from week one. + // + // Two reasons, and the first is arithmetic. TW-02 compares an + // eight-week window against the eight before it, which samples only + // about a third of a linear 26-week slope — a 40% total decline + // shows up as roughly 13% and never reaches the rule's −20%. The + // first version of this generator did exactly that: every RED + // enterprise was visibly declining and TW-02 still fired five times + // across the whole portfolio, all of them on noise. + // + // The second is that a linear slide is not what stress looks like. + // Receipts hold up while an enterprise draws on savings and + // goodwill, then fall away faster once those are gone. Starting the + // decline at 40% of the window and curving it makes the recent + // window genuinely worse than the prior one, which is the shape the + // rule was written to detect. + double onset = Math.max(0.0, (progress - 0.40) / 0.60); + double trend = 1.0 + drift * Math.pow(onset, 1.5); + // Week-to-week noise, deliberately smaller than the drift it sits on; + // a series whose noise swamps its trend would make TW-02 fire on + // arithmetic rather than on behaviour. + double noise = 0.92 + r.nextDouble() * 0.16; + + BigDecimal inflow = weeklyBase + .multiply(BigDecimal.valueOf(Math.max(0.05, trend * noise))) + .setScale(2, RoundingMode.HALF_UP); + // Outflow trails inflow — a trading enterprise pays out most of what + // it takes in, and the gap is roughly its margin. + BigDecimal outflow = inflow + .multiply(BigDecimal.valueOf(0.70 + r.nextDouble() * 0.15)) + .setScale(2, RoundingMode.HALF_UP); + + int inflowCount = 12 + r.nextInt(40); + + UpiSummary row = upiRepository + .findByEnterpriseIdAndWeekStart(e.getEnterpriseId(), weekStart) + .orElseGet(UpiSummary::new); + // Never overwrite a real row, even one reached through this path. + if (row.getId() != null && !SOURCE.equals(row.getSource())) continue; + + row.setEnterpriseId(e.getEnterpriseId()); + row.setWeekStart(weekStart); + row.setInflowAmount(inflow); + row.setOutflowAmount(outflow); + row.setInflowCount(inflowCount); + row.setOutflowCount(Math.max(1, inflowCount / 3)); + row.setSource(SOURCE); + row.setConsentRef(consentRef); + // Weekly aggregates are known at the end of their week, not during + // it, so a run scored mid-week must not see this week's total as + // though it had already closed. + row.setValidForDate(weekStart.plusDays(6)); + row.setObservedAt(observedAt); + upiRepository.save(row); + written++; + } + return written; + } + + /** + * The enterprise's own weekly money-in, from its ledger. Falls back to the + * instalment when there is no ledger to read — an enterprise still has to + * turn over more than its EMI for the loan to have been written at all. + */ + private BigDecimal weeklyLedgerInflow(Enterprise e) { + LocalDate from = LocalDate.now().minusWeeks(12); + List entries = ledgerRepository + .findByEnterpriseIdAndEntryDateGreaterThanEqualOrderByEntryDateDesc(e.getEnterpriseId(), from); + + BigDecimal inflow = entries.stream() + .filter(l -> "IN".equals(l.getKind())) + .map(LedgerEntry::getAmount) + .filter(Objects::nonNull) + .reduce(BigDecimal.ZERO, BigDecimal::add); + + if (inflow.signum() > 0) { + return inflow.divide(BigDecimal.valueOf(12), 2, RoundingMode.HALF_UP); + } + BigDecimal instalment = e.getMonthlyInstalment(); + if (instalment == null || instalment.signum() <= 0) return BigDecimal.ZERO; + // Roughly three times the instalment a month, spread over four weeks. + return instalment.multiply(BigDecimal.valueOf(3)) + .divide(BigDecimal.valueOf(4), 2, RoundingMode.HALF_UP); + } + + /** How much of the UPI table is simulated, for the honesty banner. */ + public Coverage coverage() { + List all = upiRepository.findAll(); + long simulated = all.stream().filter(u -> SOURCE.equals(u.getSource())).count(); + long aa = all.stream().filter(u -> "ACCOUNT_AGGREGATOR".equals(u.getSource())).count(); + return new Coverage(all.size(), simulated, aa, all.size() - simulated - aa, aa > 0); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/ingestion/NoFeedPriceSimulationService.java b/src/main/java/com/lincfusion/udyamlinc/ingestion/NoFeedPriceSimulationService.java new file mode 100644 index 0000000..82831bf --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/ingestion/NoFeedPriceSimulationService.java @@ -0,0 +1,236 @@ +package com.lincfusion.udyamlinc.ingestion; + +import com.lincfusion.udyamlinc.master.Commodity; +import com.lincfusion.udyamlinc.master.CommodityRepository; +import com.lincfusion.udyamlinc.master.District; +import com.lincfusion.udyamlinc.master.DistrictRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Random; + +/** + * Weekly price series for the commodities no API publishes, written with + * {@code source = SIMULATED}. + * + *

Milk, egg, green fodder and sugarcane have no usable feed and are not going + * to acquire one: milk comes from a dairy federation, egg from NECC's HTML + * pages, fodder trades in one market in Haryana, and cane never reaches a mandi + * because it is sold to mills at a set price. Manual entry exists in this system + * precisely because of them. + * + *

Their absence is not evenly distributed — it lands on two whole sector + * packs. {@code price_z_score}, {@code output_price_change} and + * {@code margin_squeeze} all read the sector's OUTPUT commodity, and dairy's + * output is milk while poultry's is egg. So a hundred and two enterprises had + * three features permanently null, and margin squeeze — the single derivation + * this project exists to compute — could not be evaluated for either pack. A + * poultry unit whose feed cost is known and whose egg price is unknowable is + * exactly the case the deck opens with. + * + *

Same discipline as {@code UpiSimulationService} and {@code + * LoanRecord.source}: these rows sit in the same table the real feed writes to, + * flow through the same derivations, and carry a marker that survives forever. + * The ingestion upsert keys on {@code (district, commodity, week, source)}, so a + * real MANUAL or AGMARKNET row for the same week coexists rather than colliding, + * and nothing here can overwrite one. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class NoFeedPriceSimulationService { + + public static final String SOURCE = "SIMULATED"; + + /** Three years, to match the Agmarknet backfill the seasonal mean is built from. */ + private static final int WEEKS = 156; + + /** + * Base price per unit and the annual seasonal swing, by commodity code. + * Levels are ordinary Indian farm-gate figures — a milk procurement rate + * around ₹38/litre, eggs around ₹5.60 each so ₹560 a hundred — because a + * simulated series that is plausible in level as well as in shape is the one + * an officer can sanity-check on sight. + * + *

{@code trendPerYear} is deliberately small. A steady annual climb makes + * the newest observation the highest the series has ever been, which + * {@code price_z_score} reads as a permanent anomaly — 5%/year compounding + * over the three-year window swamped a ±9% year-to-year shift and left + * simulated sectors averaging a z of 1.9 against real Agmarknet data's 0.44. + * Year-over-year variation has to dominate the trend, not the other way + * round. + * + * @param weeklyNoise fractional week-to-week variation, and the parameter that + * decides whether {@code price_z_score} means anything. + *

A z-score divides the deviation from a seasonal mean by + * that mean's standard deviation. On a series that trends + * steadily with almost no noise, the standard deviation is + * tiny and the newest value sits at the top of its own + * distribution by construction — so every enterprise + * scores 1.5 to 2.0 sigma and TW-05 fires on all of them. The + * first version of these indices used ±1.5% and fired TW-05 + * for 197 of 263 enterprises, which is not an early warning, + * it is a light that is always on. Noise has to be large + * enough relative to the annual trend for "unusual" to keep + * its meaning. + */ + private record Profile(double base, double seasonalAmplitude, double trendPerYear, + int peakWeek, double weeklyNoise) { + } + + private static final Map PROFILES = Map.of( + // Milk: procurement rate rises in the lean summer flush-off and eases + // after the monsoon, and drifts up a few percent a year. + // + // The level is anchored, not invented. data.gov.in resource + // 44724649-2fc1-40bb-85b8-8979832c8dd2 publishes state-wise + // cooperative procurement prices for June 2018: cow ₹25.12, mixed + // ₹30.52, buffalo ₹39.10 per litre. It is a two-year snapshot rather + // than a series, so it cannot be ingested — but it pins the level. + // ₹25.12 carried forward at roughly 5% a year to 2026 gives ~₹37, so + // 38 is a defensible cow-milk procurement rate with a citation + // behind it instead of a guess. + "MILK", new Profile(38.0, 0.08, 0.025, 22, 0.035), + // Egg: the sharpest seasonal of the four. Winter demand lifts the + // NECC rate hard and summer flattens it — the swing that makes a + // poultry margin squeeze real rather than theoretical. + "EGG", new Profile(560.0, 0.22, 0.02, 50, 0.055), + // Green fodder: scarce and dear in the dry months before the monsoon. + "FODDER", new Profile(310.0, 0.18, 0.025, 18, 0.06), + // Cane is an administered price, so it barely moves within a season + // and steps up between them. + // Genuinely near-flat, because it is an administered price. Its + // z-score is meaningless for the same reason a real administered + // price's would be, which is honest rather than a defect. + "SUGARCANE", new Profile(340.0, 0.03, 0.04, 1, 0.012), + + // Index series, base 100. Deliberately gentler than the commodity + // profiles: a processed-food or retail price does not swing like a + // mandi rate, because processing and retail absorb input shocks into + // margin before passing them on. That is the whole point of the + // margin_squeeze these feed — an input spike against a sticky output + // is exactly what squeezes a processor, and giving the output the + // same volatility as the input would cancel the signal out. + // Noise here is set relative to the annual trend, not to taste. At + // ±1.5% against a 5%/year drift these produced z-scores of 1.5–2.0 + // for almost every enterprise; at ±4–5% the newest observation is no + // longer an outlier merely for being the newest. + "PROCESSED_FOOD_INDEX", new Profile(100.0, 0.04, 0.015, 48, 0.042), + "RETAIL_BASKET_INDEX", new Profile(100.0, 0.03, 0.015, 44, 0.038), + "HANDLOOM_INDEX", new Profile(100.0, 0.06, 0.015, 42, 0.045), + // Yarn tracks cotton, so it is the most volatile of the four indices + // and peaks with the post-harvest cotton arrival window. + "YARN_INDEX", new Profile(100.0, 0.11, 0.015, 6, 0.065)); + + private final CommodityRepository commodityRepository; + private final DistrictRepository districtRepository; + private final MarketSignalRepository signalRepository; + + public record SimulationResult(int commodities, int districts, int weeksWritten, String message) { + } + + @Transactional + public SimulationResult simulate() { + LocalDate thisWeek = Weeks.startOf(LocalDate.now()); + LocalDateTime observedAt = LocalDateTime.now(); + List districts = districtRepository.findByActiveTrue(); + + int commodities = 0, written = 0; + + for (Map.Entry entry : PROFILES.entrySet()) { + Commodity commodity = commodityRepository.findByCode(entry.getKey()).orElse(null); + if (commodity == null) { + log.warn("No commodity {} to simulate prices for", entry.getKey()); + continue; + } + commodities++; + for (District district : districts) { + written += writeSeries(district, commodity, entry.getValue(), thisWeek, observedAt); + } + } + + String message = "%d commodities × %d districts · %d weeks written" + .formatted(commodities, districts.size(), written); + log.info("No-feed price simulation — {}", message); + return new SimulationResult(commodities, districts.size(), written, message); + } + + /** + * One district's series for one commodity. + * + *

Level, seasonal swing and a slow annual trend, plus district-to-district + * variation drawn from the district code so Sehore and Kolar do not report an + * identical milk rate. Deterministic throughout: the same build produces the + * same series, which is what lets a demo be talked through twice. + */ + private int writeSeries(District district, Commodity commodity, Profile profile, + LocalDate thisWeek, LocalDateTime observedAt) { + Random r = new Random((district.getCode() + ":" + commodity.getCode()).hashCode()); + // ±8% of level between districts, held constant for the whole series — + // a regional price difference, not weekly noise. + double regional = 0.92 + r.nextDouble() * 0.16; + int written = 0; + + for (int i = 0; i < WEEKS; i++) { + LocalDate weekStart = thisWeek.minusWeeks(i); + int isoWeek = Weeks.isoWeek(weekStart); + + // Seasonal term peaks at the profile's peak week and troughs opposite it. + double phase = 2 * Math.PI * (isoWeek - profile.peakWeek()) / 52.0; + double seasonal = 1.0 + profile.seasonalAmplitude() * Math.cos(phase); + // Trend runs forward in time, so the oldest week carries the least. + double years = (double) (WEEKS - 1 - i) / 52.0; + double trend = 1.0 + profile.trendPerYear() * years; + + // A year-level shift on top of the trend, so the series has good + // years and bad ones instead of climbing forever. + // + // Without it the newest observation is the highest the series has + // ever been, every time, and price_z_score — which measures the + // latest price against its own seasonal history — reads that as a + // permanent anomaly. Real Agmarknet series in this same database + // average a z of 0.44; the first monotonic version of these + // generators averaged 1.9 and tripped TW-05 for half the portfolio. + // Year-to-year variation is what makes "unusually high" a statement + // about this year rather than about the direction of time. + int yearIndex = (WEEKS - 1 - i) / 52; + double yearShift = 1.0 + (new Random((district.getCode() + ":" + commodity.getCode() + + ":y" + yearIndex).hashCode()).nextDouble() * 2 - 1) * 0.13; + double noise = 1.0 + (r.nextDouble() * 2 - 1) * profile.weeklyNoise(); + + BigDecimal modal = BigDecimal.valueOf( + profile.base() * regional * seasonal * trend * yearShift * noise) + .setScale(2, RoundingMode.HALF_UP); + + MarketSignal signal = signalRepository + .findByDistrictIdAndCommodityIdAndWeekStartAndSource( + district.getId(), commodity.getId(), weekStart, SOURCE) + .orElseGet(MarketSignal::new); + + signal.setDistrictId(district.getId()); + signal.setCommodityId(commodity.getId()); + signal.setWeekStart(weekStart); + signal.setModalPrice(modal); + signal.setMinPrice(modal.multiply(BigDecimal.valueOf(0.94)).setScale(2, RoundingMode.HALF_UP)); + signal.setMaxPrice(modal.multiply(BigDecimal.valueOf(1.07)).setScale(2, RoundingMode.HALF_UP)); + signal.setArrivalCount(null); + signal.setUnit(commodity.getUnit()); + signal.setSource(SOURCE); + // A weekly aggregate is known once its week has closed. + signal.setValidForDate(weekStart.plusDays(6)); + signal.setObservedAt(observedAt); + signal.setUpdatedAt(observedAt); + signalRepository.save(signal); + written++; + } + return written; + } +}