feat(simulation): fill what has no feed, marked so it can never pass as real
Nine of fifteen features now reach 263/263 and all six trip-wires fire for the
first time — TW-02 and TW-05 had never fired in the project's life. That
coverage comes partly from simulation, and every simulated row says so:
source = SIMULATED, upserted on a key that includes source, and refused if the
existing row is real even with overwrite=true. Real and simulated coexist; the
simulator can never overwrite a fact.
UpiSimulationService writes 26 weeks of upi_summary for each enterprise and
respects consentActive — a simulator that ignored consent would erase the very
distinction NO_CONSENT exists to hold.
NoFeedPriceSimulationService covers milk, egg, fodder and sugarcane, which have
no usable API anywhere: data.gov.in carries only annual state averages, NECC
publishes HTML, and cane is sold to mills at a set price and never reaches a
mandi. Milk's level is anchored on a real 2018 cooperative procurement figure
carried forward.
Calibrating a generated series took three passes, each plausible as a chart and
wrong as an input to a specific rule. Real Agmarknet series average a
price_z_score of 0.44, which is what exposed all of it:
- Too smooth. Index series at +/-1.5% weekly noise gave a tiny seasonal stdev,
so the newest value was an outlier by construction and TW-05 fired for 197
of 263. Noise must be large relative to trend.
- Monotonic. Even at +/-4% noise, a 5%/yr trend compounding over three years
swamped a +/-9% year shift, so every newest week was a record. Trend cut to
1.5-2.5%/yr and the year shift widened to +/-13%, so year-over-year
variation dominates. Portfolio mean z 1.39 -> 0.91, TW-05 197 -> 72.
- Wrong window. A linear decline over 26 weeks moves an 8-vs-8-week comparison
by only a third of its total, so a 40% drift showed as 13% and never reached
TW-02's -20% threshold. The decline now starts at 40% of the window and
curves. Check a generated series against the rule's own window, not against
its total drift.
Alerts are cumulative, so stale rows from the miscalibrated runs had to be
cleared: risk_alert held 206 TW-05 rows while the corrected run raised one.
The four output indices are categorically weaker than the rest and are labelled
separately in the console. Milk and egg are real prices lacking an API; a
finished sari has no price series to lack.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,49 @@
|
|||||||
|
package com.lincfusion.udyamlinc.enterprise;
|
||||||
|
|
||||||
|
import com.lincfusion.udyamlinc.common.ApiResponse;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The simulated-UPI surface.
|
||||||
|
*
|
||||||
|
* <p>Mounted under {@code /admin} so the existing rule applies — any officer may
|
||||||
|
* read the coverage, only a supervisor may generate. Kept separate from the
|
||||||
|
* Account Aggregator integration it stands in for, so that when consent lands
|
||||||
|
* the real ingestion is a new endpoint beside this one rather than a change
|
||||||
|
* inside it.
|
||||||
|
*/
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/admin/upi")
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class UpiAdminController {
|
||||||
|
|
||||||
|
private final UpiSimulationService simulationService;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate 26 weeks of weekly aggregates for every consented enterprise that
|
||||||
|
* has none. Rows carry {@code source = SIMULATED} and are never written over
|
||||||
|
* a real Account Aggregator row.
|
||||||
|
*/
|
||||||
|
@PostMapping("/simulate")
|
||||||
|
public ApiResponse<UpiSimulationService.SimulationResult> simulate(
|
||||||
|
@RequestParam(defaultValue = "false") boolean overwrite) {
|
||||||
|
UpiSimulationService.SimulationResult result = simulationService.simulate(overwrite);
|
||||||
|
return ApiResponse.ok(result.message(), result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How much of the UPI table is simulated. The console reads this to badge the
|
||||||
|
* behavioural family honestly — a screen showing an inflow trend without
|
||||||
|
* saying where it came from is the failure this whole marker exists to
|
||||||
|
* prevent.
|
||||||
|
*/
|
||||||
|
@GetMapping("/coverage")
|
||||||
|
public ApiResponse<UpiSimulationService.Coverage> coverage() {
|
||||||
|
UpiSimulationService.Coverage c = simulationService.coverage();
|
||||||
|
String message = c.consented()
|
||||||
|
? "%d rows · %d from Account Aggregator · %d simulated".formatted(c.total(), c.accountAggregator(), c.simulated())
|
||||||
|
: "%d rows, all simulated — no Account Aggregator consent exists yet".formatted(c.total());
|
||||||
|
return ApiResponse.ok(message, c);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
package com.lincfusion.udyamlinc.enterprise;
|
||||||
|
|
||||||
|
import com.lincfusion.udyamlinc.ingestion.Weeks;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Weekly UPI aggregates for the demo portfolio, written with
|
||||||
|
* {@code source = SIMULATED} so nothing can mistake them for consented data.
|
||||||
|
*
|
||||||
|
* <p>Account Aggregator consent is not built — FIU registration under Sahamati
|
||||||
|
* is a relationship with a long lead time, not a sprint — so
|
||||||
|
* {@code upi_inflow_change} is null with {@code NO_CONSENT} for every enterprise
|
||||||
|
* and <strong>TW-02 has never been able to fire</strong>. That leaves a whole
|
||||||
|
* behavioural family dark in a console whose job is to show behaviour changing.
|
||||||
|
*
|
||||||
|
* <p>The rule this follows is the one {@code LoanRecord.source} already
|
||||||
|
* establishes: simulated data is fine in a prototype provided the real seam
|
||||||
|
* exists beside it and the two can never be confused. So these rows go into the
|
||||||
|
* same {@link UpiSummary} table an Account Aggregator would write to, through
|
||||||
|
* the same feature derivation, and carry a marker that survives in the database
|
||||||
|
* forever. When AA consent lands, real rows arrive alongside these and
|
||||||
|
* {@link #coverage()} reports the split.
|
||||||
|
*
|
||||||
|
* <h2>What is deliberately not simulated</h2>
|
||||||
|
*
|
||||||
|
* <p>Only enterprises with {@code consentActive} get rows. An enterprise that
|
||||||
|
* has not consented stays null with {@code NO_CONSENT}, because the point of
|
||||||
|
* that reason code is to distinguish "we may not look" from "we looked and found
|
||||||
|
* nothing" — and a simulator that ignored consent would erase the distinction
|
||||||
|
* the schema was built to hold.
|
||||||
|
*
|
||||||
|
* <p>Individual transactions are never generated, only weekly counts and totals.
|
||||||
|
* A consent artefact permits the aggregate; simulating more than the real thing
|
||||||
|
* would be allowed to collect would make this a worse rehearsal, not a better
|
||||||
|
* one.
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class UpiSimulationService {
|
||||||
|
|
||||||
|
/** The marker that keeps simulated inflow distinguishable forever. */
|
||||||
|
public static final String SOURCE = "SIMULATED";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Twenty-six weeks. {@code upi_inflow_change} compares an eight-week window
|
||||||
|
* against the eight before it and returns INSUFFICIENT_HISTORY below sixteen,
|
||||||
|
* so anything less than that is a generator that produces no feature. The
|
||||||
|
* extra ten weeks give the trailing query something to be point-in-time
|
||||||
|
* correct about.
|
||||||
|
*/
|
||||||
|
private static final int WEEKS = 26;
|
||||||
|
|
||||||
|
private final EnterpriseRepository enterpriseRepository;
|
||||||
|
private final UpiSummaryRepository upiRepository;
|
||||||
|
private final LedgerEntryRepository ledgerRepository;
|
||||||
|
|
||||||
|
public record SimulationResult(int enterprisesWritten, int weeksWritten, int skippedNoConsent,
|
||||||
|
int skippedExisting, String message) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public record Coverage(long total, long simulated, long accountAggregator, long other,
|
||||||
|
boolean consented) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param overwrite replace existing SIMULATED rows. Rows from any other
|
||||||
|
* source are never touched — once a real Account Aggregator
|
||||||
|
* feed exists for an enterprise, the simulator must not be
|
||||||
|
* able to overwrite it, whatever it is asked to do.
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public SimulationResult simulate(boolean overwrite) {
|
||||||
|
LocalDate thisWeek = Weeks.startOf(LocalDate.now());
|
||||||
|
LocalDateTime observedAt = LocalDateTime.now();
|
||||||
|
|
||||||
|
int enterprises = 0, weeks = 0, noConsent = 0, existing = 0;
|
||||||
|
|
||||||
|
for (Enterprise e : enterpriseRepository.findAll()) {
|
||||||
|
if (!Boolean.TRUE.equals(e.getConsentActive())) {
|
||||||
|
noConsent++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<UpiSummary> already = upiRepository
|
||||||
|
.findTrailing(e.getEnterpriseId(), thisWeek, observedAt);
|
||||||
|
boolean hasReal = already.stream().anyMatch(u -> !SOURCE.equals(u.getSource()));
|
||||||
|
if (hasReal) {
|
||||||
|
existing++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!already.isEmpty() && !overwrite) {
|
||||||
|
existing++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
int written = writeSeries(e, thisWeek, observedAt);
|
||||||
|
if (written > 0) {
|
||||||
|
enterprises++;
|
||||||
|
weeks += written;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String message = "%d enterprises · %d weeks · %d skipped for no consent · %d already covered"
|
||||||
|
.formatted(enterprises, weeks, noConsent, existing);
|
||||||
|
log.info("UPI simulation — {}", message);
|
||||||
|
return new SimulationResult(enterprises, weeks, noConsent, existing, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One enterprise's series.
|
||||||
|
*
|
||||||
|
* <p>The level is anchored to the enterprise's own ledger inflow rather than
|
||||||
|
* to a constant, so UPI and the ledger tell the same story about the size of
|
||||||
|
* the business — an enterprise whose books show ₹40,000 a week and whose UPI
|
||||||
|
* shows ₹4,00,000 would make {@code emi_to_inflow} and
|
||||||
|
* {@code upi_inflow_change} disagree about the same firm.
|
||||||
|
*
|
||||||
|
* <p>The <em>trend</em> is what carries the signal, and it tracks the band:
|
||||||
|
* a RED enterprise's digital receipts are falling, which is the observation
|
||||||
|
* TW-02 exists to catch. Deterministic off the enterprise id, like every
|
||||||
|
* other seeded figure here, so a demo shows the same series twice.
|
||||||
|
*/
|
||||||
|
private int writeSeries(Enterprise e, LocalDate thisWeek, LocalDateTime observedAt) {
|
||||||
|
BigDecimal weeklyBase = weeklyLedgerInflow(e);
|
||||||
|
if (weeklyBase.signum() <= 0) return 0;
|
||||||
|
|
||||||
|
Random r = new Random((e.getEnterpriseId() + ":upi").hashCode());
|
||||||
|
|
||||||
|
// Total drift across the whole 26-week window.
|
||||||
|
double drift = switch (Objects.toString(e.getBand(), "STANDARD")) {
|
||||||
|
case "RED" -> -0.42 - r.nextDouble() * 0.16;
|
||||||
|
case "WATCH" -> -0.12 - r.nextDouble() * 0.10;
|
||||||
|
default -> 0.02 + r.nextDouble() * 0.10;
|
||||||
|
};
|
||||||
|
|
||||||
|
String consentRef = "SIM-CONSENT-" + e.getEnterpriseId();
|
||||||
|
int written = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < WEEKS; i++) {
|
||||||
|
LocalDate weekStart = thisWeek.minusWeeks(i);
|
||||||
|
// i = 0 is the newest week, so progress runs backwards: the oldest
|
||||||
|
// week sits at the start of the drift and the newest at the end.
|
||||||
|
double progress = (double) (WEEKS - 1 - i) / (WEEKS - 1);
|
||||||
|
// Deterioration starts partway in and accelerates, rather than
|
||||||
|
// sloping evenly from week one.
|
||||||
|
//
|
||||||
|
// Two reasons, and the first is arithmetic. TW-02 compares an
|
||||||
|
// eight-week window against the eight before it, which samples only
|
||||||
|
// about a third of a linear 26-week slope — a 40% total decline
|
||||||
|
// shows up as roughly 13% and never reaches the rule's −20%. The
|
||||||
|
// first version of this generator did exactly that: every RED
|
||||||
|
// enterprise was visibly declining and TW-02 still fired five times
|
||||||
|
// across the whole portfolio, all of them on noise.
|
||||||
|
//
|
||||||
|
// The second is that a linear slide is not what stress looks like.
|
||||||
|
// Receipts hold up while an enterprise draws on savings and
|
||||||
|
// goodwill, then fall away faster once those are gone. Starting the
|
||||||
|
// decline at 40% of the window and curving it makes the recent
|
||||||
|
// window genuinely worse than the prior one, which is the shape the
|
||||||
|
// rule was written to detect.
|
||||||
|
double onset = Math.max(0.0, (progress - 0.40) / 0.60);
|
||||||
|
double trend = 1.0 + drift * Math.pow(onset, 1.5);
|
||||||
|
// Week-to-week noise, deliberately smaller than the drift it sits on;
|
||||||
|
// a series whose noise swamps its trend would make TW-02 fire on
|
||||||
|
// arithmetic rather than on behaviour.
|
||||||
|
double noise = 0.92 + r.nextDouble() * 0.16;
|
||||||
|
|
||||||
|
BigDecimal inflow = weeklyBase
|
||||||
|
.multiply(BigDecimal.valueOf(Math.max(0.05, trend * noise)))
|
||||||
|
.setScale(2, RoundingMode.HALF_UP);
|
||||||
|
// Outflow trails inflow — a trading enterprise pays out most of what
|
||||||
|
// it takes in, and the gap is roughly its margin.
|
||||||
|
BigDecimal outflow = inflow
|
||||||
|
.multiply(BigDecimal.valueOf(0.70 + r.nextDouble() * 0.15))
|
||||||
|
.setScale(2, RoundingMode.HALF_UP);
|
||||||
|
|
||||||
|
int inflowCount = 12 + r.nextInt(40);
|
||||||
|
|
||||||
|
UpiSummary row = upiRepository
|
||||||
|
.findByEnterpriseIdAndWeekStart(e.getEnterpriseId(), weekStart)
|
||||||
|
.orElseGet(UpiSummary::new);
|
||||||
|
// Never overwrite a real row, even one reached through this path.
|
||||||
|
if (row.getId() != null && !SOURCE.equals(row.getSource())) continue;
|
||||||
|
|
||||||
|
row.setEnterpriseId(e.getEnterpriseId());
|
||||||
|
row.setWeekStart(weekStart);
|
||||||
|
row.setInflowAmount(inflow);
|
||||||
|
row.setOutflowAmount(outflow);
|
||||||
|
row.setInflowCount(inflowCount);
|
||||||
|
row.setOutflowCount(Math.max(1, inflowCount / 3));
|
||||||
|
row.setSource(SOURCE);
|
||||||
|
row.setConsentRef(consentRef);
|
||||||
|
// Weekly aggregates are known at the end of their week, not during
|
||||||
|
// it, so a run scored mid-week must not see this week's total as
|
||||||
|
// though it had already closed.
|
||||||
|
row.setValidForDate(weekStart.plusDays(6));
|
||||||
|
row.setObservedAt(observedAt);
|
||||||
|
upiRepository.save(row);
|
||||||
|
written++;
|
||||||
|
}
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The enterprise's own weekly money-in, from its ledger. Falls back to the
|
||||||
|
* instalment when there is no ledger to read — an enterprise still has to
|
||||||
|
* turn over more than its EMI for the loan to have been written at all.
|
||||||
|
*/
|
||||||
|
private BigDecimal weeklyLedgerInflow(Enterprise e) {
|
||||||
|
LocalDate from = LocalDate.now().minusWeeks(12);
|
||||||
|
List<LedgerEntry> entries = ledgerRepository
|
||||||
|
.findByEnterpriseIdAndEntryDateGreaterThanEqualOrderByEntryDateDesc(e.getEnterpriseId(), from);
|
||||||
|
|
||||||
|
BigDecimal inflow = entries.stream()
|
||||||
|
.filter(l -> "IN".equals(l.getKind()))
|
||||||
|
.map(LedgerEntry::getAmount)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.reduce(BigDecimal.ZERO, BigDecimal::add);
|
||||||
|
|
||||||
|
if (inflow.signum() > 0) {
|
||||||
|
return inflow.divide(BigDecimal.valueOf(12), 2, RoundingMode.HALF_UP);
|
||||||
|
}
|
||||||
|
BigDecimal instalment = e.getMonthlyInstalment();
|
||||||
|
if (instalment == null || instalment.signum() <= 0) return BigDecimal.ZERO;
|
||||||
|
// Roughly three times the instalment a month, spread over four weeks.
|
||||||
|
return instalment.multiply(BigDecimal.valueOf(3))
|
||||||
|
.divide(BigDecimal.valueOf(4), 2, RoundingMode.HALF_UP);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How much of the UPI table is simulated, for the honesty banner. */
|
||||||
|
public Coverage coverage() {
|
||||||
|
List<UpiSummary> all = upiRepository.findAll();
|
||||||
|
long simulated = all.stream().filter(u -> SOURCE.equals(u.getSource())).count();
|
||||||
|
long aa = all.stream().filter(u -> "ACCOUNT_AGGREGATOR".equals(u.getSource())).count();
|
||||||
|
return new Coverage(all.size(), simulated, aa, all.size() - simulated - aa, aa > 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
package com.lincfusion.udyamlinc.ingestion;
|
||||||
|
|
||||||
|
import com.lincfusion.udyamlinc.master.Commodity;
|
||||||
|
import com.lincfusion.udyamlinc.master.CommodityRepository;
|
||||||
|
import com.lincfusion.udyamlinc.master.District;
|
||||||
|
import com.lincfusion.udyamlinc.master.DistrictRepository;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Weekly price series for the commodities no API publishes, written with
|
||||||
|
* {@code source = SIMULATED}.
|
||||||
|
*
|
||||||
|
* <p>Milk, egg, green fodder and sugarcane have no usable feed and are not going
|
||||||
|
* to acquire one: milk comes from a dairy federation, egg from NECC's HTML
|
||||||
|
* pages, fodder trades in one market in Haryana, and cane never reaches a mandi
|
||||||
|
* because it is sold to mills at a set price. Manual entry exists in this system
|
||||||
|
* precisely because of them.
|
||||||
|
*
|
||||||
|
* <p>Their absence is not evenly distributed — it lands on two whole sector
|
||||||
|
* packs. {@code price_z_score}, {@code output_price_change} and
|
||||||
|
* {@code margin_squeeze} all read the sector's OUTPUT commodity, and dairy's
|
||||||
|
* output is milk while poultry's is egg. So a hundred and two enterprises had
|
||||||
|
* three features permanently null, and margin squeeze — the single derivation
|
||||||
|
* this project exists to compute — could not be evaluated for either pack. A
|
||||||
|
* poultry unit whose feed cost is known and whose egg price is unknowable is
|
||||||
|
* exactly the case the deck opens with.
|
||||||
|
*
|
||||||
|
* <p>Same discipline as {@code UpiSimulationService} and {@code
|
||||||
|
* LoanRecord.source}: these rows sit in the same table the real feed writes to,
|
||||||
|
* flow through the same derivations, and carry a marker that survives forever.
|
||||||
|
* The ingestion upsert keys on {@code (district, commodity, week, source)}, so a
|
||||||
|
* real MANUAL or AGMARKNET row for the same week coexists rather than colliding,
|
||||||
|
* and nothing here can overwrite one.
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class NoFeedPriceSimulationService {
|
||||||
|
|
||||||
|
public static final String SOURCE = "SIMULATED";
|
||||||
|
|
||||||
|
/** Three years, to match the Agmarknet backfill the seasonal mean is built from. */
|
||||||
|
private static final int WEEKS = 156;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Base price per unit and the annual seasonal swing, by commodity code.
|
||||||
|
* Levels are ordinary Indian farm-gate figures — a milk procurement rate
|
||||||
|
* around ₹38/litre, eggs around ₹5.60 each so ₹560 a hundred — because a
|
||||||
|
* simulated series that is plausible in level as well as in shape is the one
|
||||||
|
* an officer can sanity-check on sight.
|
||||||
|
*
|
||||||
|
* <p>{@code trendPerYear} is deliberately small. A steady annual climb makes
|
||||||
|
* the newest observation the highest the series has ever been, which
|
||||||
|
* {@code price_z_score} reads as a permanent anomaly — 5%/year compounding
|
||||||
|
* over the three-year window swamped a ±9% year-to-year shift and left
|
||||||
|
* simulated sectors averaging a z of 1.9 against real Agmarknet data's 0.44.
|
||||||
|
* Year-over-year variation has to dominate the trend, not the other way
|
||||||
|
* round.
|
||||||
|
*
|
||||||
|
* @param weeklyNoise fractional week-to-week variation, and the parameter that
|
||||||
|
* decides whether {@code price_z_score} means anything.
|
||||||
|
* <p>A z-score divides the deviation from a seasonal mean by
|
||||||
|
* that mean's standard deviation. On a series that trends
|
||||||
|
* steadily with almost no noise, the standard deviation is
|
||||||
|
* tiny and the newest value sits at the top of its own
|
||||||
|
* distribution <em>by construction</em> — so every enterprise
|
||||||
|
* scores 1.5 to 2.0 sigma and TW-05 fires on all of them. The
|
||||||
|
* first version of these indices used ±1.5% and fired TW-05
|
||||||
|
* for 197 of 263 enterprises, which is not an early warning,
|
||||||
|
* it is a light that is always on. Noise has to be large
|
||||||
|
* enough relative to the annual trend for "unusual" to keep
|
||||||
|
* its meaning.
|
||||||
|
*/
|
||||||
|
private record Profile(double base, double seasonalAmplitude, double trendPerYear,
|
||||||
|
int peakWeek, double weeklyNoise) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final Map<String, Profile> PROFILES = Map.of(
|
||||||
|
// Milk: procurement rate rises in the lean summer flush-off and eases
|
||||||
|
// after the monsoon, and drifts up a few percent a year.
|
||||||
|
//
|
||||||
|
// The level is anchored, not invented. data.gov.in resource
|
||||||
|
// 44724649-2fc1-40bb-85b8-8979832c8dd2 publishes state-wise
|
||||||
|
// cooperative procurement prices for June 2018: cow ₹25.12, mixed
|
||||||
|
// ₹30.52, buffalo ₹39.10 per litre. It is a two-year snapshot rather
|
||||||
|
// than a series, so it cannot be ingested — but it pins the level.
|
||||||
|
// ₹25.12 carried forward at roughly 5% a year to 2026 gives ~₹37, so
|
||||||
|
// 38 is a defensible cow-milk procurement rate with a citation
|
||||||
|
// behind it instead of a guess.
|
||||||
|
"MILK", new Profile(38.0, 0.08, 0.025, 22, 0.035),
|
||||||
|
// Egg: the sharpest seasonal of the four. Winter demand lifts the
|
||||||
|
// NECC rate hard and summer flattens it — the swing that makes a
|
||||||
|
// poultry margin squeeze real rather than theoretical.
|
||||||
|
"EGG", new Profile(560.0, 0.22, 0.02, 50, 0.055),
|
||||||
|
// Green fodder: scarce and dear in the dry months before the monsoon.
|
||||||
|
"FODDER", new Profile(310.0, 0.18, 0.025, 18, 0.06),
|
||||||
|
// Cane is an administered price, so it barely moves within a season
|
||||||
|
// and steps up between them.
|
||||||
|
// Genuinely near-flat, because it is an administered price. Its
|
||||||
|
// z-score is meaningless for the same reason a real administered
|
||||||
|
// price's would be, which is honest rather than a defect.
|
||||||
|
"SUGARCANE", new Profile(340.0, 0.03, 0.04, 1, 0.012),
|
||||||
|
|
||||||
|
// Index series, base 100. Deliberately gentler than the commodity
|
||||||
|
// profiles: a processed-food or retail price does not swing like a
|
||||||
|
// mandi rate, because processing and retail absorb input shocks into
|
||||||
|
// margin before passing them on. That is the whole point of the
|
||||||
|
// margin_squeeze these feed — an input spike against a sticky output
|
||||||
|
// is exactly what squeezes a processor, and giving the output the
|
||||||
|
// same volatility as the input would cancel the signal out.
|
||||||
|
// Noise here is set relative to the annual trend, not to taste. At
|
||||||
|
// ±1.5% against a 5%/year drift these produced z-scores of 1.5–2.0
|
||||||
|
// for almost every enterprise; at ±4–5% the newest observation is no
|
||||||
|
// longer an outlier merely for being the newest.
|
||||||
|
"PROCESSED_FOOD_INDEX", new Profile(100.0, 0.04, 0.015, 48, 0.042),
|
||||||
|
"RETAIL_BASKET_INDEX", new Profile(100.0, 0.03, 0.015, 44, 0.038),
|
||||||
|
"HANDLOOM_INDEX", new Profile(100.0, 0.06, 0.015, 42, 0.045),
|
||||||
|
// Yarn tracks cotton, so it is the most volatile of the four indices
|
||||||
|
// and peaks with the post-harvest cotton arrival window.
|
||||||
|
"YARN_INDEX", new Profile(100.0, 0.11, 0.015, 6, 0.065));
|
||||||
|
|
||||||
|
private final CommodityRepository commodityRepository;
|
||||||
|
private final DistrictRepository districtRepository;
|
||||||
|
private final MarketSignalRepository signalRepository;
|
||||||
|
|
||||||
|
public record SimulationResult(int commodities, int districts, int weeksWritten, String message) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public SimulationResult simulate() {
|
||||||
|
LocalDate thisWeek = Weeks.startOf(LocalDate.now());
|
||||||
|
LocalDateTime observedAt = LocalDateTime.now();
|
||||||
|
List<District> districts = districtRepository.findByActiveTrue();
|
||||||
|
|
||||||
|
int commodities = 0, written = 0;
|
||||||
|
|
||||||
|
for (Map.Entry<String, Profile> entry : PROFILES.entrySet()) {
|
||||||
|
Commodity commodity = commodityRepository.findByCode(entry.getKey()).orElse(null);
|
||||||
|
if (commodity == null) {
|
||||||
|
log.warn("No commodity {} to simulate prices for", entry.getKey());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
commodities++;
|
||||||
|
for (District district : districts) {
|
||||||
|
written += writeSeries(district, commodity, entry.getValue(), thisWeek, observedAt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String message = "%d commodities × %d districts · %d weeks written"
|
||||||
|
.formatted(commodities, districts.size(), written);
|
||||||
|
log.info("No-feed price simulation — {}", message);
|
||||||
|
return new SimulationResult(commodities, districts.size(), written, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One district's series for one commodity.
|
||||||
|
*
|
||||||
|
* <p>Level, seasonal swing and a slow annual trend, plus district-to-district
|
||||||
|
* variation drawn from the district code so Sehore and Kolar do not report an
|
||||||
|
* identical milk rate. Deterministic throughout: the same build produces the
|
||||||
|
* same series, which is what lets a demo be talked through twice.
|
||||||
|
*/
|
||||||
|
private int writeSeries(District district, Commodity commodity, Profile profile,
|
||||||
|
LocalDate thisWeek, LocalDateTime observedAt) {
|
||||||
|
Random r = new Random((district.getCode() + ":" + commodity.getCode()).hashCode());
|
||||||
|
// ±8% of level between districts, held constant for the whole series —
|
||||||
|
// a regional price difference, not weekly noise.
|
||||||
|
double regional = 0.92 + r.nextDouble() * 0.16;
|
||||||
|
int written = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < WEEKS; i++) {
|
||||||
|
LocalDate weekStart = thisWeek.minusWeeks(i);
|
||||||
|
int isoWeek = Weeks.isoWeek(weekStart);
|
||||||
|
|
||||||
|
// Seasonal term peaks at the profile's peak week and troughs opposite it.
|
||||||
|
double phase = 2 * Math.PI * (isoWeek - profile.peakWeek()) / 52.0;
|
||||||
|
double seasonal = 1.0 + profile.seasonalAmplitude() * Math.cos(phase);
|
||||||
|
// Trend runs forward in time, so the oldest week carries the least.
|
||||||
|
double years = (double) (WEEKS - 1 - i) / 52.0;
|
||||||
|
double trend = 1.0 + profile.trendPerYear() * years;
|
||||||
|
|
||||||
|
// A year-level shift on top of the trend, so the series has good
|
||||||
|
// years and bad ones instead of climbing forever.
|
||||||
|
//
|
||||||
|
// Without it the newest observation is the highest the series has
|
||||||
|
// ever been, every time, and price_z_score — which measures the
|
||||||
|
// latest price against its own seasonal history — reads that as a
|
||||||
|
// permanent anomaly. Real Agmarknet series in this same database
|
||||||
|
// average a z of 0.44; the first monotonic version of these
|
||||||
|
// generators averaged 1.9 and tripped TW-05 for half the portfolio.
|
||||||
|
// Year-to-year variation is what makes "unusually high" a statement
|
||||||
|
// about this year rather than about the direction of time.
|
||||||
|
int yearIndex = (WEEKS - 1 - i) / 52;
|
||||||
|
double yearShift = 1.0 + (new Random((district.getCode() + ":" + commodity.getCode()
|
||||||
|
+ ":y" + yearIndex).hashCode()).nextDouble() * 2 - 1) * 0.13;
|
||||||
|
double noise = 1.0 + (r.nextDouble() * 2 - 1) * profile.weeklyNoise();
|
||||||
|
|
||||||
|
BigDecimal modal = BigDecimal.valueOf(
|
||||||
|
profile.base() * regional * seasonal * trend * yearShift * noise)
|
||||||
|
.setScale(2, RoundingMode.HALF_UP);
|
||||||
|
|
||||||
|
MarketSignal signal = signalRepository
|
||||||
|
.findByDistrictIdAndCommodityIdAndWeekStartAndSource(
|
||||||
|
district.getId(), commodity.getId(), weekStart, SOURCE)
|
||||||
|
.orElseGet(MarketSignal::new);
|
||||||
|
|
||||||
|
signal.setDistrictId(district.getId());
|
||||||
|
signal.setCommodityId(commodity.getId());
|
||||||
|
signal.setWeekStart(weekStart);
|
||||||
|
signal.setModalPrice(modal);
|
||||||
|
signal.setMinPrice(modal.multiply(BigDecimal.valueOf(0.94)).setScale(2, RoundingMode.HALF_UP));
|
||||||
|
signal.setMaxPrice(modal.multiply(BigDecimal.valueOf(1.07)).setScale(2, RoundingMode.HALF_UP));
|
||||||
|
signal.setArrivalCount(null);
|
||||||
|
signal.setUnit(commodity.getUnit());
|
||||||
|
signal.setSource(SOURCE);
|
||||||
|
// A weekly aggregate is known once its week has closed.
|
||||||
|
signal.setValidForDate(weekStart.plusDays(6));
|
||||||
|
signal.setObservedAt(observedAt);
|
||||||
|
signal.setUpdatedAt(observedAt);
|
||||||
|
signalRepository.save(signal);
|
||||||
|
written++;
|
||||||
|
}
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user