From 5401496a64fa6c2b0dedccd7e3602cb672b466eb Mon Sep 17 00:00:00 2001 From: LincPay Date: Thu, 6 Aug 2026 14:12:45 +0530 Subject: [PATCH] feat(manual-entry): bulk and single entry for the feeds no API serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Egg, milk and fodder have no usable API for the pilot districts — NECC publishes HTML, dairy rates exist only as annual state averages, and every recent Agmarknet fodder row is one market in Ambala. This is how a bank gets that data in. Upload stages a PREVIEW batch and writes nothing. It reports what would land, what was rejected and why, and which rows would overwrite an existing value. Only commit touches market_signal. Effective dating: effective_date is the date the fact describes and comes from the file; observedAt is when we learned it and is stamped at commit. The uploader cannot reach observedAt — a March rate uploaded in August must stay invisible to a March scoring run, or the model trains on data that did not exist yet. Rejections name themselves: UNKNOWN_DISTRICT, UNKNOWN_COMMODITY, FUTURE_DATE, BAD_DATE, BAD_NUMBER, UNIT_MISMATCH, PRICE_ORDER, DUPLICATE_IN_FILE, MISSING_REQUIRED. A price far from last week warns but is accepted — a genuine price shock is the event this system exists to catch, and rejecting it would be backwards. Rows are written with source=MANUAL, which the unique index keeps separate from a feed's own row for the same week, so manual data never silently replaces a government feed. Also distinguishes INSUFFICIENT_HISTORY from NO_SIGNAL in the price derivations. A newly onboarded manual feed spends its first weeks below the four-week minimum, and reporting NO_SIGNAL there reads as a broken integration rather than a series still filling up. Result: margin_squeeze and output_price_change compute for the first time — 57 of 142 enterprises, exactly the DAIRY (30) and POULTRY (27) sectors that have an output commodity. Co-Authored-By: Claude Opus 5 (1M context) --- pom.xml | 8 + .../features/FeatureEngineeringService.java | 11 +- .../ingestion/MarketSignalRepository.java | 7 + .../manualentry/ManualEntryBatch.java | 72 ++++ .../ManualEntryBatchRepository.java | 13 + .../manualentry/ManualEntryController.java | 132 +++++++ .../manualentry/ManualEntryLine.java | 102 +++++ .../ManualEntryLineRepository.java | 14 + .../manualentry/ManualEntryPreview.java | 30 ++ .../manualentry/ManualMarketEntryService.java | 368 ++++++++++++++++++ .../udyamlinc/manualentry/TabularParser.java | 187 +++++++++ 11 files changed, 943 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatch.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatchRepository.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLineRepository.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryPreview.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/ManualMarketEntryService.java create mode 100644 src/main/java/com/lincfusion/udyamlinc/manualentry/TabularParser.java diff --git a/pom.xml b/pom.xml index c726a1c..729fcbd 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,14 @@ provided + + + org.apache.poi + poi-ooxml + 5.3.0 + + org.springframework.boot diff --git a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java index 4fad867..f81a388 100644 --- a/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java +++ b/src/main/java/com/lincfusion/udyamlinc/features/FeatureEngineeringService.java @@ -139,10 +139,12 @@ public class FeatureEngineeringService { BigDecimal weightedSum = BigDecimal.ZERO; int totalWeight = 0; boolean anyHistory = false; + boolean anyRows = false; for (SectorCommodity link : commodities) { List series = marketSignalRepository.findTrailing( districtId, link.getCommodityId(), asOfDate, observedAsOf); + if (!series.isEmpty()) anyRows = true; BigDecimal change = changeVsTrailingMean(series); if (change == null) continue; anyHistory = true; @@ -152,7 +154,14 @@ public class FeatureEngineeringService { } if (!anyHistory || totalWeight == 0) { - return Computed.missing(FeatureCatalog.NO_SIGNAL); + // "Nobody has ever sent us a price" and "the series has started but is + // still too short to mean anything" are different facts, and only the + // second one gets better on its own. A newly onboarded manual feed + // reporting NO_SIGNAL reads as a broken integration for the weeks it + // spends filling up. + return Computed.missing(anyRows + ? FeatureCatalog.INSUFFICIENT_HISTORY + : FeatureCatalog.NO_SIGNAL); } return Computed.of(weightedSum.divide(BigDecimal.valueOf(totalWeight), 4, RoundingMode.HALF_UP), staleness); } diff --git a/src/main/java/com/lincfusion/udyamlinc/ingestion/MarketSignalRepository.java b/src/main/java/com/lincfusion/udyamlinc/ingestion/MarketSignalRepository.java index 2a7aea7..aebf4da 100644 --- a/src/main/java/com/lincfusion/udyamlinc/ingestion/MarketSignalRepository.java +++ b/src/main/java/com/lincfusion/udyamlinc/ingestion/MarketSignalRepository.java @@ -35,4 +35,11 @@ public interface MarketSignalRepository extends JpaRepository findFirstByDistrictIdOrderByValidForDateDesc(Long districtId); + + /** + * The last value before a given week, from any source — what a manually + * entered price is sanity-checked against. + */ + Optional findFirstByDistrictIdAndCommodityIdAndWeekStartLessThanOrderByWeekStartDesc( + Long districtId, Long commodityId, LocalDate weekStart); } diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatch.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatch.java new file mode 100644 index 0000000..921d4d7 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatch.java @@ -0,0 +1,72 @@ +package com.lincfusion.udyamlinc.manualentry; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.LocalDateTime; + +/** + * One upload. The audit anchor for every manually-sourced number in the system: + * a supervisor asking "where did Bhopal's March milk rate come from" gets a + * filename, a person and a timestamp, not a shrug. + * + *

A batch is staged as {@code PREVIEW} and writes nothing. Only + * {@code COMMITTED} has touched {@code market_signal}. Uploading four hundred + * rows and discovering the district column was misspelled afterwards is not a + * recoverable situation, so nothing lands until someone has seen what will. + */ +@Entity +@Table(name = "manual_entry_batch", indexes = { + @Index(name = "ix_manual_batch_status", columnList = "status"), + @Index(name = "ix_manual_batch_uploaded", columnList = "uploadedAt"), +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ManualEntryBatch { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** MARKET_SIGNAL | LOAN_RECORD — which template was uploaded. */ + @Column(nullable = false, length = 32) + private String entryType; + + @Column(length = 260) + private String filename; + + /** The username from the JWT. Never a display name — this is an audit field. */ + @Column(nullable = false, length = 64) + private String uploadedBy; + + @Column(nullable = false) + private LocalDateTime uploadedAt; + + /** PREVIEW | COMMITTED | DISCARDED. */ + @Column(nullable = false, length = 16) + private String status; + + private LocalDateTime committedAt; + + @Column(length = 64) + private String committedBy; + + private Integer rowsParsed; + private Integer rowsAccepted; + private Integer rowsRejected; + + /** How many accepted rows would replace a value that already exists. */ + private Integer rowsOverwriting; + + @Column(length = 500) + private String note; + + @PrePersist + void onCreate() { + if (uploadedAt == null) uploadedAt = LocalDateTime.now(); + if (status == null) status = "PREVIEW"; + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatchRepository.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatchRepository.java new file mode 100644 index 0000000..6706696 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryBatchRepository.java @@ -0,0 +1,13 @@ +package com.lincfusion.udyamlinc.manualentry; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface ManualEntryBatchRepository extends JpaRepository { + + List findAllByOrderByUploadedAtDesc(Pageable pageable); + + List findByEntryTypeOrderByUploadedAtDesc(String entryType, Pageable pageable); +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java new file mode 100644 index 0000000..12ad7a8 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryController.java @@ -0,0 +1,132 @@ +package com.lincfusion.udyamlinc.manualentry; + +import com.lincfusion.udyamlinc.common.ApiResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +/** + * Manual and bulk entry of the signals no API serves. + * + *

Mounted under {@code /admin}, so the existing rule applies unchanged: any + * officer may read the batch history, only a supervisor may upload or commit. + * That is deliberate — the audit trail is worth more if the people who can read + * it outnumber the people who can write to it. + */ +@RestController +@RequestMapping("/admin/manual") +@RequiredArgsConstructor +public class ManualEntryController { + + private final ManualMarketEntryService marketEntryService; + private final ManualEntryBatchRepository batchRepository; + private final ManualEntryLineRepository lineRepository; + + // ── template ──────────────────────────────────────────── + + /** + * The column names the parser accepts, with one filled-in example row. + * Served as CSV because every spreadsheet opens it and nothing has to be + * generated. + */ + @GetMapping(value = "/template/market", produces = "text/csv") + public ResponseEntity marketTemplate() { + String csv = """ + district_code,commodity_code,effective_date,modal_price,min_price,max_price,unit,note + BHOPAL,EGG,2026-08-05,610,600,620,HUNDRED,NECC Bhopal zone rate + SEHORE,MILK,2026-07-31,38.50,,,LITRE,Sanchi procurement rate for July + """; + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"udyamlinc-market-template.csv\"") + .contentType(MediaType.parseMediaType("text/csv")) + .body(csv.getBytes(StandardCharsets.UTF_8)); + } + + // ── upload → preview → commit ─────────────────────────── + + /** Stages the file and reports what it would do. Writes no market signal. */ + @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ApiResponse> upload(@RequestParam("file") MultipartFile file, + Authentication authentication) throws IOException { + if (file == null || file.isEmpty()) { + return ApiResponse.fail("No file uploaded"); + } + ManualEntryPreview preview = marketEntryService.upload(file, username(authentication)); + return ApiResponse.ok( + "%d parsed · %d accepted · %d rejected · %d would overwrite".formatted( + preview.lines().size(), preview.accepted().size(), + preview.rejected().size(), preview.overwriting().size()), + describe(preview)); + } + + @PostMapping("/{batchId}/commit") + public ApiResponse> commit(@PathVariable Long batchId, Authentication authentication) { + ManualEntryPreview committed = marketEntryService.commit(batchId, username(authentication)); + return ApiResponse.ok( + "%d market signals written".formatted(committed.accepted().size()), + describe(committed)); + } + + @DeleteMapping("/{batchId}") + public ApiResponse discard(@PathVariable Long batchId, Authentication authentication) { + return ApiResponse.ok("Batch discarded", marketEntryService.discard(batchId, username(authentication))); + } + + /** One row, validated and committed in a single call. For the monthly milk rate. */ + @PostMapping("/entry") + public ApiResponse> enterOne(@RequestBody Map row, + Authentication authentication) { + ManualEntryPreview result = marketEntryService.enterOne(row, username(authentication)); + return result.accepted().isEmpty() + ? ApiResponse.ok("Rejected — see the line for the reason", describe(result)) + : ApiResponse.ok("Entry recorded", describe(result)); + } + + // ── history ───────────────────────────────────────────── + + @GetMapping("/batches") + public ApiResponse> batches(@RequestParam(required = false) String entryType, + @RequestParam(defaultValue = "30") int limit) { + PageRequest page = PageRequest.of(0, Math.min(limit, 200)); + return ApiResponse.ok(entryType == null + ? batchRepository.findAllByOrderByUploadedAtDesc(page) + : batchRepository.findByEntryTypeOrderByUploadedAtDesc(entryType, page)); + } + + @GetMapping("/batches/{batchId}") + public ApiResponse> batch(@PathVariable Long batchId) { + ManualEntryBatch batch = batchRepository.findById(batchId) + .orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId)); + return ApiResponse.ok(describe( + new ManualEntryPreview(batch, lineRepository.findByBatchIdOrderByRowNumber(batchId)))); + } + + // ── shaping ───────────────────────────────────────────── + + /** + * Accepted, rejected and overwriting stay separate all the way to the UI — + * see {@link ManualEntryPreview}. + */ + private Map describe(ManualEntryPreview preview) { + return Map.of( + "batch", preview.batch(), + "accepted", preview.accepted(), + "rejected", preview.rejected(), + "overwriting", preview.overwriting(), + "warnings", preview.warnings()); + } + + private String username(Authentication authentication) { + return authentication == null ? "unknown" : authentication.getName(); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java new file mode 100644 index 0000000..18c4168 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLine.java @@ -0,0 +1,102 @@ +package com.lincfusion.udyamlinc.manualentry; + +import jakarta.persistence.*; +import lombok.*; + +import java.math.BigDecimal; +import java.time.LocalDate; + +/** + * One row of an upload, kept whether or not it was accepted. + * + *

A rejected row is data too: "we uploaded four hundred rows and three + * hundred silently vanished" is the failure this table exists to prevent. Each + * rejection carries the reason that produced it. + * + *

{@code priorModalPrice} records what an accepted row displaced, because a + * correction is only auditable if the thing it corrected is still legible. + */ +@Entity +@Table(name = "manual_entry_line", indexes = { + @Index(name = "ix_manual_line_batch", columnList = "batchId"), +}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ManualEntryLine { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private Long batchId; + + /** 1-based row number in the uploaded file, so a reject points somewhere. */ + @Column(nullable = false) + private Integer rowNumber; + + // ── as supplied, before resolution ────────────────────── + + @Column(length = 64) + private String rawDistrict; + + @Column(length = 64) + private String rawCommodity; + + @Column(length = 32) + private String rawEffectiveDate; + + @Column(length = 32) + private String rawModalPrice; + + // ── as resolved ───────────────────────────────────────── + + private Long districtId; + private Long commodityId; + + /** The date the fact describes, as entered. Never the upload time. */ + private LocalDate effectiveDate; + + /** Monday of the ISO week {@link #effectiveDate} falls in. */ + private LocalDate weekStart; + + @Column(precision = 12, scale = 2) + private BigDecimal modalPrice; + + @Column(precision = 12, scale = 2) + private BigDecimal minPrice; + + @Column(precision = 12, scale = 2) + private BigDecimal maxPrice; + + @Column(length = 16) + private String unit; + + // ── verdict ───────────────────────────────────────────── + + @Column(nullable = false) + private boolean accepted; + + /** + * UNKNOWN_DISTRICT | UNKNOWN_COMMODITY | FUTURE_DATE | BAD_DATE | BAD_NUMBER + * | UNIT_MISMATCH | PRICE_ORDER | DUPLICATE_IN_FILE | MISSING_REQUIRED + */ + @Column(length = 32) + private String rejectReason; + + @Column(length = 300) + private String rejectDetail; + + /** Set when the row is accepted but sits outside the sanity band. Not a rejection. */ + @Column(length = 300) + private String warning; + + /** True when committing this row replaces an existing manual value. */ + private boolean overwriting; + + @Column(precision = 12, scale = 2) + private BigDecimal priorModalPrice; +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLineRepository.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLineRepository.java new file mode 100644 index 0000000..1d1d598 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryLineRepository.java @@ -0,0 +1,14 @@ +package com.lincfusion.udyamlinc.manualentry; + +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface ManualEntryLineRepository extends JpaRepository { + + List findByBatchIdOrderByRowNumber(Long batchId); + + List findByBatchIdAndAcceptedOrderByRowNumber(Long batchId, boolean accepted); + + void deleteByBatchId(Long batchId); +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryPreview.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryPreview.java new file mode 100644 index 0000000..c3adb51 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualEntryPreview.java @@ -0,0 +1,30 @@ +package com.lincfusion.udyamlinc.manualentry; + +import java.util.List; + +/** + * What an upload would do, before it does it. + * + *

The three groups are kept apart deliberately. A console that merges them + * into one list of "rows" hides the only two questions the person clicking + * Commit actually has: what will not land, and what will land on top of + * something that is already there. + */ +public record ManualEntryPreview(ManualEntryBatch batch, List lines) { + + public List accepted() { + return lines.stream().filter(ManualEntryLine::isAccepted).toList(); + } + + public List rejected() { + return lines.stream().filter(line -> !line.isAccepted()).toList(); + } + + public List overwriting() { + return lines.stream().filter(line -> line.isAccepted() && line.isOverwriting()).toList(); + } + + public List warnings() { + return lines.stream().filter(line -> line.isAccepted() && line.getWarning() != null).toList(); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualMarketEntryService.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualMarketEntryService.java new file mode 100644 index 0000000..50aa851 --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/ManualMarketEntryService.java @@ -0,0 +1,368 @@ +package com.lincfusion.udyamlinc.manualentry; + +import com.lincfusion.udyamlinc.ingestion.MarketSignal; +import com.lincfusion.udyamlinc.ingestion.MarketSignalRepository; +import com.lincfusion.udyamlinc.ingestion.Weeks; +import com.lincfusion.udyamlinc.master.Commodity; +import com.lincfusion.udyamlinc.master.CommodityRepository; +import com.lincfusion.udyamlinc.master.District; +import com.lincfusion.udyamlinc.master.DistrictRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; + +/** + * Manual and bulk entry of market signals, for the commodities no API serves: + * NECC egg rates, dairy-federation milk procurement, green fodder. + * + *

Two dates, and only one of them is the uploader's. + * {@code effective_date} is the date the fact describes and is entered. + * {@code observedAt} is when we learned it, and is always the commit time — a + * bank uploading March's milk rate in August has told us something in August. + * Setting {@code observedAt} from the file would let a backdated upload appear + * inside a March scoring run, which is how a model learns from data that did not + * exist yet. The uploader cannot reach that field. + * + *

Nothing is written until it has been seen. An upload is staged as a + * PREVIEW batch and returns what would land, what was rejected and why, and + * which rows would overwrite an existing value. Only {@code commit} touches + * {@code market_signal}. + * + *

Rows are written with {@code source = MANUAL}, which the unique index on + * {@code (district, commodity, week, source)} keeps separate from a feed's own + * row for the same week. Manual data never silently replaces a government feed. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class ManualMarketEntryService { + + public static final String SOURCE = "MANUAL"; + public static final String ENTRY_TYPE = "MARKET_SIGNAL"; + + private static final List DATE_FORMATS = List.of( + DateTimeFormatter.ISO_LOCAL_DATE, + DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("d/M/yyyy"), + DateTimeFormatter.ofPattern("dd-MM-yyyy"), + DateTimeFormatter.ofPattern("dd/MM/yy")); + + private final TabularParser parser; + private final ManualEntryBatchRepository batchRepository; + private final ManualEntryLineRepository lineRepository; + private final MarketSignalRepository signalRepository; + private final DistrictRepository districtRepository; + private final CommodityRepository commodityRepository; + + /** + * How far a price may sit from the last known value before the row is + * flagged. A warning, never a rejection: a genuine price shock is the event + * this whole system exists to catch, and rejecting it would be precisely + * backwards. + */ + @Value("${udyam.manual.outlier-ratio:3.0}") + private double outlierRatio; + + // ── upload → preview ──────────────────────────────────── + + @Transactional + public ManualEntryPreview upload(MultipartFile file, String username) throws IOException { + TabularParser.Sheet sheet = parser.parse(file); + + ManualEntryBatch batch = batchRepository.save(ManualEntryBatch.builder() + .entryType(ENTRY_TYPE) + .filename(file.getOriginalFilename()) + .uploadedBy(username) + .status("PREVIEW") + .rowsParsed(sheet.rows().size()) + .build()); + + Map districts = new HashMap<>(); + districtRepository.findByActiveTrue().forEach(d -> districts.put(key(d.getCode()), d)); + Map commodities = new HashMap<>(); + commodityRepository.findAll().forEach(c -> commodities.put(key(c.getCode()), c)); + + Set seenInFile = new HashSet<>(); + List lines = new ArrayList<>(); + + int rowNumber = 1; + for (Map row : sheet.rows()) { + lines.add(validate(row, ++rowNumber, batch.getId(), districts, commodities, seenInFile)); + } + + lineRepository.saveAll(lines); + + int accepted = (int) lines.stream().filter(ManualEntryLine::isAccepted).count(); + int overwriting = (int) lines.stream().filter(l -> l.isAccepted() && l.isOverwriting()).count(); + batch.setRowsAccepted(accepted); + batch.setRowsRejected(lines.size() - accepted); + batch.setRowsOverwriting(overwriting); + batchRepository.save(batch); + + log.info("Manual upload {} staged — {} parsed, {} accepted, {} rejected, {} overwriting", + batch.getId(), lines.size(), accepted, lines.size() - accepted, overwriting); + + return new ManualEntryPreview(batch, lines); + } + + // ── validation ────────────────────────────────────────── + + private ManualEntryLine validate(Map row, int rowNumber, Long batchId, + Map districts, Map commodities, + Set seenInFile) { + + String rawDistrict = TabularParser.get(row, "district_code", "district"); + String rawCommodity = TabularParser.get(row, "commodity_code", "commodity"); + String rawDate = TabularParser.get(row, "effective_date", "date", "as_of_date"); + String rawModal = TabularParser.get(row, "modal_price", "price", "rate"); + + ManualEntryLine line = ManualEntryLine.builder() + .batchId(batchId) + .rowNumber(rowNumber) + .rawDistrict(rawDistrict) + .rawCommodity(rawCommodity) + .rawEffectiveDate(rawDate) + .rawModalPrice(rawModal) + .accepted(false) + .build(); + + if (rawDistrict == null || rawCommodity == null || rawDate == null || rawModal == null) { + return reject(line, "MISSING_REQUIRED", + "district_code, commodity_code, effective_date and modal_price are all required"); + } + + District district = districts.get(key(rawDistrict)); + if (district == null) { + return reject(line, "UNKNOWN_DISTRICT", "no active district with code '" + rawDistrict + "'"); + } + Commodity commodity = commodities.get(key(rawCommodity)); + if (commodity == null) { + return reject(line, "UNKNOWN_COMMODITY", "no commodity with code '" + rawCommodity + "'"); + } + + LocalDate effective = parseDate(rawDate); + if (effective == null) { + return reject(line, "BAD_DATE", "could not read '" + rawDate + "' as a date"); + } + // A price for a date that has not happened is a forecast. This table holds + // observations, and the difference is the system's whole premise. + if (effective.isAfter(LocalDate.now())) { + return reject(line, "FUTURE_DATE", effective + " is in the future"); + } + + BigDecimal modal = parseNumber(rawModal); + if (modal == null) { + return reject(line, "BAD_NUMBER", "could not read '" + rawModal + "' as a price"); + } + BigDecimal min = parseNumber(TabularParser.get(row, "min_price", "minimum_price")); + BigDecimal max = parseNumber(TabularParser.get(row, "max_price", "maximum_price")); + if (min != null && max != null && min.compareTo(max) > 0) { + return reject(line, "PRICE_ORDER", "min_price " + min + " exceeds max_price " + max); + } + if (min != null && modal.compareTo(min) < 0 || max != null && modal.compareTo(max) > 0) { + return reject(line, "PRICE_ORDER", "modal_price " + modal + " sits outside min/max"); + } + + String unit = TabularParser.get(row, "unit"); + if (unit != null && commodity.getUnit() != null + && !unit.trim().equalsIgnoreCase(commodity.getUnit())) { + // ₹/litre landing in a ₹/quintal column is not a small error — it is a + // hundredfold one, and every derived feature would inherit it. + return reject(line, "UNIT_MISMATCH", + commodity.getCode() + " is recorded in " + commodity.getUnit() + ", not " + unit); + } + + LocalDate weekStart = Weeks.startOf(effective); + String fileKey = district.getId() + ":" + commodity.getId() + ":" + weekStart; + if (!seenInFile.add(fileKey)) { + return reject(line, "DUPLICATE_IN_FILE", + "another row already covers " + commodity.getCode() + " / " + district.getCode() + + " for the week of " + weekStart); + } + + line.setDistrictId(district.getId()); + line.setCommodityId(commodity.getId()); + line.setEffectiveDate(effective); + line.setWeekStart(weekStart); + line.setModalPrice(modal); + line.setMinPrice(min); + line.setMaxPrice(max); + line.setUnit(commodity.getUnit()); + line.setAccepted(true); + + signalRepository + .findByDistrictIdAndCommodityIdAndWeekStartAndSource( + district.getId(), commodity.getId(), weekStart, SOURCE) + .ifPresent(existing -> { + line.setOverwriting(true); + line.setPriorModalPrice(existing.getModalPrice()); + }); + + flagOutlier(line, district, commodity, modal, weekStart); + return line; + } + + /** + * Compares against the most recent earlier value from any source, so a + * manual egg rate is sanity-checked against the last manual egg rate rather + * than against nothing at all. + */ + private void flagOutlier(ManualEntryLine line, District district, Commodity commodity, + BigDecimal modal, LocalDate weekStart) { + signalRepository + .findFirstByDistrictIdAndCommodityIdAndWeekStartLessThanOrderByWeekStartDesc( + district.getId(), commodity.getId(), weekStart) + .ifPresent(previous -> { + BigDecimal before = previous.getModalPrice(); + if (before == null || before.signum() == 0) return; + double ratio = modal.doubleValue() / before.doubleValue(); + if (ratio > outlierRatio || ratio < 1 / outlierRatio) { + line.setWarning("modal_price " + modal + " is " + Math.round(ratio * 100) + + "% of the previous week's " + before + " — accepted, but check the unit"); + } + }); + } + + private ManualEntryLine reject(ManualEntryLine line, String reason, String detail) { + line.setAccepted(false); + line.setRejectReason(reason); + line.setRejectDetail(detail); + return line; + } + + // ── commit ────────────────────────────────────────────── + + /** + * Writes the accepted rows. {@code observedAt} is stamped here, once, for the + * whole batch — the moment the institution accepted the data. + */ + @Transactional + public ManualEntryPreview commit(Long batchId, String username) { + ManualEntryBatch batch = batchRepository.findById(batchId) + .orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId)); + if (!"PREVIEW".equals(batch.getStatus())) { + throw new IllegalStateException("Batch " + batchId + " is " + batch.getStatus() + + ", and only a PREVIEW batch can be committed"); + } + + List accepted = lineRepository.findByBatchIdAndAcceptedOrderByRowNumber(batchId, true); + LocalDateTime observedAt = LocalDateTime.now(); + + for (ManualEntryLine line : accepted) { + MarketSignal signal = signalRepository + .findByDistrictIdAndCommodityIdAndWeekStartAndSource( + line.getDistrictId(), line.getCommodityId(), line.getWeekStart(), SOURCE) + .orElseGet(() -> MarketSignal.builder() + .districtId(line.getDistrictId()) + .commodityId(line.getCommodityId()) + .weekStart(line.getWeekStart()) + .source(SOURCE) + .build()); + + signal.setModalPrice(line.getModalPrice()); + signal.setMinPrice(line.getMinPrice()); + signal.setMaxPrice(line.getMaxPrice()); + signal.setUnit(line.getUnit()); + // One hand-entered observation for the week, not an aggregate of daily + // arrivals — the count says so rather than implying a depth it lacks. + signal.setArrivalCount(1); + signal.setValidForDate(line.getEffectiveDate()); + signal.setObservedAt(observedAt); + signalRepository.save(signal); + } + + batch.setStatus("COMMITTED"); + batch.setCommittedAt(observedAt); + batch.setCommittedBy(username); + batchRepository.save(batch); + + log.info("Manual batch {} committed by {} — {} market signals written", + batchId, username, accepted.size()); + + return new ManualEntryPreview(batch, lineRepository.findByBatchIdOrderByRowNumber(batchId)); + } + + @Transactional + public ManualEntryBatch discard(Long batchId, String username) { + ManualEntryBatch batch = batchRepository.findById(batchId) + .orElseThrow(() -> new IllegalArgumentException("No such batch: " + batchId)); + if ("COMMITTED".equals(batch.getStatus())) { + throw new IllegalStateException("Batch " + batchId + " is already committed and cannot be discarded"); + } + batch.setStatus("DISCARDED"); + batch.setCommittedBy(username); + batchRepository.save(batch); + return batch; + } + + // ── single-row entry ──────────────────────────────────── + + /** + * The monthly milk rate does not deserve a spreadsheet. Same validation, same + * audit trail — a one-row batch, committed in the same call. + */ + @Transactional + public ManualEntryPreview enterOne(Map row, String username) { + ManualEntryBatch batch = batchRepository.save(ManualEntryBatch.builder() + .entryType(ENTRY_TYPE) + .filename("(single entry)") + .uploadedBy(username) + .status("PREVIEW") + .rowsParsed(1) + .build()); + + Map districts = new HashMap<>(); + districtRepository.findByActiveTrue().forEach(d -> districts.put(key(d.getCode()), d)); + Map commodities = new HashMap<>(); + commodityRepository.findAll().forEach(c -> commodities.put(key(c.getCode()), c)); + + ManualEntryLine line = validate(row, 1, batch.getId(), districts, commodities, new HashSet<>()); + lineRepository.save(line); + + batch.setRowsAccepted(line.isAccepted() ? 1 : 0); + batch.setRowsRejected(line.isAccepted() ? 0 : 1); + batch.setRowsOverwriting(line.isOverwriting() ? 1 : 0); + batchRepository.save(batch); + + if (!line.isAccepted()) { + return new ManualEntryPreview(batch, List.of(line)); + } + return commit(batch.getId(), username); + } + + // ── helpers ───────────────────────────────────────────── + + private LocalDate parseDate(String raw) { + for (DateTimeFormatter format : DATE_FORMATS) { + try { + return LocalDate.parse(raw.trim(), format); + } catch (Exception ignored) { + // try the next shape + } + } + return null; + } + + private BigDecimal parseNumber(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return new BigDecimal(raw.replace(",", "").replace("₹", "").trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private String key(String code) { + return code == null ? "" : code.trim().toUpperCase(Locale.ROOT); + } +} diff --git a/src/main/java/com/lincfusion/udyamlinc/manualentry/TabularParser.java b/src/main/java/com/lincfusion/udyamlinc/manualentry/TabularParser.java new file mode 100644 index 0000000..6fea97f --- /dev/null +++ b/src/main/java/com/lincfusion/udyamlinc/manualentry/TabularParser.java @@ -0,0 +1,187 @@ +package com.lincfusion.udyamlinc.manualentry; + +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.*; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Reads an uploaded .xlsx or .csv into rows keyed by column header. + * + *

Header driven rather than positional, because a bank exporting from its own + * system reorders columns and inserts its own. Lookup is case-, space- and + * underscore-insensitive for the same reason {@code DataGovClient} reads that + * way: {@code Effective Date}, {@code effective_date} and {@code EFFECTIVEDATE} + * are the same intent, and rejecting two of them teaches the uploader nothing. + * + *

Every value is returned as trimmed text. Nothing is coerced here — parsing + * a date or a price is a validation decision, and validation owns the reason a + * bad one is rejected. + */ +@Component +@Slf4j +public class TabularParser { + + /** A parsed file: the header names as supplied, and one map per data row. */ + public record Sheet(List headers, List> rows) { + } + + public Sheet parse(MultipartFile file) throws IOException { + String name = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase(Locale.ROOT); + return name.endsWith(".csv") ? parseCsv(file) : parseWorkbook(file); + } + + // ── xlsx ──────────────────────────────────────────────── + + private Sheet parseWorkbook(MultipartFile file) throws IOException { + try (Workbook workbook = WorkbookFactory.create(file.getInputStream())) { + org.apache.poi.ss.usermodel.Sheet sheet = workbook.getSheetAt(0); + if (sheet == null) return new Sheet(List.of(), List.of()); + + Row headerRow = sheet.getRow(sheet.getFirstRowNum()); + if (headerRow == null) return new Sheet(List.of(), List.of()); + + List headers = new ArrayList<>(); + for (int c = 0; c < headerRow.getLastCellNum(); c++) { + headers.add(text(headerRow.getCell(c))); + } + + DataFormatter formatter = new DataFormatter(); + List> rows = new ArrayList<>(); + for (int r = headerRow.getRowNum() + 1; r <= sheet.getLastRowNum(); r++) { + Row row = sheet.getRow(r); + if (row == null) continue; + + Map values = new LinkedHashMap<>(); + boolean anyValue = false; + for (int c = 0; c < headers.size(); c++) { + String header = headers.get(c); + if (header == null || header.isBlank()) continue; + String value = cellText(row.getCell(c), formatter); + values.put(header, value); + if (value != null && !value.isBlank()) anyValue = true; + } + // A row of empty cells is spreadsheet residue, not a rejected row — + // reporting it would bury the rejections that matter. + if (anyValue) rows.add(values); + } + return new Sheet(headers, rows); + } + } + + private String cellText(Cell cell, DataFormatter formatter) { + if (cell == null) return null; + // A date cell must not become "45231" — the effective date is the whole + // point of the upload, and a serial number would parse as a bad date + // rather than the date the uploader typed. + if (cell.getCellType() == CellType.NUMERIC && DateUtil.isCellDateFormatted(cell)) { + return cell.getLocalDateTimeCellValue().toLocalDate().toString(); + } + if (cell.getCellType() == CellType.NUMERIC) { + double d = cell.getNumericCellValue(); + return d == Math.floor(d) && !Double.isInfinite(d) + ? String.valueOf((long) d) + : BigDecimal.valueOf(d).stripTrailingZeros().toPlainString(); + } + String value = formatter.formatCellValue(cell); + return value == null ? null : value.trim(); + } + + private String text(Cell cell) { + if (cell == null) return null; + String value = new DataFormatter().formatCellValue(cell); + return value == null ? null : value.trim(); + } + + // ── csv ───────────────────────────────────────────────── + + private Sheet parseCsv(MultipartFile file) throws IOException { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8))) { + + String headerLine = reader.readLine(); + if (headerLine == null) return new Sheet(List.of(), List.of()); + // A UTF-8 BOM would otherwise ride along on the first header name and + // stop it matching anything. + if (!headerLine.isEmpty() && headerLine.charAt(0) == '') { + headerLine = headerLine.substring(1); + } + + List headers = splitCsv(headerLine); + List> rows = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + if (line.isBlank()) continue; + List cells = splitCsv(line); + Map values = new LinkedHashMap<>(); + for (int i = 0; i < headers.size(); i++) { + String header = headers.get(i); + if (header == null || header.isBlank()) continue; + values.put(header, i < cells.size() ? cells.get(i) : null); + } + rows.add(values); + } + return new Sheet(headers, rows); + } + } + + /** Minimal RFC-4180 split: quoted fields, doubled quotes, commas inside quotes. */ + private List splitCsv(String line) { + List out = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean inQuotes = false; + for (int i = 0; i < line.length(); i++) { + char ch = line.charAt(i); + if (inQuotes) { + if (ch == '"') { + if (i + 1 < line.length() && line.charAt(i + 1) == '"') { + current.append('"'); + i++; + } else { + inQuotes = false; + } + } else { + current.append(ch); + } + } else if (ch == '"') { + inQuotes = true; + } else if (ch == ',') { + out.add(current.toString().trim()); + current.setLength(0); + } else { + current.append(ch); + } + } + out.add(current.toString().trim()); + return out; + } + + // ── header lookup ─────────────────────────────────────── + + /** + * Reads a column by any of its accepted names. Returns null for a blank cell + * so "absent" and "empty" reach validation as the same thing. + */ + public static String get(Map row, String... names) { + for (String name : names) { + for (Map.Entry entry : row.entrySet()) { + if (entry.getKey() != null && normalise(entry.getKey()).equals(normalise(name))) { + String value = entry.getValue(); + return value == null || value.isBlank() ? null : value.trim(); + } + } + } + return null; + } + + private static String normalise(String s) { + return s.replace("_", "").replace(" ", "").replace("-", "").toLowerCase(Locale.ROOT); + } +}