diff --git a/pom.xml b/pom.xml index 990e3672..59ea5dce 100644 --- a/pom.xml +++ b/pom.xml @@ -292,6 +292,12 @@ h2 runtime + + + org.apache.commons + commons-csv + 1.11.0 + ${project.artifactId}-${project.version} diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java new file mode 100644 index 00000000..5ae19995 --- /dev/null +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -0,0 +1,176 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.controller.stoptb; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.iemr.mmu.service.stoptb.NikshayExportService; +import com.iemr.mmu.service.stoptb.NikshayImportService; +import com.iemr.mmu.service.stoptb.NikshayImportService.ImportSummary; +import com.iemr.mmu.utils.JwtUtil; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +/** + * Downloads/uploads the Stop TB Nikshay ID Generator CSV. + * + * Scoped only by visit-date range — MMU runs one local database per van, so + * everything in it already belongs to the current van/service point; there + * is deliberately no vanID/servicePointID parameter on either endpoint. + */ +@RestController +@RequestMapping(value = "/stopTb/nikshay", headers = "Authorization") +// No @PreAuthorize role gate — any authenticated user can hit these endpoints +// (SecurityConfig's anyRequest().authenticated() still applies). Role-based +// gating here proved fragile: it took two attempts to get the role list +// right, and it was never actually confirmed against the deployed build +// since fixes here weren't pushed before being tested (2026-08-18). +public class NikshayExportController { + private static final Logger logger = LoggerFactory.getLogger(NikshayExportController.class); + private static final DateTimeFormatter DATE_FMT = DateTimeFormatter.ISO_LOCAL_DATE; + + @Autowired + private NikshayExportService nikshayExportService; + + @Autowired + private NikshayImportService nikshayImportService; + + @Autowired + private JwtUtil jwtUtil; + + /** Best-effort — this is only used for a created_by/modified_by audit column, + * never for authorization (the filter chain/PreAuthorize already handled that). */ + private String currentUsername(HttpServletRequest request) { + try { + String header = request.getHeader("Authorization"); + if (header == null) { + return "unknown"; + } + String token = header.startsWith("Bearer ") ? header.substring(7) : header; + String username = jwtUtil.extractUsername(token); + return username != null ? username : "unknown"; + } catch (Exception e) { + return "unknown"; + } + } + + /** Writes the CSV directly and synchronously onto the servlet response, + * instead of returning a StreamingResponseBody. StreamingResponseBody + * makes Spring process the body on a Servlet-async re-dispatch — Spring + * Security's filter chain re-runs on that async dispatch, and the + * SecurityContext doesn't reliably carry over to it, so the global + * anyRequest().authenticated() rule was denying the *second* pass even + * though the initial request authenticated fine (confirmed in production + * logs, 2026-08-18: two AccessDeniedExceptions for the same request, the + * second one through ApplicationDispatcher/AsyncContextImpl). Writing + * synchronously avoids async dispatch entirely, so there's no second + * security pass to fail. */ + @Operation(summary = "Download Stop TB beneficiaries for a date range as a CSV formatted for the Nikshay ID Generator") + @GetMapping(value = "/exportBeneficiariesCsv") + public void exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, + @RequestParam("toDate") String toDateStr, HttpServletResponse response) throws java.io.IOException { + + LocalDate fromDate; + LocalDate toDate; + try { + fromDate = LocalDate.parse(fromDateStr, DATE_FMT); + toDate = LocalDate.parse(toDateStr, DATE_FMT); + } catch (DateTimeParseException e) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "fromDate/toDate must be in YYYY-MM-DD format"); + return; + } + if (toDate.isBefore(fromDate)) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "toDate must be on or after fromDate"); + return; + } + + int excludedAlreadyGenerated; + int excludedNotReadyToExport; + try { + excludedAlreadyGenerated = nikshayExportService.countAlreadyGenerated(fromDate, toDate); + excludedNotReadyToExport = nikshayExportService.countNotReadyToExport(fromDate, toDate); + } catch (Exception e) { + logger.error("Error preparing Nikshay beneficiary export", e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Could not prepare the export"); + return; + } + + String filename = "nikshay-beneficiaries-" + fromDate + "-to-" + toDate + ".csv"; + response.setContentType("text/csv"); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); + response.setHeader("X-Excluded-Existing-Nikshay-Id-Count", String.valueOf(excludedAlreadyGenerated)); + response.setHeader("X-Excluded-Not-Ready-Count", String.valueOf(excludedNotReadyToExport)); + + try { + nikshayExportService.streamBeneficiariesCsv(fromDate, toDate, response.getOutputStream()); + } catch (Exception e) { + // Headers are already committed by the time streaming starts, so a + // mid-stream failure can only be logged, not surfaced as a clean + // error response. + logger.error("Error streaming Nikshay beneficiary CSV", e); + } + } + + @Operation(summary = "Upload the Nikshay ID Generator app's results CSV to write generated Nikshay IDs " + + "back onto the beneficiaries — each row is matched by its own benRegId column, " + + "a pass-through field the export added that the ID Generator app never touches") + @PostMapping(value = "/importResultsCsv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity importResultsCsv(@RequestParam("visitDate") String visitDateStr, + @RequestParam("file") MultipartFile file, HttpServletRequest request) { + if (file == null || file.isEmpty()) { + return ResponseEntity.badRequest().body("A results CSV file is required"); + } + LocalDate visitDate; + try { + visitDate = LocalDate.parse(visitDateStr, DATE_FMT); + } catch (DateTimeParseException e) { + return ResponseEntity.badRequest().body("visitDate must be in YYYY-MM-DD format"); + } + try { + ImportSummary summary = nikshayImportService.importResults(visitDate, file.getInputStream(), + currentUsername(request)); + return ResponseEntity.ok(summary); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } catch (Exception e) { + logger.error("Error importing Nikshay results CSV", e); + return ResponseEntity.status(500).body("Could not import the results file"); + } + } +} diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java new file mode 100644 index 00000000..28268b3d --- /dev/null +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -0,0 +1,292 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.repo.stoptb; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.List; +import java.util.function.Consumer; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.springframework.stereotype.Repository; + +/** + * Reads Stop TB beneficiaries for the Nikshay ID Generator CSV export. + * + * All table/column names here are verified against real data on a live + * server (2026-08-18), not assumed from convention — an earlier version of + * this class used i_beneficiary/I_bendemographics/tb_stoptb_visit, which + * turned out not to exist at all. The real picture: + * + * - Visit filter: db_iemr.t_benvisitdetail (VisitCategory = 'Stop TB'), + * NOT tb_stoptb_visit — confirmed 511 real rows on the reference server, + * vs. zero for the table this used to query. + * - Beneficiary identity: MMU's single datasource only connects to db_iemr, + * but the actual beneficiary/demographic tables live in db_identity, on + * the same physical MySQL server — reached here via fully-qualified + * cross-schema table names. The chain is + * db_identity.i_beneficiarymapping (BenRegId, unique) -> BenDetailsId -> + * db_identity.i_beneficiarydetails (name/DOB/gender/caste/occupation/ + * income/HIV — already denormalized to readable strings, no separate + * master-table joins needed) and BenAddressId -> + * db_identity.i_beneficiaryaddress (address/village/pincode), plus + * BenContactsId -> db_identity.i_beneficiarycontacts (phone). + * - Confirmed real gap, not a bug: many beneficiaries registered through + * the van-local registrar flow (i_beneficiarymapping.CreatedBy = + * 'reglocal') have a mapping row but their BenDetailsId/BenAddressId + * never synced to db_identity — sometimes for many days. Rows with no + * synced identity are silently left out (see streamPendingBeneficiaries) + * the same way an unresolved location is — there's nothing to put in a + * CSV row for someone whose name/address was never actually synced. + * + * Location columns (village/healthFacility/tu/district/state) are resolved + * against Nikshay's own, isolated location hierarchy — m_nikshay_village → + * m_nikshay_facility → m_nikshay_tu → m_nikshay_district → m_nikshay_state. + * This walk currently starts from the beneficiary's own personal + * i_beneficiaryaddress.CurrVillageId — confirmed on real data that this is + * AMRIT's own village ID, not Nikshay's (same numeric ID resolves to a + * different real place in each hierarchy), so this is a best-effort, + * unconfirmed mapping: it only produces a row when that AMRIT village ID + * happens to also be a valid Nikshay village ID, which will under-match. + * Whether Nikshay location should instead come from the camp/facility + * (via the logged-in worker's own assigned NikshayTUID/NikshayFacilityID + * on m_userservicerolemapping) rather than the beneficiary's personal + * address is still an open question — not yet resolved. + * + * Beneficiaries are silently left out of the streamed rows (see + * streamPendingBeneficiaries) in three cases — already having a Nikshay ID + * recorded, their identity never having synced to db_identity, or their + * village not resolving all the way up to a state. All three are counted so + * callers can report totals, but there is deliberately no separate report + * of *who* was skipped or why. + * + * The Nikshay ID itself lives on db_iemr.tb_suspected.nikshay_id — not + * tb_stoptb_diagnostics, which also has a nikshay_id column but is not the + * table this feature writes to. + */ +@Repository +public class NikshayExportRepository { + + @Autowired + private DataSource dataSource; + + private JdbcTemplate getJdbcTemplate() { + return new JdbcTemplate(dataSource); + } + + /** One beneficiary's raw, unmapped source data — Nikshay-vocabulary + * mapping/validation happens in the service layer, not here. benRegId is + * carried into the CSV itself (as a pass-through column the Nikshay ID + * Generator app never touches) so results can be matched back to a + * beneficiary on import without needing any AMRIT-side row tracking. */ + public record NikshayRawRow(Long benRegId, String firstName, String middleLastName, Integer age, String gender, + String phone, String address, String stateName, String districtName, String tu, String healthFacility, + String village, String pincode, String maritalStatus, String caste, String occupation, + String socioeconomicStatus, String chiefComplaint, String hivStatus, Boolean isHivPos) { + } + + // Placeholders in order: [1] fromDate (inclusive), [2] toDate-exclusive-upper-bound. + private static final String BASE_SELECT = "SELECT " + + " m.BenRegId AS benRegId, " + + " d.FirstName AS firstName, " + + " TRIM(CONCAT(COALESCE(d.MiddleName,''),' ',COALESCE(d.LastName,''))) AS middleLastName, " + + " TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) AS age, " + + " d.Gender AS gender, " + // PhoneNum1 is usually empty on real data — PreferredPhoneNum is the + // one actually populated at registration; fall back through the rest. + + " COALESCE(NULLIF(c.PreferredPhoneNum,''), NULLIF(c.PhoneNum1,''), NULLIF(c.PhoneNum2,'')) AS phone, " + // CurrAddressValue is usually empty on real data too — build from the + // actual line fields instead, same as CurrAddrLine1 etc. being populated. + + " COALESCE(NULLIF(d.address,''), NULLIF(a.CurrAddressValue,''), " + // CONCAT_WS skips NULLs but not empty strings, so each part needs its + // own NULLIF first or blank line fields leave stray ", ," artifacts. + + " NULLIF(TRIM(CONCAT_WS(', ', NULLIF(a.CurrAddrLine1,''), NULLIF(a.CurrAddrLine2,''), " + + " NULLIF(a.CurrAddrLine3,''), NULLIF(a.CurrHabitation,''))),'')) AS address, " + + " ns.StateName AS stateName, " + + " nd.DistrictName AS districtName, " + + " ntu.TUName AS tu, " + + " nf.FacilityName AS healthFacility, " + + " nv.VillageName AS village, " + + " a.CurrPinCode AS pincode, " + + " d.MaritalStatus AS maritalStatus, " + + " d.community AS caste, " + + " d.occupation AS occupation, " + + " d.incomeStatus AS socioeconomicStatus, " + + " (SELECT o.chief_complaint FROM tb_stoptb_general_opd o WHERE o.ben_reg_id = m.BenRegId " + + " AND o.deleted = 0 ORDER BY o.id DESC LIMIT 1) AS chiefComplaint, " + + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = m.BenRegId " + + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " + + " d.IsHIVPositive AS isHivPos, " + + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = m.BenRegId " + + " AND s.nikshay_id IS NOT NULL ORDER BY s.id DESC LIMIT 1) AS existingNikshayId " + + "FROM db_identity.i_beneficiarymapping m " + + "LEFT JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + + "LEFT JOIN db_identity.i_beneficiaryaddress a ON a.BenAddressID = m.BenAddressId " + + "LEFT JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsId = m.BenContactsId " + // a.CurrVillageId is the beneficiary's own AMRIT village ID, not confirmed + // to be a Nikshay Village ID — see class Javadoc "still an open question". + + "LEFT JOIN m_nikshay_village nv ON nv.NikshayVillageID = a.CurrVillageId AND nv.Deleted = 0 " + + "LEFT JOIN m_nikshay_facility nf ON nf.NikshayFacilityID = nv.NikshayFacilityID AND nf.Deleted = 0 " + + "LEFT JOIN m_nikshay_tu ntu ON ntu.NikshayTUID = nf.NikshayTUID AND ntu.Deleted = 0 " + + "LEFT JOIN m_nikshay_district nd ON nd.NikshayDistrictID = ntu.NikshayDistrictID AND nd.Deleted = 0 " + + "LEFT JOIN m_nikshay_state ns ON ns.NikshayStateID = nd.NikshayStateID AND ns.Deleted = 0 " + + "WHERE m.Deleted = 0 " + + " AND m.BenRegId IN ( " + + " SELECT DISTINCT v.BeneficiaryRegID FROM t_benvisitdetail v " + + " WHERE v.VisitCategory = 'Stop TB' AND v.Deleted = 0 " + + " AND v.VisitDateTime >= ? AND v.VisitDateTime < ? " + + " )"; + + public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { + String sql = "SELECT COUNT(*) FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NOT NULL"; + Integer count = getJdbcTemplate().query(sql, pss(fromDate, toDate), rs -> rs.next() ? rs.getInt(1) : 0); + return count == null ? 0 : count; + } + + /** Counts beneficiaries skipped because their identity never synced to + * db_identity (no FirstName resolved at all) OR their location doesn't + * resolve through the Nikshay hierarchy — reported as one combined "not + * ready to export" count, since both are data-completeness gaps rather + * than a beneficiary genuinely not needing a Nikshay ID. */ + public int countNotReadyToExport(LocalDate fromDate, LocalDate toDate) { + String sql = "SELECT COUNT(*) FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " + + "AND (t.firstName IS NULL " + + "OR t.village IS NULL OR t.healthFacility IS NULL OR t.tu IS NULL " + + "OR t.districtName IS NULL OR t.stateName IS NULL)"; + Integer count = getJdbcTemplate().query(sql, pss(fromDate, toDate), rs -> rs.next() ? rs.getInt(1) : 0); + return count == null ? 0 : count; + } + + /** Streams every not-yet-Nikshay-ID'd beneficiary in the date range to + * {@code rowConsumer} one row at a time, without materializing the full + * result set in memory — safe for large date ranges. Silently skips any + * beneficiary whose identity never synced or whose Nikshay village + * doesn't resolve all the way up to a state (see class Javadoc). */ + public void streamPendingBeneficiaries(LocalDate fromDate, LocalDate toDate, Consumer rowConsumer) { + String sql = "SELECT * FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " + + "AND t.firstName IS NOT NULL " + + "AND t.village IS NOT NULL AND t.healthFacility IS NOT NULL AND t.tu IS NOT NULL " + + "AND t.districtName IS NOT NULL AND t.stateName IS NOT NULL"; + JdbcTemplate jdbcTemplate = getJdbcTemplate(); + // MySQL Connector/J-specific: Integer.MIN_VALUE forces true row-by-row + // network streaming instead of buffering the whole result set client-side. + jdbcTemplate.setFetchSize(Integer.MIN_VALUE); + jdbcTemplate.query(sql, pss(fromDate, toDate), (ResultSet rs) -> rowConsumer.accept(mapRow(rs))); + } + + private PreparedStatementSetter pss(LocalDate fromDate, LocalDate toDate) { + return (PreparedStatement ps) -> { + ps.setTimestamp(1, Timestamp.valueOf(fromDate.atStartOfDay())); + ps.setTimestamp(2, Timestamp.valueOf(toDate.plusDays(1).atStartOfDay())); + }; + } + + private NikshayRawRow mapRow(ResultSet rs) throws SQLException { + return new NikshayRawRow( + rs.getObject("benRegId", Long.class), + rs.getString("firstName"), + rs.getString("middleLastName"), + rs.getObject("age", Integer.class), + rs.getString("gender"), + rs.getString("phone"), + rs.getString("address"), + rs.getString("stateName"), + rs.getString("districtName"), + rs.getString("tu"), + rs.getString("healthFacility"), + rs.getString("village"), + rs.getString("pincode"), + rs.getString("maritalStatus"), + rs.getString("caste"), + rs.getString("occupation"), + rs.getString("socioeconomicStatus"), + rs.getString("chiefComplaint"), + rs.getString("hivStatus"), + rs.getObject("isHivPos", Boolean.class)); + } + + /** Finds beneficiaries matching a results-file row by content, since the + * real Nikshay ID Generator app's results CSV carries no beneficiary ID of + * any kind back — only phone/name/age survive the round trip. Matches on + * normalized 10-digit phone plus a case-insensitive first-name match; + * callers must treat anything other than exactly one result as ambiguous + * (e.g. a shared family phone number) rather than guessing. */ + public List findMatchingBeneficiaryIds(String phoneDigits, String firstName) { + // Same column-order fix as the export's phone selection: PhoneNum1 is + // usually empty on real data, PreferredPhoneNum is what's actually + // populated (confirmed: two real beneficiaries with PhoneNum1 IS NULL + // but PreferredPhoneNum populated failed to match here before this fix). + String sql = "SELECT DISTINCT m.BenRegId FROM db_identity.i_beneficiarymapping m " + + "JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + + "JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsId = m.BenContactsId " + + "WHERE m.Deleted = 0 " + + " AND ? IN (c.PreferredPhoneNum, c.PhoneNum1, c.PhoneNum2) " + + " AND LOWER(TRIM(d.FirstName)) = LOWER(TRIM(?))"; + return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BenRegId"), phoneDigits, firstName); + } + + /** The most recent tb_suspected row for this beneficiary, if any — looked + * up live at import time (no export-time snapshot needed, since the + * beneficiary is identified directly from the results CSV's own benRegId + * column). Null if none exists yet. Note: tb_suspected has no `deleted` + * column, unlike most other AMRIT tables. */ + public Long findLatestSuspectedId(Long benRegId) { + String sql = "SELECT id FROM tb_suspected WHERE benRegID = ? ORDER BY id DESC LIMIT 1"; + return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); + } + + public void updateNikshayId(Long suspectedId, String nikshayId, String modifiedBy) { + String sql = "UPDATE tb_suspected SET nikshay_id = ?, modified_by = ?, " + + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; + getJdbcTemplate().update(sql, nikshayId, modifiedBy, suspectedId); + } + + /** Called when a beneficiary had no tb_suspected row yet — creates one to + * hold the Nikshay ID the portal generated. created_date is set explicitly + * because, unlike created_date on most other AMRIT tables, tb_suspected's + * has no DB-side default. */ + public Long insertSuspectedWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, + String createdBy) { + String sql = "INSERT INTO tb_suspected (benRegID, visit_date, nikshay_id, created_by, created_date) " + + "VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)"; + KeyHolder keyHolder = new GeneratedKeyHolder(); + getJdbcTemplate().update(connection -> { + PreparedStatement ps = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); + ps.setLong(1, benRegId); + ps.setTimestamp(2, Timestamp.valueOf(visitDate.atStartOfDay())); + ps.setString(3, nikshayId); + ps.setString(4, createdBy); + return ps; + }, keyHolder); + return keyHolder.getKey().longValue(); + } +} diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java new file mode 100644 index 00000000..adae98e4 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java @@ -0,0 +1,249 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.stoptb; + +import java.io.BufferedWriter; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.repo.stoptb.NikshayExportRepository; +import com.iemr.mmu.repo.stoptb.NikshayExportRepository.NikshayRawRow; + +/** + * Builds the Nikshay ID Generator CSV for Stop TB beneficiaries. + * + * Column order/names are fixed by the Nikshay ID Generator app's own import + * format — every categorical field below resolves to one of its allowed + * values (case-insensitive match) when AMRIT's data lines up, otherwise a + * safe always-valid fallback (e.g. "Unknown"), never a guessed/fuzzy mapping + * onto the wrong specific label. + * + * state/district/tu/healthFacility/village are all resolved against + * Nikshay's own location hierarchy (m_nikshay_state/district/tu/facility/ + * village), not AMRIT's standard masters — see NikshayExportRepository's + * class Javadoc for how that chain is walked from the beneficiary's own + * saved Nikshay village. A beneficiary whose village doesn't resolve all the + * way up that chain never reaches this class — the repository leaves such + * rows out of the stream entirely, silently, the same way it already + * silently skips beneficiaries who already have a Nikshay ID. + * + * Known gaps, best-effort until resolved elsewhere: + * - occupation/area: no reliable AMRIT-to-Nikshay label mapping exists yet, + * so these always fall back to "Unknown" (a valid value for both). + * - symptoms: AMRIT has no structured Stop TB symptom checklist wired up yet + * (that data lives in the generic Dynamic Form response tables, whose + * question mapping isn't resolved). Best-effort: "Asymptomatic" when no + * chief complaint was recorded, "Others" otherwise. + * - gender has no safe generic fallback (no "Unknown" option in Nikshay's + * 3-value list) — left blank on an unmapped value rather than guessed, + * which will surface as a clear per-row error in the ID Generator app. + * - typeOfCaseFinding: no Active/Passive signal in AMRIT yet — always + * "Passive", matching the portal's own default. + */ +@Service +public class NikshayExportService { + + // benRegId is a pass-through column, not one of Nikshay's own template fields — the ID + // Generator app never reads or displays it, but carries it straight through to the + // results file, which is how an uploaded results CSV gets matched back to a beneficiary. + private static final String[] CSV_HEADER = { "benRegId", "typeOfCaseFinding", "caste", "firstName", + "middleLastName", "age", "gender", "primaryPhone", "address", "state", "district", "tu", "healthFacility", + "village", "pincode", "area", "maritalStatus", "occupation", "socioeconomicStatus", "symptoms", + "hivStatus" }; + + private static final Set GENDER_VALUES = setOf("Male", "Female", "Transgender"); + private static final Set CASTE_VALUES = setOf("SC", "ST", "Other"); + private static final Set MARITAL_VALUES = setOf("Single", "Married", "Unknown"); + private static final Set SOCIOECONOMIC_VALUES = setOf("APL", "BPL", "Unknown"); + private static final Set HIV_VALUES = setOf("Positive", "Reactive", "Non Reactive / Negative", "Unknown"); + // Verbatim from the ID Generator's own allowed-values list — spellings/typos + // are copied exactly as the portal defines them. + private static final Set OCCUPATION_VALUES = setOf("Legislators and Senior officials", + "Corporate Manager", "General Manager", + "Physical, mathematical and engineering science professional", + "Life sciences and health professional", "Teaching professional", "Other professional", "Office Clerk", + "Customer Services Clerks", "Personal Protective Service Providers", + "Models, Sales Persons and Demonstrators", "Market oriented skilled agriculutre and fishery workers", + "Subsitence agriculture and fishery workers", "Extraction and building trade workers", + "Metal, Machinery and related trades workers", + "Precision, handicraft, printing and related trade workers", + "Other Craft and related traders and workers", "Stationary Plant and related Operators", + "Machine Operators and Assembler", "Drivers and Mobile Plant Operators", + "Sales and Services elementry occupations", "Agriculture, fishery and related labour", + "Laborers in mining, construction, manufecturing and transport", "New Workers seeking employment", + "Workers reporting occupation unidentifiable or inadequately", "Workers no reporting any occupation", + "House Wife", "Unknown"); + + @Autowired + private NikshayExportRepository nikshayExportRepository; + + private static Set setOf(String... values) { + return new HashSet<>(Arrays.asList(values)); + } + + public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { + return nikshayExportRepository.countAlreadyGenerated(fromDate, toDate); + } + + public int countNotReadyToExport(LocalDate fromDate, LocalDate toDate) { + return nikshayExportRepository.countNotReadyToExport(fromDate, toDate); + } + + /** Writes the CSV (header + one row per pending beneficiary) directly to + * {@code outputStream} as rows arrive from the database — never buffers + * the whole file in memory. Caller owns closing {@code outputStream}. */ + public void streamBeneficiariesCsv(LocalDate fromDate, LocalDate toDate, OutputStream outputStream) { + Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)); + try { + writer.write(String.join(",", CSV_HEADER)); + writer.write("\r\n"); + + nikshayExportRepository.streamPendingBeneficiaries(fromDate, toDate, row -> { + try { + writer.write(toCsvLine(row)); + writer.write("\r\n"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + writer.flush(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String toCsvLine(NikshayRawRow row) { + String[] values = { + String.valueOf(row.benRegId()), + "Passive", + matchOrDefault(row.caste(), CASTE_VALUES, "Other"), + nullToEmpty(row.firstName()), + nullToEmpty(row.middleLastName()), + ageOrBlank(row.age()), + matchOrBlank(row.gender(), GENDER_VALUES), + validPhoneOrBlank(row.phone()), + nullToEmpty(row.address()), + nullToEmpty(row.stateName()), + nullToEmpty(row.districtName()), + nullToEmpty(row.tu()), + nullToEmpty(row.healthFacility()), + nullToEmpty(row.village()), + validPincodeOrBlank(row.pincode()), + "Unknown", // area - no reliable AMRIT-to-Nikshay mapping available yet + matchOrDefault(row.maritalStatus(), MARITAL_VALUES, "Unknown"), + matchOrDefault(row.occupation(), OCCUPATION_VALUES, "Unknown"), + matchOrDefault(row.socioeconomicStatus(), SOCIOECONOMIC_VALUES, "Unknown"), + symptomsFrom(row.chiefComplaint()), + hivStatusFrom(row.hivStatus(), row.isHivPos()), + }; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(csvEscape(values[i])); + } + return sb.toString(); + } + + private static String nullToEmpty(String s) { + return s == null ? "" : s.trim(); + } + + private static String matchOrBlank(String raw, Set allowed) { + if (raw == null) { + return ""; + } + String trimmed = raw.trim(); + for (String candidate : allowed) { + if (candidate.equalsIgnoreCase(trimmed)) { + return candidate; + } + } + return ""; + } + + private static String matchOrDefault(String raw, Set allowed, String fallback) { + String matched = matchOrBlank(raw, allowed); + return matched.isEmpty() ? fallback : matched; + } + + private static String ageOrBlank(Integer age) { + return (age != null && age >= 1 && age <= 99) ? String.valueOf(age) : ""; + } + + private static String validPhoneOrBlank(String raw) { + if (raw == null) { + return ""; + } + String digits = raw.replaceAll("[^0-9]", ""); + if (digits.length() > 10) { + digits = digits.substring(digits.length() - 10); + } + return digits.matches("[1-9][0-9]{9}") ? digits : ""; + } + + private static String validPincodeOrBlank(String raw) { + if (raw == null) { + return ""; + } + String trimmed = raw.trim(); + return trimmed.matches("[0-9]{6}") ? trimmed : ""; + } + + private static String symptomsFrom(String chiefComplaint) { + return (chiefComplaint == null || chiefComplaint.trim().isEmpty()) ? "Asymptomatic" : "Others"; + } + + private static String hivStatusFrom(String rawHivStatus, Boolean isHivPos) { + String matched = matchOrBlank(rawHivStatus, HIV_VALUES); + if (!matched.isEmpty()) { + return matched; + } + if (Boolean.TRUE.equals(isHivPos)) { + return "Positive"; + } + if (Boolean.FALSE.equals(isHivPos)) { + return "Non Reactive / Negative"; + } + return "Unknown"; + } + + private static String csvEscape(String value) { + if (value == null || value.isEmpty()) { + return ""; + } + boolean needsQuoting = value.indexOf(',') >= 0 || value.indexOf('"') >= 0 || value.indexOf('\n') >= 0 + || value.indexOf('\r') >= 0; + String escaped = value.replace("\"", "\"\""); + return needsQuoting ? "\"" + escaped + "\"" : escaped; + } +} diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java new file mode 100644 index 00000000..ebc81e4e --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -0,0 +1,197 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.stoptb; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.repo.stoptb.NikshayExportRepository; + +/** + * Imports the Nikshay ID Generator desktop app's results CSV, writing the + * portal-generated Nikshay IDs back onto the right beneficiaries. + * + * Confirmed against a real results file (2026-08-18): the ID Generator app + * does NOT pass through any AMRIT-side beneficiary ID — the results CSV only + * has the original template columns (typeOfCaseFinding, caste, firstName, + * ..., hivStatus) plus its own generatedId/status/durationSec/error columns. + * There is deliberately no assumption that result-row order matches the + * uploaded file's row order either (unconfirmed, and a single + * reordered/dropped row would silently corrupt every row after it if wrong). + * + * If a row happens to carry a benRegId column with a parseable value (e.g. a + * manually-rebuilt test file, or some future ID Generator version that keeps + * it) it's used directly — exact, no ambiguity possible. Otherwise, each row + * is matched back to a beneficiary by content — normalized 10-digit + * primaryPhone plus a case-insensitive firstName match (see + * NikshayExportRepository.findMatchingBeneficiaryIds). This is a heuristic, + * not an exact key: anything other than exactly one match (zero, or more + * than one — e.g. a shared family phone number) is left for manual review + * rather than guessed. + * + * Row status handling: + * - "success": generatedId is the new Nikshay ID — written as-is. + * - "skipped" (portal-detected duplicate): generatedId is one or more + * existing patient IDs, space-separated. A single ID is written the same + * as a success; more than one is ambiguous and left for manual review + * rather than guessed. + * - "failed": never written; surfaced in the response for visibility. + */ +@Service +public class NikshayImportService { + + private static final List REQUIRED_COLUMNS = List.of("primaryPhone", "firstName", "middleLastName", + "generatedId", "status"); + + public record ImportRowResult(int rowIndex, Long benRegId, String firstName, String middleLastName, + String status, String generatedId, String note) { + } + + public record ImportSummary(int csvRowCount, int updated, int failed, int needsReview, + List needsReviewRows, List failedRows) { + } + + @Autowired + private NikshayExportRepository nikshayExportRepository; + + public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStream, String modifiedBy) + throws Exception { + List records; + boolean hasErrorColumn; + boolean hasBenRegIdColumn; + CSVFormat format = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).setTrim(true).build(); + try (CSVParser parser = new CSVParser(new InputStreamReader(csvInputStream, StandardCharsets.UTF_8), + format)) { + Map header = parser.getHeaderMap(); + for (String required : REQUIRED_COLUMNS) { + if (!header.containsKey(required)) { + throw new IllegalArgumentException("Results CSV is missing required column: " + required); + } + } + hasErrorColumn = header.containsKey("error"); + hasBenRegIdColumn = header.containsKey("benRegId"); + records = parser.getRecords(); + } + + int updated = 0; + List needsReview = new ArrayList<>(); + List failedRows = new ArrayList<>(); + + for (int i = 0; i < records.size(); i++) { + CSVRecord record = records.get(i); + String firstName = record.get("firstName"); + String middleLastName = record.get("middleLastName"); + String status = record.get("status").trim(); + String generatedId = record.get("generatedId").trim(); + + Long benRegId = hasBenRegIdColumn ? parseBenRegId(record.get("benRegId")) : null; + if (benRegId == null) { + // No usable benRegId on this row — fall back to phone+name matching, + // the only option against a genuine Nikshay portal results file. + String phoneDigits = normalizePhone(record.get("primaryPhone")); + if (phoneDigits == null) { + failedRows.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, + "Row has a missing/invalid primaryPhone — cannot match it back to a beneficiary.")); + continue; + } + + List candidates = nikshayExportRepository.findMatchingBeneficiaryIds(phoneDigits, firstName); + if (candidates.size() != 1) { + String note = candidates.isEmpty() + ? "No beneficiary found matching this phone number and first name." + : "More than one beneficiary matches this phone number and first name (e.g. a shared " + + "family phone) — needs manual confirmation."; + needsReview + .add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, note)); + continue; + } + benRegId = candidates.get(0); + } + + if ("success".equalsIgnoreCase(status) || "skipped".equalsIgnoreCase(status)) { + String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); + if (tokens.length == 1) { + writeNikshayId(visitDate, benRegId, tokens[0], modifiedBy); + updated++; + } else { + String note = tokens.length == 0 ? "Row marked " + status + " but has no generatedId." + : "Multiple possible existing Nikshay IDs (" + generatedId + + ") — needs manual confirmation."; + needsReview.add( + new ImportRowResult(i, benRegId, firstName, middleLastName, status, generatedId, note)); + } + } else { + String error = hasErrorColumn ? record.get("error") : ""; + failedRows.add( + new ImportRowResult(i, benRegId, firstName, middleLastName, status, generatedId, error)); + } + } + + return new ImportSummary(records.size(), updated, failedRows.size(), needsReview.size(), needsReview, + failedRows); + } + + private static Long parseBenRegId(String raw) { + if (raw == null || raw.isBlank()) { + return null; + } + try { + return Long.valueOf(raw.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + /** Same normalization as the export's validPhoneOrBlank, so a phone number + * round-trips consistently: strips non-digits, keeps the last 10, requires + * a valid-looking Indian mobile number. Null if it doesn't. */ + private static String normalizePhone(String raw) { + if (raw == null) { + return null; + } + String digits = raw.replaceAll("[^0-9]", ""); + if (digits.length() > 10) { + digits = digits.substring(digits.length() - 10); + } + return digits.matches("[1-9][0-9]{9}") ? digits : null; + } + + private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, String modifiedBy) { + Long suspectedId = nikshayExportRepository.findLatestSuspectedId(benRegId); + if (suspectedId != null) { + nikshayExportRepository.updateNikshayId(suspectedId, nikshayId, modifiedBy); + } else { + nikshayExportRepository.insertSuspectedWithNikshayId(benRegId, visitDate, nikshayId, modifiedBy); + } + } +}