fix(sync): faster changes-feed first download (100 beneficiaries per page)

src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java

@@ -123,4 +123,14 @@ List<Long> findBeneficiaryIdsAwaitingManualEntry(@Param("orderType") String orde
             "AND (:providerServiceMapId IS NULL OR b.providerServiceMapId = :providerServiceMapId))")
     List<Object[]> findLatestOrderStatusByBeneficiary(@Param("orderType") String orderType,
             @Param("villageId") Integer villageId, @Param("providerServiceMapId") Integer providerServiceMapId);
+
+    // True when the beneficiary already has an order of this type that a new referral must not
+    // duplicate: any row created since dayStart (whatever its status, so a FAILED push is left for
+    // the nurse's Retry instead of being re-sent every sweep), or an open/resolved order from an
+    // earlier day, which createAndPushOrder would return instead of creating a new one anyway.
+    @Query("SELECT COUNT(o) > 0 FROM DiagnosticOrder o WHERE o.beneficiaryId = :beneficiaryId " +
+            "AND o.orderType = :orderType AND o.deleted = false " +
+            "AND (o.createdDate >= :dayStart OR o.status IN ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'MANUAL_ENTRY'))")
+    boolean hasOrderTodayOrOpen(@Param("beneficiaryId") Long beneficiaryId, @Param("orderType") String orderType,
+            @Param("dayStart") Timestamp dayStart);
 }

src/main/java/com/iemr/flw/repo/iemr/TBScreeningRepo.java

@@ -50,4 +50,11 @@ public interface TBScreeningRepo extends JpaRepository<TBScreening, Long> {
     @Modifying
     @Query("UPDATE TBScreening t SET t.vanSerialNo = t.id WHERE t.id = :id")
     void updateVanSerialNo(@Param("id") Long id);
+
+    // This van's screenings since dayStart that referred the beneficiary for a chest X-ray or a
+    // sputum test; read by the hub's referral-order sweep.
+    @Query("SELECT tbs FROM TBScreening tbs WHERE tbs.vanID = :vanID AND tbs.deleted = false " +
+            "AND tbs.visitDate >= :dayStart " +
+            "AND (tbs.referredForDigitalChestXray = true OR tbs.referredForSputumCollection = true)")
+    List<TBScreening> findReferredSince(@Param("vanID") Integer vanID, @Param("dayStart") Timestamp dayStart);
 }

src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java

@@ -266,12 +266,19 @@
     // Used by the STOP-TB changes feed; a BenRegId with no mapping or address is left out.
     @Override
     public Map<Long, String> getBeneficiaryRecordsByRegIds(List<Long> benRegIds, String authorisation) {
-        List<RMNCHMBeneficiaryaddress> addresses = new ArrayList<>();
-        for (Long benRegId : benRegIds) {
-            RMNCHMBeneficiarymapping m = beneficiaryRepo.getById(BigInteger.valueOf(benRegId));
-            if (m == null || m.getBenAddressId() == null) continue;
-            RMNCHMBeneficiaryaddress a = beneficiaryRepo.getAddressById(m.getBenAddressId());
-            if (a != null) addresses.add(a);
+        // Two queries for the whole page; the per-beneficiary lookups stay as the fallback.
+        List<RMNCHMBeneficiaryaddress> addresses;
+        try {
+            addresses = worklistPagePrefetch.addressesForRegIds(benRegIds);
+        } catch (Exception e) {
+            logger.warn("Beneficiary address prefetch failed, using per-beneficiary queries: " + e.getMessage());
+            addresses = new ArrayList<>();
+            for (Long benRegId : benRegIds) {
+                RMNCHMBeneficiarymapping m = beneficiaryRepo.getById(BigInteger.valueOf(benRegId));
+                if (m == null || m.getBenAddressId() == null) continue;
+                RMNCHMBeneficiaryaddress a = beneficiaryRepo.getAddressById(m.getBenAddressId());
+                if (a != null) addresses.add(a);
+            }
         }
         Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create();
         Map<Long, String> records = new LinkedHashMap<>();

src/main/java/com/iemr/flw/service/impl/StopTBReferralOrderService.java

@@ -0,0 +1,230 @@
+package com.iemr.flw.service.impl;
+
+import com.iemr.flw.domain.identity.RMNCHMBeneficiarydetail;
+import com.iemr.flw.domain.identity.RMNCHMBeneficiarymapping;
+import com.iemr.flw.domain.iemr.TBScreening;
+import com.iemr.flw.dto.DiagnosticOrderRequestDto;
+import com.iemr.flw.masterEnum.DiagnosticOrderType;
+import com.iemr.flw.repo.identity.BeneficiaryRepo;
+import com.iemr.flw.repo.iemr.DiagnosticOrderRepo;
+import com.iemr.flw.repo.iemr.TBScreeningRepo;
+import com.iemr.flw.service.CampConfigService;
+import com.iemr.flw.service.DiagnosticOrderService;
+import jakarta.annotation.PreDestroy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.support.TransactionSynchronization;
+import org.springframework.transaction.support.TransactionSynchronizationManager;
+
+import java.math.BigInteger;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Creates the diagnostic orders a TB screening refers the beneficiary for (chest X-ray, sputum
+ * TrueNat) on the camp hub itself, from the screening record it receives.
+ *
+ * The tablet used to place these orders with its own order/push call right after saving the
+ * screening. When the tablet could not reach the hub at that moment the order was marked FAILED on
+ * the tablet only and never retried, and the registrar who fills the screening has no access to
+ * the X-ray/TrueNat screens to retry it. The screening itself always reaches the hub later through
+ * normal sync, so the hub creating the order from it removes the dependency on the tablet being
+ * connected at save time.
+ *
+ * Two triggers, both limited to screenings dated today (hub local date) on this camp's van:
+ * - right after a screening save commits (single background thread, so a slow or unreachable
+ *   device never holds up screening sync);
+ * - a periodic sweep for referred screenings that still have no order, covering screenings saved
+ *   before this existed and any attempt that died.
+ *
+ * Duplicates are prevented by DiagnosticOrderService's per-beneficiary lock and dedup, and by
+ * skipping beneficiaries that already have an order of that type today or an open one.
+ * Only runs where a camp van is configured, so the central server never sends orders to devices.
+ */
+@Service
+public class StopTBReferralOrderService {
+
+    private static final Logger logger = LoggerFactory.getLogger(StopTBReferralOrderService.class);
+
+    static final String ORDER_EVENT = "STOP_TB_REFERRAL";
+
+    @Autowired
+    private DiagnosticOrderService diagnosticOrderService;
+
+    @Autowired
+    private DiagnosticOrderRepo diagnosticOrderRepo;
+
+    @Autowired
+    private TBScreeningRepo tbScreeningRepo;
+
+    @Autowired
+    private BeneficiaryRepo beneficiaryRepo;
+
+    @Autowired
+    private CampConfigService campConfigService;
+
+    private final ExecutorService executor = Executors.newSingleThreadExecutor(r -> {
+        Thread t = new Thread(r, "stoptb-referral-orders");
+        t.setDaemon(true);
+        return t;
+    });
+
+    // Set while a sweep's batch is queued or running, so a backlog (e.g. an unreachable device
+    // holding each push to its timeout) is not queued again on every tick.
+    private final AtomicBoolean sweepRunning = new AtomicBoolean(false);
+
+    /** One saved screening's referral flags. */
+    public record Referral(Long beneficiaryRegID, boolean xray, boolean sputum) {
+        boolean any() {
+            return xray || sputum;
+        }
+    }
+
+    /**
+     * Queues order creation for the given referrals once the caller's transaction commits (or right
+     * away when there is none), so the orders are created against committed screening rows.
+     */
+    public void createAfterCommit(List<Referral> referrals) {
+        if (!campConfigService.isCampConfigured()) return;
+        List<Referral> pending = referrals.stream().filter(Referral::any).toList();
+        if (pending.isEmpty()) return;
+        Runnable submit = () -> executor.submit(() -> pending.forEach(this::createOrders));
+        if (TransactionSynchronizationManager.isSynchronizationActive()) {
+            TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
+                @Override
+                public void afterCommit() {
+                    submit.run();
+                }
+            });
+        } else {
+            submit.run();
+        }
+    }
+
+    // Runs on the shared scheduler thread, so it only reads and hands the work to the referral
+    // thread: an order push can wait on the device for its full timeout.
+    @Scheduled(fixedDelayString = "${diagnostic.referral-sweep.tick-ms:120000}",
+            initialDelayString = "${diagnostic.referral-sweep.initial-delay-ms:60000}")
+    public void sweep() {
+        if (!campConfigService.isCampConfigured()) return;
+        if (!sweepRunning.compareAndSet(false, true)) return;
+        try {
+            List<TBScreening> referred = tbScreeningRepo.findReferredSince(campConfigService.getVanID(), startOfToday());
+            List<Referral> referrals = new ArrayList<>();
+            for (TBScreening s : referred) {
+                referrals.add(new Referral(s.getBenRegID(),
+                        Boolean.TRUE.equals(s.getReferredForDigitalChestXray()),
+                        Boolean.TRUE.equals(s.getReferredForSputumCollection())));
+            }
+            if (referrals.isEmpty()) {
+                sweepRunning.set(false);
+                return;
+            }
+            logger.debug("Referral sweep: {} referred screening(s) today", referrals.size());
+            executor.submit(() -> {
+                try {
+                    referrals.forEach(this::createOrders);
+                } finally {
+                    sweepRunning.set(false);
+                }
+            });
+        } catch (RuntimeException e) {
+            sweepRunning.set(false);
+            logger.error("Referral sweep failed: {}", e.getMessage(), e);
+        }
+    }
+
+    void createOrders(Referral referral) {
+        try {
+            if (referral.beneficiaryRegID() == null) return;
+            if (referral.xray()) createIfMissing(referral.beneficiaryRegID(), DiagnosticOrderType.XRAY_CHEST);
+            if (referral.sputum()) createIfMissing(referral.beneficiaryRegID(), DiagnosticOrderType.MTB);
+        } catch (Exception e) {
+            logger.error("Referral orders failed for benRegID={}: {}", referral.beneficiaryRegID(), e.getMessage(), e);
+        }
+    }
+
+    private void createIfMissing(Long beneficiaryRegID, DiagnosticOrderType orderType) {
+        if (!screenedToday(beneficiaryRegID)) return;
+        BigInteger benId = beneficiaryRepo.getBenIdFromRegID(beneficiaryRegID);
+        if (benId == null) {
+            logger.warn("Referral order skipped: no beneficiaryID for benRegID={}", beneficiaryRegID);
+            return;
+        }
+        Long beneficiaryId = benId.longValue();
+        if (diagnosticOrderRepo.hasOrderTodayOrOpen(beneficiaryId, orderType.name(), startOfToday())) return;
+
+        DiagnosticOrderRequestDto.PatientDto patient = patientFor(beneficiaryRegID);
+        if (patient == null) {
+            logger.warn("Referral order skipped: no beneficiary details for benRegID={}", beneficiaryRegID);
+            return;
+        }
+        DiagnosticOrderRequestDto request = new DiagnosticOrderRequestDto();
+        request.setBeneficiaryId(beneficiaryId);
+        request.setOrderType(orderType.name());
+        request.setOrderEvent(ORDER_EVENT);
+        request.setPatient(patient);
+        try {
+            diagnosticOrderService.createAndPushOrderAsSystem(request);
+            logger.info("Referral order created for beneficiaryId={}, orderType={}", beneficiaryId, orderType);
+        } catch (Exception e) {
+            logger.error("Referral order failed for beneficiaryId={}, orderType={}: {}",
+                    beneficiaryId, orderType, e.getMessage());
+        }
+    }
+
+    // The latest screening must be today's: an order is for the patient who is at the camp now,
+    // not for one screened on an earlier day whose record only reached the hub today.
+    private boolean screenedToday(Long beneficiaryRegID) {
+        List<TBScreening> latest = tbScreeningRepo.findLatestByBenRegID(beneficiaryRegID,
+                org.springframework.data.domain.PageRequest.of(0, 1));
+        return !latest.isEmpty() && latest.get(0).getVisitDate() != null
+                && !latest.get(0).getVisitDate().before(startOfToday());
+    }
+
+    DiagnosticOrderRequestDto.PatientDto patientFor(Long beneficiaryRegID) {
+        List<RMNCHMBeneficiarymapping> mappings =
+                beneficiaryRepo.findByBenRegIdFromMapping(BigInteger.valueOf(beneficiaryRegID));
+        if (mappings.isEmpty() || mappings.get(0).getBenDetailsId() == null) return null;
+        RMNCHMBeneficiarydetail detail = beneficiaryRepo.getDetailsById(mappings.get(0).getBenDetailsId());
+        if (detail == null) return null;
+
+        DiagnosticOrderRequestDto.PatientDto patient = new DiagnosticOrderRequestDto.PatientDto();
+        patient.setFirstName(detail.getFirstName() != null ? detail.getFirstName().trim() : "");
+        patient.setLastName(detail.getLastName() != null ? detail.getLastName().trim() : "");
+        patient.setDateOfBirth(detail.getDob() != null ? new SimpleDateFormat("yyyy-MM-dd").format(detail.getDob()) : null);
+        patient.setSex(sexOf(detail.getGender(), detail.getGenderId()));
+        return patient;
+    }
+
+    // Same three values the tablet sent (Male / Female / Other).
+    static String sexOf(String gender, Integer genderId) {
+        if (gender != null) {
+            if (gender.trim().equalsIgnoreCase("Male")) return "Male";
+            if (gender.trim().equalsIgnoreCase("Female")) return "Female";
+        }
+        if (genderId != null) {
+            if (genderId == 1) return "Male";
+            if (genderId == 2) return "Female";
+        }
+        return "Other";
+    }
+
+    static Timestamp startOfToday() {
+        return Timestamp.valueOf(LocalDate.now().atStartOfDay());
+    }
+
+    @PreDestroy
+    void shutdown() {
+        executor.shutdown();
+    }
+}

src/main/java/com/iemr/flw/service/impl/StopTBServiceImpl.java

@@ -76,6 +76,9 @@ public class StopTBServiceImpl implements StopTBService {
     @Autowired
     private TBStopVisitService tbStopVisitService;
 
+    @Autowired
+    private StopTBReferralOrderService referralOrderService;
+
     @Autowired
     private BenAnthropometryRepo benAnthropometryRepo;
 
@@ -225,6 +228,7 @@ private Map<String, Object> examToMap(StopTBGeneralExamination e) {
     @Transactional
     public List<Map<String, Object>> saveNurseTBScreening(List<Map<String, Object>> dataList) throws Exception {
         List<Map<String, Object>> results = new ArrayList<>();
+        List<StopTBReferralOrderService.Referral> referrals = new ArrayList<>();
         Integer vanID = campConfigService.getVanID();
         Integer parkingPlaceID = campConfigService.getParkingPlaceID();
         for (Map<String, Object> data : dataList) {
@@ -300,12 +304,17 @@ public List<Map<String, Object>> saveNurseTBScreening(List<Map<String, Object>>
 
             tbScreeningRepo.save(screening);
             if (isNew) tbScreeningRepo.updateVanSerialNo(screening.getId());
+            referrals.add(new StopTBReferralOrderService.Referral(beneficiaryRegID,
+                    Boolean.TRUE.equals(screening.getReferredForDigitalChestXray()),
+                    Boolean.TRUE.equals(screening.getReferredForSputumCollection())));
 
             Map<String, Object> result = new HashMap<>();
             result.put("beneficiaryRegID", beneficiaryRegID);
             result.put("visitCode", visit.getVisitCode());
             results.add(result);
         }
+        // The hub, not the tablet, places the X-ray / TrueNat orders the screening refers for.
+        referralOrderService.createAfterCommit(referrals);
         return results;
     }
 

src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java

@@ -43,10 +43,12 @@ public class StopTBSyncServiceImpl implements StopTBSyncService {
     static final int DEFAULT_LIMIT = 200;
     static final int MAX_LIMIT = 500;
     /**
-     * Beneficiary records are the expensive ones (several identity lookups and a FHIR call each), so a
-     * page carries at most this many; the token resumes from the last one sent.
+     * Beneficiary records are the most expensive to build, so a page carries at most this many; the
+     * token resumes from the last one sent. They are now built with one query per table for the whole
+     * page (WorklistPagePrefetch, health IDs read locally), so the cap was raised from 25 to cut the
+     * number of pages a new tablet needs for its first download.
      */
-    static final int MAX_BENEFICIARIES_PER_PAGE = 25;
+    static final int MAX_BENEFICIARIES_PER_PAGE = 100;
     /** Margin under the oldest open transaction (see safePoint); lines younger than this wait. */
     static final int SAFE_LAG_SECONDS = 5;
     /** The hub keeps 30 days of log; a token older than this may point at cleaned-up lines. */

src/main/java/com/iemr/flw/service/impl/WorklistPagePrefetch.java

@@ -71,7 +71,7 @@
     public List<RMNCHMBeneficiaryaddress> addressesInOrder(List<BigInteger> ids) {
         Map<BigInteger, RMNCHMBeneficiaryaddress> byId = new HashMap<>();
         putAll(byId, entities(identityEm, RMNCHMBeneficiaryaddress.class,
                 "SELECT * FROM db_identity.i_beneficiaryaddress WHERE BenAddressID IN (%s)",
                 new LinkedHashSet<>(ids)), RMNCHMBeneficiaryaddress::getBenAddressID);
         return ids.stream().map(byId::get).filter(Objects::nonNull).collect(Collectors.toList());
     }
@@ -153,6 +153,33 @@
         return page;
     }
 
+    /**
+     * The address row of each beneficiary, in the order of regIds, using one query for the mappings
+     * and one for the addresses instead of two per beneficiary. If a beneficiary has several mappings,
+     * the newest one with an address is used. A beneficiary with no mapping or no address is left out,
+     * as the per-row lookup did.
+     */
+    public List<RMNCHMBeneficiaryaddress> addressesForRegIds(List<Long> regIds) {
+        if (regIds.isEmpty()) return Collections.emptyList();
+        Map<Long, BigInteger> addressIdByRegId = new HashMap<>();
+        for (Object[] r : rows(identityEm,
+                "SELECT BenRegId, BenAddressId FROM db_identity.i_beneficiarymapping WHERE BenRegId IN (%s) ORDER BY BenMapId DESC",
+                new LinkedHashSet<>(regIds))) {
+            if (r[1] != null) addressIdByRegId.putIfAbsent(toLong(r[0]), toBigInteger(r[1]));
+        }
+        Map<BigInteger, RMNCHMBeneficiaryaddress> addressById = new HashMap<>();
+        putAll(addressById, entities(identityEm, RMNCHMBeneficiaryaddress.class,
+                "SELECT * FROM db_identity.i_beneficiaryaddress WHERE BenAddressID IN (%s)",
+                new LinkedHashSet<>(addressIdByRegId.values())), RMNCHMBeneficiaryaddress::getBenAddressID);
+        List<RMNCHMBeneficiaryaddress> out = new ArrayList<>();
+        for (Long regId : regIds) {
+            BigInteger addressId = addressIdByRegId.get(regId);
+            RMNCHMBeneficiaryaddress a = addressId == null ? null : addressById.get(addressId);
+            if (a != null) out.add(a);
+        }
+        return out;
+    }
+
     @SuppressWarnings("unchecked")
     private static <T> List<T> entities(EntityManager em, Class<T> type, String sql, Collection<?> ids) {
         if (ids.isEmpty()) return Collections.emptyList();