From 44d036e7cfde0f9f539a9b7a8042268c1034ad41 Mon Sep 17 00:00:00 2001
From: shvngisingh <Shvngisingh@google.com>
Date: Mon, 10 Aug 2026 09:07:00 +0000
Subject: [PATCH 1/2] Update CDC500 state query to use new TimeSeries and
 Observation tables

---
 scripts/us_cdc/cdc500_state/process.py | 99 +++++++++++++++-----------
 1 file changed, 58 insertions(+), 41 deletions(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index 0115b96e43..ea9b2ecbe7 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -22,49 +22,66 @@
     os.mkdir(_OUTPUT_FILE_PATH)
 
 query = """
-SELECT distinct * from(
-SELECT 
-  statvar, 
-  SUBSTR(observation_about,0,8) as observation_about,
-  observation_date, 
-  CONCAT('dcAggregate/',measurement_method) as measurement_method,
-  population_statvar,
-  SUM(CAST(pop_count AS FLOAT64))*100/SUM(CAST(population AS FLOAT64)) as percent
-FROM 
-(
+WITH cdc_sv AS (
   SELECT
-    SVO1.variable_measured as statvar,
-    SVO1.observation_about as observation_about,
-    SVO1.observation_date as observation_date,
-    SVO1.value as percent,
-    SVO1.measurement_method as measurement_method,
-    SVO2.variable_measured as population_statvar,
-    SVO2.value as population,
-    CAST(SVO2.value AS FLOAT64) * CAST(SVO1.value AS FLOAT64) / 100 as pop_count
-  FROM `datcom-store.dc_kg_latest.StatVarObservation` as SVO1
-  JOIN `datcom-store.dc_kg_latest.StatVarObservation` as SVO2 ON TRUE
-  JOIN (
-    # Get the statvars and corresponding population statvar
-    # with ‘Percent_’ replaced with ‘Count_’ and
-    # dropping the non-age, non-gender constraints.
+    variable_measured AS cdc500,
+    CONCAT('Count_', REGEXP_SUBSTR(variable_measured, '(Person_.*ale|Person_.*Years|Person)')) AS pop_statvar
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries`
+  WHERE provenance = 'dc/base/CDC500'
+    AND variable_measured LIKE 'Percent_%'
+  GROUP BY cdc500, pop_statvar
+),
+svo_percent AS (
+  SELECT
+    O.variable_measured AS statvar,
+    O.entity1 AS observation_about,
+    O.date AS observation_date,
+    O.value AS percent,
+    T.measurement_method AS measurement_method,
+    cdc_sv.pop_statvar
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation` AS O
+  JOIN `datcom-store.spanner_dc_graph_prod_DEFAULT.TimeSeries` AS T
+    ON O.variable_measured = T.variable_measured
+    AND O.entity1 = T.entity1
+    AND O.facet_id = T.facet_id
+  JOIN cdc_sv ON O.variable_measured = cdc_sv.cdc500
+  WHERE O.entity1 LIKE 'geoId/%'
+),
+svo_count AS (
+  SELECT
+    variable_measured AS population_statvar,
+    entity1 AS observation_about,
+    date AS observation_date,
+    value AS population
+  FROM `datcom-store.spanner_dc_graph_prod_DEFAULT.Observation`
+  WHERE entity1 LIKE 'geoId/%'
+    AND variable_measured IN (SELECT DISTINCT pop_statvar FROM cdc_sv)
+)
+SELECT DISTINCT * FROM (
+  SELECT 
+    statvar, 
+    SUBSTR(observation_about, 0, 8) AS observation_about,
+    observation_date, 
+    CONCAT('dcAggregate/', measurement_method) AS measurement_method,
+    population_statvar,
+    SUM(CAST(pop_count AS FLOAT64)) * 100 / SUM(CAST(population AS FLOAT64)) AS percent
+  FROM (
     SELECT
-      SVO.variable_measured as CDC500,
-      CONCAT('Count_', REGEXP_SUBSTR(SVO.variable_measured, '(Person_.*ale|Person_.*Years|Person)')) as pop_statvar
-      FROM `datcom-store.dc_kg_latest.StatVarObservation` as SVO
-    WHERE
-      SVO.prov_id = 'dc/base/CDC500'
-      AND SVO.variable_measured like 'Percent_%'
-      GROUP BY CDC500, pop_statvar
-   ) AS CDC_SV ON TRUE
-  WHERE
-    SVO1.prov_id = 'dc/base/CDC500'
-    AND SVO1.variable_measured LIKE 'Percent%'
-    AND SVO1.observation_about = SVO2.observation_about
-    AND SVO1.observation_date = SVO2.observation_date
-    AND SVO1.variable_measured = CDC_SV.CDC500
-    AND SVO2.variable_measured = CDC_SV.pop_statvar
-    AND SVO1.observation_about like "geoId/%"
-) group by 1,2,3,4,5
+      p.statvar,
+      p.observation_about,
+      p.observation_date,
+      p.percent,
+      p.measurement_method,
+      c.population_statvar,
+      c.population,
+      CAST(c.population AS FLOAT64) * CAST(p.percent AS FLOAT64) / 100 AS pop_count
+    FROM svo_percent AS p
+    JOIN svo_count AS c
+      ON p.observation_about = c.observation_about
+      AND p.observation_date = c.observation_date
+      AND p.pop_statvar = c.population_statvar
+  )
+  GROUP BY 1, 2, 3, 4, 5
 )
 """
 

From ebc79b75c1a48b9163aa7f0d88c16a43275e0609 Mon Sep 17 00:00:00 2001
From: shvngisingh <shvngisingh@google.com>
Date: Mon, 10 Aug 2026 17:16:40 +0530
Subject: [PATCH 2/2] Update scripts/us_cdc/cdc500_state/process.py

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 scripts/us_cdc/cdc500_state/process.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/us_cdc/cdc500_state/process.py b/scripts/us_cdc/cdc500_state/process.py
index ea9b2ecbe7..63c8e25bda 100644
--- a/scripts/us_cdc/cdc500_state/process.py
+++ b/scripts/us_cdc/cdc500_state/process.py
@@ -64,7 +64,7 @@
     observation_date, 
     CONCAT('dcAggregate/', measurement_method) AS measurement_method,
     population_statvar,
-    SUM(CAST(pop_count AS FLOAT64)) * 100 / SUM(CAST(population AS FLOAT64)) AS percent
+    SAFE_DIVIDE(SUM(CAST(pop_count AS FLOAT64)) * 100, SUM(CAST(population AS FLOAT64))) AS percent
   FROM (
     SELECT
       p.statvar,