From 6efe60c7884b75137980ca364f5395106619496e Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Thu, 6 Aug 2026 11:32:27 +0000 Subject: [PATCH 1/3] added download.py --- .../Death_Data_Cross_Tabulation_pvmap.csv | 533 +++++++++--------- statvar_imports/nyu_diabetes/texas/README.md | 49 +- .../nyu_diabetes/texas/download.py | 147 +++++ .../nyu_diabetes/texas/manifest.json | 12 +- 4 files changed, 439 insertions(+), 302 deletions(-) create mode 100644 statvar_imports/nyu_diabetes/texas/download.py diff --git a/statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv b/statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv index 1cce206232..6f6491510c 100644 --- a/statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv +++ b/statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv @@ -1,275 +1,258 @@ -key,prop,val,, -Diabetes mellitus,causeOfDeath,DiabetesMellitus,populationType,MortalityEvent -2011,observationDate,2011,value,{Number} -2012,observationDate,2012,value,{Number} -2013,observationDate,2013,value,{Number} -2014,observationDate,2014,value,{Number} -2015,observationDate,2015,value,{Number} -2016,observationDate,2016,value,{Number} -2017,observationDate,2017,value,{Number} -2018,observationDate,2018,value,{Number} -2019,observationDate,2019,value,{Number} -2020,observationDate,2020,value,{Number} -2021,observationDate,2021,value,{Number} -2022,observationDate,2022,value,{Number} -2023,observationDate,2023,value,{Number} -2024,observationDate,2024,value,{Number} -2025,observationDate,2025,value,{Number} -2026,observationDate,2026,value,{Number} -2027,observationDate,2027,value,{Number} -2028,observationDate,2028,value,{Number} -Anderson,observationAbout,geoId/48001,, -Andrews,observationAbout,geoId/48003,, -Angelina,observationAbout,geoId/48005,, -Aransas,observationAbout,geoId/48007,, -Archer,observationAbout,geoId/48009,, -Armstrong,observationAbout,geoId/48011,, -Atascosa,observationAbout,geoId/48013,, -Austin,observationAbout,geoId/48015,, -Bailey,observationAbout,geoId/48017,, -Bandera,observationAbout,geoId/48019,, -Bastrop,observationAbout,geoId/48021,, -Baylor,observationAbout,geoId/48023,, -Bee,observationAbout,geoId/48025,, -Bell,observationAbout,geoId/48027,, -Bexar,observationAbout,geoId/48029,, -Blanco,observationAbout,geoId/48031,, -Borden,observationAbout,geoId/48033,, -Bosque,observationAbout,geoId/48035,, -Bowie,observationAbout,geoId/48037,, -Brazoria,observationAbout,geoId/48039,, -Brazos,observationAbout,geoId/48041,, -Brewster,observationAbout,geoId/48043,, -Briscoe,observationAbout,geoId/48045,, -Brooks,observationAbout,geoId/48047,, -Brown,observationAbout,geoId/48049,, -Burleson,observationAbout,geoId/48051,, -Burnet,observationAbout,geoId/48053,, -Caldwell,observationAbout,geoId/48055,, -Calhoun,observationAbout,geoId/48057,, -Callahan,observationAbout,geoId/48059,, -Cameron,observationAbout,geoId/48061,, -Camp,observationAbout,geoId/48063,, -Carson,observationAbout,geoId/48065,, -Cass,observationAbout,geoId/48067,, -Castro,observationAbout,geoId/48069,, -Chambers,observationAbout,geoId/48071,, -Cherokee,observationAbout,geoId/48073,, -Childress,observationAbout,geoId/48075,, -Clay,observationAbout,geoId/48077,, -Cochran,observationAbout,geoId/48079,, -Coke,observationAbout,geoId/48081,, -Coleman,observationAbout,geoId/48083,, -Collin,observationAbout,geoId/48085,, -Collingsworth,observationAbout,geoId/48087,, -Colorado,observationAbout,geoId/48089,, -Comal,observationAbout,geoId/48091,, -Comanche,observationAbout,geoId/48093,, -Concho,observationAbout,geoId/48095,, -Cooke,observationAbout,geoId/48097,, -Coryell,observationAbout,geoId/48099,, -Cottle,observationAbout,geoId/48101,, -Crane,observationAbout,geoId/48103,, -Crockett,observationAbout,geoId/48105,, -Crosby,observationAbout,geoId/48107,, -Culberson,observationAbout,geoId/48109,, -Dallam,observationAbout,geoId/48111,, -Dallas,observationAbout,geoId/48113,, -Dawson,observationAbout,geoId/48115,, -Deaf Smith,observationAbout,geoId/48117,, -Delta,observationAbout,geoId/48119,, -Denton,observationAbout,geoId/48121,, -Dewitt,observationAbout,geoId/48123,, -Dickens,observationAbout,geoId/48125,, -Dimmit,observationAbout,geoId/48127,, -Donley,observationAbout,geoId/48129,, -Duval,observationAbout,geoId/48131,, -Eastland,observationAbout,geoId/48133,, -Ector,observationAbout,geoId/48135,, -Edwards,observationAbout,geoId/48137,, -El Paso,observationAbout,geoId/48139,, -Ellis,observationAbout,geoId/48141,, -Erath,observationAbout,geoId/48143,, -Falls,observationAbout,geoId/48145,, -Fannin,observationAbout,geoId/48147,, -Fayette,observationAbout,geoId/48149,, -Fisher,observationAbout,geoId/48151,, -Floyd,observationAbout,geoId/48153,, -Foard,observationAbout,geoId/48155,, -Fort Bend,observationAbout,geoId/48157,, -Franklin,observationAbout,geoId/48159,, -Freestone,observationAbout,geoId/48161,, -Frio,observationAbout,geoId/48163,, -Gaines,observationAbout,geoId/48165,, -Galveston,observationAbout,geoId/48167,, -Garza,observationAbout,geoId/48169,, -Gillespie,observationAbout,geoId/48171,, -Glasscock,observationAbout,geoId/48173,, -Goliad,observationAbout,geoId/48175,, -Gonzales,observationAbout,geoId/48177,, -Gray,observationAbout,geoId/48179,, -Grayson,observationAbout,geoId/48181,, -Gregg,observationAbout,geoId/48183,, -Grimes,observationAbout,geoId/48185,, -Guadalupe,observationAbout,geoId/48187,, -Hale,observationAbout,geoId/48189,, -Hall,observationAbout,geoId/48191,, -Hamilton,observationAbout,geoId/48193,, -Hansford,observationAbout,geoId/48195,, -Hardeman,observationAbout,geoId/48197,, -Hardin,observationAbout,geoId/48199,, -Harris,observationAbout,geoId/48201,, -Harrison,observationAbout,geoId/48203,, -Hartley,observationAbout,geoId/48205,, -Haskell,observationAbout,geoId/48207,, -Hays,observationAbout,geoId/48209,, -Hemphill,observationAbout,geoId/48211,, -Henderson,observationAbout,geoId/48213,, -Hidalgo,observationAbout,geoId/48215,, -Hill,observationAbout,geoId/48217,, -Hockley,observationAbout,geoId/48219,, -Hood,observationAbout,geoId/48221,, -Hopkins,observationAbout,geoId/48223,, -Houston,observationAbout,geoId/48225,, -Howard,observationAbout,geoId/48227,, -Hudspeth,observationAbout,geoId/48229,, -Hunt,observationAbout,geoId/48231,, -Hutchinson,observationAbout,geoId/48233,, -Irion,observationAbout,geoId/48235,, -Jack,observationAbout,geoId/48237,, -Jackson,observationAbout,geoId/48239,, -Jasper,observationAbout,geoId/48241,, -Jeff Davis,observationAbout,geoId/48243,, -Jefferson,observationAbout,geoId/48245,, -Jim Hogg,observationAbout,geoId/48247,, -Jim Wells,observationAbout,geoId/48249,, -Johnson,observationAbout,geoId/48251,, -Jones,observationAbout,geoId/48253,, -Karnes,observationAbout,geoId/48255,, -Kaufman,observationAbout,geoId/48257,, -Kendall,observationAbout,geoId/48259,, -Kenedy,observationAbout,geoId/48261,, -Kent,observationAbout,geoId/48263,, -Kerr,observationAbout,geoId/48265,, -Kimble,observationAbout,geoId/48267,, -King,observationAbout,geoId/48269,, -Kinney,observationAbout,geoId/48271,, -Kleberg,observationAbout,geoId/48273,, -Knox,observationAbout,geoId/48275,, -La Salle,observationAbout,geoId/48277,, -Lamar,observationAbout,geoId/48279,, -Lamb,observationAbout,geoId/48281,, -Lampasas,observationAbout,geoId/48283,, -Lavaca,observationAbout,geoId/48285,, -Lee,observationAbout,geoId/48287,, -Leon,observationAbout,geoId/48289,, -Liberty,observationAbout,geoId/48291,, -Limestone,observationAbout,geoId/48293,, -Lipscomb,observationAbout,geoId/48295,, -Live Oak,observationAbout,geoId/48297,, -Llano,observationAbout,geoId/48299,, -Loving,observationAbout,geoId/48301,, -Lubbock,observationAbout,geoId/48303,, -Lynn,observationAbout,geoId/48305,, -Madison,observationAbout,geoId/48307,, -Marion,observationAbout,geoId/48309,, -Martin,observationAbout,geoId/48311,, -Mason,observationAbout,geoId/48313,, -Matagorda,observationAbout,geoId/48315,, -Maverick,observationAbout,geoId/48317,, -Mcculloch,observationAbout,geoId/48319,, -Mclennan,observationAbout,geoId/48321,, -Mcmullen,observationAbout,geoId/48323,, -Medina,observationAbout,geoId/48325,, -Menard,observationAbout,geoId/48327,, -Midland,observationAbout,geoId/48329,, -Milam,observationAbout,geoId/48331,, -Mills,observationAbout,geoId/48333,, -Mitchell,observationAbout,geoId/48335,, -Montague,observationAbout,geoId/48337,, -Montgomery,observationAbout,geoId/48339,, -Moore,observationAbout,geoId/48341,, -Morris,observationAbout,geoId/48343,, -Motley,observationAbout,geoId/48345,, -Nacogdoches,observationAbout,geoId/48347,, -Navarro,observationAbout,geoId/48349,, -Newton,observationAbout,geoId/48351,, -Nolan,observationAbout,geoId/48353,, -Nueces,observationAbout,geoId/48355,, -Ochiltree,observationAbout,geoId/48357,, -Oldham,observationAbout,geoId/48359,, -Orange,observationAbout,geoId/48361,, -Palo Pinto,observationAbout,geoId/48363,, -Panola,observationAbout,geoId/48365,, -Parker,observationAbout,geoId/48367,, -Parmer,observationAbout,geoId/48369,, -Pecos,observationAbout,geoId/48371,, -Polk,observationAbout,geoId/48373,, -Potter,observationAbout,geoId/48375,, -Presidio,observationAbout,geoId/48377,, -Rains,observationAbout,geoId/48379,, -Randall,observationAbout,geoId/48381,, -Reagan,observationAbout,geoId/48383,, -Real,observationAbout,geoId/48385,, -Red River,observationAbout,geoId/48387,, -Reeves,observationAbout,geoId/48389,, -Refugio,observationAbout,geoId/48391,, -Roberts,observationAbout,geoId/48393,, -Robertson,observationAbout,geoId/48395,, -Rockwall,observationAbout,geoId/48397,, -Runnels,observationAbout,geoId/48399,, -Rusk,observationAbout,geoId/48401,, -Sabine,observationAbout,geoId/48403,, -San Augustine,observationAbout,geoId/48405,, -San Jacinto,observationAbout,geoId/48407,, -San Patricio,observationAbout,geoId/48409,, -San Saba,observationAbout,geoId/48411,, -Schleicher,observationAbout,geoId/48413,, -Scurry,observationAbout,geoId/48415,, -Shackelford,observationAbout,geoId/48417,, -Shelby,observationAbout,geoId/48419,, -Sherman,observationAbout,geoId/48421,, -Smith,observationAbout,geoId/48423,, -Somervell,observationAbout,geoId/48425,, -Starr,observationAbout,geoId/48427,, -Stephens,observationAbout,geoId/48429,, -Sterling,observationAbout,geoId/48431,, -Stonewall,observationAbout,geoId/48433,, -Sutton,observationAbout,geoId/48435,, -Swisher,observationAbout,geoId/48437,, -Tarrant,observationAbout,geoId/48439,, -Taylor,observationAbout,geoId/48441,, -Terrell,observationAbout,geoId/48443,, -Terry,observationAbout,geoId/48445,, -Throckmorton,observationAbout,geoId/48447,, -Titus,observationAbout,geoId/48449,, -Tom Green,observationAbout,geoId/48451,, -Travis,observationAbout,geoId/48453,, -Trinity,observationAbout,geoId/48455,, -Tyler,observationAbout,geoId/48457,, -Upshur,observationAbout,geoId/48459,, -Upton,observationAbout,geoId/48461,, -Uvalde,observationAbout,geoId/48463,, -Val Verde,observationAbout,geoId/48465,, -Van Zandt,observationAbout,geoId/48467,, -Victoria,observationAbout,geoId/48469,, -Walker,observationAbout,geoId/48471,, -Waller,observationAbout,geoId/48473,, -Ward,observationAbout,geoId/48475,, -Washington,observationAbout,geoId/48477,, -Webb,observationAbout,geoId/48479,, -Wharton,observationAbout,geoId/48481,, -Wheeler,observationAbout,geoId/48483,, -Wichita,observationAbout,geoId/48485,, -Wilbarger,observationAbout,geoId/48487,, -Willacy,observationAbout,geoId/48489,, -Williamson,observationAbout,geoId/48491,, -Wilson,observationAbout,geoId/48493,, -Winkler,observationAbout,geoId/48495,, -Wise,observationAbout,geoId/48497,, -Wood,observationAbout,geoId/48499,, -Yoakum,observationAbout,geoId/48501,, -Young,observationAbout,geoId/48503,, -Zapata,observationAbout,geoId/48505,, -Zavala,observationAbout,geoId/48507,, - +key,p1,v1,p2,v2 +#Diabetes mellitus (E10-E14),causeOfDeath,DiabetesMellitus,populationType,MortalityEvent +Year,observationDate,{Year},#Eval,"Year = re.sub(r'\\D', '',Data)" +"Anderson County, TX",observationAbout,geoId/48001,, +"Andrews County, TX",observationAbout,geoId/48003,, +"Angelina County, TX",observationAbout,geoId/48005,, +"Aransas County, TX",observationAbout,geoId/48007,, +"Archer County, TX",observationAbout,geoId/48009,, +"Armstrong County, TX",observationAbout,geoId/48011,, +"Atascosa County, TX",observationAbout,geoId/48013,, +"Austin County, TX",observationAbout,geoId/48015,, +"Bailey County, TX",observationAbout,geoId/48017,, +"Bandera County, TX",observationAbout,geoId/48019,, +"Bastrop County, TX",observationAbout,geoId/48021,, +"Baylor County, TX",observationAbout,geoId/48023,, +"Bee County, TX",observationAbout,geoId/48025,, +"Bell County, TX",observationAbout,geoId/48027,, +"Bexar County, TX",observationAbout,geoId/48029,, +"Blanco County, TX",observationAbout,geoId/48031,, +"Borden County, TX",observationAbout,geoId/48033,, +"Bosque County, TX",observationAbout,geoId/48035,, +"Bowie County, TX",observationAbout,geoId/48037,, +"Brazoria County, TX",observationAbout,geoId/48039,, +"Brazos County, TX",observationAbout,geoId/48041,, +"Brewster County, TX",observationAbout,geoId/48043,, +"Briscoe County, TX",observationAbout,geoId/48045,, +"Brooks County, TX",observationAbout,geoId/48047,, +"Brown County, TX",observationAbout,geoId/48049,, +"Burleson County, TX",observationAbout,geoId/48051,, +"Burnet County, TX",observationAbout,geoId/48053,, +"Caldwell County, TX",observationAbout,geoId/48055,, +"Calhoun County, TX",observationAbout,geoId/48057,, +"Callahan County, TX",observationAbout,geoId/48059,, +"Cameron County, TX",observationAbout,geoId/48061,, +"Camp County, TX",observationAbout,geoId/48063,, +"Carson County, TX",observationAbout,geoId/48065,, +"Cass County, TX",observationAbout,geoId/48067,, +"Castro County, TX",observationAbout,geoId/48069,, +"Chambers County, TX",observationAbout,geoId/48071,, +"Cherokee County, TX",observationAbout,geoId/48073,, +"Childress County, TX",observationAbout,geoId/48075,, +"Clay County, TX",observationAbout,geoId/48077,, +"Cochran County, TX",observationAbout,geoId/48079,, +"Coke County, TX",observationAbout,geoId/48081,, +"Coleman County, TX",observationAbout,geoId/48083,, +"Collin County, TX",observationAbout,geoId/48085,, +"Collingsworth County, TX",observationAbout,geoId/48087,, +"Colorado County, TX",observationAbout,geoId/48089,, +"Comal County, TX",observationAbout,geoId/48091,, +"Comanche County, TX",observationAbout,geoId/48093,, +"Concho County, TX",observationAbout,geoId/48095,, +"Cooke County, TX",observationAbout,geoId/48097,, +"Coryell County, TX",observationAbout,geoId/48099,, +"Cottle County, TX",observationAbout,geoId/48101,, +"Crane County, TX",observationAbout,geoId/48103,, +"Crockett County, TX",observationAbout,geoId/48105,, +"Crosby County, TX",observationAbout,geoId/48107,, +"Culberson County, TX",observationAbout,geoId/48109,, +"Dallam County, TX",observationAbout,geoId/48111,, +"Dallas County, TX",observationAbout,geoId/48113,, +"Dawson County, TX",observationAbout,geoId/48115,, +"Deaf Smith County, TX",observationAbout,geoId/48117,, +"Delta County, TX",observationAbout,geoId/48119,, +"Denton County, TX",observationAbout,geoId/48121,, +"DeWitt County, TX",observationAbout,geoId/48123,, +"Dickens County, TX",observationAbout,geoId/48125,, +"Dimmit County, TX",observationAbout,geoId/48127,, +"Donley County, TX",observationAbout,geoId/48129,, +"Duval County, TX",observationAbout,geoId/48131,, +"Eastland County, TX",observationAbout,geoId/48133,, +"Ector County, TX",observationAbout,geoId/48135,, +"Edwards County, TX",observationAbout,geoId/48137,, +"El Paso County, TX",observationAbout,geoId/48139,, +"Ellis County, TX",observationAbout,geoId/48141,, +"Erath County, TX",observationAbout,geoId/48143,, +"Falls County, TX",observationAbout,geoId/48145,, +"Fannin County, TX",observationAbout,geoId/48147,, +"Fayette County, TX",observationAbout,geoId/48149,, +"Fisher County, TX",observationAbout,geoId/48151,, +"Floyd County, TX",observationAbout,geoId/48153,, +"Foard County, TX",observationAbout,geoId/48155,, +"Fort Bend County, TX",observationAbout,geoId/48157,, +"Franklin County, TX",observationAbout,geoId/48159,, +"Freestone County, TX",observationAbout,geoId/48161,, +"Frio County, TX",observationAbout,geoId/48163,, +"Gaines County, TX",observationAbout,geoId/48165,, +"Galveston County, TX",observationAbout,geoId/48167,, +"Garza County, TX",observationAbout,geoId/48169,, +"Gillespie County, TX",observationAbout,geoId/48171,, +"Glasscock County, TX",observationAbout,geoId/48173,, +"Goliad County, TX",observationAbout,geoId/48175,, +"Gonzales County, TX",observationAbout,geoId/48177,, +"Gray County, TX",observationAbout,geoId/48179,, +"Grayson County, TX",observationAbout,geoId/48181,, +"Gregg County, TX",observationAbout,geoId/48183,, +"Grimes County, TX",observationAbout,geoId/48185,, +"Guadalupe County, TX",observationAbout,geoId/48187,, +"Hale County, TX",observationAbout,geoId/48189,, +"Hall County, TX",observationAbout,geoId/48191,, +"Hamilton County, TX",observationAbout,geoId/48193,, +"Hansford County, TX",observationAbout,geoId/48195,, +"Hardeman County, TX",observationAbout,geoId/48197,, +"Hardin County, TX",observationAbout,geoId/48199,, +"Harris County, TX",observationAbout,geoId/48201,, +"Harrison County, TX",observationAbout,geoId/48203,, +"Hartley County, TX",observationAbout,geoId/48205,, +"Haskell County, TX",observationAbout,geoId/48207,, +"Hays County, TX",observationAbout,geoId/48209,, +"Hemphill County, TX",observationAbout,geoId/48211,, +"Henderson County, TX",observationAbout,geoId/48213,, +"Hidalgo County, TX",observationAbout,geoId/48215,, +"Hill County, TX",observationAbout,geoId/48217,, +"Hockley County, TX",observationAbout,geoId/48219,, +"Hood County, TX",observationAbout,geoId/48221,, +"Hopkins County, TX",observationAbout,geoId/48223,, +"Houston County, TX",observationAbout,geoId/48225,, +"Howard County, TX",observationAbout,geoId/48227,, +"Hudspeth County, TX",observationAbout,geoId/48229,, +"Hunt County, TX",observationAbout,geoId/48231,, +"Hutchinson County, TX",observationAbout,geoId/48233,, +"Irion County, TX",observationAbout,geoId/48235,, +"Jack County, TX",observationAbout,geoId/48237,, +"Jackson County, TX",observationAbout,geoId/48239,, +"Jasper County, TX",observationAbout,geoId/48241,, +"Jeff Davis County, TX",observationAbout,geoId/48243,, +"Jefferson County, TX",observationAbout,geoId/48245,, +"Jim Hogg County, TX",observationAbout,geoId/48247,, +"Jim Wells County, TX",observationAbout,geoId/48249,, +"Johnson County, TX",observationAbout,geoId/48251,, +"Jones County, TX",observationAbout,geoId/48253,, +"Karnes County, TX",observationAbout,geoId/48255,, +"Kaufman County, TX",observationAbout,geoId/48257,, +"Kendall County, TX",observationAbout,geoId/48259,, +"Kenedy County, TX",observationAbout,geoId/48261,, +"Kent County, TX",observationAbout,geoId/48263,, +"Kerr County, TX",observationAbout,geoId/48265,, +"Kimble County, TX",observationAbout,geoId/48267,, +"King County, TX",observationAbout,geoId/48269,, +"Kinney County, TX",observationAbout,geoId/48271,, +"Kleberg County, TX",observationAbout,geoId/48273,, +"Knox County, TX",observationAbout,geoId/48275,, +"La Salle County, TX",observationAbout,geoId/48277,, +"Lamar County, TX",observationAbout,geoId/48279,, +"Lamb County, TX",observationAbout,geoId/48281,, +"Lampasas County, TX",observationAbout,geoId/48283,, +"Lavaca County, TX",observationAbout,geoId/48285,, +"Lee County, TX",observationAbout,geoId/48287,, +"Leon County, TX",observationAbout,geoId/48289,, +"Liberty County, TX",observationAbout,geoId/48291,, +"Limestone County, TX",observationAbout,geoId/48293,, +"Lipscomb County, TX",observationAbout,geoId/48295,, +"Live Oak County, TX",observationAbout,geoId/48297,, +"Llano County, TX",observationAbout,geoId/48299,, +"Loving County, TX",observationAbout,geoId/48301,, +"Lubbock County, TX",observationAbout,geoId/48303,, +"Lynn County, TX",observationAbout,geoId/48305,, +"Madison County, TX",observationAbout,geoId/48307,, +"Marion County, TX",observationAbout,geoId/48309,, +"Martin County, TX",observationAbout,geoId/48311,, +"Mason County, TX",observationAbout,geoId/48313,, +"Matagorda County, TX",observationAbout,geoId/48315,, +"Maverick County, TX",observationAbout,geoId/48317,, +"Mcculloch County, TX",observationAbout,geoId/48319,, +"McLennan County, TX",observationAbout,geoId/48321,, +"Mcmullen County, TX",observationAbout,geoId/48323,, +"Medina County, TX",observationAbout,geoId/48325,, +"Menard County, TX",observationAbout,geoId/48327,, +"Midland County, TX",observationAbout,geoId/48329,, +"Milam County, TX",observationAbout,geoId/48331,, +"Mills County, TX",observationAbout,geoId/48333,, +"Mitchell County, TX",observationAbout,geoId/48335,, +"Montague County, TX",observationAbout,geoId/48337,, +"Montgomery County, TX",observationAbout,geoId/48339,, +"Moore County, TX",observationAbout,geoId/48341,, +"Morris County, TX",observationAbout,geoId/48343,, +"Motley County, TX",observationAbout,geoId/48345,, +"Nacogdoches County, TX",observationAbout,geoId/48347,, +"Navarro County, TX",observationAbout,geoId/48349,, +"Newton County, TX",observationAbout,geoId/48351,, +"Nolan County, TX",observationAbout,geoId/48353,, +"Nueces County, TX",observationAbout,geoId/48355,, +"Ochiltree County, TX",observationAbout,geoId/48357,, +"Oldham County, TX",observationAbout,geoId/48359,, +"Orange County, TX",observationAbout,geoId/48361,, +"Palo Pinto County, TX",observationAbout,geoId/48363,, +"Panola County, TX",observationAbout,geoId/48365,, +"Parker County, TX",observationAbout,geoId/48367,, +"Parmer County, TX",observationAbout,geoId/48369,, +"Pecos County, TX",observationAbout,geoId/48371,, +"Polk County, TX",observationAbout,geoId/48373,, +"Potter County, TX",observationAbout,geoId/48375,, +"Presidio County, TX",observationAbout,geoId/48377,, +"Rains County, TX",observationAbout,geoId/48379,, +"Randall County, TX",observationAbout,geoId/48381,, +"Reagan County, TX",observationAbout,geoId/48383,, +"Real County, TX",observationAbout,geoId/48385,, +"Red River County, TX",observationAbout,geoId/48387,, +"Reeves County, TX",observationAbout,geoId/48389,, +"Refugio County, TX",observationAbout,geoId/48391,, +"Roberts County, TX",observationAbout,geoId/48393,, +"Robertson County, TX",observationAbout,geoId/48395,, +"Rockwall County, TX",observationAbout,geoId/48397,, +"Runnels County, TX",observationAbout,geoId/48399,, +"Rusk County, TX",observationAbout,geoId/48401,, +"Sabine County, TX",observationAbout,geoId/48403,, +"San Augustine County, TX",observationAbout,geoId/48405,, +"San Jacinto County, TX",observationAbout,geoId/48407,, +"San Patricio County, TX",observationAbout,geoId/48409,, +"San Saba County, TX",observationAbout,geoId/48411,, +"Schleicher County, TX",observationAbout,geoId/48413,, +"Scurry County, TX",observationAbout,geoId/48415,, +"Shackelford County, TX",observationAbout,geoId/48417,, +"Shelby County, TX",observationAbout,geoId/48419,, +"Sherman County, TX",observationAbout,geoId/48421,, +"Smith County, TX",observationAbout,geoId/48423,, +"Somervell County, TX",observationAbout,geoId/48425,, +"Starr County, TX",observationAbout,geoId/48427,, +"Stephens County, TX",observationAbout,geoId/48429,, +"Sterling County, TX",observationAbout,geoId/48431,, +"Stonewall County, TX",observationAbout,geoId/48433,, +"Sutton County, TX",observationAbout,geoId/48435,, +"Swisher County, TX",observationAbout,geoId/48437,, +"Tarrant County, TX",observationAbout,geoId/48439,, +"Taylor County, TX",observationAbout,geoId/48441,, +"Terrell County, TX",observationAbout,geoId/48443,, +"Terry County, TX",observationAbout,geoId/48445,, +"Throckmorton County, TX",observationAbout,geoId/48447,, +"Titus County, TX",observationAbout,geoId/48449,, +"Tom Green County, TX",observationAbout,geoId/48451,, +"Travis County, TX",observationAbout,geoId/48453,, +"Trinity County, TX",observationAbout,geoId/48455,, +"Tyler County, TX",observationAbout,geoId/48457,, +"Upshur County, TX",observationAbout,geoId/48459,, +"Upton County, TX",observationAbout,geoId/48461,, +"Uvalde County, TX",observationAbout,geoId/48463,, +"Val Verde County, TX",observationAbout,geoId/48465,, +"Van Zandt County, TX",observationAbout,geoId/48467,, +"Victoria County, TX",observationAbout,geoId/48469,, +"Walker County, TX",observationAbout,geoId/48471,, +"Waller County, TX",observationAbout,geoId/48473,, +"Ward County, TX",observationAbout,geoId/48475,, +"Washington County, TX",observationAbout,geoId/48477,, +"Webb County, TX",observationAbout,geoId/48479,, +"Wharton County, TX",observationAbout,geoId/48481,, +"Wheeler County, TX",observationAbout,geoId/48483,, +"Wichita County, TX",observationAbout,geoId/48485,, +"Wilbarger County, TX",observationAbout,geoId/48487,, +"Willacy County, TX",observationAbout,geoId/48489,, +"Williamson County, TX",observationAbout,geoId/48491,, +"Wilson County, TX",observationAbout,geoId/48493,, +"Winkler County, TX",observationAbout,geoId/48495,, +"Wise County, TX",observationAbout,geoId/48497,, +"Wood County, TX",observationAbout,geoId/48499,, +"Yoakum County, TX",observationAbout,geoId/48501,, +"Young County, TX",observationAbout,geoId/48503,, +"Zapata County, TX",observationAbout,geoId/48505,, +"Zavala County, TX",observationAbout,geoId/48507,, +Deaths,value,{Number},measuredProperty,count \ No newline at end of file diff --git a/statvar_imports/nyu_diabetes/texas/README.md b/statvar_imports/nyu_diabetes/texas/README.md index 1d6bb5f9c9..b8fc544708 100644 --- a/statvar_imports/nyu_diabetes/texas/README.md +++ b/statvar_imports/nyu_diabetes/texas/README.md @@ -22,32 +22,39 @@ This import utilizes official mortality data from the Texas Department of Health ### ⚙️ Workflow -The workflow for this data import involves one step: Processing file. +The workflow for this data import involves two main steps: downloading the necessary files and then processing them. -#### Step 1: Download the File -Select the following options from the website -1. Data Table Builder -2. Select a Year - All -3. Geographic Level -County -4. Geographic Area - All -5. Breakdown Category - Leading Cause of Death+ -6. Breakdown Details - Diabetes mellitus +#### Step 1: Download the Source Data + +To acquire the necessary data files, execute the download script `download.py`: + +```bash +python3 download.py +``` + +The script connects to the CDC WONDER saved query (`https://wonder.cdc.gov/controller/saved/D158/D516F171`), automates the session agreement, and downloads the county-level diabetes mortality dataset to `input_files/Death_Data_Cross_Tabulation_data.csv`. + +### Autorefresh type + +This import uses a fully automated refresh process. + +----- #### Step 2: Process the Files -Run the processing script to generate the final output artifacts. +After successfully downloading the files, run the processing script to generate the final output artifacts. #### **Execution Commands** -The final output is generated by processing the downloaded data using the stat_var_processor.py script, which is located in the data/tools/statvar_importer/ directory. - - ```bash - python3 stat_var_processor.py \ - --input_data=../../statvar_imports/nyu_diabetes/texas/input_files/Death_Data_Cross_Tabulation_data.csv \ - --pv_map=../../statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv \ - --config_file=../../statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_metadata.csv \ - --output_path=../../statvar_imports/nyu_diabetes/texas/output_files/Texas_Diabetes \ - --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ - --places_resolved_csv=place_resolver.csv - ``` +The final output is generated by processing the downloaded data using the `stat_var_processor.py` script located in the `data/tools/statvar_importer/` directory: + +```bash +python3 stat_var_processor.py \ +--input_data=../../statvar_imports/nyu_diabetes/texas/input_files/Death_Data_Cross_Tabulation_data.csv \ +--pv_map=../../statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_pvmap.csv \ +--config_file=../../statvar_imports/nyu_diabetes/texas/Death_Data_Cross_Tabulation_metadata.csv \ +--output_path=../../statvar_imports/nyu_diabetes/texas/output_files/texas_output \ +--existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ +--places_resolved_csv=place_resolver.csv +``` diff --git a/statvar_imports/nyu_diabetes/texas/download.py b/statvar_imports/nyu_diabetes/texas/download.py new file mode 100644 index 0000000000..6e74928b64 --- /dev/null +++ b/statvar_imports/nyu_diabetes/texas/download.py @@ -0,0 +1,147 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import io +import os +import sys +from urllib.parse import urljoin +from pathlib import Path +from absl import app, logging +from bs4 import BeautifulSoup +import requests +from retry import retry + +script_dir = os.path.dirname(os.path.abspath(__file__)) +INPUT_DIR = os.path.join(script_dir, "input_files") +OUTPUT_CSV = os.path.join(INPUT_DIR, "Death_Data_Cross_Tabulation_data.csv") + +SOURCE_URL = "https://wonder.cdc.gov/controller/saved/D158/D516F171" + + +@retry(tries=3, delay=5, backoff=2) +def download_cdc_wonder_data(saved_url: str) -> str: + """Automates CDC WONDER saved request session and downloads dataset as TSV/CSV text. + + Args: + saved_url: Saved request URL from CDC WONDER. + + Returns: + The raw response text (TSV formatted dataset). + """ + session = requests.Session() + + logging.info(f"Connecting to CDC WONDER saved query: {saved_url}") + res = session.get(saved_url, timeout=60) + res.raise_for_status() + + soup = BeautifulSoup(res.text, "lxml") + form = soup.find("form", id="wonderform") + if not form: + raise ValueError("Could not find initial wonderform on CDC WONDER page.") + + action = urljoin(saved_url, form.get("action")) + agree_inputs = [ + (inp.get("name"), inp.get("value", "")) + for inp in form.find_all("input") + if inp.get("name") + ] + agree_inputs.append(("action-I Agree", "I Agree")) + + logging.info("Submitting Data Use Agreement (I Agree)...") + res_agree = session.post(action, data=agree_inputs, timeout=60) + res_agree.raise_for_status() + + soup_req = BeautifulSoup(res_agree.text, "lxml") + form_req = soup_req.find("form", id="wonderform") + if not form_req: + raise ValueError("Could not find request form after agreeing to terms.") + + action_req = urljoin(saved_url, form_req.get("action")) + + # Extract all pre-populated query parameters from the form + post_data = [] + for el in form_req.find_all(["input", "select", "textarea"]): + name = el.get("name") + if not name: + continue + if el.name == "input": + itype = el.get("type", "text").lower() + if itype in ["submit", "button", "reset", "image"]: + continue + if itype in ["checkbox", "radio"]: + if el.has_attr("checked"): + post_data.append((name, el.get("value", "on"))) + else: + post_data.append((name, el.get("value", ""))) + elif el.name == "select": + selected_opts = [ + opt for opt in el.find_all("option") if opt.has_attr("selected") + ] + if selected_opts: + for opt in selected_opts: + post_data.append((name, opt.get("value", ""))) + else: + if not el.has_attr("multiple"): + first_opt = el.find("option") + if first_opt: + post_data.append((name, first_opt.get("value", ""))) + elif el.name == "textarea": + post_data.append((name, el.text or "")) + + # Submit query with action-Send + post_data.append(("action-Send", "Send")) + + logging.info("Submitting query request to CDC WONDER...") + res_data = session.post(action_req, data=post_data, timeout=120) + res_data.raise_for_status() + + return res_data.text + + +def save_tsv_as_csv(raw_text: str, output_filepath: str): + """Converts TSV text from CDC WONDER into CSV format and writes to disk. + + Args: + raw_text: TSV content string. + output_filepath: Target CSV file path. + """ + Path(os.path.dirname(output_filepath)).mkdir(parents=True, exist_ok=True) + tsv_reader = csv.reader(io.StringIO(raw_text), delimiter="\t") + + row_count = 0 + with open(output_filepath, "w", newline="", encoding="utf-8") as f: + csv_writer = csv.writer(f) + for row in tsv_reader: + if row: + csv_writer.writerow(row) + row_count += 1 + + logging.info(f"Successfully saved {row_count} rows to {output_filepath}") + + +def main(_): + logging.info("Starting CDC WONDER download process...") + raw_data = download_cdc_wonder_data(SOURCE_URL) + + if not raw_data or "County Code" not in raw_data: + logging.fatal("Downloaded data is empty or missing expected headers.") + sys.exit(1) + + save_tsv_as_csv(raw_data, OUTPUT_CSV) + logging.info(f"CDC WONDER download finished successfully: {OUTPUT_CSV}") + + +if __name__ == "__main__": + app.run(main) diff --git a/statvar_imports/nyu_diabetes/texas/manifest.json b/statvar_imports/nyu_diabetes/texas/manifest.json index a46cf39645..ab0daaf401 100644 --- a/statvar_imports/nyu_diabetes/texas/manifest.json +++ b/statvar_imports/nyu_diabetes/texas/manifest.json @@ -5,19 +5,19 @@ "curator_emails": [ "support@datacommons.org" ], - "provenance_url": "https://healthdata.dshs.texas.gov/dashboard/births-and-deaths/deaths", - "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.", + "provenance_url": "https://wonder.cdc.gov/controller/saved/D158/D516F171", + "provenance_description": "This import utilizes official diabetes mortality data from the Texas Department of Health, detailing county-level death statistics.", "scripts": [ "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=Death_Data_Cross_Tabulation_pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/texas_output" ], "source_files": [ - "input_files/*.xlsx" + "input_files/*.csv" ], "import_inputs": [ { - "template_mcf": "output_files/tennessee_output.tmcf", - "cleaned_csv": "output_files/tennessee_output.csv" + "template_mcf": "output_files/texas_output.tmcf", + "cleaned_csv": "output_files/texas_output.csv" } ], "cron_schedule": "0 05 * * 5" From 6ac7425cb330ed027c0b03f99f6659fd943aabe2 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Thu, 6 Aug 2026 13:25:55 +0000 Subject: [PATCH 2/3] additional changes --- statvar_imports/nyu_diabetes/texas/download.py | 2 +- statvar_imports/nyu_diabetes/texas/manifest.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/statvar_imports/nyu_diabetes/texas/download.py b/statvar_imports/nyu_diabetes/texas/download.py index 6e74928b64..039b716308 100644 --- a/statvar_imports/nyu_diabetes/texas/download.py +++ b/statvar_imports/nyu_diabetes/texas/download.py @@ -30,7 +30,7 @@ SOURCE_URL = "https://wonder.cdc.gov/controller/saved/D158/D516F171" -@retry(tries=3, delay=5, backoff=2) +@retry(tries=10, delay=5, backoff=2) def download_cdc_wonder_data(saved_url: str) -> str: """Automates CDC WONDER saved request session and downloads dataset as TSV/CSV text. diff --git a/statvar_imports/nyu_diabetes/texas/manifest.json b/statvar_imports/nyu_diabetes/texas/manifest.json index ab0daaf401..e43517820f 100644 --- a/statvar_imports/nyu_diabetes/texas/manifest.json +++ b/statvar_imports/nyu_diabetes/texas/manifest.json @@ -6,18 +6,18 @@ "support@datacommons.org" ], "provenance_url": "https://wonder.cdc.gov/controller/saved/D158/D516F171", - "provenance_description": "This import utilizes official diabetes mortality data from the Texas Department of Health, detailing county-level death statistics.", + "provenance_description": "This import utilizes official diabetes mortality data from CDC WONDER, detailing county-level death statistics.", "scripts": [ "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=Death_Data_Cross_Tabulation_pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/texas_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=Death_Data_Cross_Tabulation_pvmap.csv --config_file=Death_Data_Cross_Tabulation_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/Texas_Diabetes" ], "source_files": [ "input_files/*.csv" ], "import_inputs": [ { - "template_mcf": "output_files/texas_output.tmcf", - "cleaned_csv": "output_files/texas_output.csv" + "template_mcf": "output_files/Texas_Diabetes.tmcf", + "cleaned_csv": "output_files/Texas_Diabetes.csv" } ], "cron_schedule": "0 05 * * 5" From 6fbbddc4080b74df5da71cb72c1244b67d1f746e Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 7 Aug 2026 07:59:02 +0000 Subject: [PATCH 3/3] added counters --- statvar_imports/nyu_diabetes/texas/manifest.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/statvar_imports/nyu_diabetes/texas/manifest.json b/statvar_imports/nyu_diabetes/texas/manifest.json index e43517820f..eab70691cf 100644 --- a/statvar_imports/nyu_diabetes/texas/manifest.json +++ b/statvar_imports/nyu_diabetes/texas/manifest.json @@ -9,10 +9,11 @@ "provenance_description": "This import utilizes official diabetes mortality data from CDC WONDER, detailing county-level death statistics.", "scripts": [ "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=Death_Data_Cross_Tabulation_pvmap.csv --config_file=Death_Data_Cross_Tabulation_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/Texas_Diabetes" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=Death_Data_Cross_Tabulation_pvmap.csv --config_file=Death_Data_Cross_Tabulation_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/Texas_Diabetes --output_counters=counters/Texas_Diabetes_counters.csv" ], "source_files": [ - "input_files/*.csv" + "input_files/*.csv", + "counters/*.csv" ], "import_inputs": [ {