diff --git a/README.md b/README.md index 8134d2b..6f6162b 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ No application code changes required. ## Catalog -The catalog currently contains 100 operational datasets spanning natural +The catalog currently contains 141 operational datasets spanning natural hazards, weather, water, climate, flood risk, drought, space weather, global disaster alerts, public health, clinical research, cybersecurity, package graphs, legislation, sanctions screening, government spending and procurement, @@ -158,8 +158,10 @@ broadband, bridges, EV charging, consumer finance, education, K-12 directories, housing prices, rents, food and product recalls, elections, European statistics, live transit feeds, aviation, provider directories, drinking water, preprints, pageviews, agriculture, tropical cyclones, crime, companies, occupations, -global forecasts, live OSM, food products, nursing homes, LEI, and euro-area -statistics. The YAML files in +global forecasts, live OSM, food products, nursing homes, LEI, euro-area +statistics, patents, vital statistics, mortgages, OECD and national statistics, +Mauna Loa CO2, sea ice, caselaw, lobbying, Certificate Transparency, and +public GitHub activity. The YAML files in [`data/datasets`](data/datasets) are the source of truth for the current list. ## Deploy to Vercel diff --git a/data/datasets/cdc-uscs-cancer-statistics.yaml b/data/datasets/cdc-uscs-cancer-statistics.yaml new file mode 100644 index 0000000..3f1f64f --- /dev/null +++ b/data/datasets/cdc-uscs-cancer-statistics.yaml @@ -0,0 +1,79 @@ +id: cdc-uscs-cancer-statistics +name: U.S. Cancer Statistics +description: > + Official federal cancer incidence and mortality statistics for building + state comparison and cancer-burden dashboards. +theme: Health, Food & Safety +url: https://www.cdc.gov/united-states-cancer-statistics/index.html +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - CSV + - JSON + - Excel +license: U.S. Public Domain with CDC attribution and use requirements +license_url: https://www.cdc.gov/other/agencymaterials.html +url_checks: + source_marker: United States Cancer Statistics + license_marker: public domain +domains: + - Public Health + - Epidemiology +data_types: + - Tabular + - Aggregated Data +tasks: + - Incidence Comparison + - Trend Analysis + - Public Health Planning +difficulty: intermediate +geography: + - United States +temporal_coverage: USCS incidence and mortality years published by CDC and NCI +update_frequency: annual +provider: Centers for Disease Control and Prevention +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + USCS is the official federal combination of NPCR and SEER incidence with + NVSS mortality. Start with a public state-level incidence table, not the + SEER*Stat research file. Rates are age-adjusted to a standard population + and are not individual case listings. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the USCS page and locate Data Visualizations or public-use tables. + - Download or query one recent state incidence file. + - Keep state, cancer site, year, and age-adjusted rate. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.cdc.gov/resource/5kmb-vjq2.json", + params={"$limit": 50, "$order": "year DESC"}, + timeout=30, + ) + response.raise_for_status() + stats = pd.DataFrame(response.json()) + stats["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(stats.head()) + first_project: + title: Compare state cancer incidence rows + goal: Test whether a USCS public table can power a bounded state-burden sketch. + steps: + - Keep geography, site, year, and age-adjusted rate from the extract. + - Rank a single site-year by rate and flag missing geographies. + - Explain that USCS public tables are aggregated rates, not SEER research microdata, and cannot identify individuals. diff --git a/data/datasets/census-lehd-lodes.yaml b/data/datasets/census-lehd-lodes.yaml new file mode 100644 index 0000000..03f158e --- /dev/null +++ b/data/datasets/census-lehd-lodes.yaml @@ -0,0 +1,73 @@ +id: census-lehd-lodes +name: Census LEHD Origin-Destination Employment Statistics +description: > + Block-level origin-destination jobs data for building commute, workforce, and + local employment-map prototypes. +theme: Geospatial & Infrastructure +url: https://lehd.ces.census.gov/data/ +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 50 +formats: + - CSV + - GZIP +license: U.S. Census Bureau data-use terms +license_url: https://www.census.gov/data/developers/about/terms-of-service.html +url_checks: + source_marker: LEHD Origin-Destination Employment Statistics + license_marker: Terms of Service +domains: + - Labor Economics + - Local Economics + - Transportation +data_types: + - Tabular + - Geospatial + - Origin-Destination +tasks: + - Commute Analysis + - Workforce Mapping + - Site Selection +difficulty: intermediate +geography: + - United States +temporal_coverage: annual LODES vintages with state-specific start years +update_frequency: annual +provider: U.S. Census Bureau +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + LODES Origin-Destination files count jobs connecting home and work census + blocks. Start with one small state's OD main file. Counts are synthesised + from administrative records, some states are missing in some years, and + block-to-block files are large. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the LEHD data page and choose LODES OD for one small state. + - Download the JT00 OD main gzip for the latest vintage listed. + - Keep workplace block, residence block, and total jobs. + python: + packages: + - pandas + code: | + import pandas as pd + + jobs = pd.read_csv( + "https://lehd.ces.census.gov/data/lodes/LODES8/vt/od/vt_od_main_JT00_2022.csv.gz", + nrows=200, + ) + print(jobs.head()) + first_project: + title: Inspect Vermont LODES job flows + goal: Test whether a one-state OD extract can power a bounded commute sketch. + steps: + - Keep workplace geocode, residence geocode, and total jobs from the first rows. + - Sum jobs by workplace census tract prefix and list empty geocodes. + - Explain that LODES is synthesised administrative data, some states are omitted, and a sample of rows is not a statewide commute census. diff --git a/data/datasets/certificate-transparency-crtsh.yaml b/data/datasets/certificate-transparency-crtsh.yaml new file mode 100644 index 0000000..572bbeb --- /dev/null +++ b/data/datasets/certificate-transparency-crtsh.yaml @@ -0,0 +1,77 @@ +id: certificate-transparency-crtsh +name: Certificate Transparency Search +description: > + Issued TLS certificates indexed from public CT logs for building domain + certificate monitors and issuance alerts. +theme: Technology & Cybersecurity +url: https://certificate.transparency.dev/ +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - JSON +license: Certificate Transparency log public data / crt.sh use +license_url: https://certificate.transparency.dev/ +url_checks: + source_marker: Certificate Transparency + license_marker: Certificate Transparency +domains: + - Cybersecurity + - Internet Infrastructure +data_types: + - Event Data + - Registry Data + - Tabular +tasks: + - Certificate Monitoring + - Alerting + - Asset Discovery +difficulty: beginner +geography: + - Global +temporal_coverage: certificates submitted to participating CT logs +update_frequency: continuous +provider: Sectigo crt.sh +source_type: company +last_verified: 2026-08-18 +getting_started: + overview: > + crt.sh queries public Certificate Transparency logs. Start with one + registered domain and a JSON search. CT logs are the origin; crt.sh is the + query surface. Logs can lag, expired certificates remain, and this is not a + complete inventory of every private PKI. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A domain you are authorized to inspect + access_steps: + - Open crt.sh and try one Identity search in the browser. + - Request JSON for that domain with output=json. + - Keep issuer name, common name, and not-after date. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://crt.sh/", + params={"q": "example.com", "output": "json"}, + timeout=60, + ) + response.raise_for_status() + certificates = pd.DataFrame(response.json()) + certificates["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(certificates[["issuer_name", "common_name", "not_after"]].head()) + first_project: + title: List certificates for one domain + goal: Test whether a CT search can power a bounded issuance monitor. + steps: + - Deduplicate certificate ids and keep issuer, common name, and expiry. + - Flag certificates whose not-after date is in the past. + - Explain that CT coverage depends on participating logs and is not a private-PKI inventory. diff --git a/data/datasets/chrome-ux-report.yaml b/data/datasets/chrome-ux-report.yaml new file mode 100644 index 0000000..71ecd88 --- /dev/null +++ b/data/datasets/chrome-ux-report.yaml @@ -0,0 +1,76 @@ +id: chrome-ux-report +name: Chrome UX Report API +description: > + Real-user Chrome experience metrics for building Core Web Vitals monitors + from origin-level CrUX records. +theme: Technology & Cybersecurity +url: https://developer.chrome.com/docs/crux +access_type: + - api +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.01 +formats: + - JSON +license: Google APIs Terms of Service +license_url: https://developers.google.com/terms +url_checks: + source_marker: Chrome UX Report + license_marker: Google APIs Terms of Service +domains: + - Web Performance + - Software +data_types: + - Aggregated Data + - Time Series +tasks: + - Performance Monitoring + - Origin Comparison + - Alerting +difficulty: beginner +geography: + - Global +temporal_coverage: rolling 28-day CrUX collection windows +update_frequency: daily +provider: Google +source_type: company +last_verified: 2026-08-18 +getting_started: + overview: > + The CrUX API returns origin-level p75 metrics such as LCP and INP. Start + with one origin you are authorized to monitor. CrUX covers opted-in Chrome + users, not synthetic tests, and small origins can be missing. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A Google API key saved in the CRUX_API_KEY environment variable + access_steps: + - Enable the Chrome UX Report API in Google Cloud and save CRUX_API_KEY. + - Read the query endpoint documentation. + - Request metrics for one HTTPS origin. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.post( + "https://chromeuxreport.googleapis.com/v1/records:queryRecord", + params={"key": os.environ["CRUX_API_KEY"]}, + json={"origin": "https://www.example.com"}, + timeout=30, + ) + response.raise_for_status() + record = pd.json_normalize(response.json()) + print(record) + first_project: + title: Read one origin's Core Web Vitals + goal: Test whether CrUX can power a bounded real-user performance card. + steps: + - Confirm p75 LCP, INP, and CLS fields returned for the origin. + - Compare those p75 values to the published "good" thresholds. + - Explain that CrUX is opted-in Chrome field data and that small origins can be absent. diff --git a/data/datasets/copernicus-era5.yaml b/data/datasets/copernicus-era5.yaml new file mode 100644 index 0000000..c9a4130 --- /dev/null +++ b/data/datasets/copernicus-era5.yaml @@ -0,0 +1,92 @@ +id: copernicus-era5 +name: Copernicus ERA5 Reanalysis +description: > + Global hourly climate reanalysis for building location-specific weather-history + tools from a tightly bounded ERA5 extract. +theme: Environment & Hazards +url: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 50 +formats: + - NetCDF + - GRIB +license: Licence to use Copernicus Products +license_url: https://cds.climate.copernicus.eu/licences/licence-to-use-copernicus-products +url_checks: + source_marker: ERA5 hourly data on single levels + license_marker: Licence to use Copernicus Products +domains: + - Climate + - Weather + - Environmental Science +data_types: + - Gridded Data + - Time Series +tasks: + - Climate Analysis + - Historical Weather Reconstruction + - Site Screening +difficulty: advanced +geography: + - Global +temporal_coverage: 1940-present +update_frequency: daily +provider: Copernicus Climate Change Service +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + ERA5 is a global reanalysis on the Climate Data Store. Start with one + variable, one hour, and a few-kilometer bounding box. Reanalysis blends + observations and a model, so it is not a station measurement, and CDS + jobs are queued rather than instant. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A CDS account, accepted ERA5 licence, and CDSAPI_KEY environment variable + access_steps: + - Register at the Climate Data Store and accept the ERA5 dataset licence on the download form. + - Save your personal access token in CDSAPI_KEY. + - Retrieve 2-metre temperature for one hour over a tiny bounding box as NetCDF. + python: + packages: + - cdsapi + - xarray + - netCDF4 + code: | + import os + import cdsapi + import xarray as xr + + client = cdsapi.Client( + url="https://cds.climate.copernicus.eu/api", + key=os.environ["CDSAPI_KEY"], + ) + client.retrieve( + "reanalysis-era5-single-levels", + { + "product_type": "reanalysis", + "variable": "2m_temperature", + "year": "2024", + "month": "07", + "day": "15", + "time": "12:00", + "area": [40.8, -74.05, 40.7, -73.95], + "format": "netcdf", + }, + "era5_nyc_sample.nc", + ) + sample = xr.open_dataset("era5_nyc_sample.nc") + print(sample) + first_project: + title: Reconstruct one hour of 2-metre temperature + goal: Test whether a tiny ERA5 extract can power a site-history prototype. + steps: + - Confirm the NetCDF contains 2-metre temperature for the requested hour and bounding box. + - Convert temperature from kelvin to Celsius and record the grid-cell center used. + - Explain that ERA5 is a reanalysis grid, not a weather-station observation, and that queued CDS jobs can delay retrieval. diff --git a/data/datasets/dailymed-drug-labels.yaml b/data/datasets/dailymed-drug-labels.yaml new file mode 100644 index 0000000..e4e1abb --- /dev/null +++ b/data/datasets/dailymed-drug-labels.yaml @@ -0,0 +1,81 @@ +id: dailymed-drug-labels +name: DailyMed Drug Labels +description: > + Structured FDA drug-label (SPL) documents for building label lookup, + section-search, and medication-reference tools. +theme: Health, Food & Safety +url: https://dailymed.nlm.nih.gov/dailymed/app-support-web-services.cfm +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 5 +formats: + - JSON + - XML + - ZIP +license: NLM copyright and data-use terms +license_url: https://www.nlm.nih.gov/copyright.html +url_checks: + source_marker: DailyMed + license_marker: NLM Copyright Information +domains: + - Pharmaceuticals + - Health Care + - Regulation +data_types: + - Documents + - Text + - Registry Data +tasks: + - Label Lookup + - Section Search + - Reference Building +difficulty: beginner +geography: + - United States +temporal_coverage: currently published SPL labels with historical versions +update_frequency: daily +provider: U.S. National Library of Medicine +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + DailyMed REST returns Structured Product Labeling metadata and sections. + Start with one drug name search and a single SET ID. Labels are regulatory + documents, not a complete picture of off-label use, and they can lag + manufacturer updates. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the DailyMed web-services help page. + - Search drug names for one generic and request the first page. + - Keep SET ID, title, and last-updated date. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://dailymed.nlm.nih.gov/dailymed/services/v2/spls.json", + params={"drug_name": "ibuprofen", "pagesize": 20, "page": 1}, + timeout=30, + ) + response.raise_for_status() + labels = pd.DataFrame(response.json()["data"]) + labels["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(labels.head()) + first_project: + title: List ibuprofen SPL titles + goal: Test whether DailyMed search can power a bounded label-lookup inbox. + steps: + - Keep SET ID, title, and published date from the first page. + - Deduplicate SET IDs and list titles that look like different dosage forms. + - Explain that SPL is the currently published label text and is not a complete clinical-use database. diff --git a/data/datasets/eia-weekly-natural-gas.yaml b/data/datasets/eia-weekly-natural-gas.yaml new file mode 100644 index 0000000..bc1fab0 --- /dev/null +++ b/data/datasets/eia-weekly-natural-gas.yaml @@ -0,0 +1,87 @@ +id: eia-weekly-natural-gas +name: EIA Weekly Natural Gas Storage +description: > + Weekly U.S. working-gas in storage for building inventory monitors and + energy-market dashboards. +theme: Markets & Economics +url: https://www.eia.gov/opendata/index.php/api +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV +license: U.S. Public Domain with EIA source acknowledgment +license_url: https://www.eia.gov/about/copyrights_reuse.php +url_checks: + source_marker: OPEN DATA + license_marker: Public domain and use of EIA content +domains: + - Energy + - Commodities + - Macroeconomics +data_types: + - Time Series + - Tabular +tasks: + - Market Monitoring + - Trend Analysis + - Forecast Evaluation +difficulty: intermediate +geography: + - United States +temporal_coverage: weekly natural-gas storage series with history varying by region +update_frequency: weekly +provider: U.S. Energy Information Administration +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + EIA's weekly underground storage route returns working-gas inventories. + Start with the Lower 48 total and a short length. These are storage + statistics, not Henry Hub spot prices, and weekly values can be revised. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free EIA API key saved in the EIA_API_KEY environment variable + access_steps: + - Register for a free key on the EIA Open Data site, or reuse EIA_API_KEY. + - Confirm the natural-gas weekly storage route. + - Request a bounded Lower 48 working-gas sample. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.eia.gov/v2/natural-gas/stor/wkly/data/", + params={ + "api_key": os.environ["EIA_API_KEY"], + "frequency": "weekly", + "data[0]": "value", + "facets[duoarea][]": "NUS", + "length": 20, + }, + timeout=30, + ) + response.raise_for_status() + storage = pd.DataFrame(response.json()["response"]["data"]) + storage["period"] = pd.to_datetime(storage["period"]) + storage["value"] = pd.to_numeric(storage["value"], errors="coerce") + storage["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(storage[["period", "duoarea", "value", "units"]].head()) + first_project: + title: Track Lower 48 weekly gas storage + goal: Test whether a short storage window can power an inventory-change alert. + steps: + - Sort by period and compute week-to-week storage changes. + - Flag the largest injections and withdrawals in the extract. + - Explain that weekly storage is not a spot price and that EIA can revise previously published weeks. diff --git a/data/datasets/entsoe-transparency.yaml b/data/datasets/entsoe-transparency.yaml new file mode 100644 index 0000000..9b21741 --- /dev/null +++ b/data/datasets/entsoe-transparency.yaml @@ -0,0 +1,87 @@ +id: entsoe-transparency +name: ENTSO-E Transparency Platform +description: > + European electricity generation, load, and day-ahead prices for building + grid-monitoring and power-market dashboards. +theme: Markets & Economics +url: https://www.entsoe.eu/data/transparency-platform/ +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - XML + - CSV +license: ENTSO-E Transparency Platform terms of use +license_url: https://www.entsoe.eu/data/transparency-platform/ +url_checks: + source_marker: Transparency Platform + license_marker: Electricity Market Transparency +domains: + - Energy + - Electricity + - Commodities +data_types: + - Time Series + - Tabular +tasks: + - Grid Monitoring + - Market Monitoring + - Forecast Evaluation +difficulty: intermediate +geography: + - Europe +temporal_coverage: platform series from 2015-present with bidding-zone changes +update_frequency: near real time +provider: ENTSO-E +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + The Transparency Platform REST API returns XML for load, generation, and + prices. Start with one bidding zone and one day of actual total load. Zone + codes change over time, and this is not a retail bill. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free Transparency Platform token saved in the ENTSOE_API_KEY environment variable + access_steps: + - Register on the Transparency Platform and create a REST API token. + - Save the token in ENTSOE_API_KEY. + - Request one day of actual total load for one bidding zone. + python: + packages: + - pandas + - requests + - lxml + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://web-api.tp.entsoe.eu/api", + params={ + "securityToken": os.environ["ENTSOE_API_KEY"], + "documentType": "A65", + "processType": "A16", + "outBiddingZone_Domain": "10Y1001A1001A83F", + "periodStart": "202407010000", + "periodEnd": "202407020000", + }, + timeout=60, + ) + response.raise_for_status() + print(response.text[:1000]) + points = pd.read_xml(response.content, xpath=".//Point") + print(points.head()) + first_project: + title: Chart one day of German load + goal: Test whether a Transparency extract can power a bounded grid-load monitor. + steps: + - Parse Point positions and quantities from the XML document. + - Plot load over the requested day and note missing intervals. + - Explain that bidding-zone codes change and that platform load is not a household electricity price. diff --git a/data/datasets/epa-ghgrp.yaml b/data/datasets/epa-ghgrp.yaml new file mode 100644 index 0000000..e80d03c --- /dev/null +++ b/data/datasets/epa-ghgrp.yaml @@ -0,0 +1,78 @@ +id: epa-ghgrp +name: EPA Greenhouse Gas Reporting Program +description: > + Facility-level U.S. greenhouse-gas emissions reports for building industrial + emissions maps and sector comparison tools. +theme: Environment & Hazards +url: https://www.epa.gov/ghgreporting/data-sets +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - Excel + - CSV + - JSON +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: GHG Reporting Program Data Sets + license_marker: federal government materials +domains: + - Climate + - Environmental Health + - Regulation +data_types: + - Tabular + - Facility Records +tasks: + - Emissions Monitoring + - Facility Comparison + - Mapping +difficulty: intermediate +geography: + - United States +temporal_coverage: 2010-present reporting years +update_frequency: annual +provider: U.S. Environmental Protection Agency +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + GHGRP publishes annual facility emissions for large U.S. reporters. Start + with one state and one reporting year from Envirofacts. Reported totals are + self-reported under the rule, cover large emitters rather than every source, + and are not the EPA Inventory of U.S. Greenhouse Gas Emissions. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the GHGRP data sets page and note that Envirofacts can return JSON. + - Query one state and keep facility name, latitude, longitude, and CO2e. + - Record the reporting year used in the query. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.epa.gov/efservice/PUB_DIM_FACILITY/STATE/CO/JSON", + timeout=60, + ) + response.raise_for_status() + facilities = pd.DataFrame(response.json()) + print(facilities.head()) + first_project: + title: Map Colorado GHGRP facilities + goal: Test whether facility records can power a bounded industrial-emissions locator. + steps: + - Keep facility identifier, name, county, and coordinates from the bounded extract. + - Count reporters by county and list names missing coordinates. + - Explain that GHGRP covers large reporters for a reporting year and is not a complete emissions inventory. diff --git a/data/datasets/faostat-food-agriculture.yaml b/data/datasets/faostat-food-agriculture.yaml new file mode 100644 index 0000000..f5e80e9 --- /dev/null +++ b/data/datasets/faostat-food-agriculture.yaml @@ -0,0 +1,84 @@ +id: faostat-food-agriculture +name: FAOSTAT Food and Agriculture +description: > + Country food, agriculture, and land-use statistics for building commodity + and food-security comparison tools. +theme: Demographics & Development +url: https://www.fao.org/faostat/en/#data +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV + - Excel +license: FAO terms and conditions +license_url: https://www.fao.org/contact-us/terms/en +url_checks: + source_marker: FAOSTAT + license_marker: Terms and Conditions +domains: + - Agriculture + - Food Science + - International Statistics +data_types: + - Time Series + - Tabular +tasks: + - Country Comparison + - Commodity Monitoring + - Food Security Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: item- and country-specific FAOSTAT series +update_frequency: annual +provider: Food and Agriculture Organization of the United Nations +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + FAOSTAT's JSON API returns one domain, item, and area at a time. Start + with wheat production for one country. National questionnaires, modelled + estimates, and unit changes all affect comparisons. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open FAOSTAT Data and choose the Crops and livestock products domain. + - Request one item, one area, and a short year list. + - Cite FAO with the domain and access date. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://fenixservices.fao.org/faostat/api/v1/en/data/QCL", + params={ + "area": 231, + "item": 15, + "element": 5510, + "year": "2018,2019,2020,2021,2022", + }, + timeout=30, + ) + response.raise_for_status() + crops = pd.DataFrame(response.json()["data"]) + crops["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(crops.head()) + first_project: + title: Track U.S. wheat production + goal: Test whether one FAOSTAT series can power a bounded commodity sketch. + steps: + - Keep area, item, year, value, and unit from the extract. + - Plot production over the requested years and note missing years. + - Cite FAO and explain that questionnaire coverage and unit definitions can change across years. diff --git a/data/datasets/fema-nfip-redacted-claims.yaml b/data/datasets/fema-nfip-redacted-claims.yaml new file mode 100644 index 0000000..82667fb --- /dev/null +++ b/data/datasets/fema-nfip-redacted-claims.yaml @@ -0,0 +1,85 @@ +id: fema-nfip-redacted-claims +name: FEMA NFIP Redacted Claims +description: > + Redacted National Flood Insurance Program claims for building local flood-loss + histories and insurance-signal prototypes. +theme: Environment & Hazards +url: https://www.fema.gov/openfema-data-page/fima-nfip-redacted-claims-v2 +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV + - Parquet +license: OpenFEMA Terms and Conditions +license_url: https://www.fema.gov/about/openfema/terms-conditions +url_checks: + source_marker: FIMA NFIP Redacted Claims + license_marker: OpenFEMA Terms and Conditions +domains: + - Natural Hazards + - Housing + - Emergency Management +data_types: + - Event Data + - Tabular + - Geospatial +tasks: + - Loss Analysis + - Community Comparison + - Insurance Research +difficulty: intermediate +geography: + - United States +temporal_coverage: historical NFIP claims through the current OpenFEMA extract +update_frequency: monthly +provider: Federal Emergency Management Agency +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + OpenFEMA publishes redacted NFIP claim records with occupancy, dates, and + paid amounts. Start with one state and a recent year. Claims are insured + losses, not a complete flood census, and redaction plus policy changes + limit neighborhood-level conclusions. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A two-letter U.S. state code + access_steps: + - Read the OpenFEMA NFIP claims page and terms before querying. + - Request a bounded v2 page for one state ordered by date of loss. + - Keep occupancy, loss date, and building and contents payments. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://www.fema.gov/api/open/v2/FimaNfipClaims", + params={ + "$filter": "state eq 'FL' and yearOfLoss eq 2024", + "$select": "id,state,countyCode,yearOfLoss,dateOfLoss,occupancyType,amountPaidOnBuildingClaim,amountPaidOnContentsClaim", + "$top": 100, + }, + timeout=60, + ) + response.raise_for_status() + claims = pd.DataFrame(response.json()["FimaNfipClaims"]) + claims["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(claims.head()) + first_project: + title: Summarize recent Florida NFIP claims + goal: Test whether a bounded claims extract can power a county flood-loss sketch. + steps: + - Convert payment fields to numbers and drop rows missing loss dates. + - Sum building and contents payments by county code for the selected year. + - Explain that NFIP claims cover insured properties only and that redaction prevents household-level identification. diff --git a/data/datasets/gfw-tree-cover-loss.yaml b/data/datasets/gfw-tree-cover-loss.yaml new file mode 100644 index 0000000..b37d360 --- /dev/null +++ b/data/datasets/gfw-tree-cover-loss.yaml @@ -0,0 +1,84 @@ +id: gfw-tree-cover-loss +name: Global Forest Watch Tree Cover Loss +description: > + Annual tree-cover loss statistics for building deforestation monitors and + country comparison tools from satellite-derived forest change. +theme: Environment & Hazards +url: https://data-api.globalforestwatch.org/ +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - JSON +license: Creative Commons Attribution 4.0 International +license_url: https://creativecommons.org/licenses/by/4.0/ +url_checks: + source_marker: GFW DATA API + license_marker: Attribution 4.0 International +domains: + - Climate + - Forestry + - Earth Observation +data_types: + - Tabular + - Geospatial + - Time Series +tasks: + - Deforestation Monitoring + - Country Comparison + - Trend Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: 2001-present annual loss years +update_frequency: annual +provider: World Resources Institute +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + Global Forest Watch exposes UMD/Hansen tree-cover loss through a SQL query + API. Start with one year and the top countries by hectares lost. Loss is a + stand-replacement disturbance, not a land-use class, and it includes fire, + harvest, and conversion. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the GFW Data API documentation and locate the umd_tree_cover_loss dataset. + - Query one loss year aggregated by ISO3 code with a small row limit. + - Cite Hansen/UMD/Google/USGS/NASA and WRI when presenting results. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data-api.globalforestwatch.org/dataset/umd_tree_cover_loss/latest/query", + params={ + "sql": ( + "SELECT iso, SUM(umd_tree_cover_loss__ha) AS loss_ha " + "FROM data WHERE umd_tree_cover_loss__year = 2023 " + "GROUP BY iso ORDER BY loss_ha DESC LIMIT 15" + ) + }, + timeout=60, + ) + response.raise_for_status() + loss = pd.DataFrame(response.json()["data"]) + loss["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(loss) + first_project: + title: Rank countries by 2023 tree-cover loss + goal: Test whether GFW country totals can power a bounded deforestation leaderboard. + steps: + - Confirm the query returned ISO3 codes and hectare totals for one loss year. + - Rank the bounded result and note countries with missing or zero values. + - Explain that tree-cover loss is not the same as deforestation of natural forest and includes harvest and fire. diff --git a/data/datasets/github-archive.yaml b/data/datasets/github-archive.yaml new file mode 100644 index 0000000..4b5ec64 --- /dev/null +++ b/data/datasets/github-archive.yaml @@ -0,0 +1,91 @@ +id: github-archive +name: GitHub Archive +description: > + Hourly public GitHub event archives for building repository-activity monitors + without using BigQuery. +theme: Technology & Cybersecurity +url: https://www.gharchive.org/ +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.02 +size_gb_max: 0.2 +formats: + - JSON + - GZIP +license: GitHub Terms of Service for public events, with GH Archive access terms +license_url: https://docs.github.com/en/site-policy/github-terms/github-terms-of-service +url_checks: + source_marker: GH Archive + license_marker: GitHub Terms of Service +domains: + - Open Source + - Software + - Attention Measurement +data_types: + - Event Data + - JSON Lines +tasks: + - Activity Monitoring + - Trend Analysis + - Repository Research +difficulty: intermediate +geography: + - Global +temporal_coverage: 2011-present hourly public GitHub events +update_frequency: near real time +provider: GH Archive +source_type: community +last_verified: 2026-08-18 +getting_started: + overview: > + GH Archive stores hourly gzip JSON of public GitHub events. Start with one + UTC hour and stream the first events rather than loading the whole file. + Do not use BigQuery for this guide, and public events still fall under + GitHub's terms. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open gharchive.org and note the hourly URL pattern. + - Stream one recent hourly gzip. + - Keep event type and repository name from the first records. + python: + packages: + - pandas + - requests + code: | + import gzip + import json + import pandas as pd + import requests + + response = requests.get( + "https://data.gharchive.org/2026-08-17-15.json.gz", + stream=True, + timeout=60, + ) + response.raise_for_status() + rows = [] + with gzip.GzipFile(fileobj=response.raw) as archive: + for line in archive: + event = json.loads(line) + rows.append( + { + "type": event.get("type"), + "repo": (event.get("repo") or {}).get("name"), + } + ) + if len(rows) >= 50: + break + events = pd.DataFrame(rows) + print(events.head()) + first_project: + title: Count event types in one GitHub hour + goal: Test whether a streamed hour can power a bounded activity monitor. + steps: + - Stream the first 50 events from one hourly gzip and keep type plus repository. + - Count event types in that sample. + - Explain that the sample is not the full hour, BigQuery is out of scope here, and GitHub terms apply to public event data. diff --git a/data/datasets/govinfo-uscourts.yaml b/data/datasets/govinfo-uscourts.yaml new file mode 100644 index 0000000..8625ecc --- /dev/null +++ b/data/datasets/govinfo-uscourts.yaml @@ -0,0 +1,86 @@ +id: govinfo-uscourts +name: GovInfo U.S. Courts Opinions +description: > + Published federal court opinions from GPO GovInfo for building bounded case + lookup and docket-research tools. +theme: Government & Policy +url: https://www.govinfo.gov/help/uscourts +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 5 +formats: + - JSON + - XML + - PDF +license: U.S. Government public data / GPO terms of use +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: United States Courts Opinions + license_marker: federal government materials +domains: + - Justice + - Public Policy + - Legislation +data_types: + - Documents + - Text +tasks: + - Case Research + - Legal Monitoring + - Search +difficulty: intermediate +geography: + - United States +temporal_coverage: participating federal courts with collection-specific start dates +update_frequency: daily +provider: U.S. Government Publishing Office +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + GovInfo's USCOURTS collection hosts published federal opinions. Start with + one court package id through the API. Coverage is not every federal case, + unpublished opinions are often missing, and the collection is not a + citator. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free api.data.gov key saved in the DATA_GOV_API_KEY environment variable + access_steps: + - Register for a free api.data.gov key and save it in DATA_GOV_API_KEY. + - Open the USCOURTS collection page and copy one court collection code. + - Request a small search of opinion granules for that court. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.govinfo.gov/search", + params={ + "api_key": os.environ["DATA_GOV_API_KEY"], + "query": "collection:USCOURTS courtCode:ca9", + "pageSize": 20, + "offsetMark": "*", + }, + timeout=30, + ) + response.raise_for_status() + opinions = pd.json_normalize(response.json()["results"]) + opinions["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(opinions.head()) + first_project: + title: List recent Ninth Circuit opinions + goal: Test whether GovInfo search can power a bounded federal-opinion inbox. + steps: + - Keep package id, title, and dateIssued from the search page. + - Sort by date and list records missing a court code. + - Explain that USCOURTS coverage is incomplete and that search hits are not a citator or docket history. diff --git a/data/datasets/grants-gov-opportunities.yaml b/data/datasets/grants-gov-opportunities.yaml new file mode 100644 index 0000000..9b456a1 --- /dev/null +++ b/data/datasets/grants-gov-opportunities.yaml @@ -0,0 +1,83 @@ +id: grants-gov-opportunities +name: Grants.gov Opportunities +description: > + Federal grant opportunity postings for building funding inboxes and agency + opportunity monitors. +theme: Government & Policy +url: https://grants.gov/api/api-guide +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Grants.gov RESTful Application Programming Interface + license_marker: federal government materials +domains: + - Research Funding + - Public Spending + - Government +data_types: + - Tabular + - Documents +tasks: + - Opportunity Monitoring + - Agency Comparison + - Alerting +difficulty: beginner +geography: + - United States +temporal_coverage: posted, forecasted, and closed federal opportunities +update_frequency: daily +provider: Grants.gov +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + Grants.gov search2 returns opportunity summaries without authentication. + Start with one keyword and posted status. A posting is not an award, and + eligibility, close dates, and attachments must be read on the opportunity + itself. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Grants.gov API guide and note that search2 uses POST. + - Search one keyword among posted opportunities with a small row count. + - Keep opportunity number, title, agency, and close date. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.post( + "https://api.grants.gov/v1/api/search2", + json={ + "keyword": "climate", + "oppStatuses": "posted", + "rows": 20, + "startRecordNum": 0, + }, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + opportunities = pd.DataFrame(payload.get("data", {}).get("oppHits", payload.get("oppHits", []))) + opportunities["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(opportunities.head()) + first_project: + title: List posted climate grant opportunities + goal: Test whether search2 can power a bounded federal-funding inbox. + steps: + - Keep opportunity number, title, agency, and close date from the bounded response. + - Sort by close date and flag records missing an agency name. + - Explain that a posted opportunity is not an award and that eligibility is defined on the full notice. diff --git a/data/datasets/hmda-loan-applications.yaml b/data/datasets/hmda-loan-applications.yaml new file mode 100644 index 0000000..e0a246f --- /dev/null +++ b/data/datasets/hmda-loan-applications.yaml @@ -0,0 +1,83 @@ +id: hmda-loan-applications +name: HMDA Loan Application Register +description: > + Home Mortgage Disclosure Act loan-application records for building local + mortgage-market and fair-lending research tools. +theme: Markets & Economics +url: https://cfpb.github.io/api/hmda/ +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0.1 +size_gb_max: 20 +formats: + - CSV + - ZIP +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: HMDA API documentation + license_marker: federal government materials +domains: + - Housing + - Consumer Finance + - Financial Regulation +data_types: + - Tabular + - Loan Records +tasks: + - Mortgage Market Analysis + - Fair Lending Research + - Geographic Comparison +difficulty: intermediate +geography: + - United States +temporal_coverage: annual HMDA filings from 2007-present with schema changes +update_frequency: annual +provider: Consumer Financial Protection Bureau +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + HMDA LAR records loan applications, originations, and denials. Start with + one year and one state through the Data Browser. Applicant race and + ethnicity fields have disclosure rules, and a LAR row is not a credit + decision you can reverse-engineer. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the HMDA Data Publication page and choose the Data Browser. + - Filter to one year and one state with a small set of fields. + - Keep action taken, loan amount, and census tract. + python: + packages: + - pandas + - requests + code: | + from io import StringIO + import pandas as pd + import requests + + response = requests.get( + "https://ffiec.cfpb.gov/v2/data-browser-api/view/csv", + params={ + "years": "2023", + "states": "VT", + "actions_taken": "1", + }, + timeout=120, + ) + response.raise_for_status() + loans = pd.read_csv(StringIO(response.text), nrows=200) + print(loans.head()) + first_project: + title: Profile Vermont originations + goal: Test whether a one-state LAR extract can power a bounded mortgage-volume sketch. + steps: + - Keep action taken, loan amount, and census tract from the first rows. + - Summarize origination counts and median loan amount. + - Explain that HMDA is an application register with suppressed fields and is not a credit-scoring file. diff --git a/data/datasets/idmc-internal-displacement.yaml b/data/datasets/idmc-internal-displacement.yaml new file mode 100644 index 0000000..ab05a09 --- /dev/null +++ b/data/datasets/idmc-internal-displacement.yaml @@ -0,0 +1,77 @@ +id: idmc-internal-displacement +name: IDMC Internal Displacement +description: > + Country-year internal displacement figures for building humanitarian + comparison tools alongside refugee statistics. +theme: Demographics & Development +url: https://www.internal-displacement.org/database/displacement-data +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - Excel + - CSV +license: Creative Commons Attribution for Intergovernmental Organisations +license_url: https://creativecommons.org/licenses/by/3.0/igo/ +url_checks: + source_marker: Global Internal Displacement Database + license_marker: Attribution +domains: + - Forced Displacement + - Humanitarian Development +data_types: + - Tabular + - Time Series + - Population Data +tasks: + - Displacement Analysis + - Country Comparison + - Humanitarian Monitoring +difficulty: beginner +geography: + - Global +temporal_coverage: 2008-present GIDD stocks and flows +update_frequency: annual +provider: Internal Displacement Monitoring Centre +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + IDMC's Global Internal Displacement Database publishes country-year stocks + and new displacements from conflict and disasters. Start with the Excel + download. Figures are expert-validated estimates, not household surveys, + and they are not UNHCR refugee counts. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the IDMC displacement-data page and download the GIDD Excel file. + - Load the conflict-and-disasters sheet. + - Keep ISO3, year, conflict displacements, and disaster displacements. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + from io import BytesIO + + response = requests.get( + "https://www.internal-displacement.org/sites/default/files/2024-05/IDMC_Internal_Displacement_Conflict-Violence_Disasters.xlsx", + timeout=60, + ) + response.raise_for_status() + displacement = pd.read_excel(BytesIO(response.content), sheet_name=0) + print(displacement.head()) + first_project: + title: Compare conflict and disaster displacements + goal: Test whether GIDD country-year rows can power a bounded humanitarian sketch. + steps: + - Keep country, year, and conflict versus disaster displacement columns. + - Rank one year by new displacements and list missing ISO3 codes. + - Explain that GIDD figures are curated estimates, not UNHCR refugee statistics, and can be revised. diff --git a/data/datasets/ilostat-labour-statistics.yaml b/data/datasets/ilostat-labour-statistics.yaml new file mode 100644 index 0000000..4997c17 --- /dev/null +++ b/data/datasets/ilostat-labour-statistics.yaml @@ -0,0 +1,83 @@ +id: ilostat-labour-statistics +name: ILOSTAT Labour Statistics +description: > + Comparable labour-force, employment, and wage indicators for building + country labour-market dashboards from ILO SDMX. +theme: Markets & Economics +url: https://www.ilo.org/data-and-statistics +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - JSON + - CSV + - SDMX +license: ILO rights and permissions / Creative Commons Attribution 4.0 +license_url: https://www.ilo.org/rights-and-permissions +url_checks: + source_marker: ILOSTAT + license_marker: Creative Commons +domains: + - Labor Economics + - International Statistics + - Employment +data_types: + - Time Series + - Tabular +tasks: + - Country Comparison + - Labor Market Monitoring + - Trend Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: country-specific labour series published by ILOSTAT +update_frequency: monthly +provider: International Labour Organization +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + ILOSTAT SDMX returns labour indicators such as unemployment rate by sex + and age. Start with one country and one indicator. National definitions + differ, modelled estimates are not survey microdata, and coverage gaps + are common. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open ILOSTAT and copy one dataflow identifier such as UNE_2EAP_SEX_AGE_RT. + - Request SDMX-JSON for one country and a recent year range. + - Cite ILOSTAT with the indicator code. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://sdmx.ilo.org/rest/data/ILO,DF_UNE_2EAP_SEX_AGE_RT/A.USA.SEX_T.AGE_YTHADULT_YGE15", + headers={"Accept": "application/vnd.sdmx.data+json;version=1.0.0"}, + timeout=30, + ) + response.raise_for_status() + observations = response.json()["data"]["dataSets"][0]["observations"] + unemployment = pd.DataFrame( + [{"position": key, "value": value[0]} for key, value in observations.items()] + ) + unemployment["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(unemployment.tail()) + first_project: + title: Track U.S. ILO unemployment + goal: Test whether one ILOSTAT series can power a bounded labour-market chart. + steps: + - Confirm observation values returned for the selected series key. + - Plot the latest published rates and note missing years. + - Explain that ILO modelled series can differ from BLS household-survey rates and are not microdata. diff --git a/data/datasets/irs-soi-tax-stats.yaml b/data/datasets/irs-soi-tax-stats.yaml new file mode 100644 index 0000000..31a41f2 --- /dev/null +++ b/data/datasets/irs-soi-tax-stats.yaml @@ -0,0 +1,72 @@ +id: irs-soi-tax-stats +name: IRS SOI Tax Stats +description: > + Statistics of Income tables on individual and business tax returns for + building income, filing, and geographic tax-profile tools. +theme: Markets & Economics +url: https://www.irs.gov/statistics/soi-tax-stats-statistics-of-income +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - Excel + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Statistics of Income + license_marker: federal government materials +domains: + - Public Finance + - Taxation + - Regional Economics +data_types: + - Tabular + - Aggregated Data +tasks: + - Income Analysis + - Geographic Comparison + - Public Finance Research +difficulty: intermediate +geography: + - United States +temporal_coverage: tax years published in SOI annual and geographic products +update_frequency: annual +provider: Internal Revenue Service +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + SOI publishes aggregated individual-income tables, including ZIP-code + products. Start with one state's ZIP-code AGI table. These are tabulated + tax returns, not household surveys, and ZIP totals can be suppressed for + disclosure. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the SOI Statistics of Income page and locate Individual Income Tax ZIP Code Data. + - Download one recent tax-year CSV or Excel file. + - Keep ZIP code, AGI class, number of returns, and total AGI. + python: + packages: + - pandas + code: | + import pandas as pd + + soi = pd.read_csv( + "https://www.irs.gov/pub/irs-soi/22zpallagi.csv", + nrows=200, + ) + print(soi.head()) + first_project: + title: Profile ZIP-code AGI classes + goal: Test whether a SOI ZIP extract can power a bounded local-income sketch. + steps: + - Keep state, ZIP, AGI class, return counts, and AGI from the first rows. + - Sum returns by AGI class for one state abbreviation in the extract. + - Explain that SOI ZIP tables are tax-return tabulations with disclosure suppression, not ACS income estimates. diff --git a/data/datasets/medsl-county-returns.yaml b/data/datasets/medsl-county-returns.yaml new file mode 100644 index 0000000..01dfe1f --- /dev/null +++ b/data/datasets/medsl-county-returns.yaml @@ -0,0 +1,71 @@ +id: medsl-county-returns +name: MEDSL County Presidential Returns +description: > + County-level U.S. presidential election returns for building result maps and + turnout comparison tools from MIT Election Data and Science Lab files. +theme: Government & Policy +url: https://electionlab.mit.edu/data +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - CSV + - DTA +license: Creative Commons CC0 1.0 Universal +license_url: https://creativecommons.org/publicdomain/zero/1.0/ +url_checks: + source_marker: Election Data + license_marker: CC0 +domains: + - Elections + - Public Policy +data_types: + - Tabular + - Geospatial +tasks: + - Election Analysis + - Mapping + - Turnout Comparison +difficulty: beginner +geography: + - United States +temporal_coverage: presidential general elections compiled by MEDSL +update_frequency: occasional +provider: MIT Election Data and Science Lab +source_type: academic +last_verified: 2026-08-18 +getting_started: + overview: > + MEDSL publishes county presidential returns on Harvard Dataverse. Start + with one election year and two-party totals. County shapefiles, third-party + votes, and certified state files can disagree, and this is not precinct + data. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the MEDSL data page and follow the county presidential returns Dataverse record. + - Download the CSV for the latest presidential cycle. + - Keep year, county FIPS, Democratic votes, and Republican votes. + python: + packages: + - pandas + code: | + import pandas as pd + + returns = pd.read_csv( + "https://dataverse.harvard.edu/api/access/datafile/6188156", + encoding="latin-1", + ) + print(returns.head()) + first_project: + title: Map one year's county presidential margins + goal: Test whether county returns can power a bounded result-map prototype. + steps: + - Filter to one year and keep county FIPS plus Democratic and Republican votes. + - Compute two-party vote share and list counties with missing FIPS. + - Explain that county totals can differ from certified state files and are not precinct results. diff --git a/data/datasets/nchs-provisional-mortality.yaml b/data/datasets/nchs-provisional-mortality.yaml new file mode 100644 index 0000000..68614f6 --- /dev/null +++ b/data/datasets/nchs-provisional-mortality.yaml @@ -0,0 +1,83 @@ +id: nchs-provisional-mortality +name: NCHS Provisional Mortality +description: > + Weekly provisional U.S. death counts by jurisdiction for building mortality + monitors and excess-death research prototypes. +theme: Health, Food & Safety +url: https://data.cdc.gov/NCHS/Weekly-Provisional-Counts-of-Deaths-by-State-and-/muzy-jte6 +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.1 +formats: + - JSON + - CSV +license: U.S. Public Domain with CDC attribution and use requirements +license_url: https://www.cdc.gov/other/agencymaterials.html +url_checks: + source_marker: Weekly Provisional Counts of Deaths + license_marker: public domain +domains: + - Public Health + - Demographics +data_types: + - Time Series + - Tabular + - Vital Statistics +tasks: + - Mortality Monitoring + - Geographic Comparison + - Trend Analysis +difficulty: beginner +geography: + - United States +temporal_coverage: weekly provisional death counts with lagged completeness +update_frequency: weekly +provider: National Center for Health Statistics +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + NCHS publishes weekly provisional death counts on data.cdc.gov. Start with + one jurisdiction and all-cause deaths. Provisional counts are incomplete, + lag varies by state, and they are not cause-specific research files from + WONDER. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Weekly Provisional Counts dataset page. + - Query one jurisdiction through the Socrata JSON endpoint. + - Keep week-ending date, jurisdiction, and all-cause deaths. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.cdc.gov/resource/muzy-jte6.json", + params={ + "$where": "jurisdiction_of_occurrence='California'", + "$limit": 52, + "$order": "week_ending_date DESC", + }, + timeout=30, + ) + response.raise_for_status() + deaths = pd.DataFrame(response.json()) + deaths["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(deaths.head()) + first_project: + title: Chart California weekly deaths + goal: Test whether provisional counts can power a bounded mortality sparkline. + steps: + - Convert week-ending dates and all-cause counts to usable types. + - Plot the latest year of weekly deaths and note weeks that may still be incomplete. + - Explain that provisional NCHS counts lag and are not the final vital-statistics file. diff --git a/data/datasets/nhanes.yaml b/data/datasets/nhanes.yaml new file mode 100644 index 0000000..8ea45e8 --- /dev/null +++ b/data/datasets/nhanes.yaml @@ -0,0 +1,78 @@ +id: nhanes +name: NHANES Public Data Files +description: > + Cycle-based examination, laboratory, and questionnaire files for building + nutrition and health-survey prototypes from public NHANES tables. +theme: Health, Food & Safety +url: https://wwwn.cdc.gov/nchs/nhanes/default.aspx +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 2 +formats: + - XPT + - CSV +license: U.S. Public Domain with CDC attribution and use requirements +license_url: https://www.cdc.gov/other/agencymaterials.html +url_checks: + source_marker: National Health and Nutrition Examination Survey + license_marker: public domain +domains: + - Public Health + - Nutrition + - Survey Research +data_types: + - Survey Microdata + - Tabular +tasks: + - Health Survey Analysis + - Nutrition Research + - Prevalence Estimation +difficulty: intermediate +geography: + - United States +temporal_coverage: NHANES cycles from 1999-present with two-year public files +update_frequency: occasional +provider: National Center for Health Statistics +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + NHANES public files include demographics, examination, and lab tables by + cycle. Start with one cycle's demographics XPT. Unweighted row counts are + not national estimates; use interview weights, and do not treat a cycle + extract as clinical diagnosis data. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the NHANES page and choose one public cycle such as 2021-2023. + - Download the demographics XPT file. + - Keep sequence number, interview weight, age, and sex. + python: + packages: + - pandas + - pyreadstat + code: | + import pandas as pd + import pyreadstat + import requests + from io import BytesIO + + response = requests.get( + "https://wwwn.cdc.gov/Nchs/Nhanes/2021-2023/DEMO_L.xpt", + timeout=60, + ) + response.raise_for_status() + demographics, _meta = pyreadstat.read_xport(BytesIO(response.content)) + print(demographics.head()) + first_project: + title: Describe one NHANES demographics file + goal: Test whether a public cycle file can power a bounded survey-profile sketch. + steps: + - Keep sequence number, interview weight, age, and sex from the demographics table. + - Compare unweighted counts with a weighted age summary using the interview weight. + - Explain that unweighted rows are not U.S. prevalence estimates and that NHANES is not a diagnostic file. diff --git a/data/datasets/nhtsa-fars.yaml b/data/datasets/nhtsa-fars.yaml new file mode 100644 index 0000000..c3f156b --- /dev/null +++ b/data/datasets/nhtsa-fars.yaml @@ -0,0 +1,86 @@ +id: nhtsa-fars +name: NHTSA Fatality Analysis Reporting System +description: > + Census of fatal U.S. motor-vehicle crashes for building road-safety maps and + fatality-trend monitors. +theme: Health, Food & Safety +url: https://www.nhtsa.gov/nhtsa-datasets-and-apis +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0.05 +size_gb_max: 2 +formats: + - CSV + - SAS + - ZIP +license: NHTSA public-information terms +license_url: https://www.nhtsa.gov/about-nhtsa/terms-use +url_checks: + source_marker: NHTSA Datasets and APIs + license_marker: may be distributed or copied +domains: + - Transportation + - Vehicle Safety + - Public Health +data_types: + - Event Data + - Tabular + - Crash Records +tasks: + - Safety Analysis + - Mapping + - Trend Analysis +difficulty: intermediate +geography: + - United States +temporal_coverage: annual FARS files from 1975-present +update_frequency: annual +provider: National Highway Traffic Safety Administration +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + FARS is a census of fatal crashes on U.S. public roads. Start with one + year's accident CSV. It does not include non-fatal crashes, and FARS is + not the same as NHTSA recall records. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the FARS file-download directory and choose one annual National folder. + - Download the accident CSV from that year. + - Keep state, county, date, and fatality count. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://crashviewer.nhtsa.dot.gov/CrashAPI/FARSData/GetFARSData", + params={ + "dataset": "Accident", + "FromYear": "2022", + "ToYear": "2022", + "State": "12", + "format": "json", + }, + timeout=60, + ) + response.raise_for_status() + crashes = pd.DataFrame(response.json().get("Results", [[]])[0]) + crashes["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(crashes.head()) + first_project: + title: Profile one state's fatal crashes + goal: Test whether a FARS extract can power a bounded road-safety sketch. + steps: + - Keep crash identifier, date, county, and fatality count from the bounded year. + - Count fatal crashes by county and list records missing coordinates if present. + - Explain that FARS covers fatal crashes only and is not NHTSA's vehicle-recall file. diff --git a/data/datasets/noaa-gml-co2.yaml b/data/datasets/noaa-gml-co2.yaml new file mode 100644 index 0000000..b83e334 --- /dev/null +++ b/data/datasets/noaa-gml-co2.yaml @@ -0,0 +1,73 @@ +id: noaa-gml-co2 +name: NOAA GML Mauna Loa CO2 +description: > + Monthly atmospheric carbon dioxide measurements from Mauna Loa for building + climate-trend monitors and educational concentration trackers. +theme: Environment & Hazards +url: https://gml.noaa.gov/ccgg/trends/data.html +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.001 +formats: + - CSV + - Text +license: U.S. Government public data / NOAA disclaimer +license_url: https://www.noaa.gov/disclaimer +url_checks: + source_marker: Trends in CO2 + license_marker: public domain +domains: + - Climate + - Environmental Science +data_types: + - Time Series + - Tabular +tasks: + - Trend Analysis + - Climate Monitoring +difficulty: beginner +geography: + - Global +temporal_coverage: 1958-present +update_frequency: monthly +provider: NOAA Global Monitoring Laboratory +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + NOAA GML publishes monthly mean carbon dioxide mole fractions from the Mauna + Loa observatory as a small CSV. Start with the in-situ monthly file. These + values represent one high-altitude Pacific site after quality control, not + a global average or a local urban concentration. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Trends in CO2 data page and identify the Mauna Loa monthly mean CSV. + - Load the file, skipping comment rows that begin with a hash. + - Keep year, month, and the seasonally adjusted monthly mean. + python: + packages: + - pandas + code: | + import pandas as pd + + co2 = pd.read_csv( + "https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_mm_mlo.csv", + comment="#", + ) + co2["date"] = pd.to_datetime( + dict(year=co2["year"], month=co2["month"], day=1) + ) + print(co2[["date", "average", "deseasonalized"]].tail()) + first_project: + title: Chart Mauna Loa monthly CO2 + goal: Test whether the monthly file can power a simple concentration-trend monitor. + steps: + - Drop rows where the monthly average is a missing-value sentinel. + - Plot the monthly average and the deseasonalized series over the full record. + - Explain that Mauna Loa is one background site and is not a global or city-level CO2 product. diff --git a/data/datasets/noaa-ndbc-buoys.yaml b/data/datasets/noaa-ndbc-buoys.yaml new file mode 100644 index 0000000..697dc5f --- /dev/null +++ b/data/datasets/noaa-ndbc-buoys.yaml @@ -0,0 +1,81 @@ +id: noaa-ndbc-buoys +name: NOAA NDBC Buoy Observations +description: > + Real-time marine weather and wave observations from moored buoys for building + coastal condition monitors and marine-operations alerts. +theme: Environment & Hazards +url: https://www.ndbc.noaa.gov/ +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.02 +formats: + - Text + - CSV +license: U.S. Government public data / NOAA disclaimer +license_url: https://www.noaa.gov/disclaimer +url_checks: + source_marker: National Data Buoy Center + license_marker: public domain +domains: + - Weather + - Marine Operations + - Coastal Data +data_types: + - Time Series + - Sensor Observations +tasks: + - Operational Monitoring + - Alerting + - Marine Planning +difficulty: beginner +geography: + - United States +temporal_coverage: station-specific realtime and historical observations +update_frequency: near real time +provider: NOAA National Data Buoy Center +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + NDBC publishes a station-page text file of recent meteorological and wave + observations for each buoy. Start with one station identifier such as 41001. + Sensors drop out, quality flags matter, and a single buoy is not a coastal + forecast. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An NDBC station identifier + access_steps: + - Open the NDBC station map and copy one buoy identifier. + - Retrieve that station's realtime text file. + - Keep observation time, wind speed, wave height, and air temperature. + python: + packages: + - pandas + code: | + import pandas as pd + + observations = pd.read_csv( + "https://www.ndbc.noaa.gov/data/realtime2/41001.txt", + sep=r"\s+", + skiprows=[1], + na_values=["MM"], + ) + observations["observed_at_utc"] = pd.to_datetime( + observations[["#YY", "MM", "DD", "hh", "mm"]].rename( + columns={"#YY": "year", "MM": "month", "DD": "day", "hh": "hour", "mm": "minute"} + ), + utc=True, + ) + print(observations[["observed_at_utc", "WSPD", "WVHT", "ATMP"]].head()) + first_project: + title: Monitor one Atlantic buoy + goal: Test whether a single station file can power a bounded wind-and-wave condition board. + steps: + - Convert the timestamp columns to UTC and drop rows missing wind speed or wave height. + - Plot the latest hours of wind speed and significant wave height. + - Explain that one buoy cannot substitute for a coastal forecast and that MM placeholders mark missing sensors. diff --git a/data/datasets/nsf-awards.yaml b/data/datasets/nsf-awards.yaml new file mode 100644 index 0000000..6c6c93d --- /dev/null +++ b/data/datasets/nsf-awards.yaml @@ -0,0 +1,76 @@ +id: nsf-awards +name: NSF Awards API +description: > + National Science Foundation award records for building research-funding + monitors and institution comparison tools. +theme: Government & Policy +url: https://resources.research.gov/common/webapi/awardapisearch-v1.htm +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - JSON + - XML +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: NSF Awards API + license_marker: federal government materials +domains: + - Research Funding + - Public Spending +data_types: + - Tabular + - Documents +tasks: + - Funding Analysis + - Institution Comparison + - Topic Research +difficulty: beginner +geography: + - United States +temporal_coverage: historical and active NSF awards +update_frequency: daily +provider: U.S. National Science Foundation +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The NSF Awards API searches Research.gov award records by keyword. Start + with one term and a 25-record page. Award amounts can span multiple years, + and NSF is not a complete picture of U.S. research funding. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Awards API documentation and note the 25-result page limit. + - Search one keyword with JSON output. + - Keep award id, title, institution, and amount. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.nsf.gov/services/v1/awards.json", + params={"keyword": "quantum", "rpp": 25}, + timeout=30, + ) + response.raise_for_status() + awards = pd.json_normalize(response.json()["response"]["award"]) + awards["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(awards[["id", "title", "awardeeName", "estimatedTotalAmt"]].head()) + first_project: + title: List recent NSF quantum awards + goal: Test whether a keyword search can power a bounded funding inbox. + steps: + - Keep award id, title, awardee, and estimated total amount from the first page. + - Sum amounts by institution and flag missing amounts. + - Explain that page size is capped and that NSF awards are not all U.S. research funding. diff --git a/data/datasets/nsidc-sea-ice-index.yaml b/data/datasets/nsidc-sea-ice-index.yaml new file mode 100644 index 0000000..a33e2a7 --- /dev/null +++ b/data/datasets/nsidc-sea-ice-index.yaml @@ -0,0 +1,78 @@ +id: nsidc-sea-ice-index +name: NSIDC Sea Ice Index +description: > + Arctic and Antarctic sea-ice extent and concentration time series for + building seasonal ice monitors and climate-comparison tools. +theme: Environment & Hazards +url: https://nsidc.org/data/g02135/versions/3 +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - CSV + - GeoTIFF + - PNG +license: NASA NSIDC DAAC data use and citation guidance +license_url: https://nsidc.org/about/data-use-and-copyright +url_checks: + source_marker: Sea Ice Index + license_marker: Citation Policies +domains: + - Climate + - Cryosphere + - Earth Observation +data_types: + - Time Series + - Geospatial + - Tabular +tasks: + - Climate Monitoring + - Trend Analysis + - Seasonal Comparison +difficulty: beginner +geography: + - Global +temporal_coverage: 1978-present +update_frequency: daily +provider: National Snow and Ice Data Center +source_type: academic +last_verified: 2026-08-18 +getting_started: + overview: > + The Sea Ice Index provides daily and monthly Northern Hemisphere extent + CSV files derived from passive-microwave satellite retrievals. Start with + the monthly North CSV. Extent is a thresholded ice-covered area, not ice + thickness or volume, and near-real-time values can be revised. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the G02135 dataset page and locate the Sea Ice Index CSV archives. + - Download the Northern Hemisphere monthly extent file. + - Keep date, extent, and missing flags before plotting. + python: + packages: + - pandas + code: | + import pandas as pd + + ice = pd.read_csv( + "https://noaadata.apps.nsidc.org/NOAA/G02135/north/monthly/data/N_09_extent_v3.0.csv", + ) + ice.columns = [column.strip() for column in ice.columns] + ice["extent"] = pd.to_numeric(ice["extent"], errors="coerce") + ice["date"] = pd.to_datetime( + dict(year=ice["year"], month=ice["mo"], day=1) + ) + print(ice[["date", "extent", "area"]].tail()) + first_project: + title: Track September Arctic sea-ice extent + goal: Describe how September Northern Hemisphere extent has changed across the published record. + steps: + - Convert year and month to dates and drop rows with missing extent. + - Plot September extent and mark the minimum year in the extract. + - Explain that extent is not thickness or volume and that near-real-time months can be revised. diff --git a/data/datasets/ntsb-aviation-accidents.yaml b/data/datasets/ntsb-aviation-accidents.yaml new file mode 100644 index 0000000..7c361e8 --- /dev/null +++ b/data/datasets/ntsb-aviation-accidents.yaml @@ -0,0 +1,82 @@ +id: ntsb-aviation-accidents +name: NTSB Aviation Accident Database +description: > + Civil aviation accident and incident investigations for building safety + monitors and event-history tools. +theme: Geospatial & Infrastructure +url: https://www.ntsb.gov/Pages/AviationQuery.aspx +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON + - CSV + - XML +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Aviation Accident Database + license_marker: federal government materials +domains: + - Aviation + - Transportation + - Public Safety +data_types: + - Event Data + - Tabular + - Geospatial +tasks: + - Safety Analysis + - Event Monitoring + - Mapping +difficulty: beginner +geography: + - United States +temporal_coverage: historical and current NTSB civil aviation investigations +update_frequency: daily +provider: National Transportation Safety Board +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + NTSB CAROL and the aviation query page expose accident records with + aircraft, location, and injury severity. Start with one year and one + state. Preliminary reports change, this is not a live ADS-B feed, and it + is not a complete global accident census. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the NTSB aviation query page and note CAROL as the current retrieval tool. + - Request a bounded JSON export for one year. + - Keep NTSB number, event date, location, and highest injury. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.ntsb.gov/carol-repgen/api/Aviation/ReportMain/GenerateRecentFile/6", + params={"fileType": "json"}, + timeout=60, + ) + response.raise_for_status() + payload = response.json() + accidents = pd.DataFrame(payload if isinstance(payload, list) else payload.get("data", payload)) + accidents["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(accidents.head()) + first_project: + title: List recent U.S. aviation accidents + goal: Test whether an NTSB extract can power a bounded safety-event inbox. + steps: + - Keep NTSB number, event date, location, and injury severity from the extract. + - Count events by highest injury and list records missing coordinates. + - Explain that preliminary investigations change and that NTSB records are not live aircraft positions. diff --git a/data/datasets/oecd-sdmx-statistics.yaml b/data/datasets/oecd-sdmx-statistics.yaml new file mode 100644 index 0000000..e76ea9c --- /dev/null +++ b/data/datasets/oecd-sdmx-statistics.yaml @@ -0,0 +1,84 @@ +id: oecd-sdmx-statistics +name: OECD SDMX Statistics +description: > + Comparable OECD economic and social indicators for building country dashboards + and release-monitoring tools from a bounded SDMX query. +theme: Markets & Economics +url: https://data-explorer.oecd.org/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON + - CSV + - SDMX +license: Creative Commons Attribution 4.0 International +license_url: https://creativecommons.org/licenses/by/4.0/ +url_checks: + source_marker: Data Explorer + license_marker: Attribution 4.0 International +domains: + - International Economics + - Macroeconomics + - International Statistics +data_types: + - Time Series + - Tabular +tasks: + - Country Comparison + - Economic Monitoring + - Trend Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: series-specific coverage published with each OECD dataset +update_frequency: monthly +provider: Organisation for Economic Co-operation and Development +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + OECD publishes SDMX-JSON for named datasets such as quarterly GDP growth. + Start with one dataset, one country, and a short time window. Member-country + coverage is not global, and vintages can revise previous values. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open OECD Data Explorer and copy one dataset identifier such as QNA. + - Request SDMX-JSON for one country and a few periods. + - Cite OECD with the dataset code and access date. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://sdmx.oecd.org/public/rest/data/OECD.SDD.NAD,DSD_NAMAIN1@DF_QNA,1.1/Q.AUS.S1.S1.B1GQ.L.XDC.LR.GY._Z", + headers={"Accept": "application/vnd.sdmx.data+json;charset=utf-8;version=1.0"}, + timeout=30, + ) + response.raise_for_status() + observations = response.json()["data"]["dataSets"][0]["observations"] + rows = [ + {"position": key, "value": value[0]} + for key, value in observations.items() + ] + gdp = pd.DataFrame(rows) + gdp["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(gdp.head()) + first_project: + title: Track Australian quarterly GDP growth + goal: Test whether one OECD SDMX series can power a bounded growth monitor. + steps: + - Confirm the response contains observation values for the selected series key. + - Sort by time position and plot the latest published growth rates. + - Cite OECD and explain that member-economy series are not a world aggregate and can be revised. diff --git a/data/datasets/ons-statistics.yaml b/data/datasets/ons-statistics.yaml new file mode 100644 index 0000000..2cf627e --- /dev/null +++ b/data/datasets/ons-statistics.yaml @@ -0,0 +1,79 @@ +id: ons-statistics +name: ONS Statistics API +description: > + Official UK statistical datasets for building local labour, prices, and + population tools from Office for National Statistics. +theme: Demographics & Development +url: https://developer.ons.gov.uk/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON + - CSV +license: Open Government Licence v3.0 +license_url: https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/ +url_checks: + source_marker: ONS developer hub + license_marker: Open Government Licence +domains: + - Demographics + - Labor Economics + - International Statistics +data_types: + - Time Series + - Tabular +tasks: + - Local Statistics + - Country Comparison + - Trend Analysis +difficulty: beginner +geography: + - United Kingdom +temporal_coverage: dataset-specific ONS time series +update_frequency: monthly +provider: Office for National Statistics +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The ONS API returns dataset metadata and observations. Start with one + edition of CPIH or a labour dataset and a few observations. UK statistics + have geography codes that do not match U.S. FIPS, and some tables are + still Excel-only. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the ONS developer hub and pick one dataset id. + - Request the latest edition metadata, then a small observation slice. + - Cite ONS under the Open Government Licence. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.beta.ons.gov.uk/v1/datasets/cpih01/editions/time-series/versions/1/observations", + params={"time": "latest", "geography": "K02000001"}, + timeout=30, + ) + response.raise_for_status() + observations = pd.json_normalize(response.json().get("observations", [])) + observations["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(observations.head()) + first_project: + title: Read a UK CPIH observation + goal: Test whether the ONS API can power a bounded UK prices card. + steps: + - Confirm an observation value returned for the selected dataset and geography. + - Record the edition version and observation time used. + - Cite ONS under OGL and explain that UK geography codes are not U.S. FIPS and that editions are versioned. diff --git a/data/datasets/openfda-device-events.yaml b/data/datasets/openfda-device-events.yaml new file mode 100644 index 0000000..48d9af2 --- /dev/null +++ b/data/datasets/openfda-device-events.yaml @@ -0,0 +1,81 @@ +id: openfda-device-events +name: openFDA Device Adverse Events +description: > + FDA medical-device adverse-event reports for exploring reported device + problems and building bounded safety-research tools. +theme: Health, Food & Safety +url: https://open.fda.gov/apis/device/event/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.1 +formats: + - JSON + - ZIP +license: Creative Commons CC0 1.0 Universal, with noted third-party exceptions +license_url: https://open.fda.gov/terms/ +url_checks: + source_marker: Device Adverse Event Overview + license_marker: Creative Commons CC0 1.0 Universal +domains: + - Medical Devices + - Public Health + - Regulation +data_types: + - Event Data + - Aggregated Data +tasks: + - Safety Signal Analysis + - Trend Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: 1991-present device event reports +update_frequency: quarterly +provider: U.S. Food and Drug Administration +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The openFDA device-event API searches MAUDE reports and can return + server-side counts. Start with one device generic name and a bounded + receive-date window. Voluntary reports cannot establish causality, + incidence, or comparative device safety. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the device-event field documentation and choose one generic device name. + - Query a bounded receive-date period and count reported product problems. + - Preserve the exact query and retrieval timestamp. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.fda.gov/device/event.json", + params={ + "search": 'device.generic_name:"INFUSION PUMP" AND date_received:[20250101 TO 20251231]', + "count": "product_problems.exact", + }, + timeout=30, + ) + response.raise_for_status() + problems = pd.DataFrame(response.json()["results"]) + problems["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(problems.head(20)) + first_project: + title: Summarize reported infusion-pump problems + goal: Identify the most frequently reported product-problem terms for one device type in a bounded year. + steps: + - Confirm the API returned product-problem counts for the generic-name query. + - Rank the bounded counts and note terms that may reflect duplicate or incomplete reports. + - State that MAUDE reports do not establish causality, incidence, or comparative device safety. diff --git a/data/datasets/pubchem-compounds.yaml b/data/datasets/pubchem-compounds.yaml new file mode 100644 index 0000000..d749159 --- /dev/null +++ b/data/datasets/pubchem-compounds.yaml @@ -0,0 +1,79 @@ +id: pubchem-compounds +name: PubChem Compounds +description: > + Chemical identifiers, properties, and bioassay links for building compound + lookup and structure-reference tools. +theme: Research & Reference +url: https://pubchem.ncbi.nlm.nih.gov/docs/pug-rest +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 20 +formats: + - JSON + - XML + - CSV +license: NCBI PubChem data-use and copyright terms +license_url: https://www.ncbi.nlm.nih.gov/home/about/policies/ +url_checks: + source_marker: PUG REST + license_marker: Copyright +domains: + - Chemistry + - Biomedical Research + - Research +data_types: + - Registry Data + - Tabular + - Documents +tasks: + - Compound Lookup + - Property Retrieval + - Reference Building +difficulty: beginner +geography: + - Global +temporal_coverage: continuously updated compound records +update_frequency: daily +provider: National Center for Biotechnology Information +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + PubChem PUG REST returns compound records by name or CID. Start with one + chemical name. Properties can come from predicted or deposited sources, + name resolution can hit multiple CIDs, and this is not a regulatory label. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the PUG REST documentation and choose the compound-name path. + - Request JSON properties for one name such as aspirin. + - Keep CID, molecular formula, and molecular weight. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/aspirin/property/MolecularFormula,MolecularWeight,IUPACName/JSON", + timeout=30, + ) + response.raise_for_status() + compounds = pd.DataFrame(response.json()["PropertyTable"]["Properties"]) + compounds["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(compounds) + first_project: + title: Look up aspirin identifiers + goal: Test whether PUG REST can power a bounded compound-reference card. + steps: + - Confirm CID, formula, and molecular weight returned for the name query. + - Compare multiple CIDs if the name resolves to more than one compound. + - Explain that PubChem properties may be predicted or deposited and are not a DailyMed label. diff --git a/data/datasets/regulations-gov-dockets.yaml b/data/datasets/regulations-gov-dockets.yaml new file mode 100644 index 0000000..ea3145e --- /dev/null +++ b/data/datasets/regulations-gov-dockets.yaml @@ -0,0 +1,85 @@ +id: regulations-gov-dockets +name: Regulations.gov Dockets +description: > + Federal rulemaking dockets, documents, and public comments for building + comment-period monitors and agency docket trackers. +theme: Government & Policy +url: https://open.gsa.gov/api/regulationsgov/ +access_type: + - api +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON +license: Regulations.gov API terms of service +license_url: https://open.gsa.gov/api/regulationsgov/ +url_checks: + source_marker: Regulations.gov API + license_marker: api.data.gov +domains: + - Regulation + - Public Policy + - Legislation +data_types: + - Documents + - Event Data + - Tabular +tasks: + - Rulemaking Monitoring + - Comment Analysis + - Alerting +difficulty: intermediate +geography: + - United States +temporal_coverage: federal dockets and comments available through regulations.gov +update_frequency: daily +provider: U.S. General Services Administration +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The regulations.gov v4 API searches dockets and documents with an + api.data.gov key. Start with one agency and posted documents. Comment text + is public input, not agency policy, and bulk comment jobs have separate + rate limits. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free api.data.gov key saved in the DATA_GOV_API_KEY environment variable + access_steps: + - Register for a free api.data.gov key and save it in DATA_GOV_API_KEY. + - Read the v4 documents search parameters. + - Request a bounded page of EPA proposed rules. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.regulations.gov/v4/documents", + params={ + "api_key": os.environ["DATA_GOV_API_KEY"], + "filter[agencyId]": "EPA", + "filter[documentType]": "Proposed Rule", + "page[size]": 20, + "sort": "-postedDate", + }, + timeout=30, + ) + response.raise_for_status() + documents = pd.json_normalize(response.json()["data"]) + documents["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(documents.head()) + first_project: + title: Track recent EPA proposed rules + goal: Test whether v4 document search can power a bounded comment-period monitor. + steps: + - Keep document id, title, posted date, and comment-end date from the page. + - Flag documents whose comment period is still open. + - Explain that comments are public submissions and that the API is not the signed final rule. diff --git a/data/datasets/ripe-stat.yaml b/data/datasets/ripe-stat.yaml new file mode 100644 index 0000000..e31fe79 --- /dev/null +++ b/data/datasets/ripe-stat.yaml @@ -0,0 +1,76 @@ +id: ripe-stat +name: RIPEstat Data API +description: > + Internet routing, prefix, and ASN statistics for building network-visibility + and prefix-lookup tools. +theme: Technology & Cybersecurity +url: https://stat.ripe.net/docs/data-api/ripestat-data-api +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.02 +formats: + - JSON +license: RIPEstat terms and Creative Commons Attribution +license_url: https://www.ripe.net/about-us/legal/ +url_checks: + source_marker: About the Data API + license_marker: RIPE NCC +domains: + - Internet Infrastructure + - Cybersecurity +data_types: + - Registry Data + - Time Series + - Tabular +tasks: + - Prefix Lookup + - Routing Analysis + - Network Monitoring +difficulty: beginner +geography: + - Global +temporal_coverage: continuously updated routing and registration views +update_frequency: near real time +provider: RIPE NCC +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + RIPEstat data API returns JSON for a resource such as an IP prefix or ASN. + Start with prefix-overview for one well-known prefix. Routing views change, + WHOIS fields can be stale, and this is not packet capture. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the RIPEstat data API documentation and choose prefix-overview. + - Query one prefix such as 8.8.8.0/24. + - Keep announced ASN, holder, and resource. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://stat.ripe.net/data/prefix-overview/data.json", + params={"resource": "8.8.8.0/24"}, + timeout=30, + ) + response.raise_for_status() + overview = pd.json_normalize(response.json()["data"]) + overview["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(overview) + first_project: + title: Look up a public DNS prefix + goal: Test whether prefix-overview can power a bounded routing card. + steps: + - Confirm the response contains the queried resource and announced ASN list. + - Record holder and ASN from the overview. + - Explain that RIPEstat is a routing and registration view, not live packet data, and WHOIS can lag. diff --git a/data/datasets/senate-lda-filings.yaml b/data/datasets/senate-lda-filings.yaml new file mode 100644 index 0000000..e0f7659 --- /dev/null +++ b/data/datasets/senate-lda-filings.yaml @@ -0,0 +1,79 @@ +id: senate-lda-filings +name: Senate LDA Lobbying Disclosures +description: > + Federal lobbying registrations and activity reports for building client and + issue disclosure monitors from Senate LDA filings. +theme: Government & Policy +url: https://lda.gov/api/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - XML +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: LDA API + license_marker: federal government materials +domains: + - Public Policy + - Regulation + - Legislation +data_types: + - Tabular + - Documents +tasks: + - Disclosure Monitoring + - Client Research + - Legislative Monitoring +difficulty: intermediate +geography: + - United States +temporal_coverage: LDA registrations and quarterly activity reports +update_frequency: daily +provider: United States Senate +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The Senate LDA API lists lobbying filings, registrants, and clients. Start + with a small page of recent filings. Reported income is self-disclosed, + issue codes are broad, and a filing is not a complete map of influence. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the LDA API documentation and choose the filings list endpoint. + - Request one page of recent filings. + - Keep filing UUID, registrant, client, and filing type. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://lda.gov/api/v1/filings/", + params={"page_size": 25, "ordering": "-dt_posted"}, + timeout=30, + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + filings = pd.json_normalize(response.json()["results"]) + filings["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(filings.head()) + first_project: + title: Scan recent LDA filings + goal: Test whether the filings list can power a bounded lobbying-disclosure inbox. + steps: + - Keep filing identifier, registrant name, client name, and posted date. + - Count filings by registrant on the first page and list missing client names. + - Explain that LDA reports are self-disclosed summaries and are not a complete record of lobbying contacts. diff --git a/data/datasets/smithsonian-gvp-volcanoes.yaml b/data/datasets/smithsonian-gvp-volcanoes.yaml new file mode 100644 index 0000000..27d2bb0 --- /dev/null +++ b/data/datasets/smithsonian-gvp-volcanoes.yaml @@ -0,0 +1,87 @@ +id: smithsonian-gvp-volcanoes +name: Smithsonian Global Volcanism Program +description: > + Holocene volcano locations and eruption records for building global volcanic + activity maps and nearby-hazard lookup tools. +theme: Environment & Hazards +url: https://volcano.si.edu/ +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.02 +formats: + - Excel + - CSV + - JSON +license: Smithsonian GVP data-use and citation guidance +license_url: https://volcano.si.edu/gvp_webservices.cfm +url_checks: + source_marker: Global Volcanism Program + license_marker: web services +domains: + - Natural Hazards + - Earth Science + - Emergency Management +data_types: + - Geospatial + - Event Data + - Registry Data +tasks: + - Hazard Mapping + - Eruption Monitoring + - Place Lookup +difficulty: beginner +geography: + - Global +temporal_coverage: Holocene volcanoes and historical eruption records +update_frequency: weekly +provider: Smithsonian Institution +source_type: academic +last_verified: 2026-08-18 +getting_started: + overview: > + GVP publishes a Holocene volcano list with coordinates, country, and + primary volcano type. Start with one country. Presence in the Holocene + list is not an eruption nowcast, and eruption records are compiled from + heterogeneous historical sources. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the GVP website and locate the volcano list download or web service. + - Retrieve Holocene volcanoes and keep volcano number, name, country, and coordinates. + - Cite Smithsonian GVP when presenting a map or list. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://volcano.si.edu/geoserver/GVP-VOTW/ows", + params={ + "service": "WFS", + "version": "1.0.0", + "request": "GetFeature", + "typeName": "GVP-VOTW:Smithsonian_VOTW_Holocene_Volcanoes", + "outputFormat": "application/json", + "CQL_FILTER": "Country='Iceland'", + }, + timeout=60, + ) + response.raise_for_status() + volcanoes = pd.json_normalize(response.json()["features"]) + print(volcanoes.head()) + first_project: + title: Map Iceland's Holocene volcanoes + goal: Test whether GVP coordinates can power a bounded volcanic-place lookup. + steps: + - Keep volcano name, volcano number, and coordinates from the Iceland extract. + - Plot the points and list names missing coordinates. + - Explain that the Holocene catalog is not a live eruption feed and that historical eruption dates vary in quality. diff --git a/data/datasets/statcan-web-data.yaml b/data/datasets/statcan-web-data.yaml new file mode 100644 index 0000000..25bd25b --- /dev/null +++ b/data/datasets/statcan-web-data.yaml @@ -0,0 +1,79 @@ +id: statcan-web-data +name: Statistics Canada Web Data Service +description: > + Canadian statistical tables for building provincial labour, prices, and + population tools from Statistics Canada SDMX. +theme: Demographics & Development +url: https://www.statcan.gc.ca/en/developers/wds +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - JSON + - CSV + - SDMX +license: Statistics Canada Open Licence +license_url: https://www.statcan.gc.ca/en/reference/licence +url_checks: + source_marker: Web Data Service + license_marker: Statistics Canada Open Licence +domains: + - Demographics + - Labor Economics + - International Statistics +data_types: + - Time Series + - Tabular +tasks: + - Provincial Comparison + - Economic Monitoring + - Trend Analysis +difficulty: intermediate +geography: + - Canada +temporal_coverage: table-specific Statistics Canada series +update_frequency: monthly +provider: Statistics Canada +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The Web Data Service returns cube metadata and vector series as JSON. Start + with one vector id such as a national unemployment rate. Cube ids change + when tables are terminated, and bilingual labels can differ. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the WDS documentation and copy one vector identifier from a CANSIM table. + - POST getDataFromVectorsAndLatestNPeriods for a few periods. + - Cite Statistics Canada under the Open Licence. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.post( + "https://www150.statcan.gc.ca/t1/wds/rest/getDataFromVectorsAndLatestNPeriods", + json=[{"vectorId": 2062811, "latestN": 12}], + timeout=30, + ) + response.raise_for_status() + series = pd.json_normalize(response.json()[0]["object"]["vectorDataPoint"]) + series["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(series.head()) + first_project: + title: Chart a Canadian labour vector + goal: Test whether one WDS vector can power a bounded provincial-or-national labour sketch. + steps: + - Confirm twelve vector data points returned for the selected vector id. + - Plot the reference periods and values. + - Cite Statistics Canada and explain that terminated cubes change identifiers and that this is not U.S. BLS data. diff --git a/data/datasets/treasury-debt-to-the-penny.yaml b/data/datasets/treasury-debt-to-the-penny.yaml new file mode 100644 index 0000000..df4d75f --- /dev/null +++ b/data/datasets/treasury-debt-to-the-penny.yaml @@ -0,0 +1,85 @@ +id: treasury-debt-to-the-penny +name: Treasury Debt to the Penny +description: > + Daily U.S. public debt outstanding for building national-debt monitors and + fiscal-dashboard prototypes. +theme: Markets & Economics +url: https://fiscaldata.treasury.gov/datasets/debt-to-the-penny/debt-to-the-penny +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - JSON + - CSV + - XML +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Debt to the Penny + license_marker: federal government materials +domains: + - Public Finance + - Fixed Income + - Macroeconomics +data_types: + - Time Series + - Tabular +tasks: + - Fiscal Monitoring + - Trend Analysis + - Dashboarding +difficulty: beginner +geography: + - United States +temporal_coverage: 1993-present daily totals with earlier incomplete splits +update_frequency: daily +provider: U.S. Department of the Treasury Bureau of the Fiscal Service +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + Fiscal Data reports total public debt outstanding each business day, split + into debt held by the public and intragovernmental holdings. Start with + the latest 30 records. This is debt outstanding, not a deficit, and other + Fiscal Data tables calculate TPDO slightly differently. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Debt to the Penny dataset page and copy the API endpoint. + - Request the latest records sorted by record date descending. + - Keep record date, debt held by the public, and total public debt outstanding. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/debt_to_penny", + params={ + "fields": "record_date,debt_held_public_amt,intragov_hold_amt,tot_pub_debt_out_amt", + "sort": "-record_date", + "page[size]": 30, + }, + timeout=30, + ) + response.raise_for_status() + debt = pd.DataFrame(response.json()["data"]) + debt["record_date"] = pd.to_datetime(debt["record_date"]) + debt["tot_pub_debt_out_amt"] = pd.to_numeric(debt["tot_pub_debt_out_amt"]) + print(debt.head()) + first_project: + title: Chart recent total public debt + goal: Test whether daily TPDO can power a bounded national-debt sparkline. + steps: + - Convert amounts to numbers and sort by record date. + - Plot total public debt outstanding and the public versus intragovernmental split. + - Explain that TPDO is not the deficit and that other Fiscal Data tables treat Federal Financing Bank debt differently. diff --git a/data/datasets/un-comtrade.yaml b/data/datasets/un-comtrade.yaml new file mode 100644 index 0000000..e4dc9dc --- /dev/null +++ b/data/datasets/un-comtrade.yaml @@ -0,0 +1,87 @@ +id: un-comtrade +name: UN Comtrade +description: > + Official merchandise trade statistics for building partner and commodity + monitors from a bounded Comtrade extract. +theme: Markets & Economics +url: https://uncomtrade.org/docs/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV +license: United Nations website terms of use +license_url: https://www.un.org/en/about-us/terms-of-use +url_checks: + source_marker: UN Comtrade Help Center + license_marker: Terms of Use +domains: + - International Economics + - Trade + - Commodities +data_types: + - Time Series + - Tabular +tasks: + - Trade Monitoring + - Country Comparison + - Market Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: reporter-specific annual and monthly trade statistics +update_frequency: monthly +provider: United Nations Statistics Division +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + Comtrade preview returns up to 500 trade records without a key. Start with + one reporter, one year, and total merchandise trade. Mirror statistics + disagree, confidential HS lines are suppressed, and a preview is not a + complete bulk extract. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open Comtrade Plus and note the preview API for unauthenticated queries. + - Request one reporter, one period, and HS total trade with a small record limit. + - Keep reporter, partner, flow, and primary value. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://comtradeplus.un.org/api/preview/C/A/HS", + params={ + "reporterCode": 842, + "period": 2023, + "partnerCode": 0, + "cmdCode": "TOTAL", + "flowCode": "X,M", + "maxRecords": 100, + }, + timeout=60, + ) + response.raise_for_status() + payload = response.json() + trade = pd.DataFrame(payload.get("data", payload)) + trade["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(trade.head()) + first_project: + title: Compare U.S. total merchandise exports and imports + goal: Test whether a preview extract can power a bounded trade-balance sketch. + steps: + - Keep reporter, period, flow, and primary value from the preview response. + - Compare export and import totals for the selected year. + - Explain that preview rows are capped, partners can be aggregated, and confidential commodities are omitted. diff --git a/data/datasets/un-sc-consolidated-list.yaml b/data/datasets/un-sc-consolidated-list.yaml new file mode 100644 index 0000000..1d0f6a1 --- /dev/null +++ b/data/datasets/un-sc-consolidated-list.yaml @@ -0,0 +1,74 @@ +id: un-sc-consolidated-list +name: UN Security Council Consolidated List +description: > + UN Security Council sanctions designations for building bounded screening + alerts that stay inside list-matching limits. +theme: Government & Policy +url: https://main.un.org/securitycouncil/en/content/un-sc-consolidated-list +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.02 +formats: + - XML + - HTML +license: United Nations website terms of use +license_url: https://www.un.org/en/about-us/terms-of-use +url_checks: + source_marker: UN Security Council Consolidated List + license_marker: Terms of Use +domains: + - Regulation + - Compliance + - Finance +data_types: + - Registry Data + - Tabular +tasks: + - Alerting + - Search + - Compliance Monitoring +difficulty: beginner +geography: + - Global +temporal_coverage: currently listed designations with historical updates +update_frequency: continuous +provider: United Nations Security Council +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + The Security Council publishes a consolidated XML list of designated + individuals and entities. Start with exact primary names. A string match is + not a confirmed hit, aliases matter, and this list is not OFAC or a + complete global sanctions product. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the consolidated list page and locate the English XML download. + - Load INDIVIDUAL and ENTITY records from the XML. + - Keep DATAID, first and last names or entity name, and listed-on date. + python: + packages: + - pandas + - lxml + code: | + import pandas as pd + + xml = pd.read_xml( + "https://scsanctions.un.org/resources/xml/en/consolidated.xml", + xpath=".//INDIVIDUAL", + ) + xml["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(xml.head()) + first_project: + title: Build an exact-name UN list lookup + goal: Test whether primary individual names can support a bounded screening prototype. + steps: + - Load individuals and keep DATAID plus first and last name fields. + - Compare a short authorized name list using exact uppercase matching only. + - Explain that aliases, transliteration, and entity records require the rest of the XML and that a match is not a legal determination. diff --git a/data/datasets/uspto-open-data-portal.yaml b/data/datasets/uspto-open-data-portal.yaml new file mode 100644 index 0000000..e65f137 --- /dev/null +++ b/data/datasets/uspto-open-data-portal.yaml @@ -0,0 +1,76 @@ +id: uspto-open-data-portal +name: USPTO Open Data Portal +description: > + U.S. patent bibliographic bulk files for building assignee, classification, + and filing-volume research tools. +theme: Research & Reference +url: https://www.uspto.gov/learning-and-resources/data-and-statistics +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 50 +formats: + - XML + - ZIP + - CSV +license: USPTO website terms of use +license_url: https://www.uspto.gov/terms-use-uspto-websites +url_checks: + source_marker: Open Data Portal + license_marker: Terms of Use +domains: + - Research + - Intellectual Property + - Innovation +data_types: + - Documents + - Tabular + - Registry Data +tasks: + - Patent Research + - Assignee Analysis + - Classification Research +difficulty: intermediate +geography: + - United States +temporal_coverage: weekly patent grant and application bulk files +update_frequency: weekly +provider: United States Patent and Trademark Office +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + USPTO Open Data Portal is the current bulk and API home after PatentsView + search shut down. Start by confirming the portal page, then download one + weekly bibliographic zip only after checking its size. Full-text weekly + files are large, and this is not the retired PatentsView search API. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - Enough disk for one weekly zip + access_steps: + - Open USPTO data and statistics and follow the Open Data Portal link. + - Choose one weekly patent-grant bulk file and note its size before downloading. + - Parse a handful of invention-title fields rather than the whole week. + python: + packages: + - requests + code: | + import requests + + response = requests.get( + "https://data.uspto.gov/home", + timeout=30, + ) + response.raise_for_status() + print("Open Data Portal" in response.text) + print(response.url) + first_project: + title: Reach USPTO Open Data Portal + goal: Test whether the current USPTO portal can support a bounded patent-download workflow. + steps: + - Confirm the portal page identifies Open Data Portal before downloading anything. + - Choose one weekly bibliographic file in the portal and record its published week and size. + - Explain that weekly XML dumps are large and that PatentsView's search API is no longer the access path. diff --git a/e2e/discovery.spec.ts b/e2e/discovery.spec.ts index 607e2a1..97aa7a0 100644 --- a/e2e/discovery.spec.ts +++ b/e2e/discovery.spec.ts @@ -654,7 +654,7 @@ test("application copy reflows across supported viewport widths", async ({ page }); test("every dataset guide reflows on a mobile viewport", async ({ page }) => { - test.setTimeout(90_000); + test.setTimeout(Math.max(150_000, getAllDatasets().length * 1_200)); await page.setViewportSize({ width: 360, height: 900 }); for (const dataset of getAllDatasets()) { diff --git a/src/lib/datasets.test.ts b/src/lib/datasets.test.ts index 79aaf0e..9cdf8b0 100644 --- a/src/lib/datasets.test.ts +++ b/src/lib/datasets.test.ts @@ -86,14 +86,14 @@ describe("loadDatasets", () => { getAllDatasets().map((dataset) => [dataset.id, dataset.theme]), ); const groups = { - "Environment & Hazards": ["airnow-air-quality", "epa-airdata-daily-summaries", "epa-echo-drinking-water", "epa-toxics-release-inventory", "fema-national-flood-hazard-layer", "gdacs-disaster-alerts", "nasa-firms", "nasa-power-daily", "noaa-ibtracs", "noaa-swpc-space-weather", "nws-weather-api", "noaa-ncei-daily-summaries", "noaa-tides-currents", "openfema-disaster-declarations", "us-drought-monitor", "usgs-earthquakes", "usgs-water-data", "met-norway-locationforecast", "noaa-storm-events"], - "Government & Policy": ["congress-gov-legislation", "fec-campaign-finance", "federal-register-documents", "ofac-sdn-list", "open-states-legislation", "sam-gov-contract-opportunities", "usaspending-federal-awards", "legislation-gov-uk", "uk-police-street-crime", "fbi-crime-data-explorer", "nih-reporter-projects"], - "Markets & Economics": ["bea-regional-gdp-income", "bls-public-data-api", "census-international-trade", "cfpb-consumer-complaints", "eia-weekly-petroleum-status", "fhfa-house-price-index", "fred-economic-series", "hud-fair-market-rents", "imf-world-economic-outlook", "kalshi-market-data", "polymarket-markets", "sec-edgar-apis", "treasury-securities-auctions", "gleif-lei", "companies-house-uk", "fdic-bank-find", "cftc-commitment-of-traders", "census-county-business-patterns", "ecb-statistical-data-warehouse"], - "Health, Food & Safety": ["cdc-fluview-ilinet", "cdc-places", "clinicaltrials-studies", "cms-care-compare-hospitals", "cms-open-payments", "cpsc-product-recalls", "nhtsa-vehicle-recalls", "nppes-npi-registry", "openfda-drug-adverse-events", "openfda-food-enforcement", "usda-fooddata-central", "open-food-facts", "cms-nursing-homes", "cdc-social-vulnerability-index", "fda-orange-book"], - "Geospatial & Infrastructure": ["bts-airline-on-time", "census-tiger-line", "eia-hourly-electric-grid", "fcc-national-broadband-map", "fhwa-national-bridge-inventory", "fta-ntd-monthly-ridership", "mobility-database-feeds", "natural-earth", "nrel-alt-fuel-stations", "overture-maps-places", "osm-overpass", "ourairports"], - "Research & Reference": ["arxiv-preprints", "crossref-works", "gbif-species-occurrences", "openalex-scholarly-works", "pubmed-citations", "wikimedia-pageviews", "wikidata-query"], - "Technology & Cybersecurity": ["cisa-known-exploited-vulnerabilities", "deps-dev-package-graph", "mitre-attack-enterprise", "nvd-cve", "osv-open-source-vulnerabilities", "first-epss", "openssf-scorecard"], - "Demographics & Development": ["acs-five-year-estimates", "college-scorecard", "eurostat-statistics", "nces-common-core-of-data", "unhcr-refugee-population", "usda-nass-quick-stats", "world-development-indicators", "onet-occupations", "worldpop-population", "who-gho-indicators"], + "Environment & Hazards": ["airnow-air-quality", "epa-airdata-daily-summaries", "epa-echo-drinking-water", "epa-toxics-release-inventory", "fema-national-flood-hazard-layer", "gdacs-disaster-alerts", "nasa-firms", "nasa-power-daily", "noaa-ibtracs", "noaa-swpc-space-weather", "nws-weather-api", "noaa-ncei-daily-summaries", "noaa-tides-currents", "openfema-disaster-declarations", "us-drought-monitor", "usgs-earthquakes", "usgs-water-data", "met-norway-locationforecast", "noaa-storm-events", "noaa-gml-co2", "nsidc-sea-ice-index", "noaa-ndbc-buoys", "epa-ghgrp", "fema-nfip-redacted-claims", "gfw-tree-cover-loss", "smithsonian-gvp-volcanoes", "copernicus-era5"], + "Government & Policy": ["congress-gov-legislation", "fec-campaign-finance", "federal-register-documents", "ofac-sdn-list", "open-states-legislation", "sam-gov-contract-opportunities", "usaspending-federal-awards", "legislation-gov-uk", "uk-police-street-crime", "fbi-crime-data-explorer", "nih-reporter-projects", "nsf-awards", "grants-gov-opportunities", "senate-lda-filings", "regulations-gov-dockets", "govinfo-uscourts", "medsl-county-returns", "un-sc-consolidated-list"], + "Markets & Economics": ["bea-regional-gdp-income", "bls-public-data-api", "census-international-trade", "cfpb-consumer-complaints", "eia-weekly-petroleum-status", "fhfa-house-price-index", "fred-economic-series", "hud-fair-market-rents", "imf-world-economic-outlook", "kalshi-market-data", "polymarket-markets", "sec-edgar-apis", "treasury-securities-auctions", "gleif-lei", "companies-house-uk", "fdic-bank-find", "cftc-commitment-of-traders", "census-county-business-patterns", "ecb-statistical-data-warehouse", "oecd-sdmx-statistics", "un-comtrade", "ilostat-labour-statistics", "irs-soi-tax-stats", "treasury-debt-to-the-penny", "hmda-loan-applications", "eia-weekly-natural-gas", "entsoe-transparency"], + "Health, Food & Safety": ["cdc-fluview-ilinet", "cdc-places", "clinicaltrials-studies", "cms-care-compare-hospitals", "cms-open-payments", "cpsc-product-recalls", "nhtsa-vehicle-recalls", "nppes-npi-registry", "openfda-drug-adverse-events", "openfda-food-enforcement", "usda-fooddata-central", "open-food-facts", "cms-nursing-homes", "cdc-social-vulnerability-index", "fda-orange-book", "nchs-provisional-mortality", "nhanes", "cdc-uscs-cancer-statistics", "nhtsa-fars", "openfda-device-events", "dailymed-drug-labels"], + "Geospatial & Infrastructure": ["bts-airline-on-time", "census-tiger-line", "eia-hourly-electric-grid", "fcc-national-broadband-map", "fhwa-national-bridge-inventory", "fta-ntd-monthly-ridership", "mobility-database-feeds", "natural-earth", "nrel-alt-fuel-stations", "overture-maps-places", "osm-overpass", "ourairports", "census-lehd-lodes", "ntsb-aviation-accidents"], + "Research & Reference": ["arxiv-preprints", "crossref-works", "gbif-species-occurrences", "openalex-scholarly-works", "pubmed-citations", "wikimedia-pageviews", "wikidata-query", "uspto-open-data-portal", "pubchem-compounds"], + "Technology & Cybersecurity": ["cisa-known-exploited-vulnerabilities", "deps-dev-package-graph", "mitre-attack-enterprise", "nvd-cve", "osv-open-source-vulnerabilities", "first-epss", "openssf-scorecard", "github-archive", "chrome-ux-report", "certificate-transparency-crtsh", "ripe-stat"], + "Demographics & Development": ["acs-five-year-estimates", "college-scorecard", "eurostat-statistics", "nces-common-core-of-data", "unhcr-refugee-population", "usda-nass-quick-stats", "world-development-indicators", "onet-occupations", "worldpop-population", "who-gho-indicators", "faostat-food-agriculture", "ons-statistics", "statcan-web-data", "idmc-internal-displacement"], } as const; const grouped = Object.values(groups).flat(); expect([...Object.keys(themes)].sort()).toEqual([...grouped].sort()); diff --git a/src/lib/provider-validation.test.ts b/src/lib/provider-validation.test.ts index 896c66c..59fc0a3 100644 --- a/src/lib/provider-validation.test.ts +++ b/src/lib/provider-validation.test.ts @@ -246,6 +246,37 @@ const validBodies = { elements: [{ type: "node", id: 1, lat: 40.75, lon: -73.98 }], }), "ourairports": "id,ident,type,name,latitude_deg,longitude_deg\n1,KSEA,large_airport,Seattle,47.45,-122.31", + "nsf-awards": JSON.stringify({ + response: { award: [{ id: "1234567", title: "Example quantum award" }] }, + }), + "treasury-debt-to-the-penny": JSON.stringify({ + data: [{ record_date: "2026-08-14", tot_pub_debt_out_amt: "37000000000000.00" }], + }), + "noaa-gml-co2": "year,month,decimal date,average,deseasonalized\n2026,1,2026.042,427.01,425.50\n", + "pubchem-compounds": JSON.stringify({ + PropertyTable: { Properties: [{ CID: 2244, MolecularFormula: "C9H8O4" }] }, + }), + "dailymed-drug-labels": JSON.stringify({ + data: [{ setid: "example", title: "IBUPROFEN" }], + }), + "openfda-device-events": JSON.stringify({ + meta: { results: { total: 1 } }, + results: [{ mdr_report_key: "123" }], + }), + "fema-nfip-redacted-claims": JSON.stringify({ + FimaNfipClaims: [{ id: "abc", state: "FL" }], + }), + "ripe-stat": JSON.stringify({ + data: { resource: "8.8.8.0/24" }, + }), + "nchs-provisional-mortality": JSON.stringify([ + { jurisdiction_of_occurrence: "California" }, + ]), + "noaa-ndbc-buoys": "#YY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD\n2026 08 17 12 00 180 8.0 10.0 1.5 8.0 6.0 180\n", + "ons-statistics": JSON.stringify({ + id: "cpih01", + title: "Consumer Prices Index including owner occupiers' housing costs", + }), } as const; const contentTypes = { @@ -311,6 +342,17 @@ const contentTypes = { "met-norway-locationforecast": "application/json", "osm-overpass": "application/json", "ourairports": "text/csv", + "nsf-awards": "application/json", + "treasury-debt-to-the-penny": "application/json", + "noaa-gml-co2": "text/csv", + "pubchem-compounds": "application/json", + "dailymed-drug-labels": "application/json", + "openfda-device-events": "application/json", + "fema-nfip-redacted-claims": "application/json", + "ripe-stat": "application/json", + "nchs-provisional-mortality": "application/json", + "noaa-ndbc-buoys": "text/plain", + "ons-statistics": "application/json", } as const; function response( @@ -435,6 +477,18 @@ describe("checkProviderContract", () => { fetchImpl: vi.fn().mockResolvedValue(response("HTML", "application/octet-stream")), }), ).resolves.toEqual(["download is not a ZIP archive"]); + + await expect( + checkProviderContract("noaa-gml-co2", { + fetchImpl: vi.fn().mockResolvedValue(response("not-co2", "text/csv")), + }), + ).resolves.toEqual(["CSV is missing year or average columns"]); + + await expect( + checkProviderContract("noaa-ndbc-buoys", { + fetchImpl: vi.fn().mockResolvedValue(response("no sensors", "text/plain")), + }), + ).resolves.toEqual(["buoy file is missing WSPD or WVHT"]); }); it("reports Error and non-Error request failures", async () => { diff --git a/src/lib/provider-validation.ts b/src/lib/provider-validation.ts index d623240..62fec65 100644 --- a/src/lib/provider-validation.ts +++ b/src/lib/provider-validation.ts @@ -804,6 +804,130 @@ const contracts = { : "CSV is missing ident, type, name, or latitude_deg"; }, }, + "nsf-awards": { + url: "https://api.nsf.gov/services/v1/awards.json?keyword=quantum&rpp=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + response: z.object({ + award: z.array(z.object({ id: z.string(), title: z.string() })).min(1), + }), + }), + ), + }, + "treasury-debt-to-the-penny": { + url: "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/debt_to_penny?fields=record_date,tot_pub_debt_out_amt&sort=-record_date&page[size]=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + data: z.array(z.object({ + record_date: z.string(), + tot_pub_debt_out_amt: z.string(), + })).min(1), + }), + ), + }, + "noaa-gml-co2": { + url: "https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_mm_mlo.csv", + range: "bytes=0-4096", + contentTypes: ["text/csv", "text/plain", "application/octet-stream"], + validate(body: Uint8Array) { + const text = new TextDecoder().decode(body); + return text.includes("year") && (text.includes("average") || text.includes("deseasonalized")) + ? null + : "CSV is missing year or average columns"; + }, + }, + "pubchem-compounds": { + url: "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/aspirin/property/MolecularFormula,MolecularWeight/JSON", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + PropertyTable: z.object({ + Properties: z.array(z.object({ + CID: z.number(), + MolecularFormula: z.string(), + })).min(1), + }), + }), + ), + }, + "dailymed-drug-labels": { + url: "https://dailymed.nlm.nih.gov/dailymed/services/v2/spls.json?drug_name=ibuprofen&pagesize=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + data: z.array(z.record(z.string(), z.unknown())).min(1), + }), + ), + }, + "openfda-device-events": { + url: "https://api.fda.gov/device/event.json?search=date_received:%5B20240101+TO+20241231%5D&limit=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + meta: z.object({ + results: z.object({ total: z.number() }), + }), + results: z.array(z.object({ + mdr_report_key: z.string().optional(), + }).passthrough()).min(1), + }), + ), + }, + "fema-nfip-redacted-claims": { + url: "https://www.fema.gov/api/open/v2/FimaNfipClaims?$top=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + FimaNfipClaims: z.array(z.object({ + id: z.string(), + state: z.string().optional(), + }).passthrough()).min(1), + }), + ), + }, + "ripe-stat": { + url: "https://stat.ripe.net/data/prefix-overview/data.json?resource=8.8.8.0/24", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + data: z.object({ + resource: z.string(), + }).passthrough(), + }), + ), + }, + "nchs-provisional-mortality": { + url: "https://data.cdc.gov/resource/muzy-jte6.json?$limit=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.array(z.object({ + jurisdiction_of_occurrence: z.string(), + }).passthrough()).min(1), + ), + }, + "noaa-ndbc-buoys": { + url: "https://www.ndbc.noaa.gov/data/realtime2/41001.txt", + range: "bytes=0-2048", + contentTypes: ["text/plain", "text/csv", "application/octet-stream"], + validate(body: Uint8Array) { + const header = new TextDecoder().decode(body); + return header.includes("WSPD") && header.includes("WVHT") + ? null + : "buoy file is missing WSPD or WVHT"; + }, + }, + "ons-statistics": { + url: "https://api.beta.ons.gov.uk/v1/datasets/cpih01", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + id: z.string(), + title: z.string().optional(), + }).passthrough(), + ), + }, } satisfies Record< string, { diff --git a/src/lib/url-validation.test.ts b/src/lib/url-validation.test.ts index 9a648df..772093d 100644 --- a/src/lib/url-validation.test.ts +++ b/src/lib/url-validation.test.ts @@ -465,6 +465,34 @@ describe("checkUrl", () => { "2026-11-15", 2, ], + [ + "https://volcano.si.edu/", + 403, + "Smithsonian GVP blocks automated validation from some regions; reconfirmed 2026-08-18", + "2026-11-16", + 2, + ], + [ + "https://volcano.si.edu/gvp_webservices.cfm", + 403, + "Smithsonian GVP blocks automated validation from some regions; reconfirmed 2026-08-18", + "2026-11-16", + 2, + ], + [ + "https://lda.gov/api/", + 403, + "Senate LDA blocks automated validation from some regions; reconfirmed 2026-08-18", + "2026-11-16", + 2, + ], + [ + "https://main.un.org/securitycouncil/en/content/un-sc-consolidated-list", + 403, + "UN Security Council pages block automated validation from some regions; reconfirmed 2026-08-18", + "2026-11-16", + 2, + ], ])( "allows an exact, unexpired protected-URL exception for %s", async (url, status, reason, expires, expectedAttempts) => { diff --git a/src/lib/url-validation.ts b/src/lib/url-validation.ts index 4fcb382..77d8d50 100644 --- a/src/lib/url-validation.ts +++ b/src/lib/url-validation.ts @@ -153,6 +153,42 @@ const STATUS_EXCEPTIONS = new Map([ expires: "2026-11-15", }, ], + [ + "https://volcano.si.edu/", + { + statuses: [403], + reason: + "Smithsonian GVP blocks automated validation from some regions; reconfirmed 2026-08-18", + expires: "2026-11-16", + }, + ], + [ + "https://volcano.si.edu/gvp_webservices.cfm", + { + statuses: [403], + reason: + "Smithsonian GVP blocks automated validation from some regions; reconfirmed 2026-08-18", + expires: "2026-11-16", + }, + ], + [ + "https://lda.gov/api/", + { + statuses: [403], + reason: + "Senate LDA blocks automated validation from some regions; reconfirmed 2026-08-18", + expires: "2026-11-16", + }, + ], + [ + "https://main.un.org/securitycouncil/en/content/un-sc-consolidated-list", + { + statuses: [403], + reason: + "UN Security Council pages block automated validation from some regions; reconfirmed 2026-08-18", + expires: "2026-11-16", + }, + ], ]); function isReachable(status: number | null): boolean {