diff --git a/README.md b/README.md index b0a8e8f..8134d2b 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ No application code changes required. ## Catalog -The catalog currently contains 76 operational datasets spanning natural +The catalog currently contains 100 operational datasets spanning natural hazards, weather, water, climate, flood risk, drought, space weather, global disaster alerts, public health, clinical research, cybersecurity, package graphs, legislation, sanctions screening, government spending and procurement, @@ -157,7 +157,9 @@ petroleum inventories, prediction markets, geospatial analysis, places, broadband, bridges, EV charging, consumer finance, education, K-12 directories, housing prices, rents, food and product recalls, elections, European statistics, live transit feeds, aviation, provider directories, drinking water, preprints, -pageviews, agriculture, and tropical cyclones. The YAML files in +pageviews, agriculture, tropical cyclones, crime, companies, occupations, +global forecasts, live OSM, food products, nursing homes, LEI, and euro-area +statistics. The YAML files in [`data/datasets`](data/datasets) are the source of truth for the current list. ## Deploy to Vercel diff --git a/data/datasets/cdc-social-vulnerability-index.yaml b/data/datasets/cdc-social-vulnerability-index.yaml new file mode 100644 index 0000000..55f3828 --- /dev/null +++ b/data/datasets/cdc-social-vulnerability-index.yaml @@ -0,0 +1,88 @@ +id: cdc-social-vulnerability-index +name: CDC ATSDR Social Vulnerability Index +description: > + Census-tract and county social-vulnerability rankings for building emergency + planning, outreach, and hazard-equity tools. +theme: Health, Food & Safety +url: https://www.atsdr.cdc.gov/place-health/php/svi/svi-data-documentation-download.html +access_type: + - download + - api +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 0.5 +formats: + - CSV + - JSON +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Social Vulnerability Index + license_marker: federal government materials +domains: + - Community Health + - Emergency Management + - Demographics +data_types: + - Tabular + - Geospatial + - Index Scores +tasks: + - Risk Assessment + - Geographic Analysis + - Emergency Planning +difficulty: intermediate +geography: + - United States +temporal_coverage: 2000-2022 SVI releases +update_frequency: occasional +provider: Agency for Toxic Substances and Disease Registry +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + SVI ranks communities on socioeconomic, household, minority-status, and + housing themes derived from ACS. Start with 2022 county ranks for the + United States file. Percentiles are relative within one vintage, so do not + compare 2020 and 2022 ranks as a time series. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the SVI data-download page and read the comparison-over-time warning. + - Query a bounded page of 2022 county ranks from the public feature service. + - Keep FIPS codes, county names, and the overall RPL_THEMES percentile. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://onemap.cdc.gov/onemapservices/rest/services/SVI/" + "CDC_ATSDR_Social_Vulnerability_Index_2022_USA/FeatureServer/1/query", + params={ + "where": "1=1", + "outFields": "ST,STATE,STCNTY,COUNTY,RPL_THEMES,E_TOTPOP", + "returnGeometry": "false", + "resultRecordCount": 50, + "f": "json", + }, + timeout=30, + ) + response.raise_for_status() + counties = pd.DataFrame( + [feature["attributes"] for feature in response.json()["features"]] + ) + print(counties.head()) + first_project: + title: Rank counties by 2022 overall SVI + goal: Test whether SVI percentiles can flag counties for emergency-planning outreach. + steps: + - Keep state, county, FIPS, population, and RPL_THEMES. + - Sort by overall percentile and report how many ranks are missing. + - Explain that percentiles are relative within 2022 and should not be subtracted from an earlier SVI vintage. diff --git a/data/datasets/census-county-business-patterns.yaml b/data/datasets/census-county-business-patterns.yaml new file mode 100644 index 0000000..8e73158 --- /dev/null +++ b/data/datasets/census-county-business-patterns.yaml @@ -0,0 +1,86 @@ +id: census-county-business-patterns +name: Census County Business Patterns +description: > + Annual establishment, employment, and payroll counts by industry and county + for building local industry-mix and site-selection tools. +theme: Markets & Economics +url: https://www.census.gov/programs-surveys/cbp.html +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0.001 +size_gb_max: 2 +formats: + - JSON + - CSV +license: U.S. Census Bureau API Terms of Service +license_url: https://www.census.gov/data/developers/about/terms-of-service.html +url_checks: + source_marker: County Business Patterns + license_marker: Terms of Service Agreement +domains: + - Local Economics + - Business + - Labor Economics +data_types: + - Tabular + - Survey Estimates +tasks: + - Market Sizing + - Site Selection + - Industry Analysis +difficulty: intermediate +geography: + - United States +temporal_coverage: 1986-present +update_frequency: annual +provider: U.S. Census Bureau +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + County Business Patterns counts establishments with paid employees by + NAICS and geography. Start with one state and the all-industry NAICS code. + Employment is a March 12 snapshot, payroll can be suppressed, and CBP + excludes most government and self-employed activity. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free Census Data API key saved in the CENSUS_API_KEY environment variable + access_steps: + - Request a free Census Data API key and open the CBP program page. + - Review the current year variables for establishments, employment, and payroll. + - Request all-industry county totals for one state. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.census.gov/data/2023/cbp", + params={ + "get": "NAME,ESTAB,EMP,PAYANN,NAICS2017_LABEL", + "for": "county:*", + "in": "state:08", + "NAICS2017": "00", + "key": os.environ["CENSUS_API_KEY"], + }, + timeout=30, + ) + response.raise_for_status() + rows = response.json() + counties = pd.DataFrame(rows[1:], columns=rows[0]) + print(counties.head()) + first_project: + title: Rank one state's counties by establishment counts + goal: Test whether CBP can describe which counties hold the most paid employers. + steps: + - Keep county name, establishments, employment, and annual payroll as numbers. + - Rank counties by establishments and flag suppressed payroll cells. + - Explain that March employment, NAICS suppression, and the exclusion of most government work limit a local industry story. diff --git a/data/datasets/cftc-commitment-of-traders.yaml b/data/datasets/cftc-commitment-of-traders.yaml new file mode 100644 index 0000000..c590db9 --- /dev/null +++ b/data/datasets/cftc-commitment-of-traders.yaml @@ -0,0 +1,88 @@ +id: cftc-commitment-of-traders +name: CFTC Commitments of Traders +description: > + Weekly futures positioning by trader category for building commodities, + rates, and speculative-position monitors. +theme: Markets & Economics +url: https://www.cftc.gov/MarketReports/CommitmentsofTraders/index.htm +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.5 +formats: + - JSON + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Commitments of Traders + license_marker: federal government materials +domains: + - Commodities + - Finance + - Capital Markets +data_types: + - Time Series + - Tabular +tasks: + - Market Monitoring + - Positioning Analysis + - Trend Analysis +difficulty: intermediate +geography: + - United States +temporal_coverage: 1986-present weekly reports +update_frequency: weekly +provider: Commodity Futures Trading Commission +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The COT reports break down open interest for futures markets with enough + large traders. Start with one contract in the legacy combined dataset. + Categories are regulatory, not strategy labels, and Tuesday positions are + published later in the week. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Commitments of Traders page and the public reporting environment. + - Request a bounded page of legacy combined records for one market. + - Keep report date, market name, and commercial versus noncommercial long/short. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://publicreporting.cftc.gov/resource/jun7-fc8e.json", + params={ + "$limit": 20, + "$order": "report_date_as_yyyy_mm_dd DESC", + "contract_market_name": "GOLD", + }, + timeout=30, + ) + response.raise_for_status() + cot = pd.DataFrame(response.json()) + print(cot[[ + "report_date_as_yyyy_mm_dd", + "contract_market_name", + "open_interest_all", + "noncomm_positions_long_all", + "noncomm_positions_short_all", + ]].head()) + first_project: + title: Track speculative positioning in one futures market + goal: Test whether weekly COT records can power a bounded positioning monitor. + steps: + - Keep report date, market name, open interest, and noncommercial long/short. + - Compute net noncommercial positioning and note the publication lag from Tuesday. + - Explain that trader categories are CFTC reporting classes, not a forecast of price direction. diff --git a/data/datasets/cms-nursing-homes.yaml b/data/datasets/cms-nursing-homes.yaml new file mode 100644 index 0000000..904058c --- /dev/null +++ b/data/datasets/cms-nursing-homes.yaml @@ -0,0 +1,92 @@ +id: cms-nursing-homes +name: CMS Nursing Home Compare +description: > + Medicare-certified nursing home ratings, staffing, and inspection records + for building local long-term-care comparison tools. +theme: Health, Food & Safety +url: https://data.cms.gov/provider-data/api/1/metastore/schemas/dataset/items/4pq5-n9py +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.2 +formats: + - JSON + - CSV +license: CMS public-data principles +license_url: https://www.cms.gov/data-research/cms-data/cms-data-principles-and-operating-norms +url_checks: + source_marker: Provider Information + license_marker: CMS data is a public good +domains: + - Health Care + - Health Services + - Long-Term Care +data_types: + - Tabular + - Geospatial +tasks: + - Provider Comparison + - Geographic Analysis + - Quality Monitoring +difficulty: beginner +geography: + - United States +temporal_coverage: current Medicare-certified nursing homes +update_frequency: monthly +provider: Centers for Medicare & Medicaid Services +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + Nursing Home Provider Information lists certified facilities with Five-Star + ratings, staffing, and inspection dates. Start with one state and the + stable dataset identifier. Ratings can be missing, inspections lag events, + and a star rating is not a clinical recommendation. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Provider Information dataset and read the data dictionary. + - Query the stable dataset identifier for a bounded page of facilities. + - Preserve CMS Certification Number values as text. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.cms.gov/provider-data/api/1/datastore/query/4pq5-n9py/0", + params={ + "limit": 20, + "conditions[0][property]": "state", + "conditions[0][value]": "CA", + "conditions[0][operator]": "=", + }, + timeout=30, + ) + response.raise_for_status() + homes = pd.DataFrame(response.json()["results"]) + print( + homes[ + [ + "cms_certification_number_ccn", + "provider_name", + "state", + "overall_rating", + ] + ].head(20) + ) + first_project: + title: Compare nursing homes within one state + goal: Summarize Five-Star ratings and staffing for facilities in one state. + steps: + - Filter records by state while preserving CMS Certification Numbers as text. + - Summarize overall ratings and report missingness before ranking facilities. + - Explain why inspection lag, missing ratings, and resident mix limit a quality comparison. diff --git a/data/datasets/companies-house-uk.yaml b/data/datasets/companies-house-uk.yaml new file mode 100644 index 0000000..66230cd --- /dev/null +++ b/data/datasets/companies-house-uk.yaml @@ -0,0 +1,79 @@ +id: companies-house-uk +name: Companies House Register +description: > + Public UK company, officer, and filing records for building company-status, + officer, and industry-lookup tools. +theme: Markets & Economics +url: https://developer.company-information.service.gov.uk/get-started +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 10 +formats: + - JSON + - CSV +license: Companies House public register terms +license_url: https://www.gov.uk/guidance/companies-house-data-products +url_checks: + source_marker: Get started with the Companies House API + license_marker: impose no rules or requirements +domains: + - Business + - Corporate Filings + - Regulation +data_types: + - Registry Data + - Documents +tasks: + - Entity Resolution + - Company Research + - Screening +difficulty: beginner +geography: + - United Kingdom +temporal_coverage: live companies on the public register +update_frequency: daily +provider: Companies House +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The Companies House API returns public register records for UK companies. + Start with one company number after creating a free API key. Register data + can lag filings, dissolved companies remain searchable, and you remain + responsible for data-protection rules when publishing officer details. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free Companies House API key saved in the COMPANIES_HOUSE_API_KEY environment variable + access_steps: + - Register a Companies House developer account and create an API key. + - Read the data-products page, including the notice that the public register has no reuse licence restrictions. + - Request one company profile by company number. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.company-information.service.gov.uk/company/00000006", + auth=(os.environ["COMPANIES_HOUSE_API_KEY"], ""), + timeout=30, + ) + response.raise_for_status() + profile = pd.json_normalize(response.json()) + print(profile.filter(regex="company_name|company_number|company_status|sic").head()) + first_project: + title: Look up one UK company's status and SIC codes + goal: Test whether the public register can power a company-status card. + steps: + - Keep company number, name, status, and SIC codes. + - Record the accounts and confirmation-statement due dates if present. + - Explain that officer and address fields are personal data and that register lag can make a "live" status stale. diff --git a/data/datasets/ecb-statistical-data-warehouse.yaml b/data/datasets/ecb-statistical-data-warehouse.yaml new file mode 100644 index 0000000..e04e15d --- /dev/null +++ b/data/datasets/ecb-statistical-data-warehouse.yaml @@ -0,0 +1,86 @@ +id: ecb-statistical-data-warehouse +name: ECB Statistical Data Warehouse +description: > + Euro-area official statistics on exchange rates, prices, and monetary + aggregates for building policy-rate and inflation monitors. +theme: Markets & Economics +url: https://data.ecb.europa.eu/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV + - SDMX +license: ECB copyright and reuse notice +license_url: https://www.ecb.europa.eu/home/disclaimer/html/index.en.html +url_checks: + source_marker: ECB Data Portal + license_marker: Disclaimer +domains: + - Macroeconomics + - Finance + - International Statistics +data_types: + - Time Series +tasks: + - Trend Analysis + - Policy Monitoring + - Market Research +difficulty: intermediate +geography: + - Europe +temporal_coverage: euro-area series with varying starts +update_frequency: daily +provider: European Central Bank +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + The ECB SDMX 2.1 API returns official euro-area statistics. Start with one + exchange-rate series and a short observation window. Series keys are dense, + vintages can be revised, and a rate observation is not a trading signal. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the ECB Data Portal and the SDMX web-service help. + - Request the last ten daily USD/EUR reference rates. + - Keep the series key, observation date, and value. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data-api.ecb.europa.eu/service/data/EXR/D.USD.EUR.SP00.A", + params={"lastNObservations": 10, "format": "jsondata"}, + headers={"Accept": "application/json"}, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + series = payload["dataSets"][0]["series"] + observations = next(iter(series.values()))["observations"] + dates = payload["structure"]["dimensions"]["observation"][0]["values"] + rows = [ + {"date": dates[int(index)]["id"], "usd_per_eur": value[0]} + for index, value in observations.items() + ] + rates = pd.DataFrame(rows) + print(rates) + first_project: + title: Chart recent USD per EUR reference rates + goal: Test whether one ECB series can power a bounded FX monitor. + steps: + - Keep observation dates and USD-per-EUR values from the EXR dataflow. + - Plot the last ten daily observations and record the series key. + - Explain that ECB reference rates are not transaction prices and can be revised. diff --git a/data/datasets/fbi-crime-data-explorer.yaml b/data/datasets/fbi-crime-data-explorer.yaml new file mode 100644 index 0000000..832e599 --- /dev/null +++ b/data/datasets/fbi-crime-data-explorer.yaml @@ -0,0 +1,79 @@ +id: fbi-crime-data-explorer +name: FBI Crime Data Explorer +description: > + Uniform Crime Reporting summaries and NIBRS extracts for building U.S. crime + trend and agency-comparison tools. +theme: Government & Policy +url: https://www.justice.gov/developer +access_type: + - api + - download +api_key_required: true +free_to_access: true +size_gb_min: 0 +size_gb_max: 5 +formats: + - JSON + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: FBI Crime Data API + license_marker: federal government materials +domains: + - Public Safety + - Crime + - Justice +data_types: + - Event Data + - Time Series +tasks: + - Trend Analysis + - Geographic Analysis + - Agency Comparison +difficulty: intermediate +geography: + - United States +temporal_coverage: 1985-present summarized UCR series +update_frequency: annual +provider: Federal Bureau of Investigation +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The FBI Crime Data API returns Uniform Crime Reporting summaries. Start + with national participation or one state series after requesting a free + api.data.gov key. Agency coverage is voluntary, SRS and NIBRS are not + interchangeable, and recent years can be incomplete. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - A free api.data.gov key saved in the FBI_CDE_API_KEY environment variable + access_steps: + - Open Crime Data Explorer and request a free key at api.data.gov. + - Read the API documentation for summarized state estimates. + - Request a bounded violent-crime series for one state. + python: + packages: + - pandas + - requests + code: | + import os + import pandas as pd + import requests + + response = requests.get( + "https://api.usa.gov/crime/fbi/sapi/api/estimates/states/CA/2020/2022", + params={"api_key": os.environ["FBI_CDE_API_KEY"]}, + timeout=30, + ) + response.raise_for_status() + estimates = pd.DataFrame(response.json().get("results", response.json())) + print(estimates.head()) + first_project: + title: Compare one state's reported violent crime across three years + goal: Test whether CDE estimates can power a bounded state crime trend. + steps: + - Keep year, state, and violent-crime counts from the estimates response. + - Chart the three-year series and record any missing years. + - Explain that voluntary reporting, SRS-to-NIBRS transition, and population changes limit year-to-year comparison. diff --git a/data/datasets/fda-orange-book.yaml b/data/datasets/fda-orange-book.yaml new file mode 100644 index 0000000..1c73d54 --- /dev/null +++ b/data/datasets/fda-orange-book.yaml @@ -0,0 +1,81 @@ +id: fda-orange-book +name: FDA Orange Book +description: > + Approved drug products with therapeutic-equivalence, patent, and exclusivity + dates for building generic-entry and substitution-research tools. +theme: Health, Food & Safety +url: https://www.fda.gov/drugs/drug-approvals-and-databases/orange-book-data-files +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - TXT + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Orange Book Data Files + license_marker: federal government materials +domains: + - Pharmaceuticals + - Regulation + - Health Care +data_types: + - Registry Data + - Tabular +tasks: + - Market Research + - Patent Research + - Product Lookup +difficulty: intermediate +geography: + - United States +temporal_coverage: currently listed approved products +update_frequency: monthly +provider: U.S. Food and Drug Administration +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The Orange Book lists FDA-approved products with therapeutic-equivalence + codes, patents, and exclusivity. Start with products.txt from the published + zip. Exclusivity dates are not a guaranteed generic-entry day, and omitted + patents can still block substitution. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Orange Book data-files page and the federal public-data copyright page. + - Download the published zip and read products.txt with the documented tilde delimiter. + - Keep application number, trade name, and therapeutic-equivalence code. + python: + packages: + - pandas + - requests + code: | + import io + import zipfile + + import pandas as pd + import requests + + response = requests.get( + "https://www.fda.gov/media/76860/download", + timeout=30, + ) + response.raise_for_status() + with zipfile.ZipFile(io.BytesIO(response.content)) as archive: + products = pd.read_csv(archive.open("products.txt"), sep="~") + sample = products[products["Trade_Name"].str.upper() == "LIPITOR"] + print(sample[["Appl_No", "Trade_Name", "Ingredient", "TE_Code", "Approval_Date"]].head()) + first_project: + title: Inspect one brand's approved products and exclusivity + goal: Test whether Orange Book fields can support a bounded generic-entry review. + steps: + - Keep application number, trade name, ingredient, and TE code for Lipitor. + - Join patent.txt or exclusivity.txt on application number only when those dates are present. + - Explain that remaining patents, litigation, and supply can move generic entry after an exclusivity date. diff --git a/data/datasets/fdic-bank-find.yaml b/data/datasets/fdic-bank-find.yaml new file mode 100644 index 0000000..f512f57 --- /dev/null +++ b/data/datasets/fdic-bank-find.yaml @@ -0,0 +1,86 @@ +id: fdic-bank-find +name: FDIC BankFind Suite +description: > + FDIC-insured institution, branch, financial, and failure records for building + bank-directory, branch-map, and failure-history tools. +theme: Markets & Economics +url: https://api.fdic.gov/banks/docs +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: BankFind Suite + license_marker: federal government materials +domains: + - Finance + - Banking + - Regulation +data_types: + - Registry Data + - Geospatial + - Time Series +tasks: + - Directory Search + - Geographic Analysis + - Failure Analysis +difficulty: beginner +geography: + - United States +temporal_coverage: 1934-present failures with current institutions +update_frequency: weekly +provider: Federal Deposit Insurance Corporation +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + BankFind Suite exposes institution and branch records through a public API. + Start with one state abbreviation and a small result limit. A key is not + required today, though FDIC documents a registration path. Branch lists are + not a measure of service quality, and failure records are historical. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the BankFind API documentation and note the Elastic-style filter syntax. + - Request a bounded page of Iowa institutions. + - Keep certificate numbers, names, and city fields as text. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.fdic.gov/banks/institutions", + params={ + "filters": 'STALP:IA', + "fields": "NAME,CERT,CITY,STALP,ACTIVE", + "limit": 20, + "format": "json", + }, + timeout=30, + ) + response.raise_for_status() + banks = pd.json_normalize( + [row["data"] for row in response.json()["data"]] + ) + print(banks[["NAME", "CERT", "CITY", "STALP", "ACTIVE"]].head()) + first_project: + title: List active FDIC-insured institutions in one state + goal: Test whether BankFind can power a bounded state bank directory. + steps: + - Keep certificate number, name, city, and active flag. + - Count active versus inactive institutions and flag missing cities. + - Explain that a branch or certificate list is not a safety rating and that failure history lives on a separate endpoint. diff --git a/data/datasets/first-epss.yaml b/data/datasets/first-epss.yaml new file mode 100644 index 0000000..8bb871f --- /dev/null +++ b/data/datasets/first-epss.yaml @@ -0,0 +1,78 @@ +id: first-epss +name: FIRST Exploit Prediction Scoring System +description: > + Daily CVE exploit-probability scores for building patch-priority queues that + go beyond severity ratings and known-exploited lists. +theme: Technology & Cybersecurity +url: https://www.first.org/epss/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - JSON + - CSV +license: FIRST EPSS Usage Agreement +license_url: https://www.first.org/epss/faq +url_checks: + source_marker: Exploit Prediction Scoring System + license_marker: published freely via CSV download and API +domains: + - Cybersecurity + - Software Security +data_types: + - Scores + - Event Data +tasks: + - Vulnerability Monitoring + - Risk Assessment + - Alerting +difficulty: beginner +geography: + - Global +temporal_coverage: 2021-present +update_frequency: daily +provider: Forum of Incident Response and Security Teams +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + EPSS estimates the probability that a published CVE will be exploited in + the next 30 days. Start with one CVE and the current daily score. EPSS is + a likelihood model, not proof of exploitation, and it does not replace + asset exposure or CISA Known Exploited Vulnerabilities evidence. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the EPSS FAQ, including the request to attribute FIRST when scores appear in a product. + - Request the current score for one CVE from the public API. + - Keep the CVE identifier, EPSS score, percentile, and retrieval time. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.first.org/data/v1/epss", + params={"cve": "CVE-2024-3400"}, + timeout=30, + ) + response.raise_for_status() + scores = pd.DataFrame(response.json()["data"]) + scores["retrieved_at_utc"] = pd.Timestamp.now(tz="UTC") + print(scores[["cve", "epss", "percentile", "date"]].head()) + first_project: + title: Compare one CVE's exploit probability with its KEV status + goal: Test whether EPSS can rank a published CVE before treating it as an active incident. + steps: + - Retrieve the current EPSS score and percentile for CVE-2024-3400. + - Record whether the same CVE appears on an authorized Known Exploited Vulnerabilities list. + - Explain why a high EPSS score is not the same as confirmed exploitation or organizational exposure. diff --git a/data/datasets/gleif-lei.yaml b/data/datasets/gleif-lei.yaml new file mode 100644 index 0000000..d9119c9 --- /dev/null +++ b/data/datasets/gleif-lei.yaml @@ -0,0 +1,79 @@ +id: gleif-lei +name: GLEIF Legal Entity Identifier Index +description: > + Global legal-entity identifiers and reference data for building company + lookup, ownership, and sanctions-screening tools. +theme: Markets & Economics +url: https://www.gleif.org/en/lei-data/gleif-api/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.1 +size_gb_max: 2 +formats: + - JSON + - XML +license: CC0 1.0 Universal +license_url: https://www.gleif.org/en/meta/lei-data-terms-of-use/ +url_checks: + source_marker: GLEIF API + license_marker: CC0 licence +domains: + - Finance + - Business + - Compliance +data_types: + - Registry Data + - Relational Data +tasks: + - Entity Resolution + - Ownership Analysis + - Screening +difficulty: beginner +geography: + - Global +temporal_coverage: current LEI records with daily golden copies +update_frequency: daily +provider: Global Legal Entity Identifier Foundation +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + GLEIF publishes Legal Entity Identifiers and reference data under CC0. + Start with one known LEI rather than the concatenated golden copy. LEI + coverage is not universal, reference data can lag the entity's own filings, + and an LEI is not a credit, sanctions, or beneficial-ownership decision. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the GLEIF API page and the LEI Data Terms of Use, including the CC0 licence. + - Request one LEI record from the public API. + - Keep the LEI, legal name, jurisdiction, and registration status. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.gleif.org/api/v1/lei-records", + params={"filter[lei]": "5493001KJTIIGC8Y1R12"}, + headers={"Accept": "application/vnd.api+json"}, + timeout=30, + ) + response.raise_for_status() + records = pd.json_normalize(response.json()["data"]) + print(records.filter(regex="id|legalName|jurisdiction|status").head()) + first_project: + title: Look up one legal entity by LEI + goal: Test whether GLEIF can power a company-identity card from a single identifier. + steps: + - Keep the LEI, legal name, jurisdiction, and registration status. + - Record the last update time and any missing address fields. + - Explain that an LEI does not by itself confirm ownership, sanctions status, or that the entity is still operating. diff --git a/data/datasets/legislation-gov-uk.yaml b/data/datasets/legislation-gov-uk.yaml new file mode 100644 index 0000000..0c89196 --- /dev/null +++ b/data/datasets/legislation-gov-uk.yaml @@ -0,0 +1,89 @@ +id: legislation-gov-uk +name: legislation.gov.uk +description: > + Consolidated UK legislation in machine-readable form for building statute + lookup, citation, and change-monitoring tools. +theme: Government & Policy +url: https://www.legislation.gov.uk/developer +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 5 +formats: + - XML + - HTML + - JSON +license: Open Government Licence v3.0 +license_url: https://www.legislation.gov.uk/developer +url_checks: + source_marker: Legislation API + license_marker: Open Government Licence v3.0 +domains: + - Legislation + - Public Policy + - Regulation +data_types: + - Documents + - Legal Text +tasks: + - Document Search + - Change Monitoring + - Citation Research +difficulty: beginner +geography: + - United Kingdom +temporal_coverage: historic and in-force UK legislation +update_frequency: daily +provider: The National Archives +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + legislation.gov.uk publishes consolidated UK Acts and statutory instruments + through a documented HTTP API. Start with one well-known Act and its XML + representation. The service is free to reuse commercially under OGL v3.0; + some EU-derived items carry an additional Commission reuse notice. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the developer zone and read the Open Government Licence attribution requirement. + - Request the Data Protection Act 2018 as XML. + - Keep the official URI when you cite a section in any downstream product. + python: + packages: + - pandas + - requests + code: | + import xml.etree.ElementTree as ET + import pandas as pd + import requests + + response = requests.get( + "https://www.legislation.gov.uk/ukpga/2018/12/data.xml", + timeout=30, + ) + response.raise_for_status() + root = ET.fromstring(response.content) + ns = {"a": "http://www.legislation.gov.uk/namespaces/legislation"} + title = root.findtext(".//a:Title", default="", namespaces=ns) + sections = [ + { + "number": item.findtext("a:Number", default="", namespaces=ns), + "title": item.findtext("a:Title", default="", namespaces=ns), + } + for item in root.findall(".//a:P1group", ns)[:20] + ] + print(title) + print(pd.DataFrame(sections).head()) + first_project: + title: Extract section titles from one UK Act + goal: Test whether legislation.gov.uk URIs can power a bounded statute browser. + steps: + - Parse the Data Protection Act 2018 XML and keep official section numbers. + - List the first twenty section titles and flag any missing numbers. + - Attribute Crown copyright under OGL v3.0 and note that EU-origin items may need a second acknowledgement. diff --git a/data/datasets/met-norway-locationforecast.yaml b/data/datasets/met-norway-locationforecast.yaml new file mode 100644 index 0000000..2d1bee2 --- /dev/null +++ b/data/datasets/met-norway-locationforecast.yaml @@ -0,0 +1,88 @@ +id: met-norway-locationforecast +name: MET Norway Locationforecast +description: > + Global point weather forecasts for building location-specific planning tools + outside the U.S. National Weather Service footprint. +theme: Environment & Hazards +url: https://api.met.no/weatherapi/locationforecast/2.0/documentation +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.01 +formats: + - JSON +license: CC BY 4.0 and Norwegian Licence for Open Government Data 2.0 +license_url: https://api.met.no/doc/License +url_checks: + source_marker: Weather forecasts for any location on earth + license_marker: Norwegian Licence for Open Government Data +domains: + - Weather + - Forecasting + - Emergency Management +data_types: + - Time Series + - Forecast Data +tasks: + - Forecasting + - Operational Planning + - Alerting +difficulty: beginner +geography: + - Global +temporal_coverage: current forecasts out to about nine days +update_frequency: continuous +provider: Norwegian Meteorological Institute +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + Locationforecast returns JSON weather for any latitude and longitude. + Start with the compact product for one coordinate. Identify the client in + User-Agent or MET Norway returns 403. Forecasts change, and this product is + not an official warning service. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the Locationforecast documentation and licensing page, including commercial reuse under CC BY and NLOD. + - Choose one coordinate and the compact JSON method. + - Send a unique User-Agent with a contact URL. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.met.no/weatherapi/locationforecast/2.0/compact", + params={"lat": 59.91, "lon": 10.75}, + headers={ + "User-Agent": ( + "TrilemmaDataCatalogExample/1.0 " + "(https://data.trilemma.foundation)" + ) + }, + timeout=30, + ) + response.raise_for_status() + timeseries = pd.json_normalize( + response.json()["properties"]["timeseries"] + ) + print( + timeseries.filter( + regex="time|air_temperature|wind_speed|precipitation_amount" + ).head() + ) + first_project: + title: Inspect Oslo's next forecast hours + goal: Test whether Locationforecast can power a bounded local planning card. + steps: + - Keep time, air temperature, wind speed, and precipitation for the first hours. + - Convert times to timezone-aware timestamps before ranking the wettest hour. + - Credit MET Norway and explain that a forecast is not an official warning. diff --git a/data/datasets/nih-reporter-projects.yaml b/data/datasets/nih-reporter-projects.yaml new file mode 100644 index 0000000..923f3d7 --- /dev/null +++ b/data/datasets/nih-reporter-projects.yaml @@ -0,0 +1,92 @@ +id: nih-reporter-projects +name: NIH RePORTER Projects +description: > + Funded NIH and HHS project records for building research-funding monitors, + institution dashboards, and topic-prospecting tools. +theme: Government & Policy +url: https://api.reporter.nih.gov/ +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: RePORTER Project + license_marker: federal government materials +domains: + - Research Funding + - Biomedical Research + - Public Spending +data_types: + - Tabular + - Documents +tasks: + - Funding Analysis + - Topic Research + - Institution Comparison +difficulty: intermediate +geography: + - United States +temporal_coverage: active and historical NIH-funded projects +update_frequency: weekly +provider: National Institutes of Health +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + RePORTER exposes project abstracts, awards, institutes, and organizations + through a JSON POST search. Start with one fiscal year and one keyword. + Award amounts can be multi-year, subprojects can double-count activity, + and the API is not a complete picture of all U.S. biomedical funding. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the RePORTER API documentation and note that search uses POST, not GET. + - Search one fiscal year for a single keyword with a small result limit. + - Keep application IDs, organization names, and award amounts. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.post( + "https://api.reporter.nih.gov/v2/projects/search", + json={ + "criteria": { + "fiscal_years": [2025], + "advanced_text_search": { + "search_field": "terms", + "search_text": "CRISPR", + }, + }, + "limit": 25, + "include_fields": [ + "ApplId", + "ProjectTitle", + "Organization", + "AwardAmount", + "FiscalYear", + ], + }, + timeout=30, + ) + response.raise_for_status() + projects = pd.json_normalize(response.json()["results"]) + print(projects.head()) + first_project: + title: List recent NIH CRISPR project awards + goal: Test whether RePORTER can power a bounded funding inbox for one research topic. + steps: + - Keep application ID, title, organization, fiscal year, and award amount. + - Sum award amounts by organization and flag missing amounts. + - Explain that multi-year awards and subprojects can inflate totals if treated as independent grants. diff --git a/data/datasets/noaa-storm-events.yaml b/data/datasets/noaa-storm-events.yaml new file mode 100644 index 0000000..8e6f827 --- /dev/null +++ b/data/datasets/noaa-storm-events.yaml @@ -0,0 +1,94 @@ +id: noaa-storm-events +name: NOAA Storm Events Database +description: > + National Weather Service storm reports for building U.S. severe-weather + history, loss, and event-type analysis tools. +theme: Environment & Hazards +url: https://www.ncei.noaa.gov/stormevents/ftp.jsp +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 2 +formats: + - CSV +license: U.S. Government public data / federal copyright guidance +license_url: https://www.usa.gov/government-copyright +url_checks: + source_marker: Storm Events Database + license_marker: federal government materials +domains: + - Weather + - Natural Hazards + - Emergency Management +data_types: + - Event Data + - Tabular +tasks: + - Event Analysis + - Loss Analysis + - Geographic Analysis +difficulty: intermediate +geography: + - United States +temporal_coverage: 1950-present +update_frequency: monthly +provider: NOAA National Centers for Environmental Information +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + Storm Events publishes yearly details, locations, and fatalities CSVs. + Start with one recent details file, not the full 1950-present archive. + Reports are NWS-entered events, so counts are not a complete census of + every storm and damage figures can be estimated. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the bulk-download page and the directory of yearly CSV files. + - Choose the latest details file for one year from the directory listing. + - Read a bounded row sample instead of concatenating every historical year. + python: + packages: + - pandas + - requests + code: | + import gzip + import re + from io import BytesIO + + import pandas as pd + import requests + + listing = requests.get( + "https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/", + timeout=30, + ) + listing.raise_for_status() + names = re.findall( + r'href="(StormEvents_details-ftp_v1.0_d2025_c\d+\.csv\.gz)"', + listing.text, + ) + filename = sorted(names)[-1] + archive = requests.get( + "https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/" + + filename, + timeout=120, + ) + archive.raise_for_status() + events = pd.read_csv( + BytesIO(gzip.decompress(archive.content)), + nrows=200, + low_memory=False, + ) + print(events[["EVENT_ID", "STATE", "EVENT_TYPE", "BEGIN_DATE_TIME"]].head()) + first_project: + title: Count 2025 storm-event types in a bounded sample + goal: Test whether one yearly details file can power a local hazard-history table. + steps: + - Keep event ID, state, event type, and begin time from the first 200 rows. + - Count event types and flag missing states before ranking hazards. + - Explain that Storm Events is a reported-event archive, not a complete storm census. diff --git a/data/datasets/onet-occupations.yaml b/data/datasets/onet-occupations.yaml new file mode 100644 index 0000000..7cbe0f3 --- /dev/null +++ b/data/datasets/onet-occupations.yaml @@ -0,0 +1,74 @@ +id: onet-occupations +name: O*NET Occupational Database +description: > + U.S. occupation titles, skills, and task descriptions for building job-profile + and labor-market research tools from published database files. +theme: Demographics & Development +url: https://www.onetcenter.org/database.html +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - CSV + - JSON + - Excel +license: Creative Commons Attribution 4.0 +license_url: https://www.onetcenter.org/license_db.html +url_checks: + source_marker: O*NET 30.3 Database + license_marker: Creative Commons Attribution 4.0 International License +domains: + - Labor Economics + - Occupations + - Workforce Development +data_types: + - Tabular + - Taxonomy +tasks: + - Labor Market Research + - Job Profiling + - Skills Analysis +difficulty: beginner +geography: + - United States +temporal_coverage: current O*NET-SOC occupations with quarterly updates +update_frequency: quarterly +provider: National Center for O*NET Development +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + O*NET publishes occupation codes, titles, and descriptions as downloadable + database files under CC BY 4.0. Start with the Occupation Data CSV, not the + paid-product Web Services free-tier. Titles are a taxonomy, not a vacancy + count, and O*NET is a trademark of USDOL/ETA. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the database page and read the CC BY attribution requirements. + - Download the published Occupation Data file, not services.onetcenter.org. + - Keep O*NET-SOC codes as text and credit O*NET 30.3 Database. + python: + packages: + - pandas + code: | + import pandas as pd + + occupations = pd.read_csv( + "https://www.onetcenter.org/dl_files/database/db_30_3_text/Occupation%20Data.txt", + sep="\t", + ) + print(occupations.head()) + print(occupations.columns.tolist()) + first_project: + title: Browse published O*NET occupation titles + goal: Test whether the Occupation Data file can power a bounded job-profile lookup. + steps: + - Keep O*NET-SOC code, title, and description as published. + - Search for one occupation family by title keywords and count matching codes. + - Attribute O*NET 30.3 Database under CC BY 4.0 and explain that titles are not a measure of hiring demand. diff --git a/data/datasets/open-food-facts.yaml b/data/datasets/open-food-facts.yaml new file mode 100644 index 0000000..4894e3c --- /dev/null +++ b/data/datasets/open-food-facts.yaml @@ -0,0 +1,83 @@ +id: open-food-facts +name: Open Food Facts +description: > + Collaborative packaged-food product records for building barcode lookup, + label, and nutrition-comparison tools. +theme: Health, Food & Safety +url: https://world.openfoodfacts.org/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.1 +size_gb_max: 20 +formats: + - JSON + - CSV +license: Open Database License +license_url: https://world.openfoodfacts.org/terms-of-use +url_checks: + source_marker: Open Food Facts + license_marker: Open Database License +domains: + - Nutrition + - Food Science + - Consumer Safety +data_types: + - Product Data + - Tabular +tasks: + - Product Lookup + - Label Analysis + - Nutrition Comparison +difficulty: beginner +geography: + - Global +temporal_coverage: continuously updated product records +update_frequency: continuous +provider: Open Food Facts +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + Open Food Facts publishes packaged-product facts, including Nutri-Score + and ingredient lists, under ODbL. Start with one barcode and a custom + User-Agent. Contributor data can be incomplete, images have a separate + share-alike licence, and the database is not medical advice. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the terms of use, including attribution and share-alike for derivative databases. + - Request one product by barcode with a descriptive User-Agent. + - Keep the barcode, product name, and Nutri-Score if present. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://world.openfoodfacts.org/api/v2/product/737628064502.json", + headers={ + "User-Agent": ( + "TrilemmaDataCatalogExample/1.0 " + "(https://data.trilemma.foundation)" + ) + }, + timeout=30, + ) + response.raise_for_status() + product = pd.json_normalize(response.json()["product"]) + print(product.filter(regex="code|product_name|nutriscore|brands").head()) + first_project: + title: Look up one packaged food by barcode + goal: Test whether Open Food Facts can power a bounded product-fact card. + steps: + - Keep barcode, product name, brands, and Nutri-Score. + - Record missing nutrition or ingredient fields instead of filling them from other sites. + - Attribute Open Food Facts under ODbL and note that images and third-party packaging rights are separate. diff --git a/data/datasets/openssf-scorecard.yaml b/data/datasets/openssf-scorecard.yaml new file mode 100644 index 0000000..f415f66 --- /dev/null +++ b/data/datasets/openssf-scorecard.yaml @@ -0,0 +1,79 @@ +id: openssf-scorecard +name: OpenSSF Scorecard +description: > + Automated security-practice scores for public repositories for building + dependency-acceptance checks and maintainer-risk reviews. +theme: Technology & Cybersecurity +url: https://scorecard.dev/ +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.01 +formats: + - JSON +license: Community Data License Agreement Permissive 2.0 +license_url: https://cdla.dev/permissive-2-0/ +url_checks: + source_marker: Quickly assess open source projects + license_marker: Community Data License Agreement +domains: + - Software Security + - Open Source + - Cybersecurity +data_types: + - Scores + - Repository Metadata +tasks: + - Risk Assessment + - Dependency Review + - Security Monitoring +difficulty: intermediate +geography: + - Global +temporal_coverage: continuously rescored public repositories +update_frequency: continuous +provider: Open Source Security Foundation +source_type: nonprofit +last_verified: 2026-08-18 +getting_started: + overview: > + OpenSSF Scorecard publishes precomputed checks for public GitHub projects. + Start with one well-known repository and read the aggregate score plus a + few high-risk checks. Scores describe repository practices, not whether a + specific package version is vulnerable. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the Scorecard project page and the CDLA Permissive 2.0 data licence. + - Request the published score for github.com/ossf/scorecard. + - Keep the date, aggregate score, and check names before comparing projects. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://api.scorecard.dev/projects/github.com/ossf/scorecard", + timeout=30, + ) + response.raise_for_status() + payload = response.json() + checks = pd.json_normalize(payload.get("checks", [])) + checks["repo"] = payload.get("repo", {}).get("name") + checks["score"] = payload.get("score") + print(payload.get("repo"), payload.get("score")) + print(checks[["name", "score", "reason"]].head(10)) + first_project: + title: Review one repository's Scorecard checks + goal: Decide whether Scorecard can support an accept-or-review rule for a new dependency. + steps: + - Store the aggregate score and the Dangerous-Workflow, Branch-Protection, and Maintained checks. + - Flag any check scored below 5 and record the published reason text. + - State that Scorecard does not replace CVE monitoring or a review of the exact package version you would install. diff --git a/data/datasets/osm-overpass.yaml b/data/datasets/osm-overpass.yaml new file mode 100644 index 0000000..898798e --- /dev/null +++ b/data/datasets/osm-overpass.yaml @@ -0,0 +1,88 @@ +id: osm-overpass +name: OpenStreetMap Overpass API +description: > + Live OpenStreetMap features for building bounded place, amenity, and + infrastructure lookup tools without downloading the planet. +theme: Geospatial & Infrastructure +url: https://wiki.openstreetmap.org/wiki/Overpass_API +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.05 +formats: + - JSON + - XML +license: Open Database License +license_url: https://www.openstreetmap.org/copyright +url_checks: + source_marker: Overpass API + license_marker: Open Database License +domains: + - Mapping + - Infrastructure + - Places +data_types: + - Geospatial + - OpenStreetMap Features +tasks: + - Geographic Analysis + - Place Lookup + - Infrastructure Mapping +difficulty: intermediate +geography: + - Global +temporal_coverage: continuously updated OpenStreetMap features +update_frequency: continuous +provider: OpenStreetMap contributors +source_type: community +last_verified: 2026-08-18 +getting_started: + overview: > + Overpass returns live OSM features for a bounding box or named area. Start + with one city bbox and a LIMIT, not a planet dump. Send a descriptive + User-Agent. Results are volunteer-mapped, ODbL share-alike applies to + derivative databases, and coverage varies by city. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the Overpass API wiki and the OpenStreetMap copyright page. + - Write a QL query with a bounding box and a max number of features. + - POST or GET the query with a descriptive User-Agent. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + query = """ + [out:json][timeout:25]; + node["amenity"="cafe"](40.74,-74.02,40.76,-73.99); + out 20; + """ + response = requests.get( + "https://overpass-api.de/api/interpreter", + params={"data": query}, + headers={ + "User-Agent": ( + "TrilemmaDataCatalogExample/1.0 " + "(https://data.trilemma.foundation)" + ) + }, + timeout=30, + ) + response.raise_for_status() + cafes = pd.json_normalize(response.json()["elements"]) + print(cafes.filter(regex="id|lat|lon|tags.name").head()) + first_project: + title: List cafes inside a Manhattan bounding box + goal: Test whether a bounded Overpass query can power a neighborhood amenity list. + steps: + - Keep OSM id, coordinates, and name for up to 20 cafes. + - Count unnamed nodes instead of filling names from other maps. + - Attribute OpenStreetMap contributors under ODbL and do not publish a planet extract from this guide. diff --git a/data/datasets/ourairports.yaml b/data/datasets/ourairports.yaml new file mode 100644 index 0000000..df8b314 --- /dev/null +++ b/data/datasets/ourairports.yaml @@ -0,0 +1,77 @@ +id: ourairports +name: OurAirports +description: > + Community airport, runway, and navaid records for building aviation + directories and map layers from a nightly public-domain CSV. +theme: Geospatial & Infrastructure +url: https://ourairports.com/data/ +access_type: + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 0.05 +formats: + - CSV +license: Public Domain +license_url: https://ourairports.com/data/ +url_checks: + source_marker: airports.csv + license_marker: All data is released to the Public Domain +domains: + - Aviation + - Transportation + - Mapping +data_types: + - Tabular + - Geospatial +tasks: + - Directory Search + - Geographic Analysis + - Infrastructure Mapping +difficulty: beginner +geography: + - Global +temporal_coverage: continuously updated community airport records +update_frequency: daily +provider: OurAirports +source_type: community +last_verified: 2026-08-18 +getting_started: + overview: > + OurAirports publishes a nightly airports.csv covering worldwide aerodromes. + Start with large airports in one country. Community coordinates and names + can be incomplete, and the file is not an official aeronautical chart. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the data page and note the public-domain terms. + - Download airports.csv from the published GitHub Pages mirror. + - Filter to one ISO country and large airports before mapping. + python: + packages: + - pandas + code: | + import pandas as pd + + airports = pd.read_csv( + "https://davidmegginson.github.io/ourairports-data/airports.csv" + ) + us_large = airports[ + (airports["iso_country"] == "US") + & (airports["type"] == "large_airport") + ] + print( + us_large[ + ["ident", "name", "municipality", "latitude_deg", "longitude_deg"] + ].head() + ) + first_project: + title: Map large U.S. airports from the public-domain dump + goal: Test whether OurAirports can power a bounded aviation directory layer. + steps: + - Keep ident, name, municipality, and coordinates for U.S. large airports. + - Count missing municipalities before plotting. + - Explain that community airport records are not a substitute for official NASR or AIP charts. diff --git a/data/datasets/uk-police-street-crime.yaml b/data/datasets/uk-police-street-crime.yaml new file mode 100644 index 0000000..6cba365 --- /dev/null +++ b/data/datasets/uk-police-street-crime.yaml @@ -0,0 +1,79 @@ +id: uk-police-street-crime +name: UK Police Street-Level Crime +description: > + Monthly street-level crime and outcome records for building neighbourhood + safety monitors in England, Wales, and Northern Ireland. +theme: Government & Policy +url: https://data.police.uk/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 2 +formats: + - JSON + - CSV +license: Open Government Licence v3.0 +license_url: https://data.police.uk/about/ +url_checks: + source_marker: open data about crime and policing + license_marker: Open Government Licence v3.0 +domains: + - Public Safety + - Crime + - Local Government +data_types: + - Event Data + - Geospatial +tasks: + - Geographic Analysis + - Trend Analysis + - Neighbourhood Monitoring +difficulty: beginner +geography: + - United Kingdom +temporal_coverage: 2014-present street-level records +update_frequency: monthly +provider: data.police.uk +source_type: government +last_verified: 2026-08-18 +getting_started: + overview: > + The Police API returns street-level crimes for a point or custom area. + Start with one coordinate pair and the most recently published month. + Locations are snapped to anonymised map points, reporting coverage varies + by force, and counts are not official statistics. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the data.police.uk about page and the Open Government Licence notice. + - Request street-level crimes for one latitude and longitude. + - Keep category, month, and the anonymised street name. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://data.police.uk/api/crimes-street/all-crime", + params={"lat": 51.5074, "lng": -0.1278}, + timeout=30, + ) + response.raise_for_status() + crimes = pd.json_normalize(response.json()) + print(crimes[["id", "category", "month", "location.street.name"]].head(20)) + print(crimes["category"].value_counts().head()) + first_project: + title: Summarise recent street crime around one point + goal: Test whether category counts can power a neighbourhood safety snapshot. + steps: + - Keep crime id, category, month, and anonymised street name. + - Count records by category and flag missing coordinates. + - Explain that points are snapped for privacy and that force coverage and lag limit comparison across areas. diff --git a/data/datasets/who-gho-indicators.yaml b/data/datasets/who-gho-indicators.yaml new file mode 100644 index 0000000..d96d533 --- /dev/null +++ b/data/datasets/who-gho-indicators.yaml @@ -0,0 +1,81 @@ +id: who-gho-indicators +name: WHO Global Health Observatory +description: > + Official country health indicators for building life-expectancy, mortality, + and coverage comparison tools. +theme: Demographics & Development +url: https://www.who.int/data/gho/info/gho-odata-api +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 1 +formats: + - JSON + - XML +license: Creative Commons Attribution 4.0 with WHO additional terms +license_url: https://data.who.int/about/data/terms-and-conditions +url_checks: + source_marker: GHO OData API + license_marker: Creative Commons Attribution 4.0 International License +domains: + - Public Health + - International Statistics + - Demographics +data_types: + - Time Series + - Tabular +tasks: + - International Comparison + - Trend Analysis + - Policy Monitoring +difficulty: beginner +geography: + - Global +temporal_coverage: indicator series with country-specific starts +update_frequency: annual +provider: World Health Organization +source_type: intergovernmental +last_verified: 2026-08-18 +getting_started: + overview: > + The GHO OData API returns WHO health indicators by country and year. Start + with life expectancy at birth for one country. Values can lag, sex and age + splits differ by indicator, and WHO forbids implying endorsement. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the GHO OData examples and the data.who.int terms, including CC BY 4.0 and no-endorsement rules. + - Request one indicator filtered to one country and a recent year. + - Keep indicator code, SpatialDim, TimeDim, and NumericValue. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://ghoapi.azureedge.net/api/WHOSIS_000001", + params={"$filter": "SpatialDim eq 'USA'", "$top": "20"}, + timeout=30, + ) + response.raise_for_status() + observations = pd.DataFrame(response.json()["value"]) + print( + observations[ + ["IndicatorCode", "SpatialDim", "TimeDim", "Dim1", "NumericValue"] + ].head() + ) + first_project: + title: Chart U.S. life expectancy at birth from GHO + goal: Test whether one WHO indicator can power a bounded national trend card. + steps: + - Keep year, sex dimension, and NumericValue for WHOSIS_000001 in the United States. + - Plot the series and report missing years before interpreting a change. + - Attribute WHO under CC BY 4.0 and state that the chart is not a WHO-endorsed product. diff --git a/data/datasets/wikidata-query.yaml b/data/datasets/wikidata-query.yaml new file mode 100644 index 0000000..3ae1e34 --- /dev/null +++ b/data/datasets/wikidata-query.yaml @@ -0,0 +1,93 @@ +id: wikidata-query +name: Wikidata Query Service +description: > + CC0 structured knowledge-graph facts for building entity lookup, taxonomy, + and reference-enrichment tools. +theme: Research & Reference +url: https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service +access_type: + - api +api_key_required: false +free_to_access: true +size_gb_min: 0 +size_gb_max: 0.01 +formats: + - JSON + - CSV +license: Creative Commons CC0 1.0 +license_url: https://www.wikidata.org/wiki/Wikidata:Licensing +url_checks: + source_marker: Wikidata:SPARQL query service + license_marker: Creative Commons CC0 License +domains: + - Knowledge Graphs + - Reference Data + - Research +data_types: + - Linked Data + - Entity Data +tasks: + - Entity Lookup + - Knowledge Graph Query + - Reference Enrichment +difficulty: intermediate +geography: + - Global +temporal_coverage: continuously updated Wikidata statements +update_frequency: continuous +provider: Wikimedia Foundation +source_type: community +last_verified: 2026-08-18 +getting_started: + overview: > + Wikidata SPARQL returns CC0 statements for entities and properties. Start + with one class, one country, and a LIMIT. Send a descriptive User-Agent. + Query results can be incomplete, contested, or delayed relative to the + source that a statement cites. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Open the Query Service and read the CC0 licensing page. + - Write a bounded SPARQL query with a LIMIT instead of selecting the whole graph. + - Request JSON results with a descriptive User-Agent. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + query = """ + SELECT ?item ?itemLabel WHERE { + ?item wdt:P31 wd:Q515. + ?item wdt:P17 wd:Q30. + SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } + } + LIMIT 10 + """ + response = requests.get( + "https://query.wikidata.org/sparql", + params={"query": query, "format": "json"}, + headers={ + "User-Agent": ( + "TrilemmaDataCatalogExample/1.0 " + "(https://data.trilemma.foundation)" + ), + "Accept": "application/sparql-results+json", + }, + timeout=30, + ) + response.raise_for_status() + bindings = response.json()["results"]["bindings"] + cities = pd.json_normalize(bindings) + print(cities.head()) + first_project: + title: List a handful of U.S. cities from Wikidata + goal: Test whether a bounded SPARQL query can seed a reference lookup table. + steps: + - Keep item IRI and English label for ten cities. + - Record missing labels instead of guessing names from other sources. + - Treat results as CC0 statements that still need a cited source before publication. diff --git a/data/datasets/worldpop-population.yaml b/data/datasets/worldpop-population.yaml new file mode 100644 index 0000000..9c63de7 --- /dev/null +++ b/data/datasets/worldpop-population.yaml @@ -0,0 +1,79 @@ +id: worldpop-population +name: WorldPop Population Estimates +description: > + Country-level high-resolution population grids for building catchment, + coverage, and demographic-denominator tools. +theme: Demographics & Development +url: https://www.worldpop.org/sdi/introapi/ +access_type: + - api + - download +api_key_required: false +free_to_access: true +size_gb_min: 0.01 +size_gb_max: 20 +formats: + - JSON + - GeoTIFF +license: Creative Commons Attribution 4.0 +license_url: https://hub.worldpop.org/data/licence.txt +url_checks: + source_marker: WorldPop REST API + license_marker: WorldPop datasets are licensed under the Creative Commons Attribution +domains: + - Demographics + - Population + - Geospatial +data_types: + - Geospatial + - Gridded Data +tasks: + - Population Estimation + - Geographic Analysis + - Coverage Analysis +difficulty: intermediate +geography: + - Global +temporal_coverage: 2000-2020 global per-country estimates +update_frequency: occasional +provider: WorldPop, University of Southampton +source_type: academic +last_verified: 2026-08-18 +getting_started: + overview: > + WorldPop REST metadata lists per-country population rasters. Start with one + ISO3 code and the unconstrained 2000-2020 series, not the global mosaic. + Grid values are modeled estimates, not a census count, and some derived + layers use a different share-alike licence. + prerequisites: + - Python 3.10 or newer + - A notebook environment such as Jupyter or Google Colab + - An internet connection + access_steps: + - Read the REST API basics and the CC BY 4.0 licence text. + - Request metadata for one country instead of downloading a continental mosaic. + - Keep ISO3, year, DOI, and the published file URL. + python: + packages: + - pandas + - requests + code: | + import pandas as pd + import requests + + response = requests.get( + "https://www.worldpop.org/rest/data/pop/wpgp", + params={"iso3": "LUX"}, + timeout=30, + ) + response.raise_for_status() + layers = pd.DataFrame(response.json()["data"]) + sample = layers[layers["popyear"].astype(str) == "2020"] + print(sample[["iso3", "popyear", "doi", "data_format", "files"]].head()) + first_project: + title: List Luxembourg's 2020 WorldPop raster metadata + goal: Test whether REST metadata can locate one country unconstrained population file. + steps: + - Keep ISO3, year, DOI, and the published GeoTIFF path for 2020. + - Confirm the file is a country raster rather than a global mosaic. + - Cite WorldPop under CC BY 4.0 and explain that pixel counts are modeled estimates, not census enumerations. diff --git a/src/lib/datasets.test.ts b/src/lib/datasets.test.ts index 3c31a63..79aaf0e 100644 --- a/src/lib/datasets.test.ts +++ b/src/lib/datasets.test.ts @@ -86,14 +86,14 @@ describe("loadDatasets", () => { getAllDatasets().map((dataset) => [dataset.id, dataset.theme]), ); const groups = { - "Environment & Hazards": ["airnow-air-quality", "epa-airdata-daily-summaries", "epa-echo-drinking-water", "epa-toxics-release-inventory", "fema-national-flood-hazard-layer", "gdacs-disaster-alerts", "nasa-firms", "nasa-power-daily", "noaa-ibtracs", "noaa-swpc-space-weather", "nws-weather-api", "noaa-ncei-daily-summaries", "noaa-tides-currents", "openfema-disaster-declarations", "us-drought-monitor", "usgs-earthquakes", "usgs-water-data"], - "Government & Policy": ["congress-gov-legislation", "fec-campaign-finance", "federal-register-documents", "ofac-sdn-list", "open-states-legislation", "sam-gov-contract-opportunities", "usaspending-federal-awards"], - "Markets & Economics": ["bea-regional-gdp-income", "bls-public-data-api", "census-international-trade", "cfpb-consumer-complaints", "eia-weekly-petroleum-status", "fhfa-house-price-index", "fred-economic-series", "hud-fair-market-rents", "imf-world-economic-outlook", "kalshi-market-data", "polymarket-markets", "sec-edgar-apis", "treasury-securities-auctions"], - "Health, Food & Safety": ["cdc-fluview-ilinet", "cdc-places", "clinicaltrials-studies", "cms-care-compare-hospitals", "cms-open-payments", "cpsc-product-recalls", "nhtsa-vehicle-recalls", "nppes-npi-registry", "openfda-drug-adverse-events", "openfda-food-enforcement", "usda-fooddata-central"], - "Geospatial & Infrastructure": ["bts-airline-on-time", "census-tiger-line", "eia-hourly-electric-grid", "fcc-national-broadband-map", "fhwa-national-bridge-inventory", "fta-ntd-monthly-ridership", "mobility-database-feeds", "natural-earth", "nrel-alt-fuel-stations", "overture-maps-places"], - "Research & Reference": ["arxiv-preprints", "crossref-works", "gbif-species-occurrences", "openalex-scholarly-works", "pubmed-citations", "wikimedia-pageviews"], - "Technology & Cybersecurity": ["cisa-known-exploited-vulnerabilities", "deps-dev-package-graph", "mitre-attack-enterprise", "nvd-cve", "osv-open-source-vulnerabilities"], - "Demographics & Development": ["acs-five-year-estimates", "college-scorecard", "eurostat-statistics", "nces-common-core-of-data", "unhcr-refugee-population", "usda-nass-quick-stats", "world-development-indicators"], + "Environment & Hazards": ["airnow-air-quality", "epa-airdata-daily-summaries", "epa-echo-drinking-water", "epa-toxics-release-inventory", "fema-national-flood-hazard-layer", "gdacs-disaster-alerts", "nasa-firms", "nasa-power-daily", "noaa-ibtracs", "noaa-swpc-space-weather", "nws-weather-api", "noaa-ncei-daily-summaries", "noaa-tides-currents", "openfema-disaster-declarations", "us-drought-monitor", "usgs-earthquakes", "usgs-water-data", "met-norway-locationforecast", "noaa-storm-events"], + "Government & Policy": ["congress-gov-legislation", "fec-campaign-finance", "federal-register-documents", "ofac-sdn-list", "open-states-legislation", "sam-gov-contract-opportunities", "usaspending-federal-awards", "legislation-gov-uk", "uk-police-street-crime", "fbi-crime-data-explorer", "nih-reporter-projects"], + "Markets & Economics": ["bea-regional-gdp-income", "bls-public-data-api", "census-international-trade", "cfpb-consumer-complaints", "eia-weekly-petroleum-status", "fhfa-house-price-index", "fred-economic-series", "hud-fair-market-rents", "imf-world-economic-outlook", "kalshi-market-data", "polymarket-markets", "sec-edgar-apis", "treasury-securities-auctions", "gleif-lei", "companies-house-uk", "fdic-bank-find", "cftc-commitment-of-traders", "census-county-business-patterns", "ecb-statistical-data-warehouse"], + "Health, Food & Safety": ["cdc-fluview-ilinet", "cdc-places", "clinicaltrials-studies", "cms-care-compare-hospitals", "cms-open-payments", "cpsc-product-recalls", "nhtsa-vehicle-recalls", "nppes-npi-registry", "openfda-drug-adverse-events", "openfda-food-enforcement", "usda-fooddata-central", "open-food-facts", "cms-nursing-homes", "cdc-social-vulnerability-index", "fda-orange-book"], + "Geospatial & Infrastructure": ["bts-airline-on-time", "census-tiger-line", "eia-hourly-electric-grid", "fcc-national-broadband-map", "fhwa-national-bridge-inventory", "fta-ntd-monthly-ridership", "mobility-database-feeds", "natural-earth", "nrel-alt-fuel-stations", "overture-maps-places", "osm-overpass", "ourairports"], + "Research & Reference": ["arxiv-preprints", "crossref-works", "gbif-species-occurrences", "openalex-scholarly-works", "pubmed-citations", "wikimedia-pageviews", "wikidata-query"], + "Technology & Cybersecurity": ["cisa-known-exploited-vulnerabilities", "deps-dev-package-graph", "mitre-attack-enterprise", "nvd-cve", "osv-open-source-vulnerabilities", "first-epss", "openssf-scorecard"], + "Demographics & Development": ["acs-five-year-estimates", "college-scorecard", "eurostat-statistics", "nces-common-core-of-data", "unhcr-refugee-population", "usda-nass-quick-stats", "world-development-indicators", "onet-occupations", "worldpop-population", "who-gho-indicators"], } as const; const grouped = Object.values(groups).flat(); expect([...Object.keys(themes)].sort()).toEqual([...grouped].sort()); diff --git a/src/lib/provider-validation.test.ts b/src/lib/provider-validation.test.ts index a0df753..896c66c 100644 --- a/src/lib/provider-validation.test.ts +++ b/src/lib/provider-validation.test.ts @@ -204,6 +204,48 @@ const validBodies = { "deps-dev-package-graph": JSON.stringify({ versionKey: { system: "PYPI", name: "requests", version: "2.32.3" }, }), + "first-epss": JSON.stringify({ + data: [{ cve: "CVE-2024-3400", epss: "0.9", percentile: "0.99" }], + }), + "openssf-scorecard": JSON.stringify({ + score: 8.5, + repo: { name: "github.com/ossf/scorecard" }, + }), + "legislation-gov-uk": "Data Protection Act 2018", + "uk-police-street-crime": JSON.stringify([ + { category: "anti-social-behaviour", month: "2026-01" }, + ]), + "gleif-lei": JSON.stringify({ + data: [{ id: "5493001KJTIIGC8Y1R12", type: "lei-records" }], + }), + "fdic-bank-find": JSON.stringify({ + data: [{ data: { NAME: "Example Bank", CERT: "1" } }], + }), + "cftc-commitment-of-traders": JSON.stringify([ + { contract_market_name: "GOLD", open_interest_all: "1" }, + ]), + "ecb-statistical-data-warehouse": JSON.stringify({ + dataSets: [{ series: {} }], + }), + "open-food-facts": JSON.stringify({ + product: { code: "737628064502", product_name: "Example" }, + }), + "cms-nursing-homes": JSON.stringify({ + results: [{ cms_certification_number_ccn: "000000", provider_name: "Example" }], + }), + "who-gho-indicators": JSON.stringify({ + value: [{ IndicatorCode: "WHOSIS_000001", SpatialDim: "USA" }], + }), + "wikidata-query": JSON.stringify({ + results: { bindings: [{ item: { type: "uri", value: "http://www.wikidata.org/entity/Q5" } }] }, + }), + "met-norway-locationforecast": JSON.stringify({ + properties: { timeseries: [{ time: "2026-08-18T00:00:00Z" }] }, + }), + "osm-overpass": JSON.stringify({ + elements: [{ type: "node", id: 1, lat: 40.75, lon: -73.98 }], + }), + "ourairports": "id,ident,type,name,latitude_deg,longitude_deg\n1,KSEA,large_airport,Seattle,47.45,-122.31", } as const; const contentTypes = { @@ -254,6 +296,21 @@ const contentTypes = { "cdc-fluview-ilinet": "application/json", "cms-open-payments": "application/json", "deps-dev-package-graph": "application/json", + "first-epss": "application/json", + "openssf-scorecard": "application/json", + "legislation-gov-uk": "application/xml", + "uk-police-street-crime": "application/json", + "gleif-lei": "application/vnd.api+json", + "fdic-bank-find": "application/json", + "cftc-commitment-of-traders": "application/json", + "ecb-statistical-data-warehouse": "application/json", + "open-food-facts": "application/json", + "cms-nursing-homes": "application/json", + "who-gho-indicators": "application/json", + "wikidata-query": "application/sparql-results+json", + "met-norway-locationforecast": "application/json", + "osm-overpass": "application/json", + "ourairports": "text/csv", } as const; function response( diff --git a/src/lib/provider-validation.ts b/src/lib/provider-validation.ts index aa8ccc7..d623240 100644 --- a/src/lib/provider-validation.ts +++ b/src/lib/provider-validation.ts @@ -648,6 +648,162 @@ const contracts = { }).passthrough(), ), }, + "first-epss": { + url: "https://api.first.org/data/v1/epss?cve=CVE-2024-3400", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + data: z.array( + z.object({ + cve: z.string(), + epss: z.union([z.string(), z.number()]), + percentile: z.union([z.string(), z.number()]), + }).passthrough(), + ).min(1), + }).passthrough(), + ), + }, + "openssf-scorecard": { + url: "https://api.scorecard.dev/projects/github.com/ossf/scorecard", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + score: z.number(), + repo: z.object({ + name: z.string(), + }).passthrough(), + }).passthrough(), + ), + }, + "legislation-gov-uk": { + url: "https://www.legislation.gov.uk/ukpga/2018/12/section/1/data.xml", + contentTypes: ["application/xml", "text/xml", "application/xhtml+xml"], + validate(body: Uint8Array) { + return new TextDecoder().decode(body).toLowerCase().includes("legislation") + ? null + : "XML is missing legislation markup"; + }, + }, + "uk-police-street-crime": { + url: "https://data.police.uk/api/crimes-street/all-crime?lat=51.5074&lng=-0.1278&date=2026-01", + contentTypes: ["application/json"], + validate: jsonValidator( + z.array( + z.object({ + category: z.string(), + }).passthrough(), + ).min(1), + ), + }, + "gleif-lei": { + url: "https://api.gleif.org/api/v1/lei-records?filter[lei]=5493001KJTIIGC8Y1R12", + contentTypes: ["application/vnd.api+json", "application/json"], + validate: jsonValidator( + z.object({ + data: z.array( + z.object({ + id: z.string(), + }).passthrough(), + ).min(1), + }).passthrough(), + ), + }, + "fdic-bank-find": { + url: "https://api.fdic.gov/banks/institutions?filters=STALP:IA&limit=1&format=json", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + data: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + ), + }, + "cftc-commitment-of-traders": { + url: "https://publicreporting.cftc.gov/resource/jun7-fc8e.json?$limit=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.array(z.record(z.string(), z.unknown())).min(1), + ), + }, + "ecb-statistical-data-warehouse": { + url: "https://data-api.ecb.europa.eu/service/data/EXR/D.USD.EUR.SP00.A?lastNObservations=1&format=jsondata", + contentTypes: ["application/json", "application/vnd.sdmx.data+json"], + validate: jsonValidator( + z.object({ + dataSets: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + ), + }, + "open-food-facts": { + url: "https://world.openfoodfacts.org/api/v2/product/737628064502.json", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + product: z.object({ + code: z.string(), + }).passthrough(), + }).passthrough(), + ), + }, + "cms-nursing-homes": { + url: "https://data.cms.gov/provider-data/api/1/datastore/query/4pq5-n9py/0?limit=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + results: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + ), + }, + "who-gho-indicators": { + url: "https://ghoapi.azureedge.net/api/WHOSIS_000001?$filter=SpatialDim%20eq%20'USA'&$top=1", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + value: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + ), + }, + "wikidata-query": { + url: "https://query.wikidata.org/sparql?query=SELECT%20%3Fitem%20WHERE%20%7B%20wd%3AQ42%20wdt%3AP31%20%3Fitem%20%7D%20LIMIT%201&format=json", + contentTypes: ["application/sparql-results+json", "application/json"], + validate: jsonValidator( + z.object({ + results: z.object({ + bindings: z.array(z.record(z.string(), z.unknown())).min(1), + }), + }).passthrough(), + ), + }, + "met-norway-locationforecast": { + url: "https://api.met.no/weatherapi/locationforecast/2.0/compact?lat=59.91&lon=10.75", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + properties: z.object({ + timeseries: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + }).passthrough(), + ), + }, + "osm-overpass": { + url: "https://overpass-api.de/api/interpreter?data=%5Bout%3Ajson%5D%5Btimeout%3A10%5D%3Bnode%5B%22amenity%22%3D%22cafe%22%5D(40.748,-73.988,40.751,-73.985)%3Bout%201%3B", + contentTypes: ["application/json"], + validate: jsonValidator( + z.object({ + elements: z.array(z.record(z.string(), z.unknown())).min(1), + }).passthrough(), + ), + }, + "ourairports": { + url: "https://davidmegginson.github.io/ourairports-data/airports.csv", + range: "bytes=0-65535", + contentTypes: ["text/csv", "text/plain", "application/octet-stream"], + validate(body: Uint8Array) { + const header = new TextDecoder().decode(body).split(/\r?\n/, 1).join(); + return ["ident", "type", "name", "latitude_deg"].every((field) => header.includes(field)) + ? null + : "CSV is missing ident, type, name, or latitude_deg"; + }, + }, } satisfies Record< string, {