From 62005964c5e031455b57f2501577dd06816dcfbb Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 13:48:57 -0400 Subject: [PATCH 01/26] Added .env CENSUS_API_KEY to Justfile --- backend/data_collection/acs5.py | 9 +++++---- backend/data_collection/base.py | 5 +++-- backend/data_collection/qcew.py | 1 - justfile | 6 ++++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/backend/data_collection/acs5.py b/backend/data_collection/acs5.py index 42618a0..e6e48ed 100644 --- a/backend/data_collection/acs5.py +++ b/backend/data_collection/acs5.py @@ -11,6 +11,7 @@ Use --append to merge new rows into existing files instead of overwriting. """ +import os import time import pandas as pd @@ -19,9 +20,10 @@ from app_utils.census import tidy_census from data_collection.base import ALL_GEOS -API_KEY = ( - "29af5488bbdb8c7d9f67b7f4ff9c9151e8c2bd0a" # TODO: Get this as a .env variable!!! -) +# Define API key through the .env file +API_KEY = os.environ.get("CENSUS_API_KEY") + + BASE_URL = "https://api.census.gov/data/{year}/acs/acs5/profile" STATE_FIPS = "50" # Vermont TABLES = { @@ -30,7 +32,6 @@ "DP04": "Housing", "DP05": "Demographic", } -YEARS = list(range(2009, 2025)) STORAGE_LOCATION = "Data/Census/ACS_5" ID_VARS = ["year", "geo_type", "table", "NAME", "state", "county"] diff --git a/backend/data_collection/base.py b/backend/data_collection/base.py index 106fed8..6fb229c 100644 --- a/backend/data_collection/base.py +++ b/backend/data_collection/base.py @@ -21,6 +21,7 @@ replaces rather than duplicates those rows), then writes the merged result. """ +import os import time from dataclasses import dataclass @@ -29,10 +30,10 @@ from app_utils.census import split_name_col -API_KEY = "29af5488bbdb8c7d9f67b7f4ff9c9151e8c2bd0a" +# Define API key through the .env file +API_KEY = os.environ.get("CENSUS_API_KEY") BASE_URL = "https://api.census.gov/data/{year}/acs/acs5" STATE_FIPS = "50" -# YEARS = list(range(2009, 2025)) STORAGE_LOCATION = "Data/Census/ACS_5" # --------------------------------------------------------------------------- diff --git a/backend/data_collection/qcew.py b/backend/data_collection/qcew.py index 9d5b0ef..7ab9695 100644 --- a/backend/data_collection/qcew.py +++ b/backend/data_collection/qcew.py @@ -86,7 +86,6 @@ ] BASE_URL = "https://data.bls.gov/cew/data/api/{year}/{q}/area/{fips}.csv" -# YEARS = list(range(2009, 2024)) QUARTERS = [1, 2, 3, 4] diff --git a/justfile b/justfile index faec1b9..e6a7eaa 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,8 @@ ## Set up environment ## export DATA_DIR := justfile_directory() / "Data" +# Load environment variables +set dotenv-filename := ".env.local" ################ # CLI Development # @@ -95,8 +97,7 @@ check-frontend: npx tsc --noEmit -################ -# ETL (Pipeline) Container +## ETL (Pipeline) Container ################ # --------- 1. Data Collection (E) --------------------- @@ -108,6 +109,7 @@ build-collection: # Get the data for a specified year [working-directory("backend")] get-data year: build-collection + echo "Using API key: $CENSUS_API_KEY" podman run --rm -v "$(pwd)/Data:/data:z" -e DATA_DIR=/data localhost/vdc-collection {{year}} From 2208c34d65043de069369b19a8f3bde25793e1a9 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 13:56:19 -0400 Subject: [PATCH 02/26] justfile documentation --- justfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index e6a7eaa..66b4fc3 100644 --- a/justfile +++ b/justfile @@ -106,7 +106,7 @@ check-frontend: build-collection: podman build -t localhost/vdc-collection -f ETL/dockerfile.collect . -# Get the data for a specified year +# Collect the data for a specified year and add to lake.RAW tables [working-directory("backend")] get-data year: build-collection echo "Using API key: $CENSUS_API_KEY" @@ -114,7 +114,7 @@ get-data year: build-collection # --------- 2. Data Cleaning (T) --------------------- -# build and run the backend CLEANING image +# Run each RAW table through it's data cleaning script [working-directory("backend")] transform-data: podman build -t localhost/vdc-cleaning -f ETL/dockerfile.clean . @@ -122,15 +122,14 @@ transform-data: # --------- 3. Data Loading (L) --------------------- -# build and run the backend LOADING image (loads cleaned tables into a DuckDB) +# Load the lake.CLEANED tables into a DuckDB database [working-directory("backend")] load-data: podman build -t localhost/vdc-loading -f ETL/dockerfile.load . podman run --rm -v "$(pwd)/Data:/data:z" localhost/vdc-loading - -# --------- FULL PIPELINE RUN (ETL) --------------------- +# Collect (E), clean (T), and load (L) the data (Full pipeline run) [working-directory("backend")] run-etl year: # Collect the data for a certain year From eeeb78e50451ee6e1083abde1a94c0fcc0725437 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 14:06:59 -0400 Subject: [PATCH 03/26] Added build-lake recipe to justfile --- backend/ETL/dockerfile.lake | 24 +++++++++++++++++++ backend/data_cleaning/clean_cdc.py | 2 +- backend/data_cleaning/clean_demographics.py | 2 +- .../data_cleaning/clean_dependency_ratio.py | 2 +- .../clean_derived_time_series.py | 2 +- backend/data_cleaning/clean_economic.py | 2 +- backend/data_cleaning/clean_education.py | 2 +- backend/data_cleaning/clean_flood.py | 2 +- .../clean_health_insurance_coverage.py | 2 +- .../clean_historic_population.py | 2 +- backend/data_cleaning/clean_housing.py | 2 +- backend/data_cleaning/clean_qcew.py | 2 +- backend/data_cleaning/clean_wastewater.py | 2 +- backend/data_cleaning/clean_zoning.py | 2 +- backend/{datastore => }/lake_build.py | 0 backend/run_data_collection.py | 2 +- backend/run_data_loading.py | 2 +- backend/tests/test_lake.py | 2 +- justfile | 7 ++++++ 19 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 backend/ETL/dockerfile.lake rename backend/{datastore => }/lake_build.py (100%) diff --git a/backend/ETL/dockerfile.lake b/backend/ETL/dockerfile.lake new file mode 100644 index 0000000..63dde14 --- /dev/null +++ b/backend/ETL/dockerfile.lake @@ -0,0 +1,24 @@ +FROM python:3.12-slim-trixie +COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /uvx /bin/ + +# Set working directory +WORKDIR /code +RUN useradd -m -u 1000 app && chown app:app /code /home/app +USER app + +# Dependencies +COPY --chown=app:app pyproject.toml uv.lock /code/ +RUN --mount=type=cache,target=/home/app/.cache/uv,uid=1000,gid=1000 \ + uv sync --frozen --no-dev --no-install-project + +# Spatial extension installation +RUN uv run python -c "import duckdb; duckdb.connect().execute('INSTALL spatial')" + +# The source data collection scripts (source --> destination) +COPY --chown=app:app ./lake_build.py /code/lake_build.py + +ENV DATA_DIR=/data +ENV PYTHONPATH=/code + +# Command to run the DuckLake builder script (container entrypoint) +ENTRYPOINT ["uv", "run", "python", "-u", "-m", "lake_build"] \ No newline at end of file diff --git a/backend/data_cleaning/clean_cdc.py b/backend/data_cleaning/clean_cdc.py index fc6ea6a..edde5e3 100644 --- a/backend/data_cleaning/clean_cdc.py +++ b/backend/data_cleaning/clean_cdc.py @@ -14,7 +14,7 @@ from sklearn.decomposition import PCA from build.core_functions import bin_measures -from datastore.lake_build import con +from lake_build import con # Columns we'd like excluded from the cleaned tables, IF they exist on that # particular RAW table. Tract- and county-level releases don't always share diff --git a/backend/data_cleaning/clean_demographics.py b/backend/data_cleaning/clean_demographics.py index d75cfe4..d35b901 100644 --- a/backend/data_cleaning/clean_demographics.py +++ b/backend/data_cleaning/clean_demographics.py @@ -12,7 +12,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_dependency_ratio.py b/backend/data_cleaning/clean_dependency_ratio.py index 1142e6e..e11e858 100644 --- a/backend/data_cleaning/clean_dependency_ratio.py +++ b/backend/data_cleaning/clean_dependency_ratio.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_derived_time_series.py b/backend/data_cleaning/clean_derived_time_series.py index 2d616c7..3818ee4 100644 --- a/backend/data_cleaning/clean_derived_time_series.py +++ b/backend/data_cleaning/clean_derived_time_series.py @@ -29,7 +29,7 @@ import numpy as np import pandas as pd -from datastore.lake_build import con +from lake_build import con UNAVAILABLE_SENTINEL = -666666666.0 diff --git a/backend/data_cleaning/clean_economic.py b/backend/data_cleaning/clean_economic.py index d83ff54..1cc43c8 100644 --- a/backend/data_cleaning/clean_economic.py +++ b/backend/data_cleaning/clean_economic.py @@ -12,7 +12,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_education.py b/backend/data_cleaning/clean_education.py index 248fe8d..65dcf3c 100644 --- a/backend/data_cleaning/clean_education.py +++ b/backend/data_cleaning/clean_education.py @@ -12,7 +12,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_flood.py b/backend/data_cleaning/clean_flood.py index 67440d1..38d057c 100644 --- a/backend/data_cleaning/clean_flood.py +++ b/backend/data_cleaning/clean_flood.py @@ -9,7 +9,7 @@ python -m ETL.data_cleaning.clean_flood """ -from datastore.lake_build import con +from lake_build import con ## LOAD SPATIAL EXTENSION FUNCTION -------------------- diff --git a/backend/data_cleaning/clean_health_insurance_coverage.py b/backend/data_cleaning/clean_health_insurance_coverage.py index 96a0993..d282915 100644 --- a/backend/data_cleaning/clean_health_insurance_coverage.py +++ b/backend/data_cleaning/clean_health_insurance_coverage.py @@ -13,7 +13,7 @@ import numpy as np import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_historic_population.py b/backend/data_cleaning/clean_historic_population.py index 91b3e57..fdc71b7 100644 --- a/backend/data_cleaning/clean_historic_population.py +++ b/backend/data_cleaning/clean_historic_population.py @@ -11,7 +11,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_housing.py b/backend/data_cleaning/clean_housing.py index fd23158..0063c6b 100644 --- a/backend/data_cleaning/clean_housing.py +++ b/backend/data_cleaning/clean_housing.py @@ -12,7 +12,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_qcew.py b/backend/data_cleaning/clean_qcew.py index d697484..58bb490 100644 --- a/backend/data_cleaning/clean_qcew.py +++ b/backend/data_cleaning/clean_qcew.py @@ -12,7 +12,7 @@ import pandas as pd -from datastore.lake_build import con +from lake_build import con def read_raw_data() -> pd.DataFrame: diff --git a/backend/data_cleaning/clean_wastewater.py b/backend/data_cleaning/clean_wastewater.py index 3754f74..ad39e94 100644 --- a/backend/data_cleaning/clean_wastewater.py +++ b/backend/data_cleaning/clean_wastewater.py @@ -10,7 +10,7 @@ python -m data_cleaning.clean_wastewater """ -from datastore.lake_build import con +from lake_build import con # TODO: Path is useful when sql files are created! # from build import BACKEND diff --git a/backend/data_cleaning/clean_zoning.py b/backend/data_cleaning/clean_zoning.py index 1b72578..573396d 100644 --- a/backend/data_cleaning/clean_zoning.py +++ b/backend/data_cleaning/clean_zoning.py @@ -14,7 +14,7 @@ import pandas as pd from app_utils.sql_render import render_sql -from datastore.lake_build import con +from lake_build import con SQL_PATH = Path(__file__).resolve().parent / "sql" # Town and zoning-district boundaries were digitised separately, so subtracting diff --git a/backend/datastore/lake_build.py b/backend/lake_build.py similarity index 100% rename from backend/datastore/lake_build.py rename to backend/lake_build.py diff --git a/backend/run_data_collection.py b/backend/run_data_collection.py index c43525c..5a4142a 100644 --- a/backend/run_data_collection.py +++ b/backend/run_data_collection.py @@ -27,7 +27,7 @@ wastewater, zoning, ) -from datastore.lake_build import insert_year, replace_table +from lake_build import insert_year, replace_table # Datasets WITH year columns (longitudinal) YEARLY_SCRAPERS = [acs5, demographics, economic, education, housing, qcew] diff --git a/backend/run_data_loading.py b/backend/run_data_loading.py index 0af2c2b..a9cd94e 100644 --- a/backend/run_data_loading.py +++ b/backend/run_data_loading.py @@ -15,7 +15,7 @@ import duckdb # DuckLake connection -from datastore.lake_build import con +from lake_build import con # New DuckDB connection (where CLEANED lake tables will go) db_con = duckdb.connect() diff --git a/backend/tests/test_lake.py b/backend/tests/test_lake.py index 5ccb831..8a45e7f 100644 --- a/backend/tests/test_lake.py +++ b/backend/tests/test_lake.py @@ -5,7 +5,7 @@ python -m tests.test_lake """ -from datastore.lake_build import con +from lake_build import con def inspect_schema(schema: str) -> None: diff --git a/justfile b/justfile index 66b4fc3..96a8bc7 100644 --- a/justfile +++ b/justfile @@ -100,6 +100,13 @@ check-frontend: ## ETL (Pipeline) Container ################ +# --------- Pre-step: Lake Builder --------------------- +[working-directory("backend")] +build-lake: + podman build -t localhost/vdc-lake -f ETL/dockerfile.lake . + podman run --rm -v "$(pwd)/Data:/data:z" -e DATA_DIR=/data localhost/vdc-lake + + # --------- 1. Data Collection (E) --------------------- # build the backend COLLECTION image [working-directory("backend")] From b94837f8e450e77b03eec3cf6b313e2d2866c68a Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 14:28:35 -0400 Subject: [PATCH 04/26] fixed lake_build.py bug and updated dockerfiles --- backend/ETL/dockerfile.clean | 2 +- backend/ETL/dockerfile.collect | 2 +- backend/ETL/dockerfile.load | 2 +- backend/lake_build.py | 6 +++--- justfile | 8 ++++++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/ETL/dockerfile.clean b/backend/ETL/dockerfile.clean index 478d9c8..d400888 100644 --- a/backend/ETL/dockerfile.clean +++ b/backend/ETL/dockerfile.clean @@ -22,7 +22,7 @@ COPY --chown=app:app ./api /code/api COPY --chown=app:app ./logger /code/logger # TODO: Remove the dependency on app_utils COPY --chown=app:app ./app_utils /code/app_utils -COPY --chown=app:app ./datastore /code/datastore +COPY --chown=app:app ./lake_build.py /code/lake_build.py ENV DATA_DIR=/data diff --git a/backend/ETL/dockerfile.collect b/backend/ETL/dockerfile.collect index 03df0a8..7b9bde5 100644 --- a/backend/ETL/dockerfile.collect +++ b/backend/ETL/dockerfile.collect @@ -21,7 +21,7 @@ COPY --chown=app:app ./run_data_collection.py /code/run_data_collection.py # TODO: Remove the dependency on app_utils COPY --chown=app:app ./app_utils /code/app_utils -COPY --chown=app:app ./datastore /code/datastore +COPY --chown=app:app ./lake_build.py /code/lake_build.py ENV DATA_DIR=/data diff --git a/backend/ETL/dockerfile.load b/backend/ETL/dockerfile.load index 99d98a9..4d5962c 100644 --- a/backend/ETL/dockerfile.load +++ b/backend/ETL/dockerfile.load @@ -19,7 +19,7 @@ COPY --chown=app:app ./run_data_loading.py /code/run_data_loading.py # TODO: Remove the dependency on app_utils COPY --chown=app:app ./app_utils /code/app_utils # For lake connection import -COPY --chown=app:app ./datastore /code/datastore +COPY --chown=app:app ./lake_build.py /code/lake_build.py ENV DATA_DIR=/data diff --git a/backend/lake_build.py b/backend/lake_build.py index e6df53e..acc9612 100644 --- a/backend/lake_build.py +++ b/backend/lake_build.py @@ -25,7 +25,7 @@ # Attach DuckLake catalog con.execute( - f""" + f"""--sql ATTACH '{LAKE_PATH.as_posix()}' AS lake ( @@ -37,8 +37,8 @@ ) # Create schemas in the lake catalog -con.execute("""--sql CREATE SCHEMA IF NOT EXISTS lake.RAW""") -con.execute("""--sql CREATE SCHEMA IF NOT EXISTS lake.CLEANED""") +con.execute("""CREATE SCHEMA IF NOT EXISTS lake.RAW""") +con.execute("""CREATE SCHEMA IF NOT EXISTS lake.CLEANED""") def insert_year(name: str, df: pd.DataFrame, year: int): diff --git a/justfile b/justfile index 96a8bc7..c99c8b2 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ export DATA_DIR := justfile_directory() / "Data" # Load environment variables -set dotenv-filename := ".env.local" +set dotenv-filename := ".env" ################ # CLI Development # @@ -117,7 +117,11 @@ build-collection: [working-directory("backend")] get-data year: build-collection echo "Using API key: $CENSUS_API_KEY" - podman run --rm -v "$(pwd)/Data:/data:z" -e DATA_DIR=/data localhost/vdc-collection {{year}} + podman run --rm \ + -v "$(pwd)/Data:/data:z" \ + -e DATA_DIR=/data \ + -e CENSUS_API_KEY="$CENSUS_API_KEY" \ + localhost/vdc-collection {{year}} # --------- 2. Data Cleaning (T) --------------------- From 9d7e9eaac3d0b1753d842a3ce6289801bc42996e Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 15:06:38 -0400 Subject: [PATCH 05/26] Attached duckdb to DuckLake (warehouse.duckdb) --- .gitignore | 5 +++ backend/run_data_loading.py | 75 ++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 75ed795..5a84823 100644 --- a/.gitignore +++ b/.gitignore @@ -16,9 +16,14 @@ venv/ backend/Data/Parcels/ #DuckLake files + +# DuckLake catalogue backend/Data/lake backend/Data/lake.files/ +# DuckDB Database with CLEAN tables +backend/Data/warehouse.duckdb + # Local planning/notes files claude_todo.md claude-work-done.md diff --git a/backend/run_data_loading.py b/backend/run_data_loading.py index a9cd94e..cf66688 100644 --- a/backend/run_data_loading.py +++ b/backend/run_data_loading.py @@ -12,16 +12,16 @@ python -m run_data_loading """ -import duckdb +import os +from pathlib import Path -# DuckLake connection -from lake_build import con +import duckdb -# New DuckDB connection (where CLEANED lake tables will go) -db_con = duckdb.connect() +ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = Path(os.getenv("DATA_DIR", ROOT / "Data")) -def get_cleaned_tables() -> list[str]: +def get_cleaned_tables(db_con: duckdb.DuckDBPyConnection) -> list[str]: """ List all active DuckLake tables within the 'CLEANED' lake schema. """ @@ -34,33 +34,64 @@ def get_cleaned_tables() -> list[str]: WHERE s.schema_name = 'CLEANED' AND t.end_snapshot IS NULL AND s.end_snapshot IS NULL + ORDER BY table_name """ - tables = [row[0] for row in con.execute(query).fetchall()] + tables = [row[0] for row in db_con.execute(query).fetchall()] return tables def create_duckdb(): - # Fetch CLEANED table names - tables = get_cleaned_tables() + warehouse_path = DATA_DIR / "warehouse.duckdb" + lake_db_path = DATA_DIR / "lake" + storage_path = DATA_DIR / "lake.files" + + db_con = duckdb.connect(str(warehouse_path)) + + db_con.execute( + f""" + ATTACH '{lake_db_path.as_posix()}' + AS lake ( + TYPE ducklake, + DATA_PATH '{storage_path.as_posix()}', + OVERRIDE_DATA_PATH TRUE + ) + """ + ) + + tables = get_cleaned_tables(db_con) total_tables = len(tables) - # Each table gets written to the DuckDB as its own table - try: - for i, table in enumerate(tables): - df = con.sql(f"SELECT * FROM lake.CLEANED.{table}").df() # noqa: F841 - db_con.execute(f"CREATE OR REPLACE TABLE {table} AS SELECT * FROM df") - - print( - f"\r{i + 1} / {total_tables} tables added to the database", - end="", - flush=True, + failed = [] + + for i, table in enumerate(tables): + try: + db_con.execute( + f''' + CREATE OR REPLACE TABLE "{table}" + AS + SELECT * + FROM lake.CLEANED."{table}" + ''' ) + except Exception as e: + failed.append((table, str(e))) + + print( + f"\r{i + 1}/{total_tables} processed", + end="", + flush=True, + ) + + if failed: + print(f"\n{len(failed)} tables failed:") + for table, error in failed: + print(f" {table}: {error}") - except Exception as e: - print(f"Database creation failed: {e}") + db_con.execute("DETACH lake") + db_con.close() - print("\nDATABASE COMPLETED") + print("\nDATABASE COMPLETED!") def main(): From f417b988bb74b72d9c49cf4e82cf550bc4fa4ce1 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 18 Aug 2026 17:06:18 -0400 Subject: [PATCH 06/26] Added year RANGE for etl process instead of singular year --- backend/data_collection/acs5.py | 84 +++++++++++++++---------- backend/data_collection/demographics.py | 32 ++++++---- backend/data_collection/economic.py | 41 +++++++----- backend/data_collection/education.py | 41 +++++++----- backend/data_collection/housing.py | 33 +++++++--- backend/data_collection/qcew.py | 27 ++++---- backend/lake_build.py | 20 +++--- backend/run_data_collection.py | 19 +++--- justfile | 8 +-- 9 files changed, 187 insertions(+), 118 deletions(-) diff --git a/backend/data_collection/acs5.py b/backend/data_collection/acs5.py index e6e48ed..6509fd1 100644 --- a/backend/data_collection/acs5.py +++ b/backend/data_collection/acs5.py @@ -35,6 +35,8 @@ STORAGE_LOCATION = "Data/Census/ACS_5" ID_VARS = ["year", "geo_type", "table", "NAME", "state", "county"] +YEARS = range(2009, 2025) + # Default geos list in (label, for_clause, in_clause) format GEOS = [(k, *v) for k, v in ALL_GEOS.items()] @@ -71,21 +73,25 @@ def fetch_table(year, table, for_clause, in_clause): return None -def run_acs5_scrape(year: int = 2024, geos: list = GEOS, append: bool = False): +def run_acs5_scrape(years: range = YEARS, geos: list = GEOS, append: bool = False): + """ + Collect ACS 5-year profile tables for multiple years. + """ # Collect raw frames per table all_frames = {table: [] for table in TABLES} - print(f"\n=== {year} ===") - for geo_label, for_clause, in_clause in geos: - for table in TABLES: - print(f" {table} / {geo_label}...") - df = fetch_table(year, table, for_clause, in_clause) - if df is not None: - df["geo_type"] = geo_label - all_frames[table].append(df) - time.sleep(0.1) - - # Save wide + tidy per table + for year in years: + print(f"\n=== {year} ===") + for geo_label, for_clause, in_clause in geos: + for table in TABLES: + print(f" {table} / {geo_label}...") + df = fetch_table(year, table, for_clause, in_clause) + if df is not None: + df["geo_type"] = geo_label + all_frames[table].append(df) + time.sleep(0.1) + + # Save wide + tidy per table results = {} for table, frames in all_frames.items(): if not frames: @@ -107,7 +113,7 @@ def run_acs5_scrape(year: int = 2024, geos: list = GEOS, append: bool = False): # wide_csv_path = f"{STORAGE_LOCATION}/{title}.csv" if append: - new_names = set(combined["NAME"].unique()) + new_names = set(combined["year", "geo_type", "NAME"].unique()) # --- Wide --- try: existing_wide = pd.read_parquet(wide_parquet_path) @@ -124,31 +130,41 @@ def run_acs5_scrape(year: int = 2024, geos: list = GEOS, append: bool = False): # print(f"Saved wide: {title} ({len(combined):,} rows)") # Tidy: run per-year so column labels are year-accurate - tidy_frames = [] + # Tidy: run per-year so column labels are year-accurate + tidy_frames = [] + + for year in sorted(combined["year"].unique()): year_df = combined[combined["year"] == year] - if not year_df.empty: - try: - tidy_year = tidy_census(year_df, year=year, id_vars=ID_VARS) - tidy_year["table"] = table - tidy_frames.append(tidy_year) - except Exception as e: - print(f" SKIP tidy {year} / {table}: {e}") - if tidy_frames: - tidy = pd.concat(tidy_frames, ignore_index=True) + if year_df.empty: + continue + + try: + tidy_year = tidy_census( + year_df, + year=year, + id_vars=ID_VARS, + ) + tidy_year["table"] = table + tidy_frames.append(tidy_year) - label = TABLES[table] + except Exception as e: + print(f" SKIP tidy {year} / {table}: {e}") - results[f"acs5_{label.lower()}"] = tidy - # tidy_parquet_path = f"{STORAGE_LOCATION}/{title}_tidy.parquet" - # tidy_csv_path = f"{STORAGE_LOCATION}/{title}_tidy.csv" + if tidy_frames: + tidy = pd.concat(tidy_frames, ignore_index=True) + + label = TABLES[table] + results[f"acs5_{label.lower()}"] = tidy + # tidy_parquet_path = f"{STORAGE_LOCATION}/{title}_tidy.parquet" + # tidy_csv_path = f"{STORAGE_LOCATION}/{title}_tidy.csv" - # No separate append needed for tidy: it's derived from the - # already-merged wide frame, so it naturally contains all geos. + # No separate append needed for tidy: it's derived from the + # already-merged wide frame, so it naturally contains all geos. - # tidy.to_csv(tidy_csv_path, index=False) - # tidy.to_parquet(tidy_parquet_path, index=False) - # print(f"Saved tidy: {title}_tidy ({len(tidy):,} rows)") + # tidy.to_csv(tidy_csv_path, index=False) + # tidy.to_parquet(tidy_parquet_path, index=False) + # print(f"Saved tidy: {title}_tidy ({len(tidy):,} rows)") return results @@ -172,13 +188,13 @@ def merge_tidy_tables(): return -def collect(year: int = 2024, geos=GEOS, append=False): +def collect(years: range = YEARS, geos=GEOS, append=False): """ Collect ACS profile tables and return tidy datasets. """ tables = run_acs5_scrape( - year=year, + years=years, geos=geos, append=append, ) diff --git a/backend/data_collection/demographics.py b/backend/data_collection/demographics.py index ed6e02b..72d63b1 100644 --- a/backend/data_collection/demographics.py +++ b/backend/data_collection/demographics.py @@ -8,6 +8,8 @@ Output: vt_acs5_b_demographics_tidy.parquet """ +import pandas as pd + from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape # --------------------------------------------------------------------------- @@ -25,6 +27,8 @@ ("75 Plus", range(23, 26), range(47, 50)), ] +YEARS = range(2009, 2025) + def _b01001_codes(male_r, female_r): return [f"B01001_{str(i).zfill(3)}E" for i in male_r] + [ @@ -87,26 +91,32 @@ def _all_b01001_vars(): } -def collect(year: int = 2024, geos=None, append=False): +def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: if geos is None: geos = [(k, *ALL_GEOS[k]) for k in ALL_GEOS] - df = run_acs_b_scrape( - fetch_specs, - var_groups, - "vt_acs5_b_demographics_tidy.parquet", - year=year, - geos=geos, - append=append, - ) + frames = [] + for year in years: + df = run_acs_b_scrape( + fetch_specs, + var_groups, + "vt_acs5_b_demographics_tidy.parquet", + year=year, + geos=geos, + append=append, + ) + if df is not None: + frames.append(df) - return df + return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() if __name__ == "__main__": import argparse p = argparse.ArgumentParser(description="Scrape ACS B-table demographics data.") + p.add_argument("--start-year", type=int, default=2009) + p.add_argument("--end-year", type=int, default=2024) p.add_argument( "--geos", nargs="+", @@ -124,7 +134,7 @@ def collect(year: int = 2024, geos=None, append=False): selected_geos = [(k, *ALL_GEOS[k]) for k in args.geos] df = collect( - year=args.year, + years=range(args.start_year, args.end_year + 1), geos=selected_geos, append=args.append, ) diff --git a/backend/data_collection/economic.py b/backend/data_collection/economic.py index de6a212..0f84a28 100644 --- a/backend/data_collection/economic.py +++ b/backend/data_collection/economic.py @@ -20,6 +20,8 @@ Output: vt_acs5_b_economic_tidy.parquet """ +import pandas as pd + from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape SL = "Labor Force" @@ -70,40 +72,49 @@ "B19301": ["B19301_001E"], } +YEARS = range(2009, 2025) + -def collect(year: int = 2024, geos=None, append=False): +def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: if geos is None: geos = [(k, *ALL_GEOS[k]) for k in ALL_GEOS] - return run_acs_b_scrape( - fetch_specs, - var_groups, - "vt_acs5_b_economic_tidy.parquet", - year=year, - geos=geos, - append=append, - ) + frames = [] + for year in years: + df = run_acs_b_scrape( + fetch_specs, + var_groups, + "vt_acs5_b_economic_tidy.parquet", + year=year, + geos=geos, + append=append, + ) + if df is not None: + frames.append(df) + + return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(description="Scrape ACS B-table economic data.") - parser.add_argument("year", type=int, nargs="?", default=2024) - parser.add_argument( + p = argparse.ArgumentParser(description="Scrape ACS B-table economic data.") + p.add_argument("--start-year", type=int, default=2009) + p.add_argument("--end-year", type=int, default=2024) + p.add_argument( "--geos", nargs="+", choices=list(ALL_GEOS), default=list(ALL_GEOS), ) - parser.add_argument("--append", action="store_true") + p.add_argument("--append", action="store_true") - args = parser.parse_args() + args = p.parse_args() selected_geos = [(k, *ALL_GEOS[k]) for k in args.geos] df = collect( - year=args.year, + years=range(args.start_year, args.end_year + 1), geos=selected_geos, append=args.append, ) diff --git a/backend/data_collection/education.py b/backend/data_collection/education.py index 35f543c..076abf2 100644 --- a/backend/data_collection/education.py +++ b/backend/data_collection/education.py @@ -12,6 +12,8 @@ Output: vt_acs5_b_education_tidy.parquet """ +import pandas as pd + from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape TOTAL = "B15003_001E" @@ -37,40 +39,49 @@ "B15003": [TOTAL] + [f"B15003_{str(i).zfill(3)}E" for i in range(2, 26)], } +YEARS = range(2009, 2025) + -def collect(year: int = 2024, geos=None, append=False): +def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: if geos is None: geos = [(k, *ALL_GEOS[k]) for k in ALL_GEOS] - return run_acs_b_scrape( - fetch_specs, - var_groups, - "vt_acs5_b_education_tidy.parquet", - year=year, - geos=geos, - append=append, - ) + frames = [] + for year in years: + df = run_acs_b_scrape( + fetch_specs, + var_groups, + "vt_acs5_b_education_tidy.parquet", + year=year, + geos=geos, + append=append, + ) + if df is not None: + frames.append(df) + + return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(description="Scrape ACS B-table education data.") - parser.add_argument("year", type=int, nargs="?", default=2024) - parser.add_argument( + p = argparse.ArgumentParser(description="Scrape ACS B-table education data.") + p.add_argument("--start-year", type=int, default=2009) + p.add_argument("--end-year", type=int, default=2024) + p.add_argument( "--geos", nargs="+", choices=list(ALL_GEOS), default=list(ALL_GEOS), ) - parser.add_argument("--append", action="store_true") + p.add_argument("--append", action="store_true") - args = parser.parse_args() + args = p.parse_args() selected_geos = [(k, *ALL_GEOS[k]) for k in args.geos] df = collect( - year=args.year, + years=range(args.start_year, args.end_year + 1), geos=selected_geos, append=args.append, ) diff --git a/backend/data_collection/housing.py b/backend/data_collection/housing.py index 0bcc58a..108fad9 100644 --- a/backend/data_collection/housing.py +++ b/backend/data_collection/housing.py @@ -15,6 +15,8 @@ Output: vt_acs5_b_housing_tidy.parquet """ +import pandas as pd + from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape S = "Housing" @@ -42,24 +44,35 @@ } -def collect(year: int = 2024, geos=None, append=False): +YEARS = range(2009, 2025) + + +def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: if geos is None: geos = [(k, *ALL_GEOS[k]) for k in ALL_GEOS] - return run_acs_b_scrape( - fetch_specs, - var_groups, - "vt_acs5_b_housing_tidy.parquet", - year=year, - geos=geos, - append=append, - ) + frames = [] + for year in years: + df = run_acs_b_scrape( + fetch_specs, + var_groups, + "vt_acs5_b_housing_tidy.parquet", + year=year, + geos=geos, + append=append, + ) + if df is not None: + frames.append(df) + + return pd.concat(frames, ignore_index=True) if frames else pd.DataFrame() if __name__ == "__main__": import argparse p = argparse.ArgumentParser(description="Scrape ACS B-table housing data.") + p.add_argument("--start-year", type=int, default=2009) + p.add_argument("--end-year", type=int, default=2024) p.add_argument( "--geos", nargs="+", @@ -77,7 +90,7 @@ def collect(year: int = 2024, geos=None, append=False): selected_geos = [(k, *ALL_GEOS[k]) for k in args.geos] df = collect( - year=args.year, + years=range(args.start_year, args.end_year + 1), geos=selected_geos, append=args.append, ) diff --git a/backend/data_collection/qcew.py b/backend/data_collection/qcew.py index 7ab9695..d34eb57 100644 --- a/backend/data_collection/qcew.py +++ b/backend/data_collection/qcew.py @@ -88,6 +88,8 @@ BASE_URL = "https://data.bls.gov/cew/data/api/{year}/{q}/area/{fips}.csv" QUARTERS = [1, 2, 3, 4] +YEARS = range(2009, 2025) + # --------------------------------------------------------------------------- # Fetching @@ -214,18 +216,17 @@ def process_county(area_fips: str, county_name: str, year: int) -> pd.DataFrame: # --------------------------------------------------------------------------- -def run_qcew_scrape(year: int) -> pd.DataFrame: +def run_qcew_scrape(years: range = YEARS) -> pd.DataFrame: STORAGE_PATH.mkdir(parents=True, exist_ok=True) all_frames = [] - - for fips, name in VT_COUNTIES.items(): - print(f"\n=== {name} County ({fips}) ===") - df = process_county(fips, name, year) - if not df.empty: - all_frames.append(df) - print(f" {len(df):,} rows") - else: - print(" No data") + for year in years: + for fips, name in VT_COUNTIES.items(): + print(f"\n=== {name} County ({fips}) ===") + df = process_county(fips, name, year) + if not df.empty: + all_frames.append(df) + else: + print("No data") if not all_frames: print("No data fetched.") @@ -235,13 +236,11 @@ def run_qcew_scrape(year: int) -> pd.DataFrame: combined.sort_values(["County", "year", "quarter", "sector"], inplace=True) combined.reset_index(drop=True, inplace=True) - # combined.to_parquet(OUTPUT_FILE, index=False) - # print(f"\nDone. {len(combined):,} rows → {OUTPUT_FILE}") return combined -def collect(year: int = 2024): - df = run_qcew_scrape(year) +def collect(years: range = YEARS): + df = run_qcew_scrape(years) return df diff --git a/backend/lake_build.py b/backend/lake_build.py index acc9612..8658548 100644 --- a/backend/lake_build.py +++ b/backend/lake_build.py @@ -1,5 +1,6 @@ import os from pathlib import Path +from typing import Iterable, Union import duckdb import geopandas as gpd @@ -41,14 +42,18 @@ con.execute("""CREATE SCHEMA IF NOT EXISTS lake.CLEANED""") -def insert_year(name: str, df: pd.DataFrame, year: int): +def insert_year(name: str, df: pd.DataFrame, years: Union[int, Iterable[int]]): """ - Insert or replace one year's data in a DuckLake table. + Insert or replace data for specific year(s) in a DuckLake table. """ - if "year" not in map(str.lower, df.columns): raise ValueError(f"DataFrame for {name!r} does not contain a 'year' column.") + if isinstance(years, int): + years_list = [years] + else: + years_list = list(years) + if isinstance(df, gpd.GeoDataFrame): df = df.copy() df["geometry"] = df.geometry.to_wkb() @@ -81,19 +86,20 @@ def insert_year(name: str, df: pd.DataFrame, year: int): ) return - # Remove this year's existing data. + # Remove existing data for the target years con.execute( f""" DELETE FROM lake.{schema}.{table} - WHERE year = ? + WHERE year IN ({",".join("?" for _ in years_list)}) """, - [year], + years_list, ) - # Insert the replacement. + # Insert the replacement data con.execute( f""" INSERT INTO lake.{schema}.{table} + BY NAME SELECT * FROM tmp_df """ ) diff --git a/backend/run_data_collection.py b/backend/run_data_collection.py index 5a4142a..f5f03df 100644 --- a/backend/run_data_collection.py +++ b/backend/run_data_collection.py @@ -42,15 +42,17 @@ zoning, ] +YEARS = range(2009, 2025) -def run_scraper(scraper, yearly=False, year=None): + +def run_scraper(scraper, yearly: bool = False, years: range = YEARS): name = scraper.__name__.split(".")[-1] try: print(f"Running {name}...") if yearly: - outputs = scraper.collect(year=year) + outputs = scraper.collect(years) else: outputs = scraper.collect() @@ -62,7 +64,7 @@ def run_scraper(scraper, yearly=False, year=None): print(f"Loading {full_name}") # If the dataset is longitudinal, replace or append that year's data if yearly: - insert_year(full_name, df, year) + insert_year(full_name, df, years) # If a static dataset, replace the whole table else: replace_table(full_name, df) @@ -74,9 +76,9 @@ def run_scraper(scraper, yearly=False, year=None): raise -def run_master_scrape(year: int): +def run_master_scrape(start_year: int = 2009, end_year: int = 2024): for scraper in YEARLY_SCRAPERS: - run_scraper(scraper, yearly=True, year=year) + run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) for scraper in STATIC_SCRAPERS: run_scraper(scraper, yearly=False) @@ -85,12 +87,13 @@ def run_master_scrape(year: int): def main(): # Accepts the year argument from justfile for collection parser = argparse.ArgumentParser() - parser.add_argument("year", type=int) + parser.add_argument("start_year", type=int) + parser.add_argument("end_year", type=int) args = parser.parse_args() - print(f"Collecting data for {args.year}") + print(f"Collecting data from {args.start_year} to {args.end_year}") - run_master_scrape(args.year) + run_master_scrape(args.start_year, args.end_year) if __name__ == "__main__": diff --git a/justfile b/justfile index c99c8b2..375252d 100644 --- a/justfile +++ b/justfile @@ -115,13 +115,13 @@ build-collection: # Collect the data for a specified year and add to lake.RAW tables [working-directory("backend")] -get-data year: build-collection +get-data start_year end_year: build-collection echo "Using API key: $CENSUS_API_KEY" podman run --rm \ -v "$(pwd)/Data:/data:z" \ -e DATA_DIR=/data \ -e CENSUS_API_KEY="$CENSUS_API_KEY" \ - localhost/vdc-collection {{year}} + localhost/vdc-collection {{start_year}} {{end_year}} # --------- 2. Data Cleaning (T) --------------------- @@ -142,9 +142,9 @@ load-data: # Collect (E), clean (T), and load (L) the data (Full pipeline run) [working-directory("backend")] -run-etl year: +run-etl start_year end_year: # Collect the data for a certain year - just get-data {{year}} + just get-data {{start_year}} {{end_year}} # Clean the RAW populated lake tables into CLEANED just transform-data # Load CLEANED tables into DuckDB instance From 653b422a6e210936a1babda15690e9437f3b1db8 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 11:57:35 -0400 Subject: [PATCH 07/26] updated acs5 routes to point to new db --- backend/query/acs5.py | 23 ++++++++++++++++------- backend/query/processed_db.py | 15 ++++++++++++--- frontend/next-env.d.ts | 2 +- justfile | 2 +- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/backend/query/acs5.py b/backend/query/acs5.py index 252fd17..afcbb06 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -23,18 +23,22 @@ # Per-dataset FIXED filters, expressed as {column: [values]} and folded into the # FilterSource (these replace the old raw-SQL base_conditions) QUERY_CONFIG = { - "demographics": {"table": "acs5_b10_census", "fixed_filters": {}}, - "education": {"table": "acs5_b15003_education", "fixed_filters": {}}, - "housing": {"table": "acs5_b_housing", "fixed_filters": {}}, + "demographics": {"table": "acs5_demographics_tidy", "fixed_filters": {}}, + "education": {"table": "acs5_education_tidy", "fixed_filters": {}}, + "housing": {"table": "acs5_housing_tidy", "fixed_filters": {}}, "labor_force": { - "table": "acs5_b_economic", + "table": "acs5_economics_tidy", "fixed_filters": {"Section": ["Labor Force"]}, }, - "income": {"table": "acs5_b_economic", "fixed_filters": {"Section": ["Income"]}}, + "income": { + "table": "acs5_economics_tidy", + "fixed_filters": {"Section": ["Income"]}, + }, "median_age": { - "table": "acs5_b10_census", + "table": "acs5Demographics_medianAge_timeseries", "fixed_filters": {"Variable": ["Median Age"]}, }, + # TODO: Create a "snapshot" table to database "snapshot": {"table": "acs5_snapshot", "fixed_filters": {}}, } @@ -93,7 +97,9 @@ def get_acs5_tidy(dataset: str, filters: dict | None = None) -> pd.DataFrame: def get_unemployment_rate_ts(filters: dict | None = None) -> pd.DataFrame: - source = _acs5_source(table="acs5_unemployment_rate", filters=filters) + source = _acs5_source( + table="acs5Economics_unemploymentRate_timeseries", filters=filters + ) sql, params = sql_filter_block(sql_path / "unemployment_rate.sql", [source]) @@ -106,6 +112,7 @@ def get_unemployment_rate_ts(filters: dict | None = None) -> pd.DataFrame: return result +# FIXME: Link to new database table name (broken for now) def get_median_earnings(filters: dict | None = None) -> pd.DataFrame: source = _acs5_source(table="acs5_median_earnings", filters=filters) @@ -120,6 +127,7 @@ def get_median_earnings(filters: dict | None = None) -> pd.DataFrame: return result +# FIXME: Link to new database table name (broken for now) def get_snapshot(filters: dict | None = None) -> pd.DataFrame: source = _acs5_source(table="snapshot", filters=filters) @@ -134,5 +142,6 @@ def get_snapshot(filters: dict | None = None) -> pd.DataFrame: return result +# FIXME: Link to new database table name (broken for now) def get_acs5_filters(): return filter_tree(ACS5_FILTER_COLS, ACS5_TREE_LABELS, "acs5_info") diff --git a/backend/query/processed_db.py b/backend/query/processed_db.py index af6c879..bbd373b 100644 --- a/backend/query/processed_db.py +++ b/backend/query/processed_db.py @@ -16,7 +16,9 @@ logger = logging.getLogger(__name__) proc_dir = Path(__file__).resolve().parent.parent / "Data" / "_Processed" -DATA_DIR = Path(os.environ.get("DATA_DIR", proc_dir)) + +BACKEND_DIR = Path(__file__).resolve().parents[1] +DATA_DIR = Path(os.environ.get("DATA_DIR", BACKEND_DIR / "Data")) def _load_spatial(con: duckdb.DuckDBPyConnection) -> None: @@ -46,11 +48,18 @@ def _load_spatial(con: duckdb.DuckDBPyConnection) -> None: def _build() -> duckdb.DuckDBPyConnection: - print(DATA_DIR) path = Path(proc_dir / "all_data.duckdb") con = duckdb.connect(path, read_only=True) _load_spatial(con) return con -DB = _build() +def _build_etl_db() -> duckdb.DuckDBPyConnection: + print(f"DATA DIRECTORY PATH: {DATA_DIR}") + path = Path(DATA_DIR / "warehouse.duckdb") + con = duckdb.connect(path, read_only=True) + _load_spatial(con) + return con + + +DB = _build_etl_db() diff --git a/frontend/next-env.d.ts b/frontend/next-env.d.ts index 20e7bcf..c4b7818 100644 --- a/frontend/next-env.d.ts +++ b/frontend/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import './.next/dev/types/routes.d.ts'; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/justfile b/justfile index 375252d..cc7d105 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,6 @@ ## Set up environment ## -export DATA_DIR := justfile_directory() / "Data" +export DATA_DIR := justfile_directory() / "backend" / "Data" # Load environment variables set dotenv-filename := ".env" From 62c9e3d48fd91f6836661c0f950f8aaa2d4fbebd Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 11:59:32 -0400 Subject: [PATCH 08/26] future fixes noted --- backend/api/routes/post_routes/post_acs5_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 6ef9ef5..87aea8a 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -4,7 +4,6 @@ from api.metadata_registry import get_metadata from api.models import DPSeriesRequest, FilterRequest, make_response -from query.processed_db import DB # TODO: Simplify / Refactor this script using the new query folder functions from query.acs5 import ( @@ -13,6 +12,7 @@ get_snapshot, get_unemployment_rate_ts, ) +from query.processed_db import DB logger = logging.getLogger(__name__) router = APIRouter() @@ -70,14 +70,14 @@ async def tidy_unemployment_rate(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("unemployment_rate")) -# Median Earnings +# Median Earnings (FIXME) @router.post("/load/acs5-db/tidy/median-earnings") async def tidy_median_earnings(request: FilterRequest): rows = get_median_earnings(filters=request.filters) return make_response(data=rows, metadata=get_metadata("median_earnings")) -# Geography Snapshot Variables +# Geography Snapshot Variables (FIXME) @router.post("/load/acs5-db/tidy/snapshot") async def tidy_snapshot(request: FilterRequest): rows = get_snapshot(filters=request.filters) From dd22580fdd8afac08c159be63af1424fb22d1837 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 12:14:25 -0400 Subject: [PATCH 09/26] cdc route updates --- backend/api/routes/post_routes/post_cdc.py | 3 ++- backend/api/schema.json | 2 +- backend/query/cdc.py | 5 +++-- backend/query/sql/cdc/county_places.sql | 2 +- backend/query/sql/cdc/tract_places.sql | 2 +- backend/tests/test_sql_render.py | 4 ++-- frontend/src/components/FilterRedux/filterDefs.ts | 4 ++-- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/api/routes/post_routes/post_cdc.py b/backend/api/routes/post_routes/post_cdc.py index 47b34b8..c019bae 100644 --- a/backend/api/routes/post_routes/post_cdc.py +++ b/backend/api/routes/post_routes/post_cdc.py @@ -9,7 +9,7 @@ @router.post("/load/mapping/cdc/places/single") async def cdc_single_geojson(request: FilterRequest): - source = request_to_source(request, "cdc_county_places", "default") + source = request_to_source(request, "cdc_places_county", "default") data = single_var_geojson([source]) return data @@ -38,6 +38,7 @@ async def cdc_comparison_tract(specs: list[FilterSpec]) -> APIResponse: return make_response(data=geojson, metadata={"legend": legend}) +# (FIXME) @router.post("/load/mapping/cdc/places/pca_summary") async def cdc_pca(specs: list[FilterSpec]) -> APIResponse: return make_response(data=get_cdc_county_pca(), metadata={}) diff --git a/backend/api/schema.json b/backend/api/schema.json index e019a8e..d797271 100644 --- a/backend/api/schema.json +++ b/backend/api/schema.json @@ -35,7 +35,7 @@ "Value": "val" } }, - "cdc_county_places": { + "cdc_places_county": { "join_key": "LocationID", "join_type": "inner", "value_col": "Data_Value", diff --git a/backend/query/cdc.py b/backend/query/cdc.py index d00454d..30e9970 100644 --- a/backend/query/cdc.py +++ b/backend/query/cdc.py @@ -81,7 +81,7 @@ def _measure_cutpoints(measures: list[str]) -> tuple[list[float], list[float]]: """Bin edges for each measure from the precomputed cdc_edges table.""" params: list = [] where_string = compile_where({"Measure": measures}, params) - sql = f"SELECT * FROM cdc_county_edges {where_string}" + sql = f"SELECT * FROM cdc_edges_county {where_string}" edges = DB.execute(sql, params).df() edges_x = ( edges[edges["Measure"] == measures[0]].drop(columns="Measure").iloc[0].tolist() @@ -106,7 +106,7 @@ def dual_var_comparison( # Both measures ride in one merged FilterSource so the shared places.sql # template serves the single- and dual-variable cases alike. - table = "cdc_county_places" if geoLevel == "county_places" else "cdc_tract_places" + table = "cdc_places_county" if geoLevel == "county_places" else "cdc_places_tract" merged = FilterSource(filter_table=table, filters={"Measure": measures}) sql_path = sql_dir / f"{geoLevel}.sql" sql, params = sql_filter_block(sql_path, [merged]) @@ -149,6 +149,7 @@ def dual_var_comparison( return geojson, legend +# FIXME: Add PCA table in data_cleaning/clean_cdc.py script (broken for now) def get_cdc_county_pca(): df = DB.execute("""--sql SELECT i.LocationID, ROUND(i.pca_score, 2) AS "Health Burden", c.CountyName diff --git a/backend/query/sql/cdc/county_places.sql b/backend/query/sql/cdc/county_places.sql index da1cdb0..f01befa 100644 --- a/backend/query/sql/cdc/county_places.sql +++ b/backend/query/sql/cdc/county_places.sql @@ -7,6 +7,6 @@ SELECT c.CountyFIPS, c.CountyName, ST_ASGEOJSON(c.geom) AS geometry -FROM cdc_county_places AS p +FROM cdc_places_county AS p LEFT JOIN vermont_counties AS c ON p.LocationID = c.CountyFIPS {{ where_string }} diff --git a/backend/query/sql/cdc/tract_places.sql b/backend/query/sql/cdc/tract_places.sql index 7ff166f..3542680 100644 --- a/backend/query/sql/cdc/tract_places.sql +++ b/backend/query/sql/cdc/tract_places.sql @@ -6,6 +6,6 @@ SELECT ROUND(p.natl_pct * 100, 2) AS natl_pct, ST_ASGEOJSON(c.geometry) AS geometry, c.name -FROM cdc_tract_places AS p +FROM cdc_places_tract AS p LEFT JOIN vermont_tracts AS c ON p.LocationID = c.LocationID {{ where_string }} diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index 76fa137..a195857 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -12,7 +12,7 @@ from sqlfluff.core import FluffConfig, Linter from api.models import FilterSource, RangeFilter -from sql_render import ( +from app_utils.sql_render import ( compile_filters, compile_where, filter_clauses, @@ -50,7 +50,7 @@ "query/sql/acs5/unemployment_rate.sql": [WHERE_SOURCE], "query/sql/cdc/county_places.sql": [ FilterSource( - filter_table="cdc_county_places", + filter_table="cdc_places_county", filters={"Measure": ["Depression among adults"]}, ) ], diff --git a/frontend/src/components/FilterRedux/filterDefs.ts b/frontend/src/components/FilterRedux/filterDefs.ts index 31b836e..877f5e9 100644 --- a/frontend/src/components/FilterRedux/filterDefs.ts +++ b/frontend/src/components/FilterRedux/filterDefs.ts @@ -2,12 +2,12 @@ import { filterDef } from './filterTypes'; export const cdc_filtering: filterDef[] = [ { - filter_table: 'cdc_county_places', + filter_table: 'cdc_places_county', filter_style: 'Cascade', label: 'Variable 1', }, { - filter_table: 'cdc_county_places', + filter_table: 'cdc_places_county', filter_style: 'Cascade', label: 'Variable 2', }, From e2df8bcfba184dd0cdceac6b53dda094e2a315ce Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 12:21:49 -0400 Subject: [PATCH 10/26] wastewater database updates --- backend/query/sql/wastewater/service_area_geo_query.sql | 4 ++-- backend/query/sql/wastewater/soil_suitability_geo_query.sql | 6 +++--- backend/query/sql/wastewater/waste_treatment_geo_query.sql | 4 ++-- .../query/sql/wastewater/waste_treatment_permit_table.sql | 4 ++-- backend/query/wastewater.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/query/sql/wastewater/service_area_geo_query.sql b/backend/query/sql/wastewater/service_area_geo_query.sql index bb47f2e..5a783b9 100644 --- a/backend/query/sql/wastewater/service_area_geo_query.sql +++ b/backend/query/sql/wastewater/service_area_geo_query.sql @@ -20,7 +20,7 @@ FROM ( ) ) ) AS feature - FROM service_areas_service_area_info AS i - INNER JOIN service_areas_service_area_geom AS g USING (ID) + FROM VersoWastewater_serviceAreas_info AS i + INNER JOIN VersoWastewater_serviceAreas_geom AS g USING (ID) {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/wastewater/soil_suitability_geo_query.sql b/backend/query/sql/wastewater/soil_suitability_geo_query.sql index 80c10ff..3e6234f 100644 --- a/backend/query/sql/wastewater/soil_suitability_geo_query.sql +++ b/backend/query/sql/wastewater/soil_suitability_geo_query.sql @@ -21,9 +21,9 @@ FROM ( ) ) ) AS feature - FROM soil_suitability_info_soil_suit AS i - INNER JOIN soil_suitability_geom_soil_suit AS g USING (ID) - LEFT JOIN soil_suitability_soil_suitability_colors AS c + FROM VersoWastewater_soilSuitability_info AS i + INNER JOIN VersoWastewater_soilSuitability_geom AS g USING (ID) + LEFT JOIN VersoWastewater_soilSuitability_colors AS c ON i.Suitability = c.soil_suitability {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/wastewater/waste_treatment_geo_query.sql b/backend/query/sql/wastewater/waste_treatment_geo_query.sql index a95d1cc..8c29a8e 100644 --- a/backend/query/sql/wastewater/waste_treatment_geo_query.sql +++ b/backend/query/sql/wastewater/waste_treatment_geo_query.sql @@ -23,7 +23,7 @@ FROM ( ) ) ) AS feature - FROM treatment_facilities_treatment_facility_info AS i - INNER JOIN treatment_facilities_treatment_facility_geom AS g USING (ID) + FROM VersoWastewater_treatmentFacilities_info AS i + INNER JOIN VersoWastewater_treatmentFacilities_geom AS g USING (ID) {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/wastewater/waste_treatment_permit_table.sql b/backend/query/sql/wastewater/waste_treatment_permit_table.sql index 51de522..d274676 100644 --- a/backend/query/sql/wastewater/waste_treatment_permit_table.sql +++ b/backend/query/sql/wastewater/waste_treatment_permit_table.sql @@ -5,5 +5,5 @@ SELECT p.NPDESPermitID AS "NPDES Permit ID", p.PermitLink AS "Permit Link", p.PermitteeName AS "Permittee Name" -FROM treatment_facilities_treatment_facility_info AS i -INNER JOIN treatment_facilities_treatment_facility_permit_info AS p USING (ID) +FROM VersoWastewater_treatmentFacilities_info AS i +INNER JOIN VersoWastewater_treatmentFacilitiesPermits_info AS p USING (ID) diff --git a/backend/query/wastewater.py b/backend/query/wastewater.py index fe65d9b..7b1cd0e 100644 --- a/backend/query/wastewater.py +++ b/backend/query/wastewater.py @@ -4,7 +4,7 @@ **Created**: 2026-07-06 **Description**: - Functions for serving wastewater data to the API from the parquet files. + Functions for serving wastewater data to the API from the database tables. """ import logging From 931ffedb7e0f9d68dc45640eb6bf04b07c91e0fb Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 12:28:52 -0400 Subject: [PATCH 11/26] updated zoning routes to database --- backend/query/sql/zoning/agg_info_table.sql | 4 ++-- backend/query/sql/zoning/agg_rules_table.sql | 4 ++-- backend/query/sql/zoning/geo_query.sql | 10 +++++----- backend/query/sql/zoning/info_table.sql | 4 ++-- backend/query/sql/zoning/rules.sql | 6 +++--- backend/query/sql/zoning/rules_table.sql | 2 +- backend/query/sql/zoning/unzoned.sql | 2 +- backend/query/zoning.py | 2 ++ backend/tests/test_sql_render.py | 4 +++- design/archive/zoning-four-table-migration.md | 6 +++--- 10 files changed, 24 insertions(+), 20 deletions(-) diff --git a/backend/query/sql/zoning/agg_info_table.sql b/backend/query/sql/zoning/agg_info_table.sql index 25d241c..357b2a5 100644 --- a/backend/query/sql/zoning/agg_info_table.sql +++ b/backend/query/sql/zoning/agg_info_table.sql @@ -3,7 +3,7 @@ SELECT i.District_Type AS "District Type", SUM(i.Acres) AS Acres, ANY_VALUE(c.hex_color) AS hex_color -FROM zoning_info AS i -LEFT JOIN zoning_colors AS c ON i.District_Type = c.district_type +FROM VersoZoning_info AS i +LEFT JOIN VersoZoning_colors AS c ON i.District_Type = c.district_type {{ join_filter_block }} GROUP BY i.District_Type diff --git a/backend/query/sql/zoning/agg_rules_table.sql b/backend/query/sql/zoning/agg_rules_table.sql index a997d00..be1f659 100644 --- a/backend/query/sql/zoning/agg_rules_table.sql +++ b/backend/query/sql/zoning/agg_rules_table.sql @@ -3,8 +3,8 @@ SELECT r.use_type, r.val, SUM(i.Acres) AS Acres -FROM zoning_rules AS r -INNER JOIN zoning_info AS i USING (OBJECT_ID) +FROM VersoZoning_rules AS r +INNER JOIN VersoZoning_info AS i USING (OBJECT_ID) {{ join_filter_block }} WHERE r.rule = 'Allowance' diff --git a/backend/query/sql/zoning/geo_query.sql b/backend/query/sql/zoning/geo_query.sql index e1daa13..96bb617 100644 --- a/backend/query/sql/zoning/geo_query.sql +++ b/backend/query/sql/zoning/geo_query.sql @@ -9,9 +9,9 @@ filtered AS ( i.District_Name, c.rgba, g.geom - FROM zoning_info AS i - INNER JOIN zoning_geom AS g USING (OBJECT_ID) - LEFT JOIN zoning_colors AS c ON i.District_Type = c.district_type + FROM VersoZoning_info AS i + INNER JOIN VersoZoning_geom AS g USING (OBJECT_ID) + LEFT JOIN VersoZoning_colors AS c ON i.District_Type = c.district_type {{ join_filter_block }} ), @@ -49,8 +49,8 @@ county_area AS ( SELECT i.County, ST_Area_Spheroid(ST_Union_Agg(g.geom)) / 4046.8564224 AS total_acres - FROM zoning_info AS i - INNER JOIN zoning_geom AS g USING (OBJECT_ID) + FROM VersoZoning_info AS i + INNER JOIN VersoZoning_geom AS g USING (OBJECT_ID) GROUP BY i.County ), diff --git a/backend/query/sql/zoning/info_table.sql b/backend/query/sql/zoning/info_table.sql index fa7b75a..04e3f7b 100644 --- a/backend/query/sql/zoning/info_table.sql +++ b/backend/query/sql/zoning/info_table.sql @@ -5,6 +5,6 @@ SELECT i.District_Type AS "District Type", ROUND(i.Acres, 2) AS Acres, c.hex_color -FROM zoning_info AS i -LEFT JOIN zoning_colors AS c ON i.District_Type = c.district_type +FROM VersoZoning_info AS i +LEFT JOIN VersoZoning_colors AS c ON i.District_Type = c.district_type {{ join_filter_block }} diff --git a/backend/query/sql/zoning/rules.sql b/backend/query/sql/zoning/rules.sql index 607676b..58433d6 100644 --- a/backend/query/sql/zoning/rules.sql +++ b/backend/query/sql/zoning/rules.sql @@ -21,8 +21,8 @@ FROM ( ) ) ) AS feature - FROM zoning_info AS i - INNER JOIN zoning_geom AS g USING (OBJECT_ID) - LEFT JOIN zoning_colors AS c ON i.District_Type = c.district_type + FROM VersoZoning_info AS i + INNER JOIN VersoZoning_geom AS g USING (OBJECT_ID) + LEFT JOIN VersoZoning_colors AS c ON i.District_Type = c.district_type {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/zoning/rules_table.sql b/backend/query/sql/zoning/rules_table.sql index 86a5c84..e66a250 100644 --- a/backend/query/sql/zoning/rules_table.sql +++ b/backend/query/sql/zoning/rules_table.sql @@ -4,5 +4,5 @@ SELECT r.use_type, r.rule, r.val -FROM zoning_rules AS r +FROM VersoZoning_rules AS r {{ join_filter_block }} diff --git a/backend/query/sql/zoning/unzoned.sql b/backend/query/sql/zoning/unzoned.sql index 23b62ea..0ff88f3 100644 --- a/backend/query/sql/zoning/unzoned.sql +++ b/backend/query/sql/zoning/unzoned.sql @@ -22,5 +22,5 @@ FROM ( ) ) ) AS feature - FROM zoning_empty_geom + FROM VersoZoning_empty_geom ) AS features; diff --git a/backend/query/zoning.py b/backend/query/zoning.py index 68f222d..af3360a 100644 --- a/backend/query/zoning.py +++ b/backend/query/zoning.py @@ -3,6 +3,8 @@ Fitz Koch **Created**: 2026-06-01 +**Updated**: + 2026-08-20 **Description**: Functions for serving zoning_info data to the API from the parquet files. """ diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index a195857..aef98eb 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -182,7 +182,9 @@ def test_left_join(self): def test_spatial_join(self): src = FilterSource( - filter_table="zoning_geom", join_key="geom", join_type="spatial_intersect" + filter_table="VersoZoning_geom", + join_key="geom", + join_type="spatial_intersect", ) _, join = compile_filters([src], []) assert join == "JOIN f0 ON ST_Intersects(g.geom, f0.geom)" diff --git a/design/archive/zoning-four-table-migration.md b/design/archive/zoning-four-table-migration.md index fe4084b..cd57722 100644 --- a/design/archive/zoning-four-table-migration.md +++ b/design/archive/zoning-four-table-migration.md @@ -201,9 +201,9 @@ def geojson(filters: dict | None = None) -> dict: c.hex_color AS hex_color, c.rgba AS rgba, ST_AsGeoJSON(ST_Simplify(g.geom, 0.0001)) AS geometry - FROM zoning_info i - JOIN zoning_geom g USING (OBJECT_ID) - LEFT JOIN zoning_colors c ON c.district_type = i.District_Type + FROM VersoZoning_info i + JOIN VersoZoning_geom g USING (OBJECT_ID) + LEFT JOIN VersoZoning_colors c ON c.district_type = i.District_Type {where} """, params, From e7975aa490aa58e9a64601e8dd4a8d21976f659a Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 12:36:18 -0400 Subject: [PATCH 12/26] more zoning table name changes --- backend/.sqlfluff | 2 +- backend/api/routes/post_routes/post_zoning.py | 6 +++--- backend/api/schema.json | 14 +++++++------- backend/notebooks/zoning/runtime_test.qmd | 15 ++++++++------- backend/query/zoning.py | 2 +- backend/tests/test_sql_render.py | 4 ++-- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/backend/.sqlfluff b/backend/.sqlfluff index 57f8427..734d261 100644 --- a/backend/.sqlfluff +++ b/backend/.sqlfluff @@ -27,7 +27,7 @@ case_sensitive = True [sqlfluff:templater:jinja:context] table = acs5_b10_census where_string = WHERE "Measure" IN ($1) -cte_filter_block = WITH f0 AS (SELECT DISTINCT OBJECT_ID FROM zoning_info WHERE "County" IN ($1)) +cte_filter_block = WITH f0 AS (SELECT DISTINCT OBJECT_ID FROM VersoZoning_info WHERE "County" IN ($1)) join_filter_block = JOIN f0 USING (OBJECT_ID) info_string = OBJECT_ID, County rule_string = CAST("Residential_Min_Lot" AS VARCHAR) AS residential_min_lot diff --git a/backend/api/routes/post_routes/post_zoning.py b/backend/api/routes/post_routes/post_zoning.py index f5d1855..68010fa 100644 --- a/backend/api/routes/post_routes/post_zoning.py +++ b/backend/api/routes/post_routes/post_zoning.py @@ -35,21 +35,21 @@ async def zoning_unzoned(): @router.post("/load/mapping/zoning/standard") async def zoning_geojson_info(request: FilterRequest): - source = request_to_source(request, "zoning_info", "default") + source = request_to_source(request, "VersoZoning_info", "default") data = get_zoning_geojson([source]) return Response(content=data, media_type="application/json") @router.post("/load/data/zoning/aggregated") async def acreage_response(request: FilterRequest): - source = request_to_source(request, "zoning_info", "default") + source = request_to_source(request, "VersoZoning_info", "default") agg, table = get_zoning_aggregated_acres([source]) return make_response(data=agg, metadata=get_metadata("zoning"), tableData=table) @router.post("/load/data/zoning/allowances") async def zoning_allowances(request: FilterRequest): - source = request_to_source(request, "zoning_info", "default") + source = request_to_source(request, "VersoZoning_info", "default") agg, table = get_zoning_allowances([source]) return make_response( data=agg, diff --git a/backend/api/schema.json b/backend/api/schema.json index d797271..3a49b87 100644 --- a/backend/api/schema.json +++ b/backend/api/schema.json @@ -1,6 +1,6 @@ { "default": { - "zoning_info": { + "VersoZoning_info": { "join_key": "OBJECT_ID", "join_type": "inner", "columns": { @@ -10,7 +10,7 @@ "District Name": "District_Name" } }, - "zoning_wide": { + "VersoZoning_wide": { "join_key": "OBJECT_ID", "join_type": "inner", "columns": { @@ -24,7 +24,7 @@ "Planned Unit Development": "PUD_Allowance" } }, - "zoning_rules": { + "VersoZoning_rules": { "join_key": "OBJECT_ID", "join_type": "inner", "value_col": "val", @@ -47,7 +47,7 @@ "Prevalence Measure": "Data_Value_Type" } }, - "soil_suitability_info_soil_suit": { + "VersoWastewater_soilSuitability_info": { "join_key": "ID", "join_type": "inner", "columns": { @@ -56,7 +56,7 @@ "Soil Suitability Level": "Suitability" } }, - "treatment_facilities_treatment_facility_info": { + "VersoWastewater_treatmentFacilities_info": { "join_key": "ID", "join_type": "inner", "columns": { @@ -66,7 +66,7 @@ "Town": "TownName" } }, - "treatment_facilities_treatment_facility_permit_info": { + "VersoWastewater_treatmentFacilitiesPermits_info": { "join_key": "ID", "join_type": "inner", "columns": { @@ -76,7 +76,7 @@ "Town": "TownName" } }, - "service_areas_service_area_info": { + "VersoWastewater_serviceAreas_info": { "join_key": "ID", "join_type": "inner", "columns": { diff --git a/backend/notebooks/zoning/runtime_test.qmd b/backend/notebooks/zoning/runtime_test.qmd index 2371e06..55faedc 100644 --- a/backend/notebooks/zoning/runtime_test.qmd +++ b/backend/notebooks/zoning/runtime_test.qmd @@ -152,15 +152,16 @@ def serve_dataframe(filters: dict | None = None): c.hex_color AS hex_color, c.rgba AS rgba, ST_AsGeoJSON(ST_Simplify(g.geom, 0.0001)) AS geometry - FROM zoning_info i - JOIN zoning_geom g USING (OBJECT_ID) - LEFT JOIN zoning_colors c ON c.district_type = i.District_Type + FROM VersoZoning_info i + JOIN VersoZoning_geom g USING (OBJECT_ID) + LEFT JOIN VersoZoning_colors c ON c.district_type = i.District_Type {where} """, ).df() return df -filter_1 = {"County": ["Chittenden"], "Jurisdiction" : ["Hinesburg"]} + +filter_1 = {"County": ["Chittenden"], "Jurisdiction": ["Hinesburg"]} serve_dataframe(filter_1) # print(build_where_query_from_filters(filter_1, FCOLS)) @@ -190,9 +191,9 @@ def serve_geojson(filters: dict | None = None): ) ) ) AS feature - FROM zoning_info i - JOIN zoning_geom g USING (OBJECT_ID) - LEFT JOIN zoning_colors c ON c.district_type = i.District_Type + FROM VersoZoning_info i + JOIN VersoZoning_geom g USING (OBJECT_ID) + LEFT JOIN VersoZoning_colors c ON c.district_type = i.District_Type {where} ) """).fetchone()[0] diff --git a/backend/query/zoning.py b/backend/query/zoning.py index af3360a..2e85596 100644 --- a/backend/query/zoning.py +++ b/backend/query/zoning.py @@ -6,7 +6,7 @@ **Updated**: 2026-08-20 **Description**: - Functions for serving zoning_info data to the API from the parquet files. + Functions for serving VersoZoning_info data to the API from the database. """ import logging diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index aef98eb..740a0ce 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -29,14 +29,14 @@ # --------------------------------------------------------------------------- WHERE_SOURCE = FilterSource( - filter_table="acs5_b10_census", + filter_table="acs5_demographics_tidy", filters={ "NAME": ["Vergennes", "Addison town"], "year": RangeFilter(min=2015, max=2020), }, ) CTE_SOURCE = FilterSource( - filter_table="zoning_info", + filter_table="VersoZoning_info", filters={"County": ["Addison"]}, join_key="OBJECT_ID", join_type="inner", From ed6509d1b70c573d6b15e25fa9ad48b45d4c12f8 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 20 Aug 2026 16:46:28 -0400 Subject: [PATCH 13/26] updated all mapping api endpoints to work with the frontend --- backend/.sqlfluff | 2 +- .../api/routes/get_routes/get_wholedata.py | 32 ++++- .../api/routes/post_routes/post_acs5_db.py | 113 ++++++++++++++++-- backend/api/routes/post_routes/post_census.py | 96 ++++----------- .../api/routes/post_routes/post_wastewater.py | 14 ++- backend/api/schema.json | 8 +- backend/data_cleaning/clean_wastewater.py | 27 ++++- backend/query/acs5.py | 104 +++++++++++++--- backend/query/processed_db.py | 1 - .../sql/wastewater/service_area_geo_query.sql | 2 +- .../wastewater/soil_suitability_geo_query.sql | 13 +- .../wastewater/waste_treatment_geo_query.sql | 2 +- backend/query/sql/zoning/geo_query.sql | 8 +- backend/query/sql/zoning/unzoned.sql | 7 +- backend/query/wastewater.py | 7 +- backend/run_data_cleaning.py | 22 ++-- backend/run_data_collection.py | 38 +++--- backend/tests/test_sql_render.py | 2 +- .../src/app/mapping/[slug]/page_content.tsx | 26 +++- .../components/Charts/configs/ChartDefs.tsx | 22 +--- .../src/components/FilterRedux/filterDefs.ts | 2 +- frontend/src/components/mapping/index.tsx | 18 ++- 22 files changed, 376 insertions(+), 190 deletions(-) diff --git a/backend/.sqlfluff b/backend/.sqlfluff index 734d261..22b13c1 100644 --- a/backend/.sqlfluff +++ b/backend/.sqlfluff @@ -25,7 +25,7 @@ exclude_rules = CP02, CP03, RF04, RF05, ST06, ST07 case_sensitive = True [sqlfluff:templater:jinja:context] -table = acs5_b10_census +table = acs5_demographics_tidy where_string = WHERE "Measure" IN ($1) cte_filter_block = WITH f0 AS (SELECT DISTINCT OBJECT_ID FROM VersoZoning_info WHERE "County" IN ($1)) join_filter_block = JOIN f0 USING (OBJECT_ID) diff --git a/backend/api/routes/get_routes/get_wholedata.py b/backend/api/routes/get_routes/get_wholedata.py index 18e5cbf..f073bc5 100644 --- a/backend/api/routes/get_routes/get_wholedata.py +++ b/backend/api/routes/get_routes/get_wholedata.py @@ -5,6 +5,7 @@ from app_utils import data_loading from app_utils.flooding import add_flood_color +from query.processed_db import DB logger = logging.getLogger(__name__) @@ -16,14 +17,33 @@ def read_root(): return {"Default Message": "No endpoint specified"} -# Flood Endpoint (Hardcoded for now) +# Flood Endpoint @router.get("/load/mapping/flood_legal") async def read_flood_data(): - data = data_loading.masterload(name="flood_legal") - # Re-apply zone-based colors at serve time so the static JSON - # does not need to be regenerated when the color scheme changes. - data = add_flood_color(data) - return json.loads(data.to_json()) + result = DB.execute("""--sql + SELECT + *, + ST_AsGeoJSON(geometry)::JSON AS geometry_json + FROM FEMA_floodHazard_geom + """).df() + + result = add_flood_color(result) + + features = [] + for _, row in result.iterrows(): + properties = row.drop(["geometry", "geometry_json"]).to_dict() + features.append( + { + "type": "Feature", + "geometry": json.loads(row["geometry_json"]), + "properties": properties, + } + ) + + return { + "type": "FeatureCollection", + "features": features, + } # Soil Septic Endpoint (Hardcoded for now) diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 87aea8a..935a158 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -8,9 +8,9 @@ # TODO: Simplify / Refactor this script using the new query folder functions from query.acs5 import ( get_acs5_tidy, - get_median_earnings, + get_acs5_timeseries, + get_median_earnings_ts, get_snapshot, - get_unemployment_rate_ts, ) from query.processed_db import DB @@ -21,6 +21,10 @@ # TODO: Percents might need to be weighted averages instead of simple averages for statewide aggregation # TODO: In DB, add an aggregated statewide VT row to each table for easier aggregation requests +# ----------------------------- +# CENSUS TIDY FORMAT TABLES +# ----------------------------- + # Demographics @router.post("/load/acs5-db/tidy/demographics") @@ -36,6 +40,7 @@ async def tidy_education(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("education")) +# Housing @router.post("/load/acs5-db/tidy/housing") async def tidy_housing(request: FilterRequest): rows = get_acs5_tidy(dataset="housing", filters=request.filters) @@ -56,24 +61,108 @@ async def tidy_income(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("income")) +# ----------------------------- +# CENSUS TIMESERIES TABLES +# ----------------------------- + + +##### DEMOGRAPHICS ##### +# Age Dependency Ratio +@router.post("/load/acs5-db/timeseries/demographics/age-dependency-ratio") +async def get_age_dependency_ratio(request: FilterRequest): + rows = get_acs5_timeseries( + category="demographics", dataset="age_dependency_ratio", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("demographics")) + + # Median Age -@router.post("/load/acs5-db/tidy/demographics/median-age") -async def tidy_median_age(request: FilterRequest): - rows = get_acs5_tidy(dataset="demographics", filters=request.filters) +@router.post("/load/acs5-db/timeseries/demographics/median-age") +async def get_median_age(request: FilterRequest): + rows = get_acs5_timeseries( + category="demographics", dataset="median_age", filters=request.filters + ) return make_response(data=rows, metadata=get_metadata("demographics")) +# Historic Population +@router.post("/load/acs5-db/timeseries/demographics/historic-population") +async def get_historic_population(request: FilterRequest): + rows = get_acs5_timeseries( + category="demographics", dataset="historic_population", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("demographics")) + + +##### ECONOMICS ##### +# Heath Insurance Coverage +@router.post("/load/acs5-db/timeseries/economics/health-insurance") +async def get_health_insurance(request: FilterRequest): + rows = get_acs5_timeseries( + category="economics", dataset="health_insurance", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("labor_force")) + + +# Median Household Income +@router.post("/load/acs5-db/timeseries/economics/median-hh-income") +async def get_household_income(request: FilterRequest): + rows = get_acs5_timeseries( + category="economics", dataset="household_income", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("income")) + + +# Per Capita Income +@router.post("/load/acs5-db/timeseries/economics/per-capita-income") +async def get_per_capita_income(request: FilterRequest): + rows = get_acs5_timeseries( + category="economics", dataset="per_capita_income", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("income")) + + # Unemployment Rate -@router.post("/load/acs5-db/tidy/unemployment-rate") -async def tidy_unemployment_rate(request: FilterRequest): - rows = get_unemployment_rate_ts(filters=request.filters) +@router.post("/load/acs5-db/timeseries/economics/unemployment-rate") +async def get_unemployment_rate(request: FilterRequest): + rows = get_acs5_timeseries( + category="economics", dataset="unemployment_rate", filters=request.filters + ) return make_response(data=rows, metadata=get_metadata("unemployment_rate")) -# Median Earnings (FIXME) -@router.post("/load/acs5-db/tidy/median-earnings") -async def tidy_median_earnings(request: FilterRequest): - rows = get_median_earnings(filters=request.filters) +##### HOUSING ##### +# Total Housing Units +@router.post("/load/acs5-db/timeseries/housing/total-units") +async def get_housing_units(request: FilterRequest): + rows = get_acs5_timeseries( + category="housing", dataset="housing_units", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("housing")) + + +# Median Home Value +@router.post("/load/acs5-db/timeseries/housing/median-home-value") +async def get_median_home_value(request: FilterRequest): + rows = get_acs5_timeseries( + category="housing", dataset="median_home_value", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("housing")) + + +# Vacancy Rates +@router.post("/load/acs5-db/timeseries/housing/vacancy-rates") +async def get_vacancy_rates(request: FilterRequest): + rows = get_acs5_timeseries( + category="housing", dataset="vacancy_rates", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("housing")) + + +# Median Earnings (FIXME: broken) +@router.post("/load/acs5-db/timeseries/median-earnings") +async def get_median_earnings(request: FilterRequest): + rows = get_median_earnings_ts(filters=request.filters) return make_response(data=rows, metadata=get_metadata("median_earnings")) diff --git a/backend/api/routes/post_routes/post_census.py b/backend/api/routes/post_routes/post_census.py index 8df4109..9bbaf3b 100644 --- a/backend/api/routes/post_routes/post_census.py +++ b/backend/api/routes/post_routes/post_census.py @@ -3,12 +3,11 @@ from fastapi import APIRouter, HTTPException from api.models import FilterRequest, make_response -from app_utils import data_loading, timeseries_db +from app_utils import timeseries_db from app_utils.df_filtering import ( filter_from_request, - mass_filter_from_requests, ) -from app_utils.housing import housing_df_metric_dict +from query.processed_db import DB router = APIRouter() @@ -16,81 +15,30 @@ DATADIR = Path(__file__).parent.parent.parent.parent / "Data" CENSUS_DATADIR = DATADIR / "Census" + +CENSUS_DATASETS = ["demographics", "economics", "housing", "social"] + # Maps (category, subcategory) to the timeseries_db view name. # These subcategories are served via DuckDB instead of pandas/CSV. _TIMESERIES_VIEWS: dict[tuple[str, str], str] = { - ("housing", "median_home_value"): "median_home_value", - ("housing", "median_smoc"): "median_smoc", - ("economic", "median_earnings"): "median_earnings", - ("economic", "unemployment_rate"): "unemployment_rate", - ("economic", "commute_habits"): "commute_habits", - ("economic", "commute_time"): "commute_time", - ("demographic", "historic_population"): "historic_population", -} - -CENSUS_DATASETS = { - "housing": { - "main": CENSUS_DATADIR / "VT_HOUSING_ALL.fgb", - # time-series subcategories handled via _TIMESERIES_VIEWS / DuckDB - }, - "economic": { - "main": CENSUS_DATADIR / "VT_ECONOMIC_ALL.fgb", - # time-series subcategories handled via _TIMESERIES_VIEWS / DuckDB - }, - "demographic": { - "main": CENSUS_DATADIR / "VT_DEMOGRAPHIC_ALL.fgb", - # time-series subcategories handled via _TIMESERIES_VIEWS / DuckDB - }, - "social": {"main": CENSUS_DATADIR / "VT_SOCIAL_ALL.fgb"}, + ("demographics", "historic_population"): "VCGI_historicPopulation_timeseries", + ( + "demographics", + "age_dependency_ratio", + ): "acs5Demographics_ageDependencyRatio_timeseries", + ("demographics", "median_age"): "acs5Demographics_medianAge_timeseries", + ("economics", "health_insurance"): "acs5Economics_healthInsurance_timeseries", + ("economics", "household_income"): "acs5Economics_medianHouseholdIncome_timeseries", + ("economics", "per_capita_income"): "acs5Economics_perCapitaIncome_timeseries", + ("economics", "unemployment_rate"): "acs5Economics_unemploymentRate_timeseries", + ("housing", "housing_units"): "acs5Housing_housingUnits_timeseries", + ("housing", "median_home_value"): "acs5Housing_medianHomeValue_timeseries", + ("housing", "vacancy_rates"): "acs5Demographics_vacancyRates_timeseries", } -# Load the Census "Main" Dataset by Cateogory (housing, economic, demographic, social) -@router.post("/load/census/{category}") -async def read_census_data(category: str, request: FilterRequest): - if category not in CENSUS_DATASETS: - raise HTTPException( - status_code=404, detail=f"Census category '{category}' was not found" - ) - - data = data_loading.load_census_data(CENSUS_DATASETS[category]["main"]) - data = filter_from_request(data, request) - metadata = {} - - return make_response(data, metadata) - - -@router.post("/load/census/housing/snapshot") -async def get_housing_snapshot(request: FilterRequest): - dfs = data_loading.masterload("census_housing") - dfs = mass_filter_from_requests(dfs, request) - metrics, plot_dfs = housing_df_metric_dict(dfs) - - # Convert metrics to JSON-serializable - metrics_json = {k: float(v) if v is not None else None for k, v in metrics.items()} - - # Convert plot dataframes - plot_data = {k: v.to_dict(orient="records") for k, v in plot_dfs.items()} - - response = {"metrics": metrics_json, "plot_data": plot_data} - - # Filter response if specific includes requested - if request and request.include: - filtered_response = {} - if "metrics" in request.include: - filtered_response["metrics"] = metrics_json - - # Filter plot_data to only included charts - plot_includes = [i for i in request.include if i in plot_dfs] - if plot_includes: - filtered_response["plot_data"] = {k: plot_data[k] for k in plot_includes} - - return filtered_response - - return response - - -# Load the Census Dataset by `category`(housing, economic, etc.) and `subcategory`(special csv files) +# Load the Census Dataset by `category`(housing, economic, etc.) +# and `subcategory`(special time series tables) @router.post("/load/census/{category}/{subcategory}") async def read_census_data_subcat( category: str, request: FilterRequest, subcategory: str = "main" @@ -105,6 +53,7 @@ async def read_census_data_subcat( if ts_key in _TIMESERIES_VIEWS: view_name = _TIMESERIES_VIEWS[ts_key] filters = request.filters if request else None + data = timeseries_db.query_timeseries(view_name, filters) if data.empty: raise HTTPException( @@ -120,7 +69,8 @@ async def read_census_data_subcat( detail=f"Census subcategory '{subcategory}' was not found in category '{category}'", ) - data = data_loading.load_census_data(CENSUS_DATASETS[category][subcategory]) + # data = data_loading.load_census_data(CENSUS_DATASETS[category][subcategory]) + data = DB.execute(f"SELECT * FROM acs5_{category}_tidy") data = filter_from_request(data, request) metadata = {} diff --git a/backend/api/routes/post_routes/post_wastewater.py b/backend/api/routes/post_routes/post_wastewater.py index 61aeacd..b0679f4 100644 --- a/backend/api/routes/post_routes/post_wastewater.py +++ b/backend/api/routes/post_routes/post_wastewater.py @@ -4,10 +4,10 @@ from api.metadata_registry import get_metadata from api.models import FilterRequest, make_response from query import ( + get_soil_suit_geojson, get_waste_service_areas_geojson, get_waste_treatment_facility_geojson, get_waste_treatment_facility_permits, - get_soil_suit_geojson, ) router = APIRouter() @@ -15,7 +15,7 @@ @router.post("/load/mapping/wastewater/service_area") async def wastewater_service_geojson(request: FilterRequest): - source = request_to_source(request, "service_areas_service_area_info", "default") + source = request_to_source(request, "VersoWastewater_serviceAreas_info", "default") data = get_waste_service_areas_geojson([source]) return Response(content=data, media_type="application/json") @@ -23,17 +23,17 @@ async def wastewater_service_geojson(request: FilterRequest): @router.post("/load/mapping/wastewater/treatment_facility") async def wastewater_facility_geojson(request: FilterRequest): source = request_to_source( - request, "treatment_facilities_treatment_facility_info", "default" + request, "VersoWastewater_treatmentFacilities_info", "default" ) data = get_waste_treatment_facility_geojson([source]) return Response(content=data, media_type="application/json") -@router.post("/load/wastewater/zoning/facility_permits") +@router.post("/load/mapping/wastewater/treatment_facility/permits") async def wastewater_facility_permits(request: FilterRequest): # TODO: the json table might be wrong, check later source = request_to_source( - request, "treatment_facilities_treatment_facility_permit_info", "default" + request, "VersoWastewater_treatmentFacilitiesPermits_info", "default" ) table = get_waste_treatment_facility_permits([source]) return make_response(data=table, metadata=get_metadata("zoning")) @@ -41,6 +41,8 @@ async def wastewater_facility_permits(request: FilterRequest): @router.post("/load/mapping/wastewater/septic_soil_suitability") async def wastewater_soil_suit_geojson(request: FilterRequest): - source = request_to_source(request, "soil_suitability_info_soil_suit", "default") + source = request_to_source( + request, "VersoWastewater_soilSuitability_info", "default" + ) data = get_soil_suit_geojson([source]) return Response(content=data, media_type="application/json") diff --git a/backend/api/schema.json b/backend/api/schema.json index 3a49b87..49891cd 100644 --- a/backend/api/schema.json +++ b/backend/api/schema.json @@ -48,7 +48,7 @@ } }, "VersoWastewater_soilSuitability_info": { - "join_key": "ID", + "join_key": "OGC_FID", "join_type": "inner", "columns": { "Regional Planning Commission": "RPC", @@ -57,7 +57,7 @@ } }, "VersoWastewater_treatmentFacilities_info": { - "join_key": "ID", + "join_key": "Facility_ID", "join_type": "inner", "columns": { "Regional Planning Commission": "RPC", @@ -67,7 +67,7 @@ } }, "VersoWastewater_treatmentFacilitiesPermits_info": { - "join_key": "ID", + "join_key": "Facility_ID", "join_type": "inner", "columns": { "Regional Planning Commission": "RPC", @@ -77,7 +77,7 @@ } }, "VersoWastewater_serviceAreas_info": { - "join_key": "ID", + "join_key": "Area_ID", "join_type": "inner", "columns": { "Regional Planning Commission": "RPC", diff --git a/backend/data_cleaning/clean_wastewater.py b/backend/data_cleaning/clean_wastewater.py index ad39e94..b238378 100644 --- a/backend/data_cleaning/clean_wastewater.py +++ b/backend/data_cleaning/clean_wastewater.py @@ -23,10 +23,10 @@ def _load_spatial() -> None: Load the spatial extension, installing it first if necessary. """ try: - con.execute("""--sql LOAD spatial""") + con.execute("""LOAD spatial""") except Exception: - con.execute("""--sql INSTALL spatial""") - con.execute("""--sql LOAD spatial""") + con.execute("""INSTALL spatial""") + con.execute("""LOAD spatial""") ## ADD UNIQUE ID COLUMNS -------------------- @@ -241,6 +241,25 @@ def build_suitability_geom() -> None: ) +def build_suitability_colors() -> None: + con.execute( + """--sql + CREATE OR REPLACE TABLE soilSuitability_colors ( + soil_suitability TEXT PRIMARY KEY, + hex_color TEXT NOT NULL, + rgba TEXT NOT NULL + ); + + INSERT INTO soilSuitability_colors VALUES + ('Well Suited', '#2ca02c', '[44, 160, 44, 180]'), + ('Moderately Suited', '#ffcc00', '[255, 204, 0, 180]'), + ('Marginally Suited', '#fd7e14', '[253, 126, 20, 180]'), + ('Not Suited', '#dc3545', '[220, 53, 69, 180]'), + ('Not Rated', '#6c757d', '[108, 117, 125, 180]'); + """ + ) + + ## STORMWATER MANAGEMENT TABLES -------------------- def build_stormwater_info() -> None: # "Type" labels derived from VERSO WIM GitHub pages @@ -314,6 +333,7 @@ def clean(): # Soil suitability Tables build_suitability_info() build_suitability_geom() + build_suitability_colors() # Stormwater Management Tables build_stormwater_info() @@ -332,6 +352,7 @@ def add_to_lake(): "soilSuitability_info", # NOTE: The `soilSuitabilitygeom` dataset below is too large for git storage. Add to .gitignore "soilSuitability_geom", + "soilSuitability_colors", "stormwaterManagement_info", "stormwaterManagement_geom", ] diff --git a/backend/query/acs5.py b/backend/query/acs5.py index afcbb06..7b7f8f4 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -23,23 +23,64 @@ # Per-dataset FIXED filters, expressed as {column: [values]} and folded into the # FilterSource (these replace the old raw-SQL base_conditions) QUERY_CONFIG = { - "demographics": {"table": "acs5_demographics_tidy", "fixed_filters": {}}, - "education": {"table": "acs5_education_tidy", "fixed_filters": {}}, - "housing": {"table": "acs5_housing_tidy", "fixed_filters": {}}, - "labor_force": { - "table": "acs5_economics_tidy", - "fixed_filters": {"Section": ["Labor Force"]}, + "demographics": { + "table": "acs5_demographics_tidy", + "fixed_filters": {}, + "timeseries": { + "age_dependency_ratio": { + "table": "acs5Demographics_ageDependencyRatio_timeseries", + "fixed_filters": {}, + }, + "median_age": { + "table": "acs5Demographics_medianAge_timeseries", + "fixed_filters": {}, + }, + "historic_population": { + "table": "VCGI_historicPopulation_timeseries", + "fixed_filters": {}, + }, + }, }, - "income": { + "economics": { "table": "acs5_economics_tidy", - "fixed_filters": {"Section": ["Income"]}, + "fixed_filters": {}, + "timeseries": { + "health_insurance": { + "table": "acs5Economics_healthInsurance_timeseries", + "fixed_filters": {}, + }, + "household_income": { + "table": "acs5Economics_medianHouseholdIncome_timeseries", + "fixed_filters": {}, + }, + "per_capita_income": { + "table": "acs5Economics_perCapitaIncome_timeseries", + "fixed_filters": {}, + }, + "unemployment_rate": { + "table": "acs5Economics_unemploymentRate_timeseries", + "fixed_filters": {}, + }, + }, }, - "median_age": { - "table": "acs5Demographics_medianAge_timeseries", - "fixed_filters": {"Variable": ["Median Age"]}, + "housing": { + "table": "acs5_housing_tidy", + "fixed_filters": {}, + "timeseries": { + "housing_units": { + "table": "acs5Housing_housingUnits_timeseries", + "fixed_filters": {}, + }, + "median_home_value": { + "table": "acs5Housing_medianHomeValue_timeseries", + "fixed_filters": {}, + }, + "vacancy_rates": { + "table": "acs5Housing_vacancyRates_timeseries", + "fixed_filters": {}, + }, + }, }, - # TODO: Create a "snapshot" table to database - "snapshot": {"table": "acs5_snapshot", "fixed_filters": {}}, } # frontend filter label -> database column. Location and the year range both @@ -85,7 +126,7 @@ def get_acs5_tidy(dataset: str, filters: dict | None = None) -> pd.DataFrame: result = DB.execute(sql, params).df() - if result is None: + if result.empty: logger.error( "ACS5 tidy query returned no rows for dataset: %s, filters: %s", dataset, @@ -96,6 +137,39 @@ def get_acs5_tidy(dataset: str, filters: dict | None = None) -> pd.DataFrame: return result +def get_acs5_timeseries( + category: str, + dataset: str, + filters: dict | None = None, +) -> pd.DataFrame: + category_config = QUERY_CONFIG[category] + config = category_config["timeseries"][dataset] + + source = _acs5_source( + table=config["table"], + filters=filters, + fixed_filters=config.get("fixed_filters", {}), + ) + + sql, params = sql_filter_block(sql_path / "acs5_tidy.sql", [source]) + + result = DB.execute(sql, params).df() + + if result is None: + logger.error( + "ACS5 timeseries query returned no rows for category=%s, " + "dataset=%s, filters=%s", + category, + dataset, + filters, + ) + raise ValueError( + f" No results for timeseries: {category}/{dataset}, filters: {filters}" + ) + + return result + + def get_unemployment_rate_ts(filters: dict | None = None) -> pd.DataFrame: source = _acs5_source( table="acs5Economics_unemploymentRate_timeseries", filters=filters @@ -113,7 +187,7 @@ def get_unemployment_rate_ts(filters: dict | None = None) -> pd.DataFrame: # FIXME: Link to new database table name (broken for now) -def get_median_earnings(filters: dict | None = None) -> pd.DataFrame: +def get_median_earnings_ts(filters: dict | None = None) -> pd.DataFrame: source = _acs5_source(table="acs5_median_earnings", filters=filters) sql, params = sql_filter_block(sql_path / "median_earnings.sql", [source]) diff --git a/backend/query/processed_db.py b/backend/query/processed_db.py index bbd373b..8670326 100644 --- a/backend/query/processed_db.py +++ b/backend/query/processed_db.py @@ -55,7 +55,6 @@ def _build() -> duckdb.DuckDBPyConnection: def _build_etl_db() -> duckdb.DuckDBPyConnection: - print(f"DATA DIRECTORY PATH: {DATA_DIR}") path = Path(DATA_DIR / "warehouse.duckdb") con = duckdb.connect(path, read_only=True) _load_spatial(con) diff --git a/backend/query/sql/wastewater/service_area_geo_query.sql b/backend/query/sql/wastewater/service_area_geo_query.sql index 5a783b9..aca3f2e 100644 --- a/backend/query/sql/wastewater/service_area_geo_query.sql +++ b/backend/query/sql/wastewater/service_area_geo_query.sql @@ -21,6 +21,6 @@ FROM ( ) ) AS feature FROM VersoWastewater_serviceAreas_info AS i - INNER JOIN VersoWastewater_serviceAreas_geom AS g USING (ID) + INNER JOIN VersoWastewater_serviceAreas_geom AS g USING (Area_ID) {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/wastewater/soil_suitability_geo_query.sql b/backend/query/sql/wastewater/soil_suitability_geo_query.sql index 3e6234f..96a937f 100644 --- a/backend/query/sql/wastewater/soil_suitability_geo_query.sql +++ b/backend/query/sql/wastewater/soil_suitability_geo_query.sql @@ -1,4 +1,3 @@ -{{ cte_filter_block }} SELECT json_object( 'type', 'FeatureCollection', @@ -8,7 +7,10 @@ FROM ( SELECT json_object( 'type', 'Feature', - 'geometry', ST_AsGeoJSON(ST_Simplify(g.geom, 0.0001))::JSON, + 'geometry', + ST_AsGeoJSON( + ST_Simplify(g.geometry, 0.0001) + )::JSON, 'properties', json_object( 'Suitability', i.Suitability, 'Acres', ROUND(i.Acres, 2), @@ -22,8 +24,9 @@ FROM ( ) ) AS feature FROM VersoWastewater_soilSuitability_info AS i - INNER JOIN VersoWastewater_soilSuitability_geom AS g USING (ID) + INNER JOIN VersoWastewater_soilSuitability_geom AS g + ON i.OGC_FID = g.OGC_FID LEFT JOIN VersoWastewater_soilSuitability_colors AS c ON i.Suitability = c.soil_suitability - {{ join_filter_block }} -) AS features + {{ where_string }} +) AS features; \ No newline at end of file diff --git a/backend/query/sql/wastewater/waste_treatment_geo_query.sql b/backend/query/sql/wastewater/waste_treatment_geo_query.sql index 8c29a8e..9ad0783 100644 --- a/backend/query/sql/wastewater/waste_treatment_geo_query.sql +++ b/backend/query/sql/wastewater/waste_treatment_geo_query.sql @@ -24,6 +24,6 @@ FROM ( ) ) AS feature FROM VersoWastewater_treatmentFacilities_info AS i - INNER JOIN VersoWastewater_treatmentFacilities_geom AS g USING (ID) + INNER JOIN VersoWastewater_treatmentFacilities_geom AS g USING (Facility_ID) {{ join_filter_block }} ) AS features diff --git a/backend/query/sql/zoning/geo_query.sql b/backend/query/sql/zoning/geo_query.sql index 96bb617..fa9df79 100644 --- a/backend/query/sql/zoning/geo_query.sql +++ b/backend/query/sql/zoning/geo_query.sql @@ -8,7 +8,7 @@ filtered AS ( i.Municipal_Name, i.District_Name, c.rgba, - g.geom + g.geometry FROM VersoZoning_info AS i INNER JOIN VersoZoning_geom AS g USING (OBJECT_ID) LEFT JOIN VersoZoning_colors AS c ON i.District_Type = c.district_type @@ -19,7 +19,7 @@ features AS ( SELECT JSON_OBJECT( 'type', 'Feature', - 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(geom, 0.0001))::JSON, + 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(geometry, 0.0001))::JSON, 'properties', JSON_OBJECT( 'District Type', District_Type, 'Acres', Acres, @@ -40,7 +40,7 @@ features AS ( matched_area AS ( SELECT County, - ST_Area_Spheroid(ST_Union_Agg(geom)) / 4046.8564224 AS matched_acres + ST_Area_Spheroid(ST_Union_Agg(geometry)) / 4046.8564224 AS matched_acres FROM filtered GROUP BY County ), @@ -48,7 +48,7 @@ matched_area AS ( county_area AS ( SELECT i.County, - ST_Area_Spheroid(ST_Union_Agg(g.geom)) / 4046.8564224 AS total_acres + ST_Area_Spheroid(ST_Union_Agg(g.geometry)) / 4046.8564224 AS total_acres FROM VersoZoning_info AS i INNER JOIN VersoZoning_geom AS g USING (OBJECT_ID) GROUP BY i.County diff --git a/backend/query/sql/zoning/unzoned.sql b/backend/query/sql/zoning/unzoned.sql index 0ff88f3..ec917b5 100644 --- a/backend/query/sql/zoning/unzoned.sql +++ b/backend/query/sql/zoning/unzoned.sql @@ -13,7 +13,12 @@ FROM ( SELECT JSON_OBJECT( 'type', 'Feature', - 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(geom, 0.0001))::JSON, + 'geometry', ST_ASGEOJSON( + ST_SIMPLIFY( + ST_GEOMFROMWKB(geom), + 0.0001::DOUBLE + ) + )::JSON, 'properties', JSON_OBJECT( 'rgba_color', JSON_ARRAY(170, 170, 170, 160), 'tooltip', JSON_OBJECT( diff --git a/backend/query/wastewater.py b/backend/query/wastewater.py index 7b1cd0e..ade57a7 100644 --- a/backend/query/wastewater.py +++ b/backend/query/wastewater.py @@ -32,6 +32,7 @@ def get_waste_service_areas_geojson(sources: list[FilterSource]): def get_waste_treatment_facility_geojson(sources: list[FilterSource]): sql, params = sql_filter_block(sql_dir / "waste_treatment_geo_query.sql", sources) result = DB.execute(sql, params).fetchone() + print(f"TREATMENT FACILITY GEOM RESULT: {result}") if result is None: logger.error("geo query returned no rows for filters: %s", sources) raise ValueError(f"no results for filters: {sources}") @@ -46,7 +47,11 @@ def get_waste_treatment_facility_permits(sources: list[FilterSource]) -> pd.Data def get_soil_suit_geojson(sources: list[FilterSource]): - sql, params = sql_filter_block(sql_dir / "soil_suitability_geo_query.sql", sources) + sql, params = sql_filter_block( + sql_dir / "soil_suitability_geo_query.sql", + sources, + ) + result = DB.execute(sql, params).fetchone() if result is None: logger.error("geo query returned no rows for filters: %s", sources) diff --git a/backend/run_data_cleaning.py b/backend/run_data_cleaning.py index a7f8105..994abb4 100644 --- a/backend/run_data_cleaning.py +++ b/backend/run_data_cleaning.py @@ -14,10 +14,6 @@ def get_cleaners(): if ispkg: continue - # Skip private/helper modules - if module_name.startswith("_"): - continue - module = import_module(f"data_cleaning.{module_name}") # Only include modules that expose a main() function @@ -27,6 +23,15 @@ def get_cleaners(): return cleaners +def get_cleaner(module_name: str): + """ + Return all cleaning scripts in the backend/data_cleaning folder + """ + + module = import_module(f"data_cleaning.{module_name}") + return module + + # def create_duckdb_version(): # tables = con.execute( # """--sql @@ -40,10 +45,11 @@ def get_cleaners(): def run_master_clean(): - for cleaner in get_cleaners(): - print(f"Running {cleaner.__name__.split('.')[-1]}...") - cleaner.main() - print(f"Completed {cleaner.__name__.split('.')[-1]}") + # for cleaner in get_cleaners(): + cleaner = get_cleaner("clean_wastewater") + print(f"Running {cleaner.__name__.split('.')[-1]}...") + cleaner.main() + print(f"Completed {cleaner.__name__.split('.')[-1]}") def main(): diff --git a/backend/run_data_collection.py b/backend/run_data_collection.py index f5f03df..801bf8a 100644 --- a/backend/run_data_collection.py +++ b/backend/run_data_collection.py @@ -14,32 +14,32 @@ import argparse from data_collection import ( - acs5, - cdc, - demographics, - economic, - education, - fips, - flood, - historic_population, - housing, - qcew, + # acs5, + # cdc, + # demographics, + # economic, + # education, + # fips, + # flood, + # historic_population, + # housing, + # qcew, wastewater, - zoning, + # zoning, ) from lake_build import insert_year, replace_table # Datasets WITH year columns (longitudinal) -YEARLY_SCRAPERS = [acs5, demographics, economic, education, housing, qcew] +# YEARLY_SCRAPERS = [acs5, demographics, economic, education, housing, qcew] # Datasets WITHOUT year columns (static) STATIC_SCRAPERS = [ - cdc, - fips, - flood, - historic_population, + # cdc, + # fips, + # flood, + # historic_population, wastewater, - zoning, + # zoning, ] YEARS = range(2009, 2025) @@ -77,8 +77,8 @@ def run_scraper(scraper, yearly: bool = False, years: range = YEARS): def run_master_scrape(start_year: int = 2009, end_year: int = 2024): - for scraper in YEARLY_SCRAPERS: - run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) + # for scraper in YEARLY_SCRAPERS: + # run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) for scraper in STATIC_SCRAPERS: run_scraper(scraper, yearly=False) diff --git a/backend/tests/test_sql_render.py b/backend/tests/test_sql_render.py index 740a0ce..ce19a3f 100644 --- a/backend/tests/test_sql_render.py +++ b/backend/tests/test_sql_render.py @@ -207,7 +207,7 @@ def test_where_string_injected(self): sql, params = sql_filter_block( BACKEND / "query/sql/acs5/acs5_tidy.sql", [WHERE_SOURCE] ) - assert "FROM acs5_b10_census" in sql + assert "FROM acs5_demographics_tidy" in sql assert 'WHERE "NAME" IN ($1, $2)' in sql assert params == ["Vergennes", "Addison town", 2015.0, 2020.0] assert "{{" not in sql and "{%" not in sql diff --git a/frontend/src/app/mapping/[slug]/page_content.tsx b/frontend/src/app/mapping/[slug]/page_content.tsx index 4f08284..dd4a584 100644 --- a/frontend/src/app/mapping/[slug]/page_content.tsx +++ b/frontend/src/app/mapping/[slug]/page_content.tsx @@ -33,28 +33,38 @@ const SOIL_RPCS = [ // takes precedence over this dynamic segment. const MAP_CONFIG: Record< string, - { title: string; initialURL?: string; filterURL?: string; dataURL?: string } + { + title: string; + initialURL?: string; + filterURL?: string; + dataURL?: string; + initialMethod?: 'GET' | 'POST'; + } > = { 'flood-legal': { title: 'Flood Insurance', initialURL: `${BASE_API_URL}/load/mapping/flood_legal`, + initialMethod: 'GET', }, 'soil-suitability': { title: 'Soil Suitability', initialURL: `${BASE_API_URL}/load/mapping/wastewater/septic_soil_suitability`, - filterURL: `${BASE_API_URL}/filters/tree?filter_table=soil_suitability_info_soil_suit`, + initialMethod: 'POST', + filterURL: `${BASE_API_URL}/filters/tree?filter_table=VersoWastewater_soilSuitability_info`, dataURL: `${BASE_API_URL}/load/mapping/wastewater/septic_soil_suitability`, }, 'treatment-facilities': { title: 'Wastewater Treatment Facilities', initialURL: `${BASE_API_URL}/load/mapping/wastewater/treatment_facility`, - filterURL: `${BASE_API_URL}/filters/tree?filter_table=treatment_facilities_treatment_facility_info`, + initialMethod: 'POST', + filterURL: `${BASE_API_URL}/filters/tree?filter_table=VersoWastewater_treatmentFacilities_info`, dataURL: `${BASE_API_URL}/load/mapping/wastewater/treatment_facility`, }, 'service-areas': { title: 'Wastewater Service Areas', initialURL: `${BASE_API_URL}/load/mapping/wastewater/service_area`, - filterURL: `${BASE_API_URL}/filters/tree?filter_table=service_areas_service_area_info`, + initialMethod: 'POST', + filterURL: `${BASE_API_URL}/filters/tree?filter_table=VersoWastewater_serviceAreas_info`, dataURL: `${BASE_API_URL}/load/mapping/wastewater/service_area`, }, }; @@ -83,7 +93,11 @@ export default function MappingContent() { setLoading(true); axios - .get(config.initialURL) + .request({ + url: config.initialURL, + method: config.initialMethod ?? 'GET', + data: {}, + }) .then((res) => setData(res.data)) .catch(console.error) .finally(() => setLoading(false)); @@ -103,6 +117,8 @@ export default function MappingContent() { .finally(() => setLoading(false)); }, [rpc]); + console.log('MAP DATA:', data); + return ( diff --git a/frontend/src/components/Charts/configs/ChartDefs.tsx b/frontend/src/components/Charts/configs/ChartDefs.tsx index 194df26..b286cb5 100644 --- a/frontend/src/components/Charts/configs/ChartDefs.tsx +++ b/frontend/src/components/Charts/configs/ChartDefs.tsx @@ -71,18 +71,6 @@ export const chartDefs: ChartDef[] = [ { key: 'Acres', label: 'Total Acres' }, ], }, - // tenure bar chart removed — not adding enough information (4.3) - // { - // id: 'tenure', - // title: 'Housing Tenure', - // categories: ['Housing'], - // xField: 'Occupied Tenure', - // yField: 'Value', - // subtype: 'CompareDiffPerXBarChart', - // chartParams: { colorScheme: 'schemeAccent', legendLabels: ['Main', 'Compare'] }, - // dataKey: 'plot_data.tenure_df', - // url: `${BASE_API_URL}/load/census/housing/snapshot`, - // }, { id: 'demographics', title: 'Changes in Age Composition', @@ -127,7 +115,7 @@ export const chartDefs: ChartDef[] = [ { id: 'demographics_population', title: 'Historic Population Estimates', - url: `${BASE_API_URL}/load/census/demographic/historic_population`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/demographics/historic-population`, xField: '', yField: '', subtype: 'renderTableEstimates', // signals to the renderer to use TableStack not ChartStack @@ -154,7 +142,7 @@ export const chartDefs: ChartDef[] = [ { id: 'median_age', title: 'Median Age', - url: `${BASE_API_URL}/load/acs5-db/tidy/demographics/median-age`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/demographics/median-age`, xField: '', yField: '', subtype: 'renderTableEstimates', @@ -278,7 +266,7 @@ export const chartDefs: ChartDef[] = [ { id: 'unemployment_rate', title: 'Unemployment Rate', - url: `${BASE_API_URL}/load/acs5-db/tidy/unemployment-rate`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/economics/unemployment-rate`, xField: '', yField: '', subtype: 'renderTable', @@ -321,7 +309,7 @@ export const chartDefs: ChartDef[] = [ { id: 'median_hh_income', title: 'Median Household Income', - url: `${BASE_API_URL}/load/acs5-db/tidy/income`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/economics/median-hh-income`, xField: '', yField: '', categories: ['Labor & Economy'], @@ -337,7 +325,7 @@ export const chartDefs: ChartDef[] = [ { id: 'per_capita_income', title: 'Per Capita Income', - url: `${BASE_API_URL}/load/acs5-db/tidy/income`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/economics/per-capita-income`, xField: '', yField: '', categories: ['Labor & Economy'], diff --git a/frontend/src/components/FilterRedux/filterDefs.ts b/frontend/src/components/FilterRedux/filterDefs.ts index 877f5e9..c44c38a 100644 --- a/frontend/src/components/FilterRedux/filterDefs.ts +++ b/frontend/src/components/FilterRedux/filterDefs.ts @@ -15,7 +15,7 @@ export const cdc_filtering: filterDef[] = [ export const zoning_filtering: filterDef[] = [ { - filter_table: 'zoning_wide', + filter_table: 'VersoZoning_wide', filter_style: 'Checkbox', label: '', }, diff --git a/frontend/src/components/mapping/index.tsx b/frontend/src/components/mapping/index.tsx index 4b1c644..0eeb4e2 100644 --- a/frontend/src/components/mapping/index.tsx +++ b/frontend/src/components/mapping/index.tsx @@ -137,13 +137,21 @@ export default function VTMap({ new GeoJsonLayer({ id: 'geojson', data: geojson, - filled: true, - getFillColor, - getLineColor: [80, 80, 80, 80], - lineWidthMinPixels: 0.5, + + // Points + pointType: 'circle', + getPointRadius: 7, + pointRadiusUnits: 'pixels', + pointRadiusMinPixels: 8, + pointRadiusMaxPixels: 12, + + getFillColor: [30, 100, 220, 255], + getLineColor: [20, 70, 160, 255], + lineWidthMinPixels: 1, + pickable: true, autoHighlight: true, - highlightColor: [222, 102, 0, 200], + highlightColor: [255, 255, 255, 255], onHover, }), showCountyLines && From 74bd9202419167a0298782f5fa885e72d7d88432 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Fri, 21 Aug 2026 17:51:36 -0400 Subject: [PATCH 14/26] cdc, acs, qcew, fips, fema route changes --- .../api/routes/post_routes/post_acs5_db.py | 18 ++- backend/api/routes/post_routes/post_qcew.py | 4 +- backend/data_cleaning/clean_cdc.py | 89 ++++++++--- backend/data_cleaning/clean_fips.py | 146 ++++++++++++++++++ backend/data_cleaning/clean_flood.py | 2 +- .../clean_historic_population.py | 14 +- backend/data_cleaning/clean_snapshot.py | 89 +++++++++++ backend/data_collection/acs5.py | 101 ++++++------ backend/query/acs5.py | 68 ++++---- backend/query/cdc.py | 42 ++--- backend/query/sql/acs5/acs5_timeseries.sql | 4 + backend/query/sql/acs5/median_earnings.sql | 9 -- backend/query/sql/acs5/snapshot.sql | 8 - backend/query/sql/acs5/unemployment_rate.sql | 8 - backend/query/sql/cdc/county_places.sql | 8 +- backend/query/sql/cdc/tract_places.sql | 8 +- backend/run_data_cleaning.py | 30 +--- backend/run_data_collection.py | 38 ++--- .../src/components/Charts/TrendCharts.tsx | 12 +- frontend/src/components/mapping/index.tsx | 11 +- 20 files changed, 469 insertions(+), 240 deletions(-) create mode 100644 backend/data_cleaning/clean_fips.py create mode 100644 backend/data_cleaning/clean_snapshot.py create mode 100644 backend/query/sql/acs5/acs5_timeseries.sql delete mode 100644 backend/query/sql/acs5/median_earnings.sql delete mode 100644 backend/query/sql/acs5/snapshot.sql delete mode 100644 backend/query/sql/acs5/unemployment_rate.sql diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 935a158..8b04b90 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -10,7 +10,6 @@ get_acs5_tidy, get_acs5_timeseries, get_median_earnings_ts, - get_snapshot, ) from query.processed_db import DB @@ -47,14 +46,14 @@ async def tidy_housing(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("housing")) -# Labor Force +# Labor Force (FIXME: broken) @router.post("/load/acs5-db/tidy/labor-force") async def tidy_labor_force(request: FilterRequest): rows = get_acs5_tidy(dataset="labor_force", filters=request.filters) return make_response(data=rows, metadata=get_metadata("labor_force")) -# Income +# Income (FIXME: broken) @router.post("/load/acs5-db/tidy/income") async def tidy_income(request: FilterRequest): rows = get_acs5_tidy(dataset="income", filters=request.filters) @@ -88,9 +87,14 @@ async def get_median_age(request: FilterRequest): # Historic Population @router.post("/load/acs5-db/timeseries/demographics/historic-population") async def get_historic_population(request: FilterRequest): + filters = {key: value for key, value in request.filters.items() if key != "year"} + rows = get_acs5_timeseries( - category="demographics", dataset="historic_population", filters=request.filters + category="demographics", + dataset="historic_population", + filters=filters, ) + return make_response(data=rows, metadata=get_metadata("demographics")) @@ -166,11 +170,11 @@ async def get_median_earnings(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("median_earnings")) -# Geography Snapshot Variables (FIXME) +# Geography Snapshot Variables @router.post("/load/acs5-db/tidy/snapshot") async def tidy_snapshot(request: FilterRequest): - rows = get_snapshot(filters=request.filters) - return make_response(data=rows, metadata=get_metadata("snapshot")) + rows = get_acs5_tidy(dataset="snapshot", filters=request.filters) + return make_response(data=rows, metadata=get_metadata("demographics")) # --------------------------------------------------------------------------- diff --git a/backend/api/routes/post_routes/post_qcew.py b/backend/api/routes/post_routes/post_qcew.py index 85b8f04..f28c553 100644 --- a/backend/api/routes/post_routes/post_qcew.py +++ b/backend/api/routes/post_routes/post_qcew.py @@ -36,7 +36,7 @@ def _first(label: str): if not county and is_statewide: query = """ SELECT year, quarter, quarter_label, sector, employment_4qma - FROM qcew_employment + FROM qcew_sectorEmployment_timeseries WHERE sector != 'Total' ORDER BY year, quarter, sector """ @@ -44,7 +44,7 @@ def _first(label: str): elif county: query = """ SELECT year, quarter, quarter_label, sector, employment_4qma - FROM qcew_employment + FROM qcew_sectorEmployment_timeseries WHERE sector != 'Total' AND County = ? ORDER BY year, quarter, sector diff --git a/backend/data_cleaning/clean_cdc.py b/backend/data_cleaning/clean_cdc.py index edde5e3..8d101fe 100644 --- a/backend/data_cleaning/clean_cdc.py +++ b/backend/data_cleaning/clean_cdc.py @@ -45,32 +45,73 @@ def get_sme_indicators() -> str: def build_PCA_table(us_df: pd.DataFrame) -> pd.DataFrame: """ - Builds a 2-Principal Component DataFrame - for county-level CDC indicators + Builds a 2-component PCA score for Vermont counties. + + PCA is fit using the full national county dataset. Vermont county + observations are then standardized using the national means and + standard deviations before being projected into the fitted PCA space. + + Returns: + DataFrame containing LocationID and the first PCA component score. """ - ## select only shared columns - vt_df = us_df[us_df["stateabbr"] == "VT"].copy() - pv = us_df.pivot(columns="measure", values="data_value", index="locationid").dropna( - axis=0, how="any" + # Build a wide national dataset: + # rows = counties + # columns = CDC measures + pv = us_df.pivot_table( + index="locationid", + columns="measure", + values="data_value", + aggfunc="first", + ).dropna(axis=0, how="any") + + # Build Vermont-wide dataset using the same measures + vt_df = us_df[us_df["stateabbr"].eq("VT")].copy() + + pv_vt = vt_df.pivot_table( + index="locationid", + columns="measure", + values="data_value", + aggfunc="first", ) - pv_vt = vt_df.pivot(columns="measure", values="data_value", index="locationid") - shared = pv.columns.intersection(pv_vt.dropna(axis=1, how="all").columns) - pv = pv[shared] - pv_vt = pv_vt[shared] - - ## standardize US to build column - mean, std = pv.mean(), pv.std() - pv = (pv - mean) / std + + # Keep only measures that exist in both datasets and have + # complete national data. + shared = pv.columns.intersection(pv_vt.columns) + + pv = pv[shared].dropna(axis=1, how="all") + pv_vt = pv_vt[pv.columns] + + # Only retain Vermont counties with complete data for all + # measures used in the PCA. + pv_vt = pv_vt.dropna(axis=0, how="any") + + # Standardize using NATIONAL parameters. + mean = pv.mean() + std = pv.std() + + # Avoid division by zero for constant measures. + valid = std > 0 + pv = pv.loc[:, valid] + pv_vt = pv_vt.loc[:, valid] + mean = mean[valid] + std = std[valid] + + pv_standardized = (pv - mean) / std + vt_standardized = (pv_vt - mean) / std + + # Fit PCA using national observations. pca = PCA(n_components=2) - pca.fit(pv) + pca.fit(pv_standardized) + + # Project Vermont observations into national PCA space. + scores = pca.transform(vt_standardized) - # standardize the VT, transform, add back in, and return - pv_vt = (pv_vt - mean) / std - assert list(pv_vt.columns) == list(pv.columns), "measure columns misaligned" - scores = pca.transform(pv_vt) - pv_vt["pca_score"] = scores[:, 0] - pv_vt = pv_vt.reset_index() - return pv_vt + return pd.DataFrame( + { + "LocationID": pv_vt.index, + "pca_score": scores[:, 0], + } + ) def add_national_percentile(us_df: pd.DataFrame) -> pd.DataFrame: @@ -154,7 +195,7 @@ def clean() -> dict[str, pd.DataFrame]: ) # PCA is fit on the national county data and applied to Vermont - # pca_county = build_PCA_table(county_us) + pca_county = build_PCA_table(county_us) # Tract: national data is needed for the national percentile _, tract_places, tract_edges = build_places_table( @@ -168,7 +209,7 @@ def clean() -> dict[str, pd.DataFrame]: "cdc_edges_county": county_edges, "cdc_places_tract": tract_places, "cdc_edges_tract": tract_edges, - # "cdc_pca_county": pca_county, + "cdc_pca_county": pca_county, } diff --git a/backend/data_cleaning/clean_fips.py b/backend/data_cleaning/clean_fips.py new file mode 100644 index 0000000..92ad49b --- /dev/null +++ b/backend/data_cleaning/clean_fips.py @@ -0,0 +1,146 @@ +""" +**Author**: + Ian Sargent +**Created**: + 2026-07-16 +**Description**: + Data cleaning script for the raw boundary line tables in the DuckLake. + + Standardizes boundary column names to match the legacy processed boundary + tables used throughout the application. + + County: + CountyFIPS + CountyName + geom + + Town: + FIPS_ID + TOWN_NAME + geometry + + Tract: + LocationID + name + geometry + +Run with: + python -m data_cleaning.clean_boundaries +""" + +from lake_build import con + +## LOAD SPATIAL EXTENSION FUNCTION -------------------- + + +def _load_spatial() -> None: + """ + Load the spatial extension, installing it first if necessary. + """ + try: + con.execute("LOAD spatial") + except Exception: + con.execute("INSTALL spatial") + con.execute("LOAD spatial") + + +## BUILD CLEANED VIEWS -------------------- + + +def build_county_lines(): + """ + Clean VT county boundary lines. + + Standardizes county identifiers and names to the legacy boundary + column conventions: + CNTYGEOID -> CountyFIPS + CNTYNAME -> CountyName + geometry -> geom + """ + con.execute( + """--sql + CREATE OR REPLACE VIEW vt_county_lines AS + SELECT + CNTYGEOID AS CountyFIPS, + CNTYNAME AS CountyName, + geometry + FROM lake.RAW.vt_county_lines + """ + ) + + +def build_town_lines(): + """ + Clean VT town boundary lines. + + Standardizes town identifiers and names to the legacy boundary + column conventions: + GEOID -> FIPS_ID + NAME -> TOWN_NAME + geom -> geometry + """ + con.execute( + """--sql + CREATE OR REPLACE VIEW vt_town_lines AS + SELECT + GEOID AS FIPS_ID, + TRIM(SPLIT_PART("NAME", ',', 1)) AS TOWN_NAME, + geometry + FROM lake.RAW.vt_town_lines + """ + ) + + +def build_tract_lines(): + """ + Clean VT Census tract boundary lines. + + Standardizes the tract identifier to LocationID. + """ + con.execute( + """--sql + CREATE OR REPLACE VIEW vt_tract_lines AS + SELECT + GEOID AS LocationID, + NAMELSAD AS name, + geometry + FROM lake.RAW.vt_tract_lines + """ + ) + + +## WRITE CLEANED TABLES -------------------- + + +def add_to_lake(): + table_names = [ + "vt_county_lines", + "vt_town_lines", + "vt_tract_lines", + ] + + for name in table_names: + con.execute( + f"""--sql + CREATE OR REPLACE TABLE lake.CLEANED.{name}_geom AS + SELECT * + FROM {name} + """ + ) + + +def clean(): + _load_spatial() + + build_county_lines() + build_town_lines() + build_tract_lines() + + +def main(): + clean() + add_to_lake() + + +if __name__ == "__main__": + main() diff --git a/backend/data_cleaning/clean_flood.py b/backend/data_cleaning/clean_flood.py index 38d057c..bcbb333 100644 --- a/backend/data_cleaning/clean_flood.py +++ b/backend/data_cleaning/clean_flood.py @@ -6,7 +6,7 @@ **Description**: Data cleaning script for the raw `flood` table in the DuckLake Run with: -python -m ETL.data_cleaning.clean_flood +python -m data_cleaning.clean_flood """ from lake_build import con diff --git a/backend/data_cleaning/clean_historic_population.py b/backend/data_cleaning/clean_historic_population.py index fdc71b7..a287c14 100644 --- a/backend/data_cleaning/clean_historic_population.py +++ b/backend/data_cleaning/clean_historic_population.py @@ -49,10 +49,10 @@ def long_format(df: pd.DataFrame): df, id_vars=["geoid", "town", "county", "geo_type"], value_vars=year_cols, - var_name="Year", + var_name="year", value_name="Population", ) - df_long["Year"] = df_long["Year"].astype(int) + df_long["year"] = df_long["year"].astype(int) return df_long @@ -82,7 +82,7 @@ def add_population_aggregations(df: pd.DataFrame): """ df["county_geoid"] = df["geoid"].astype(str).str[:5] # County-level aggregation - county_df = df.groupby(["county_geoid", "county", "Year"], as_index=False)[ + county_df = df.groupby(["county_geoid", "county", "year"], as_index=False)[ "Population" ].sum() county_df["NAME"] = county_df["county"] + " County, Vermont" @@ -90,13 +90,13 @@ def add_population_aggregations(df: pd.DataFrame): county_df = county_df.rename(columns={"county_geoid": "geoid"}) # State-level aggregation - state_df = df.groupby("Year", as_index=False)["Population"].sum() + state_df = df.groupby("year", as_index=False)["Population"].sum() state_df["NAME"] = "Vermont" state_df["geoid"] = "50" # Vermont's state FIPS code state_df["geo_type"] = "state" # Align columns before concatenating - cols = ["geoid", "NAME", "Year", "Population", "geo_type"] + cols = ["geoid", "NAME", "year", "Population", "geo_type"] town_df = df[cols] county_df = county_df[cols] @@ -112,12 +112,12 @@ def clean(): raw_df = read_raw_data() # Clean column names clean_column_names(raw_df) - # Melt DataFrame into long format (Cols: "geoid", "NAME", "Year", "Population", "geo_type") + # Melt DataFrame into long format (Cols: "geoid", "NAME", "year", "Population", "geo_type") df_long = long_format(raw_df) # Add a census-style "NAME" column for easier filtering df_long_clean = add_NAME_column(df_long) # Reorder columns - column_order = ["geoid", "NAME", "county", "town", "Year", "Population", "geo_type"] + column_order = ["geoid", "NAME", "county", "town", "year", "Population", "geo_type"] df = df_long_clean[column_order] # Append county + state aggregations (total sum) df = add_population_aggregations(df) diff --git a/backend/data_cleaning/clean_snapshot.py b/backend/data_cleaning/clean_snapshot.py new file mode 100644 index 0000000..41211c9 --- /dev/null +++ b/backend/data_cleaning/clean_snapshot.py @@ -0,0 +1,89 @@ +""" +**Author**: + Ian Sargent +**Created**: + 2026-08-21 +**Description**: + Data cleaning script for creating an ACS5 snapshot table by + combining selected indicators from multiple RAW tables in DuckLake. +**Run with**: + python -m data_cleaning.clean_snapshot +""" + +import pandas as pd + +from lake_build import con + + +def read_raw_data() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """ + Read the source tables from the RAW schema in DuckLake. + """ + + dem_df = con.execute( + """--sql + SELECT * + FROM lake.RAW.demographics + """ + ).df() + + housing_df = con.execute( + """--sql + SELECT * + FROM lake.RAW.housing + """ + ).df() + + econ_df = con.execute( + """--sql + SELECT * + FROM lake.RAW.economic + """ + ).df() + + return dem_df, housing_df, econ_df + + +def clean() -> pd.DataFrame: + """ + Select the snapshot indicators from the source datasets + and combine them into a single dataframe. + """ + + dem_df, housing_df, econ_df = read_raw_data() + + dem_vars = dem_df[dem_df["Variable"].isin(["Population (ACS)", "Median Age"])] + + housing_vars = housing_df[housing_df["Variable"].isin(["Median Home Value"])] + + econ_vars = econ_df[ + econ_df["Variable"].isin( + ["Labor Force Participation Rate (16+)", "Median Household Income"] + ) + ] + + combined = pd.concat([dem_vars, econ_vars, housing_vars], ignore_index=True) + + return combined + + +def add_to_lake(clean_df: pd.DataFrame): + """ + Writes the cleaned snapshot dataframe + to the CLEANED schema in DuckLake. + """ + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.acs5_snapshot_indicators_tidy AS + SELECT * FROM clean_df + """ + ) + + +def main(): + clean_df = clean() + add_to_lake(clean_df) + + +if __name__ == "__main__": + main() diff --git a/backend/data_collection/acs5.py b/backend/data_collection/acs5.py index 6509fd1..fa91d70 100644 --- a/backend/data_collection/acs5.py +++ b/backend/data_collection/acs5.py @@ -82,89 +82,82 @@ def run_acs5_scrape(years: range = YEARS, geos: list = GEOS, append: bool = Fals for year in years: print(f"\n=== {year} ===") + for geo_label, for_clause, in_clause in geos: for table in TABLES: print(f" {table} / {geo_label}...") - df = fetch_table(year, table, for_clause, in_clause) + + df = fetch_table( + year, + table, + for_clause, + in_clause, + ) + if df is not None: df["geo_type"] = geo_label all_frames[table].append(df) + time.sleep(0.1) - # Save wide + tidy per table results = {} + + # Process each ACS profile table independently for table, frames in all_frames.items(): if not frames: print(f" No frames for {table}, skipping.") continue label = TABLES[table] - title = f"vt_acs5_{label}_data" - combined = pd.concat(frames, ignore_index=True, sort=False) + + combined = pd.concat( + frames, + ignore_index=True, + sort=False, + ) # Key columns to front front = [c for c in ID_VARS if c in combined.columns] rest = [c for c in combined.columns if c not in front] + combined = combined[front + rest] - combined.sort_values(["year", "geo_type", "NAME"], inplace=True) - combined.reset_index(drop=True, inplace=True) - wide_parquet_path = f"{STORAGE_LOCATION}/{title}.parquet" - # wide_csv_path = f"{STORAGE_LOCATION}/{title}.csv" + combined.sort_values( + ["year", "geo_type", "NAME"], + inplace=True, + ) - if append: - new_names = set(combined["year", "geo_type", "NAME"].unique()) - # --- Wide --- - try: - existing_wide = pd.read_parquet(wide_parquet_path) - existing_wide = existing_wide[~existing_wide["NAME"].isin(new_names)] - combined = pd.concat([existing_wide, combined], ignore_index=True) - combined.sort_values(["year", "geo_type", "NAME"], inplace=True) - combined.reset_index(drop=True, inplace=True) - print(f" Wide append: kept {len(existing_wide):,} existing rows.") - except FileNotFoundError: - pass - - # combined.to_csv(wide_csv_path, index=False) - # combined.to_parquet(wide_parquet_path, index=False) - # print(f"Saved wide: {title} ({len(combined):,} rows)") + combined.reset_index(drop=True, inplace=True) # Tidy: run per-year so column labels are year-accurate - # Tidy: run per-year so column labels are year-accurate - tidy_frames = [] + tidy_frames = [] - for year in sorted(combined["year"].unique()): - year_df = combined[combined["year"] == year] + for year in sorted(combined["year"].unique()): + year_df = combined[combined["year"] == year] - if year_df.empty: - continue + if year_df.empty: + continue - try: - tidy_year = tidy_census( - year_df, - year=year, - id_vars=ID_VARS, + try: + tidy_year = tidy_census( + year_df, + year=year, + id_vars=ID_VARS, + ) + + tidy_year["table"] = table + tidy_frames.append(tidy_year) + + except Exception as e: + print(f" SKIP tidy {year} / {table}: {e}") + + if tidy_frames: + tidy = pd.concat( + tidy_frames, + ignore_index=True, ) - tidy_year["table"] = table - tidy_frames.append(tidy_year) - - except Exception as e: - print(f" SKIP tidy {year} / {table}: {e}") - - if tidy_frames: - tidy = pd.concat(tidy_frames, ignore_index=True) - - label = TABLES[table] - results[f"acs5_{label.lower()}"] = tidy - # tidy_parquet_path = f"{STORAGE_LOCATION}/{title}_tidy.parquet" - # tidy_csv_path = f"{STORAGE_LOCATION}/{title}_tidy.csv" - - # No separate append needed for tidy: it's derived from the - # already-merged wide frame, so it naturally contains all geos. - # tidy.to_csv(tidy_csv_path, index=False) - # tidy.to_parquet(tidy_parquet_path, index=False) - # print(f"Saved tidy: {title}_tidy ({len(tidy):,} rows)") + results[f"acs5_{label.lower()}"] = tidy return results diff --git a/backend/query/acs5.py b/backend/query/acs5.py index 7b7f8f4..3e9725b 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -63,6 +63,16 @@ }, }, }, + "labor_force": { + "table": "acs5_economics_tidy", + "fixed_filters": {"Section": ["Labor Force"]}, + "timeseries": {}, + }, + "income": { + "table": "acs5_economics_tidy", + "fixed_filters": {"Section": ["Income"]}, + "timeseries": {}, + }, "housing": { "table": "acs5_housing_tidy", "fixed_filters": {}, @@ -81,6 +91,16 @@ }, }, }, + "education": { + "table": "acs5_education_tidy", + "fixed_filters": {}, + "timeseries": {}, + }, + "snapshot": { + "table": "acs5_snapshot_indicators_tidy", + "fixed_filters": {}, + "timeseries": {}, + }, } # frontend filter label -> database column. Location and the year range both @@ -142,20 +162,25 @@ def get_acs5_timeseries( dataset: str, filters: dict | None = None, ) -> pd.DataFrame: - category_config = QUERY_CONFIG[category] - config = category_config["timeseries"][dataset] + try: + config = QUERY_CONFIG[category]["timeseries"][dataset] + except KeyError as e: + raise ValueError(f"Unknown ACS5 timeseries: {category}/{dataset}") from e source = _acs5_source( table=config["table"], filters=filters, - fixed_filters=config.get("fixed_filters", {}), + fixed_filters=config.get("fixed_filters"), ) - sql, params = sql_filter_block(sql_path / "acs5_tidy.sql", [source]) + sql, params = sql_filter_block( + sql_path / "acs5_timeseries.sql", + [source], + ) result = DB.execute(sql, params).df() - if result is None: + if result.empty: logger.error( "ACS5 timeseries query returned no rows for category=%s, " "dataset=%s, filters=%s", @@ -164,28 +189,12 @@ def get_acs5_timeseries( filters, ) raise ValueError( - f" No results for timeseries: {category}/{dataset}, filters: {filters}" + f"No results for timeseries: {category}/{dataset}, filters: {filters}" ) return result -def get_unemployment_rate_ts(filters: dict | None = None) -> pd.DataFrame: - source = _acs5_source( - table="acs5Economics_unemploymentRate_timeseries", filters=filters - ) - - sql, params = sql_filter_block(sql_path / "unemployment_rate.sql", [source]) - - result = DB.execute(sql, params).df() - - if result is None or result.empty: - logger.error("Unemployment rate query returned no rows for filters=%s", filters) - raise ValueError("no results for unemployment_rate query") - - return result - - # FIXME: Link to new database table name (broken for now) def get_median_earnings_ts(filters: dict | None = None) -> pd.DataFrame: source = _acs5_source(table="acs5_median_earnings", filters=filters) @@ -201,21 +210,6 @@ def get_median_earnings_ts(filters: dict | None = None) -> pd.DataFrame: return result -# FIXME: Link to new database table name (broken for now) -def get_snapshot(filters: dict | None = None) -> pd.DataFrame: - source = _acs5_source(table="snapshot", filters=filters) - - sql, params = sql_filter_block(sql_path / "snapshot.sql", [source]) - - result = DB.execute(sql, params).df() - - if result is None or result.empty: - logger.error("Snapshot query returned no rows for filters=%s", filters) - raise ValueError("no results for snapshot query") - - return result - - # FIXME: Link to new database table name (broken for now) def get_acs5_filters(): return filter_tree(ACS5_FILTER_COLS, ACS5_TREE_LABELS, "acs5_info") diff --git a/backend/query/cdc.py b/backend/query/cdc.py index 30e9970..8c33d66 100644 --- a/backend/query/cdc.py +++ b/backend/query/cdc.py @@ -46,7 +46,7 @@ def single_var_geojson(sources: list[FilterSource]): "geometry": json.loads(r.geometry), "properties": { "rgba_color": RAMP[int(r.bin)], - "tooltip": {"__title__": r.Measure, "value": r.Data_Value}, + "tooltip": {"__title__": r.measure, "value": r.data_value}, }, } ) @@ -54,10 +54,10 @@ def single_var_geojson(sources: list[FilterSource]): def widen_dual_var(df, measures): - cols = ["LocationID", "geometry", "Data_Value", "bin", "natl_pct", "CountyName"] - m1 = df[df.Measure == measures[0]][[c for c in cols if c in df.columns]] - m2 = df[df.Measure == measures[1]][["LocationID", "Data_Value", "bin"]] - wide = m1.merge(m2, on="LocationID", suffixes=("_1", "_2")) + cols = ["locationid", "geometry", "data_value", "bin", "natl_pct", "CountyName"] + m1 = df[df.measure == measures[0]][[c for c in cols if c in df.columns]] + m2 = df[df.measure == measures[1]][["locationid", "data_value", "bin"]] + wide = m1.merge(m2, on="locationid", suffixes=("_1", "_2")) return wide @@ -84,10 +84,10 @@ def _measure_cutpoints(measures: list[str]) -> tuple[list[float], list[float]]: sql = f"SELECT * FROM cdc_edges_county {where_string}" edges = DB.execute(sql, params).df() edges_x = ( - edges[edges["Measure"] == measures[0]].drop(columns="Measure").iloc[0].tolist() + edges[edges["measure"] == measures[0]].drop(columns="measure").iloc[0].tolist() ) edges_y = ( - edges[edges["Measure"] == measures[1]].drop(columns="Measure").iloc[0].tolist() + edges[edges["measure"] == measures[1]].drop(columns="measure").iloc[0].tolist() ) return edges_x, edges_y @@ -109,9 +109,9 @@ def dual_var_comparison( table = "cdc_places_county" if geoLevel == "county_places" else "cdc_places_tract" merged = FilterSource(filter_table=table, filters={"Measure": measures}) sql_path = sql_dir / f"{geoLevel}.sql" + sql, params = sql_filter_block(sql_path, [merged]) df = DB.execute(sql, params).df() - print(df.head()) df = widen_dual_var(df, measures) @@ -122,8 +122,8 @@ def dual_var_comparison( tooltip = { "__title__": "Variable Comparison", # "County": r.CountyName, - f"{measures[0]}": r.Data_Value_1, - f"{measures[1]}": r.Data_Value_2, + f"{measures[0]}": r.data_value_1, + f"{measures[1]}": r.data_value_2, "National Percentage": r.natl_pct, } ## add in County Name if we're in county space. @@ -149,14 +149,20 @@ def dual_var_comparison( return geojson, legend -# FIXME: Add PCA table in data_cleaning/clean_cdc.py script (broken for now) def get_cdc_county_pca(): - df = DB.execute("""--sql - SELECT i.LocationID, ROUND(i.pca_score, 2) AS "Health Burden", c.CountyName - FROM cdc_countyPcaData AS i - LEFT JOIN vermont_counties AS c ON i.LocationID = c.CountyFIPS - """).df() + df = DB.execute( + """--sql + SELECT + i.LocationID, + ROUND(i.pca_score, 2) AS "Health Burden", + c.CountyName + FROM cdc_pca_county AS i + LEFT JOIN vt_county_lines_geom AS c + ON i.LocationID = c.CountyFIPS + """ + ).df() + df["CountyName"] = df["CountyName"].str.title() df = df.sort_values(by="CountyName") - ret = df[["CountyName", "Health Burden"]].to_dict(orient="records") - return ret + + return df[["CountyName", "Health Burden"]].to_dict(orient="records") diff --git a/backend/query/sql/acs5/acs5_timeseries.sql b/backend/query/sql/acs5/acs5_timeseries.sql new file mode 100644 index 0000000..1b0fda1 --- /dev/null +++ b/backend/query/sql/acs5/acs5_timeseries.sql @@ -0,0 +1,4 @@ +SELECT * +FROM {{ table }} +{{ where_string }} +ORDER BY year \ No newline at end of file diff --git a/backend/query/sql/acs5/median_earnings.sql b/backend/query/sql/acs5/median_earnings.sql deleted file mode 100644 index 0ea7e7f..0000000 --- a/backend/query/sql/acs5/median_earnings.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT - year, - NAME, - Value, - -- quoted: the case change from `variable` is intentional (frontend key) - variable AS "Variable" -- noqa: RF06 -FROM acs5_median_earnings -{{ where_string }} -ORDER BY year diff --git a/backend/query/sql/acs5/snapshot.sql b/backend/query/sql/acs5/snapshot.sql deleted file mode 100644 index c300f44..0000000 --- a/backend/query/sql/acs5/snapshot.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - year, - NAME, - Value, - Variable -FROM acs5_snapshot -{{ where_string }} -ORDER BY year diff --git a/backend/query/sql/acs5/unemployment_rate.sql b/backend/query/sql/acs5/unemployment_rate.sql deleted file mode 100644 index 5413bcd..0000000 --- a/backend/query/sql/acs5/unemployment_rate.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - year, - NAME, - Value, - Value AS Percent -FROM acs5_unemployment_rate -{{ where_string }} -ORDER BY year diff --git a/backend/query/sql/cdc/county_places.sql b/backend/query/sql/cdc/county_places.sql index f01befa..2debc1d 100644 --- a/backend/query/sql/cdc/county_places.sql +++ b/backend/query/sql/cdc/county_places.sql @@ -6,7 +6,9 @@ SELECT ROUND(p.natl_pct * 100, 2) AS natl_pct, c.CountyFIPS, c.CountyName, - ST_ASGEOJSON(c.geom) AS geometry + ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry FROM cdc_places_county AS p -LEFT JOIN vermont_counties AS c ON p.LocationID = c.CountyFIPS -{{ where_string }} +LEFT JOIN vt_county_lines_geom AS c + ON p.LocationID = c.CountyFIPS + +{{ where_string }} \ No newline at end of file diff --git a/backend/query/sql/cdc/tract_places.sql b/backend/query/sql/cdc/tract_places.sql index 3542680..1f8fc78 100644 --- a/backend/query/sql/cdc/tract_places.sql +++ b/backend/query/sql/cdc/tract_places.sql @@ -4,8 +4,10 @@ SELECT p.Data_Value, p.bin, ROUND(p.natl_pct * 100, 2) AS natl_pct, - ST_ASGEOJSON(c.geometry) AS geometry, + ST_AsGeoJSON(c.geometry) AS geometry, c.name FROM cdc_places_tract AS p -LEFT JOIN vermont_tracts AS c ON p.LocationID = c.LocationID -{{ where_string }} +LEFT JOIN vt_tract_lines_geom AS c + ON p.LocationID = c.LocationID + +{{ where_string }} \ No newline at end of file diff --git a/backend/run_data_cleaning.py b/backend/run_data_cleaning.py index 994abb4..89823f6 100644 --- a/backend/run_data_cleaning.py +++ b/backend/run_data_cleaning.py @@ -23,33 +23,11 @@ def get_cleaners(): return cleaners -def get_cleaner(module_name: str): - """ - Return all cleaning scripts in the backend/data_cleaning folder - """ - - module = import_module(f"data_cleaning.{module_name}") - return module - - -# def create_duckdb_version(): -# tables = con.execute( -# """--sql -# SELECT table_name -# FROM duckdb_tables -# WHERE database_name = 'lake' -# AND schema_name = 'CLEANED' -# """).fetchall() - -# for table in tables - - def run_master_clean(): - # for cleaner in get_cleaners(): - cleaner = get_cleaner("clean_wastewater") - print(f"Running {cleaner.__name__.split('.')[-1]}...") - cleaner.main() - print(f"Completed {cleaner.__name__.split('.')[-1]}") + for cleaner in get_cleaners(): + print(f"Running {cleaner.__name__.split('.')[-1]}...") + cleaner.main() + print(f"Completed {cleaner.__name__.split('.')[-1]}") def main(): diff --git a/backend/run_data_collection.py b/backend/run_data_collection.py index 801bf8a..f5f03df 100644 --- a/backend/run_data_collection.py +++ b/backend/run_data_collection.py @@ -14,32 +14,32 @@ import argparse from data_collection import ( - # acs5, - # cdc, - # demographics, - # economic, - # education, - # fips, - # flood, - # historic_population, - # housing, - # qcew, + acs5, + cdc, + demographics, + economic, + education, + fips, + flood, + historic_population, + housing, + qcew, wastewater, - # zoning, + zoning, ) from lake_build import insert_year, replace_table # Datasets WITH year columns (longitudinal) -# YEARLY_SCRAPERS = [acs5, demographics, economic, education, housing, qcew] +YEARLY_SCRAPERS = [acs5, demographics, economic, education, housing, qcew] # Datasets WITHOUT year columns (static) STATIC_SCRAPERS = [ - # cdc, - # fips, - # flood, - # historic_population, + cdc, + fips, + flood, + historic_population, wastewater, - # zoning, + zoning, ] YEARS = range(2009, 2025) @@ -77,8 +77,8 @@ def run_scraper(scraper, yearly: bool = False, years: range = YEARS): def run_master_scrape(start_year: int = 2009, end_year: int = 2024): - # for scraper in YEARLY_SCRAPERS: - # run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) + for scraper in YEARLY_SCRAPERS: + run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) for scraper in STATIC_SCRAPERS: run_scraper(scraper, yearly=False) diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 2025baa..5663263 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -418,7 +418,7 @@ export const MedianAgeTrendChart = ({ }) => single( chart, - { seriesKey: 'Median Age', valueField: 'Value', format: 'years' }, + { seriesKey: null, valueField: 'Median_Age', format: 'years' }, view, ); @@ -510,7 +510,7 @@ export const UnemploymentTrendChart = ({ chart, { seriesKey: null, - valueField: 'Value', + valueField: 'Unemployment_Rate', format: 'percent', decimals: 1, }, @@ -527,8 +527,8 @@ export const HouseholdIncomeTrendChart = ({ single( chart, { - seriesKey: 'Median Household Income', - valueField: 'Value', + seriesKey: null, + valueField: 'Median_Household_Income', format: 'currency', showHelperText: false, }, @@ -545,8 +545,8 @@ export const PerCapitaIncomeTrendChart = ({ single( chart, { - seriesKey: 'Per Capita Income', - valueField: 'Value', + seriesKey: null, + valueField: 'Per_Capita_Income', format: 'currency', showHelperText: false, }, diff --git a/frontend/src/components/mapping/index.tsx b/frontend/src/components/mapping/index.tsx index 0eeb4e2..7f320a2 100644 --- a/frontend/src/components/mapping/index.tsx +++ b/frontend/src/components/mapping/index.tsx @@ -137,18 +137,13 @@ export default function VTMap({ new GeoJsonLayer({ id: 'geojson', data: geojson, - - // Points pointType: 'circle', - getPointRadius: 7, pointRadiusUnits: 'pixels', - pointRadiusMinPixels: 8, + pointRadiusMinPixels: 12, pointRadiusMaxPixels: 12, - - getFillColor: [30, 100, 220, 255], + getFillColor, getLineColor: [20, 70, 160, 255], - lineWidthMinPixels: 1, - + lineWidthMinPixels: 2, pickable: true, autoHighlight: true, highlightColor: [255, 255, 255, 255], From 4fa2054c24aafd59e5bc7e9b16706844992095f6 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Mon, 24 Aug 2026 09:37:39 -0400 Subject: [PATCH 15/26] fixed cdc tract bug --- backend/query/sql/cdc/tract_places.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/query/sql/cdc/tract_places.sql b/backend/query/sql/cdc/tract_places.sql index 1f8fc78..1273e1f 100644 --- a/backend/query/sql/cdc/tract_places.sql +++ b/backend/query/sql/cdc/tract_places.sql @@ -4,7 +4,7 @@ SELECT p.Data_Value, p.bin, ROUND(p.natl_pct * 100, 2) AS natl_pct, - ST_AsGeoJSON(c.geometry) AS geometry, + ST_ASGEOJSON(ST_GeomFromWKB(c.geometry)) AS geometry, c.name FROM cdc_places_tract AS p LEFT JOIN vt_tract_lines_geom AS c From 2e226efdc1df55205d5b1dc0c56260b024e80401 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Mon, 24 Aug 2026 10:02:28 -0400 Subject: [PATCH 16/26] add unemployment rate to economic data collection --- .../data_cleaning/clean_derived_time_series.py | 8 -------- backend/data_collection/economic.py | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/backend/data_cleaning/clean_derived_time_series.py b/backend/data_cleaning/clean_derived_time_series.py index 3818ee4..e00fd29 100644 --- a/backend/data_cleaning/clean_derived_time_series.py +++ b/backend/data_cleaning/clean_derived_time_series.py @@ -101,14 +101,6 @@ class DatasetConfig: output_table="acs5Housing_vacancyRates_timeseries", keep_variable_col=True, ), - "unemployment_rate": DatasetConfig( - source_table="acs5_economic", - variables=["Unemployment Rate"], - value_source_col="Value", - output_value_col="Unemployment_Rate", - output_table="acs5Economics_unemploymentRate_timeseries", - extra_where_statement="Measure = 'Percent'", - ), } diff --git a/backend/data_collection/economic.py b/backend/data_collection/economic.py index 0f84a28..29e8cce 100644 --- a/backend/data_collection/economic.py +++ b/backend/data_collection/economic.py @@ -1,6 +1,6 @@ """ Fetch ACS 5-Year economic data for Vermont: - B23025 – Employment Status (labor force participation, 16+) + B23025 – Employment Status (labor force participation, 16+ and unemployment) B23001 – Sex by Age by Employment Status (prime-age 25-54 LFP) B19013 – Median Household Income B19301 – Per Capita Income @@ -37,6 +37,7 @@ "B23001_125E", "B23001_132E", # female 25-54 ] + _PRIME_TOTAL = [ "B23001_024E", "B23001_031E", @@ -55,6 +56,12 @@ ["B23025_002E"], ["B23025_001E"], ), + VarGroup( + "Unemployment Rate (16+)", + SL, + ["B23025_003E"], + ["B23025_002E"], + ), VarGroup( "Prime-Age Labor Force Participation Rate (25-54)", SL, @@ -66,7 +73,11 @@ ] fetch_specs = { - "B23025": ["B23025_001E", "B23025_002E"], + "B23025": [ + "B23025_001E", + "B23025_002E", + "B23025_003E", + ], "B23001": _PRIME_TOTAL + _PRIME_IN_LF, "B19013": ["B19013_001E"], "B19301": ["B19301_001E"], From cd3358cff5479f8e26ee9bc2cbcf4ab1f96070fc Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Mon, 24 Aug 2026 11:10:54 -0400 Subject: [PATCH 17/26] developing acs5 cleaner script --- backend/data_cleaning/clean_acs5.py | 143 ++++++++++++++++++++++++++++ backend/data_collection/acs5.py | 1 - 2 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 backend/data_cleaning/clean_acs5.py diff --git a/backend/data_cleaning/clean_acs5.py b/backend/data_cleaning/clean_acs5.py new file mode 100644 index 0000000..c49a80b --- /dev/null +++ b/backend/data_cleaning/clean_acs5.py @@ -0,0 +1,143 @@ +""" +**Author**: + Ian Sargent +**Created**: + 2026-08-24 +**Description**: + Clean the raw ACS-5 Data Profile (DP) tables from DuckLake and build: + - Individual cleaned DP tables + - A combined DP table + - A county GEOID lookup table + + DP02 – Social + DP03 – Economic + DP04 – Housing + DP05 – Demographic + +**Run with**: + python -m data_cleaning.acs5 +""" + +import pandas as pd + +from lake_build import con + +# Data Profile tables +DP_TABLES = { + "DP02": ("acs5_social", "dp_social"), + "DP03": ("acs5_economic", "dp_economic"), + "DP04": ("acs5_housing", "dp_housing"), + "DP05": ("acs5_demographic", "dp_demographic"), +} + +# County GEOIDs for the county_geom table +COUNTY_GEOIDS = { + "Addison County, Vermont": 50001, + "Bennington County, Vermont": 50003, + "Caledonia County, Vermont": 50005, + "Chittenden County, Vermont": 50007, + "Essex County, Vermont": 50009, + "Franklin County, Vermont": 50011, + "Grand Isle County, Vermont": 50013, + "Lamoille County, Vermont": 50015, + "Orange County, Vermont": 50017, + "Orleans County, Vermont": 50019, + "Rutland County, Vermont": 50021, + "Washington County, Vermont": 50023, + "Windham County, Vermont": 50025, + "Windsor County, Vermont": 50027, +} + + +def read_raw_data() -> dict[str, pd.DataFrame]: + """ + Read the raw ACS-5 DP tables from DuckLake. + """ + tables = {} + + for dp, (raw_table, _) in DP_TABLES.items(): + tables[dp] = con.execute( + f"""--sql + SELECT * + FROM lake.RAW.{raw_table} + """ + ).df() + + return tables + + +def clean_dp_tables(raw_tables: dict[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: + """ + Clean the individual ACS-5 DP tables. + """ + cleaned = {} + + for dp, df in raw_tables.items(): + # Future cleaning steps can go in here! + cleaned[dp] = df + + return cleaned + + +def build_county_geoids(): + """ + Create the county GEOID table. + """ + values = ", ".join(f"('{name}', {geoid})" for name, geoid in COUNTY_GEOIDS.items()) + + con.execute( + f"""--sql + CREATE OR REPLACE TABLE lake.CLEANED.vt_county_geoids AS + SELECT * FROM (VALUES {values}) AS t(NAME, GEOID) + """ + ) + + +def add_dp_tables(cleaned: dict[str, pd.DataFrame]): + """ + Write each DP table to the CLEANED DuckLake schema. + """ + for dp, df in cleaned.items(): + table = DP_TABLES[dp][1] + con.execute( + f"""--sql + CREATE OR REPLACE TABLE lake.CLEANED.{table} AS + SELECT * + FROM df + """ + ) + + +def build_dp_combined(): + """ + Combine the four DP tables into one table. + """ + tables = [table_name for _, table_name in DP_TABLES.values()] + + union = "\nUNION ALL\n".join( + f"SELECT * FROM lake.CLEANED.{table}" for table in tables + ) + + con.execute( + f"""--sql + CREATE OR REPLACE TABLE lake.CLEANED.dp_combined AS + {union} + """ + ) + + +def clean(): + raw_tables = read_raw_data() + cleaned = clean_dp_tables(raw_tables) + + add_dp_tables(cleaned) + build_dp_combined() + build_county_geoids() + + +def main(): + clean() + + +if __name__ == "__main__": + main() diff --git a/backend/data_collection/acs5.py b/backend/data_collection/acs5.py index fa91d70..1538451 100644 --- a/backend/data_collection/acs5.py +++ b/backend/data_collection/acs5.py @@ -175,7 +175,6 @@ def merge_tidy_tables(): if tidy_frames: combined = pd.concat(tidy_frames, ignore_index=True) # combined.to_parquet(f"{STORAGE_LOCATION}/vt_acs5_combined_TIDY.parquet", index=False) - # print(f"Combined tidy saved: {len(combined):,} rows") return combined return From 76aac018c5aca2aba2808939cef52e2ced58e975 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Mon, 24 Aug 2026 14:16:09 -0400 Subject: [PATCH 18/26] updating acs5 dp_combined route --- Data/lake | Bin 0 -> 2109440 bytes .../api/routes/post_routes/post_acs5_db.py | 23 ++++--- backend/data_cleaning/clean_acs5.py | 63 ++++++++++++------ backend/query/acs5.py | 4 -- .../src/components/Charts/TrendCharts.tsx | 4 +- .../components/Charts/configs/ChartDefs.tsx | 2 +- 6 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 Data/lake diff --git a/Data/lake b/Data/lake new file mode 100644 index 0000000000000000000000000000000000000000..adb4067c0795c603c0a933a381227c480b0166fc GIT binary patch literal 2109440 zcmeI*dyHjgK>+YGJF`3c-j;`LrL~pD#Kh2T!wLnBVd*v&fu>ZN3Q=zE&OI|bb{@;j z-G!pk1tlpYB>vF?iYUQKpaL-^#RrKL4Wz{qOR!PI7>v;%sZlh<#+Z)ZcOLiL$GJ0i zW_G)GcYn9tx#xAy`Ofcqo^$W_oyY#@bwB-^uleMskKJ|4>yIU)y-weB!%yCDWcao> z-kiSt$jp%=uRK0{o497~`&az!&rCJG^cS!H#ltWA%IvSLzqPdl2oNAZfB*pk1PBly zK!Ctj1%Bn1ZaedV2Y&UV6ZJdTySuY|DxWGAOm~)>AV7cs0RjXF5FkK+009C7#$Dj) z-#UH8lTW_&sWO~veXnwPl8k=+@B9Bs%@hFw1PBlyK!5-N0t5&U7zcr$`{LwP3y;n0 zEy6O@O0JnncC?a-R-M|9IOOt5D_t4b>%L;`QgbnlE3fJfQn|8y@5#ZHhk{Le zzPZq7rL)ab3t51BK7F8A(Ky{)I5qhB152kC7FHY0g@xsJq#4KF?qeS~*<4AN+KoJX z1L1x_&(0gQA$)P~HmwlYbf!O?F>>SH&lIl^!MoSiW<&psB4#!l^H=o6d}FbB^5p!| zTucFztz>U8TNhQsBXO$c1ZzV6$phO8`V0Gm-fm{W-qlKm2ko3C>B)Q2dk2!_&bq`@ zBk?bnZA;<>9f>S=&(1HTG5#)VCHp%IHW!22%ieoq2paWSc~bi=RvK}WftCALmfsQZ8msN*%HY2Dhm7A2CClg-^vT2vQX!%L0rWrv=~>5Ix%vegW+Ygk?=&hlL3g^kklrqXO*bPP?k zJJRHLx+=MlYJX%f)po_~1(|MttdnkIf?aK_&YVbFrxwyyBSdp(w1l)AGEywnL$mMP zB&dVsF{`5Px~h#6Gh=sI5@O}AUtE3`+0|yPiIFda)!S$cUeg;RRn?ZA)JTbMc%;0r+Fhs$8}m!8w7*_!C5Nhw>LB}1 zc`{ADJ-Mw@eo5~spKh+qH$ywVlM45rlT^6BJWg$6=)mfUm9*6gdB3I|-AXRvBpl-N zp{}BS3x&$no%ozzT9{u7hFOCqyWY%-m2gorJdN(nuS;t$FSFutv01ih`s(qTl=-C> zZ7XFS?3~GBb7o~Z=Dz$DuJ{jxN-)gkO{KZy@h}sv1Y;PoSD1>Mhec(}@63Jqnlzc1 z&F&YZ$!m(;ZTT(VqmXn1GU?XHr|V5ZHqT#AB%OXMUrUm|N+zBxFI>)bQWb<)bc}&Q zR#b%x%c9Wg6lCLDwu}MVc?d$NJ+i1O^LV@>2=^Jzg!=&TiavjZ+Y;j&Z{t%>j&Jns z5`AS_j_-Lb#bjfX^N^(bqvbpVVZ=i5vi_isPf;W!*}X>>RGlndU*?^M%vGCdMrbLH zOHgyj*gER8Znu0%KG}($4JYd3Pd=B=dy`ImI=L7#Pw#ox&eda_zf3b?!@|A5p4c3> zSgftV@<4{aXQ5i?Is5vVaW@&40_PHKSkxz)o~7fSg3ezl-Jie5B`u$qB4)%YYpNej z^!0z2H$qXAp*>f|b5~9B*b~n@8m}_$%h!cSIm_SqjTp^)$2XeCC&3Aci?W`owz}b5 zEZ*63V`>cQmoW%JmTN{RBhN!nM=Tu2xDHe#4Hik`n~?UaWxLDIyv;MdCg&j`^<#`d z5EL0MKEhTphmTirOW>g~qS2pyb4w1nQqS#Ol-EuH5&{o4+9_CF;0tMxUfA4h%2$43~_ zS#9Hi4EhCkrpYI_*G22{O0XEeBBqeg4RZ7@_aQ;8&Memq3$Z&}JeZcs9@cp;>QKyv ziEz8AH~EKCdn62s5eCxO>gWqE?wnf8tKDNSgh33}>m0sL3fQ;0d}?JTt&jdgp%eRD z+77*<(}mXHX@~d^>0_ONEcS;k`eBQ_$9v3CU7Zzcc>ex{XDRP=j#+dQEy{@Xd*2W5 z=QUDa#C44Kgg%)|tFZ%iU#U`K@bvcjWL5)oR0i`)vsr(z*cG^I>EI|W!v)S}H4yT_ zr>@;rL|5a_cvy1@|y7#RZr#Z4o*|%-WZ^&M=$z)c%ghJ@5fqw6W!i`l=zwljO9JL{|$HGtx;WMG^ol7PrP9zie zUp5~W#?cz`;T)<>(&YZog`?Q)JG;uw#(k`ZD?6oU!B7l0QTFrPD2=b*qjPva(f&*x zBI=_r*jD!sBU684|YA|-9 zcjiQMX)eu@b!ZrOLk~pi{$ktEmT0AYl~XI()jZWcv7C*o)isbnNEb89i;MH^FhJPq zY8ba=;B%qNdA2?uj38D&eVc4CIZUU?gHt_4-1WngLwRQFB#SV#%tpx~9P)xp7F~ti z@=CjrzVqZth^%5vkWg1nug1j#`E+@QYBOg)v=e_I~htQr?9$Eli4ThZFah<7!o_KBJrBfs$9H3qk| zF3li}coS=Sd~Ul5N%yU`n9T;iSGzRRa(~LWG!;h7(P5;NsoBsz4!3ybl0zrLRQNq+ zxRMGrPZ%I!W+e?{4z?O$q>ADSBMiBhk5m!rr}&DsT)B_?;y!lt>?7MI`%`TeHvfg% zqaD6$HQXFLJRgoW+gu2P6~+Pn@}s>v-_h2N_TB2_!m-1}VSaLD*oFSf z%>~K~*9pi*5m_IQYRA=7PY^oY00-B11*FS;l$;;#r`Cz|9C#Bm^#RG!M<=<2I}!0B zHw5%d`MNJ(Sr4;`!w+oQwciL+ykbJ@3|~=dW?4TJ2I1aX8LV(6`E1DOJq!MMU*Cdn z?C)FfSQy&ocf=5%!&9w9VogKwP*P# z{Ol;iM`%^Vi?^TZ%ls4JJI>9#a?`Sj(sT5!WrV>X)&(cK((GKy)ooFo#yUY9C=LdX zeinf`$-ao5PI|0@QC>53bv_N>L3c-3-MMaNS-nP?+J|PYL6~6&t>C+p(qLSzACRHh zt~MZ9a;OW?p1OtAvMj6v0m<&{49}DyvB#J!Uc;|~LnfQ9E_%ba-1mp?>UtJ@w|~KZ zW)Fra6G8aMH+S`Y)FTWLt)gj;1hegFPz8tH|!bXXnCMhT$SoueeaFy&DLij z?VaV2WT|;rYAR1MgUYJVkY3JdZ6EGp4R6@|;C=u2>3{gs4~1XE6gQq8_}%Xm3v$*i z)Q91KH=p}+%I*}`Hj=7ZH=Nd%&*Hv#3cdHN2WsF@#&3H6olF0odbO%8iZ<+Ek$4s> zt-PI5wJA181%qKJQ*r(K(2~2XMvB%@Z42Z4N(`i(Vdq|+N)jr>YP#N|cWItty3T!~ zrfpZGtihxAr-I&nZ#-9~*G*J-;rBIXidU$;{wX}+q-VjGf2eQ4BjNY&JzG3@W#58l z!-DEQaCwmss>QG{d+j-wmp|bou8W6S?@O)n`>pa~Bm_9Vwon>|qs1Q$7KiIXm)@7S({P@^KDPg+_1n18|>`) zL{T1OrN(9=b%TKP9%H1rJkbeAg_VnB(lKg^*Sgmby+!KxuI(-#!*9Ia+Bg4r$YNEA z8aDXj&`Ri85c`wlC2IJk@|oh5T~>7NBAow*aardHNlYVM87Z{KN|Oz1g^qlr9z;)T)k^)Qxoi;%Kh2|r8dk1!YAgI|;}$ts|Xp-ehGRZ_Vy{2KZKN>4?m ztS-~ay6((<)g{w8E*~YWan5dPug>j+Dwh;_W8@;?d2ck9m2L4fr7IiN)0acf@MiYF z!iDBNY4{OZ{GM<=315d*PXo*zQ5bJv?q08NNAV=XbI?7q>bV$i5cKpYTkr9STkSX* zqp_#;|6k!2%r>5JT0QVHf7WC9JlOKFv(xLTuG1y>K=l~c@PybH=J1oj>?$Rs$Iwr= zT3z#$*R4G{cHijFh7Ffd@4I$eZq3`Scy}Ohb?1={Q|?o_n$t&u>AJAG5Xt_Rx-JwPSpQj*Sba z7MB{$h56=c8h%W_?h}+l8v5#?9@^@$c4ZUNI{^X&2oNAZfB=CJ1!7A+zUns5*Npki z$(nmCwILp1J9n;p?^2f!dX`-fnIaEl%+{UX6;}7@^OfiRYs~X^n|{Ra@j5?hDZl$Y ze<7|dvM%vh>^HeIi{}y3@}7RJF;2@xx~^*Es%-Kd;w4#N_b1*SxKAiwADC%Z>!mEHcmF%C(^L+-jiWr>)`Qp?&NRp$+j>5d}DVx zmB0Udc+2*?E8p);H{5va)XY7{UVGh*H>WQ@GIQj}D~}((Yc(A0@O*P=wYf5vF0~Ke zdF#iH_oO$>&M%~^hi97$t7({O%s+P7zh#H% zJ$jP#hbOL|$TwQ+-_|?<0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNBU3w-Y_Kk>6qeeOrExw;QzqL|;EB$ILS!{3^F+iSo6k%vC}#<>sN z{fF=Vn^*k#=Z;@>^w%GF|GQrCw$C2<_NdIlpx0(YtSX!>vbezxCMK+IQjx#cM6RUR6wA9^SLhP3_2*#LpcLA3R)q@^IWZ zyzcmo;seE7`T5?Z<_HiVK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+0D(GzN3MUPvkSx#lkXwb!}pj009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk7nZ$xA9JZ009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5(bUEq=H-*nr<&)oLtb8B7%2oNAZfWXBrP_`J(o))Z literal 0 HcmV?d00001 diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 8b04b90..970740a 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -46,6 +46,13 @@ async def tidy_housing(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("housing")) +# Economics +@router.post("/load/acs5-db/tidy/economics") +async def tidy_economics(request: FilterRequest): + rows = get_acs5_tidy(dataset="economics", filters=request.filters) + return make_response(data=rows, metadata=get_metadata("labor_force")) + + # Labor Force (FIXME: broken) @router.post("/load/acs5-db/tidy/labor-force") async def tidy_labor_force(request: FilterRequest): @@ -127,12 +134,12 @@ async def get_per_capita_income(request: FilterRequest): # Unemployment Rate -@router.post("/load/acs5-db/timeseries/economics/unemployment-rate") -async def get_unemployment_rate(request: FilterRequest): - rows = get_acs5_timeseries( - category="economics", dataset="unemployment_rate", filters=request.filters - ) - return make_response(data=rows, metadata=get_metadata("unemployment_rate")) +# @router.post("/load/acs5-db/timeseries/economics/unemployment-rate") +# async def get_unemployment_rate(request: FilterRequest): +# rows = get_acs5_timeseries( +# category="economics", dataset="unemployment_rate", filters=request.filters +# ) +# return make_response(data=rows, metadata=get_metadata("unemployment_rate")) ##### HOUSING ##### @@ -205,7 +212,7 @@ async def dp_combined_tree(): rows = DB.execute( """--sql SELECT DISTINCT "table", Category, Subcategory, Variable, Measure - FROM acs5_dp_combined + FROM acs5_dp_combined_tidy ORDER BY "table", Category, Subcategory, Variable, Measure """ ).df() @@ -221,7 +228,7 @@ async def dp_combined_series(request: DPSeriesRequest): """--sql SELECT CAST(year AS INTEGER) AS year, CAST(Value AS DOUBLE) AS Value - FROM acs5_dp_combined + FROM acs5_dp_combined_tidy WHERE NAME = ? AND "table" = ? AND Category = ? diff --git a/backend/data_cleaning/clean_acs5.py b/backend/data_cleaning/clean_acs5.py index c49a80b..c713422 100644 --- a/backend/data_cleaning/clean_acs5.py +++ b/backend/data_cleaning/clean_acs5.py @@ -15,7 +15,7 @@ DP05 – Demographic **Run with**: - python -m data_cleaning.acs5 +python -m data_cleaning.clean_acs5 """ import pandas as pd @@ -54,14 +54,9 @@ def read_raw_data() -> dict[str, pd.DataFrame]: Read the raw ACS-5 DP tables from DuckLake. """ tables = {} - - for dp, (raw_table, _) in DP_TABLES.items(): - tables[dp] = con.execute( - f"""--sql - SELECT * - FROM lake.RAW.{raw_table} - """ - ).df() + for dp, (raw_table_name, dp_table_name) in DP_TABLES.items(): + query = f"SELECT * FROM lake.RAW.{raw_table_name}" + tables[dp] = con.execute(query).df() return tables @@ -96,31 +91,57 @@ def build_county_geoids(): def add_dp_tables(cleaned: dict[str, pd.DataFrame]): """ Write each DP table to the CLEANED DuckLake schema. + + Adds a `table` column containing the DP table identifier + (e.g. DP02, DP03, DP04, DP05). """ for dp, df in cleaned.items(): - table = DP_TABLES[dp][1] - con.execute( - f"""--sql - CREATE OR REPLACE TABLE lake.CLEANED.{table} AS - SELECT * - FROM df - """ - ) + table_name = DP_TABLES[dp][1] + + df = df.copy() + df["table"] = dp + + con.register("tmp_df", df) + + try: + con.execute( + f""" + CREATE OR REPLACE TABLE lake.CLEANED.{table_name} AS + SELECT + * + FROM tmp_df + """ + ) + finally: + con.unregister("tmp_df") def build_dp_combined(): """ - Combine the four DP tables into one table. + Combine the four cleaned DP tables into one tidy table. """ + tables = [table_name for _, table_name in DP_TABLES.values()] union = "\nUNION ALL\n".join( - f"SELECT * FROM lake.CLEANED.{table}" for table in tables + f""" + SELECT + NAME, + "table", + Category, + Subcategory, + Variable, + Measure, + year, + Value + FROM lake.CLEANED.{table_name} + """ + for table_name in tables ) con.execute( - f"""--sql - CREATE OR REPLACE TABLE lake.CLEANED.dp_combined AS + f""" + CREATE OR REPLACE TABLE lake.CLEANED.acs5_dp_combined_tidy AS {union} """ ) diff --git a/backend/query/acs5.py b/backend/query/acs5.py index 3e9725b..b11b8b3 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -57,10 +57,6 @@ "table": "acs5Economics_perCapitaIncome_timeseries", "fixed_filters": {}, }, - "unemployment_rate": { - "table": "acs5Economics_unemploymentRate_timeseries", - "fixed_filters": {}, - }, }, }, "labor_force": { diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 5663263..7af3262 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -509,8 +509,8 @@ export const UnemploymentTrendChart = ({ single( chart, { - seriesKey: null, - valueField: 'Unemployment_Rate', + seriesKey: 'Unemployment Rate (16+)', + valueField: 'Value', format: 'percent', decimals: 1, }, diff --git a/frontend/src/components/Charts/configs/ChartDefs.tsx b/frontend/src/components/Charts/configs/ChartDefs.tsx index b286cb5..be93de3 100644 --- a/frontend/src/components/Charts/configs/ChartDefs.tsx +++ b/frontend/src/components/Charts/configs/ChartDefs.tsx @@ -266,7 +266,7 @@ export const chartDefs: ChartDef[] = [ { id: 'unemployment_rate', title: 'Unemployment Rate', - url: `${BASE_API_URL}/load/acs5-db/timeseries/economics/unemployment-rate`, + url: `${BASE_API_URL}/load/acs5-db/tidy/labor-force`, xField: '', yField: '', subtype: 'renderTable', From b7c4e142a0494ac31cc763ca03d62ba18f1d10d8 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 25 Aug 2026 13:06:29 -0400 Subject: [PATCH 19/26] fixed unemployment rate census calculation --- Data/lake | Bin 2109440 -> 0 bytes backend/data_collection/economic.py | 5 +++-- .../src/components/Charts/TrendCharts.tsx | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 Data/lake diff --git a/Data/lake b/Data/lake deleted file mode 100644 index adb4067c0795c603c0a933a381227c480b0166fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2109440 zcmeI*dyHjgK>+YGJF`3c-j;`LrL~pD#Kh2T!wLnBVd*v&fu>ZN3Q=zE&OI|bb{@;j z-G!pk1tlpYB>vF?iYUQKpaL-^#RrKL4Wz{qOR!PI7>v;%sZlh<#+Z)ZcOLiL$GJ0i zW_G)GcYn9tx#xAy`Ofcqo^$W_oyY#@bwB-^uleMskKJ|4>yIU)y-weB!%yCDWcao> z-kiSt$jp%=uRK0{o497~`&az!&rCJG^cS!H#ltWA%IvSLzqPdl2oNAZfB*pk1PBly zK!Ctj1%Bn1ZaedV2Y&UV6ZJdTySuY|DxWGAOm~)>AV7cs0RjXF5FkK+009C7#$Dj) z-#UH8lTW_&sWO~veXnwPl8k=+@B9Bs%@hFw1PBlyK!5-N0t5&U7zcr$`{LwP3y;n0 zEy6O@O0JnncC?a-R-M|9IOOt5D_t4b>%L;`QgbnlE3fJfQn|8y@5#ZHhk{Le zzPZq7rL)ab3t51BK7F8A(Ky{)I5qhB152kC7FHY0g@xsJq#4KF?qeS~*<4AN+KoJX z1L1x_&(0gQA$)P~HmwlYbf!O?F>>SH&lIl^!MoSiW<&psB4#!l^H=o6d}FbB^5p!| zTucFztz>U8TNhQsBXO$c1ZzV6$phO8`V0Gm-fm{W-qlKm2ko3C>B)Q2dk2!_&bq`@ zBk?bnZA;<>9f>S=&(1HTG5#)VCHp%IHW!22%ieoq2paWSc~bi=RvK}WftCALmfsQZ8msN*%HY2Dhm7A2CClg-^vT2vQX!%L0rWrv=~>5Ix%vegW+Ygk?=&hlL3g^kklrqXO*bPP?k zJJRHLx+=MlYJX%f)po_~1(|MttdnkIf?aK_&YVbFrxwyyBSdp(w1l)AGEywnL$mMP zB&dVsF{`5Px~h#6Gh=sI5@O}AUtE3`+0|yPiIFda)!S$cUeg;RRn?ZA)JTbMc%;0r+Fhs$8}m!8w7*_!C5Nhw>LB}1 zc`{ADJ-Mw@eo5~spKh+qH$ywVlM45rlT^6BJWg$6=)mfUm9*6gdB3I|-AXRvBpl-N zp{}BS3x&$no%ozzT9{u7hFOCqyWY%-m2gorJdN(nuS;t$FSFutv01ih`s(qTl=-C> zZ7XFS?3~GBb7o~Z=Dz$DuJ{jxN-)gkO{KZy@h}sv1Y;PoSD1>Mhec(}@63Jqnlzc1 z&F&YZ$!m(;ZTT(VqmXn1GU?XHr|V5ZHqT#AB%OXMUrUm|N+zBxFI>)bQWb<)bc}&Q zR#b%x%c9Wg6lCLDwu}MVc?d$NJ+i1O^LV@>2=^Jzg!=&TiavjZ+Y;j&Z{t%>j&Jns z5`AS_j_-Lb#bjfX^N^(bqvbpVVZ=i5vi_isPf;W!*}X>>RGlndU*?^M%vGCdMrbLH zOHgyj*gER8Znu0%KG}($4JYd3Pd=B=dy`ImI=L7#Pw#ox&eda_zf3b?!@|A5p4c3> zSgftV@<4{aXQ5i?Is5vVaW@&40_PHKSkxz)o~7fSg3ezl-Jie5B`u$qB4)%YYpNej z^!0z2H$qXAp*>f|b5~9B*b~n@8m}_$%h!cSIm_SqjTp^)$2XeCC&3Aci?W`owz}b5 zEZ*63V`>cQmoW%JmTN{RBhN!nM=Tu2xDHe#4Hik`n~?UaWxLDIyv;MdCg&j`^<#`d z5EL0MKEhTphmTirOW>g~qS2pyb4w1nQqS#Ol-EuH5&{o4+9_CF;0tMxUfA4h%2$43~_ zS#9Hi4EhCkrpYI_*G22{O0XEeBBqeg4RZ7@_aQ;8&Memq3$Z&}JeZcs9@cp;>QKyv ziEz8AH~EKCdn62s5eCxO>gWqE?wnf8tKDNSgh33}>m0sL3fQ;0d}?JTt&jdgp%eRD z+77*<(}mXHX@~d^>0_ONEcS;k`eBQ_$9v3CU7Zzcc>ex{XDRP=j#+dQEy{@Xd*2W5 z=QUDa#C44Kgg%)|tFZ%iU#U`K@bvcjWL5)oR0i`)vsr(z*cG^I>EI|W!v)S}H4yT_ zr>@;rL|5a_cvy1@|y7#RZr#Z4o*|%-WZ^&M=$z)c%ghJ@5fqw6W!i`l=zwljO9JL{|$HGtx;WMG^ol7PrP9zie zUp5~W#?cz`;T)<>(&YZog`?Q)JG;uw#(k`ZD?6oU!B7l0QTFrPD2=b*qjPva(f&*x zBI=_r*jD!sBU684|YA|-9 zcjiQMX)eu@b!ZrOLk~pi{$ktEmT0AYl~XI()jZWcv7C*o)isbnNEb89i;MH^FhJPq zY8ba=;B%qNdA2?uj38D&eVc4CIZUU?gHt_4-1WngLwRQFB#SV#%tpx~9P)xp7F~ti z@=CjrzVqZth^%5vkWg1nug1j#`E+@QYBOg)v=e_I~htQr?9$Eli4ThZFah<7!o_KBJrBfs$9H3qk| zF3li}coS=Sd~Ul5N%yU`n9T;iSGzRRa(~LWG!;h7(P5;NsoBsz4!3ybl0zrLRQNq+ zxRMGrPZ%I!W+e?{4z?O$q>ADSBMiBhk5m!rr}&DsT)B_?;y!lt>?7MI`%`TeHvfg% zqaD6$HQXFLJRgoW+gu2P6~+Pn@}s>v-_h2N_TB2_!m-1}VSaLD*oFSf z%>~K~*9pi*5m_IQYRA=7PY^oY00-B11*FS;l$;;#r`Cz|9C#Bm^#RG!M<=<2I}!0B zHw5%d`MNJ(Sr4;`!w+oQwciL+ykbJ@3|~=dW?4TJ2I1aX8LV(6`E1DOJq!MMU*Cdn z?C)FfSQy&ocf=5%!&9w9VogKwP*P# z{Ol;iM`%^Vi?^TZ%ls4JJI>9#a?`Sj(sT5!WrV>X)&(cK((GKy)ooFo#yUY9C=LdX zeinf`$-ao5PI|0@QC>53bv_N>L3c-3-MMaNS-nP?+J|PYL6~6&t>C+p(qLSzACRHh zt~MZ9a;OW?p1OtAvMj6v0m<&{49}DyvB#J!Uc;|~LnfQ9E_%ba-1mp?>UtJ@w|~KZ zW)Fra6G8aMH+S`Y)FTWLt)gj;1hegFPz8tH|!bXXnCMhT$SoueeaFy&DLij z?VaV2WT|;rYAR1MgUYJVkY3JdZ6EGp4R6@|;C=u2>3{gs4~1XE6gQq8_}%Xm3v$*i z)Q91KH=p}+%I*}`Hj=7ZH=Nd%&*Hv#3cdHN2WsF@#&3H6olF0odbO%8iZ<+Ek$4s> zt-PI5wJA181%qKJQ*r(K(2~2XMvB%@Z42Z4N(`i(Vdq|+N)jr>YP#N|cWItty3T!~ zrfpZGtihxAr-I&nZ#-9~*G*J-;rBIXidU$;{wX}+q-VjGf2eQ4BjNY&JzG3@W#58l z!-DEQaCwmss>QG{d+j-wmp|bou8W6S?@O)n`>pa~Bm_9Vwon>|qs1Q$7KiIXm)@7S({P@^KDPg+_1n18|>`) zL{T1OrN(9=b%TKP9%H1rJkbeAg_VnB(lKg^*Sgmby+!KxuI(-#!*9Ia+Bg4r$YNEA z8aDXj&`Ri85c`wlC2IJk@|oh5T~>7NBAow*aardHNlYVM87Z{KN|Oz1g^qlr9z;)T)k^)Qxoi;%Kh2|r8dk1!YAgI|;}$ts|Xp-ehGRZ_Vy{2KZKN>4?m ztS-~ay6((<)g{w8E*~YWan5dPug>j+Dwh;_W8@;?d2ck9m2L4fr7IiN)0acf@MiYF z!iDBNY4{OZ{GM<=315d*PXo*zQ5bJv?q08NNAV=XbI?7q>bV$i5cKpYTkr9STkSX* zqp_#;|6k!2%r>5JT0QVHf7WC9JlOKFv(xLTuG1y>K=l~c@PybH=J1oj>?$Rs$Iwr= zT3z#$*R4G{cHijFh7Ffd@4I$eZq3`Scy}Ohb?1={Q|?o_n$t&u>AJAG5Xt_Rx-JwPSpQj*Sba z7MB{$h56=c8h%W_?h}+l8v5#?9@^@$c4ZUNI{^X&2oNAZfB=CJ1!7A+zUns5*Npki z$(nmCwILp1J9n;p?^2f!dX`-fnIaEl%+{UX6;}7@^OfiRYs~X^n|{Ra@j5?hDZl$Y ze<7|dvM%vh>^HeIi{}y3@}7RJF;2@xx~^*Es%-Kd;w4#N_b1*SxKAiwADC%Z>!mEHcmF%C(^L+-jiWr>)`Qp?&NRp$+j>5d}DVx zmB0Udc+2*?E8p);H{5va)XY7{UVGh*H>WQ@GIQj}D~}((Yc(A0@O*P=wYf5vF0~Ke zdF#iH_oO$>&M%~^hi97$t7({O%s+P7zh#H% zJ$jP#hbOL|$TwQ+-_|?<0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNBU3w-Y_Kk>6qeeOrExw;QzqL|;EB$ILS!{3^F+iSo6k%vC}#<>sN z{fF=Vn^*k#=Z;@>^w%GF|GQrCw$C2<_NdIlpx0(YtSX!>vbezxCMK+IQjx#cM6RUR6wA9^SLhP3_2*#LpcLA3R)q@^IWZ zyzcmo;seE7`T5?Z<_HiVK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+0D(GzN3MUPvkSx#lkXwb!}pj009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk7nZ$xA9JZ009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5(bUEq=H-*nr<&)oLtb8B7%2oNAZfWXBrP_`J(o))Z diff --git a/backend/data_collection/economic.py b/backend/data_collection/economic.py index 29e8cce..9275b65 100644 --- a/backend/data_collection/economic.py +++ b/backend/data_collection/economic.py @@ -57,10 +57,10 @@ ["B23025_001E"], ), VarGroup( - "Unemployment Rate (16+)", + "Unemployment Rate", SL, + ["B23025_005E"], ["B23025_003E"], - ["B23025_002E"], ), VarGroup( "Prime-Age Labor Force Participation Rate (25-54)", @@ -77,6 +77,7 @@ "B23025_001E", "B23025_002E", "B23025_003E", + "B23025_005E", ], "B23001": _PRIME_TOTAL + _PRIME_IN_LF, "B19013": ["B19013_001E"], diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 7af3262..930e6b7 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -509,8 +509,8 @@ export const UnemploymentTrendChart = ({ single( chart, { - seriesKey: 'Unemployment Rate (16+)', - valueField: 'Value', + seriesKey: 'Unemployment Rate', + valueField: 'Percent', format: 'percent', decimals: 1, }, From 3d00d0ebd038bf935601f4b9a54ac104c1c47b3d Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Tue, 25 Aug 2026 14:14:30 -0400 Subject: [PATCH 20/26] fixed median earnings cleaning + api route --- .../api/routes/post_routes/post_acs5_db.py | 22 ++---- .../data_cleaning/clean_median_earnings.py | 78 +++++++++++++++++++ backend/query/acs5.py | 4 + .../src/components/Charts/TrendCharts.tsx | 12 ++- .../components/Charts/configs/ChartDefs.tsx | 2 +- 5 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 backend/data_cleaning/clean_median_earnings.py diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 970740a..395ab34 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -9,7 +9,6 @@ from query.acs5 import ( get_acs5_tidy, get_acs5_timeseries, - get_median_earnings_ts, ) from query.processed_db import DB @@ -133,13 +132,13 @@ async def get_per_capita_income(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("income")) -# Unemployment Rate -# @router.post("/load/acs5-db/timeseries/economics/unemployment-rate") -# async def get_unemployment_rate(request: FilterRequest): -# rows = get_acs5_timeseries( -# category="economics", dataset="unemployment_rate", filters=request.filters -# ) -# return make_response(data=rows, metadata=get_metadata("unemployment_rate")) +# Median Earnings (FIXME: broken) +@router.post("/load/acs5-db/timeseries/economics/median-earnings") +async def get_median_earnings(request: FilterRequest): + rows = get_acs5_timeseries( + category="economics", dataset="median_earnings", filters=request.filters + ) + return make_response(data=rows, metadata=get_metadata("income")) ##### HOUSING ##### @@ -170,13 +169,6 @@ async def get_vacancy_rates(request: FilterRequest): return make_response(data=rows, metadata=get_metadata("housing")) -# Median Earnings (FIXME: broken) -@router.post("/load/acs5-db/timeseries/median-earnings") -async def get_median_earnings(request: FilterRequest): - rows = get_median_earnings_ts(filters=request.filters) - return make_response(data=rows, metadata=get_metadata("median_earnings")) - - # Geography Snapshot Variables @router.post("/load/acs5-db/tidy/snapshot") async def tidy_snapshot(request: FilterRequest): diff --git a/backend/data_cleaning/clean_median_earnings.py b/backend/data_cleaning/clean_median_earnings.py new file mode 100644 index 0000000..033a83c --- /dev/null +++ b/backend/data_cleaning/clean_median_earnings.py @@ -0,0 +1,78 @@ +""" +**Author**: + Ian Sargent +**Created**: + 2026-08-25 +**Description**: + Data cleaning script for median earnings (Male, Female, All Workers). + Derived from the `RAW.acs5_economic` DuckLake table +**Run with**: +python -m data_cleaning.clean_median_earnings +""" + +import numpy as np +import pandas as pd + +from lake_build import con + + +def read_raw_data() -> pd.DataFrame: + raw_df = con.execute( + """--sql + SELECT year, NAME, Subcategory AS 'Variable', Value, geo_type + FROM lake.RAW.acs5_economic + WHERE Category LIKE '%INCOME AND BENEFITS ' || chr(40) || 'IN 2024%' + AND Subcategory IN ( + 'Median earnings for male full-time, year-round workers (dollars)', + 'Median earnings for female full-time, year-round workers (dollars)', + 'Median earnings for workers (dollars)' + ) + AND Variable = 'Total' + AND Measure = 'Estimate' + ORDER BY year; + """ + ).df() + + return raw_df + + +def change_dtype(df: pd.DataFrame): + df["Value"] = pd.to_numeric(df["Value"], errors="coerce") + + return df + + +def replace_unavailable_data(df: pd.DataFrame): + df["Value"] = df["Value"].replace(-666666666.0, np.nan) + + return df + + +def clean(): + raw_df = read_raw_data() + df = change_dtype(raw_df) + df = replace_unavailable_data(df) + + return df + + +def add_to_lake(clean_df: pd.DataFrame): + """ + Writes the cleaned, long-format health_insurance_coverage dataframe + to the CLEANED schema in DuckLake. + """ + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.acs5Economics_medianEarnings_timeseries AS + SELECT * FROM clean_df + """ + ) + + +def main(): + clean_df = clean() + add_to_lake(clean_df) + + +if __name__ == "__main__": + main() diff --git a/backend/query/acs5.py b/backend/query/acs5.py index b11b8b3..b826463 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -57,6 +57,10 @@ "table": "acs5Economics_perCapitaIncome_timeseries", "fixed_filters": {}, }, + "median_earnings": { + "table": "acs5Economics_medianEarnings_timeseries", + "fixed_filters": {}, + }, }, }, "labor_force": { diff --git a/frontend/src/components/Charts/TrendCharts.tsx b/frontend/src/components/Charts/TrendCharts.tsx index 930e6b7..82798f2 100644 --- a/frontend/src/components/Charts/TrendCharts.tsx +++ b/frontend/src/components/Charts/TrendCharts.tsx @@ -624,15 +624,21 @@ export const EarningsTrendChart = ({ series: [ { key: 'Male Full-Time Workers', - matchVariable: 'DP03_0093', + matchVariable: + 'Median earnings for male full-time, year-round workers (dollars)', color: '#161E54', }, { key: 'Female Full-Time Workers', - matchVariable: 'DP03_0094', + matchVariable: + 'Median earnings for female full-time, year-round workers (dollars)', color: '#F16D34', }, - { key: 'All Workers', matchVariable: 'DP03_0092', color: '#9BB0C1' }, + { + key: 'All Workers', + matchVariable: 'Median earnings for workers (dollars)', + color: '#9BB0C1', + }, ], }, view, diff --git a/frontend/src/components/Charts/configs/ChartDefs.tsx b/frontend/src/components/Charts/configs/ChartDefs.tsx index be93de3..89ef679 100644 --- a/frontend/src/components/Charts/configs/ChartDefs.tsx +++ b/frontend/src/components/Charts/configs/ChartDefs.tsx @@ -282,7 +282,7 @@ export const chartDefs: ChartDef[] = [ { id: 'earnings', title: 'Median Earnings - Value', - url: `${BASE_API_URL}/load/acs5-db/tidy/median-earnings`, + url: `${BASE_API_URL}/load/acs5-db/timeseries/economics/median-earnings`, xField: '', yField: '', subtype: 'renderTableEstimates', From 2eb20b00dd985a4db99c2c798caeecf1469a2880 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Wed, 26 Aug 2026 10:03:25 -0400 Subject: [PATCH 21/26] renamed db connection script --- .../api/routes/get_routes/get_wholedata.py | 2 +- .../api/routes/post_routes/post_acs5_db.py | 2 +- backend/api/routes/post_routes/post_census.py | 2 +- backend/api/routes/post_routes/post_qcew.py | 2 +- backend/app_utils/timeseries_db.py | 2 +- backend/query/__init__.py | 2 +- backend/query/acs5.py | 2 +- backend/query/cdc.py | 2 +- backend/query/core_functions.py | 2 +- .../{processed_db.py => production_db.py} | 31 +++---------------- backend/query/wastewater.py | 2 +- backend/query/zoning.py | 2 +- 12 files changed, 16 insertions(+), 37 deletions(-) rename backend/query/{processed_db.py => production_db.py} (50%) diff --git a/backend/api/routes/get_routes/get_wholedata.py b/backend/api/routes/get_routes/get_wholedata.py index f073bc5..011b9ea 100644 --- a/backend/api/routes/get_routes/get_wholedata.py +++ b/backend/api/routes/get_routes/get_wholedata.py @@ -5,7 +5,7 @@ from app_utils import data_loading from app_utils.flooding import add_flood_color -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) diff --git a/backend/api/routes/post_routes/post_acs5_db.py b/backend/api/routes/post_routes/post_acs5_db.py index 395ab34..baf2421 100644 --- a/backend/api/routes/post_routes/post_acs5_db.py +++ b/backend/api/routes/post_routes/post_acs5_db.py @@ -10,7 +10,7 @@ get_acs5_tidy, get_acs5_timeseries, ) -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) router = APIRouter() diff --git a/backend/api/routes/post_routes/post_census.py b/backend/api/routes/post_routes/post_census.py index 9bbaf3b..55bf7e2 100644 --- a/backend/api/routes/post_routes/post_census.py +++ b/backend/api/routes/post_routes/post_census.py @@ -7,7 +7,7 @@ from app_utils.df_filtering import ( filter_from_request, ) -from query.processed_db import DB +from query.production_db import DB router = APIRouter() diff --git a/backend/api/routes/post_routes/post_qcew.py b/backend/api/routes/post_routes/post_qcew.py index f28c553..db0a95c 100644 --- a/backend/api/routes/post_routes/post_qcew.py +++ b/backend/api/routes/post_routes/post_qcew.py @@ -3,7 +3,7 @@ from api.metadata_registry import get_metadata from api.models import FilterRequest, make_response -from query.processed_db import DB +from query.production_db import DB router = APIRouter() diff --git a/backend/app_utils/timeseries_db.py b/backend/app_utils/timeseries_db.py index 7d8df18..ef84d57 100644 --- a/backend/app_utils/timeseries_db.py +++ b/backend/app_utils/timeseries_db.py @@ -6,7 +6,7 @@ import logging -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) diff --git a/backend/query/__init__.py b/backend/query/__init__.py index fcb068b..b951332 100644 --- a/backend/query/__init__.py +++ b/backend/query/__init__.py @@ -2,7 +2,7 @@ from query.cdc import dual_var_comparison, get_cdc_county_pca, single_var_geojson from query.core_functions import filter_options, filter_tree -from query.processed_db import DB +from query.production_db import DB from query.wastewater import ( get_soil_suit_geojson, get_waste_service_areas_geojson, diff --git a/backend/query/acs5.py b/backend/query/acs5.py index b826463..c883e8c 100644 --- a/backend/query/acs5.py +++ b/backend/query/acs5.py @@ -15,7 +15,7 @@ from api.models import FilterSource, RangeFilter from app_utils.sql_render import sql_filter_block from query.core_functions import filter_tree -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) sql_path = Path(__file__).resolve().parent / "sql" / "acs5" diff --git a/backend/query/cdc.py b/backend/query/cdc.py index 8c33d66..1a29a2d 100644 --- a/backend/query/cdc.py +++ b/backend/query/cdc.py @@ -17,7 +17,7 @@ from api.models import FilterSource from app_utils.sql_render import compile_where, sql_filter_block -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) sql_dir = Path(__file__).resolve().parent / "sql" / "cdc" diff --git a/backend/query/core_functions.py b/backend/query/core_functions.py index fa568f9..4256f49 100644 --- a/backend/query/core_functions.py +++ b/backend/query/core_functions.py @@ -13,7 +13,7 @@ import logging from api.models import FilterResponse, RangeDescriptor -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) diff --git a/backend/query/processed_db.py b/backend/query/production_db.py similarity index 50% rename from backend/query/processed_db.py rename to backend/query/production_db.py index 8670326..27e4b60 100644 --- a/backend/query/processed_db.py +++ b/backend/query/production_db.py @@ -3,9 +3,12 @@ Fitz Koch **Created**: 2026-06-01 +**Updated**: + 2026-08-26 **Description**: - Pull in all the parquet files into active api. - Uses pre-run ETL from consolidate.py in the build section. + Establishes the DB connection to the finalized + database, `warehouse.duckdb`, which is derived + from the CLEANED DuckLake tables. """ import logging @@ -30,30 +33,6 @@ def _load_spatial(con: duckdb.DuckDBPyConnection) -> None: con.execute("LOAD spatial") -# def _build() -> duckdb.DuckDBPyConnection: -# con = duckdb.connect(":memory:") -# _load_spatial(con) - -# parquets = sorted(proc_dir.rglob("*.parquet")) -# if not parquets: -# logger.warning("No parquet files found under %s", proc_dir) - -# for path in parquets: -# name = f"{path.parent.name}_{path.stem}" -# con.execute(f"""--sql -# CREATE TABLE "{name}" AS SELECT * FROM read_parquet('{path}') -# """) -# logger.info("Loaded table %s from %s", name, path) -# return con - - -def _build() -> duckdb.DuckDBPyConnection: - path = Path(proc_dir / "all_data.duckdb") - con = duckdb.connect(path, read_only=True) - _load_spatial(con) - return con - - def _build_etl_db() -> duckdb.DuckDBPyConnection: path = Path(DATA_DIR / "warehouse.duckdb") con = duckdb.connect(path, read_only=True) diff --git a/backend/query/wastewater.py b/backend/query/wastewater.py index ade57a7..ec8c9df 100644 --- a/backend/query/wastewater.py +++ b/backend/query/wastewater.py @@ -14,7 +14,7 @@ from api.models import FilterSource from app_utils.sql_render import sql_filter_block -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) sql_dir = Path(__file__).resolve().parent / "sql" / "wastewater" diff --git a/backend/query/zoning.py b/backend/query/zoning.py index 2e85596..3febee7 100644 --- a/backend/query/zoning.py +++ b/backend/query/zoning.py @@ -16,7 +16,7 @@ from api.models import FilterSource from app_utils.sql_render import render_sql, sql_filter_block -from query.processed_db import DB +from query.production_db import DB logger = logging.getLogger(__name__) sql_dir = Path(__file__).resolve().parent / "sql" / "zoning" From 15e0eb0b2e8cd0bff1f4b3e327aef786c5ef4e63 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Wed, 26 Aug 2026 16:07:24 -0400 Subject: [PATCH 22/26] updating year generalizability. (no hardcoded years) --- backend/api/models/request_models.py | 5 ++++- backend/data_cleaning/clean_median_earnings.py | 8 ++++++-- backend/data_collection/acs5.py | 7 +++++-- backend/data_collection/base.py | 5 ++++- backend/data_collection/demographics.py | 8 ++++++-- backend/data_collection/economic.py | 8 ++++++-- backend/data_collection/education.py | 8 ++++++-- backend/data_collection/housing.py | 7 +++++-- backend/data_collection/qcew.py | 5 ++++- backend/run_data_collection.py | 7 +++++-- design/archive/notes/data_coverage.md | 4 ++-- .../app/data-comparison/dp-explorer/page.tsx | 4 +++- .../components/Charts/configs/ChartDefs.tsx | 6 ++++-- frontend/src/components/profile/SetProfile.tsx | 18 ++++++++++++++---- .../src/components/profile/profileStore.ts | 2 +- 15 files changed, 75 insertions(+), 27 deletions(-) diff --git a/backend/api/models/request_models.py b/backend/api/models/request_models.py index 0d46728..add7635 100644 --- a/backend/api/models/request_models.py +++ b/backend/api/models/request_models.py @@ -1,7 +1,10 @@ +from datetime import datetime from typing import Literal from pydantic import BaseModel, model_validator +MAX_YEAR = datetime.now().year - 2 + class RangeFilter(BaseModel): min: float | None = None @@ -30,7 +33,7 @@ class DPSeriesRequest(BaseModel): variable: str measure: str year_min: int = 2009 - year_max: int = 2024 + year_max: int = MAX_YEAR join_types = Literal["inner", "left", "spatial_intersect"] diff --git a/backend/data_cleaning/clean_median_earnings.py b/backend/data_cleaning/clean_median_earnings.py index 033a83c..c5b89ad 100644 --- a/backend/data_cleaning/clean_median_earnings.py +++ b/backend/data_cleaning/clean_median_earnings.py @@ -10,18 +10,22 @@ python -m data_cleaning.clean_median_earnings """ +from datetime import datetime + import numpy as np import pandas as pd from lake_build import con +INFLATION_YEAR = datetime.now().year - 2 + def read_raw_data() -> pd.DataFrame: raw_df = con.execute( - """--sql + f"""--sql SELECT year, NAME, Subcategory AS 'Variable', Value, geo_type FROM lake.RAW.acs5_economic - WHERE Category LIKE '%INCOME AND BENEFITS ' || chr(40) || 'IN 2024%' + WHERE Category LIKE '%INCOME AND BENEFITS ' || chr(40) || 'IN {INFLATION_YEAR}%' AND Subcategory IN ( 'Median earnings for male full-time, year-round workers (dollars)', 'Median earnings for female full-time, year-round workers (dollars)', diff --git a/backend/data_collection/acs5.py b/backend/data_collection/acs5.py index 1538451..60ed069 100644 --- a/backend/data_collection/acs5.py +++ b/backend/data_collection/acs5.py @@ -1,7 +1,7 @@ """ Fetch ACS 5-Year Data Profile tables (DP02-DP05) for Vermont Geographies: counties + county subdivisions + Vermont statewide + United States -Years: 2009-2024 +Years: 2009 - Latest published data Output: one wide CSV + parquet per table, plus tidy parquet per table Credit: Written largely by Claude, with some fine-tuning and troubleshooting by Fitz Koch @@ -13,6 +13,7 @@ import os import time +from datetime import datetime import pandas as pd import requests @@ -35,7 +36,9 @@ STORAGE_LOCATION = "Data/Census/ACS_5" ID_VARS = ["year", "geo_type", "table", "NAME", "state", "county"] -YEARS = range(2009, 2025) +MAX_YEAR = datetime.now().year - 1 + +YEARS = range(2009, MAX_YEAR) # Default geos list in (label, for_clause, in_clause) format GEOS = [(k, *v) for k, v in ALL_GEOS.items()] diff --git a/backend/data_collection/base.py b/backend/data_collection/base.py index 6fb229c..5e854be 100644 --- a/backend/data_collection/base.py +++ b/backend/data_collection/base.py @@ -24,6 +24,7 @@ import os import time from dataclasses import dataclass +from datetime import datetime import pandas as pd import requests @@ -51,6 +52,8 @@ GEOS = [(k, *v) for k, v in ALL_GEOS.items()] +MAX_YEAR = datetime.now().year - 2 + # --------------------------------------------------------------------------- # Data structures # --------------------------------------------------------------------------- @@ -151,7 +154,7 @@ def run_acs_b_scrape( fetch_specs: dict[str, list[str]], var_groups: list[VarGroup], output_filename: str, - year: int = 2024, + year: int = MAX_YEAR, geos: list = GEOS, append: bool = False, ) -> None: diff --git a/backend/data_collection/demographics.py b/backend/data_collection/demographics.py index 72d63b1..c8fef05 100644 --- a/backend/data_collection/demographics.py +++ b/backend/data_collection/demographics.py @@ -8,6 +8,8 @@ Output: vt_acs5_b_demographics_tidy.parquet """ +from datetime import datetime + import pandas as pd from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape @@ -27,7 +29,9 @@ ("75 Plus", range(23, 26), range(47, 50)), ] -YEARS = range(2009, 2025) +MAX_YEAR = datetime.now().year - 1 + +YEARS = range(2009, MAX_YEAR) def _b01001_codes(male_r, female_r): @@ -116,7 +120,7 @@ def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: p = argparse.ArgumentParser(description="Scrape ACS B-table demographics data.") p.add_argument("--start-year", type=int, default=2009) - p.add_argument("--end-year", type=int, default=2024) + p.add_argument("--end-year", type=int, default=MAX_YEAR - 1) p.add_argument( "--geos", nargs="+", diff --git a/backend/data_collection/economic.py b/backend/data_collection/economic.py index 9275b65..0e8cd6c 100644 --- a/backend/data_collection/economic.py +++ b/backend/data_collection/economic.py @@ -20,6 +20,8 @@ Output: vt_acs5_b_economic_tidy.parquet """ +from datetime import datetime + import pandas as pd from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape @@ -84,7 +86,9 @@ "B19301": ["B19301_001E"], } -YEARS = range(2009, 2025) +MAX_YEAR = datetime.now().year - 1 + +YEARS = range(2009, MAX_YEAR) def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: @@ -112,7 +116,7 @@ def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: p = argparse.ArgumentParser(description="Scrape ACS B-table economic data.") p.add_argument("--start-year", type=int, default=2009) - p.add_argument("--end-year", type=int, default=2024) + p.add_argument("--end-year", type=int, default=MAX_YEAR - 1) p.add_argument( "--geos", nargs="+", diff --git a/backend/data_collection/education.py b/backend/data_collection/education.py index 076abf2..4ceb6b1 100644 --- a/backend/data_collection/education.py +++ b/backend/data_collection/education.py @@ -12,6 +12,8 @@ Output: vt_acs5_b_education_tidy.parquet """ +from datetime import datetime + import pandas as pd from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape @@ -39,7 +41,9 @@ "B15003": [TOTAL] + [f"B15003_{str(i).zfill(3)}E" for i in range(2, 26)], } -YEARS = range(2009, 2025) +MAX_YEAR = datetime.now().year - 1 + +YEARS = range(2009, MAX_YEAR) def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: @@ -67,7 +71,7 @@ def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: p = argparse.ArgumentParser(description="Scrape ACS B-table education data.") p.add_argument("--start-year", type=int, default=2009) - p.add_argument("--end-year", type=int, default=2024) + p.add_argument("--end-year", type=int, default=MAX_YEAR - 1) p.add_argument( "--geos", nargs="+", diff --git a/backend/data_collection/housing.py b/backend/data_collection/housing.py index 108fad9..d5784cd 100644 --- a/backend/data_collection/housing.py +++ b/backend/data_collection/housing.py @@ -15,6 +15,8 @@ Output: vt_acs5_b_housing_tidy.parquet """ +from datetime import datetime + import pandas as pd from data_collection.base import ALL_GEOS, VarGroup, run_acs_b_scrape @@ -43,8 +45,9 @@ "B25077": ["B25077_001E"], } +MAX_YEAR = datetime.now().year - 1 -YEARS = range(2009, 2025) +YEARS = range(2009, MAX_YEAR) def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: @@ -72,7 +75,7 @@ def collect(years: range = YEARS, geos=None, append=False) -> pd.DataFrame: p = argparse.ArgumentParser(description="Scrape ACS B-table housing data.") p.add_argument("--start-year", type=int, default=2009) - p.add_argument("--end-year", type=int, default=2024) + p.add_argument("--end-year", type=int, default=MAX_YEAR - 1) p.add_argument( "--geos", nargs="+", diff --git a/backend/data_collection/qcew.py b/backend/data_collection/qcew.py index d34eb57..ef015ee 100644 --- a/backend/data_collection/qcew.py +++ b/backend/data_collection/qcew.py @@ -18,6 +18,7 @@ """ import time +from datetime import datetime from io import StringIO from pathlib import Path @@ -88,7 +89,9 @@ BASE_URL = "https://data.bls.gov/cew/data/api/{year}/{q}/area/{fips}.csv" QUARTERS = [1, 2, 3, 4] -YEARS = range(2009, 2025) +MAX_YEAR = datetime.now().year - 1 + +YEARS = range(2009, MAX_YEAR) # --------------------------------------------------------------------------- diff --git a/backend/run_data_collection.py b/backend/run_data_collection.py index f5f03df..7b9f239 100644 --- a/backend/run_data_collection.py +++ b/backend/run_data_collection.py @@ -12,6 +12,7 @@ """ import argparse +from datetime import datetime from data_collection import ( acs5, @@ -41,8 +42,10 @@ wastewater, zoning, ] +MAX_YEAR = datetime.now().year - 1 -YEARS = range(2009, 2025) + +YEARS = range(2009, MAX_YEAR) def run_scraper(scraper, yearly: bool = False, years: range = YEARS): @@ -76,7 +79,7 @@ def run_scraper(scraper, yearly: bool = False, years: range = YEARS): raise -def run_master_scrape(start_year: int = 2009, end_year: int = 2024): +def run_master_scrape(start_year: int = 2009, end_year: int = MAX_YEAR - 1): for scraper in YEARLY_SCRAPERS: run_scraper(scraper, yearly=True, years=range(start_year, end_year + 1)) diff --git a/design/archive/notes/data_coverage.md b/design/archive/notes/data_coverage.md index 9030bc7..3c193e7 100644 --- a/design/archive/notes/data_coverage.md +++ b/design/archive/notes/data_coverage.md @@ -4,7 +4,7 @@ The scrapers in `backend/data_collection/` pull from the **ACS 5-year estimates** via the Census API. The ACS 5-year program began with the 2005–2009 dataset, released in December 2010. **2009 is the earliest year available in this product** and is the hard floor for all longitudinal tables (demographics, education, housing, labor force, income). -Current scraper config: `YEARS = list(range(2009, 2025))` in `data_collection/base.py`. +Current scraper config: `YEARS = list(range(2009, MAX_YEAR))` in `data_collection/base.py`. Education data starts at 2012 in practice (earlier tables used different variable structures). @@ -21,7 +21,7 @@ The ACS replaced the decennial Census long-form starting with the 2010 cycle. Be | Period | Source | Resolution | Town-level? | | ---------------- | ----------------------------- | ----------- | ----------------------- | -| 2009–2024 | ACS 5-year estimates | Annual | Yes | +| 2009–Present | ACS 5-year estimates | Annual | Yes | | 2005–2008 | ACS 1-year estimates | Annual | No (≥65k pop only) | | 2000, 2010, 2020 | Decennial Census (short form) | Every 10 yr | Yes (limited variables) | | 1970–2000 | Decennial Census long-form | Every 10 yr | Partial | diff --git a/frontend/src/app/data-comparison/dp-explorer/page.tsx b/frontend/src/app/data-comparison/dp-explorer/page.tsx index 946527f..505f230 100644 --- a/frontend/src/app/data-comparison/dp-explorer/page.tsx +++ b/frontend/src/app/data-comparison/dp-explorer/page.tsx @@ -308,6 +308,8 @@ export default function DPExplorerPage() { // Available years derived from fetched data const [availableYears, setAvailableYears] = useState([]); + const YEAR_MAX_OVERALL = new Date().getFullYear() - 2; + useEffect(() => { if (!isComplete) { // eslint-disable-next-line react-hooks/set-state-in-effect -- clear stale results when the selection becomes incomplete @@ -329,7 +331,7 @@ export default function DPExplorerPage() { variable, measure, year_min: 2009, - year_max: 2024, + year_max: YEAR_MAX_OVERALL, }) .then((r) => r.data); diff --git a/frontend/src/components/Charts/configs/ChartDefs.tsx b/frontend/src/components/Charts/configs/ChartDefs.tsx index 89ef679..b3bd9a7 100644 --- a/frontend/src/components/Charts/configs/ChartDefs.tsx +++ b/frontend/src/components/Charts/configs/ChartDefs.tsx @@ -2,6 +2,8 @@ import { BASE_API_URL } from '@/config'; import { ChartParams, TableColumnConfig } from '@/types/cachedCharts'; +const YEAR_MAX_OVERALL = new Date().getFullYear() - 2; + export interface TableRowDef { label: string; variable: string; @@ -96,7 +98,7 @@ export const chartDefs: ChartDef[] = [ subtype: 'CompareDiffPerXBarChart', chartParams: { legendLabels: ['Main', 'Compare'], - fixedYear: 2024, // NOTE: Temporary fix to ensure chart shows most recent year (2024). + fixedYear: YEAR_MAX_OVERALL, percentFormat: true, includeCategories: [ 'Under 18', @@ -179,7 +181,7 @@ export const chartDefs: ChartDef[] = [ subtype: 'CompareDiffPerXBarChart', chartParams: { legendLabels: ['Main', 'Compare'], - fixedYear: 2024, // NOTE: Temporary fix to ensure chart shows most recent year (2024). + fixedYear: YEAR_MAX_OVERALL, percentFormat: true, }, url: `${BASE_API_URL}/load/acs5-db/tidy/education`, diff --git a/frontend/src/components/profile/SetProfile.tsx b/frontend/src/components/profile/SetProfile.tsx index cf64b15..aa132b0 100644 --- a/frontend/src/components/profile/SetProfile.tsx +++ b/frontend/src/components/profile/SetProfile.tsx @@ -194,10 +194,20 @@ const ProfileLocationSelect: React.FC = ({ ); }; -const YEAR_MARKS = [2009, 2012, 2015, 2018, 2021, 2024].map((y) => ({ - value: y, - label: String(y), -})); +const START_YEAR = 2009; +const YEAR_STEP = 3; +const LATEST_YEAR = new Date().getFullYear() - 2; + +const YEAR_MARKS = Array.from( + { length: Math.floor((LATEST_YEAR - START_YEAR) / YEAR_STEP) + 1 }, + (_, i) => { + const year = START_YEAR + i * YEAR_STEP; + return { + value: year, + label: String(year), + }; + }, +); export const ProfileModal: React.FC = () => { const { diff --git a/frontend/src/components/profile/profileStore.ts b/frontend/src/components/profile/profileStore.ts index 9cfc970..fad7d9d 100644 --- a/frontend/src/components/profile/profileStore.ts +++ b/frontend/src/components/profile/profileStore.ts @@ -21,7 +21,7 @@ export const INTEREST_OPTIONS = [ export type Interest = (typeof INTEREST_OPTIONS)[number]; export const YEAR_MIN_OVERALL = 2009; -export const YEAR_MAX_OVERALL = 2024; +export const YEAR_MAX_OVERALL = new Date().getFullYear() - 2; interface ProfileStore { myLocation: Location; From 8c3a88bff10ca15bc60f0d7205cd9873f9c7623c Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 27 Aug 2026 10:16:28 -0400 Subject: [PATCH 23/26] prettier changes --- frontend/.prettierignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/.prettierignore b/frontend/.prettierignore index 9408be1..a1b87ad 100644 --- a/frontend/.prettierignore +++ b/frontend/.prettierignore @@ -2,4 +2,5 @@ node_modules .next out dist -public \ No newline at end of file +public +next-env.d.ts \ No newline at end of file From b0bc49ba12668ddb4aaece59363e5208ba773942 Mon Sep 17 00:00:00 2001 From: Isarge05 Date: Thu, 27 Aug 2026 12:41:50 -0400 Subject: [PATCH 24/26] updated data pipeline documentation --- design/current/Data_Engineering.md | 78 ++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/design/current/Data_Engineering.md b/design/current/Data_Engineering.md index fb9283f..c52b4ee 100644 --- a/design/current/Data_Engineering.md +++ b/design/current/Data_Engineering.md @@ -15,17 +15,75 @@ - At some point we want to have an LLM that makes interacting with all this easier. Setting up default routes it can use to get data cleanly and easily will make it more efficient and reliable - Reference: [Tidy Data](https://vita.had.co.nz/papers/tidy-data.pdf). + # Overview of Steps -1. COLLECTION: data is collected, either in direct download or via an API or scrape. API is preferred. -2. BUILD: data is processed into a clean dataset and stored in SQL tables. - - how these decisions are arrived in light of the data should be well articulated in the corresponding `.qmd` in the notebooks folder, with ample code included. -3. QUERY and FILTER: - - queries are built for the data in SQL and wrapped up in python functions. This is how the data tables are manipulated to serve the precise data the frontend needs. - - Filtering is done in reference to `backend/api/schema.json` - - see the [schema](#schema) section below for an explanation of fields - - see `backend/api/routes/get_routes/get_filters.py` and `backend/api/routes/get_routes/get_filters.py` for how those fields are used in practice. -4. API: thin wrapper of fastapi stuff around the queries. +0. **LAKE CREATION** If the DuckLake is not yet instanciated, the `just build-lake` justfile recipe will create the DuckLake instance, install the spatial extension, and establish both the `RAW` and `CLEANED` table schemas. + + * Files called upon: [`backend/lake_build.py`](../../backend/lake_build.py) + +1. **COLLECTION:** Data is collected from external sources within the [`backend/data_collection/`](../../backend/data_collection/) folder, with APIs preferred whenever available. Direct downloads and locally stored tables are used when an API is unavailable or does not provide the required data. Raw data is loaded into the DuckLake `RAW` schema with minimal transformations so that the original source data is preserved. + + * The collection process is orchestrated using the `just get-data {start_year} {end_year}` justfile recipe, which collects data given the specified year range (inclusive). + * Files called upon: [`backend/run_data_collection.py`](../../backend/run_data_collection.py) + * Separate data file collectors live within [`backend/data_collection/`](../../backend/data_collection/) + * [`acs5.py`](../../backend/data_collection/acs5.py)