diff --git a/.gitignore b/.gitignore index 89b5f19c..3342ceb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -## Build files +## Build files */Data/_Processed/ # random stuff that's too big or unneeded to push @@ -55,5 +55,10 @@ backend/logger/*.log out/ dist/ backend/Data/Census/**/*.csv +backend/Data/_Processed/parcels/ +**/.jupyter_cache/ +backend/notebooks/**/*.html +backend/notebooks/**/*_files/ +backend/notebooks/**/*.ipynb tmp/ diff --git a/backend/api/routes/post_routes/__init__.py b/backend/api/routes/post_routes/__init__.py index 321b9823..51775230 100644 --- a/backend/api/routes/post_routes/__init__.py +++ b/backend/api/routes/post_routes/__init__.py @@ -2,6 +2,7 @@ from .post_cdc import router as post_cdc_router from .post_census import router as post_census_router from .post_export import router as post_export_router +from .post_parcels import router as post_parcels_router from .post_qcew import router as post_qcew_router from .post_zoning import router as post_zoning_router from .post_wastewater import router as post_wastewater_router @@ -14,4 +15,5 @@ post_wastewater_router, post_export_router, post_cdc_router, + post_parcels_router, ] diff --git a/backend/api/routes/post_routes/post_parcels.py b/backend/api/routes/post_routes/post_parcels.py new file mode 100644 index 00000000..0d837e26 --- /dev/null +++ b/backend/api/routes/post_routes/post_parcels.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter, Response + +from api.core_functions import request_to_source +from api.models import FilterRequest +from query.parcels import get_parcels_geojson + +router = APIRouter() + + +@router.post("/load/mapping/parcels") +async def read_parcels(request: FilterRequest): + source = request_to_source(request, "parcels_info", "default") + data = get_parcels_geojson([source]) + return Response(content=data, media_type="application/json") diff --git a/backend/api/schema.json b/backend/api/schema.json index e019a8e3..a2e5d715 100644 --- a/backend/api/schema.json +++ b/backend/api/schema.json @@ -47,6 +47,13 @@ "Prevalence Measure": "Data_Value_Type" } }, + "parcels_info": { + "join_key": "OBJECTID", + "join_type": "inner", + "columns": { + "Town": "TOWN", + "Category": "CAT", + "Property Type": "PROPTYPE", "soil_suitability_info_soil_suit": { "join_key": "ID", "join_type": "inner", @@ -87,4 +94,4 @@ } } } -} +}}} diff --git a/backend/build/main.py b/backend/build/main.py index db6b7b85..199fce48 100644 --- a/backend/build/main.py +++ b/backend/build/main.py @@ -7,7 +7,7 @@ Runs all the build scripts one after the other. """ -from build import FIPS_data, acs5, cdc, wastewater, zoning +from build import FIPS_data, acs5, cdc, parcels, wastewater, zoning def main(): @@ -17,6 +17,7 @@ def main(): # information here" layer subtracts the districts from -- so it runs first. FIPS_data.main() zoning.main() + parcels.main() wastewater.main() diff --git a/backend/build/parcels.py b/backend/build/parcels.py index e69de29b..8631f0b2 100644 --- a/backend/build/parcels.py +++ b/backend/build/parcels.py @@ -0,0 +1,94 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-07-23 +**Description**: + Build script to convert the CDC-places csv data into a single vermont-based SQL table. +""" + +from pathlib import Path + +from build import BACKEND, CON + +data_directory = Path("Data") +parcels_fgb = Path("Data/parcels/parcels_f.fgb") +parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + + +proc_dir = BACKEND / "Data" / "_Processed" / "parcels" +TABLES = ["geom", "info"] +INFO_COLS = [ + "OBJECTID", + "SPAN", + "CAT", + "RESCODE", + "PARCID", + "CITYGL", + "TOWN", + "ZIPGL", + "PROPTYPE", + "DESCPROP", + "SOURCENAME", + "YEAR", + "ACRESGL", + "REAL_FLV", + "HSTED_FLV", + "IMPRV_LV", +] + + +def load_dataset(): + if not parcels_fgb.exists(): + CON.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') + """) + CON.execute( + "CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/Parcels/parcels_f.fgb')" + ) + + CON.execute(""" + CREATE OR REPLACE VIEW final_view AS + SELECT + * EXCLUDE (TOWN), + UPPER(TOWN) AS TOWN, + FROM parcels_raw; + """) + + +def load_geom(): + CON.execute(""" + CREATE OR REPLACE VIEW geom AS + SELECT OBJECTID, geom FROM final_view + """) + + +def load_info(): + CON.execute("""CREATE OR REPLACE VIEW info AS + SELECT OBJECTID, SPAN, CAT, RESCODE, PARCID, CITYGL, TOWN, ZIPGL, PROPTYPE, DESCPROP, SOURCENAME, YEAR, + ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV + FROM final_view""") + + +def final_writing(): + path = Path("Data/_Processed/parcels") + path.mkdir(parents=True, exist_ok=True) + for item in ["info", "geom"]: + CON.execute( + f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)" + ) + + +def main(): + load_dataset() + load_geom() + load_info() + final_writing() + + +if __name__ == "__main__": + main() diff --git a/backend/data_cleaning/clean_parcels.py b/backend/data_cleaning/clean_parcels.py new file mode 100644 index 00000000..740e2b0d --- /dev/null +++ b/backend/data_cleaning/clean_parcels.py @@ -0,0 +1,1048 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-8-24 +**Description**: + Data cleaning script for the raw `parcels` table in the DuckLake + Run with: +python -m ETL.data_cleaning.clean_parcels +""" + +from pathlib import Path + +import geopandas as gpd +import numpy as np +import pandas as pd +from datastore.lake_build import con +from shapely.geometry import MultiPolygon + + +## LOAD SPATIAL EXTENSION FUNCTION -------------------- +def _load_spatial() -> None: + """ + Load the spatial extension, installing it first if necessary. + """ + try: + con.execute("""--sql LOAD spatial""") + except Exception: + con.execute("""--sql INSTALL spatial""") + con.execute("""--sql LOAD spatial""") + + +def make_tables(parcels): + out = Path("Data/parcels/tables") + out.mkdir(parents=True, exist_ok=True) + + def present(cols): + return [c for c in cols if c in parcels.columns] + + geom_cols = present(["OBJECTID", "TOWN", "COUNTY", "geometry"]) + info_cols = present( + [ + "OBJECTID", + "TOWN", + "COUNTY", + "SPAN", + "PROPTYPE", + "CAT", + "CATEGORY", + "PURPOSE", + "DESCPROP", + "RESCODE", + "ACRESGL", + "AREAACRESGEOM", + "CITYGL", + "STGL", + "ADDRESS", + "SOURCENAME", + "MATCHSTAT", + "TNAME", + "INVESTMENTPROP", + "VACANTLAND", + "OOSOWNER", + "data_origin", + "EDITOR", + "EDITDATE", + ] + ) + tax_cols = present( + [ + "OBJECTID", + "TOWN", + "REAL_FLV", + "HSTED_FLV", + "NRES_FLV", + "LAND_LV", + "IMPRV_LV", + "IMPR_SHARE", + "EQUIPVAL", + "EQUIPCODE", + "INVENVAL", + "HSDECL", + "VETEXAMT", + "EXPDESC", + "STATUTE", + "EXEMPT", + "EXAMT_HS", + "EXAMT_NR", + "UVREDUC_HS", + "UVREDUC_NR", + "GLVAL_HS", + "GLVAL_NR", + ] + ) + + geom = parcels[geom_cols].copy() + info = parcels[info_cols].copy() + tax = parcels[tax_cols].copy() + + geom.to_parquet(out / "parcels_geom.parquet") + info.to_parquet(out / "parcels_info.parquet") + tax.to_parquet(out / "parcels_tax.parquet") + # edit this for later - where should I store them + g = gpd.read_parquet(out / "parcels_geom.parquet") + i = pd.read_parquet(out / "parcels_info.parquet") + t = pd.read_parquet(out / "parcels_tax.parquet") + + +def register_parcels(gdf): + df = pd.DataFrame( + gdf.to_crs(4326).assign(geometry=gdf.to_crs(4326).geometry.to_wkb()) + ) + con.register("parcels_raw", df) + # returning the gdf for the make tables + + +def build_parcels(): + con.execute("""--sql + CREATE OR REPLACE VIEW parcels AS + SELECT ST_GeomFromWKB(geometry) AS geometry, OBJECTID, TOWN, COUNTY, SPAN, PROPTYPE, CAT, CATEGORY, PURPOSE, DESCPROP, RESCODE, ACRESGL, AREAACRESGEOM, CITYGL, STGL, ADDRESS, SOURCENAME, MATCHSTAT, TNAME, INVESTMENTPROP, VACANTLAND, OOSOWNER, data_origin, EDITOR, EDITDATE, REAL_FLV, HSTED_FLV, NRES_FLV, LAND_LV, IMPRV_LV, IMPR_SHARE, EQUIPVAL, EQUIPCODE, INVENVAL, HSDECL, VETEXAMT, EXPDESC, STATUTE, EXEMPT, EXAMT_HS, EXAMT_NR, UVREDUC_HS, UVREDUC_NR, GLVAL_HS, GLVAL_NR + FROM parcels_raw + """) + + +def add_to_lake(): + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_geom AS + SELECT * + FROM geom + """ + ) + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_info AS + SELECT * + FROM info + """ + ) + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_tax AS + SELECT * + FROM tax + """ + ) + + +def clean(): + _load_spatial() + gdf = load_helper_and_clean() + register_parcels(gdf) + build_parcels() + return gdf + + +def main(): + make_tables(clean()) + add_to_lake() + + +def load_helper_and_clean(backfill): + vermont_towns_from_parcels = [ + "Addison", + "Albany", + "Alburgh", + "Andover", + "Arlington", + "Athens", + "Averill", + "Averys Gore", + "Bakersfield", + "Baltimore", + "Barnard", + "Barnet", + "Barre City", + "Barre Town", + "Barton", + "Belvidere", + "Bennington", + "Benson", + "Berkshire", + "Berlin", + "Bethel", + "Bloomfield", + "Bolton", + "Bradford", + "Braintree", + "Brandon", + "Brattleboro", + "Bridgewater", + "Bridport", + "Brighton", + "Bristol", + "Brookfield", + "Brookline", + "Brownington", + "Brunswick", + "Buels Gore", + "Burke", + "Burlington", + "Cabot", + "Calais", + "Cambridge", + "Canaan", + "Castleton", + "Cavendish", + "Charleston", + "Charlotte", + "Chelsea", + "Chester", + "Chittenden", + "Clarendon", + "Colchester", + "Concord", + "Corinth", + "Cornwall", + "Coventry", + "Craftsbury", + "Danby", + "Danville", + "Derby", + "Dorset", + "Dover", + "Dummerston", + "Duxbury", + "East Haven", + "East Montpelier", + "Eden", + "Elmore", + "Enosburgh", + "Essex Junction", + "Essex", + "Fair Haven", + "Fairfax", + "Fairfield", + "Fairlee", + "Fayston", + "Ferdinand", + "Ferrisburgh", + "Fletcher", + "Franklin", + "Georgia", + "Glastenbury", + "Glover", + "Goshen", + "Grafton", + "Granby", + "Grand Isle", + "Granville", + "Greensboro", + "Groton", + "Guildhall", + "Guilford", + "Halifax", + "Hancock", + "Hardwick", + "Hartford", + "Hartland", + "Highgate", + "Hinesburg", + "Holland", + "Hubbardton", + "Huntington", + "Hyde Park", + "Ira", + "Irasburg", + "Isle La Motte", + "Jamaica", + "Jay", + "Jericho", + "Johnson", + "Killington", + "Kirby", + "Landgrove", + "Leicester", + "Lemington", + "Lewis", + "Lincoln", + "Londonderry", + "Lowell", + "Ludlow", + "Lunenburg", + "Lyndon", + "Maidstone", + "Manchester", + "Marlboro", + "Marshfield", + "Mendon", + "Middlebury", + "Middlesex", + "Middletown Springs", + "Milton", + "Monkton", + "Montgomery", + "Montpelier", + "Moretown", + "Morgan", + "Morristown", + "Mount Holly", + "Mount Tabor", + "New Haven", + "Newark", + "Newbury", + "Newfane", + "Newport City", + "Newport Town", + "North Hero", + "Northfield", + "Norton", + "Norwich", + "Orange", + "Orwell", + "Panton", + "Pawlet", + "Peacham", + "Peru", + "Pittsfield", + "Pittsford", + "Plainfield", + "Plymouth", + "Pomfret", + "Poultney", + "Pownal", + "Proctor", + "Putney", + "Randolph", + "Reading", + "Readsboro", + "Richford", + "Richmond", + "Ripton", + "Rochester", + "Rockingham", + "Roxbury", + "Royalton", + "Rupert", + "Rutland City", + "Rutland Town", + "Ryegate", + "Saint Albans City", + "Saint Albans Town", + "Saint George", + "Saint Johnsbury", + "Salisbury", + "Sandgate", + "Searsburg", + "Shaftsbury", + "Sharon", + "Sheffield", + "Shelburne", + "Sheldon", + "Shoreham", + "Shrewsbury", + "Somerset", + "South Burlington", + "South Hero", + "Springfield", + "Stamford", + "Stannard", + "Starksboro", + "Stockbridge", + "Stowe", + "Strafford", + "Stratton", + "Sudbury", + "Sunderland", + "Sutton", + "Swanton", + "Thetford", + "Tinmouth", + "Topsham", + "Townshend", + "Troy", + "Tunbridge", + "Underhill", + "Vergennes", + "Vernon", + "Vershire", + "Victory", + "Waitsfield", + "Walden", + "Wallingford", + "Waltham", + "Wardsboro", + "Warners Grant", + "Warren Gore", + "Warren", + "Washington", + "Waterbury", + "Waterford", + "Waterville", + "Weathersfield", + "Wells", + "West Fairlee", + "West Haven", + "West Rutland", + "West Windsor", + "Westfield", + "Westford", + "Westminster", + "Westmore", + "Weston", + "Weybridge", + "Wheelock", + "Whiting", + "Whitingham", + "Williamstown", + "Williston", + "Wilmington", + "Windham", + "Windsor", + "Winhall", + "Winooski", + "Wolcott", + "Woodbury", + "Woodford", + "Woodstock", + "Worcester", + ] + + ADDISON = [ + "ADDISON", + "BRIDPORT", + "BRISTOL", + "CORNWALL", + "FERRISBURGH", + "GOSHEN", + "GRANVILLE", + "HANCOCK", + "LEICESTER", + "LINCOLN", + "MIDDLEBURY", + "MONKTON", + "NEW HAVEN", + "ORWELL", + "PANTON", + "RIPTON", + "SALISBURY", + "SHOREHAM", + "STARKSBORO", + "WALTHAM", + "WEYBRIDGE", + "WHITING", + "VERGENNES", + ] + BENNINGTON = [ + "ARLINGTON", + "BENNINGTON", + "DORSET", + "GLASTENBURY", + "LANDGROVE", + "MANCHESTER", + "PERU", + "POWNAL", + "READSBORO", + "RUPERT", + "SANDGATE", + "SEARSBURG", + "SHAFTSBURY", + "STAMFORD", + "SUNDERLAND", + "WINHALL", + "WOODFORD", + ] + CALEDONIA = [ + "BARNET", + "BURKE", + "DANVILLE", + "GROTON", + "HARDWICK", + "KIRBY", + "LYNDON", + "NEWARK", + "PEACHAM", + "RYEGATE", + "SAINT JOHNSBURY", + "SHEFFIELD", + "STANNARD", + "SUTTON", + "WALDEN", + "WATERFORD", + "WHEELOCK", + ] + CHITTENDEN = [ + "BOLTON", + "CHARLOTTE", + "COLCHESTER", + "ESSEX", + "ESSEX JUNCTION", + "HINESBURG", + "HUNTINGTON", + "JERICHO", + "MILTON", + "RICHMOND", + "SAINT GEORGE", + "SHELBURNE", + "UNDERHILL", + "WESTFORD", + "WILLISTON", + "BURLINGTON", + "WINOOSKI", + "BUELS GORE", + "SOUTH BURLINGTON", + ] + ESSEX = [ + "AVERILL", + "BLOOMFIELD", + "BRIGHTON", + "BRUNSWICK", + "CANAAN", + "CONCORD", + "EAST HAVEN", + "FERDINAND", + "GRANBY", + "GUILDHALL", + "LEMINGTON", + "LEWIS", + "LUNENBURG", + "MAIDSTONE", + "NORTON", + "VICTORY", + "AVERYS GORE", + "WARNERS GRANT", + "WARREN GORE", + ] + FRANKLIN = [ + "BAKERSFIELD", + "BERKSHIRE", + "ENOSBURGH", + "FAIRFAX", + "FAIRFIELD", + "FLETCHER", + "FRANKLIN", + "GEORGIA", + "HIGHGATE", + "MONTGOMERY", + "RICHFORD", + "SAINT ALBANS CITY", + "SAINT ALBANS TOWN", + "SHELDON", + "SWANTON", + ] + GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] + LAMOILLE = [ + "BELVIDERE", + "CAMBRIDGE", + "EDEN", + "ELMORE", + "HYDE PARK", + "JOHNSON", + "MORRISTOWN", + "STOWE", + "WATERVILLE", + "WOLCOTT", + ] + ORANGE = [ + "BRADFORD", + "BRAINTREE", + "BROOKFIELD", + "CHELSEA", + "CORINTH", + "FAIRLEE", + "NEWBURY", + "ORANGE", + "RANDOLPH", + "STRAFFORD", + "THETFORD", + "TOPSHAM", + "TUNBRIDGE", + "VERSHIRE", + "WASHINGTON", + "WEST FAIRLEE", + "WILLIAMSTOWN", + ] + ORLEANS = [ + "ALBANY", + "BARTON", + "BROWNINGTON", + "CHARLESTON", + "COVENTRY", + "CRAFTSBURY", + "DERBY", + "GLOVER", + "GREENSBORO", + "HOLLAND", + "IRASBURG", + "JAY", + "LOWELL", + "MORGAN", + "NEWPORT CITY", + "NEWPORT TOWN", + "TROY", + "WESTFIELD", + "WESTMORE", + ] + RUTLAND = [ + "BENSON", + "BRANDON", + "CASTLETON", + "CHITTENDEN", + "CLARENDON", + "DANBY", + "FAIR HAVEN", + "HUBBARDTON", + "IRA", + "KILLINGTON", + "MENDON", + "MIDDLETOWN SPRINGS", + "MOUNT HOLLY", + "MOUNT TABOR", + "PAWLET", + "PITTSFIELD", + "PITTSFORD", + "POULTNEY", + "PROCTOR", + "RUTLAND TOWN", + "RUTLAND CITY", + "SHREWSBURY", + "SUDBURY", + "TINMOUTH", + "WALLINGFORD", + "WELLS", + "WEST HAVEN", + "WEST RUTLAND", + ] + WASHINGTON = [ + "BARRE TOWN", + "BERLIN", + "CABOT", + "CALAIS", + "DUXBURY", + "EAST MONTPELIER", + "FAYSTON", + "MARSHFIELD", + "MIDDLESEX", + "MORETOWN", + "NORTHFIELD", + "PLAINFIELD", + "ROXBURY", + "WAITSFIELD", + "WARREN", + "WATERBURY", + "WOODBURY", + "WORCESTER", + "BARRE CITY", + "MONTPELIER", + ] + WINDHAM = [ + "ATHENS", + "BRATTLEBORO", + "BROOKLINE", + "DOVER", + "DUMMERSTON", + "GRAFTON", + "GUILFORD", + "HALIFAX", + "JAMAICA", + "LONDONDERRY", + "MARLBORO", + "NEWFANE", + "PUTNEY", + "ROCKINGHAM", + "SOMERSET", + "STRATTON", + "TOWNSHEND", + "VERNON", + "WARDSBORO", + "WESTMINSTER", + "WHITINGHAM", + "WILMINGTON", + "WINDHAM", + ] + WINDSOR = [ + "ANDOVER", + "BALTIMORE", + "BARNARD", + "BETHEL", + "BRIDGEWATER", + "CAVENDISH", + "CHESTER", + "HARTFORD", + "HARTLAND", + "LUDLOW", + "NORWICH", + "PLYMOUTH", + "POMFRET", + "READING", + "ROCHESTER", + "ROYALTON", + "SHARON", + "SPRINGFIELD", + "STOCKBRIDGE", + "WEATHERSFIELD", + "WEST WINDSOR", + "WESTON", + "WINDSOR", + "WOODSTOCK", + ] + counties_key = [ + ADDISON, + BENNINGTON, + CALEDONIA, + CHITTENDEN, + ESSEX, + FRANKLIN, + GRAND_ISLE, + LAMOILLE, + ORANGE, + ORLEANS, + RUTLAND, + WASHINGTON, + WINDHAM, + WINDSOR, + ] + names_key = [ + "ADDISON", + "BENNINGTON", + "CALEDONIA", + "CHITTENDEN", + "ESSEX", + "FRANKLIN", + "GRAND_ISLE", + "LAMOILLE", + "ORANGE", + "ORLEANS", + "RUTLAND", + "WASHINGTON", + "WINDHAM", + "WINDSOR", + ] + + statutes_dict = { + "3848:3849": "Business Inventory & Equipment", + "3848:38:00": "Business Inventory & Equipment", + "3840": "Charitable, Fraternal, or Rescue", + "3840;5405a(a)(4)": "Charitable/Rescue (inc. Education Tax)", + "3840;54": "Charitable/Rescue (inc. Education Tax)", + "2741": "Tax Stabilization Contract", + "24/2741": "Tax Stabilization Contract", + "3832": "Public, Pious, or Charitable", + "3832(1)": "Out-of-Town Municipal Property", + "3832(7)": "Health or Recreational Property", + "3832(7)(B)": "Non-profit Ice Skating Rink", + "3832(7B": "Non-profit Ice Skating Rink", + "5401": "Statewide Education Tax Exception", + "3752(7)": "Agricultural / Current Use", + } + + expdesc_dict = { + "Statutory": "State Law Exemption", + "Solar Plant": "Solar Energy Facility", + "Non-Approved (Voted)": "Local Town-Voted Exemption", + "Qualified Housing Units": "Affordable / Qualified Housing", + "Grandfathered": "Pre-existing Historical Exemption", + "Partial-Statutory": "Partial State Law Exemption", + "Municipal Contract (Owner Pays)": "Payment in Lieu of Taxes (PILOT)", + "Ski Lifts / Snow Making Equip": "Ski Resort Equipment", + "Court Ordered": "Judicially Mandated Exemption", + "Wind Plant": "Wind Energy Facility", + } + + rescode_dict = { + "T": "TOWN RESIDENT", + "NS": "OUT OF STATE RESIDENT", + "S": "VERMONT RESIDENT", + "C": "CORPORATION/ENTITY", + "c": "CORPORATION/ENTITY", + } + + cat_dict = { + "R1": "Residential I (Under 6 Acres)", + "R2": "Residential II (6 Acres or More)", + "M": "Miscellaneous", + "O": "Other", + "C": "Commercial", + "MHL": "Mobile Home Landed (With Land)", + "S1": "Seasonal I (Under 6 Acres)", + "MHU": "Mobile Home Unlanded (Without Land)", + "W": "Woodland", + "S2": "Seasonal II (6 Acres or More)", + "F": "Farm", + "CA": "Commercial Apartments", + "I": "Industrial", + "UE": "Utility Electric", + "UO": "Utility Other", + } + + purpose_dict = { + "R1": "PRIMARY RESIDENCE", + "R2": "PRIMARY RESIDENCE", + "MHL": "PRIMARY RESIDENCE", + "MHU": "PRIMARY RESIDENCE", + "S1": "SEASONAL PROPERTY", + "S2": "SEASONAL PROPERTY", + "W": "WOODLAND", + "F": "FARM", + "CA": "COMMERCIAL APARTMENTS", + "M": "NOT LISTED", + "O": "NOT LISTED", + "C": "COMMERCIAL/INDUSTRIAL/UTILITY", + "I": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UE": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UO": "COMMERCIAL/INDUSTRIAL/UTILITY", + } + + states = [ + "AL", + "AK", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "HI", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VA", + "WA", + "WV", + "WI", + "WY", + "VT", + "DC", + ] + + oos_dict = { + "VERMONT": "VT", + } + oos_dict.update( + dict.fromkeys( + [ + "QC", + "QC CANADA", + "PQ", + "QUEBEC", + "ON", + "ONTARIO", + "QUE", + "BC", + "ONT", + "CAN", + ], + "CANADA", + ) + ) + oos_dict.update(dict.fromkeys(["MASS", "MA."], "MA")) + oos_dict.update(dict.fromkeys(["MICHIGAN"], "MI")) + oos_dict.update(dict.fromkeys(["OHIO"], "OH")) + oos_dict.update(dict.fromkeys(["CT."], "CT")) + oos_dict.update(dict.fromkeys(["R.I."], "RI")) + oos_dict.update(dict.fromkeys(["W VA"], "WV")) + oos_dict.update(dict.fromkeys(["MARYLAND"], "MD")) + oos_dict.update(dict.fromkeys(["N CAROLINA"], "NC")) + oos_dict.update(dict.fromkeys(["NEW YORK", "N.Y.", "12513", "N Y"], "NY")) + oos_dict.update(dict.fromkeys(["FLORIDA", "FLA"], "FL")) + oos_dict.update( + dict.fromkeys( + [ + "ENGLAND", + "AE", + "UNK", + "BERMUDA", + "UK", + "VY", + "FRANCE", + "IND", + "ARUBA", + "GERMANY", + "IRELAND", + "SWITZERLAN", + "QLD AUS", + "LIN", + "0R", + "BERLIN", + "AUSTRALIA", + "NS", + "FWI", + "BAHAMAS", + ], + "FOREIGN", + ) + ) + oos_dict.update(dict.fromkeys(["VI", "PR", "GUAM"], "US TERRITORY")) + + counties_dict = {} + unknown = [] + for town in backfill["TOWN"].unique(): + for item in counties_key: + if town in item: + counties_dict[town] = names_key[(counties_key.index(item))] + # adding in sourcename as city and town + city_town_dict = {} + + city_source = backfill[ + (backfill["SOURCENAME"] == "CITY") | (backfill["SOURCENAME"] == "TOWN") + ] + + # finding if the parcel was found from a local department or not + backfill["SOURCENAME"] = np.where( + backfill["SOURCENAME"].isin(["CITY", "TOWN", "City of Burlington"]), + "LOCAL DEPARTMENT", + "NOT LOCAL DEPARTMENT", + ) + + # this code aids in the creation of a column that sees if the primary people live there, or if it is an investment property. + residential_codes = ["R1", "R2", "MHL", "MHU"] + + is_residential = backfill["CAT"].isin(residential_codes) + is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == "N") + + vt_mask = ( + backfill["STGL"].str.startswith("VT", na=False) + | backfill["STGL"].str.startswith("Vt", na=False) + | backfill["STGL"].str.startswith("05", na=False) + | backfill["STGL"].str.contains("VT", na=False) + | backfill["STGL"].str.contains("V T", na=False) + | backfill["STGL"].str.startswith("vt", na=False) + ) + backfill.loc[vt_mask, "STGL"] = "VERMONT" + canada_mask = ( + backfill["STGL"].str.contains("CANADA", na=False) + | backfill["STGL"].str.contains("Canada", na=False) + | backfill["STGL"].str.contains("QC", na=False) + ) + backfill.loc[canada_mask, "STGL"] = "CANADA" + + # mapping new items in columns + backfill["INVESTMENTPROP"] = is_residential & is_not_homestead + + backfill["VACANTLAND"] = (backfill["LAND_LV"] > 0) & ( + backfill["IMPRV_LV"].fillna(0) == 0 + ) + equipcode_dict = {"E": "ELECTRIC UTILITY", "C": "CABLE UTILITY"} + backfill["EXEMPT"] = backfill["STATUTE"].notna().map({True: "YES", False: "NO"}) + backfill["STATUTE"] = backfill["STATUTE"].map(statutes_dict).fillna("No Exemption") + backfill["EXPDESC"] = backfill["EXPDESC"].map(expdesc_dict).fillna("None") + backfill["EQUIPCODE"] = ( + backfill["EQUIPCODE"].map(equipcode_dict).fillna("NOT A UTILITY") + ) + # creating a foreign ownerpship column + + backfill["COUNTY"] = backfill["TOWN"].map(counties_dict) + backfill["RESCODE"] = backfill["RESCODE"].map(rescode_dict) + backfill["CATEGORY"] = backfill["CAT"].map(cat_dict) + backfill["PURPOSE"] = backfill["CAT"].map(purpose_dict) + # adding state abbreivaitions and owner lcoations + backfill["STGL"] = backfill["STGL"].replace(oos_dict) + catch_all_mask = ( + ~backfill["STGL"].isin(["CANADA", "FOREIGN", "US TERRITORY"]) + & backfill["STGL"].notna() + & ~backfill["STGL"].isin(states) + ) + backfill.loc[catch_all_mask, "STGL"] = "UNNAMED AMERICA" + backfill["OOSOWNER"] = backfill["STGL"].fillna("VT") != "VT" + + backfill["ADDRESS"] = backfill["E911ADDR"] + + # adding an area in acres to geometry column, a + if "AREAACRESGEOM" not in backfill: + backfill["AREAACRESGEOM"] = backfill.to_crs(32145).geometry.area / 4046.8564224 + backfill["ACREVALUE"] = np.where( + (backfill.REAL_FLV > 0) & (backfill.AREAACRESGEOM > 0), + backfill.REAL_FLV / backfill.AREAACRESGEOM, + np.nan, + ) + + residential_codes = ["R1", "R2", "MH"] + is_residential = backfill["PROPTYPE"].isin(residential_codes) + is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == "0") + + backfill = backfill.drop( + columns=[ + "LOCAPROP", + "ADDRGL2", + "ENDDATE", + "OWNER1", + "OWNER2", + "EDITNOTE", + "MAPID", + "YEAR", + "GLYEAR", + "SOURCETYPE", + "SOURCEDATE", + "EDITMETHOD", + "SHAPE_STAr", + "SHAPE_STLe", + "GLIST_SPAN", + "PARCID", + "CRHOUSPCT", + "MUNGL1PCT", + "AOEGL_HS", + "AOEGL_NR", + "HSITEVAL", + "E911ADDR", + "ADDRGL1", + "ZIPGL", + ] + ) + + bad_geom = backfill.geometry.isna() | backfill.geometry.is_empty + gdf = backfill.loc[~bad_geom].copy() + + assert gdf.geom_type.isin(["Polygon", "MultiPolygon"]).all(), ( + gdf.geom_type.value_counts() + ) + gdf["geometry"] = gdf.geometry.apply( + lambda g: MultiPolygon([g]) if g.geom_type == "Polygon" else g + ) + return gdf + + +if __name__ == "__main__": + main() diff --git a/backend/data_collection/parcels.py b/backend/data_collection/parcels.py new file mode 100644 index 00000000..95dde7cd --- /dev/null +++ b/backend/data_collection/parcels.py @@ -0,0 +1,364 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-08-24 +**Description**: + fetches the parcel dataset +""" + +import zipfile +from io import BytesIO +from pathlib import Path + +import geopandas as gpd +import pandas as pd +import requests + +# list of towns from which to gather data +VT_TOWNS = [ + "Addison", + "Albany", + "Alburgh", + "Andover", + "Arlington", + "Athens", + "Averill", + "Averys Gore", + "Bakersfield", + "Baltimore", + "Barnard", + "Barnet", + "Barre City", + "Barre Town", + "Barton", + "Belvidere", + "Bennington", + "Benson", + "Berkshire", + "Berlin", + "Bethel", + "Bloomfield", + "Bolton", + "Bradford", + "Braintree", + "Brandon", + "Brattleboro", + "Bridgewater", + "Bridport", + "Brighton", + "Bristol", + "Brookfield", + "Brookline", + "Brownington", + "Brunswick", + "Buels Gore", + "Burke", + "Burlington", + "Cabot", + "Calais", + "Cambridge", + "Canaan", + "Castleton", + "Cavendish", + "Charleston", + "Charlotte", + "Chelsea", + "Chester", + "Chittenden", + "Clarendon", + "Colchester", + "Concord", + "Corinth", + "Cornwall", + "Coventry", + "Craftsbury", + "Danby", + "Danville", + "Derby", + "Dorset", + "Dover", + "Dummerston", + "Duxbury", + "East Haven", + "East Montpelier", + "Eden", + "Elmore", + "Enosburgh", + "Essex Junction", + "Essex", + "Fair Haven", + "Fairfax", + "Fairfield", + "Fairlee", + "Fayston", + "Ferdinand", + "Ferrisburgh", + "Fletcher", + "Franklin", + "Georgia", + "Glastenbury", + "Glover", + "Goshen", + "Grafton", + "Granby", + "Grand Isle", + "Granville", + "Greensboro", + "Groton", + "Guildhall", + "Guilford", + "Halifax", + "Hancock", + "Hardwick", + "Hartford", + "Hartland", + "Highgate", + "Hinesburg", + "Holland", + "Hubbardton", + "Huntington", + "Hyde Park", + "Ira", + "Irasburg", + "Isle La Motte", + "Jamaica", + "Jay", + "Jericho", + "Johnson", + "Killington", + "Kirby", + "Landgrove", + "Leicester", + "Lemington", + "Lewis", + "Lincoln", + "Londonderry", + "Lowell", + "Ludlow", + "Lunenburg", + "Lyndon", + "Maidstone", + "Manchester", + "Marlboro", + "Marshfield", + "Mendon", + "Middlebury", + "Middlesex", + "Middletown Springs", + "Milton", + "Monkton", + "Montgomery", + "Montpelier", + "Moretown", + "Morgan", + "Morristown", + "Mount Holly", + "Mount Tabor", + "New Haven", + "Newark", + "Newbury", + "Newfane", + "Newport City", + "Newport Town", + "North Hero", + "Northfield", + "Norton", + "Norwich", + "Orange", + "Orwell", + "Panton", + "Pawlet", + "Peacham", + "Peru", + "Pittsfield", + "Pittsford", + "Plainfield", + "Plymouth", + "Pomfret", + "Poultney", + "Pownal", + "Proctor", + "Putney", + "Randolph", + "Reading", + "Readsboro", + "Richford", + "Richmond", + "Ripton", + "Rochester", + "Rockingham", + "Roxbury", + "Royalton", + "Rupert", + "Rutland City", + "Rutland Town", + "Ryegate", + "Saint Albans City", + "Saint Albans Town", + "Saint George", + "Saint Johnsbury", + "Salisbury", + "Sandgate", + "Searsburg", + "Shaftsbury", + "Sharon", + "Sheffield", + "Shelburne", + "Sheldon", + "Shoreham", + "Shrewsbury", + "Somerset", + "South Burlington", + "South Hero", + "Springfield", + "Stamford", + "Stannard", + "Starksboro", + "Stockbridge", + "Stowe", + "Strafford", + "Stratton", + "Sudbury", + "Sunderland", + "Sutton", + "Swanton", + "Thetford", + "Tinmouth", + "Topsham", + "Townshend", + "Troy", + "Tunbridge", + "Underhill", + "Vergennes", + "Vernon", + "Vershire", + "Victory", + "Waitsfield", + "Walden", + "Wallingford", + "Waltham", + "Wardsboro", + "Warners Grant", + "Warren Gore", + "Warren", + "Washington", + "Waterbury", + "Waterford", + "Waterville", + "Weathersfield", + "Wells", + "West Fairlee", + "West Haven", + "West Rutland", + "West Windsor", + "Westfield", + "Westford", + "Westminster", + "Westmore", + "Weston", + "Weybridge", + "Wheelock", + "Whiting", + "Whitingham", + "Williamstown", + "Williston", + "Wilmington", + "Windham", + "Windsor", + "Winhall", + "Winooski", + "Wolcott", + "Woodbury", + "Woodford", + "Woodstock", + "Worcester", +] + +# base url from which to get the parcels +BASE_URL = BASE = ( + "https://maps.vcgi.vermont.gov/gisdata/vcgi/packaged_zips/CadastralParcels_VTPARCELS/" +) + + +# will figure this out later +STORAGE_LOCATION = "Data/parcels" + +# --------------------------------------------------------------------------- +# VERMONT PARCELS API fetch +# --------------------------------------------------------------------------- + + +def fetch_town(town: str): + # the destination is the backfill folder in the parcels folder, with one entrance per town + destination = Path(f"{STORAGE_LOCATION}/backfill/{town}") + # making a variable for the town's SHAPEFILE being there + shp = destination / f"VTPARCELS_{town}.shp" + # we can skip athe need to download if the file is already there - this will save time on multiple go arounds + if shp.exists(): + return + # getting the info from the api + r = requests.get(f"{BASE_URL}VTPARCELS_{town}.zip", timeout=60) + # throwing an exception if the https request fails + r.raise_for_status() + # extracting the data + zipfile.ZipFile(BytesIO(r.content)).extractall(destination) + + +def standardize_town(shp_path: Path) -> gpd.GeoDataFrame: + gdf = gpd.read_file(shp_path) + gdf = gdf.to_crs(4326) + gdf["TOWN"] = gdf["TOWN"].str.upper().str.strip() + return gdf + + +# --------------------------------------------------------------------------- +# Main scrape runner +# --------------------------------------------------------------------------- + + +# need to add a failsafe - all towns might not work +def collect(): + # failed = [] + # trying each town + # we collecdt each failure and add it to the failedd list, also citing it's failure essage + for town in VT_TOWNS: + fetch_town(town) + + gdfs = [ + standardize_town( + Path(f"{STORAGE_LOCATION}/backfill/{town}") / f"VTPARCELS_{town}.shp" + ) + for town in VT_TOWNS + ] + + # concatenating what we just grabbed to backfill gpd + backfill = pd.concat(gdfs, ignore_index=True) + # changing the geometry + backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) + # contriving a stable join key + backfill["OBJECTID"] = 1_000_000 + backfill.index + # returning + return backfill + + +# def combine(towns_list) -> gpd.GeoDataFrame: +# # running the loop for each town in town list: we standardize the town after having grabbed it from sotrage +# gdfs = [ +# standardize_town( +# Path(f"{STORAGE_LOCATION}/backfill/{town}") / f"VTPARCELS_{town}.shp" +# ) +# for town in towns_list +# ] +# # concatenating what we just grabbed to backfill gpd +# backfill = pd.concat(gdfs, ignore_index=True) +# # changing the geometry +# backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) +# # contriving a stable join key +# backfill["OBJECTID"] = 1_000_000 + backfill.index +# # returning +# return backfill + + +if __name__ == "__main__": + df = collect() diff --git a/backend/geo_query.sql b/backend/geo_query.sql new file mode 100644 index 00000000..1540f9c4 --- /dev/null +++ b/backend/geo_query.sql @@ -0,0 +1,28 @@ +{{ cte_filter_block }} +SELECT + JSON_OBJECT( + 'type', 'FeatureCollection', + 'features', JSON_GROUP_ARRAY(feature) + )::VARCHAR AS fc +FROM ( + SELECT + JSON_OBJECT( + 'type', 'Feature', + 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(g.geom, 0.0001))::JSON, + 'properties', JSON_OBJECT( + 'Parcel Type', i.CAT, + 'Acres', ROUND(i.ACRESGL, 2), + 'rgba_color', '[100,150,200,150]'::JSON, + 'tooltip', JSON_OBJECT( + '__title__', 'Parcels', + 'Place', i.CITYGL, + 'Parcel Description', i.DESCPROP, + 'Acres', ROUND(i.ACRESGL, 2) + ) + ) + ) AS feature + FROM parcels_info AS i + INNER JOIN parcels_geom AS g USING (OBJECTID) + {{ join_filter_block }} +) AS features + diff --git a/backend/notebooks/parcels/build.qmd b/backend/notebooks/parcels/build.qmd new file mode 100644 index 00000000..22eb8a99 --- /dev/null +++ b/backend/notebooks/parcels/build.qmd @@ -0,0 +1,46 @@ +--- +title: "Vermont Parcels: build.py [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#root finder +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) + +import logging + +from api.models import FilterSource +from query.processed_db import DB +from sql_render import sql_filter_block +``` + +```{python} + + + +``` + diff --git a/backend/notebooks/parcels/investigation.qmd b/backend/notebooks/parcels/investigation.qmd new file mode 100644 index 00000000..4782a17f --- /dev/null +++ b/backend/notebooks/parcels/investigation.qmd @@ -0,0 +1,56 @@ +--- +title: "Investigating the features of the parcels dataset" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#ill get to this later +```{python} +from pathlib import Path +import os +import sys +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +sys.path.insert(0, str(_project_root)) +print(os.getcwd()) + +from api.models import FilterSource +from sql_render import sql_filter_block +import json +from query.processed_db import DB + + + +sql_dir = Path("query/sql/parcels") +src = FilterSource(filter_table="parcels_info", filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", join_type="inner") +sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", [src]) +print(sql) +``` + +#before we do anything, we must download all six datasets and explore +```{python} +GEOJSON_FULL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_standardized_parcels_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_INACTIVE = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_standardized_inactive_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_PARCEL_STATUS_BY_TOWN = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_DataStatus_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_PARCELS_TRANSFERS = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_PTTR_point_WM_v1_view/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + + + +``` diff --git a/backend/notebooks/parcels/local_data/population_2020.csv b/backend/notebooks/parcels/local_data/population_2020.csv new file mode 100644 index 00000000..f6971cf6 --- /dev/null +++ b/backend/notebooks/parcels/local_data/population_2020.csv @@ -0,0 +1,256 @@ +_geoid,Town,County,year1791,year1800,year1810,year1820,year1830,year1840,year1850,year1860,year1870,year1880,year1890,year1900,year1910,year1920,year1930,year1940,year1950,year1960,year1970,year1980,year1990,year2000,year2010,year2020 +5000100325,Addison,Addison,402,734,1100,1210,1306,1232,1279,1000,911,847,900,851,796,743,684,576,628,645,717,889,1023,1393,1371,1365 +5001900475,Albany,Orleans,0,0,101,253,683,920,1052,1224,1151,1138,995,1028,920,840,810,748,704,560,528,705,782,840,941,976 +5001300860,Alburgh,Grand Isle,446,750,1106,1172,1239,1344,1568,1793,1716,1614,1390,1474,1311,1491,1609,1623,1402,1123,1271,1352,1362,1952,1998,2106 +5002701300,Andover,Windsor,275,1016,957,1000,975,877,725,670,588,564,418,372,284,294,258,213,185,215,239,350,373,496,467,568 +5000301450,Arlington,Bennington,992,1597,1468,1354,1207,1038,1084,1146,1636,1532,1352,1193,1307,1370,1441,1418,1463,1605,1934,2184,2299,2397,2317,2457 +5002501900,Athens,Windham,450,459,478,507,415,358,359,382,295,284,205,180,201,123,132,136,139,142,159,250,313,340,442,380 +5000902125,Averill,Essex,0,0,0,0,1,11,7,12,14,48,43,18,15,4,9,12,20,16,8,15,7,8,24,21 +5000902162,Avery's Gore,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +5001102500,Bakersfield,Franklin,13,222,812,945,1087,1258,1523,1451,1043,1248,1162,1158,1079,980,889,827,779,664,635,852,977,1215,1322,1273 +5002702575,Baltimore,Windsor,0,174,207,204,179,155,124,116,83,71,64,55,54,62,69,85,89,90,170,181,190,250,244,229 +5002702725,Barnard,Windsor,673,1236,1648,1691,1889,1774,1647,1487,1208,1191,918,840,737,653,584,486,439,435,569,790,872,958,947,992 +5000502875,Barnet,Caledonia,477,858,1301,1488,1764,2030,2521,1994,1945,1907,1897,1763,1707,1685,2604,1596,1425,1445,1342,1338,1415,1690,1708,1663 +5002303175,Barre City,Washington,0,0,0,0,0,0,0,0,0,0,0,8448,10734,10008,11307,10909,10922,10387,10209,9824,9482,9291,9052,8491 +5002303250,Barre Town,Washington,76,919,1669,1955,2012,2126,1845,1839,1882,2060,6812,3346,4194,3882,4280,4052,4145,4580,6509,7090,7411,7602,7924,7923 +5001903550,Barton,Orleans,0,128,447,372,729,892,987,1590,1911,2364,2217,2790,3346,3506,3469,3371,3298,3066,2874,2990,2967,2780,2810,2872 +5001504375,Belvidere,Lamoille,0,0,217,198,185,207,256,366,369,400,571,428,429,363,258,220,207,155,189,218,228,294,348,358 +5000304825,Bennington,Bennington,2350,2243,2524,2485,3419,3429,3923,4369,5760,6335,6391,8033,8698,9982,10628,11257,12411,13002,14586,15815,16451,15737,15764,15333 +5002105200,Benson,Rutland,658,1164,1561,1481,1493,1403,1305,1256,1244,1104,880,844,813,807,630,572,573,549,583,739,847,1039,1056,974 +5001105425,Berkshire,Franklin,0,172,918,831,1308,1818,1955,1890,1609,1596,1421,1326,1286,1299,1234,1156,1063,965,931,1116,1190,1388,1692,1547 +5002305650,Berlin,Washington,134,684,1067,1455,1664,1598,1507,1545,1474,1380,1514,1021,1079,959,992,1111,1158,1306,2050,2454,2561,2864,2887,2849 +5002705800,Bethel,Windsor,473,913,1041,1318,1667,1886,1730,1804,1817,1693,1448,1611,1843,1782,1650,1477,1534,1356,1347,1715,1866,1968,2030,1942 +5000906325,Bloomfield,Essex,0,27,9,132,150,179,244,320,455,627,827,564,496,382,417,326,291,212,196,188,253,261,221,217 +5000706550,Bolton,Chittenden,88,219,249,306,452,470,602,632,711,674,547,486,469,390,325,287,301,237,427,715,971,971,1182,1301 +5001707375,Bradford,Orange,654,1062,1302,1411,1507,1655,1723,1689,1492,1520,1429,1338,1372,1422,1234,1507,1551,1619,1627,2191,2522,2619,2797,2790 +5001707600,Braintree,Orange,221,531,850,1037,1209,1232,1228,1225,1066,1051,854,776,760,707,635,648,626,536,751,1065,1174,1194,1246,1207 +5002107750,Brandon,Rutland,637,1076,1375,1495,1940,2194,2835,3077,3571,3280,3310,2759,2712,2874,2891,2979,3304,3329,3697,4194,4223,3917,3966,4129 +5002507900,Brattleboro,Windham,1589,1867,1891,2017,2141,2624,3816,3855,4933,5880,6862,6640,7541,8332,9816,10983,11522,11734,12239,11886,12241,12005,12046,12184 +5002708275,Bridgewater,Windsor,293,781,1154,1125,1311,1363,1311,1292,1141,1084,1124,972,874,808,741,695,903,776,783,867,895,980,936,903 +5000108575,Bridport,Addison,450,1124,1520,1511,1774,1233,1393,1298,1176,1167,1018,956,848,745,703,665,663,653,809,997,1137,1235,1218,1225 +5000908725,Brighton,Essex,0,0,0,144,105,157,193,945,1536,1691,2020,2023,2013,2280,2002,1754,1671,1545,1365,1557,1562,1260,1222,1157 +5000109025,Bristol,Addison,211,665,1179,1051,1247,1480,1334,1355,1365,1579,1828,2061,2005,1952,1832,1939,1988,2159,2744,3293,3762,3788,3894,3782 +5001709325,Brookfield,Orange,419,988,1384,1411,1677,1789,1672,1672,1269,1239,996,996,1008,860,761,808,762,597,606,959,1089,1222,1292,1244 +5002509475,Brookline,Windham,0,472,431,395,376,325,285,243,203,206,162,171,137,105,101,104,132,127,180,310,403,467,530,540 +5001909850,Brownington,Orleans,0,65,236,265,412,486,613,761,901,854,799,748,760,741,697,689,673,599,522,708,705,885,988,1042 +5000910075,Brunswick,Essex,66,86,143,124,160,130,119,212,221,193,160,106,82,89,73,86,73,62,45,82,92,107,112,88 +5000710300,Buels Gore,Chittenden,0,0,0,0,0,0,18,35,29,24,21,20,16,14,4,2,3,0,10,9,2,12,30,29 +5000510450,Burke,Caledonia,0,108,460,541,866,997,1103,1138,1162,1252,1198,1184,1183,1041,1016,988,1042,922,1053,1385,1406,1571,1753,1651 +5000710675,Burlington,Chittenden,332,816,1690,2111,3526,4271,7585,7713,14387,11365,14590,18640,20468,22779,24789,27686,33155,35531,38633,37712,39127,38889,42417,44743 +5002311125,Cabot,Washington,122,349,686,1032,1304,1440,1356,1318,1279,1242,1074,1126,1116,1036,1107,974,826,763,663,958,1043,1213,1433,1443 +5002311350,Calais,Washington,45,443,841,1111,1539,1709,1410,1409,1309,1253,1062,1101,1042,860,812,818,778,684,749,1207,1521,1529,1607,1661 +5001511500,Cambridge,Lamoille,359,733,990,1435,1613,1790,1849,1748,1651,1750,1689,1606,1696,1593,1402,1383,1435,1295,1528,2019,2667,3186,3659,3839 +5000911800,Canaan,Essex,19,44,143,227,373,378,471,408,417,637,829,934,869,982,906,872,969,1094,949,1196,1121,1078,972,896 +5002111950,Castleton,Rutland,809,1039,1420,1541,1783,1769,3016,2852,3243,2605,2396,2089,1886,1919,1794,1601,1748,1902,2837,3637,4278,4367,4717,4458 +5002712250,Cavendish,Windsor,491,922,1295,1551,1498,1427,1576,1509,1823,1276,1172,1352,1208,1319,1418,1598,1374,1223,1264,1355,1323,1470,1367,1392 +5001913150,Charleston,Orleans,0,0,56,90,564,731,1009,1160,1278,1204,1058,1025,993,921,895,834,764,668,654,851,844,895,1023,1021 +5000713300,Charlotte,Chittenden,635,1231,1679,1625,1702,1620,1634,1589,1430,1342,1240,1254,1163,1160,1089,1082,1215,1271,1802,2561,3148,3569,3754,3912 +5001713525,Chelsea,Orange,239,896,1327,1462,1958,1959,1958,1757,1526,1462,1230,1070,1074,1087,1004,1013,1025,957,983,1091,1166,1250,1238,1233 +5002713675,Chester,Windsor,981,1878,2370,2495,2320,2305,2001,2126,2052,1901,1787,1775,1784,1633,1666,1740,1981,2318,2371,2791,2832,3044,3154,3005 +5002114350,Chittenden,Rutland,159,327,446,528,610,644,675,763,802,1092,730,621,563,472,341,379,424,460,646,927,1102,1182,1258,1237 +5002114500,Clarendon,Rutland,1480,1789,1797,1712,1585,1549,1477,1237,1173,1105,928,915,857,826,883,868,1102,1091,1537,2372,2835,2811,2571,2412 +5000714875,Colchester,Chittenden,137,347,659,1192,1489,1739,2575,3041,3911,4421,5143,5352,6450,6627,2638,3031,3897,4718,8776,12629,14731,16986,17067,17524 +5000915250,Concord,Essex,49,322,232,806,1031,1024,1153,1291,1276,1612,1425,1129,1080,1102,1043,923,979,956,896,1125,1093,1196,1235,1141 +5001715700,Corinth,Orange,578,1410,1876,1907,1953,1970,1806,1627,1470,1627,1027,978,1005,936,817,822,786,775,683,904,1244,1461,1367,1455 +5000116000,Cornwall,Addison,825,1163,1279,1120,1264,1164,1155,977,969,1070,927,850,789,782,640,670,728,756,900,993,1101,1136,1185,1207 +5001916150,Coventry,Orleans,0,7,178,282,728,786,857,914,914,911,879,728,616,668,610,549,497,458,492,674,806,1014,1086,1100 +5001916300,Craftsbury,Orleans,0,229,566,605,982,1151,1223,1413,1330,1381,1271,1251,1119,1042,976,875,709,674,632,844,994,1136,1206,1343 +5002116825,Danby,Rutland,1206,1487,1730,1607,1362,1479,1535,1419,1319,1202,1084,964,1001,1007,1070,1112,990,891,910,992,1193,1292,1311,1284 +5000517125,Danville,Caledonia,574,1514,2240,2300,2631,2633,2577,2544,2216,2003,1784,1628,1564,1494,1600,1472,1312,1368,1405,1705,1917,2211,2196,2335 +5001917350,Derby,Orleans,0,178,714,925,1469,1681,1759,1908,2039,1957,2900,2361,2330,2201,2135,2118,2245,2506,3252,4222,4479,4604,4621,4579 +5000317725,Dorset,Bennington,957,1286,1294,1359,1507,1426,1700,2089,2195,2005,1696,1477,1472,1226,1120,1128,1150,1150,1293,1648,1918,2036,2031,2133 +5002517875,Dover,Windham,0,0,859,829,1216,729,709,650,635,621,524,503,377,385,278,244,252,370,555,666,994,1410,1124,1798 +5002518325,Dummerston,Windham,1490,1692,1704,1658,1592,1263,1645,1021,916,816,860,728,643,570,604,615,790,872,1295,1574,1863,1915,1864,1865 +5002318550,Duxbury,Washington,39,153,326,440,652,820,895,1000,893,894,912,778,648,631,553,854,489,546,621,877,976,1289,1337,1413 +5000921250,East Haven,Essex,0,0,0,0,33,79,94,136,191,225,236,171,194,148,99,92,85,164,197,280,269,301,290,270 +5002321925,East Montpelier,Washington,0,0,0,0,1156,1092,1447,1328,1130,972,953,1061,985,918,965,1025,1128,1200,1597,2205,2239,2578,2576,2598 +5001523500,Eden,Lamoille,0,29,224,201,461,703,668,919,958,934,851,738,751,619,568,498,496,430,513,612,840,1152,1323,1338 +5001523725,Elmore,Lamoille,12,45,157,164,442,476,504,602,637,682,593,550,563,468,382,300,312,237,292,421,573,849,855,886 +5001124050,Enosburgh,Franklin,0,143,704,932,1560,2022,2009,2066,2077,2213,2289,2054,2212,2231,2093,2082,2101,1966,1918,2070,2535,2788,2781,2810 +5000724175,Essex,Chittenden,354,729,957,1089,1664,1824,2052,1905,2022,2104,2013,2203,2714,2449,2876,3059,3931,7090,10951,14392,16498,18626,19587,22094 +5002125375,Fair Haven,Rutland,545,411,645,714,675,633,902,1378,2008,2211,2791,2999,3095,2540,2614,2245,2286,2378,2777,2819,2887,2928,2734,2736 +5001124925,Fairfax,Franklin,254,786,1301,1359,1729,1918,2111,1987,1956,1820,1523,1338,1318,1244,1249,1229,1129,1244,1366,1805,2486,3765,4285,5014 +5001125225,Fairfield,Franklin,126,901,1618,1573,2270,2448,2591,2497,2391,2172,1825,1830,1778,1632,1541,1444,1428,1225,1285,1493,1680,1800,1891,2044 +5001725675,Fairlee,Orange,463,386,983,1143,656,644,575,549,416,469,398,438,438,459,456,535,571,569,604,770,883,967,977,988 +5002325825,Fayston,Washington,0,18,149,263,458,635,684,800,694,638,533,466,452,424,318,284,172,158,292,657,846,1141,1353,1364 +5000925975,Ferdinand,Essex,0,0,0,0,0,0,0,34,33,40,73,41,213,106,18,17,10,16,14,12,23,33,32,16 +5000126300,Ferrisburgh,Addison,481,956,1647,1581,1822,1755,2075,1738,1768,1684,1601,1619,1433,1338,1285,1347,1387,1426,1875,2117,2317,2657,2775,2646 +5001126500,Fletcher,Franklin,47,200,382,497,793,1014,1084,916,865,868,793,750,737,656,667,549,485,399,456,626,941,1179,1277,1346 +5001127100,Franklin,Franklin,46,280,714,831,1129,1410,1646,1781,1612,1439,1300,1145,1108,994,1001,1021,878,796,821,1006,1068,1268,1405,1363 +5001127700,Georgia,Franklin,340,1068,1760,1703,1897,2106,2686,1547,1603,1504,1282,1280,1090,1075,1090,1008,1055,1079,1711,2818,3753,4375,4515,4845 +5000327962,Glastenbury,Bennington,34,48,76,48,59,53,52,47,119,241,181,48,29,40,7,4,1,0,0,3,7,16,8,9 +5001928075,Glover,Orleans,0,35,368,549,902,1119,1137,1244,1178,1055,970,891,932,826,860,788,727,683,649,843,820,966,1122,1114 +5000128600,Goshen,Addison,0,4,86,290,555,621,486,394,330,326,311,286,212,131,84,83,94,76,120,163,226,227,164,172 +5002528900,Grafton,Windham,0,1149,1365,1482,1439,1326,1241,1154,1008,929,817,804,729,476,453,393,422,426,465,604,602,649,679,645 +5000929125,Granby,Essex,0,69,30,49,97,105,127,132,174,194,361,182,95,70,69,76,74,56,52,70,85,86,88,81 +5001329275,Grand Isle,Grand Isle,0,0,338,698,642,724,666,708,682,749,793,851,839,808,857,791,735,624,809,1238,1642,1955,2067,2086 +5000129575,Granville,Addison,101,185,326,328,403,545,603,720,726,830,637,544,464,393,280,247,213,215,255,288,309,303,298,301 +5001930175,Greensboro,Orleans,19,280,566,625,784,883,1008,1065,1027,1061,918,874,931,906,831,768,737,600,593,677,717,770,762,811 +5000530550,Groton,Caledonia,45,248,449,595,836,928,893,939,811,1014,1040,1059,915,902,803,764,712,631,666,667,862,876,1022,984 +5000930775,Guildhall,Essex,158,296,120,529,481,470,501,552,483,558,511,455,445,376,351,313,270,248,169,202,270,268,261,262 +5002530925,Guilford,Windham,2422,2257,1872,1862,1760,1523,1389,1291,1277,1096,870,782,769,684,663,686,796,823,1108,1532,1941,2046,2121,2120 +5002531150,Halifax,Windham,1209,1600,1758,1567,1552,1399,1133,1125,1029,852,702,662,635,504,390,353,343,268,295,488,588,782,728,771 +5000131525,Hancock,Addison,56,149,311,548,472,465,430,448,430,382,283,253,287,300,303,371,391,323,283,334,340,382,323,359 +5000531825,Hardwick,Caledonia,3,260,735,867,1216,1354,1402,1369,1519,1484,1547,2466,3201,2641,2720,2605,2629,2349,2466,2613,2964,3174,3010,2920 +5002732275,Hartford,Windsor,988,1494,1831,2010,2044,2341,2159,2396,2480,2954,3740,3817,4179,4739,4888,4978,5827,6355,6477,7963,9404,10367,9952,10686 +5002732425,Hartland,Windsor,1552,1960,2352,2553,2503,2194,2063,1748,1710,1598,1393,1340,1316,1212,1266,1306,1559,1592,1806,2396,2988,3223,3393,3446 +5001133025,Highgate,Franklin,103,437,1374,1250,2038,2292,2653,2526,2260,2088,1853,1980,1758,1528,1574,1647,1681,1608,1936,2493,3020,3397,3535,3472 +5000733475,Hinesburg,Chittenden,454,933,1238,1332,1669,1682,1834,1702,1573,1330,1205,1216,1042,964,1019,1000,1120,1180,1775,2690,3780,4340,4396,4698 +5001933775,Holland,Orleans,0,0,126,100,422,605,669,748,881,913,878,838,722,714,560,533,406,376,383,473,423,588,629,632 +5002134450,Hubbardton,Rutland,410,541,734,810,865,719,701,606,606,533,506,488,455,328,307,346,332,238,228,490,576,752,706,735 +5000734600,Huntington,Chittenden,167,405,514,732,929,914,835,861,864,808,723,728,760,651,621,549,601,518,748,1161,1609,1861,1938,1934 +5001535050,Hyde Park,Lamoille,43,110,261,373,823,1080,1107,1409,1624,1716,1633,1472,1453,1323,1165,1178,1291,1219,1347,2021,2344,2847,2954,3020 +5002135425,Ira,Rutland,312,473,519,498,442,451,400,422,413,479,421,350,286,295,287,248,232,220,284,354,426,455,432,368 +5001935575,Irasburg,Orleans,0,15,392,432,860,971,1034,1131,1085,1054,999,939,983,999,924,852,711,711,775,870,907,1077,1163,1233 +5001335875,Isle La Motte,Grand Isle,47,135,623,312,459,435,476,564,497,505,551,608,510,385,352,335,295,238,262,393,408,488,471,488 +5002536175,Jamaica,Windham,263,582,996,1343,1523,1586,1606,1541,1223,1252,1074,800,716,686,570,567,597,496,590,681,754,946,1035,1005 +5001936325,Jay,Orleans,0,0,28,52,196,306,371,474,553,696,641,530,513,368,274,230,243,197,182,302,381,426,521,551 +5000736700,Jericho,Chittenden,381,728,1185,1219,1654,1685,1837,1669,1757,1687,1461,1373,1307,1138,1091,1077,1135,1425,2343,3575,4302,5015,5009,5104 +5001537075,Johnson,Lamoille,93,265,494,778,1079,1410,1381,1526,1558,1495,1462,1391,1526,1478,1378,1420,1527,1478,1927,2581,3156,3274,3446,3491 +5002137685,Killington,Rutland,32,90,116,154,432,698,578,525,462,450,451,402,409,336,298,266,283,266,558,891,738,1095,811,1407 +5000537900,Kirby,Caledonia,0,20,311,312,401,520,509,473,417,398,355,350,297,324,311,256,257,235,224,282,347,456,493,575 +5000339025,Landgrove,Bennington,31,147,299,341,385,344,357,320,302,246,220,225,160,143,104,64,80,49,104,121,134,144,158,177 +5000139325,Leicester,Addison,344,522,609,442,638,603,596,737,630,634,562,509,479,436,468,518,511,551,583,803,871,974,1100,990 +5000939700,Lemington,Essex,0,31,52,139,183,124,187,207,191,222,227,204,138,145,133,131,105,112,120,108,102,107,104,87 +5000939775,Lewis,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 +5000140075,Lincoln,Addison,0,97,225,278,639,770,1057,1070,1174,1368,1255,1152,980,841,800,745,577,481,599,870,974,1214,1271,1323 +5002540225,Londonderry,Windham,362,530,637,958,1302,1216,1274,1367,1252,1154,1010,961,962,911,799,859,953,898,1037,1510,1506,1709,1769,1919 +5001940525,Lowell,Orleans,0,0,0,144,314,431,637,813,942,1067,1178,983,1086,1005,725,615,643,617,515,573,594,738,879,887 +5002741275,Ludlow,Windsor,179,410,877,1140,1227,1363,1619,1568,1827,2005,1768,2042,2215,2421,2305,2458,2428,2386,2463,2414,2302,2449,1963,2172 +5000941425,Lunenburg,Essex,119,393,132,856,1054,1130,1123,1034,999,1038,1019,968,880,1048,1400,1374,1299,1237,1061,1138,1176,1328,1302,1246 +5000541725,Lyndon,Caledonia,59,542,1090,1296,1822,1753,1752,1695,2179,2434,2619,2956,3204,3558,3285,3144,3360,3425,3705,4924,5371,5448,5981,5491 +5000942475,Maidstone,Essex,125,152,714,165,236,271,237,259,254,286,198,206,175,171,123,96,81,78,94,100,131,105,208,211 +5000342850,Manchester,Bennington,1278,1397,1502,1508,1525,1594,1762,1688,1897,1928,1907,1955,2044,2057,2004,2139,2425,2470,2919,3261,3622,4180,4391,4484 +5002543375,Marlboro,Windham,629,1087,1245,1296,1218,1027,896,741,665,563,495,448,442,300,255,225,311,347,592,695,924,978,1078,1722 +5002343600,Marshfield,Washington,0,172,513,710,1271,1156,1102,1160,1072,1102,1121,1032,1011,898,872,901,830,891,1033,1267,1331,1496,1588,1583 +5002144125,Mendon,Rutland,34,39,111,174,432,545,504,633,612,629,570,392,321,264,251,313,334,461,743,1056,1049,1028,1059,1149 +5000144350,Middlebury,Addison,398,1263,2138,2635,3468,3162,3517,2879,3086,2995,2793,3045,2848,2914,2968,3175,4778,5305,6532,7574,8034,8183,8496,9152 +5002344500,Middlesex,Washington,60,262,401,726,1156,1270,1365,1254,1171,1087,889,883,858,762,751,817,887,770,857,1235,1514,1729,1731,1779 +5002144800,Middletown Springs,Rutland,699,1066,1207,1039,919,1057,875,712,777,823,786,746,716,567,583,493,496,381,426,603,686,823,745,794 +5000745250,Milton,Chittenden,283,786,1546,1746,2100,2134,2451,1983,2062,2006,1885,1804,1648,1523,1663,1750,1874,2022,4495,6829,8404,9479,10352,10723 +5000145550,Monkton,Addison,449,880,1248,1152,1384,1310,1246,1123,1006,1025,847,912,724,671,683,575,520,551,765,1201,1482,1759,1980,2079 +5001145850,Montgomery,Franklin,0,36,237,293,460,564,1001,1262,1423,1642,1734,1876,1721,1658,1386,1208,1091,876,651,681,823,992,1201,1184 +5002346000,Montpelier,Washington,118,890,1877,2308,1193,3725,2310,2411,3023,3219,4160,6266,7856,7125,7837,8006,8599,8782,8609,8241,8247,8035,7855,8074 +5002346225,Moretown,Washington,24,191,405,593,816,1128,1335,1410,1263,1180,956,902,686,930,889,675,883,788,904,1221,1415,1653,1658,1753 +5001946450,Morgan,Orleans,0,0,135,116,331,422,486,548,614,711,520,510,463,368,363,335,296,260,286,460,497,669,749,638 +5001546675,Morristown,Lamoille,10,144,550,726,1315,1502,1441,1751,1897,2099,2411,2583,2652,2813,2939,3130,3225,3347,4052,4448,4733,5139,5227,5434 +5002147200,Mount Holly,Rutland,0,668,922,1157,1318,1356,1534,1522,1582,1390,1214,999,871,856,727,656,567,517,687,938,1093,1241,1237,1385 +5002147425,Mount Tabor,Rutland,165,153,209,222,210,226,308,358,301,495,436,494,289,168,173,213,186,165,184,211,214,203,255,210 +5000148700,New Haven,Addison,717,1135,1688,1566,1834,1503,1663,1419,1355,1355,1224,1107,1161,1001,964,881,932,922,1039,1217,1375,1666,1727,1683 +5000547725,Newark,Caledonia,0,8,88,154,257,360,434,567,593,679,536,500,415,364,301,242,192,151,144,280,354,470,581,584 +5001748175,Newbury,Orange,873,1304,1363,1623,2252,2578,2984,2549,2241,2316,2080,2125,2036,1908,1744,1723,1667,1452,1440,1699,1985,1955,2216,2293 +5002548400,Newfane,Windham,860,1000,1276,1506,1441,1403,1304,1192,1115,1031,962,902,820,710,662,672,708,714,900,1129,1555,1680,1726,1645 +5001948850,Newport City,Orleans,0,0,0,0,0,0,0,0,0,0,0,2787,3657,4976,5094,4902,5217,5019,4664,4756,4434,5005,4589,4455 +5001948925,Newport Town,Orleans,0,50,28,52,284,591,748,1197,2050,2426,3047,1239,1236,1187,1193,1064,966,1010,1125,1319,1367,1511,1594,1526 +5001350650,North Hero,Grand Isle,125,324,552,503,638,716,730,594,601,637,550,712,496,494,485,442,407,328,364,442,502,810,803,939 +5002350275,Northfield,Washington,40,204,426,690,1412,2013,2922,4329,3410,2836,2628,2855,3226,3095,3436,3601,4314,4511,4870,5435,5610,5791,6207,5918 +5000952750,Norton,Essex,0,0,0,0,0,0,0,32,303,239,960,692,479,336,339,314,279,241,207,184,169,214,169,153 +5002752900,Norwich,Windsor,1158,1482,1812,1985,2316,2218,1978,1759,1639,1471,1304,1303,1252,1092,1371,1418,1532,1790,1966,2398,3093,3544,3414,3612 +5001753425,Orange,Orange,0,348,686,751,1016,984,1007,936,733,731,589,598,644,485,508,482,410,430,540,752,915,965,1072,1048 +5000153725,Orwell,Addison,778,1376,324,1730,1598,1688,1470,1341,1192,1351,1265,1150,1065,942,835,876,902,826,851,901,1114,1185,1250,1239 +5000153950,Panton,Addison,220,363,520,548,605,670,569,511,390,419,382,409,345,321,306,312,332,352,416,537,606,682,677,646 +5002154250,Pawlet,Rutland,1458,1938,2233,2155,1965,1748,1843,1539,1505,1696,1745,1731,1969,1413,1476,1192,1156,1112,1184,1244,1314,1394,1477,1424 +5000554400,Peacham,Caledonia,365,873,1301,1294,1351,1443,1377,1247,1141,1041,892,794,777,657,620,543,501,433,446,531,627,665,732,715 +5000355000,Peru,Bennington,239,314,445,578,0,0,567,543,500,556,445,373,242,216,156,142,197,194,243,312,324,416,375,531 +5002155450,Pittsfield,Rutland,49,164,338,453,505,615,512,493,482,555,468,435,402,343,256,259,225,254,249,396,389,427,546,504 +5002155600,Pittsford,Rutland,850,1413,1936,1916,2005,1927,2026,1839,2127,1982,1775,1866,2479,2098,2332,2093,2076,2225,2306,2590,2919,3140,2991,2862 +5002355825,Plainfield,Washington,0,256,543,660,874,880,808,822,726,729,745,716,785,781,766,832,945,966,1399,1249,1302,1286,1243,1236 +5002756050,Plymouth,Windsor,106,497,834,1112,1237,1417,1226,1252,1285,1075,765,646,482,449,331,432,348,308,283,405,440,555,619,641 +5002756350,Pomfret,Windsor,710,1105,1473,1835,1867,1774,1546,1375,1251,1139,865,777,703,732,728,686,586,600,620,856,874,997,904,916 +5002156875,Poultney,Rutland,1120,1694,1904,1956,1909,1878,2329,2278,2836,2717,3031,3108,3644,2868,3215,2781,2936,3009,3217,3196,3498,3633,3432,3020 +5000357025,Pownal,Bennington,1732,1692,1655,1812,1835,1615,1742,1731,1705,2019,1919,1976,1599,1396,1425,1402,1453,1509,2441,3269,3485,3560,3527,3258 +5002157250,Proctor,Rutland,0,0,0,0,0,0,0,0,0,0,1758,2136,2871,2789,2596,2292,1917,2102,2095,1998,1979,1877,1741,1763 +5002557700,Putney,Windham,1848,1574,1607,1547,1810,1585,1425,1163,1167,1124,1075,969,788,761,835,904,1019,1177,1727,1850,2352,2634,2702,2617 +5001758075,Randolph,Orange,893,1841,2255,2487,2743,2678,2666,2502,2829,2910,3232,3141,3191,3010,3166,3278,3499,3414,3882,4689,4764,4853,4778,4774 +5002758375,Reading,Windsor,747,1120,1565,1803,1409,1353,1171,1159,1012,953,749,649,530,483,474,437,470,472,564,647,614,707,666,687 +5000358600,Readsboro,Bennington,63,234,410,530,662,767,657,930,828,743,910,1139,1252,1173,1043,913,847,783,638,638,762,809,763,702 +5001159125,Richford,Franklin,0,113,442,440,704,914,1074,1338,1481,1818,2196,2421,2907,2842,2544,2646,2643,2316,2116,2206,2178,2321,2308,2346 +5000759275,Richmond,Chittenden,0,718,935,1014,1109,1064,1453,1400,1309,1264,1115,1057,1419,1447,1315,1225,1278,1303,2249,3159,3729,4090,4081,4167 +5000159650,Ripton,Addison,0,0,15,42,278,357,567,570,617,672,568,525,421,237,194,231,207,131,187,327,444,556,588,739 +5002760100,Rochester,Windsor,215,524,911,1148,1392,1396,1493,1607,1444,1362,1257,1250,1317,1397,1285,1129,937,879,884,1054,1181,1171,1139,1099 +5002560250,Rockingham,Windham,1235,1684,1954,2155,2272,2330,2837,2904,2854,3797,4579,5809,6207,6231,5302,5737,5499,5704,5501,5538,5484,5309,5282,4832 +5002360625,Roxbury,Washington,14,113,361,512,737,784,967,1060,916,938,768,712,618,609,594,554,465,364,354,452,575,576,691,678 +5002760850,Royalton,Windsor,748,1501,1758,1816,1893,1917,1850,1739,1679,1558,1433,1427,1452,1469,1491,1291,1331,1388,1399,2100,2389,2603,2773,2750 +5000361000,Rupert,Bennington,1034,1628,1630,1532,1318,1084,1101,1103,1017,957,861,863,825,674,691,678,713,603,582,605,654,704,714,698 +5002161225,Rutland City,Rutland,0,0,0,0,0,0,0,0,0,0,0,11499,13546,14954,17318,17082,17659,18325,19293,18436,18230,17292,16495,15807 +5002161300,Rutland Town,Rutland,1417,2125,2379,2369,2753,2708,3715,7577,9834,12149,11760,1109,1311,1270,1387,1350,1416,1542,2248,3300,3781,4038,4054,3924 +5000561525,Ryegate,Caledonia,187,415,812,994,1119,1223,1606,1098,935,1046,1126,995,1194,1188,1216,1105,996,894,830,1000,1058,1150,1174,1165 +5000162575,Salisbury,Addison,444,644,709,721,907,942,1027,853,902,775,740,692,693,635,632,581,573,575,649,881,1024,1090,1136,1221 +5000362875,Sandgate,Bennington,733,1020,1187,1185,933,776,850,805,705,681,587,482,401,283,189,187,158,93,127,234,278,353,405,387 +5000363175,Searsburg,Bennington,9,40,120,0,0,0,201,262,235,232,173,161,142,133,103,135,84,73,84,72,85,96,109,126 +5000363550,Shaftsbury,Bennington,1990,1896,1973,2022,2143,1885,1896,1936,2027,1887,1652,1857,1650,1534,1631,1577,1673,1939,2411,3001,3368,3767,3590,3598 +5002763775,Sharon,Windsor,569,1158,1363,1431,1459,1371,1240,1111,1013,1012,737,709,565,545,559,530,470,485,541,828,1211,1411,1502,1560 +5000564075,Sheffield,Caledonia,0,179,388,581,720,821,797,836,811,884,760,724,691,594,543,465,451,342,307,435,541,727,703,682 +5000764300,Shelburne,Chittenden,387,723,987,1168,1123,1098,1257,1178,1190,1095,1300,1292,1097,997,1006,1010,1365,1805,3728,5000,5871,6944,7144,7717 +5001164600,Sheldon,Franklin,0,408,883,927,2158,1734,1814,1655,1697,1529,1365,1341,1246,1473,1563,1471,1352,1281,1481,1618,1748,1990,2190,2136 +5000165050,Shoreham,Addison,701,1447,2033,1881,2137,1674,1601,1382,1225,1354,1240,1193,1098,925,948,865,829,786,790,972,1115,1222,1265,1260 +5002165275,Shrewsbury,Rutland,382,748,990,1151,1289,1218,1268,1175,1145,1235,974,935,751,620,540,537,464,445,570,866,1107,1108,1056,1096 +5002565762,Somerset,Windham,111,130,199,173,245,262,321,105,80,67,61,67,27,59,20,5,8,4,0,2,2,5,3,6 +5000766175,South Burlington,Chittenden,0,0,0,0,0,0,0,0,791,664,845,971,927,938,1023,1736,3279,6903,10032,10679,12809,15814,17904,20292 +5001367000,South Hero,Grand Isle,537,678,826,842,717,664,705,617,586,620,559,817,605,605,641,611,567,614,868,1188,1404,1696,1631,1674 +5002769550,Springfield,Windsor,1097,2032,2556,2702,2749,2625,2762,2968,2937,3144,2881,3432,4784,7202,6965,7720,9190,9934,10063,10190,9579,9078,9373,9062 +5001161675,St. Albans City,Franklin,0,0,0,0,0,0,0,0,0,0,0,6237,6381,7588,8020,8037,8552,8806,8082,7308,7339,7650,6918,6877 +5001161750,St. Albans Town,Franklin,256,901,1609,1636,2395,2702,3567,3637,7014,7193,7771,1715,1617,1583,1691,1733,1908,2303,3270,3555,4606,5086,5999,6988 +5000762050,St. George,Chittenden,57,65,68,120,135,121,127,121,111,93,106,90,109,100,84,87,117,108,477,677,705,698,674,794 +5000562200,St. Johnsbury,Caledonia,143,651,1334,1404,1592,1887,2758,3469,4665,5800,6567,7010,8098,8708,9696,9095,9292,8869,8409,7938,7608,7571,7603,7364 +5000369775,Stamford,Bennington,272,383,378,490,563,662,833,759,633,726,645,677,510,374,370,418,514,600,752,773,773,813,824,861 +5000569925,Stannard,Caledonia,0,0,0,0,0,0,0,0,228,252,239,222,206,173,154,140,116,113,88,142,148,185,216,208 +5000170075,Starksboro,Addison,0,359,726,914,1342,1263,1400,1437,1361,1249,1070,902,835,806,687,744,576,502,668,1336,1511,1898,1777,1756 +5002770375,Stockbridge,Windsor,100,432,700,964,1333,1418,1327,1264,1269,1124,894,822,737,618,460,490,427,392,389,508,618,674,736,718 +5001570525,Stowe,Lamoille,0,316,650,957,1570,1371,1771,2046,2049,1896,1886,1926,1991,1800,1654,1741,1720,1901,2388,2991,3433,4339,4314,5223 +5001770675,Strafford,Orange,844,1642,1805,1921,1935,1762,1640,1506,1290,1181,932,1000,776,601,615,598,680,548,536,731,902,1045,1098,1094 +5002570750,Stratton,Windham,95,271,265,272,312,341,286,365,294,302,222,271,86,90,55,117,54,24,104,122,121,136,216,440 +5002171050,Sudbury,Rutland,258,521,754,809,812,796,794,696,601,562,502,474,415,417,361,321,263,249,253,380,516,583,560,545 +5000371425,Sunderland,Bennington,414,557,575,496,463,438,479,667,553,655,633,518,494,409,375,442,493,566,601,768,872,850,956,1056 +5000571575,Sutton,Caledonia,0,0,433,697,1005,1068,1001,987,920,838,746,694,711,659,596,561,528,476,438,667,854,1001,1029,913 +5001171725,Swanton,Franklin,74,858,1657,607,2158,2312,2824,2678,2866,3079,3231,3745,3628,3342,3433,3543,3740,3946,4622,5141,5636,6203,6427,6701 +5001772400,Thetford,Orange,862,1478,1735,1915,2113,2065,2016,1876,1613,1529,1287,1249,1182,1089,1052,1043,1046,1049,1422,2188,2438,2617,2588,2775 +5002172925,Tinmouth,Rutland,935,973,1001,1069,1049,780,717,620,569,532,435,404,410,349,340,346,248,228,268,406,455,567,613,553 +5001773075,Topsham,Orange,126,344,814,1020,1384,1745,1668,1662,1418,1365,1187,1117,918,825,720,707,733,638,686,767,944,1142,1173,1199 +5002573300,Townshend,Windham,678,1083,1115,1406,1386,1345,1354,1376,1171,1099,865,833,817,786,633,694,584,643,668,849,1019,1149,1232,1291 +5001973525,Troy,Orleans,0,0,231,227,608,855,1008,1246,1355,1522,1673,1467,1685,1869,1098,1869,1786,1613,1457,1498,1609,1564,1662,1722 +5001773675,Tunbridge,Orange,487,1324,1640,2003,1920,1811,1786,1546,1405,1262,1011,885,918,907,903,882,774,743,791,925,1154,1309,1284,1337 +5000773975,Underhill,Chittenden,59,212,490,757,1052,1441,1599,1637,1655,1439,1301,1140,1004,896,781,760,698,730,1198,2172,2799,2980,3016,3129 +5000174650,Vergennes,Addison,201,516,835,817,999,1017,1378,1286,1670,1782,1773,1753,1463,1609,1705,1662,1736,1921,2242,2273,2578,2741,2588,2553 +5002574800,Vernon,Windham,482,480,521,627,681,705,821,726,764,652,567,578,606,656,609,559,712,865,1024,1175,1850,2141,2206,2192 +5001774950,Vershire,Orange,439,1031,1311,1290,1260,1188,1071,1053,1140,1875,954,641,446,410,368,367,284,236,299,442,560,629,730,672 +5000975175,Victory,Essex,0,0,0,6,53,140,168,212,263,321,554,321,206,125,80,104,49,46,42,56,50,97,62,70 +5002375325,Waitsfield,Washington,61,473,647,935,968,1048,1021,1005,948,938,815,760,709,682,723,706,661,658,837,1300,1422,1659,1719,1844 +5000575700,Walden,Caledonia,11,153,455,580,827,913,910,1001,992,931,810,764,739,674,664,547,481,427,442,575,703,782,935,956 +5002175925,Wallingford,Rutland,538,912,1386,1570,1740,1608,1688,1747,2023,1846,1733,1575,1719,1581,1564,1450,1482,1439,1676,1893,2184,2274,2079,2129 +5000176075,Waltham,Addison,247,247,244,264,330,283,270,263,249,248,255,264,202,204,175,184,193,186,265,394,454,479,486,446 +5002576225,Wardsboro,Windham,753,1484,2053,1010,1148,1102,1125,1005,866,766,704,637,559,380,355,401,377,322,391,505,654,854,900,869 +5000976337,Warner's Grant,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +5002376525,Warren,Washington,0,58,229,320,766,943,962,1041,1008,951,866,826,825,654,486,450,498,469,588,956,1172,1681,1705,1977 +5000976562,Warren's Gore,Washington,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,10,4,2 +5001776750,Washington,Orange,72,507,1040,1160,1374,1369,1348,1249,1113,922,820,820,762,660,697,730,650,565,667,855,937,1047,1039,1032 +5002376975,Waterbury,Washington,93,644,966,1269,1650,1992,2352,2198,2633,2297,2232,2810,3273,3542,4045,4118,4276,4303,4614,4465,4589,4915,5064,5331 +5000577125,Waterford,Caledonia,63,565,1289,1247,1358,1388,1412,1171,872,815,734,705,629,574,712,498,468,460,586,882,1190,1104,1280,1268 +5001577425,Waterville,Lamoille,0,0,0,0,488,610,753,747,574,547,577,529,485,469,307,386,409,332,397,470,532,697,673,686 +5002777500,Weathersfield,Windsor,1146,1944,2115,2301,2213,2081,1851,1766,1557,1354,1174,1089,1092,1087,1156,1075,1288,1254,2040,2534,2674,2788,2825,2842 +5002177950,Wells,Rutland,620,978,1040,986,880,740,804,642,483,665,621,606,569,521,515,420,487,419,560,815,902,1121,1150,1214 +5001779975,West Fairlee,Orange,0,391,983,1143,841,824,696,830,833,1038,561,531,446,387,405,428,363,333,337,427,633,726,652,621 +5002180875,West Haven,Rutland,0,430,679,684,724,774,718,580,713,492,412,355,363,343,280,302,232,220,240,253,273,278,264,239 +5002182300,West Rutland,Rutland,0,0,0,0,0,0,0,0,0,0,3680,2914,3427,3391,3421,2922,2487,2302,2381,2351,2448,2535,2326,2214 +5002783050,West Windsor,Windsor,0,0,0,0,0,0,1002,924,708,690,570,513,569,514,512,494,504,539,571,763,923,1067,1099,1344 +5001980200,Westfield,Orleans,0,16,149,225,353,370,502,618,721,698,763,646,613,490,448,354,358,347,375,418,422,503,536,534 +5000780350,Westford,Chittenden,63,648,1107,1218,1290,1352,1458,1231,1237,1133,1033,888,854,706,698,698,685,680,991,1413,1740,2086,2029,2062 +5002581400,Westminster,Windham,1599,1942,1925,1974,1737,1656,1721,1300,1238,1377,1265,1295,1327,1289,1324,1403,1400,1602,1875,2493,3026,3210,3178,3016 +5001981700,Westmore,Orleans,0,0,0,126,132,122,152,324,412,480,395,390,331,287,224,224,210,179,195,257,305,306,350,357 +5002782000,Weston,Windsor,0,0,629,890,972,1032,950,932,931,987,864,756,632,436,411,457,468,442,507,627,488,630,566,623 +5000183275,Weybridge,Addison,174,502,750,714,850,797,804,667,627,608,543,518,494,438,418,385,402,430,618,667,749,824,833,814 +5000583500,Wheelock,Caledonia,33,568,963,906,834,881,855,832,822,829,596,567,500,526,412,293,287,246,238,444,481,621,811,759 +5000183800,Whiting,Addison,249,404,565,609,653,659,629,542,430,455,355,361,348,302,358,312,282,304,359,379,407,380,419,405 +5002583950,Whitingham,Windham,442,868,1248,1397,1477,1391,1380,1372,1263,1240,1191,1042,969,811,734,789,816,838,1011,1043,1177,1298,1357,1344 +5001784175,Williamstown,Orange,146,839,1353,1481,1487,1620,1452,1377,1236,1038,1188,1610,1726,1526,1609,1477,1600,1553,1822,2284,2839,3225,3389,3515 +5000784475,Williston,Chittenden,469,836,1195,1538,1608,1554,1664,1472,1441,1342,1161,1176,1000,929,961,1021,1182,1484,3187,3843,4887,7650,8698,10103 +5002584700,Wilmington,Windham,645,1011,1193,1369,1367,1296,1372,1424,1246,1130,1106,1221,1229,1483,1171,1221,1169,1245,1586,1808,1968,2225,1876,2255 +5002584850,Windham,Windham,0,427,782,931,847,757,763,680,544,536,379,356,345,261,254,183,146,135,174,223,251,328,419,449 +5002784925,Windsor,Windsor,1542,2211,2757,2956,3134,2744,1928,1669,1699,2175,1846,2119,2407,3687,4359,4155,4402,4468,4158,4084,3714,3756,3553,3559 +5000385075,Winhall,Bennington,155,202,429,428,571,576,762,741,642,722,523,449,366,336,229,212,255,245,281,327,482,702,769,1182 +5000785150,Winooski,Chittenden,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5308,6036,6734,7420,7309,6318,6649,6561,7267,7997 +5001585375,Wolcott,Lamoille,32,47,124,123,492,910,909,1161,1132,1166,1158,1066,1049,932,831,772,766,633,676,986,1229,1456,1676,1670 +5002385525,Woodbury,Washington,23,23,254,432,824,1092,1070,999,902,856,810,862,824,686,529,463,449,317,399,573,766,809,906,928 +5000385675,Woodford,Bennington,60,138,254,212,395,487,423,379,371,487,353,279,187,231,137,170,198,207,286,314,331,414,424,355 +5002785975,Woodstock,Windsor,1597,2132,2672,2610,3044,3315,3041,3062,2910,2815,2545,2557,2545,2370,2469,2512,2613,2786,2608,3214,3212,3232,3048,3005 +5002386125,Worcester,Washington,25,25,41,44,432,567,702,684,775,802,725,636,694,463,471,396,445,417,505,727,906,902,998,964 diff --git a/backend/notebooks/parcels/parcels_ml.qmd b/backend/notebooks/parcels/parcels_ml.qmd new file mode 100644 index 00000000..581824d3 --- /dev/null +++ b/backend/notebooks/parcels/parcels_ml.qmd @@ -0,0 +1,270 @@ +--- +title: "Vermont Parcels: build.py [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#root finder +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) + +import logging + +from api.models import FilterSource +from query.processed_db import DB +``` + +```{python} +""" +Vermont Parcels — Spatial Statistics +Step 1 of the modeling pipeline: measure spatial autocorrelation to (a) establish +that value clustering is statistically real, (b) justify spatial cross-validation +downstream, and (c) build the weights matrix reused for spatial features. +""" +import numpy as np, pandas as pd, geopandas as gpd +import matplotlib.pyplot as plt +from libpysal.weights import Queen, KNN +from esda.moran import Moran, Moran_Local +import requests + +# --------------------------------------------------------------------------- +# TOWN LEVEL — coarse, fast, interpretable. Start here. +# --------------------------------------------------------------------------- + +FGB = "Data/parcels/all_parcels_vermont.fgb" + +parcels = gpd.read_file(FGB) +#adding IMPR_SHARE +parcels["IMPR_SHARE"] = np.where(parcels["REAL_FLV"] > 0, parcels["IMPRV_LV"].fillna(0) / parcels["REAL_FLV"], np.nan) +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + +print("Downloading town boundaries...") +r = requests.get(TOWN_URL) +r.raise_for_status() +data = r.json() + +towns_gdf = gpd.GeoDataFrame.from_features(data["features"]) + +towns_gdf.set_crs(epsg=4326, inplace=True) + +print(towns_gdf.columns) + +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +towns_gdf = towns_gdf[["TOWN", "geometry"]].copy() + +# The metric: median parcel value per town. Log because Moran's I is a correlation +# statistic and raw value is heavy-tailed enough that a few towns would dominate it. +tv = parcels.loc[parcels.REAL_FLV > 0].groupby("TOWN")["REAL_FLV"].median() +tw = towns_gdf.merge(tv.rename("med_val").reset_index(), on="TOWN", how="left") +tw["log_val"] = np.log10(tw["med_val"]) + +# CRITICAL ORDERING: drop NaN BEFORE building weights. Weights encode who-borders-whom; +# dropping a town afterward would leave the matrix describing a map that no longer exists, +# and W's row order must align exactly with the value vector. +tw = tw.dropna(subset=["log_val"]).reset_index(drop=True) + +# Queen contiguity: neighbors share ANY boundary point (corners included). +w = Queen.from_dataframe(tw, use_index=True) + +# VERMONT-SPECIFIC TRAP: Lake Champlain islands (Grand Isle, Isle La Motte, North/South +# Hero, Alburgh) share no LAND border, so they get zero neighbors. Moran's I can't handle +# island rows. Drop them in a loop — removing one town can orphan another. +print("island towns:", [tw.TOWN.iloc[i] for i in w.islands]) +while w.islands: + tw = tw.drop(index=w.islands).reset_index(drop=True) + w = Queen.from_dataframe(tw, use_index=True) + +# Row-standardize: each row sums to 1, so the spatial lag is a neighbor AVERAGE. +# Without this, towns with more neighbors get mechanically more influence. +w.transform = "r" +y = tw["log_val"].values + +# GLOBAL MORAN'S I — one number, one p-value, for all of Vermont. +# 999 permutations: shuffle values across the map 999x, compare to the real I. +mi = Moran(y, w, permutations=999) +print(f"Moran's I = {mi.I:.3f} | E[I] = {mi.EI:.3f} | p = {mi.p_sim:.3f} | z = {mi.z_sim:.2f}") +#and now making the scatterplot +z = (y - y.mean()) / y.std() # standardize the values +lag_z = w.sparse @ z # spatial lag: neighbors' average, standardized +fig, ax = plt.subplots(figsize=(7, 7)) +ax.scatter(z, lag_z, alpha=0.6, edgecolor="white", linewidth=0.5) +slope = np.polyfit(z, lag_z, 1)[0] # this slope IS Moran's I +xs = np.array([z.min(), z.max()]) +ax.plot(xs, slope * xs, "r", lw=2, label=f"slope = I = {slope:.2f}") +ax.axvline(0, color="gray", ls="--", lw=0.8); ax.axhline(0, color="gray", ls="--", lw=0.8) +ax.set_xlabel("town value (standardized)"); ax.set_ylabel("neighbors' avg value (spatial lag)") +ax.set_title("Moran scatterplot"); ax.legend(); plt.show() +# quadrants preview LISA: upper-right=HH, lower-left=LL, upper-left=LH, lower-right=HL + + +``` + +```{python} +from esda.moran import Moran_Local +#getting the lisa cluster +lisa = Moran_Local(y, w, permutations=999, seed=42) +sig = lisa.p_sim < 0.05 +labels = {1: "High-High", 2: "Low-High", 3: "Low-Low", 4: "High-Low"} # verified encoding +tw["lisa"] = ["Not significant" if not s else labels[q] for q, s in zip(lisa.q, sig)] +print(tw["lisa"].value_counts()) + +cmap = {"High-High": "#d7191c", "Low-Low": "#2c7bb6", "Low-High": "#abd9e9", + "High-Low": "#fdae61", "Not significant": "#e8e8e8"} +fig, ax = plt.subplots(figsize=(8, 11)) +for cls, color in cmap.items(): + sub = tw[tw.lisa == cls] + if len(sub): + sub.plot(ax=ax, color=color, edgecolor="white", linewidth=0.3, label=f"{cls} ({len(sub)})") +ax.legend(loc="upper left", fontsize=9); ax.axis("off") +ax.set_title("LISA clusters — median parcel value (p < 0.05)"); plt.show() + +#high high means high value parcels in a hgih value area, low-low is low cost parcels surrounded by lower cost parcels, high low is high cost parcels surrounded by poorer areas, and low high is vice versa +for cls in ["High-High", "Low-Low", "High-Low", "Low-High"]: + towns_in = tw.loc[tw.lisa == cls, "TOWN"].tolist() + print(f"\n{cls} ({len(towns_in)}):", sorted(towns_in)) +print("\nHH spans counties:", sorted(tw.loc[tw.lisa=="High-High"].merge( + parcels[["TOWN","COUNTY"]].drop_duplicates(), on="TOWN")["COUNTY"].unique())) + + +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +cc = parcels[parcels.TOWN.isin(CHITTENDEN)].copy() + +cc_v = cc.loc[cc.REAL_FLV > 0].to_crs(32145).copy() # PROJECTED CRS: distances must be metric +cc_v["logv"] = np.log10(cc_v["REAL_FLV"]) +cc_pts = cc_v.copy(); cc_pts["geometry"] = cc_v.geometry.centroid # KNN needs points + +# KNN, not contiguity: parcels are effectively points, and we WANT neighbor relationships +# that cross municipal boundaries — that's the whole finding. +# 1. Symmetrize: if A is B's neighbor, make B A's neighbor. Reduces fragmentation. +from libpysal.weights import KNN, W +wk = KNN.from_dataframe(cc_pts, k=8) +wk = W(wk.symmetrize().neighbors) # union of the directed edges +wk.transform = "r" + +# 2. Or use distance bands instead — every parcel within X meters is a neighbor. +from libpysal.weights import DistanceBand +wd = DistanceBand.from_dataframe(cc_pts, threshold=1000, binary=True) # 1km +wd.transform = "r" + +for k in [2, 5, 8, 12, 20]: + w_k = KNN.from_dataframe(cc_pts, k=k); w_k.transform = "r" + m = Moran(yk, w_k, permutations=99) + print(f"k={k:3d} → I = {m.I:.3f}, p = {m.p_sim:.3f}") + +# Fewer permutations at parcel scale — 99 shows the structure; this is the slow step. +mi_p = Moran(yk, wk, permutations=99) +print(f"parcel-level Moran's I = {mi_p.I:.3f} | p = {mi_p.p_sim:.3f}") + +lisa_p = Moran_Local(yk, wk, permutations=99, seed=42) +sigp = lisa_p.p_sim < 0.05 +cc_v["lisa"] = ["ns" if not s else labels[q] for q, s in zip(lisa_p.q, sigp)] +``` + +```{python} +fig, ax = plt.subplots(figsize=(11, 11)) +cc_v[cc_v.lisa == "ns"].plot(ax=ax, color="#ededed", linewidth=0) # gray base first +for cls, color in [("High-High","#d7191c"), ("Low-Low","#2c7bb6"), + ("Low-High","#abd9e9"), ("High-Low","#fdae61")]: + s = cc_v[cc_v.lisa == cls] + if len(s): + s.plot(ax=ax, color=color, linewidth=0, label=f"{cls} ({len(s)})") # linewidth=0: edges kill perf +ax.legend(loc="upper left"); ax.axis("off") +ax.set_title("Chittenden parcels — LISA hot/cold spots (KNN, k=8)"); plt.show() +``` + +#running some machine learning tweaks +```{python} +print(parcels.columns) +``` +#performing sklearn +#problem - it is super easy to predict population from parcel count, so first well run with only categorical value like median value and size values, and then we'll throw in the parcel count correlation that is destined to improve accuracy + +whats more, this is a small data problem (343,000 rows) predicting 255 towns +```{python} +import numpy as np, pandas as pd +from sklearn.linear_model import LassoCV, Lasso +from sklearn.preprocessing import StandardScaler +from sklearn.pipeline import make_pipeline +from sklearn.model_selection import KFold, cross_val_score +#here we group all the valid variables +feat = parcels.groupby("TOWN").agg( + #median parcel value + med_val=("REAL_FLV", lambda s: s[s > 0].median()), + #25, 50, 75th percentile + p25_val=("REAL_FLV", lambda s: s[s > 0].quantile(.25)), + p75_val=("REAL_FLV", lambda s: s[s > 0].quantile(.75)), + #median size + med_lot=("AREAACRESGEOM", "median"), + p25_lot=("AREAACRESGEOM", lambda s: s.quantile(.25)), + med_vpa=("ACREVALUE", "median"), + impr_share=("IMPR_SHARE", "median"), + pct_res=("PURPOSE", lambda s: (s == "PRIMARY RESIDENCE").mean()), + pct_seasonal=("PURPOSE", lambda s: (s == "SEASONAL PROPERTY").mean()), + pct_farmwood=("PURPOSE", lambda s: s.isin(["FARM","WOODLAND"]).mean()), + pct_commercial=("PURPOSE", lambda s: (s == "COMMERCIAL/INDUSTRIAL/UTILITY").mean()), + pct_apartments=("PURPOSE", lambda s: (s == "COMMERCIAL APARTMENTS").mean()), + pct_oos=("OOSOWNER", "mean"), + pct_investment=("INVESTMENTPROP", "mean"), + pct_vacant=("VACANTLAND", "mean"), + pct_exempt=("EXEMPT", lambda s: (s == "YES").mean()), + #THIS IS FOR SECOND PLOT ONLY + n_parcels=("OBJECTID", "size"), +) +#getting the town land area +ta = towns_gdf.to_crs(32145); ta["area_km2"] = ta.geometry.area / 1e6 +feat["area_km2"] = ta.set_index("TOWN")["area_km2"].reindex(feat.index) +#getting the density - this is for tier two only +feat["parcel_density"] = feat.n_parcels / feat.area_km2 + +# rel = "backend/notebooks/parcels/parcels_ml.qmd" +# abs = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv" +pop = pd.read_csv("/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv") +pop = pop[["Town", "year2020"]] +pop["Town"] = pop["Town"].str.upper().str.strip() +replacements = { + "ST. ALBANS CITY": "SAINT ALBANS CITY", + "ST. JOHNSBURY": "SAINT JOHNSBURY", + "ST. ALBANS TOWN": "SAINT ALBANS TOWN", + "ST. GEORGE": "SAINT GEORGE", + "WARNER'S GRANT" : "WARNERS GRANT", + "AVERY'S GORE": "AVERYS GORE", + "WARREN'S GORE" : "WARREN GORE", +} +pop["Town"] = pop["Town"].replace(replacements) +pop.loc[len(pop)] = {"Town": "ESSEX JUNCTION", "year2020": 10942} +pop.loc[pop["Town"] == "ESSEX", "year2020"] = (22094 - 10942) +stragglers = set(parcels["TOWN"]) - set(pop["Town"]) + +#making pop a series +pop_series = pop.set_index("Town")["year2020"].rename("pop") +df = feat.join(pop_series, how="inner") + +print(f"n towns matched: {len(df)} (of {len(feat)} parcel towns, {len(pop_series)} pop towns)") + + + +``` + diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd new file mode 100644 index 00000000..6969a18a --- /dev/null +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -0,0 +1,937 @@ +--- +title: "Vermont Parcels: the testing run notebook [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#ill get to this later +```{python} +from pathlib import Path +import os +import sys +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +sys.path.insert(0, str(_project_root)) +print(os.getcwd()) + +from api.models import FilterSource +from sql_render import sql_filter_block + + + +sql_dir = Path("query/sql/parcels") +src = FilterSource(filter_table="parcels_info", filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", join_type="inner") +sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", [src]) +print(sql) # ← actually read the rendered SQL; watch the CTE appear +``` + +```{python} +#testing tnat the rendering worked correctly +#correctly outputted PUTNEY town property, land with building, with 120.9 acres +import json +from query.processed_db import DB +res = DB.execute(sql, params).fetchone()[0] +fc = json.loads(res) +print(fc["features"][0]) +``` + +```{python} +from query.parcels import get_parcels_geojson, get_parcels_table, get_parcels_filters +from api.models import FilterSource + +src = FilterSource( + filter_table="parcels_info", + filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", + join_type="inner", +) + +# geojson +fc = json.loads(get_parcels_geojson([src])) +print("features:", len(fc["features"])) +print("one feature:", fc["features"][0]) + +# info table +df = get_parcels_table([src]) +print("table rows:", len(df)) +df.head() + +tree = get_parcels_filters() +print(sorted(tree.tree.keys(), key=len, reverse=True)[:10]) # first few towns + +``` +#here, I am starting to look at the towns and area of chittenden county, to figure out what this dataset ould be used for, with all sql queries + +```{python} +import duckdb +con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +con.execute("LOAD spatial") +con.execute("SHOW TABLES").df() +``` +all the towns in chittenden county, and all the ciolumns. +```{python} +chittenden = ('BURLINGTON','SOUTH BURLINGTON','ESSEX','COLCHESTER','WILLISTON', + 'SHELBURNE','WINOOSKI','MILTON','JERICHO','RICHMOND','HINESBURG', + 'CHARLOTTE','WESTFORD','UNDERHILL','ST. GEORGE','BOLTON','HUNTINGTON','BUELS GORE') + +con.execute("""DESCRIBE parcels_info""").df() +``` +First starting off with all the parcel_counts and median value for each chittenden county town, and median acres +```{python} +#median value + parcel count per town +parcels_medians = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {chittenden} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians) + +#now trying again - seeing why these three are nan +#nulls are in burlington, charlotte, bolton - now checking their actual values in the table - no way they are all null +milton = con.execute("SELECT * FROM parcels_info WHERE TOWN='MILTON'").df() +burlington = con.execute("SELECT * FROM parcels_info WHERE TOWN='BURLINGTON'").df() +charlotte = con.execute("SELECT * FROM parcels_info WHERE TOWN='CHARLOTTE'").df() +bolton = con.execute("SELECT * FROM parcels_info WHERE TOWN='BOLTON'").df() + +for item in [milton, burlington, charlotte, bolton]: + print(item[["TOWN", "REAL_FLV", "ACRESGL", "TOWN", "OBJECTID", "SPAN", "CITYGL"]].head(5)) +``` +#okay, we have a problem - there are some NAN values in towns, where there are no values for a row here and there (WHILE ALWAYS HAVING AN OBJECT ID AND TOWN NAME, BUT NOT ANY VALID IDENTIFYING INFO). however, for the towns of Burlington, Charlotte, and BOLTON, it seems there are no values for any of the identifying info - basically the surrounding info is blank, or nan, but the name of the town and the object id is present. I looked it up online and this may result for from these three municipalities doing things their own way, and from prematurely updating the way they handle thigns without having coordinated with the state of vermont. Ill check how it fares for counties foreign + +```{python} +#now checking with different counties - startin with rutland +rutland = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +parcels_medians_rutland = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {rutland} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians_rutland[["TOWN", "parcels", "median_value", "median_acres"]]) + + +#looks like the last four are misiing - Castleton, chittenden, brandon, benson + +#ill do this one more time for washington county +washington_county_towns = ["BARRE TOWN", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER"] + +parcels_medians_washington = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {washington_county_towns} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians_washington[["TOWN", "parcels", "median_value", "median_acres"]]) + + + +``` + +```{python} +con.execute(""" + SELECT COUNT(*) AS n, + SUM((REAL_FLV IS NULL)::INT) AS null_val, + SUM((SPAN IS NULL)::INT) AS null_span, + SUM((ACRESGL IS NULL)::INT) AS null_acres + FROM parcels_info + WHERE TOWN = 'BURLINGTON' +""").df() +``` + +```{python} +con.execute("SELECT DISTINCT PROPTYPE, COUNT(*) FROM parcels_info WHERE TOWN='BURLINGTON' GROUP BY PROPTYPE").df() +``` +MAJOR FINDING ALERT! Burlington, and some of the other largest cities (like Barre) in the state of vermont just didn't send in their data - Burlington's parcels, of which it has 187 (Milton, with all informatino valid, has 4.3 thousand), all but 77 arent even parcels of property, but are instead row_road, water, and row_rail prop type. this proves my theory that the problem of missing data is due to some municipalities just not sending in their data, or collaborating with VCGI like smaller towns might have, thus leaving the leftover data to represent parcels that are borders and non-private property + +#now, we must find the percentage of extant data for each town, to see which ones are a worthy candidate for valid analysis - ill also filter by county for more in-depth analysis. +```{python} + +#incipient converage table +coverage = con.execute(""" + SELECT + TOWN, + COUNT(*) AS total_features, + SUM((PROPTYPE = 'PARCEL')::INT) AS parcels, + SUM((REAL_FLV IS NOT NULL)::INT) AS has_value, + ROUND(100.0 * SUM((REAL_FLV IS NOT NULL)::INT) / COUNT(*), 1) AS pct_valued + FROM parcels_info + GROUP BY TOWN + ORDER BY pct_valued DESC +""").df() + + +#print(coverage["pct_valued"].value_counts()) +#there are 31 towns with no parcel data, and 140 that do +print(sum(coverage["pct_valued"] == 0.0)) +coverage_valid = coverage[coverage["pct_valued"] == 0.0].sort_values(by = "parcels", ascending = False) +print(coverage_valid) + +``` +#here, we investigate colcheseter, which has all parcels present, and burlington and charlotte, which have comparatively less parcels, and no accompanying information on those parcels (median values, etc). we find that 7087 of the 7277 parcels in colcehster are "valued" whereas non for the other towns are - and that the "source" is not distinct for colchester, where it is for burlington & charlotte, leading me to believe the a source of null is VGIS data, and that self provided data tends to be less informative +```{python} +con.execute(""" + SELECT TOWN, + COUNT(*) AS n, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued, + COUNT(DISTINCT SOURCENAME) AS sources + FROM parcels_info + WHERE TOWN IN ('BURLINGTON','COLCHESTER','CHARLOTTE') + GROUP BY TOWN +""").df() + +``` +Echeking the previous hypothesis. source == NAN is VGIS data +```{python} +con.execute(""" + SELECT TOWN, SOURCENAME, COUNT(*) AS n, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued + FROM parcels_info + WHERE TOWN IN ('BURLINGTON','CHARLOTTE','COLCHESTER') + GROUP BY TOWN, SOURCENAME + ORDER BY TOWN, n DESC +""").df() +``` +#really seeing if burlington only has 187 parcels, and that they are all null. in a perfect world, I got the column name wrong, but this isn't the case, and the data is missing. +```{python} +con.execute("SELECT DISTINCT TOWN FROM parcels_info WHERE TOWN ILIKE '%burl%'").df() +``` +#here, we investigate the feature source, where if the SOURCENAME is null, where it happens to be for the towns with a loarge amount of parcels (and extant data), it is likely that the data was provided and submitted by VGIS, where if it is not null, it was self submitted, which leads to strange occurrences where the City of Burlington has 187 parcels, provided by themselves, but no data to go along with it +```{python} +con.execute(""" + SELECT + CASE WHEN SOURCENAME IS NULL THEN 'VGIS standard' ELSE 'self-submitted' END AS pipeline, + COUNT(*) AS features, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued, + COUNT(DISTINCT TOWN) AS towns + FROM parcels_info + GROUP BY 1 +""").df() + +``` + +#work for now : adding a column to parcels_info for county. first we must joing sourcename to parcels info and get +#TODO: add sourcename to parcels_info then run build.py +(quick check for sql database integrity) +```{python} +# import duckdb +# con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +# con.execute("LOAD spatial") +# print(con.execute("SHOW TABLES").df()) +``` +```{python} +towns = con.execute("SELECT DISTINCT TOWN FROM parcels_info").df() +counties = [] +not_stateless = [] +#all counties +ADDISON = ["ADDISON", "BRIDPORT", "BRISTOL", "CORNWALL", "FERRISBURGH", "GOSHEN", "GRANVILLE", "HANCOCK", "LEICESTER", "LINCOLN", "MIDDLEBURY", "MONKTON", "NEW HAVEN", "ORWELL", "PANTON", "RIPTON", "SALISBURY", "SHOREHAM", "STARKSBORO", "WALTHAM", "WEYBRIDGE", "WHITING", "VERGENNES"] +BENNINGTON = ["ARLINGTON", "BENNINGTON", "DORSET", "GLASTENBURY", "LANDGROVE", "MANCHESTER", "PERU", "POWNAL", "READSBORO", "RUPERT", "SANDGATE", "SEARSBURG", "SHAFTSBURY", "STAMFORD", "SUNDERLAND", "WINHALL", "WOODFORD"] +CALEDONIA = ["BARNET", "BURKE", "DANVILLE", "GROTON", "HARDWICK", "KIRBY", "LYNDON", "NEWARK", "PEACHAM", "RYEGATE", "SAINT JOHNSBURY", "SHEFFIELD", "STANNARD", "SUTTON", "WALDEN", "WATERFORD", "WHEELOCK"] +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +ESSEX = ["AVERILL", "BLOOMFIELD", "BRIGHTON", "BRUNSWICK", "CANAAN", "CONCORD", "EAST HAVEN", "FERDINAND", "GRANBY", "GUILDHALL", "LEMINGTON", "LEWIS", "LUNENBURG", "MAIDSTONE", "NORTON", "VICTORY", "AVERYS GORE", "WARNERS GRANT", "WARREN GORE"] +FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS CITY", "SAINT ALBANS TOWN", "SHELDON", "SWANTON"] +GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] +LAMOILLE = ["BELVIDERE", "CAMBRIDGE", "EDEN", "ELMORE", "HYDE PARK", "JOHNSON", "MORRISTOWN", "STOWE", "WATERVILLE", "WOLCOTT"] +ORANGE = ["BRADFORD", "BRAINTREE", "BROOKFIELD", "CHELSEA", "CORINTH", "FAIRLEE", "NEWBURY", "ORANGE", "RANDOLPH", "STRAFFORD", "THETFORD", "TOPSHAM", "TUNBRIDGE", "VERSHIRE", "WASHINGTON", "WEST FAIRLEE", "WILLIAMSTOWN"] +ORLEANS = ["ALBANY", "BARTON", "BROWNINGTON", "CHARLESTON", "COVENTRY", "CRAFTSBURY", "DERBY", "GLOVER", "GREENSBORO", "HOLLAND", "IRASBURG", "JAY", "LOWELL", "MORGAN", "NEWPORT CITY", "NEWPORT TOWN", "TROY", "WESTFIELD", "WESTMORE"] +RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "RUTLAND CITY", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +WASHINGTON = ["BARRE TOWN", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER", "BARRE CITY", "MONTPELIER"] +WINDHAM = ["ATHENS", "BRATTLEBORO", "BROOKLINE", "DOVER", "DUMMERSTON", "GRAFTON", "GUILFORD", "HALIFAX", "JAMAICA", "LONDONDERRY", "MARLBORO", "NEWFANE", "PUTNEY", "ROCKINGHAM", "SOMERSET", "STRATTON", "TOWNSHEND", "VERNON", "WARDSBORO", "WESTMINSTER", "WHITINGHAM", "WILMINGTON", "WINDHAM"] +WINDSOR = ["ANDOVER", "BALTIMORE", "BARNARD", "BETHEL", "BRIDGEWATER", "CAVENDISH", "CHESTER", "HARTFORD", "HARTLAND", "LUDLOW", "NORWICH", "PLYMOUTH", 'POMFRET', 'READING', 'ROCHESTER', 'ROYALTON', 'SHARON', 'SPRINGFIELD', 'STOCKBRIDGE', 'WEATHERSFIELD', 'WEST WINDSOR', 'WESTON', 'WINDSOR', 'WOODSTOCK'] +counties_key = [ADDISON, BENNINGTON, CALEDONIA, CHITTENDEN, ESSEX, FRANKLIN, GRAND_ISLE, LAMOILLE, ORANGE, ORLEANS, RUTLAND, WASHINGTON, WINDHAM, WINDSOR] +names_key = ["ADDISON", "BENNINGTON", "CALEDONIA", "CHITTENDEN", "ESSEX", "FRANKLIN", "GRAND_ISLE", "LAMOILLE", "ORANGE", "ORLEANS", "RUTLAND", "WASHINGTON", "WINDHAM", "WINDSOR"] + +#perfected the 7 without a county +for i in range(len(towns)): + for item in counties_key: + if towns["TOWN"].iloc[i] in item: + counties.append(names_key[(counties_key.index(item))]) + not_stateless.append(towns["TOWN"].iloc[i]) +#adding counties to towns +towns["COUNTY"] = counties +``` +now creating the by county and town availiability map +```{python} +#town level group by +con.execute(""" +SELECT c.COUNTY, i.TOWN, + COUNT(*) AS parcels, + SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued +FROM parcels_info i +LEFT JOIN towns c USING (TOWN) +GROUP BY c.COUNTY, i.TOWN""").df() + +#county_level +con.execute("""SELECT c.COUNTY, + COUNT(*) AS parcels, + SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued +FROM parcels_info i +LEFT JOIN towns c USING (TOWN) +GROUP BY c.COUNTY ORDER BY pct_valued DESC""").df() +``` +now that we have the parcels and parcels' percentages based on county and town, we must read in the vermont VCGI boundary dataset using the api option, and build the chloropleth from there +```{python} +import geopandas as gpd + +# TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +# towns_gdf = gpd.read_file(TOWN_URL) +# print(towns_gdf.shape) +# print(towns_gdf.columns.tolist()) +# print(towns_gdf.crs) + +import geopandas as gpd +import requests + +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + + +base_url = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query" + +offset = 0 +chunk_size = 50 +all_features = [] + +# Fetch data in chunks of 50 to avoid server timeouts +while True: + params = { + "where": "1=1", + "outFields": "*", + "f": "geojson", + "resultOffset": offset, + "resultRecordCount": chunk_size + } + response = requests.get(base_url, params=params) + data = response.json() + + features = data.get("features", []) + if not features: + break + + all_features.extend(features) + + # If a chunk returns less than 50, we've reached the end + if len(features) < chunk_size: + break + + offset += chunk_size + +# Rebuild the full GeoJSON and load it into GeoPandas +full_geojson = {"type": "FeatureCollection", "features": all_features} +towns_gdf = gpd.GeoDataFrame.from_features(full_geojson, crs="EPSG:4326") + +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +#print(towns_gdf.shape) + +print(towns_gdf.columns) + + +#just certifying that the town namese are both uppercase and devoic of whitespace +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +``` +```{python} +#now, merging the town level merge i made earlier with the town_gdf geojson file - changing to have null == 0 +town_coverage = con.execute(""" + SELECT c.COUNTY, i.TOWN, + COUNT(*) AS parcels, + SUM((i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM(CASE WHEN i.REAL_FLV > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) AS pct_valued + FROM parcels_info i + LEFT JOIN towns c USING (TOWN) + GROUP BY c.COUNTY, i.TOWN +""").df() + +print(town_coverage.head(5)) + +towns_map = towns_gdf.merge(town_coverage, on="TOWN", how="left") + +#making sure that all towns are matched +print("unmatched towns:", towns_map["pct_valued"].isna().sum()) +print(towns_map[towns_map["pct_valued"].isna()]["TOWN"].tolist()) +``` +84 towns end up not represented in our coverage map, but that is alright! they don't have parcel rows sent into vgis - this just means that we will have to point that out in the chloropleth +```{python} +#diagnosing the problem at hadn +parcels_towns = set(town_coverage["TOWN"]) +boundary_towns = set(towns_gdf["TOWN"]) +print("in boundary, not parcels:", sorted(boundary_towns - parcels_towns)) +print("in parcels, not boundary:", sorted(parcels_towns - boundary_towns)) +``` +making the cholorpleths - town level +```{python} +#making a valid towns list to dissolve for the county line +towns_valid = towns_map.copy() +towns_valid["geometry"] = towns_valid["geometry"].make_valid() +counties_outline = towns_valid.dissolve(by="COUNTY") + +import matplotlib.pyplot as plt +fig, ax = plt.subplots(figsize=(8, 11)) +towns_map.plot(column="pct_valued", cmap="RdYlGn", legend=True, + edgecolor="gray", linewidth=0.3, ax=ax, + missing_kwds={"color": "lightgray", "label": "No data"}) +#counties_outline.boundary.plot(ax=ax, color="black", linewidth=2) +ax.set_title("Parcel Availability Percentage by Town") +ax.axis("off") +plt.show() +``` + +```{python} +county_coverage = town_coverage.groupby("COUNTY").apply( + lambda g: 100.0 * g["valued"].sum() / g["parcels"].sum() +).reset_index(name="pct_valued") + +towns_map_clean = towns_map.dropna(subset=["COUNTY"]).copy() +towns_map_clean["geometry"] = towns_map_clean["geometry"].make_valid() +counties_map = towns_map_clean.dissolve(by="COUNTY").merge( + county_coverage, on="COUNTY", how="left", suffixes=("_town", "")) +counties_map.plot(column="pct_valued", cmap="RdYlGn", legend=True, edgecolor="black") +``` +```{python} +#step one: making a list of the parcels to investigate +import pandas as pd +import geopandas as gpd +population_csv_path = "backend/notebooks/parcels/local_data/population_2020.csv" + +import os + +print(os.getcwd()) + +# rel_path = "local_data/population_2020.csv" +abs_path = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv" + + +print("abosulte path exists?", os.path.exists(abs_path)) + +pop = pd.read_csv(abs_path) +pop = pop.loc[:, ["_geoid", "Town", "County", "year2020"]] +pop = pop.rename(columns={'Town': 'TOWN', 'County': 'COUNTY', "year2020": "2020_POP", }) +pop["TOWN"] = pop["TOWN"].str.upper() +pop = pop.sort_values(by = "2020_POP", ascending=False) + +#print(pop) + +#adding names to list +all_pop = [] + +print("getting all towns by population:\n") +for i in range(len(pop)): + town = pop.iloc[i]["TOWN"] + popu = pop.iloc[i]["2020_POP"] + #print(str(town) + " " + str(popu)) + all_pop.append(town) + +all_pop = set(all_pop) + + +#print(pop.iloc[23:]) + + +``` +comparing parcel counts between large pop centers +```{python} +info = con.execute("""SELECT TOWN, CITYGL, PROPTYPE, DESCPROP, CAT, RESCODE FROM parcels_info""").df() + +towns = info["TOWN"].unique() +towns = set(towns) + +excluded = (all_pop - towns) +print(excluded) + + + +# excluded_df = pop[pop["TOWN"].isin(excluded)].sort_values(by="2020_POP", ascending=False) +# for _, row in excluded_df.iterrows(): +# print(row["TOWN"], row["2020_POP"]) + + + +# current = [] +# for item in vermont_towns_from_parcels: +# item = item.upper() +# current.append(item) + +# set_of_pop_parcels = set(current) - set(pop["TOWN"]) +# print(set_of_pop_parcels) + +# for item in pop["TOWN"]: +# if item.startswith("WARNER"): +# print(item) + +# what I found - no essex junction in population csv +# however, all saints are there +# in population csv, saint is ST. +# warren's gore is warren gore +# avery's gore is averys gore +# warner's grant is warners grant +``` +#ingesting all cities - +```{python} +import requests, zipfile, io +import geopandas as gpd + +vermont_towns_from_parcels = [ + "Addison", "Albany", "Alburgh", "Andover", "Arlington", "Athens", + "Averill", "Averys Gore", "Bakersfield", "Baltimore", "Barnard", + "Barnet", "Barre City", "Barre Town", "Barton", "Belvidere", + "Bennington", "Benson", "Berkshire", "Berlin", "Bethel", + "Bloomfield", "Bolton", "Bradford", "Braintree", "Brandon", + "Brattleboro", "Bridgewater", "Bridport", "Brighton", "Bristol", + "Brookfield", "Brookline", "Brownington", "Brunswick", "Buels Gore", + "Burke", "Burlington", "Cabot", "Calais", "Cambridge", "Canaan", + "Castleton", "Cavendish", "Charleston", "Charlotte", "Chelsea", + "Chester", "Chittenden", "Clarendon", "Colchester", "Concord", + "Corinth", "Cornwall", "Coventry", "Craftsbury", "Danby", + "Danville", "Derby", "Dorset", "Dover", "Dummerston", "Duxbury", + "East Haven", "East Montpelier", "Eden", "Elmore", "Enosburgh", + "Essex Junction", "Essex", "Fair Haven", "Fairfax", "Fairfield", + "Fairlee", "Fayston", "Ferdinand", "Ferrisburgh", "Fletcher", + "Franklin", "Georgia", "Glastenbury", "Glover", "Goshen", + "Grafton", "Granby", "Grand Isle", "Granville", "Greensboro", + "Groton", "Guildhall", "Guilford", "Halifax", "Hancock", + "Hardwick", "Hartford", "Hartland", "Highgate", "Hinesburg", + "Holland", "Hubbardton", "Huntington", "Hyde Park", "Ira", + "Irasburg", "Isle La Motte", "Jamaica", "Jay", "Jericho", + "Johnson", "Killington", "Kirby", "Landgrove", "Leicester", + "Lemington", "Lewis", "Lincoln", "Londonderry", "Lowell", + "Ludlow", "Lunenburg", "Lyndon", "Maidstone", "Manchester", + "Marlboro","Marshfield", "Mendon", "Middlebury", "Middlesex", + "Middletown Springs", "Milton", "Monkton", "Montgomery", + "Montpelier", "Moretown", "Morgan", "Morristown", "Mount Holly", + "Mount Tabor", "New Haven", "Newark", "Newbury", "Newfane", + "Newport City", "Newport Town", "North Hero", "Northfield", + "Norton", "Norwich", "Orange", "Orwell", "Panton", "Pawlet", + "Peacham", "Peru", "Pittsfield", "Pittsford", "Plainfield", + "Plymouth", "Pomfret", "Poultney", "Pownal", "Proctor", "Putney", + "Randolph", "Reading", "Readsboro", "Richford", "Richmond", + "Ripton", "Rochester", "Rockingham", "Roxbury", "Royalton", + "Rupert", "Rutland City", "Rutland Town", "Ryegate", + "Saint Albans City", "Saint Albans Town", "Saint George", + "Saint Johnsbury", "Salisbury", "Sandgate", "Searsburg", + "Shaftsbury", "Sharon", "Sheffield", "Shelburne", "Sheldon", + "Shoreham", "Shrewsbury", "Somerset", "South Burlington", + "South Hero", "Springfield", "Stamford", "Stannard", "Starksboro", + "Stockbridge", "Stowe", "Strafford", "Stratton", "Sudbury", + "Sunderland", "Sutton", "Swanton", "Thetford", "Tinmouth", + "Topsham", "Townshend", "Troy", "Tunbridge", "Underhill", + "Vergennes", "Vernon", "Vershire", "Victory", "Waitsfield", + "Walden", "Wallingford", "Waltham", "Wardsboro", "Warners Grant", + "Warren Gore", "Warren", "Washington", "Waterbury", "Waterford", + "Waterville", "Weathersfield", "Wells", "West Fairlee", + "West Haven", "West Rutland", "West Windsor", "Westfield", + "Westford", "Westminster", "Westmore", "Weston", "Weybridge", + "Wheelock", "Whiting", "Whitingham", "Williamstown", "Williston", + "Wilmington", "Windham", "Windsor", "Winhall", "Winooski", + "Wolcott", "Woodbury", "Woodford", "Woodstock", "Worcester" +] + + + + +BASE = "https://maps.vcgi.vermont.gov/gisdata/vcgi/packaged_zips/CadastralParcels_VTPARCELS/" + +def fetch_town(town): + r = requests.get(f"{BASE}VTPARCELS_{town}.zip", timeout=60) + r.raise_for_status() + z = zipfile.ZipFile(io.BytesIO(r.content)) + z.extractall(f"Data/parcels/backfill/{town}") + return gpd.read_file(f"Data/parcels/backfill/{town}/VTPARCELS_{town}.shp") + +for town in vermont_towns_from_parcels: + fetch_town(town) + print(town + " Done") + +print("successful bro") +#SOMETHING OF NOTE! the names are capital letter for the first character of each name only, the saints are "Saint", and there are towns and cities for a select few, like barre + +#making a list of all those i need to get +``` +making the view for all of the joined parcel towns +```{python} +import geopandas as gpd +from pathlib import Path + +#KEEP_COLS = ["SPAN","GLIST_SPAN","MAPID","PARCID","PROPTYPE","YEAR","GLYEAR", + # "TOWN","SOURCENAME","SOURCETYPE","SOURCEDATE","EDITDATE","MATCHSTAT", + # "OWNER1","CITYGL","STGL","ZIPGL","DESCPROP","LOCAPROP","CAT","RESCODE", + # "ACRESGL","REAL_FLV","HSTED_FLV","NRES_FLV","LAND_LV","IMPRV_LV", + # "EQUIPVAL","INVENVAL","HSDECL","HSITEVAL","VETEXAMT", + # "EXAMT_HS","EXAMT_NR","UVREDUC_HS","UVREDUC_NR", + # "GLVAL_HS","GLVAL_NR","CRHOUSPCT","MUNGL1PCT","AOEGL_HS","AOEGL_NR", + # "E911ADDR","geometry"] + +#all_columns = ['OGC_FID', 'OBJECTID', 'SPAN', 'GLIST_SPAN', 'MAPID', 'PARCID', 'PROPTYPE', 'YEAR', 'GLYEAR', 'TOWN', 'TNAME', 'SOURCENAME', 'SOURCETYPE', 'SOURCEDATE', 'EDITMETHOD', 'EDITOR', 'EDITDATE', 'MATCHSTAT', 'EDITNOTE', 'OWNER1', 'OWNER2', 'ADDRGL1', 'ADDRGL2', 'CITYGL', 'STGL', 'ZIPGL', 'DESCPROP', 'LOCAPROP', 'CAT', 'RESCODE', 'ACRESGL', 'REAL_FLV', 'HSTED_FLV', 'NRES_FLV', 'LAND_LV', 'IMPRV_LV', 'EQUIPVAL', 'EQUIPCODE', 'INVENVAL', 'HSDECL', 'HSITEVAL', 'VETEXAMT', 'EXPDESC', 'ENDDATE', 'STATUTE', 'EXAMT_HS', 'EXAMT_NR', 'UVREDUC_HS', 'UVREDUC_NR', 'GLVAL_HS', 'GLVAL_NR', 'CRHOUSPCT', 'MUNGL1PCT', 'AOEGL_HS', 'AOEGL_NR', 'E911ADDR', 'geom'] + +def standardize_town(shp_path: Path) -> gpd.GeoDataFrame: + gdf = gpd.read_file(shp_path) + gdf = gdf.to_crs(4326) + gdf["TOWN"] = gdf["TOWN"].str.upper().str.strip() + # missing = [c for c in KEEP_COLS if c not in gdf.columns] + # if missing: + # print(f"{shp_path.name}: missing {missing}") + # gdf = gdf[[c for c in KEEP_COLS if c in gdf.columns]] + return gdf +``` +making the new join +Design philosophy: while it might be a more efficient idea to backfill only those towns missing with complementary data from the parcels that I just ingested, my thought is that if there are parcels missing for whole towns, there cannot be a guarantee that ALL parcels for all towns are there wit hno issue, so im deciding to follow the path of creating a new dataset +```{python} +import pandas as pd +import numpy as np + +frames = [] +for town in vermont_towns_from_parcels: + shp = Path(f"Data/parcels/backfill/{town}") / f"VTPARCELS_{town}.shp" + g = standardize_town(shp) + frames.append(g) + print(town + " Done") +print("frame created successfully") + +#concatenating all the frames from the town parcels into one +backfill = pd.concat(frames, ignore_index=True) +backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) +#making teh objectid and adding +backfill["OBJECTID"] = 1_000_000 + backfill.index +``` +investigating the backfill frame, and adding the county column +also, mapping new values to old columns that I deem valid - run this after having ran the dictionary definition cell +```{python} +#print((backfill.columns)) +# print(backfill.head(5)) + +#adding a counties column - filling the dictionary +counties_dict = {} +unknown = [] +for town in backfill["TOWN"].unique(): + for item in counties_key: + if town in item: + counties_dict[town] = names_key[(counties_key.index(item))] +#adding in sourcename as city and town +city_town_dict = {} + +city_source = backfill[ + (backfill["SOURCENAME"] == "CITY") | + (backfill["SOURCENAME"] == "TOWN") +] + + +#finding if the parcel was found from a local department or not +backfill["SOURCENAME"] = np.where( + backfill["SOURCENAME"].isin(["CITY", "TOWN", "City of Burlington"]), + "LOCAL DEPARTMENT", + "NOT LOCAL DEPARTMENT" +) + +#this code aids in the creation of a column that sees if the primary people live there, or if it is an investment property. +residential_codes = ['R1', 'R2', 'MHL', "MHU"] + +is_residential = backfill["CAT"].isin(residential_codes) +is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == 'N') + +vt_mask = backfill["STGL"].str.startswith("VT", na=False) | backfill["STGL"].str.startswith("Vt", na=False) | backfill["STGL"].str.startswith("05", na=False) | backfill["STGL"].str.contains("VT", na=False) | backfill["STGL"].str.contains("V T", na=False) | backfill["STGL"].str.startswith("vt", na=False) +backfill.loc[vt_mask, "STGL"] = "VERMONT" +canada_mask = backfill["STGL"].str.contains("CANADA", na=False) | backfill["STGL"].str.contains("Canada", na=False) | backfill["STGL"].str.contains("QC", na=False) +backfill.loc[canada_mask, "STGL"] = "CANADA" + +#mapping new items in columns +backfill["INVESTMENTPROP"] = is_residential & is_not_homestead + + + +backfill["VACANTLAND"] = (backfill["LAND_LV"] > 0) & (backfill["IMPRV_LV"].fillna(0) == 0) +equipcode_dict = {"E" : "ELECTRIC UTILITY", "C": "CABLE UTILITY"} +backfill["EXEMPT"] = backfill["STATUTE"].notna().map({True:"YES", False:"NO"}) +backfill["STATUTE"] = backfill["STATUTE"].map(statutes_dict).fillna("No Exemption") +backfill["EXPDESC"] = backfill["EXPDESC"].map(expdesc_dict).fillna("None") +backfill["EQUIPCODE"] = backfill["EQUIPCODE"].map(equipcode_dict).fillna("NOT A UTILITY") +#creating a foreign ownerpship column + + + + +backfill["COUNTY"] = backfill["TOWN"].map(counties_dict) +backfill["RESCODE"] = backfill["RESCODE"].map(rescode_dict) +backfill["CATEGORY"] = backfill["CAT"].map(cat_dict) +backfill["PURPOSE"] = backfill["CAT"].map(purpose_dict) +#adding state abbreivaitions and owner lcoations +backfill["STGL"] = backfill["STGL"].replace(oos_dict) +catch_all_mask = ( + ~backfill["STGL"].isin(['CANADA', 'FOREIGN', 'US TERRITORY']) & + backfill["STGL"].notna() & + ~backfill["STGL"].isin(states) +) +backfill.loc[catch_all_mask, "STGL"] = "UNNAMED AMERICA" +backfill["OOSOWNER"] = backfill["STGL"].fillna("VT") != "VT" + +backfill["ADDRESS"] = backfill["E911ADDR"] + +#adding an area in acres to geometry column, a +if "AREAACRESGEOM" not in backfill: + backfill["AREAACRESGEOM"] = backfill.to_crs(32145).geometry.area / 4046.8564224 +backfill["ACREVALUE"] = np.where( + (backfill.REAL_FLV > 0) & (backfill.AREAACRESGEOM > 0), + backfill.REAL_FLV / backfill.AREAACRESGEOM, np.nan) +``` +RUN THIS CELL TO CHECK THE NULL COUNTS/percentage OF EACH column adn choosing to drop them. +```{python} +#inspeacting the data + +# backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "OBJECTID", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) + +print(backfill["OOSOWNER"].value_counts()) + +# print(backfill[backfill["TOWN"] == "BUELS GORE"]["STGL"].value_counts()) + +# prof = pd.DataFrame({ +# "dtype": backfill.dtypes.astype(str), +# "null_pct": (backfill.isna().mean() * 100).round(1), +# "n_unique": backfill.nunique(dropna=True), +# "sample": [backfill[c].dropna().iloc[0] if backfill[c].notna().any() else None +# for c in backfill.columns], +# }) +# prof.sort_values("null_pct", ascending=False) + +``` +```{python} + +residential_codes = ['R1', 'R2', 'MH'] +is_residential = backfill["PROPTYPE"].isin(residential_codes) +is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == '0') + +states = [ + "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", + "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", + "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", + "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", + "SD", "TN", "TX", "UT", "VA", "WA", "WV", "WI", "WY", "VT", "DC" +] + + + +#print(backfill[~backfill["STGL"].isin(states)]["STGL"].value_counts()) + +#print(backfill[backfill["STGL"] == "MA"][["STGL", "ZIPGL"]].value_counts()) + +print((backfill["STGL"].value_counts())) + + +#print(len(states_except_vt)) + +#print(backfill[["TOWN", "REAL_FLV", "CAT", "INVESTMENTPROP"]]) +#print(backfill[["CAT", "DESCPROP"]].value_counts()) +#print(backfill[["CATEGORY", "TOWN", "ACRESGL", 'REAL_FLV']].value_counts().sort_index(level="REAL_FLV", ascending=False)) + +#print(backfill[backfill["STGL"] == "MA"][["CITYGL", "STGL"]].value_counts()) + +# mismatch_count = (backfill["E911ADDR"] != backfill["ADDR1"]).sum() +# print(f"Number of mismatched addresses: {mismatch_count}") + + + +#print(backfill["CAT"].value_counts()) +#print(backfill["CAT"].isin(residential_codes).value_counts()) +#print(backfill["VACANTLAND"].value_counts()) + +# foreign_investment = backfill[backfill["VACANTLAND"] == True] +# print(foreign_investment["TOWN"].value_counts()) + + +# data_directory = Path("Data") +# parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + +# parcels_parquet = Path("Data/parcels/parcels_p.parquet") +# parcels_fgb = Path("Data/parcels/parcels_f.fgb") + + +# con = duckdb.connect() +# con.execute("INSTALL spatial; LOAD spatial;") + +# if not parcels_fgb.exists(): +# con.execute(f""" +# COPY ( +# SELECT * FROM ST_Read('{parcel_path}') +# WHERE geom IS NOT NULL +# ) +# TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') +# """) + + +# prof = pd.DataFrame({ +# "dtype": backfill.dtypes.astype(str), +# "null_pct": (backfill.isna().mean() * 100).round(1), +# "n_unique": backfill.nunique(dropna=True), +# "sample": [backfill[c].dropna().iloc[0] if backfill[c].notna().any() else None +# for c in backfill.columns], +# }) +# prof.sort_values("null_pct", ascending=False) +``` +RUN THIS TO DROP COLUMNS FROM BACKFILL +```{python} +backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) + +``` +SAVING THE BACKFILL df AS A FILE +```{python} +from pathlib import Path +import geopandas as gpd +from shapely.geometry import MultiPolygon + +out_path = Path("Data/parcels/all_parcels_vermont.fgb") +out_path.parent.mkdir(parents=True, exist_ok=True) + +bad_geom = backfill.geometry.isna() | backfill.geometry.is_empty +gdf = backfill.loc[~bad_geom].copy() + +assert gdf.geom_type.isin(["Polygon", "MultiPolygon"]).all(), gdf.geom_type.value_counts() +gdf["geometry"] = gdf.geometry.apply( + lambda g: MultiPolygon([g]) if g.geom_type == "Polygon" else g +) + +gdf.to_file(out_path, driver="FlatGeobuf") +print("wrote", out_path, "|", gdf.shape) + + +#quick check +parcels = gpd.read_file("Data/parcels/all_parcels_vermont.fgb") +print(parcels.shape, "|", parcels.crs) +print(parcels.geom_type.value_counts()) + +# quick round-trip check — did the derived cols survive with sane values? +# print(parcels[["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]].apply(lambda s: s.value_counts()).T) +# parcels.head() +``` +```{python} +print(gdf.columns) + +``` + + + + +Bottom of the page cell where I am storing the dictionaries of transmuted columns +```{python} +statutes_dict = { + '3848:3849': 'Business Inventory & Equipment', + '3848:38:00': 'Business Inventory & Equipment', + '3840': 'Charitable, Fraternal, or Rescue', + '3840;5405a(a)(4)': 'Charitable/Rescue (inc. Education Tax)', + '3840;54': 'Charitable/Rescue (inc. Education Tax)', + '2741': 'Tax Stabilization Contract', + '24/2741': 'Tax Stabilization Contract', + '3832': 'Public, Pious, or Charitable', + '3832(1)': 'Out-of-Town Municipal Property', + '3832(7)': 'Health or Recreational Property', + '3832(7)(B)': 'Non-profit Ice Skating Rink', + '3832(7B': 'Non-profit Ice Skating Rink', + '5401': 'Statewide Education Tax Exception', + '3752(7)': 'Agricultural / Current Use' +} + +expdesc_dict = { + 'Statutory': 'State Law Exemption', + 'Solar Plant': 'Solar Energy Facility', + 'Non-Approved (Voted)': 'Local Town-Voted Exemption', + 'Qualified Housing Units': 'Affordable / Qualified Housing', + 'Grandfathered': 'Pre-existing Historical Exemption', + 'Partial-Statutory': 'Partial State Law Exemption', + 'Municipal Contract (Owner Pays)': 'Payment in Lieu of Taxes (PILOT)', + 'Ski Lifts / Snow Making Equip': 'Ski Resort Equipment', + 'Court Ordered': 'Judicially Mandated Exemption', + 'Wind Plant': 'Wind Energy Facility' +} + +rescode_dict = { + "T" : "TOWN RESIDENT", + "NS" : "OUT OF STATE RESIDENT", + "S" : "VERMONT RESIDENT", + "C" : "CORPORATION/ENTITY", + "c" : "CORPORATION/ENTITY" +} + +cat_dict = { + "R1": "Residential I (Under 6 Acres)", + "R2": "Residential II (6 Acres or More)", + "M": "Miscellaneous", + "O": "Other", + "C": "Commercial", + "MHL": "Mobile Home Landed (With Land)", + "S1": "Seasonal I (Under 6 Acres)", + "MHU": "Mobile Home Unlanded (Without Land)", + "W": "Woodland", + "S2": "Seasonal II (6 Acres or More)", + "F": "Farm", + "CA": "Commercial Apartments", + "I": "Industrial", + "UE": "Utility Electric", + "UO": "Utility Other" +} + +purpose_dict = { + "R1" : "PRIMARY RESIDENCE", + "R2" : "PRIMARY RESIDENCE", + "MHL" : "PRIMARY RESIDENCE", + "MHU": "PRIMARY RESIDENCE", + "S1": "SEASONAL PROPERTY", + "S2": "SEASONAL PROPERTY", + "W": "WOODLAND", + "F": "FARM", + "CA" : "COMMERCIAL APARTMENTS", + "M": "NOT LISTED", + "O" : "NOT LISTED", + "C": "COMMERCIAL/INDUSTRIAL/UTILITY", + "I": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UE" : "COMMERCIAL/INDUSTRIAL/UTILITY", + "UO": "COMMERCIAL/INDUSTRIAL/UTILITY" +} + + +oos_dict = { + "VERMONT" : "VT", +} +oos_dict.update(dict.fromkeys(['QC', 'QC CANADA', 'PQ', "QUEBEC", "ON", "ONTARIO", "QUE", "BC", "ONT", "CAN"], 'CANADA')) +oos_dict.update(dict.fromkeys(["MASS", "MA."], 'MA')) +oos_dict.update(dict.fromkeys(["MICHIGAN"], 'MI')) +oos_dict.update(dict.fromkeys(["OHIO"], 'OH')) +oos_dict.update(dict.fromkeys(["CT."], 'CT')) +oos_dict.update(dict.fromkeys(["R.I."], 'RI')) +oos_dict.update(dict.fromkeys(["W VA"], 'WV')) +oos_dict.update(dict.fromkeys(["MARYLAND"], 'MD')) +oos_dict.update(dict.fromkeys(["N CAROLINA"], 'NC')) +oos_dict.update(dict.fromkeys(["NEW YORK", "N.Y.", "12513", "N Y"], 'NY')) +oos_dict.update(dict.fromkeys(["FLORIDA", "FLA"], 'FL')) +oos_dict.update(dict.fromkeys(["ENGLAND", "AE", "UNK", "BERMUDA", "UK", "VY", "FRANCE", "IND", "ARUBA", "GERMANY", "IRELAND", "SWITZERLAN", "QLD AUS", "LIN", "0R", "BERLIN", "AUSTRALIA", "NS", "FWI", "BAHAMAS"], 'FOREIGN')) +oos_dict.update(dict.fromkeys(["VI", "PR", "GUAM"], 'US TERRITORY')) + + +``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd new file mode 100644 index 00000000..393a88c4 --- /dev/null +++ b/backend/notebooks/parcels/table_build.qmd @@ -0,0 +1,366 @@ +--- +title: "Vermont Parcels: SQL Table Build" +author: Isaac Wedaman +date: today +description: Exploratory build of SQL tables from the VT Statewide Standardized Parcel Data. +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +## SETUP +importing required libraries and setting the project root in the working directory +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) +``` + + +## Data Collection +here, we read in the parcels from the dataset, which is kept locally. we create both a parquet and an FGB rendering of the original geojson dataset. Then, we load the spatial extension for duckdb, and create a view structure of the data, for more extensive data analysis donw the line. +```{python} +data_directory = Path("Data") +parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + +parcels_parquet = Path("Data/parcels/parcels_p.parquet") +parcels_fgb = Path("Data/parcels/parcels_f.fgb") + + +con = duckdb.connect() +con.execute("INSTALL spatial; LOAD spatial;") + +if not parcels_fgb.exists(): + con.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') + """) + +if not parcels_parquet.exists(): + con.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_parquet}' (FORMAT PARQUET) + """) + +``` + +```{python} +#creating the parcels_raw view on which to query things to learn +con.execute("CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/Parcels/parcels_f.fgb')") + +col_list = con.execute("SELECT * FROM parcels_raw LIMIT 0").df().columns.to_list() +print(col_list) +``` + +Here, we print all the columns in the parcels_raw view, of which there are many. commented out is the code to yield the total row count, with is 191156. Now looking at the column types and their respective names (what they represent in real life), and making a reproducible code chunk to investigate what each means and what it looks like + +HERE ARE THE COLUMNS +OGC_FID: feature id +OBJECTID YEAR GLYEAR: specific tax year REALFLV: total assessed value of real estate HSTED_VAL:homestead value NRES_FLV: non residentialhomestead value LAND_LV:land value, IMPROVE_LV: improvement land value, EQUIPVAL_ equipment value, INVENVAL:inventory value, HSITEVAL: home value and two surrounding acres EXAMPT_HS: exemption amount homestead. EXAMPT_NR: exemption amount non-resdiential. UVREDUC_HS: use value reduction homestead. UVREDUC_NR: use value reduction non-residential + +SPAN: school property access number - property identifier. GLIST_SPAN: grand list property span number. MAPID: municipality tax identifier. PARCID: parcel identifier. PROPTYPE: what is on the parcel itself. TOWN: town name in code. TNAME: fully written out town name. SOURCENAME: name of infor source. SOURCETYPE: original data type. SOURCEDATE: date of original map publishing. EDITMETHOD: method to draw map. EDITOR initials. EDITDATE. MATCHSTAT: how well the parcel map shape attached to grand tax record. EDITNOTE. OWNER1. OWNER2. ADDRGL1. ADDRGL2. CITYGL. STATEGL. ZIPGL. DESCPROP: description of property. LOCAPROP. CAT: category of parcel use. RESCODE: where doesthe resident live - out of state?. EQUIPCODE: taxable equipcode. HSDECL: homestead declaration. ENDDATE: end of parcel data. STATUTE - reference ot vermont law if applicable to parcel. E911 ADRR: 911 address + +ACRESGL: taxable acreage. GLVAL_HS: 1% of homestead value. GLVAL_NR: non residential - 1 percent of non residential tax. CRHOUSPCT: percentage of property subject to housing statutes. MUNGL1PCT: total value on which to calculate 1% of value per municipality. AOEGL_HS: homestead value for calculating education taxation. AOEGL_NR: non residential taxation value calculated for education property. + +geom: The actual spatial map data (the points, lines, and polygons) that draws the shape and physical boundaries of the parcel. EPSG:4326 signifies that the coordinates are mapped using standard Latitude and Longitude (WGS 84), exactly like a GPS uses. +```{python} +described_parcels_raw = con.execute("DESCRIBE parcels_raw").df() + +column_types = described_parcels_raw["column_type"].unique().tolist() +columns_by_type = {} + +for thing, item in described_parcels_raw.iterrows(): + for index, row in enumerate(column_types): + if item["column_type"] == row: + if row not in columns_by_type: + columns_by_type[row] = [] + + columns_by_type[row].append(item["column_name"]) +print(columns_by_type) + +# for key, value in columns_by_type.items(): +# print(f"{key}: {value}") + +``` + +```{python} +#now, making the reproducible code to get the values of each column +name = "ACRESGL" +col = con.execute(f"""SELECT {name} FROM parcels_raw ORDER BY {name} ASC""").df() +print(col.describe()) +print(col.value_counts()) + +``` + +## DATA ANALYSIS - FINDING WHAT IS IMPORTANT +#methods to a) count get the unique values listed, and b) get the unique values counted. my thought was to create a reproducible code chunk to investigate the unique values of each column, per the column variable +```{python} + +con.execute("SELECT YEAR FROM parcels_raw ORDER BY YEAR ASC").df() + +column = "SOURCENAME" + +con.execute(f""" +SELECT + {column}, + COUNT({column}) AS instances, + SUM(COUNT({column})) OVER () AS total_number +FROM parcels_raw +GROUP BY {column} +ORDER BY instances DESC; +""").df() +``` +the following code chunk is a method to get the unique values of a column, after having gotten their column names in a list. from there, we loop through the list of column names and count the nulls, then pring out the nulls per column, outputted as a dataframe +```{python} +#summing the nulls per column +parcel_columns = described_parcels_raw["column_name"].tolist() +nulls = [] + +for col in parcel_columns: + result = con.execute(f""" + SELECT SUM(CASE WHEN "{col}" IS NULL THEN 1 ELSE 0 END) + FROM parcels_raw; + """) + values = result.fetchone()[0] + count = values if values is not None else 0 + nulls.append(count) + +nulls_columns = pd.DataFrame({'Column': parcel_columns, 'Nulls': nulls}) + +``` +Something of note for posterity is that there were a few rows that contained 9577 null values. i sook to find out whether those absences were correalted, or random, and what that might mean with respect to the story of the data. +```{python} +#print(nulls_columns) +#found that 9577 nulls occurs the most frequently +#these are them +columns_to_read = nulls_columns[nulls_columns["Nulls"] == 9577]["Column"] +columns_to_read = columns_to_read.tolist() +print(columns_to_read) +#columns where null is 0 +no_nulls = nulls_columns[nulls_columns["Nulls"] == 0]["Column"] +no_nulls = no_nulls.tolist() +print(no_nulls) +uniques = [] +for col in no_nulls: + result = con.execute(f""" + SELECT COUNT(*), COUNT(DISTINCT {col}) + FROM parcels_raw; + """) + values = result.fetchone()[1] + count = values if values is not None else -25 + uniques.append(count) +print(uniques) +``` +This method gets a few values important for the measuring of data, similar to r's "summarize" function, which helps in a completely developed understanding of the data +```{python} +#describing the acresgl +stats = con.execute("""SELECT + COUNT(ACRESGL) AS total_count, + COUNT(*) - COUNT(ACRESGL) AS missing_values, + MIN(ACRESGL) AS min_value, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY ACRESGL) AS first_quartile, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY ACRESGL) AS median, + AVG(ACRESGL) AS mean, + PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY ACRESGL) AS third_quartile, + MAX(ACRESGL) AS max_value +FROM parcels_raw; +""").df() +print(stats.head()) +span = con.execute("""SELECT SPAN AS span, GLIST_SPAN AS glistspan, PARCID AS parcid FROM parcels_raw""").df() +print(span) + +#something of note + + +acres_gl = con.execute(f"""SELECT ACRESGL AS acresgl, COUNT(*) AS n FROM parcels_raw GROUP BY ACRESGL ORDER BY n DESC""").df() +print(acres_gl.head(10)) +# fitz_form = (ST_Area(ST_Transform(geom, 'EPSG:4326', 'EPSG:32145', always_xy := TRUE)) / 4046.86).df() +# print(fitz_form.head(10)) +``` +#trying to find the +```{python} +print(columns_to_read) +span_vs_null = con.execute("""SELECT SPAN AS span, GLIST_SPAN AS glistspan, PARCID AS parcid, TNAME, ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV FROM parcels_raw""").df() +#span_vs_null.head(10) +span_vs_null[span_vs_null["TNAME"]== "Putney"].head() +``` +there are 9577 in a few rows, as we saw above. here, Im looking to discover if those few rows are all a few correlated nulls, or random. +```{python} +#im picking to compare parcel id (PARCID) and acresgl +con.execute(""" +SELECT + SUM((PARCID IS NULL AND ACRESGL IS NULL)::INT) AS both_null, + SUM((PARCID IS NULL AND ACRESGL IS NOT NULL)::INT) AS only_a +FROM parcels_raw +""").df() +#it looks like there are both correlated!!! +#now seeing what non-null they have in common +con.execute(""" +SELECT TNAME, COUNT(*) AS n +FROM parcels_raw +WHERE PARCID IS NULL +GROUP BY TNAME +ORDER BY n DESC +""").df() +#this is an interesting story: it looks like, reading from the output of 9577 entrances that also have town == null, tat the map needs to be completed with parcels, but those parcels arent valuable, important, or pertinent enough to warrant having their information filled out + +``` + +## making sure that the geometry of the data is valid +Here, I am checking that the parcel type is indeed polygon/multigon +```{python} +#all polygon/multigon +con.execute("""SELECT ST_GeometryType(geom) AS gtype, COUNT(*) FROM parcels_raw GROUP BY gtype""").df() + +#checking the CRS (coordinate reference system) +#output 0 {'min_x': -73.4374417425543, 'min_y': 42.73912... +con.execute("SELECT ST_Extent(ST_Union_Agg(geom)) FROM parcels_raw").df() + + +``` + + + +## REASONING/CONNECTING TO REAL LIFE DATA +While searching for the correlation or backstory on what might make an entrance with TNAME == null, I found that, when also keeping in mind their property type (var name PROPTYPE), that the majority are unlisted parcels (5948), and the remaining are untaxable service parcels, in the order of row roads, water, row rail, row trail, and railroad parcels, that, I guess, are not important enough to be designated a town name +```{python} +null_tname = con.execute("""SELECT * FROM parcels_raw WHERE TNAME IS NULL""").df() + +null_tname["PROPTYPE"].value_counts() + +#checking if town name and TNAME are the same, or correlated +tnames = con.execute("""SELECT TNAME AS tname, TOWN AS town FROM parcels_raw""").df() +print(tnames["tname"].value_counts(), tnames["town"].value_counts()) + +#TOWN HAS NO NULLS +con.execute("""SELECT SUM((TOWN IS NULL)::INT), SUM((TNAME IS NULL)::INT) FROM parcels_raw""").df() +``` + +```{python} +con.execute(""" +SELECT SOURCENAME, YEAR, COUNT(*) AS n +FROM parcels_raw +WHERE TNAME IS NULL +GROUP BY SOURCENAME, YEAR +ORDER BY n DESC +""").df() +con.execute(""" +SELECT + TNAME IS NULL AS no_town, + COUNT(*) AS n, + AVG(ST_Area(geom)) AS avg_area, + MEDIAN(ST_Area(geom)) AS med_area +FROM parcels_raw +GROUP BY no_town +""").df() + +con.execute("""SELECT COUNT(*), COUNT(DISTINCT SPAN), SUM((SPAN IS NULL)::INT) FROM parcels_raw""").df() +``` + +## JOIN KEY CREATION +While SPAN (school property access number) would be the best join key, since it represents the government itself's storage manner for parcels, it isnt fully non null, and there are about 18000 rows that are non-uniue in this dataset, where the row "OBJECTID" is fully present, representing a purely trustworthy join key. +```{python} +#just checking if the row count and object id are the same length +print(con.execute("SELECT COUNT(*) FROM parcels_raw").df()) +print(con.execute("SELECT COUNT(DISTINCT OBJECTID) FROM parcels_raw").df()) +#they are, so we can use objectid as the join key, and keep span around for more techincal tasks. + + +#continuing, we are summing where both tname (9577 nulls) and span (7862 null) are both null. +print(con.execute("""SELECT + SUM((SPAN IS NULL AND TNAME IS NULL)::INT) AS both_null, + SUM((SPAN IS NULL AND TNAME IS NOT NULL)::INT) AS span_only, + SUM((SPAN IS NOT NULL AND TNAME IS NULL)::INT) AS tname_only +FROM parcels_raw""").df()) +#something of note: there are no span-null only rows, 7862 where both are nulls, and 1715 tname only nulls, meaninng that the span nulls are a subset of the tname nulls. +``` +```{python} +#before we create the tables, I thought it ould be nice to enrich the info one: +con.execute("""SELECT CAT, RESCODE, CITYGL, STGL, ZIPGL, E911ADDR, DESCPROP, LOCAPROP FROM parcels_raw""").df() +``` + +## CREATING THE INFAMOUS TABLES +```{python} +#just before I forget, I think it is good practice to have both the TOWN and TNAME columns as uppercase, so that there aren't any mixups, or Colchester != COLCHESTER, ETC. +#this is the final view from which ot ship off other changes and add them to this final iteration, is my thought. +con.execute(""" + CREATE OR REPLACE VIEW final_view AS + SELECT + * EXCLUDE (TOWN, TNAME), + UPPER(TOWN) AS TOWN, + UPPER(TNAME) AS TNAME + FROM parcels_raw; +""") +``` +### Finally sending off the tables created +Fully creating three tables, parcels_geom, which has the objectid and geom, the parcels_info table, which ahs information like the access nuber, town, source, and year, and the parcels_value table, which has the acres, and all sorts of value metrics. - got rid of value the table +```{python} +#changing the name to {thing}.parcel +print(con.execute("DESCRIBE final_view").df()["column_name"].tolist()) + +con.execute(""" +CREATE OR REPLACE VIEW geom AS +SELECT OBJECTID, geom FROM final_view +""") + +con.execute("""CREATE OR REPLACE VIEW info AS +SELECT OBJECTID, SPAN, CAT, RESCODE, PARCID, CITYGL, TOWN, ZIPGL, PROPTYPE, DESCPROP, SOURCENAME, YEAR, + ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV, SOURCENAME +FROM final_view""") + +#wont ship tax table just yet! - commented out for now +#con.execute("""CREATE OR REPLACE VIEW tax AS SELECT OBJECTID, ZIPGL, DESCPROP, REAL_FLV, EQUIPVAL, AOEGL_NR FROM final_view""") + +con.execute("""SELECT CAT FROM info""").df() +``` + +somethind of note to consider. the build.py agglomerates parquets, not FGBS, so im +```{python} +path = Path("Data/_Processed/parcels") +path.mkdir(parents=True, exist_ok=True) + +#changing the name to be more terse adn to the point, in the hopes that the glob agglomerator will add the "parcels/" prefix +#Not doing tax just yet +for item in ["info", "geom"]:#tax commented out + con.execute(f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)") +``` + +#checking that it worked - (especially the uppercasing of the town name and columns) +```{python} +con.execute(f"SELECT DISTINCT ZIPGL FROM '{path / 'info.parquet'}' LIMIT 15").df() +``` + +```{python} +import duckdb +db = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +print(db.execute("SHOW TABLES").df()) +print(db.execute("SELECT COUNT(*) FROM parcels_info").df()) +print(db.execute("SELECT COUNT(*) FROM parcels_geom").df()) +``` + +```{python} +con.execute("SELECT DISTINCT RESCODE FROM parcels_raw").df() +``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build_final.qmd b/backend/notebooks/parcels/table_build_final.qmd new file mode 100644 index 00000000..b21cc93c --- /dev/null +++ b/backend/notebooks/parcels/table_build_final.qmd @@ -0,0 +1,776 @@ +--- +title: "Vermont Parcels: making the new tables" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#HELPING FIND THE DATA AFTER KERNEL RESTART +```{python} +import os +from pathlib import Path + +print("cwd (where the fresh kernel started):", Path.cwd()) + +# climb until we hit the folder containing api/ — identical anchor to your other notebooks +root = Path.cwd() +while root != root.parent and not (root / "api").exists(): + root = root.parent +assert (root / "api").exists(), f"never found an api/ folder climbing up from {Path.cwd()}" +os.chdir(root) +print("anchored to:", Path.cwd()) + +FGB = "Data/parcels/all_parcels_vermont.fgb" +print("exists now?:", os.path.exists(FGB)) +``` +```{python} +import geopandas as gpd, pandas as pd, numpy as np +import matplotlib.pyplot as plt +from matplotlib.colors import LogNorm +from pathlib import Path + +#getting the FBG from file storage, making the parcels geo df, and then printing the shape and CRS, which should be EPSG 4326, then pirnting the type of all geom entries - hopefully it is multi polygon +FGB = "Data/parcels/all_parcels_vermont.fgb" + +parcels = gpd.read_file(FGB) +print("shape:", parcels.shape, "| CRS:", parcels.crs) +print(parcels.geom_type.value_counts()) + + +#makinga failsafe self healing join key in the case object id is missing - just so objectid is always present as a thing to join on +if "OBJECTID" not in parcels.columns: + print("!! OBJECTID not in file — minting a fresh stable id.") + parcels = parcels.reset_index(drop=True) + parcels["OBJECTID"] = 1_000_000 + parcels.index +assert parcels["OBJECTID"].is_unique, "OBJECTID not unique — unusable as a join key" +assert parcels["OBJECTID"].notna().all(), "OBJECTID has nulls" +#objects count should be 343875 +print("OBJECTID ok | n =", parcels["OBJECTID"].nunique()) + +# #checking if the columns that I created are present, and then printing the column and then checking both the true and false values of each item +# for c in ["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]: +# if c in parcels: +# print(f"{c:16s} →", dict(parcels[c].value_counts(dropna=False))) + +#getting the null rates of each important column in the dataset for informative purposes +# key = ["TOWN","COUNTY","REAL_FLV","LAND_LV","IMPRV_LV","ACRESGL","CAT","PURPOSE"] +# print((parcels[[c for c in key if c in parcels]].isna().mean() * 100).round(1)) +``` +making a reusable chloropleth filter +```{python} +import requests +#getting the town boudnaries using rest api. well need this for hte chloropleths, and making the towns gdf from the town_url +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + +# 1. Download the data robustly using requests +print("Downloading town boundaries...") +r = requests.get(TOWN_URL) +r.raise_for_status() # This will yell if the URL is broken or blocked + +# 2. Parse the JSON using Python's built-in dictionary parser +data = r.json() + +# 3. Read the features into a GeoDataFrame +towns_gdf = gpd.GeoDataFrame.from_features(data["features"]) + +# ArcGIS GeoJSON defaults to EPSG 4326 (WGS84 Lat/Lon). We must set it explicitly +# before you project it to 32145 later! +towns_gdf.set_crs(epsg=4326, inplace=True) + +print(towns_gdf.columns) + +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +towns_gdf = towns_gdf[["TOWN", "geometry"]].copy() + +print(f"Successfully loaded {len(towns_gdf)} towns!") + + +#defining the functino for a chloropleth map. +"""values: a Series whose INDEX is TOWN and whose values are the metric to color by.""" # - so we can +#see by map +# Series → 2-col DataFrame so it can be merged onto the polygons by TOWN + + +def town_choropleth(values: pd.Series, title, log=False, cmap="viridis"): + + #.values.rename takes the numbers column of the seies and names it "val". the columns are now names TOWN and val + #We now commit a left merge by town + v = values.rename("val").reset_index(); v.columns = ["TOWN", "val"] + m = towns_gdf.merge(v, on="TOWN", how="left") + #this code is for if we pass in log as a true value + if log: + #we cannot take 0 or negative numbers in our logarithmic + m.loc[m["val"] <= 0, "val"] = np.nan + fig, ax = plt.subplots(figsize=(8, 12)) + + #assigning data values - the keywords dictionary + kw = dict(column="val", cmap=cmap, legend=True, ax=ax, + edgecolor="gray", linewidth=0.3, + missing_kwds={"color": "lightgray", "label": "no data"}) + #if logarithmic - + if log: + #getting the true range after dropping nans + pos = m["val"].dropna() + #this line says that the colors should be added in a gradient based on the progression from lowest to highest value. we add a new keyword key of norm for that value + kw["norm"] = LogNorm(vmin=pos.min(), vmax=pos.max()) + + #plotting m, towns_gdf, and allthe KW dictionary, which contain the instructions for logarithmic plotting if we send them in. we also output the towns with no values for the inputted considered column. + m.plot(**kw); ax.set_title(title); ax.axis("off"); plt.show() + print("unmatched towns:", m["val"].isna().sum()) + return m +``` +before vs after backfill +```{python} +import duckdb +#connect access old table created before I updated the parcel collection method +con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) # the OLD layer lives here + +#parcels represents the updated dataset, so new_n, new_valued, and new_sum represent the updated parcels' length, parcel valuue, and sum of value +new_n = len(parcels) +new_valued = int((parcels["REAL_FLV"] > 0).sum()) +new_sum = float(parcels.loc[parcels["REAL_FLV"] > 0, "REAL_FLV"].sum()) + +#getting the old count, value per parcel, adn sum of total parcels from antiuated dataset +old = con.execute(""" + SELECT TOWN, COUNT(*) n, + SUM((REAL_FLV > 0)::INT) valued, -- ::INT casts bool→0/1 to sum + SUM(CASE WHEN REAL_FLV > 0 THEN REAL_FLV ELSE 0 END) val_sum + FROM parcels_info GROUP BY TOWN +""").df() +#getting the same old calculations +old_n, old_valued, old_sum = int(old.n.sum()), int(old.valued.sum()), float(old.val_sum.sum()) + +#printing the founds stats of the old and new datasets, with a change in row represented +print(f"BEFORE: {old_n:>8,} parcels | {old_valued:>8,} valued ({100*old_valued/old_n:4.1f}%) | ${old_sum:,.0f}") +print(f"AFTER : {new_n:>8,} parcels | {new_valued:>8,} valued ({100*new_valued/new_n:4.1f}%) | ${new_sum:,.0f}") +print(f"Δ : {new_n-old_n:>+8,} parcels | {new_valued-old_valued:>+8,} valued | ${new_sum-old_sum:+,.0f}") + +# --- tag each rebuilt parcel by how the OLD layer treated its town ---------- +# Every row here came from a town download, so "origin" really means: was this town +# already covered in the statewide layer, broken there, or absent entirely? + +#unique problem - we have the old and new datasets, but both also have broken parcels without valuation, like public property and stuff like that +old_all = set(old.TOWN) +old_broken = set(old.loc[old.valued == 0, "TOWN"]) +#quick wading function for all items inputted, if not in old dataset, the data must have been absent, if it was in the old broken set it was broken, and we return a list of what is covered in out statewide data analysis, as "covered in statewide" +def origin(t): + if t not in old_all: return "not_in_old_statewide" + if t in old_broken: return "found_in_rebuild" + return "covered_in_statewide" + +#making a new column for the provenance of parcels, either restored or absent, by running our function on town +parcels["data_origin"] = parcels["TOWN"].map(origin) + +#grouping the parcels by data_origin, and also objectid - size is row count per group - grouped by data_origin +#we also sum the real_flv for all with values over 0, meaning informed parcels +by_origin = (parcels.assign(valued=parcels["REAL_FLV"] > 0) + .groupby("data_origin") + .agg(parcels=("OBJECTID", "size"), + valued=("valued", "sum"), + grand_list=("REAL_FLV", lambda s: s[s > 0].sum()))) +print("\n", by_origin, sep="") + +#getting the first 15 of the towns that weren't included in the old parcels dataset +print("\nin rebuilt, no old-layer match:", sorted(set(parcels.TOWN) - old_all)[:15]) + +#making a plot for the provenance of each particular town's parcels +#merging the towns based on data_origin. im generalizing with a few select towns by dropping all duplicated (namely burlington), but I think its worth it +prov = towns_gdf.merge( + parcels[["TOWN","data_origin"]].drop_duplicates("TOWN"), + on="TOWN", how="left") +#being safe +prov["data_origin"] = prov["data_origin"].fillna("no parcels") + +#making the plot +fig, ax = plt.subplots(figsize=(8, 12)) + +prov.plot(column="data_origin", categorical=True, legend=True, + ax=ax, edgecolor="black", linewidth=0.3) +ax.set_title("Parcel Origin vs. the old statewide layer"); ax.axis("off"); plt.show() + +#outputting towns with no parcels - three are included +print("TOWNS WITH NO PARCELS:") +print(prov[prov["data_origin"] == "no parcels"]["TOWN"].value_counts().rename_axis(None).to_string()) +``` + +MEDIAN PARCEL VALUE + +#something to consider from these outcomes is that burlington has the most parcels, which inevitably will have some non-expensive ones, where avery's gore has 3. However, most of vermont's wealth/value can be assumed to reside there +```{python} + +#getting the median values of only the parcels where the median is positive AND PRESENT +med_val = parcels.loc[parcels["REAL_FLV"] > 0].groupby("TOWN")["REAL_FLV"].median() + +#running the choropleth function with the aggregated medians list, and passing it as a logarithmic scale +town_choropleth(med_val, "Median parcel value (REAL_FLV) — log scale", log=True, cmap="viridis") +print("highest:\n", med_val.sort_values(ascending=False).head(5)) +print("lowest:\n", med_val.sort_values().head(5)) +print(f"AVERYS GORE parcel count {len(parcels[parcels["TOWN"]=="AVERYS GORE"])}") +``` + +```{python} +# Deeded acreage (ACRESGL) is a legal figure; polygon area is what the shape actually covers. +# EPSG:32145 = VT State Plane in METERS — area is only meaningful in a projected CRS, never in 4326. + +#AGRESGL, the grand list acreage of a parcel is for taxation purposes, not actual size. we need to convert it to meters^2 per acre, and need to put it in crs(32145) to project it in meters +parcels["AREA_ACRES_GEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 + +#making the size aggregation - groupoig by town, and aggreagating by median, in this case size. dropping the nulls. +size = (parcels.groupby("TOWN") + .agg(deeded=("ACRESGL", "median"), + geom=("AREA_ACRES_GEOM", "median")) + .dropna()) + +#printing the largest and smallest lots of meters squared in vermont, median +print("largest lots:\n", size.sort_values("geom", ascending=False).head(5)) +print("smallest lots:\n", size.sort_values("geom").head(5)) + +#using the correaltion function to see the divergence or deeded acreage and true polygon plot size +print("corr deeded vs geom:", + parcels[["ACRESGL","AREA_ACRES_GEOM"]].dropna().corr().iloc[0,1].round(3)) + +town_choropleth(size["geom"], "Median parcel size (geometry acres) — log scale", log=True, cmap="cividis") +``` + +LAND VS IMPROVEMENT VALUE +```{python} +val = parcels["REAL_FLV"] +#using np.where to avoid division by zero - this is the ratio of improvment to total +parcels["IMPR_SHARE"] = np.where(val > 0, parcels["IMPRV_LV"].fillna(0) / val, np.nan) + +# Alternative denominator: assessed components (LAND+IMPRV). Often cleaner than REAL_FLV, +# which is sometimes listed independently and can disagree with LAND+IMPRV. + +#making a denominator value for land value + improvement value, with both nulls filled to 0. making a new improvement share column +den = parcels["LAND_LV"].fillna(0) + parcels["IMPRV_LV"].fillna(0) +parcels["IMPR_SHARE_ALT"] = np.where(den > 0, parcels["IMPRV_LV"].fillna(0) / den, np.nan) + +#investigating the shape of the improvment share column +print(parcels["IMPR_SHARE"].describe()) +#share > 1 shouldn't happen if REAL = LAND + IMPRV — a nonzero count flags that assumption breaking +print("share > 1 (divergences where IMPRV_LV exceeds REAL_FLV):", int((parcels["IMPR_SHARE"] > 1).sum())) + +#mad share with a min/max range for the sake of the map +med_share = parcels.groupby("TOWN")["IMPR_SHARE"].median().clip(0, 1) + + +town_choropleth(med_share, "Median improvement share (building ÷ total value)", cmap="RdYlBu_r") + +#"most built up" would represent the towns with highest improvement valuesm and the flip is the most land valued towns +print("most built-up:\n", med_share.sort_values(ascending=False).head(5)) +print("most land-heavy:\n", med_share.sort_values().head(5)) +``` + + +```{python} +#using crosstab with normalize = index to make all towns have same 100% sum no matter their total purposes +mix = pd.crosstab(parcels["TOWN"], parcels["PURPOSE"], normalize="index").mul(100).round(1) +print(mix.head()) + +#using .get to have returned to us a default value +res = mix.get("COMMERCIAL APARTMENTS", pd.Series(0.0, index=mix.index)) + + +#reindex forces these columns to exist (0 if missing), then sum across them per town +rural = mix.reindex(columns=["FARM","WOODLAND","SEASONAL PROPERTY"], fill_value=0).sum(axis=1) +print("\nmost city-like:\n", res.sort_values(ascending=False).head(10)) +print("\nmost farm/woodland/seasonal:\n", rural.sort_values(ascending=False).head(10)) + +#stacking horizontal bars for the top 12 towns with the most parcels +top = parcels["TOWN"].value_counts().head(12).index +mix.loc[top].plot(kind="barh", stacked=True, figsize=(9, 7)) +plt.xlabel("% of parcels"); plt.title("PURPOSE mix of 12 largest towns") +plt.legend(bbox_to_anchor=(1.02, 1)); plt.tight_layout(); plt.show() + +``` + +SAVING THE TABLES +```{python} + +#getting the path out for the tables +out = Path("Data/parcels/tables"); out.mkdir(parents=True, exist_ok=True) +#just making sure +def present(cols): + return [c for c in cols if c in parcels.columns] + +#just need to ensure that objectid is in all three tables that make up parcels - the objectid is the join code +geom_cols = present(["OBJECTID", "TOWN", "COUNTY", "geometry"]) +info_cols = present(["OBJECTID","TOWN","COUNTY","SPAN","PROPTYPE","CAT","CATEGORY","PURPOSE", + "DESCPROP","RESCODE","ACRESGL","AREA_ACRES_GEOM","CITYGL","STGL","ADDRESS", + "SOURCENAME","MATCHSTAT","TNAME","INVESTMENTPROP","VACANTLAND","OOSOWNER", + "data_origin","EDITOR","EDITDATE"]) +tax_cols = present(["OBJECTID","TOWN","REAL_FLV","HSTED_FLV","NRES_FLV","LAND_LV","IMPRV_LV", + "IMPR_SHARE","EQUIPVAL","EQUIPCODE","INVENVAL","HSDECL","VETEXAMT","EXPDESC", + "STATUTE","EXEMPT","EXAMT_HS","EXAMT_NR","UVREDUC_HS","UVREDUC_NR", + "GLVAL_HS","GLVAL_NR"]) + +#making a variable for the three tables that we will ship out later. we need to copy the geomcols to preserve active geometry +geom = parcels[geom_cols].copy() +info = parcels[info_cols].copy() +tax = parcels[tax_cols].copy() + +# geometry side → GeoParquet via geopandas (this write already worked for you; it's the +# pandas .to_parquet calls that collided, so only those move to DuckDB below) +geom.to_parquet(out / "parcels_geom.parquet") +info.to_parquet(out / "parcels_info.parquet") +tax.to_parquet(out / "parcels_tax.parquet") + +# info + tax → Parquet via DuckDB. DuckDB has its own pandas→arrow bridge and never triggers +# the "pandas.period already defined" extension registration that pandas.to_parquet hit. +# w = duckdb.connect() # scratch in-memory connection just for writing +# w.register("info_v", info) # expose the frames to SQL by name +# w.register("tax_v", tax) +# info_path, tax_path = str(out / "parcels_info.parquet"), str(out / "parcels_tax.parquet") +# w.execute(f"COPY (SELECT * FROM info_v) TO '{info_path}' (FORMAT PARQUET)") +# w.execute(f"COPY (SELECT * FROM tax_v) TO '{tax_path}' (FORMAT PARQUET)") +print("geom:", geom.shape, "| info:", info.shape, "| tax:", tax.shape) + +#must change info and tax tables to be read by pd, so that the system doesn't look for metadata that isn't there. +g = gpd.read_parquet(out / "parcels_geom.parquet") +i = pd.read_parquet(out / "parcels_info.parquet") +t = pd.read_parquet(out / "parcels_tax.parquet") +assert g.OBJECTID.is_unique and i.OBJECTID.is_unique and t.OBJECTID.is_unique +assert set(g.OBJECTID) == set(i.OBJECTID) == set(t.OBJECTID), "OBJECTID sets differ!" +three = g.merge(i, on="OBJECTID").merge(t, on="OBJECTID", suffixes=("_info","_tax")) +print("3-way join rows:", len(three), "== parcel count?", len(three) == len(parcels)) +``` + + +#NOW SEEKING TO ADD COMPLEXITY TO PLOTS, STARTING WITH CHECKING IF BURLINGTON IS THERE +```{python} +import seaborn as sns +#making sure that they is a column for acresage in meters squared +#also, making a column for the value per acre of each parcel +if "AREAACRESGEOM" not in parcels: + parcels["AREAACRESGEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +parcels["ACREVALUE"] = np.where( + (parcels.REAL_FLV > 0) & (parcels.AREAACRESGEOM > 0), + parcels.REAL_FLV / parcels.AREAACRESGEOM, np.nan) + +#here are the jurisdictions in chittenden county +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +cc = parcels[parcels.TOWN.isin(CHITTENDEN)].copy() + +#making sure that we can reliably get the towns in chittenden county reliably - +#saving chittenden county and it's df as cc +check = cc.groupby("TOWN").agg( + n=("OBJECTID","size"), + valued=("REAL_FLV", lambda s: int((s>0).sum())), + pct_valued=("REAL_FLV", lambda s: round(100*(s>0).mean(),1)), + med_val=("REAL_FLV", lambda s: s[s>0].median()), +).sort_values("n", ascending=False) +print(check) +``` + +"Marquee 1 — distributions per town (the single biggest depth upgrade). Ordered by median so the urban-rural gradient reads top-to-bottom, and you'll see which towns are bimodal:" +```{python} +#checking for the logarithmic progression of all property value, and making sure that the towns are in order of lowest to highest median parcel size +cc["logv"] = np.log10(cc.REAL_FLV.where(cc.REAL_FLV > 0)) +order = cc.groupby("TOWN").logv.median().sort_values().index + +#making the violin plot +plt.figure(figsize=(9, 10)) + +#checking towns vs logarithmic scaling of median parcel value per town +sns.violinplot(data=cc, y="TOWN", x="logv", order=order, + density_norm="width", cut=0) +plt.xlabel("log10 parcel value"); plt.title("Value distribution by Chittenden town") +plt.tight_layout(); plt.show() + +#DOING THE EXACT SAME THING WITH PARCEL ACRE SIZE +cc["logv_acres"] = np.log10(cc.AREAACRESGEOM.where(cc.AREAACRESGEOM > 0)) +order = cc.groupby("TOWN").logv_acres.median().sort_values().index +plt.figure(figsize=(9, 10)) +sns.violinplot(data=cc, y="TOWN", x="logv_acres", order=order, + density_norm="width", cut=0, color = "orange") +plt.xlabel("log10 parcel value"); plt.title("PARCELS ACRE distribution by Chittenden town") +plt.tight_layout(); plt.show() +``` + +"Marquee 2 — parcel-level $/acre gradient (breaks out of town aggregation entirely). This is the "more than aggregated views" you asked for — Burlington→suburb→rural fringe shows as a continuous surface:" +```{python} +FGB = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/Data/parcels/all_parcels_vermont.fgb" +parcels = gpd.read_file(FGB) +print(parcels.columns) + +if "AREAACRESGEOM" not in parcels: + parcels["AREAACRESGEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +parcels["ACREVALUE"] = np.where( + (parcels.REAL_FLV > 0) & (parcels.AREAACRESGEOM > 0), + parcels.REAL_FLV / parcels.AREAACRESGEOM, np.nan) +``` + +```{python} +#DROPPING NULLS +#ALSO DOING THIS FOR WHOLE STATE + +whole_summary = parcels.groupby("TOWN").agg( + n=("OBJECTID","size"), + valued=("REAL_FLV", lambda s: int((s>0).sum())), + pct_valued=("REAL_FLV", lambda s: round(100*(s>0).mean(),1)), + med_val=("REAL_FLV", lambda s: s[s>0].median()), +).sort_values("n", ascending=False) + +cc_proj = cc.dropna(subset=["ACREVALUE"]).to_crs(32145) +whole_proj = parcels.dropna(subset=["ACREVALUE"]).to_crs(32145) + +#getting rid of absurd outliers - first and 99th percentiles +lo, hi = cc_proj.ACREVALUE.quantile([0.01, 0.99]) + +lo, hi = whole_proj.ACREVALUE.quantile([0.05, 0.99]) + +#clipping, and making figure logarithmicly scaled +# cc_proj["vpa_clip"] = cc_proj.ACREVALUE.clip(lo, hi) +# fig, ax = plt.subplots(figsize=(11, 11)) +# cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, +# linewidth=0, norm=LogNorm(vmin=lo, vmax=hi)) +# ax.set_title("Chittenden parcels — value per acre (log)"); ax.axis("off"); plt.show() + +whole_proj["vpa_clip"] = whole_proj.ACREVALUE.clip(lo, hi) +fig, ax = plt.subplots(figsize=(12, 18)) +whole_proj.plot(column="vpa_clip", cmap="inferno", legend=True, ax=ax, + linewidth=0, norm=LogNorm(vmin=lo, vmax=hi)) +ax.set_title("Whole state of vermont — value per acre (log)"); ax.axis("off"); plt.show() +``` + +ideas for the following cell + +Absentee & seasonal geography: cc.groupby("TOWN").agg(pct_oos=("OOSOWNER","mean"), pct_seasonal=("PURPOSE", lambda s:(s=="SEASONAL PROPERTY").mean())) — Charlotte/Shelburne lakefront vs. the urban core is the contrast. +Development pressure: low IMPR_SHARE and high land value = underbuilt/speculative. Flag parcels where LAND_LV > IMPRV_LV in the core towns. +Urban/suburb/fringe tiers: hand-group the towns (core: Burlington/Winooski/S.Burlington; suburb: Williston/Essex/Colchester; fringe: Charlotte/Huntington/Bolton/Buels Gore) and compare distributions across tiers — a cleaner story than 19 separate towns. +```{python} +#cc_proj = cc.dropna(subset=["ACREVALUE"]).to_crs(32145) + +#printing the groupyby of out of state owners and purpose +check = cc.groupby("TOWN").agg( + parcel_count =("OBJECTID","size"), + pct_oos=("OOSOWNER","mean"), + pct_seasonal=("PURPOSE", lambda s:(s=="SEASONAL PROPERTY").mean()), +).sort_values("pct_seasonal", ascending=False) +#print(check) + +#checking where the sum of parcels wehre land_lv > improvement_value +land_based = cc[cc["IMPR_SHARE"]<0.50] +#print(cc["TOWN"].value_counts()) +#print(land_based["TOWN"].value_counts()) + + +#print(cc[["IMPR_SHARE", "TOWN"]].value_counts()) + +URBANCC = ["BURLINGTON", "WINOOSKI", "SOUTH BURLINGTON"] +SUBURBANCC = ["COLCHESTER", "WILLISTON", "ESSEX JUNCTION", "ESSEX"] +LAKEFRONT = ["CHARLOTTE", "SHELBURNE", "COLCHESTER"] +RURALCC = [] +for item in cc["TOWN"].unique(): + if item not in (URBANCC) and item not in (SUBURBANCC): + RURALCC.append(item) +print(RURALCC) + +#print(land_based.columns) +# a = len(land_based) + +# urb = land_based["TOWN"].isin(URBANCC).sum() +# sub = land_based["TOWN"].isin(SUBURBANCC).sum() +# rur = land_based["TOWN"].isin(RURALCC).sum() + +# print(f"{urb/a:.2f} {sub/a:.2f} {rur/a:.2f}") + +# print(land_based["TOWN"].isin(URBANCC).sum() / cc["TOWN"].isin(URBANCC).sum()) + + +# print(cc[cc["PURPOSE"]=="COMMERCIAL APARTMENTS"]["TOWN"].value_counts()) +# print(cc[cc["SOUTH BURLINGTON"] == "BUELS GORE"]["OOSOWNER"].value_counts()) + + +# fig, ax = plt.subplots(figsize=(11, 11)) +# cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, +# linewidth=0) +# ax.set_title("Chittenden parcels — seasonal vs not"); ax.axis("off"); plt.show() + +#makinga new dictionary for each town in urban, suburban, and rural, and adding those as keys to the towns int he dictionary +tier_map = {t: "urban" for t in URBANCC} +tier_map.update({t: "suburban" for t in SUBURBANCC}) +tier_map.update({t: "lakefront" for t in LAKEFRONT if t not in tier_map}) +#making rural the default of the dictionary +for t in cc["TOWN"].unique(): + tier_map.setdefault(t, "rural") + +#making a town groupby with size, the out of state owner percentage, on average, the seasonal property percentage, also on average, adn the percent underbuilt, where the ratio of improvement value to land value is less than 50%. Also adding the median value or tbe parcel at hand +town = cc.groupby("TOWN").agg( + n=("OBJECTID", "size"), + pct_oos=("OOSOWNER", "mean"), + pct_seasonal=("PURPOSE", lambda s: (s == "SEASONAL PROPERTY").mean()), + pct_underbuilt=("IMPR_SHARE", lambda s: (s < 0.50).mean()), + med_val=("REAL_FLV", lambda s: s[s > 0].median()), +) +#tier is urban, suburban, lakefront, and rural +town["tier"] = town.index.map(tier_map) + +#making the an aggregated groupby by tier, the newly created measuring system by town type, with the number of entries, the parcel count, the average of each out of state ownership, the average seasonal percentage, the average underbuilt (land is more valuable than improvement), and the mdian value of the parcel +tier_summary = (town.groupby("tier") + .apply(lambda g: pd.Series({ + "towns": len(g), + "parcels": int(g.n.sum()), + "pct_oos": np.average(g.pct_oos, weights=g.n), + "pct_seasonal": np.average(g.pct_seasonal, weights=g.n), + "pct_underbuilt": np.average(g.pct_underbuilt, weights=g.n), + "med_val": g.med_val.median(), + })) + .reindex(["urban", "suburban", "lakefront", "rural"])) +#getting a summary +print(tier_summary.round(3)) + +#making the figure +import matplotlib.pyplot as plt +colors = {"urban": "#d62728", "suburban": "#ff7f0e", "lakefront": "#1f77b4", "rural": "#2ca02c"} +fig, ax = plt.subplots(figsize=(8, 4)) + +#for item tier in ("urban", "suburban", etc) and g for twown, make the scatterplot and finding percentage +for tier, g in town.groupby("tier"): + ax.scatter(g.pct_underbuilt * 100, g.pct_oos * 100, + s=g.n / 8, alpha=0.7, color=colors[tier], label=tier, edgecolor="white", linewidth=0.5) + +#labelling every town +for name, r in town.iterrows(): + ax.annotate(name.title(), (r.pct_underbuilt * 100, r.pct_oos * 100), + fontsize=7, alpha=0.6, xytext=(3, 3), textcoords="offset points") + + +#getting the chittenden county average, to place with accuracy every town against the average +ax.axhline(town.pct_oos.mean() * 100, ls="--", color="gray", lw=0.8) +ax.axvline(town.pct_underbuilt.mean() * 100, ls="--", color="gray", lw=0.8) +ax.set_xlabel("% of parcels underbuilt (improvement < 50% of value)") +ax.set_ylabel("% out-of-state owned") +ax.set_title("Chittenden: absentee ownership vs. development pressure\n(dot size = parcel count)") +ax.legend(title="tier"); plt.tight_layout(); plt.show() + +``` + +```{python} +#making a gini function. +def gini(x): + #sorting the inputted array into order - getting rid of nan and negarive numbers + x = np.sort(np.asarray(x, float)); x = x[~np.isnan(x)]; x = x[x >= 0] + #making sure of the ability to handle an empty array or the sum of the array equalling 0 + if x.size == 0 or x.sum() == 0: return np.nan + #n is the size, and the index adds the correct index of the sorted list + n = x.size; idx = np.arange(1, n+1) + #returning the value of the gini coefficient + return (2*np.sum(idx*x)/(n*x.sum())) - (n+1)/n + + +#making a function that gets the county metrics, with a dataframe as an input +#first we get hte real value of the parcels that are over 0, then return a bunch of pertinent measuremtns of the dataframe +def county_metrics(df): + v = df.REAL_FLV[df.REAL_FLV > 0] + return pd.Series({ + "median_value": v.median(), + "median_lot_acres": df.AREAACRESGEOM.median(), + "median_val_per_acre": df.ACREVALUE.median(), + "impr_share": df.IMPR_SHARE.median(), + "pct_residential": (df.PURPOSE == "PRIMARY RESIDENCE").mean()*100, + "pct_farm_wood": df.PURPOSE.isin(["FARM","WOODLAND"]).mean()*100, + "pct_seasonal": (df.PURPOSE == "SEASONAL PROPERTY").mean()*100, + "pct_oos_owner": df.OOSOWNER.mean()*100, + "value_gini": gini(v), + }) +#applying the county metrics to the parcels dataset as it is grouped by county +cm = parcels.groupby("COUNTY").apply(county_metrics) +#standarsizing (z score) metrics so that they can be compared across counties +z = (cm - cm.mean()) / cm.std() +#order the z scores by median value per acre, which is a good metric of urban-level +z = z.loc[cm.median_val_per_acre.sort_values(ascending=False).index] +#plotting the figure, and heatmap to show standarsization +plt.figure(figsize=(10, 7)) +sns.heatmap(z, cmap="RdBu_r", center=0, cbar_kws={"label": "z-score"}) +plt.title("County profiles (standardized)"); plt.tight_layout(); plt.show() +#printing the values +print(cm.round(1)) +``` + + +```{python} +#getting the parcels where the value is over 0, and scaling it logarithmically +d = parcels.loc[parcels.REAL_FLV > 0, ["COUNTY","REAL_FLV"]].copy() +d["logv"] = np.log10(d.REAL_FLV) +#getting the average, and grouping by county +grand = d.logv.mean() +grp = d.groupby("COUNTY").logv + +#ss is sum of swaures. we are calculating the variance between the parcels within, with respect to property value. with this in mind, 93.2 percent of value is attributable to county, with 6.2 not due to county +ss_between = (grp.count() * (grp.mean() - grand)**2).sum() +ss_within = ((d.logv - grp.transform("mean"))**2).sum() +ss_total = ((d.logv - grand)**2).sum() +print(f"between-county: {ss_between/ss_total:.1%} | within-county: {ss_within/ss_total:.1%}") +``` + +conclusive urban rural divide +```{python} +#printing the towns by town acreage by -- look at what are light yellow +vpa = parcels.groupby("TOWN").ACREVALUE.median() +town_choropleth(vpa, "Median value per acre ($/acre) — log", log=True, cmap="inferno") +print(vpa.sort_values(ascending = False)) +print(vpa["CHARLOTTE"]) +``` + + + +performing machine learning on the data +```{python} +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +from sklearn.cluster import KMeans + +#getting the town areas in kilometers squared +ta = towns_gdf.to_crs(32145); ta["area_km2"] = ta.geometry.area / 1e6 +town_area = ta.set_index("TOWN")["area_km2"] + +#making a group by for the median value, the median acreage lot size, the value per acre, the improvement share, the percent residence, and the number of parcels +feat = parcels.groupby("TOWN").agg( + med_val=("REAL_FLV", lambda s: s[s>0].median()), + med_lot=("AREAACRESGEOM", "median"), + val_per_acre=("ACREVALUE", "median"), + impr_share=("IMPR_SHARE", "median"), + pct_res=("PURPOSE", lambda s: (s=="PRIMARY RESIDENCE").mean()), + n_parcels=("OBJECTID", "size"), +) +#making a custom density column, made from the number of parcels over the area +feat["parcel_density"] = feat.n_parcels / town_area.reindex(feat.index) + + +feat_cols = ["med_val","med_lot","val_per_acre","impr_share","pct_res","parcel_density"] +# +F = feat[feat_cols].copy() +for c in ["med_val","med_lot","val_per_acre","parcel_density"]: + #applying a logarithnic value to the skewed values + F[c] = np.log10(F[c].replace(0, np.nan)) +#dropping null values +F = F.dropna() +#standardizing all values for future disvovery +X = StandardScaler().fit_transform(F) + +#running pca algorithm to try and discover what drives "urbanity" +pca = PCA(n_components=2).fit(X) +#getting the strongest indicator and saving it as "urban_rural" +F["urban_rural"] = pca.transform(X)[:, 0] +#for better understanding, making the urban share more positive; creating that correaltino +if np.corrcoef(F["urban_rural"], F["impr_share"])[0,1] < 0: + F["urban_rural"] *= -1 + +#printing findings +print("PC1 explains", f"{pca.explained_variance_ratio_[0]:.1%}", "of variance [in urbanity]") +print(pd.Series(pca.components_[0], index=feat_cols).sort_values()) # loadings = what "urban" means + +F["cluster"] = KMeans(n_clusters=5, n_init=10, random_state=0).fit_predict(X) +print(feat.loc[F.index].assign(cluster=F.cluster).groupby("cluster").median()) # read the typology + +town_choropleth(F["urban_rural"], "Urban–rural index (PC1)", cmap="Spectral_r") +``` + + +```{python} +#loading in the population dataset +pop = pd.read_csv("/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv") +pop["TOWN"] = pop["Town"].str.upper() +pop = pop.set_index("TOWN")["year2020"] + +#seeing what towns are missing - discrepancy between our parcels dataset and population dataset +missing = [t for t in F.index if t not in pop.index] +print(f"towns in F with no population match ({len(missing)}):", missing[:15]) + +#using reindex to keep whatever towns are overlapping (hopefully a lot) in the same order +popdens = np.log10((pop.reindex(F.index) / town_area.reindex(F.index)).replace(0, np.nan)) + +pop_sort = popdens.sort_values(ascending=False) + +# align both series and drop any NaN pairs so the r is computed on the same towns you plot +#making a variable for the valid towns. then, getting the correlation of the urban area towns and then checking the index vs population density, and getting the r value +valid = pd.DataFrame({"urban_rural": F["urban_rural"], "log_popdens": popdens}).dropna() +r = valid["urban_rural"].corr(valid["log_popdens"]) +print(f"index vs. log population density r = {r:.3f} (n = {len(valid)} towns)") + +#creating the scatterplot +import matplotlib.pyplot as plt +fig, ax = plt.subplots(figsize=(8, 7)) +#plotting the log population density vs the urban_rural found PCA +ax.scatter(valid["log_popdens"], valid["urban_rural"], alpha=0.6, edgecolor="white", linewidth=0.5) + +#finding the towns to label by finding hte ones that dont seem to conformwith the lien of best fit +resid = valid["urban_rural"] - np.poly1d(np.polyfit(valid["log_popdens"], valid["urban_rural"], 1))(valid["log_popdens"]) +to_label = valid.loc[resid.abs().sort_values(ascending=False).head(8).index] + +#adding the annotation to each item in the to_label +for name, row in to_label.iterrows(): + ax.annotate(name.title(), (row["log_popdens"], row["urban_rural"]), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") +for name, density_val in pop_sort.head(3).items(): + if name not in to_label.index: + urban_score = F.loc[name, "urban_rural"] + ax.annotate(name.title(), (density_val, urban_score), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") +for name, density_val in pop_sort.tail().items(): + if name not in to_label.index: + urban_score = F.loc[name, "urban_rural"] + ax.annotate(name.title(), (density_val, urban_score), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") + +m, b = np.polyfit(valid["log_popdens"], valid["urban_rural"], 1) +xs = np.linspace(valid["log_popdens"].min(), valid["log_popdens"].max(), 50) +ax.plot(xs, m*xs + b, "--", color="gray", lw=1) +ax.set_xlabel("log₁₀ population density (people / km²)") +ax.set_ylabel("urban–rural index (PC1)") +ax.set_title(f"Parcel-derived urban index vs. census density (r = {r:.2f})") +plt.tight_layout(); plt.show() +``` +#an r value of 0.911 is very conclusively strong for being able to predict the urbanity level of a town based on everything but population + +IDEA FOR FUTURE Next-level (the portfolio standout you probably haven't done): spatial autocorrelation. libpysal + esda give you Moran's I (is value spatially clustered at all?) and LISA (statistically significant hot/cold spots that ignore town boundaries). uv pip install libpysal esda, build a spatial weights matrix on town or parcel centroids, and you can map "significant high-value clusters" as a defensible statistical object rather than an eyeballed choropleth. +```{python} +import numpy as np, matplotlib.pyplot as plt +from libpysal.weights import Queen, KNN +from esda.moran import Moran, Moran_Local + +# 1. attach the metric to town polygons — log because value is heavy-tailed and +# raw skew lets a few outliers dominate a correlation statistic like Moran's I +tv = parcels.loc[parcels.REAL_FLV > 0].groupby("TOWN")["REAL_FLV"].median() +tw = towns_gdf.merge(tv.rename("med_val").reset_index(), on="TOWN", how="left") +tw["log_val"] = np.log10(tw["med_val"]) + +# 2. Moran can't take NaN, and DROPPING a town changes who-borders-whom — so this +# must happen BEFORE weights are built, not after +tw = tw.dropna(subset=["log_val"]).reset_index(drop=True) + +# 3. Queen contiguity: two towns are neighbors if they share ANY boundary point +# (Queen includes corner-touches; Rook would require a shared edge) +w = Queen.from_dataframe(tw, use_index=True) +print("island towns (no land neighbor):", [tw.TOWN.iloc[i] for i in w.islands]) + +# 4. handle islands — drop them (looping, since dropping one can orphan another). +# This is the Champlain-islands fix. +while w.islands: + tw = tw.drop(index=w.islands).reset_index(drop=True) + w = Queen.from_dataframe(tw, use_index=True) + +# 5. row-standardize: each town's neighbors' weights sum to 1, so the "spatial lag" +# is a neighbor AVERAGE, not a neighbor SUM (fair across towns with 3 vs 8 neighbors) +w.transform = "r" +y = tw["log_val"].values + +# 6. global Moran's I with a 999-permutation pseudo p-value +mi = Moran(y, w, permutations=999) +print(f"Moran's I = {mi.I:.3f} | E[I] under randomness = {mi.EI:.3f}" + f" | p = {mi.p_sim:.3f} | z = {mi.z_sim:.2f}") +``` diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0164ba5b..84009a88 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -10,12 +10,18 @@ dependencies = [ "fastapi>=0.137.2", "geopandas>=1.1.3", "jinja2>=3.1.6", + "jupyter-cache>=1.0.1", "matplotlib>=3.11.0", + "nbclient>=0.11.0", + "nbformat>=5.10.4", "numpy>=2.4.6", "pandas>=3.0.3", + "pathlib>=1.0.1", "pydantic>=2.13.4", "pyogrio>=0.12.1", + "pyyaml>=6.0.3", "requests>=2.34.2", + "seaborn>=0.13.2", "scikit-learn>=1.9.0", "shapely>=2.1.2", "uvicorn>=0.49.0", diff --git a/backend/query/parcels.py b/backend/query/parcels.py new file mode 100644 index 00000000..81297049 --- /dev/null +++ b/backend/query/parcels.py @@ -0,0 +1,51 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-07-23 +**Description**: + Something of note: this is a "girder" for the sql queries between + the front end (website), and the backend database, with something to do with dbeaver as a goal in mind + +""" + +import logging +from pathlib import Path + +import pandas as pd + +from api.models import FilterSource +from query.core_functions import filter_tree +from query.processed_db import DB +from sql_render import sql_filter_block + +# changed sql directory to parcels in the sql folder +logger = logging.getLogger(__name__) +sql_dir = Path(__file__).resolve().parent / "sql" / "parcels" + + +def get_parcels_geojson(sources: list[FilterSource]): + sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", sources) + result = DB.execute(sql, params).fetchone() + if result is None: + logger.error("geo query returned no rows for filters: %s", sources) + raise ValueError(f"no results for filters: {sources}") + return result[0] + + +def get_parcels_table( + sources: list[FilterSource], +) -> pd.DataFrame: + table_sql, table_params = sql_filter_block(sql_dir / "info_table.sql", sources) + table_data = DB.execute(table_sql, table_params).df() + return table_data + + +def get_parcels_filters(): + PARCEL_FILTER_COLS = { + "Town": "TOWN", + "Category": "CAT", + "Property Type": "PROPTYPE", + } + PARCEL_TREE_LABELS = ["Town", "Category"] + return filter_tree(PARCEL_FILTER_COLS, PARCEL_TREE_LABELS, "parcels_info") diff --git a/backend/query/sql/parcels/geo_query_parcels.sql b/backend/query/sql/parcels/geo_query_parcels.sql new file mode 100644 index 00000000..b94d5dc6 --- /dev/null +++ b/backend/query/sql/parcels/geo_query_parcels.sql @@ -0,0 +1,27 @@ +{{ cte_filter_block }} +SELECT + JSON_OBJECT( + 'type', 'FeatureCollection', + 'features', JSON_GROUP_ARRAY(feature) + )::VARCHAR AS fc +FROM ( + SELECT + JSON_OBJECT( + 'type', 'Feature', + 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(g.geom, 0.0001))::JSON, + 'properties', JSON_OBJECT( + 'Parcel Type', i.CAT, + 'Acres', ROUND(i.ACRESGL, 2), + 'rgba_color', '[100,150,200,150]'::JSON, + 'tooltip', JSON_OBJECT( + '__title__', 'Parcels', + 'Place', i.CITYGL, + 'Parcel Description', i.DESCPROP, + 'Acres', ROUND(i.ACRESGL, 2) + ) + ) + ) AS feature + FROM parcels_info AS i + INNER JOIN parcels_geom AS g USING (OBJECTID) + {{ join_filter_block }} +) AS features diff --git a/backend/query/sql/parcels/info_table.sql b/backend/query/sql/parcels/info_table.sql new file mode 100644 index 00000000..bfd6e6ee --- /dev/null +++ b/backend/query/sql/parcels/info_table.sql @@ -0,0 +1,10 @@ +{{ cte_filter_block }} + +SELECT + i.CAT AS "Property Category", + i.ACRESGL AS Acres, + i.CITYGL AS "Owner City", + i.TOWN AS Town, + i.DESCPROP AS "Property Description" +FROM parcels_info AS i + {{ join_filter_block }} diff --git a/backend/uv.lock b/backend/uv.lock index 3d166877..c6795181 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -77,13 +77,19 @@ dependencies = [ { name = "fastapi" }, { name = "geopandas" }, { name = "jinja2" }, + { name = "jupyter-cache" }, { name = "matplotlib" }, + { name = "nbclient" }, + { name = "nbformat" }, { name = "numpy" }, { name = "pandas" }, + { name = "pathlib" }, { name = "pydantic" }, { name = "pyogrio" }, + { name = "pyyaml" }, { name = "requests" }, { name = "scikit-learn" }, + { name = "seaborn" }, { name = "shapely" }, { name = "uvicorn" }, { name = "xycmap" }, @@ -110,13 +116,19 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.137.2" }, { name = "geopandas", specifier = ">=1.1.3" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "jupyter-cache", specifier = ">=1.0.1" }, { name = "matplotlib", specifier = ">=3.11.0" }, + { name = "nbclient", specifier = ">=0.11.0" }, + { name = "nbformat", specifier = ">=5.10.4" }, { name = "numpy", specifier = ">=2.4.6" }, { name = "pandas", specifier = ">=3.0.3" }, + { name = "pathlib", specifier = ">=1.0.1" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pyogrio", specifier = ">=0.12.1" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.34.2" }, { name = "scikit-learn", specifier = ">=1.9.0" }, + { name = "seaborn", specifier = ">=0.13.2" }, { name = "shapely", specifier = ">=2.1.2" }, { name = "uvicorn", specifier = ">=0.49.0" }, { name = "xycmap", specifier = ">=1.0.1" }, @@ -526,6 +538,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -572,6 +631,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -699,6 +770,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "jupyter-cache" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "importlib-metadata" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "pyyaml" }, + { name = "sqlalchemy" }, + { name = "tabulate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/f7/3627358075f183956e8c4974603232b03afd4ddc7baf72c2bc9fff522291/jupyter_cache-1.0.1.tar.gz", hash = "sha256:16e808eb19e3fb67a223db906e131ea6e01f03aa27f49a7214ce6a5fec186fb9", size = 32048, upload-time = "2024-11-15T16:03:55.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/6b/67b87da9d36bff9df7d0efbd1a325fa372a43be7158effaf43ed7b22341d/jupyter_cache-1.0.1-py3-none-any.whl", hash = "sha256:9c3cafd825ba7da8b5830485343091143dff903e4d8c69db9349b728b140abf6", size = 33907, upload-time = "2024-11-15T16:03:54.021Z" }, +] + [[package]] name = "jupyter-client" version = "8.9.1" @@ -916,6 +1006,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, ] +[[package]] +name = "nbclient" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, +] + [[package]] name = "nbformat" version = "5.10.4" @@ -1052,6 +1157,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] +[[package]] +name = "pathlib" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -1839,6 +1953,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" }, + { url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" }, + { url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" }, + { url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" }, + { url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, +] + [[package]] name = "sqlfluff" version = "4.2.2" @@ -1887,6 +2029,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + [[package]] name = "tblib" version = "3.2.2" @@ -2027,3 +2178,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/f7/d0/9ec6cd4913a726a19 wheels = [ { url = "https://files.pythonhosted.org/packages/1c/6c/6bbe66d9e40d8c50552a7df4014f557ae3aa67b990a4fa3f56fd92d07204/xycmap-1.0.1-py3-none-any.whl", hash = "sha256:669652bd2049f251d713d73ae18694a4b6ef92f0f9a5c8092fc5bbd1f5fb50b8", size = 6022, upload-time = "2021-03-02T11:00:32.961Z" }, ] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fe1a8cc7..b7319548 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8106,33 +8106,6 @@ "node": ">= 6" } }, - "node_modules/framer-motion": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.42.2.tgz", - "integrity": "sha512-5XY9luDiu0oHfHBjpDthFMh0ES+122w6p/papSJBweMkO8Sn+PW2QaEgRblQBpWFnuvZS5qvarpt/hO2pjGmnw==", - "license": "MIT", - "dependencies": { - "motion-dom": "^12.42.2", - "motion-utils": "^12.39.0", - "tslib": "^2.4.0" - }, - "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" - }, - "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, "node_modules/fs-extra": { "version": "11.3.3", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.3.tgz", @@ -10986,47 +10959,6 @@ "node": "*" } }, - "node_modules/motion": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/motion/-/motion-12.42.2.tgz", - "integrity": "sha512-Atvv11yUKIid41cVrRBDVX5m8tF8kNpExRSlbpt6APClhDjtwQssgFHhQzejxw7/7YYbjHSPKBVbHo05BuJT5Q==", - "license": "MIT", - "dependencies": { - "framer-motion": "^12.42.2", - "tslib": "^2.4.0" - }, - "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" - }, - "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, - "node_modules/motion-dom": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/motion-dom/-/motion-dom-12.42.2.tgz", - "integrity": "sha512-5gIMWLp/PycBtJRJWRgjxke5n8dlvkSn2DrYW+tr3XcqAZY1xZh6BJyooJXCM8wdfM7wfMjkBJNLge1CKPUIRA==", - "license": "MIT", - "dependencies": { - "motion-utils": "^12.39.0" - } - }, - "node_modules/motion-utils": { - "version": "12.39.0", - "resolved": "https://registry.npmjs.org/motion-utils/-/motion-utils-12.39.0.tgz", - "integrity": "sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ==", - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",