From d1dbec6f8f558185220e053ee249717847d9c7e5 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Wed, 22 Jul 2026 18:59:21 -0400 Subject: [PATCH 01/12] adding parcels-integrate --- backend/notebooks/parcels/build.qmd | 21 ++ .../notebooks/parcels/parcels_run_test.qmd | 21 ++ backend/notebooks/parcels/table_build.qmd | 338 ++++++++++++++++++ backend/pyproject.toml | 5 + backend/uv.lock | 318 ++++++++++++++++ 5 files changed, 703 insertions(+) create mode 100644 backend/notebooks/parcels/build.qmd create mode 100644 backend/notebooks/parcels/parcels_run_test.qmd create mode 100644 backend/notebooks/parcels/table_build.qmd diff --git a/backend/notebooks/parcels/build.qmd b/backend/notebooks/parcels/build.qmd new file mode 100644 index 00000000..dadba445 --- /dev/null +++ b/backend/notebooks/parcels/build.qmd @@ -0,0 +1,21 @@ +--- +title: "Vermont Parcels: build.py [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- + diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd new file mode 100644 index 00000000..fdaba11a --- /dev/null +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -0,0 +1,21 @@ +--- +title: "Vermont Parcels: the testing run notebook [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#ill get to this later diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd new file mode 100644 index 00000000..cac25e94 --- /dev/null +++ b/backend/notebooks/parcels/table_build.qmd @@ -0,0 +1,338 @@ +--- +title: "Vermont Parcels: SQL Table Build" +author: Isaac Wedaman +date: today +description: Exploratory build of SQL tables from the VT Statewide Standardized Parcel Data. +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +## SETUP +importing required libraries and setting the project root in the working directory +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) +``` + + +## Data Collection +here, we read in the parcels from the dataset, which is kept locally. we create both a parquet and an FGB rendering of the original geojson dataset. Then, we load the spatial extension for duckdb, and create a view structure of the data, for more extensive data analysis donw the line. +```{python} +data_directory = Path("Data") +parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + +parcels_parquet = Path("Data/parcels/parcels_p.parquet") +parcels_fgb = Path("Data/parcels/parcels_f.fgb") + + +con = duckdb.connect() +con.execute("INSTALL spatial; LOAD spatial;") + +if not parcels_fgb.exists(): + con.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') + """) + +if not parcels_parquet.exists(): + con.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_parquet}' (FORMAT PARQUET) + """) + +``` + +```{python} +#creating the parcels_raw view on which to query things to learn +con.execute("CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/Parcels/parcels_f.fgb')") +``` + +Here, we print all the columns in the parcels_raw view, of which there are many. commented out is the code to yield the total row count, with is 191156. Now looking at the column types and their respective names (what they represent in real life), and making a reproducible code chunk to investigate what each means and what it looks like +```{python} +described_parcels_raw = con.execute("DESCRIBE parcels_raw").df() +print(described_parcels_raw) +print(con.execute("SELECT COUNT(*) AS row_count, COUNT (DISTINCT OBJECTID) AS distinct_ids, SUM((OBJECTID IS NULL)::INT) AS nulls FROM parcels_raw").df()) + +column_types = described_parcels_raw["column_type"].tolist() +columns_by_type = {} + +for thing, item in described_parcels_raw.iterrows(): + for index, row in enumerate(column_types): + if item["column_type"] == row: + if row not in columns_by_type: + columns_by_type[row] = [] + + columns_by_type[row].append(item["column_name"]) +print(columns_by_type) + +for key, value in columns_by_type.items(): + print(f"{key}: {len(value)}") + +``` + +```{python} +#now, making the reproducible code to get the values of each column +name = "ACRESGL" +col = con.execute(f"""SELECT {name} FROM parcels_raw ORDER BY {name} ASC""").df() +print(col.describe()) +print(col.value_counts()) + +``` + +## DATA ANALYSIS - FINDING WHAT IS IMPORTANT +#methods to a) count get the unique values listed, and b) get the unique values counted. my thought was to create a reproducible code chunk to investigate the unique values of each column, per the column variable +```{python} + +con.execute("SELECT YEAR FROM parcels_raw ORDER BY YEAR ASC").df() + +column = "SOURCENAME" + +con.execute(f""" +SELECT + {column}, + COUNT({column}) AS instances, + SUM(COUNT({column})) OVER () AS total_number +FROM parcels_raw +GROUP BY {column} +ORDER BY instances DESC; +""").df() +``` +the following code chunk is a method to get the unique values of a column, after having gotten their column names in a list. from there, we loop through the list of column names and count the nulls, then pring out the nulls per column, outputted as a dataframe +```{python} +#summing the nulls per column +parcel_columns = described_parcels_raw["column_name"].tolist() +nulls = [] + +for col in parcel_columns: + result = con.execute(f""" + SELECT SUM(CASE WHEN "{col}" IS NULL THEN 1 ELSE 0 END) + FROM parcels_raw; + """) + values = result.fetchone()[0] + count = values if values is not None else 0 + nulls.append(count) + +nulls_columns = pd.DataFrame({'Column': parcel_columns, 'Nulls': nulls}) + +``` +Something of note for posterity is that there were a few rows that contained 9577 null values. i sook to find out whether those absences were correalted, or random, and what that might mean with respect to the story of the data. +```{python} +#print(nulls_columns) +#found that 9577 nulls occurs the most frequently +#these are them +columns_to_read = nulls_columns[nulls_columns["Nulls"] == 9577]["Column"] +columns_to_read = columns_to_read.tolist() +print(columns_to_read) +#columns where null is 0 +no_nulls = nulls_columns[nulls_columns["Nulls"] == 0]["Column"] +no_nulls = no_nulls.tolist() +print(no_nulls) +uniques = [] +for col in no_nulls: + result = con.execute(f""" + SELECT COUNT(*), COUNT(DISTINCT {col}) + FROM parcels_raw; + """) + values = result.fetchone()[1] + count = values if values is not None else -25 + uniques.append(count) +print(uniques) +``` +This method gets a few values important for the measuring of data, similar to r's "summarize" function, which helps in a completely developed understanding of the data +```{python} +#describing the acresgl +stats = con.execute("""SELECT + COUNT(ACRESGL) AS total_count, + COUNT(*) - COUNT(ACRESGL) AS missing_values, + MIN(ACRESGL) AS min_value, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY ACRESGL) AS first_quartile, + PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY ACRESGL) AS median, + AVG(ACRESGL) AS mean, + PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY ACRESGL) AS third_quartile, + MAX(ACRESGL) AS max_value +FROM parcels_raw; +""").df() +print(stats.head()) +span = con.execute("""SELECT SPAN AS span, GLIST_SPAN AS glistspan, PARCID AS parcid FROM parcels_raw""").df() +print(span) + +#something of note + + +acres_gl = con.execute(f"""SELECT ACRESGL AS acresgl, COUNT(*) AS n FROM parcels_raw GROUP BY ACRESGL ORDER BY n DESC""").df() +print(acres_gl.head(10)) +# fitz_form = (ST_Area(ST_Transform(geom, 'EPSG:4326', 'EPSG:32145', always_xy := TRUE)) / 4046.86).df() +# print(fitz_form.head(10)) +``` +#trying to find the +```{python} +print(columns_to_read) +span_vs_null = con.execute("""SELECT SPAN AS span, GLIST_SPAN AS glistspan, PARCID AS parcid, TNAME, ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV FROM parcels_raw""").df() +#span_vs_null.head(10) +span_vs_null[span_vs_null["TNAME"]== "Putney"].head() +``` +there are 9577 in a few rows, as we saw above. here, Im looking to discover if those few rows are all a few correlated nulls, or random. +```{python} +#im picking to compare parcel id (PARCID) and acresgl +con.execute(""" +SELECT + SUM((PARCID IS NULL AND ACRESGL IS NULL)::INT) AS both_null, + SUM((PARCID IS NULL AND ACRESGL IS NOT NULL)::INT) AS only_a +FROM parcels_raw +""").df() +#it looks like there are both correlated!!! +#now seeing what non-null they have in common +con.execute(""" +SELECT TNAME, COUNT(*) AS n +FROM parcels_raw +WHERE PARCID IS NULL +GROUP BY TNAME +ORDER BY n DESC +""").df() +#this is an interesting story: it looks like, reading from the output of 9577 entrances that also have town == null, tat the map needs to be completed with parcels, but those parcels arent valuable, important, or pertinent enough to warrant having their information filled out + +``` + +## making sure that the geometry of the data is valid +Here, I am checking that the parcel type is indeed polygon/multigon +```{python} +#all polygon/multigon +con.execute("""SELECT ST_GeometryType(geom) AS gtype, COUNT(*) FROM parcels_raw GROUP BY gtype""").df() + +#checking the CRS (coordinate reference system) +#output 0 {'min_x': -73.4374417425543, 'min_y': 42.73912... +con.execute("SELECT ST_Extent(ST_Union_Agg(geom)) FROM parcels_raw").df() + + +``` + + + +## REASONING/CONNECTING TO REAL LIFE DATA +While searching for the correlation or backstory on what might make an entrance with TNAME == null, I found that, when also keeping in mind their property type (var name PROPTYPE), that the majority are unlisted parcels (5948), and the remaining are untaxable service parcels, in the order of row roads, water, row rail, row trail, and railroad parcels, that, I guess, are not important enough to be designated a town name +```{python} +null_tname = con.execute("""SELECT * FROM parcels_raw WHERE TNAME IS NULL""").df() + +null_tname["PROPTYPE"].value_counts() + +#checking if town name and TNAME are the same, or correlated +tnames = con.execute("""SELECT TNAME AS tname, TOWN AS town FROM parcels_raw""").df() +print(tnames["tname"].value_counts(), tnames["town"].value_counts()) + +#TOWN HAS NO NULLS +con.execute("""SELECT SUM((TOWN IS NULL)::INT), SUM((TNAME IS NULL)::INT) FROM parcels_raw""").df() +``` + +```{python} +con.execute(""" +SELECT SOURCENAME, YEAR, COUNT(*) AS n +FROM parcels_raw +WHERE TNAME IS NULL +GROUP BY SOURCENAME, YEAR +ORDER BY n DESC +""").df() +con.execute(""" +SELECT + TNAME IS NULL AS no_town, + COUNT(*) AS n, + AVG(ST_Area(geom)) AS avg_area, + MEDIAN(ST_Area(geom)) AS med_area +FROM parcels_raw +GROUP BY no_town +""").df() + +con.execute("""SELECT COUNT(*), COUNT(DISTINCT SPAN), SUM((SPAN IS NULL)::INT) FROM parcels_raw""").df() +``` + +## JOIN KEY CREATION +While SPAN (school property access number) would be the best join key, since it represents the government itself's storage manner for parcels, it isnt fully non null, and there are about 18000 rows that are non-uniue in this dataset, where the row "OBJECTID" is fully present, representing a purely trustworthy join key. +```{python} +#just checking if the row count and object id are the same length +print(con.execute("SELECT COUNT(*) FROM parcels_raw").df()) +print(con.execute("SELECT COUNT(DISTINCT OBJECTID) FROM parcels_raw").df()) +#they are, so we can use objectid as the join key, and keep span around for more techincal tasks. + + +#continuing, we are summing where both tname (9577 nulls) and span (7862 null) are both null. +print(con.execute("""SELECT + SUM((SPAN IS NULL AND TNAME IS NULL)::INT) AS both_null, + SUM((SPAN IS NULL AND TNAME IS NOT NULL)::INT) AS span_only, + SUM((SPAN IS NOT NULL AND TNAME IS NULL)::INT) AS tname_only +FROM parcels_raw""").df()) +#something of note: there are no span-null only rows, 7862 where both are nulls, and 1715 tname only nulls, meaninng that the span nulls are a subset of the tname nulls. +``` + +## CREATING THE INFAMOUS TABLES +```{python} +#just before I forget, I think it is good practice to have both the TOWN and TNAME columns as uppercase, so that there aren't any mixups, or Colchester != COLCHESTER, ETC. +#this is the final view from which ot ship off other changes and add them to this final iteration, is my thought. +con.execute(""" + CREATE OR REPLACE VIEW final_view AS + SELECT + * EXCLUDE (TOWN, TNAME), + UPPER(TOWN) AS TOWN, + UPPER(TNAME) AS TNAME + FROM parcels_raw; +""") +``` +### Finally sending off the tables created +Fully creating three tables, parcels_geom, which has the objectid and geom, the parcels_info table, which ahs information like the access nuber, town, source, and year, and the parcels_value table, which has the acres, and all sorts of value metrics. +```{python} +#changing the name to {thing}.parcel +con.execute(""" +CREATE OR REPLACE VIEW geom AS +SELECT OBJECTID, geom FROM final_view +""") + +con.execute(""" +CREATE OR REPLACE VIEW info AS +SELECT OBJECTID, SPAN, PARCID, TNAME, TOWN, PROPTYPE, SOURCENAME, YEAR +FROM final_view +""") + +con.execute(""" +CREATE OR REPLACE VIEW value AS +SELECT OBJECTID, ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV +FROM final_view +""") +``` + +somethind of note to consider. the build.py agglomerates parquets, not FGBS, so im +```{python} +path = Path("Data/_Processed/parcels") +path.mkdir(parents=True, exist_ok=True) + +#changing the name to be more terse adn to the point, in the hopes that the glob agglomerator will add the "parcels/" prefix +for item in ["info", "value", "geom"]: + con.execute(f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)") +``` + +#checking that it worked - (especially the uppercasing of the town name and columns) +```{python} +con.execute(f"SELECT DISTINCT TNAME FROM '{path / 'info.parquet'}' LIMIT 15").df() +``` \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 64a6f3bc..5dfba9b3 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -10,11 +10,16 @@ dependencies = [ "fastapi>=0.137.2", "geopandas>=1.1.3", "jinja2>=3.1.6", + "jupyter-cache>=1.0.1", "matplotlib>=3.11.0", + "nbclient>=0.11.0", + "nbformat>=5.10.4", "numpy>=2.4.6", "pandas>=3.0.3", + "pathlib>=1.0.1", "pydantic>=2.13.4", "pyogrio>=0.12.1", + "pyyaml>=6.0.3", "requests>=2.34.2", "shapely>=2.1.2", "uvicorn>=0.49.0", diff --git a/backend/uv.lock b/backend/uv.lock index fdd6d716..3f877bcc 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -58,6 +58,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "backend" version = "0.1.0" @@ -68,11 +77,16 @@ dependencies = [ { name = "fastapi" }, { name = "geopandas" }, { name = "jinja2" }, + { name = "jupyter-cache" }, { name = "matplotlib" }, + { name = "nbclient" }, + { name = "nbformat" }, { name = "numpy" }, { name = "pandas" }, + { name = "pathlib" }, { name = "pydantic" }, { name = "pyogrio" }, + { name = "pyyaml" }, { name = "requests" }, { name = "shapely" }, { name = "uvicorn" }, @@ -95,11 +109,16 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.137.2" }, { name = "geopandas", specifier = ">=1.1.3" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "jupyter-cache", specifier = ">=1.0.1" }, { name = "matplotlib", specifier = ">=3.11.0" }, + { name = "nbclient", specifier = ">=0.11.0" }, + { name = "nbformat", specifier = ">=5.10.4" }, { name = "numpy", specifier = ">=2.4.6" }, { name = "pandas", specifier = ">=3.0.3" }, + { name = "pathlib", specifier = ">=1.0.1" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pyogrio", specifier = ">=0.12.1" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.34.2" }, { name = "shapely", specifier = ">=2.1.2" }, { name = "uvicorn", specifier = ">=0.49.0" }, @@ -446,6 +465,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/ed/0c6b644e99fb5697d8bdcd36cdb47c52e77a63fc7a1514b1f03a6ecab955/fastapi-0.137.2-py3-none-any.whl", hash = "sha256:791d36261e916a98b25ac85ee591bc3db159394070f6d3d096d94fb378f60ce2", size = 122252, upload-time = "2026-06-18T06:58:26.074Z" }, ] +[[package]] +name = "fastjsonschema" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + [[package]] name = "fonttools" version = "4.63.0" @@ -496,6 +524,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e2/f1/fbbfef6af0bad0548f09bc28948ea3c275b4edb19e17fc5ca9900a6a634d/greenlet-3.5.3.tar.gz", hash = "sha256:a61efc018fd3eb317eeca31aba90ee9e7f26f22884a79b6c6ec715bf71bb62f1", size = 200270, upload-time = "2026-06-26T19:28:24.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, + { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7e/220a7f5824a64a60443fc03b39dfac4ea63a7fb6d481efa27eafa928e7f4/greenlet-3.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:dc133a1569ee667b2a6ef56ce551084aeefd87a5acbc4736d336d1e2edc6cfc4", size = 238141, upload-time = "2026-06-26T18:22:48.507Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/43e116ee114b28737ba7e12952a0d4e2f55944d0f84e42bc91ba7192a3c9/greenlet-3.5.3-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:fd2e02fa07485778536a036222d616ab957b1d533f36b3ed98ce725d9c9d3117", size = 288202, upload-time = "2026-06-26T18:23:49.604Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/146d218299046a43d1f029fd544b3d110d0f175a09c715c7e8da4a4a345d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0a0628d1597eb0897b62f55d1343f772405fd25f3b2a796c76874b0c2e22e8", size = 654096, upload-time = "2026-06-26T19:07:12.71Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cc/04738cafb3f45fa991ea44f9de94c47dcec964f5a972300988a6751f49d9/greenlet-3.5.3-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ebd933a6adabc298bab47731a130fe6bfb888bd934eee37810f151159544540d", size = 666304, upload-time = "2026-06-26T19:10:09.503Z" }, + { url = "https://files.pythonhosted.org/packages/ce/aa/4e0dad5e605c270c784ab911c43da6adb136ccd4d81180f763ca429a723d/greenlet-3.5.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b9d501b40e80b70e32323c799dd9b420a5577a9601469d362ae1ffb690f3a7c", size = 663635, upload-time = "2026-06-26T18:32:20.802Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/13efdbea246fe3d3b735e191fec08fb50809f53cd2383ebe123d0809e44b/greenlet-3.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a1fad1d11e7d6aab184107baa8e4ece11ccba3ec9599cd7efa5ff4d70d43256a", size = 1621252, upload-time = "2026-06-26T19:09:05.647Z" }, + { url = "https://files.pythonhosted.org/packages/f7/22/c0a336ae4a1410fd5f5121098e5bfbf1865f64c5ef80b4b5412886c4a332/greenlet-3.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fad5aec764399f1b5cc347ad250a59660f20c8f8888ea6bae1f93b769cce1154", size = 1684824, upload-time = "2026-06-26T18:31:47.738Z" }, + { url = "https://files.pythonhosted.org/packages/7a/94/91aec0030bea75c4b3244251d0de60a1f3432d1ecb53ab6c437fb5c3ba61/greenlet-3.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:7669aa24cf2a1041d6f7899575b494a3ab4cf68bfcc8609b1dc0be7272db835e", size = 240754, upload-time = "2026-06-26T18:22:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/e5/06/68d0983e79e02138f64b4d303c500c27ddb48e5e77f3debb80888a921eae/greenlet-3.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:5b4807c4082c9d1b6d9eed56fcd041863e37f2228106eef24c30ca096e238605", size = 239549, upload-time = "2026-06-26T18:22:42.996Z" }, + { url = "https://files.pythonhosted.org/packages/91/95/3e161213d7f1d378d15aa9e792093e9bfe01844680d04b7fd6e0107c9098/greenlet-3.5.3-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:271a8ea7c1024e8a0d7dd2be66dd66dda8a07193f41a17b9e924f7600f5b62be", size = 296389, upload-time = "2026-06-26T18:22:20.657Z" }, + { url = "https://files.pythonhosted.org/packages/00/92/715c44721abe2b4d1ae9abde4179411868a5bff312479f54e105d372f131/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19131729ae0ddc3c2e1ef85e650169b5e37ee32e400f215f78b94d7b0d567310", size = 653382, upload-time = "2026-06-26T19:07:14.209Z" }, + { url = "https://files.pythonhosted.org/packages/a0/83/37a10372a1090a6624cca8e74c12df1a36c2dc36429ed0255b7fb1aeee23/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1540dd8e5fc2a5aec40fbb98ef8e149fa47c89a4b4a1cf2575a14d3d1869d7a8", size = 659401, upload-time = "2026-06-26T19:10:10.876Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/d1509cad4207da559cc42986ecdd8fc67ad0d1bba2bf03023c467fd5e0f3/greenlet-3.5.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e81fa194a1d20967877bdf9c7794db2bc99063e5be36aee710c08f04c5bb087f", size = 656969, upload-time = "2026-06-26T18:32:22.272Z" }, + { url = "https://files.pythonhosted.org/packages/86/7d/eaf70de20aadca3a5884aec58362861c64ce45e7b277f47ed026926a3b89/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55cf4d777485d43110e47133cbba6d74a8885a87ec1227ef0267f9ee80c5aa21", size = 1617822, upload-time = "2026-06-26T19:09:06.893Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f9/414d38fc400ae4350d4185eaad1827676f7cf5287b9136e0ed1cbbe20a7f/greenlet-3.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:12a248ba75f6a9a236375f52296c498c89ff1d8badf32deb9eca7abd5853f7da", size = 1677983, upload-time = "2026-06-26T18:31:49.396Z" }, + { url = "https://files.pythonhosted.org/packages/e4/15/7edb977e08f9bff702fe42d6c902702786ff6b9694058b4e6a2a6ac90e57/greenlet-3.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:efc6bd60ea02e085862c74a3ef64b147ffc6f1a5ea7d9f26e7a939943f68c1e3", size = 243626, upload-time = "2026-06-26T18:24:41.485Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/93928dce91e6b3598b5e779e8d1fd6576a504640c58e78627077f6a7a91a/greenlet-3.5.3-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:ea03f2f04367845d6b58eeed276e1e56e51f0b97d8ad5a88a7d20a91dc9056cc", size = 288860, upload-time = "2026-06-26T18:22:48.07Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ca/69db42d447a1378043e2c8f19c09cbbd1263371505053c496b49066d3d16/greenlet-3.5.3-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78dbef602fda6d97d957eb7937f70c9ce9e9527330347f8f6b6f9e554a9e7a47", size = 659747, upload-time = "2026-06-26T19:07:15.565Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0b/af7ac2ef8dd41e3da1a40dda6305c23b9a03e13ba975ec916357b50f8575/greenlet-3.5.3-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f73857adb8fee13fa56c172bd11262f888c0c648f9fea113e777bb2c7904a81", size = 670419, upload-time = "2026-06-26T19:10:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/51/1e/1d51640cacbfc455dbe9f9a9f594c49e4e244f63b9971a2f4764e46cc53d/greenlet-3.5.3-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:232fec92e823addaf02d9472cf7381e24a1d046a6ced1103c5caa4c21b9dfc1d", size = 668787, upload-time = "2026-06-26T18:32:24.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/4030d5b0b5894500023f003bb054d9bb354dfbd1e186c3a296759172f5f5/greenlet-3.5.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2421c3564da9429d5586d46ca31ebb26516b5498a802cf65c041a8e8a8980d34", size = 1626305, upload-time = "2026-06-26T19:09:08.281Z" }, + { url = "https://files.pythonhosted.org/packages/0e/50/5221371c7550108dfa3c378debc41d032aa9c78e89abb01d8011cfc93289/greenlet-3.5.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e0f0d160f0b2e558e6c75f7930967183255dc9735e5f5b8cae58ee09c9576d8b", size = 1688631, upload-time = "2026-06-26T18:31:51.278Z" }, + { url = "https://files.pythonhosted.org/packages/68/5d/00d469daae3c65d2bf620b10eee82eb022127d483c6bc8c69fae6f3fbf17/greenlet-3.5.3-cp315-cp315-win_amd64.whl", hash = "sha256:dd99329bbc15ca78dcc583dba05d0b1b0bae01ab6c2174989f5aaee3e41ac930", size = 241027, upload-time = "2026-06-26T18:22:38.203Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e8/883785b44c5780ed71e83d3e4437e710470be17a2e181e8b601e2da0dc4a/greenlet-3.5.3-cp315-cp315-win_arm64.whl", hash = "sha256:499fef2acede88c1864a57bb586b4bf533c81e1b82df7ab93451cdb47dfec227", size = 240085, upload-time = "2026-06-26T18:23:54.217Z" }, + { url = "https://files.pythonhosted.org/packages/1c/da/4f4a8450962fad137c1c8981a3f1b8919d06c829993d4d476f9c525d5173/greenlet-3.5.3-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:176bc16a721fa5fc294d70b87b4dfa5fbdd251b3da5d5372735ecef9bd7d6d0c", size = 297221, upload-time = "2026-06-26T18:23:27.176Z" }, + { url = "https://files.pythonhosted.org/packages/57/66/b3bfae3e220a9b63ea539a0eea681800c69ab1aada757eae8789f183e7ce/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:629b614d2b786e89c50440e246f33eea78f58a962d0bdbbcc809e6d13605903f", size = 657221, upload-time = "2026-06-26T19:07:16.973Z" }, + { url = "https://files.pythonhosted.org/packages/7b/81/b6d4d73a709684fc77e7fa034d7c2fe82cffa9fc920fadcaa659c2626213/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2b2e857ae16f5f72142edf75f9f176fe7526ba19a2841df1420516f83831c9f2", size = 663226, upload-time = "2026-06-26T19:10:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/f5/07/e210b02b589f16e74ff48b730690e4a34ffe984219fce4f3c1a0e7ec8545/greenlet-3.5.3-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e515757e2e36bcbf1fad09a46e1557e8b1ae1797d4b44d09da7deed88ad28608", size = 660802, upload-time = "2026-06-26T18:32:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/eb/2e/5303eb3fa06bca089060f479707182a93e360683bc252acf846c3090d34e/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b363d46ed1ea431825fdb01471bb024fc08399bad1572a616e853c7684415adb", size = 1622157, upload-time = "2026-06-26T19:09:09.527Z" }, + { url = "https://files.pythonhosted.org/packages/54/70/50de47a488f14df260b50ae34fb5d56016e308b098eab02c878b5223c26a/greenlet-3.5.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e44da2f5bbdaabaf7d80b73dbb430c7035771e9f244e3c8b769715c9d8fa0a16", size = 1681159, upload-time = "2026-06-26T18:31:52.986Z" }, + { url = "https://files.pythonhosted.org/packages/a7/13/1055e1dda7882073eda533e2b96c62e55bbd2db7fda6d5ece992febc7071/greenlet-3.5.3-cp315-cp315t-win_amd64.whl", hash = "sha256:8ff8bed3e3baa20a3ea261ce00526f1898ad4801d4886fd2220580ee0ad8fadf", size = 244007, upload-time = "2026-06-26T18:22:04.353Z" }, + { url = "https://files.pythonhosted.org/packages/b4/0d/ca7d15afbdc397e3401134c9e1800d51d12b829661786187a4ad08fe484f/greenlet-3.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:b7068bd09f761f3f5b4d214c2bed063186b2a86148c740b3873e3f56d79bac31", size = 242586, upload-time = "2026-06-26T18:23:37.93Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -514,6 +589,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -605,6 +692,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "jupyter-cache" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "importlib-metadata" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "pyyaml" }, + { name = "sqlalchemy" }, + { name = "tabulate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/f7/3627358075f183956e8c4974603232b03afd4ddc7baf72c2bc9fff522291/jupyter_cache-1.0.1.tar.gz", hash = "sha256:16e808eb19e3fb67a223db906e131ea6e01f03aa27f49a7214ce6a5fec186fb9", size = 32048, upload-time = "2024-11-15T16:03:55.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/6b/67b87da9d36bff9df7d0efbd1a325fa372a43be7158effaf43ed7b22341d/jupyter_cache-1.0.1-py3-none-any.whl", hash = "sha256:9c3cafd825ba7da8b5830485343091143dff903e4d8c69db9349b728b140abf6", size = 33907, upload-time = "2024-11-15T16:03:54.021Z" }, +] + [[package]] name = "jupyter-client" version = "8.9.1" @@ -813,6 +946,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "nbclient" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, +] + +[[package]] +name = "nbformat" +version = "5.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, +] + [[package]] name = "nest-asyncio2" version = "1.7.2" @@ -934,6 +1097,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] +[[package]] +name = "pathlib" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -1378,6 +1550,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/d6/4bfbb40c9a0b42fc53c7cf442f6385db70b40f74a783130c5d0a5aa62228/pyzmq-27.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dc5dbf68a7857b59473f7df42650c621d7e8923fb03fa74a526890f4d33cc4d7", size = 575170, upload-time = "2025-09-08T23:09:01.418Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "regex" version = "2026.6.28" @@ -1465,6 +1650,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] + [[package]] name = "scipy" version = "1.18.0" @@ -1567,6 +1833,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.51" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/fe/a210d52fd1a90ecfae8a78e9d8b27e18d733d60818a8bf250ff690b75120/sqlalchemy-2.0.51-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07", size = 2157184, upload-time = "2026-06-15T16:08:50.374Z" }, + { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, + { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, + { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/94/df/de669c7054cd47c4439ac34b1b2ee8b804a794791fbb10720e997a2c87c7/sqlalchemy-2.0.51-cp313-cp313-win32.whl", hash = "sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b", size = 2117721, upload-time = "2026-06-15T16:23:12.36Z" }, + { url = "https://files.pythonhosted.org/packages/d0/8a/403c51d064196bae20a0bc2476577f83a3f8dd299719a97417086b7f2ec5/sqlalchemy-2.0.51-cp313-cp313-win_amd64.whl", hash = "sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5", size = 2143615, upload-time = "2026-06-15T16:23:13.906Z" }, + { url = "https://files.pythonhosted.org/packages/b1/49/a739be2e1d02a96a658eb71ab45d921c874249252358ad24a5bffdd02525/sqlalchemy-2.0.51-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491", size = 2158999, upload-time = "2026-06-15T16:08:51.759Z" }, + { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, + { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/24/16/3efd2ee6bc4ca4693a30a1dd17a91b606cae15d517d2a4746611d9b73ce8/sqlalchemy-2.0.51-cp314-cp314-win32.whl", hash = "sha256:08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8", size = 2120551, upload-time = "2026-06-15T16:23:15.629Z" }, + { url = "https://files.pythonhosted.org/packages/7b/78/55b12e70f45bccc40d9e483925c065027b3b98ea4cbbdf6f8c2546feaf6c/sqlalchemy-2.0.51-cp314-cp314-win_amd64.whl", hash = "sha256:96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499", size = 2146318, upload-time = "2026-06-15T16:23:17.108Z" }, + { url = "https://files.pythonhosted.org/packages/21/db/a9574ed40fed418924b1b1a3e54f47ee3963053b3d3d325a0d36b41f2c08/sqlalchemy-2.0.51-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de", size = 2178920, upload-time = "2026-06-15T15:59:56.285Z" }, + { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, + { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, + { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c9/f14fdf71bb8957e0c7e39db69bbdf12b5c80f4ef775fdfa127bf4e0d6760/sqlalchemy-2.0.51-cp314-cp314t-win32.whl", hash = "sha256:9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7", size = 2151313, upload-time = "2026-06-15T16:03:39.127Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c6/673e618e6f4f297e126d9b56ea2f6478708f6c1af4e3223835c22e2c3697/sqlalchemy-2.0.51-cp314-cp314t-win_amd64.whl", hash = "sha256:159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2", size = 2186280, upload-time = "2026-06-15T16:03:40.569Z" }, + { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, +] + [[package]] name = "sqlfluff" version = "4.2.2" @@ -1615,6 +1915,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + [[package]] name = "tblib" version = "3.2.2" @@ -1737,3 +2046,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/f7/d0/9ec6cd4913a726a19 wheels = [ { url = "https://files.pythonhosted.org/packages/1c/6c/6bbe66d9e40d8c50552a7df4014f557ae3aa67b990a4fa3f56fd92d07204/xycmap-1.0.1-py3-none-any.whl", hash = "sha256:669652bd2049f251d713d73ae18694a4b6ef92f0f9a5c8092fc5bbd1f5fb50b8", size = 6022, upload-time = "2021-03-02T11:00:32.961Z" }, ] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] From 0ee6ff2023b0ffad8fea6921112f4c7e4453bc18 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Wed, 22 Jul 2026 19:01:59 -0400 Subject: [PATCH 02/12] Ignore processed parcels outputs and notebook build artifacts --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 99776bea..41080d10 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,8 @@ backend/logger/*.log out/ dist/ backend/Data/Census/**/*.csv +backend/Data/_Processed/parcels/ +**/.jupyter_cache/ +backend/notebooks/**/*.html +backend/notebooks/**/*_files/ +backend/notebooks/**/*.ipynb From 60f33342b8e82ea2d40c8d07722a08abee8cafaa Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Thu, 23 Jul 2026 14:00:06 -0400 Subject: [PATCH 03/12] added geo_query and table_info.sql --- backend/notebooks/parcels/table_build.qmd | 19 ++----- frontend/package-lock.json | 68 ----------------------- 2 files changed, 6 insertions(+), 81 deletions(-) diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd index cac25e94..d573c55c 100644 --- a/backend/notebooks/parcels/table_build.qmd +++ b/backend/notebooks/parcels/table_build.qmd @@ -301,7 +301,7 @@ con.execute(""" """) ``` ### Finally sending off the tables created -Fully creating three tables, parcels_geom, which has the objectid and geom, the parcels_info table, which ahs information like the access nuber, town, source, and year, and the parcels_value table, which has the acres, and all sorts of value metrics. +Fully creating three tables, parcels_geom, which has the objectid and geom, the parcels_info table, which ahs information like the access nuber, town, source, and year, and the parcels_value table, which has the acres, and all sorts of value metrics. - got rid of value the table ```{python} #changing the name to {thing}.parcel con.execute(""" @@ -309,17 +309,10 @@ CREATE OR REPLACE VIEW geom AS SELECT OBJECTID, geom FROM final_view """) -con.execute(""" -CREATE OR REPLACE VIEW info AS -SELECT OBJECTID, SPAN, PARCID, TNAME, TOWN, PROPTYPE, SOURCENAME, YEAR -FROM final_view -""") - -con.execute(""" -CREATE OR REPLACE VIEW value AS -SELECT OBJECTID, ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV -FROM final_view -""") +con.execute("""CREATE OR REPLACE VIEW info AS +SELECT OBJECTID, SPAN, PARCID, TNAME, TOWN, PROPTYPE, SOURCENAME, YEAR, + ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV +FROM final_view""") ``` somethind of note to consider. the build.py agglomerates parquets, not FGBS, so im @@ -328,7 +321,7 @@ path = Path("Data/_Processed/parcels") path.mkdir(parents=True, exist_ok=True) #changing the name to be more terse adn to the point, in the hopes that the glob agglomerator will add the "parcels/" prefix -for item in ["info", "value", "geom"]: +for item in ["info", "geom"]: con.execute(f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)") ``` diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 38c0abcb..25e4d1ba 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8061,33 +8061,6 @@ "node": ">= 6" } }, - "node_modules/framer-motion": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.42.2.tgz", - "integrity": "sha512-5XY9luDiu0oHfHBjpDthFMh0ES+122w6p/papSJBweMkO8Sn+PW2QaEgRblQBpWFnuvZS5qvarpt/hO2pjGmnw==", - "license": "MIT", - "dependencies": { - "motion-dom": "^12.42.2", - "motion-utils": "^12.39.0", - "tslib": "^2.4.0" - }, - "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" - }, - "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, "node_modules/fs-extra": { "version": "11.3.3", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.3.tgz", @@ -10941,47 +10914,6 @@ "node": "*" } }, - "node_modules/motion": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/motion/-/motion-12.42.2.tgz", - "integrity": "sha512-Atvv11yUKIid41cVrRBDVX5m8tF8kNpExRSlbpt6APClhDjtwQssgFHhQzejxw7/7YYbjHSPKBVbHo05BuJT5Q==", - "license": "MIT", - "dependencies": { - "framer-motion": "^12.42.2", - "tslib": "^2.4.0" - }, - "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" - }, - "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, - "node_modules/motion-dom": { - "version": "12.42.2", - "resolved": "https://registry.npmjs.org/motion-dom/-/motion-dom-12.42.2.tgz", - "integrity": "sha512-5gIMWLp/PycBtJRJWRgjxke5n8dlvkSn2DrYW+tr3XcqAZY1xZh6BJyooJXCM8wdfM7wfMjkBJNLge1CKPUIRA==", - "license": "MIT", - "dependencies": { - "motion-utils": "^12.39.0" - } - }, - "node_modules/motion-utils": { - "version": "12.39.0", - "resolved": "https://registry.npmjs.org/motion-utils/-/motion-utils-12.39.0.tgz", - "integrity": "sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ==", - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", From d9807abe21d53fda593fa26fd23a63dde8a241ce Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Thu, 23 Jul 2026 15:31:26 -0400 Subject: [PATCH 04/12] sql first query successful! --- backend/Data/_Processed/all_data.duckdb | 4 +- backend/build/main.py | 1 + backend/build/parcels.py | 94 +++++++++++++++++++ backend/geo_query.sql | 28 ++++++ backend/notebooks/parcels/build.qmd | 25 +++++ .../notebooks/parcels/parcels_run_test.qmd | 34 +++++++ backend/notebooks/parcels/table_build.qmd | 36 +++++-- backend/query/parcels.py | 58 ++++++++++++ .../query/sql/parcels/geo_query_parcels.sql | 27 ++++++ backend/query/sql/parcels/info_table.sql | 10 ++ 10 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 backend/build/parcels.py create mode 100644 backend/geo_query.sql create mode 100644 backend/query/parcels.py create mode 100644 backend/query/sql/parcels/geo_query_parcels.sql create mode 100644 backend/query/sql/parcels/info_table.sql diff --git a/backend/Data/_Processed/all_data.duckdb b/backend/Data/_Processed/all_data.duckdb index 22dad264..b92fd758 100644 --- a/backend/Data/_Processed/all_data.duckdb +++ b/backend/Data/_Processed/all_data.duckdb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c5726aedea597dbda21132fbcf9ac52804a8e9f68c6970e1e2258a1f954959d4 -size 335556608 +oid sha256:bcf4b53b2f16086487c0e16d8ab4009bc77547685eab8bd4c2bbfecf817fb156 +size 588787712 diff --git a/backend/build/main.py b/backend/build/main.py index c091cd43..fcefe47f 100644 --- a/backend/build/main.py +++ b/backend/build/main.py @@ -14,6 +14,7 @@ def main(): acs5.main() cdc.main() zoning.main() + parcels.main() if __name__ == "__main__": diff --git a/backend/build/parcels.py b/backend/build/parcels.py new file mode 100644 index 00000000..8631f0b2 --- /dev/null +++ b/backend/build/parcels.py @@ -0,0 +1,94 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-07-23 +**Description**: + Build script to convert the CDC-places csv data into a single vermont-based SQL table. +""" + +from pathlib import Path + +from build import BACKEND, CON + +data_directory = Path("Data") +parcels_fgb = Path("Data/parcels/parcels_f.fgb") +parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + + +proc_dir = BACKEND / "Data" / "_Processed" / "parcels" +TABLES = ["geom", "info"] +INFO_COLS = [ + "OBJECTID", + "SPAN", + "CAT", + "RESCODE", + "PARCID", + "CITYGL", + "TOWN", + "ZIPGL", + "PROPTYPE", + "DESCPROP", + "SOURCENAME", + "YEAR", + "ACRESGL", + "REAL_FLV", + "HSTED_FLV", + "IMPRV_LV", +] + + +def load_dataset(): + if not parcels_fgb.exists(): + CON.execute(f""" + COPY ( + SELECT * FROM ST_Read('{parcel_path}') + WHERE geom IS NOT NULL + ) + TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') + """) + CON.execute( + "CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/Parcels/parcels_f.fgb')" + ) + + CON.execute(""" + CREATE OR REPLACE VIEW final_view AS + SELECT + * EXCLUDE (TOWN), + UPPER(TOWN) AS TOWN, + FROM parcels_raw; + """) + + +def load_geom(): + CON.execute(""" + CREATE OR REPLACE VIEW geom AS + SELECT OBJECTID, geom FROM final_view + """) + + +def load_info(): + CON.execute("""CREATE OR REPLACE VIEW info AS + SELECT OBJECTID, SPAN, CAT, RESCODE, PARCID, CITYGL, TOWN, ZIPGL, PROPTYPE, DESCPROP, SOURCENAME, YEAR, + ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV + FROM final_view""") + + +def final_writing(): + path = Path("Data/_Processed/parcels") + path.mkdir(parents=True, exist_ok=True) + for item in ["info", "geom"]: + CON.execute( + f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)" + ) + + +def main(): + load_dataset() + load_geom() + load_info() + final_writing() + + +if __name__ == "__main__": + main() diff --git a/backend/geo_query.sql b/backend/geo_query.sql new file mode 100644 index 00000000..1540f9c4 --- /dev/null +++ b/backend/geo_query.sql @@ -0,0 +1,28 @@ +{{ cte_filter_block }} +SELECT + JSON_OBJECT( + 'type', 'FeatureCollection', + 'features', JSON_GROUP_ARRAY(feature) + )::VARCHAR AS fc +FROM ( + SELECT + JSON_OBJECT( + 'type', 'Feature', + 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(g.geom, 0.0001))::JSON, + 'properties', JSON_OBJECT( + 'Parcel Type', i.CAT, + 'Acres', ROUND(i.ACRESGL, 2), + 'rgba_color', '[100,150,200,150]'::JSON, + 'tooltip', JSON_OBJECT( + '__title__', 'Parcels', + 'Place', i.CITYGL, + 'Parcel Description', i.DESCPROP, + 'Acres', ROUND(i.ACRESGL, 2) + ) + ) + ) AS feature + FROM parcels_info AS i + INNER JOIN parcels_geom AS g USING (OBJECTID) + {{ join_filter_block }} +) AS features + diff --git a/backend/notebooks/parcels/build.qmd b/backend/notebooks/parcels/build.qmd index dadba445..22eb8a99 100644 --- a/backend/notebooks/parcels/build.qmd +++ b/backend/notebooks/parcels/build.qmd @@ -18,4 +18,29 @@ execute: editor: render-on-save: true --- +#root finder +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) + +import logging + +from api.models import FilterSource +from query.processed_db import DB +from sql_render import sql_filter_block +``` + +```{python} + + + +``` diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index fdaba11a..cb8488ec 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -19,3 +19,37 @@ editor: render-on-save: true --- #ill get to this later +```{python} +from pathlib import Path +import os +import sys +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +sys.path.insert(0, str(_project_root)) +print(os.getcwd()) + +from api.models import FilterSource +from sql_render import sql_filter_block + + + +sql_dir = Path("query/sql/parcels") +src = FilterSource(filter_table="parcels_info", filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", join_type="inner") +sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", [src]) +print(sql) # ← actually read the rendered SQL; watch the CTE appear +``` + +```{python} +#testing tnat the rendering worked correctly +#correctly outputted PUTNEY town property, land with building, with 120.9 acres + +import json +from query.processed_db import DB +res = DB.execute(sql, params).fetchone()[0] +fc = json.loads(res) +print(len(fc["features"])) +print(fc["features"][0]) +``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd index d573c55c..c3f607ea 100644 --- a/backend/notebooks/parcels/table_build.qmd +++ b/backend/notebooks/parcels/table_build.qmd @@ -75,10 +75,10 @@ con.execute("CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/P Here, we print all the columns in the parcels_raw view, of which there are many. commented out is the code to yield the total row count, with is 191156. Now looking at the column types and their respective names (what they represent in real life), and making a reproducible code chunk to investigate what each means and what it looks like ```{python} described_parcels_raw = con.execute("DESCRIBE parcels_raw").df() -print(described_parcels_raw) -print(con.execute("SELECT COUNT(*) AS row_count, COUNT (DISTINCT OBJECTID) AS distinct_ids, SUM((OBJECTID IS NULL)::INT) AS nulls FROM parcels_raw").df()) +#print(described_parcels_raw) +# print(con.execute("SELECT COUNT(*) AS row_count, COUNT (DISTINCT OBJECTID) AS distinct_ids, SUM((OBJECTID IS NULL)::INT) AS nulls FROM parcels_raw").df()) -column_types = described_parcels_raw["column_type"].tolist() +column_types = described_parcels_raw["column_type"].unique().tolist() columns_by_type = {} for thing, item in described_parcels_raw.iterrows(): @@ -90,8 +90,8 @@ for thing, item in described_parcels_raw.iterrows(): columns_by_type[row].append(item["column_name"]) print(columns_by_type) -for key, value in columns_by_type.items(): - print(f"{key}: {len(value)}") +# for key, value in columns_by_type.items(): +# print(f"{key}: {value}") ``` @@ -286,6 +286,10 @@ print(con.execute("""SELECT FROM parcels_raw""").df()) #something of note: there are no span-null only rows, 7862 where both are nulls, and 1715 tname only nulls, meaninng that the span nulls are a subset of the tname nulls. ``` +```{python} +#before we create the tables, I thought it ould be nice to enrich the info one: +con.execute("""SELECT CAT, RESCODE, CITYGL, STGL, ZIPGL, E911ADDR, DESCPROP, LOCAPROP FROM parcels_raw""").df() +``` ## CREATING THE INFAMOUS TABLES ```{python} @@ -304,15 +308,22 @@ con.execute(""" Fully creating three tables, parcels_geom, which has the objectid and geom, the parcels_info table, which ahs information like the access nuber, town, source, and year, and the parcels_value table, which has the acres, and all sorts of value metrics. - got rid of value the table ```{python} #changing the name to {thing}.parcel +print(con.execute("DESCRIBE final_view").df()["column_name"].tolist()) + con.execute(""" CREATE OR REPLACE VIEW geom AS SELECT OBJECTID, geom FROM final_view """) con.execute("""CREATE OR REPLACE VIEW info AS -SELECT OBJECTID, SPAN, PARCID, TNAME, TOWN, PROPTYPE, SOURCENAME, YEAR, +SELECT OBJECTID, SPAN, CAT, RESCODE, PARCID, CITYGL, TOWN, ZIPGL, PROPTYPE, DESCPROP, SOURCENAME, YEAR, ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV FROM final_view""") + +#wont ship tax table just yet! - commented out for now +#con.execute("""CREATE OR REPLACE VIEW tax AS SELECT OBJECTID, ZIPGL, DESCPROP, REAL_FLV, EQUIPVAL, AOEGL_NR FROM final_view""") + +con.execute("""SELECT CAT FROM info""").df() ``` somethind of note to consider. the build.py agglomerates parquets, not FGBS, so im @@ -321,11 +332,20 @@ path = Path("Data/_Processed/parcels") path.mkdir(parents=True, exist_ok=True) #changing the name to be more terse adn to the point, in the hopes that the glob agglomerator will add the "parcels/" prefix -for item in ["info", "geom"]: +#Not doing tax just yet +for item in ["info", "geom"]:#tax commented out con.execute(f"COPY (SELECT * FROM {item}) TO '{path / f'{item}.parquet'}' (FORMAT PARQUET)") ``` #checking that it worked - (especially the uppercasing of the town name and columns) ```{python} -con.execute(f"SELECT DISTINCT TNAME FROM '{path / 'info.parquet'}' LIMIT 15").df() +con.execute(f"SELECT DISTINCT ZIPGL FROM '{path / 'info.parquet'}' LIMIT 15").df() +``` + +```{python} +import duckdb +db = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +print(db.execute("SHOW TABLES").df()) +print(db.execute("SELECT COUNT(*) FROM parcels_info").df()) +print(db.execute("SELECT COUNT(*) FROM parcels_geom").df()) ``` \ No newline at end of file diff --git a/backend/query/parcels.py b/backend/query/parcels.py new file mode 100644 index 00000000..bef91988 --- /dev/null +++ b/backend/query/parcels.py @@ -0,0 +1,58 @@ +""" +**Author**: + Fitz Koch +**Created**: + 2026-06-01 +**Description**: + Functions for serving zoning_info data to the API from the parquet files. +""" + +import logging +from pathlib import Path + +import pandas as pd + +from api.models import FilterSource +from query.processed_db import DB +from sql_render import sql_filter_block + +logger = logging.getLogger(__name__) +sql_dir = Path(__file__).resolve().parent / "sql" / "zoning" + + +def get_zoning_geojson(sources: list[FilterSource]): + sql, params = sql_filter_block(sql_dir / "geo_query.sql", sources) + result = DB.execute(sql, params).fetchone() + if result is None: + logger.error("geo query returned no rows for filters: %s", sources) + raise ValueError(f"no results for filters: {sources}") + return result[0] + + +def get_zoning_rules(sources: list[FilterSource]) -> str: + sql, params = sql_filter_block(sql_dir / "rules.sql", sources) + result = DB.execute(sql, params).fetchone() + if result is None: + logger.error("rules query returned no rows for filters: %s", sources) + raise ValueError(f"no results for filters: {sources}") + return result[0] + + +def get_zoning_aggregated_acres( + sources: list[FilterSource], +) -> tuple[pd.DataFrame, pd.DataFrame]: + agg_sql, agg_params = sql_filter_block(sql_dir / "agg_info_table.sql", sources) + agg_data = DB.execute(agg_sql, agg_params).df() + + table_sql, table_params = sql_filter_block(sql_dir / "info_table.sql", sources) + table_data = DB.execute(table_sql, table_params).df() + return agg_data, table_data + + +def get_zoning_allowances( + sources: list[FilterSource], +) -> tuple[pd.DataFrame, pd.DataFrame]: + agg = DB.execute(*sql_filter_block(sql_dir / "agg_rules_table.sql", sources)).df() + table = DB.execute(*sql_filter_block(sql_dir / "rules_table.sql", sources)).df() + + return agg, table diff --git a/backend/query/sql/parcels/geo_query_parcels.sql b/backend/query/sql/parcels/geo_query_parcels.sql new file mode 100644 index 00000000..b94d5dc6 --- /dev/null +++ b/backend/query/sql/parcels/geo_query_parcels.sql @@ -0,0 +1,27 @@ +{{ cte_filter_block }} +SELECT + JSON_OBJECT( + 'type', 'FeatureCollection', + 'features', JSON_GROUP_ARRAY(feature) + )::VARCHAR AS fc +FROM ( + SELECT + JSON_OBJECT( + 'type', 'Feature', + 'geometry', ST_ASGEOJSON(ST_SIMPLIFY(g.geom, 0.0001))::JSON, + 'properties', JSON_OBJECT( + 'Parcel Type', i.CAT, + 'Acres', ROUND(i.ACRESGL, 2), + 'rgba_color', '[100,150,200,150]'::JSON, + 'tooltip', JSON_OBJECT( + '__title__', 'Parcels', + 'Place', i.CITYGL, + 'Parcel Description', i.DESCPROP, + 'Acres', ROUND(i.ACRESGL, 2) + ) + ) + ) AS feature + FROM parcels_info AS i + INNER JOIN parcels_geom AS g USING (OBJECTID) + {{ join_filter_block }} +) AS features diff --git a/backend/query/sql/parcels/info_table.sql b/backend/query/sql/parcels/info_table.sql new file mode 100644 index 00000000..5f4dfe66 --- /dev/null +++ b/backend/query/sql/parcels/info_table.sql @@ -0,0 +1,10 @@ +{{ cte_filter_block }} + +SELECT + i.CAT AS "Property Category", + i.ACRESGL AS Acres, + i.CITYGL AS "Owner City", + i.TOWN AS Town, + i.DESCPROP AS "Property Description" +FROM parcels_info AS i +INNER {{ join_filter_block }} From 79bfdc16ddeed2e908a5f776cbcc1e7344f5af9f Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Fri, 24 Jul 2026 16:49:56 -0400 Subject: [PATCH 05/12] started on parcels.py in query after having finished the sql portion --- backend/notebooks/parcels/parcels_run_test.qmd | 2 -- backend/query/parcels.py | 11 +++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index cb8488ec..39568265 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -45,11 +45,9 @@ print(sql) # ← actually read the rendered SQL; watch the CTE appear ```{python} #testing tnat the rendering worked correctly #correctly outputted PUTNEY town property, land with building, with 120.9 acres - import json from query.processed_db import DB res = DB.execute(sql, params).fetchone()[0] fc = json.loads(res) -print(len(fc["features"])) print(fc["features"][0]) ``` \ No newline at end of file diff --git a/backend/query/parcels.py b/backend/query/parcels.py index bef91988..18473b22 100644 --- a/backend/query/parcels.py +++ b/backend/query/parcels.py @@ -1,10 +1,12 @@ """ **Author**: - Fitz Koch + Isaac Wedaman **Created**: - 2026-06-01 + 2026-07-23 **Description**: - Functions for serving zoning_info data to the API from the parquet files. + Something of note: this is a "girder" for the sql queries between + the front end (website), and the backend dbeaver database + """ import logging @@ -16,8 +18,9 @@ from query.processed_db import DB from sql_render import sql_filter_block +# changed sql directory to parcels in the sql folder logger = logging.getLogger(__name__) -sql_dir = Path(__file__).resolve().parent / "sql" / "zoning" +sql_dir = Path(__file__).resolve().parent / "sql" / "parcels" def get_zoning_geojson(sources: list[FilterSource]): From aadb4ecf90f7f361d9a024f39d920fb24244d28a Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Thu, 30 Jul 2026 11:51:04 -0400 Subject: [PATCH 06/12] finished query/parcels --- .../notebooks/parcels/parcels_run_test.qmd | 28 ++++++++++++- backend/query/parcels.py | 40 +++++++------------ backend/query/sql/parcels/info_table.sql | 2 +- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index 39568265..3d8e97a2 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -50,4 +50,30 @@ from query.processed_db import DB res = DB.execute(sql, params).fetchone()[0] fc = json.loads(res) print(fc["features"][0]) -``` \ No newline at end of file +``` + +```{python} +from query.parcels import get_parcels_geojson, get_parcels_table, get_parcels_filters +from api.models import FilterSource + +src = FilterSource( + filter_table="parcels_info", + filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", + join_type="inner", +) + +# geojson +fc = json.loads(get_parcels_geojson([src])) +print("features:", len(fc["features"])) +print("one feature:", fc["features"][0]) + +# info table +df = get_parcels_table([src]) +print("table rows:", len(df)) +df.head() + +tree = get_parcels_filters() +print(sorted(tree.tree.keys(), key=len, reverse=True)[:10]) # first few towns + +``` diff --git a/backend/query/parcels.py b/backend/query/parcels.py index 18473b22..81297049 100644 --- a/backend/query/parcels.py +++ b/backend/query/parcels.py @@ -5,7 +5,7 @@ 2026-07-23 **Description**: Something of note: this is a "girder" for the sql queries between - the front end (website), and the backend dbeaver database + the front end (website), and the backend database, with something to do with dbeaver as a goal in mind """ @@ -15,6 +15,7 @@ import pandas as pd from api.models import FilterSource +from query.core_functions import filter_tree from query.processed_db import DB from sql_render import sql_filter_block @@ -23,8 +24,8 @@ sql_dir = Path(__file__).resolve().parent / "sql" / "parcels" -def get_zoning_geojson(sources: list[FilterSource]): - sql, params = sql_filter_block(sql_dir / "geo_query.sql", sources) +def get_parcels_geojson(sources: list[FilterSource]): + sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", sources) result = DB.execute(sql, params).fetchone() if result is None: logger.error("geo query returned no rows for filters: %s", sources) @@ -32,30 +33,19 @@ def get_zoning_geojson(sources: list[FilterSource]): return result[0] -def get_zoning_rules(sources: list[FilterSource]) -> str: - sql, params = sql_filter_block(sql_dir / "rules.sql", sources) - result = DB.execute(sql, params).fetchone() - if result is None: - logger.error("rules query returned no rows for filters: %s", sources) - raise ValueError(f"no results for filters: {sources}") - return result[0] - - -def get_zoning_aggregated_acres( +def get_parcels_table( sources: list[FilterSource], -) -> tuple[pd.DataFrame, pd.DataFrame]: - agg_sql, agg_params = sql_filter_block(sql_dir / "agg_info_table.sql", sources) - agg_data = DB.execute(agg_sql, agg_params).df() - +) -> pd.DataFrame: table_sql, table_params = sql_filter_block(sql_dir / "info_table.sql", sources) table_data = DB.execute(table_sql, table_params).df() - return agg_data, table_data + return table_data -def get_zoning_allowances( - sources: list[FilterSource], -) -> tuple[pd.DataFrame, pd.DataFrame]: - agg = DB.execute(*sql_filter_block(sql_dir / "agg_rules_table.sql", sources)).df() - table = DB.execute(*sql_filter_block(sql_dir / "rules_table.sql", sources)).df() - - return agg, table +def get_parcels_filters(): + PARCEL_FILTER_COLS = { + "Town": "TOWN", + "Category": "CAT", + "Property Type": "PROPTYPE", + } + PARCEL_TREE_LABELS = ["Town", "Category"] + return filter_tree(PARCEL_FILTER_COLS, PARCEL_TREE_LABELS, "parcels_info") diff --git a/backend/query/sql/parcels/info_table.sql b/backend/query/sql/parcels/info_table.sql index 5f4dfe66..bfd6e6ee 100644 --- a/backend/query/sql/parcels/info_table.sql +++ b/backend/query/sql/parcels/info_table.sql @@ -7,4 +7,4 @@ SELECT i.TOWN AS Town, i.DESCPROP AS "Property Description" FROM parcels_info AS i -INNER {{ join_filter_block }} + {{ join_filter_block }} From 0afbfaed9d3884bd3ce902e219b971414492e861 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Thu, 30 Jul 2026 15:05:59 -0400 Subject: [PATCH 07/12] Add parcels query layer, API route, and schema entry --- backend/api/routes/post_routes/__init__.py | 2 ++ backend/api/routes/post_routes/post_parcels.py | 14 ++++++++++++++ backend/api/schema.json | 9 +++++++++ 3 files changed, 25 insertions(+) create mode 100644 backend/api/routes/post_routes/post_parcels.py diff --git a/backend/api/routes/post_routes/__init__.py b/backend/api/routes/post_routes/__init__.py index f2b3cbb4..fe0a639e 100644 --- a/backend/api/routes/post_routes/__init__.py +++ b/backend/api/routes/post_routes/__init__.py @@ -2,6 +2,7 @@ from .post_cdc import router as post_cdc_router from .post_census import router as post_census_router from .post_export import router as post_export_router +from .post_parcels import router as post_parcels_router from .post_qcew import router as post_qcew_router from .post_zoning import router as post_zoning_router @@ -12,4 +13,5 @@ post_qcew_router, post_export_router, post_cdc_router, + post_parcels_router, ] diff --git a/backend/api/routes/post_routes/post_parcels.py b/backend/api/routes/post_routes/post_parcels.py new file mode 100644 index 00000000..0d837e26 --- /dev/null +++ b/backend/api/routes/post_routes/post_parcels.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter, Response + +from api.core_functions import request_to_source +from api.models import FilterRequest +from query.parcels import get_parcels_geojson + +router = APIRouter() + + +@router.post("/load/mapping/parcels") +async def read_parcels(request: FilterRequest): + source = request_to_source(request, "parcels_info", "default") + data = get_parcels_geojson([source]) + return Response(content=data, media_type="application/json") diff --git a/backend/api/schema.json b/backend/api/schema.json index f3774f72..dc185af4 100644 --- a/backend/api/schema.json +++ b/backend/api/schema.json @@ -46,6 +46,15 @@ "Measure": "Measure", "Prevalence Measure": "Data_Value_Type" } + }, + "parcels_info": { + "join_key": "OBJECTID", + "join_type": "inner", + "columns": { + "Town": "TOWN", + "Category": "CAT", + "Property Type": "PROPTYPE" + } } } } From 81a4bc1e2eba5dede4f157445de341c94972fc10 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Tue, 4 Aug 2026 01:42:18 -0400 Subject: [PATCH 08/12] investigating counties --- .../notebooks/parcels/parcels_run_test.qmd | 222 ++++++++++++++++++ backend/notebooks/parcels/table_build.qmd | 6 +- 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index 3d8e97a2..b36ccabe 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -77,3 +77,225 @@ tree = get_parcels_filters() print(sorted(tree.tree.keys(), key=len, reverse=True)[:10]) # first few towns ``` +#here, I am starting to look at the towns and area of chittenden county, to figure out what this dataset ould be used for, with all sql queries + +```{python} +import duckdb +con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +con.execute("LOAD spatial") +con.execute("SHOW TABLES").df() +``` +all the towns in chittenden county, and all the ciolumns. +```{python} +chittenden = ('BURLINGTON','SOUTH BURLINGTON','ESSEX','COLCHESTER','WILLISTON', + 'SHELBURNE','WINOOSKI','MILTON','JERICHO','RICHMOND','HINESBURG', + 'CHARLOTTE','WESTFORD','UNDERHILL','ST. GEORGE','BOLTON','HUNTINGTON','BUELS GORE') + +con.execute("""DESCRIBE parcels_info""").df() +``` +First starting off with all the parcel_counts and median value for each chittenden county town, and median acres +```{python} +#median value + parcel count per town +parcels_medians = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {chittenden} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians) + +#now trying again - seeing why these three are nan +#nulls are in burlington, charlotte, bolton - now checking their actual values in the table - no way they are all null +milton = con.execute("SELECT * FROM parcels_info WHERE TOWN='MILTON'").df() +burlington = con.execute("SELECT * FROM parcels_info WHERE TOWN='BURLINGTON'").df() +charlotte = con.execute("SELECT * FROM parcels_info WHERE TOWN='CHARLOTTE'").df() +bolton = con.execute("SELECT * FROM parcels_info WHERE TOWN='BOLTON'").df() + +for item in [milton, burlington, charlotte, bolton]: + print(item[["TOWN", "REAL_FLV", "ACRESGL", "TOWN", "OBJECTID", "SPAN", "CITYGL"]].head(5)) +``` +#okay, we have a problem - there are some NAN values in towns, where there are no values for a row here and there (WHILE ALWAYS HAVING AN OBJECT ID AND TOWN NAME, BUT NOT ANY VALID IDENTIFYING INFO). however, for the towns of Burlington, Charlotte, and BOLTON, it seems there are no values for any of the identifying info - basically the surrounding info is blank, or nan, but the name of the town and the object id is present. I looked it up online and this may result for from these three municipalities doing things their own way, and from prematurely updating the way they handle thigns without having coordinated with the state of vermont. Ill check how it fares for counties foreign + +```{python} +#now checking with different counties - startin with rutland +rutland = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +parcels_medians_rutland = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {rutland} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians_rutland[["TOWN", "parcels", "median_value", "median_acres"]]) + + +#looks like the last four are misiing - Castleton, chittenden, brandon, benson + +#ill do this one more time for washington county +washington_county_towns = ["BARRE TOWN", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER"] + +parcels_medians_washington = con.execute(f""" + SELECT TOWN, + COUNT(*) AS parcels, + ROUND(MEDIAN(REAL_FLV)) AS median_value, + ROUND(MEDIAN(ACRESGL), 2) AS median_acres + FROM parcels_info + WHERE TOWN IN {washington_county_towns} + GROUP BY TOWN + ORDER BY parcels DESC +""").df() + +print(parcels_medians_washington[["TOWN", "parcels", "median_value", "median_acres"]]) + + + +``` + +```{python} +con.execute(""" + SELECT COUNT(*) AS n, + SUM((REAL_FLV IS NULL)::INT) AS null_val, + SUM((SPAN IS NULL)::INT) AS null_span, + SUM((ACRESGL IS NULL)::INT) AS null_acres + FROM parcels_info + WHERE TOWN = 'BURLINGTON' +""").df() +``` + +```{python} +con.execute("SELECT DISTINCT PROPTYPE, COUNT(*) FROM parcels_info WHERE TOWN='BURLINGTON' GROUP BY PROPTYPE").df() +``` +MAJOR FINDING ALERT! Burlington, and some of the other largest cities (like Barre) in the state of vermont just didn't send in their data - Burlington's parcels, of which it has 187 (Milton, with all informatino valid, has 4.3 thousand), all but 77 arent even parcels of property, but are instead row_road, water, and row_rail prop type. this proves my theory that the problem of missing data is due to some municipalities just not sending in their data, or collaborating with VCGI like smaller towns might have, thus leaving the leftover data to represent parcels that are borders and non-private property + +#now, we must find the percentage of extant data for each town, to see which ones are a worthy candidate for valid analysis - ill also filter by county for more in-depth analysis. +```{python} + +#incipient converage table +coverage = con.execute(""" + SELECT + TOWN, + COUNT(*) AS total_features, + SUM((PROPTYPE = 'PARCEL')::INT) AS parcels, + SUM((REAL_FLV IS NOT NULL)::INT) AS has_value, + ROUND(100.0 * SUM((REAL_FLV IS NOT NULL)::INT) / COUNT(*), 1) AS pct_valued + FROM parcels_info + GROUP BY TOWN + ORDER BY pct_valued DESC +""").df() + + +#print(coverage["pct_valued"].value_counts()) +#there are 31 towns with no parcel data, and 140 that do +print(sum(coverage["pct_valued"] == 0.0)) +coverage_valid = coverage[coverage["pct_valued"] == 0.0].sort_values(by = "parcels", ascending = False) +print(coverage_valid) + +``` +#here, we investigate colcheseter, which has all parcels present, and burlington and charlotte, which have comparatively less parcels, and no accompanying information on those parcels (median values, etc). we find that 7087 of the 7277 parcels in colcehster are "valued" whereas non for the other towns are - and that the "source" is not distinct for colchester, where it is for burlington & charlotte, leading me to believe the a source of null is VGIS data, and that self provided data tends to be less informative +```{python} +con.execute(""" + SELECT TOWN, + COUNT(*) AS n, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued, + COUNT(DISTINCT SOURCENAME) AS sources + FROM parcels_info + WHERE TOWN IN ('BURLINGTON','COLCHESTER','CHARLOTTE') + GROUP BY TOWN +""").df() + +``` +Echeking the previous hypothesis. source == NAN is VGIS data +```{python} +con.execute(""" + SELECT TOWN, SOURCENAME, COUNT(*) AS n, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued + FROM parcels_info + WHERE TOWN IN ('BURLINGTON','CHARLOTTE','COLCHESTER') + GROUP BY TOWN, SOURCENAME + ORDER BY TOWN, n DESC +""").df() +``` +#really seeing if burlington only has 187 parcels, and that they are all null. in a perfect world, I got the column name wrong, but this isn't the case, and the data is missing. +```{python} +con.execute("SELECT DISTINCT TOWN FROM parcels_info WHERE TOWN ILIKE '%burl%'").df() +``` +#here, we investigate the feature source, where if the SOURCENAME is null, where it happens to be for the towns with a loarge amount of parcels (and extant data), it is likely that the data was provided and submitted by VGIS, where if it is not null, it was self submitted, which leads to strange occurrences where the City of Burlington has 187 parcels, provided by themselves, but no data to go along with it +```{python} +con.execute(""" + SELECT + CASE WHEN SOURCENAME IS NULL THEN 'VGIS standard' ELSE 'self-submitted' END AS pipeline, + COUNT(*) AS features, + SUM((REAL_FLV IS NOT NULL)::INT) AS valued, + COUNT(DISTINCT TOWN) AS towns + FROM parcels_info + GROUP BY 1 +""").df() + +``` + +#work for now : adding a column to parcels_info for county. first we must joing sourcename to parcels info and get +#TODO: add sourcename to parcels_info then run build.py +(quick check for sql database integrity) +```{python} +# import duckdb +# con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +# con.execute("LOAD spatial") +# print(con.execute("SHOW TABLES").df()) +``` +```{python} +towns = con.execute("SELECT DISTINCT TOWN FROM parcels_info").df() +counties = [] +not_stateless = [] +#all counties +ADDISON = ["ADDISON", "BRIDPORT", "BRISTOL", "CORNWALL", "FERRISBURGH", "GOSHEN", "GRANVILLE", "HANCOCK", "LEICESTER", "LINCOLN", "MIDDLEBURY", "MONKTON", "NEW HAVEN", "ORWELL", "PANTON", "RIPTON", "SALISBURY", "SHOREHAM", "STARKSBORO", "WALTHAM", "WEYBRIDGE", "WHITING"] +BENNINGTON = ["ARLINGTON", "BENNINGTON", "DORSET", "GLASTENBURY", "LANDGROVE", "MANCHESTER", "PERU", "POWNAL", "READSBORO", "RUPERT", "SANDGATE", "SEARSBURG", "SHAFTSBURY", "STAMFORD", "SUNDERLAND", "WINHALL", "WOODFORD"] +CALEDONIA = ["BARNET", "BURKE", "DANVILLE", "GROTON", "HARDWICK", "KIRBY", "LYNDON", "NEWARK", "PEACHAM", "RYEGATE", "SAINT JOHNSBURY", "SHEFFIELD", "STANNARD", "SUTTON", "WALDEN", "WATERFORD", "WHEELOCK"] +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON"] +ESSEX = ["AVERILL", "BLOOMFIELD", "BRIGHTON", "BRUNSWICK", "CANAAN", "CONCORD", "EAST HAVEN", "FERDINAND", "GRANBY", "GUILDHALL", "LEMINGTON", "LEWIS", "LUNENBURG", "MAIDSTONE", "NORTON", "VICTORY"] +FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS", "SHELDON", "SWANTON"] +GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] +LAMOILLE = ["BELVIDERE", "CAMBRIDGE", "EDEN", "ELMORE", "HYDE PARK", "JOHNSON", "MORRISTOWN", "STOWE", "WATERVILLE", "WOLCOTT"] +ORANGE = ["BRADFORD", "BRAINTREE", "BROOKFIELD", "CHELSEA", "CORINTH", "FAIRLEE", "NEWBURY", "ORANGE", "RANDOLPH", "STRAFFORD", "THETFORD", "TOPSHAM", "TUNBRIDGE", "VERSHIRE", "WASHINGTON", "WEST FAIRLEE", "WILLIAMSTOWN"] +ORLEANS = ["ALBANY", "BARTON", "BROWNINGTON", "CHARLESTON", "COVENTRY", "CRAFTSBURY", "DERBY", "GLOVER", "GREENSBORO", "HOLLAND", "IRASBURG", "JAY", "LOWELL", "MORGAN", "NEWPORT", "TROY", "WESTFIELD", "WESTMORE"] +RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +WASHINGTON = ["BARRE", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER"] +WINDHAM = ["ATHENS", "BRATTLEBORO", "BROOKLINE", "DOVER", "DUMMERSTON", "GRAFTON", "GUILFORD", "HALIFAX", "JAMAICA", "LONDONDERRY", "MARLBORO", "NEWFANE", "PUTNEY", "ROCKINGHAM", "SOMERSET", "STRATTON", "TOWNSHEND", "VERNON", "WARDSBORO", "WESTMINSTER", "WHITINGHAM", "WILMINGTON", "WINDHAM"] +WINDSOR = ["ANDOVER", "BALTIMORE", "BARNARD", "BETHEL", "BRIDGEWATER", "CAVENDISH", "CHESTER", "HARTFORD", "HARTLAND", "LUDLOW", "NORWICH", "PLYMOUTH", 'POMFRET', 'READING', 'ROCHESTER', 'ROYALTON', 'SHARON', 'SPRINGFIELD', 'STOCKBRIDGE', 'WEATHERSFIELD', 'WEST WINDSOR', 'WESTON', 'WINDSOR', 'WOODSTOCK'] +counties_key = [ADDISON, BENNINGTON, CALEDONIA, CHITTENDEN, ESSEX, FRANKLIN, GRAND_ISLE, LAMOILLE, ORANGE, ORLEANS, RUTLAND, WASHINGTON, WINDHAM, WINDSOR] +names_key = ["ADDISON", "BENNINGTON", "CALEDONIA", "CHITTENDEN", "ESSEX", "FRANKLIN", "GRAND_ISLE", "LAMOILLE", "ORANGE", "ORLEANS", "RUTLAND", "WASHINGTON", "WINDHAM", "WINDSOR"] + + +for i in range(len(towns)): + for item in counties_key: + if towns["TOWN"].iloc[i] in item: + counties.append(names_key[(counties_key.index(item))]) + not_stateless.append(towns["TOWN"].iloc[i]) + + + +#looks good so far +print(towns) +print(counties) +print(not_stateless) + +#now seeing the outliers +cities_to_investigate = set(towns["TOWN"].to_list()) - set(not_stateless) +print(cities_to_investigate) + +# towns.head(5) + + + + + + +``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd index c3f607ea..16213a87 100644 --- a/backend/notebooks/parcels/table_build.qmd +++ b/backend/notebooks/parcels/table_build.qmd @@ -317,7 +317,7 @@ SELECT OBJECTID, geom FROM final_view con.execute("""CREATE OR REPLACE VIEW info AS SELECT OBJECTID, SPAN, CAT, RESCODE, PARCID, CITYGL, TOWN, ZIPGL, PROPTYPE, DESCPROP, SOURCENAME, YEAR, - ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV + ACRESGL, REAL_FLV, HSTED_FLV, IMPRV_LV, SOURCENAME FROM final_view""") #wont ship tax table just yet! - commented out for now @@ -348,4 +348,8 @@ db = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) print(db.execute("SHOW TABLES").df()) print(db.execute("SELECT COUNT(*) FROM parcels_info").df()) print(db.execute("SELECT COUNT(*) FROM parcels_geom").df()) +``` + +```{python} +con.execute("SELECT DISTINCT TOWN FROM parcels_raw").df() ``` \ No newline at end of file From 3b80fccd7c5efe411fe3f61239240f1385ceb013 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Tue, 4 Aug 2026 12:41:35 -0400 Subject: [PATCH 09/12] investigating chloropleth --- .../notebooks/parcels/parcels_run_test.qmd | 85 ++++++++++++++----- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index b36ccabe..bb40f68a 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -256,46 +256,89 @@ towns = con.execute("SELECT DISTINCT TOWN FROM parcels_info").df() counties = [] not_stateless = [] #all counties -ADDISON = ["ADDISON", "BRIDPORT", "BRISTOL", "CORNWALL", "FERRISBURGH", "GOSHEN", "GRANVILLE", "HANCOCK", "LEICESTER", "LINCOLN", "MIDDLEBURY", "MONKTON", "NEW HAVEN", "ORWELL", "PANTON", "RIPTON", "SALISBURY", "SHOREHAM", "STARKSBORO", "WALTHAM", "WEYBRIDGE", "WHITING"] +ADDISON = ["ADDISON", "BRIDPORT", "BRISTOL", "CORNWALL", "FERRISBURGH", "GOSHEN", "GRANVILLE", "HANCOCK", "LEICESTER", "LINCOLN", "MIDDLEBURY", "MONKTON", "NEW HAVEN", "ORWELL", "PANTON", "RIPTON", "SALISBURY", "SHOREHAM", "STARKSBORO", "WALTHAM", "WEYBRIDGE", "WHITING", "VERGENNES"] BENNINGTON = ["ARLINGTON", "BENNINGTON", "DORSET", "GLASTENBURY", "LANDGROVE", "MANCHESTER", "PERU", "POWNAL", "READSBORO", "RUPERT", "SANDGATE", "SEARSBURG", "SHAFTSBURY", "STAMFORD", "SUNDERLAND", "WINHALL", "WOODFORD"] CALEDONIA = ["BARNET", "BURKE", "DANVILLE", "GROTON", "HARDWICK", "KIRBY", "LYNDON", "NEWARK", "PEACHAM", "RYEGATE", "SAINT JOHNSBURY", "SHEFFIELD", "STANNARD", "SUTTON", "WALDEN", "WATERFORD", "WHEELOCK"] -CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON"] +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON"] ESSEX = ["AVERILL", "BLOOMFIELD", "BRIGHTON", "BRUNSWICK", "CANAAN", "CONCORD", "EAST HAVEN", "FERDINAND", "GRANBY", "GUILDHALL", "LEMINGTON", "LEWIS", "LUNENBURG", "MAIDSTONE", "NORTON", "VICTORY"] -FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS", "SHELDON", "SWANTON"] +FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS CITY", "SHELDON", "SWANTON"] GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] LAMOILLE = ["BELVIDERE", "CAMBRIDGE", "EDEN", "ELMORE", "HYDE PARK", "JOHNSON", "MORRISTOWN", "STOWE", "WATERVILLE", "WOLCOTT"] ORANGE = ["BRADFORD", "BRAINTREE", "BROOKFIELD", "CHELSEA", "CORINTH", "FAIRLEE", "NEWBURY", "ORANGE", "RANDOLPH", "STRAFFORD", "THETFORD", "TOPSHAM", "TUNBRIDGE", "VERSHIRE", "WASHINGTON", "WEST FAIRLEE", "WILLIAMSTOWN"] ORLEANS = ["ALBANY", "BARTON", "BROWNINGTON", "CHARLESTON", "COVENTRY", "CRAFTSBURY", "DERBY", "GLOVER", "GREENSBORO", "HOLLAND", "IRASBURG", "JAY", "LOWELL", "MORGAN", "NEWPORT", "TROY", "WESTFIELD", "WESTMORE"] -RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] -WASHINGTON = ["BARRE", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER"] +RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +WASHINGTON = ["BARRE TOWN", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER", "BARRE CITY", "MONTPELIER"] WINDHAM = ["ATHENS", "BRATTLEBORO", "BROOKLINE", "DOVER", "DUMMERSTON", "GRAFTON", "GUILFORD", "HALIFAX", "JAMAICA", "LONDONDERRY", "MARLBORO", "NEWFANE", "PUTNEY", "ROCKINGHAM", "SOMERSET", "STRATTON", "TOWNSHEND", "VERNON", "WARDSBORO", "WESTMINSTER", "WHITINGHAM", "WILMINGTON", "WINDHAM"] WINDSOR = ["ANDOVER", "BALTIMORE", "BARNARD", "BETHEL", "BRIDGEWATER", "CAVENDISH", "CHESTER", "HARTFORD", "HARTLAND", "LUDLOW", "NORWICH", "PLYMOUTH", 'POMFRET', 'READING', 'ROCHESTER', 'ROYALTON', 'SHARON', 'SPRINGFIELD', 'STOCKBRIDGE', 'WEATHERSFIELD', 'WEST WINDSOR', 'WESTON', 'WINDSOR', 'WOODSTOCK'] counties_key = [ADDISON, BENNINGTON, CALEDONIA, CHITTENDEN, ESSEX, FRANKLIN, GRAND_ISLE, LAMOILLE, ORANGE, ORLEANS, RUTLAND, WASHINGTON, WINDHAM, WINDSOR] names_key = ["ADDISON", "BENNINGTON", "CALEDONIA", "CHITTENDEN", "ESSEX", "FRANKLIN", "GRAND_ISLE", "LAMOILLE", "ORANGE", "ORLEANS", "RUTLAND", "WASHINGTON", "WINDHAM", "WINDSOR"] - +#perfected the 7 without a county for i in range(len(towns)): for item in counties_key: if towns["TOWN"].iloc[i] in item: counties.append(names_key[(counties_key.index(item))]) not_stateless.append(towns["TOWN"].iloc[i]) +#adding counties to towns +towns["COUNTY"] = counties +``` +now creating the by county and town availiability map +```{python} +#town level group by +con.execute(""" +SELECT c.COUNTY, i.TOWN, + COUNT(*) AS parcels, + SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued +FROM parcels_info i +LEFT JOIN towns c USING (TOWN) +GROUP BY c.COUNTY, i.TOWN""").df() + +#county_level +con.execute("""SELECT c.COUNTY, + COUNT(*) AS parcels, + SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM((i.REAL_FLV IS NOT NULL AND i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued +FROM parcels_info i +LEFT JOIN towns c USING (TOWN) +GROUP BY c.COUNTY ORDER BY pct_valued DESC""").df() +``` +now that we have the parcels and parcels' percentages based on county and town, we must read in the vermont VCGI boundary dataset using the api option, and build the chloropleth from there +```{python} +import geopandas as gpd +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +towns_gdf = gpd.read_file(TOWN_URL) +print(towns_gdf.shape) +print(towns_gdf.columns.tolist()) +print(towns_gdf.crs) +#just certifying that the town namese are both uppercase and devoic of whitespace +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +``` +```{python} +#now, merging the town level merge i made earlier with the town_gdf geojson file +town_coverage = con.execute(""" + SELECT c.COUNTY, i.TOWN, + COUNT(*) AS parcels, + SUM((i.REAL_FLV > 0)::INT) AS valued, + ROUND(100.0 * SUM((i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued + FROM parcels_info i + LEFT JOIN towns c USING (TOWN) + GROUP BY c.COUNTY, i.TOWN +""").df() -#looks good so far -print(towns) -print(counties) -print(not_stateless) - -#now seeing the outliers -cities_to_investigate = set(towns["TOWN"].to_list()) - set(not_stateless) -print(cities_to_investigate) - -# towns.head(5) - - - - +towns_map = towns_gdf.merge(town_coverage, on="TOWN", how="left") +#making sure that all towns are matched +print("unmatched towns:", towns_map["pct_valued"].isna().sum()) +print(towns_map[towns_map["pct_valued"].isna()]["TOWN"].tolist()) +``` -``` \ No newline at end of file +```{python} +#diagnosing the problem at hadn +parcels_towns = set(town_coverage["TOWN"]) +boundary_towns = set(towns_gdf["TOWN"]) +print("in boundary, not parcels:", sorted(boundary_towns - parcels_towns)) +print("in parcels, not boundary:", sorted(parcels_towns - boundary_towns)) +``` From 36052d3b4cfcaa3114998b4c6d54f670d48bddde Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Tue, 18 Aug 2026 10:39:05 -0400 Subject: [PATCH 10/12] starting the final analysis --- backend/notebooks/parcels/investigation.qmd | 56 ++ .../parcels/local_data/population_2020.csv | 256 ++++++++ .../notebooks/parcels/parcels_run_test.qmd | 550 +++++++++++++++++- backend/notebooks/parcels/table_build.qmd | 17 +- .../notebooks/parcels/table_build_final.qmd | 216 +++++++ 5 files changed, 1084 insertions(+), 11 deletions(-) create mode 100644 backend/notebooks/parcels/investigation.qmd create mode 100644 backend/notebooks/parcels/local_data/population_2020.csv create mode 100644 backend/notebooks/parcels/table_build_final.qmd diff --git a/backend/notebooks/parcels/investigation.qmd b/backend/notebooks/parcels/investigation.qmd new file mode 100644 index 00000000..4782a17f --- /dev/null +++ b/backend/notebooks/parcels/investigation.qmd @@ -0,0 +1,56 @@ +--- +title: "Investigating the features of the parcels dataset" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#ill get to this later +```{python} +from pathlib import Path +import os +import sys +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +sys.path.insert(0, str(_project_root)) +print(os.getcwd()) + +from api.models import FilterSource +from sql_render import sql_filter_block +import json +from query.processed_db import DB + + + +sql_dir = Path("query/sql/parcels") +src = FilterSource(filter_table="parcels_info", filters={"TOWN": ["PUTNEY"]}, + join_key="OBJECTID", join_type="inner") +sql, params = sql_filter_block(sql_dir / "geo_query_parcels.sql", [src]) +print(sql) +``` + +#before we do anything, we must download all six datasets and explore +```{python} +GEOJSON_FULL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_standardized_parcels_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_INACTIVE = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_standardized_inactive_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_PARCEL_STATUS_BY_TOWN = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_VTPARCELS_poly_DataStatus_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +GEOJSON_PARCELS_TRANSFERS = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Cadastral_PTTR_point_WM_v1_view/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + + + +``` diff --git a/backend/notebooks/parcels/local_data/population_2020.csv b/backend/notebooks/parcels/local_data/population_2020.csv new file mode 100644 index 00000000..f6971cf6 --- /dev/null +++ b/backend/notebooks/parcels/local_data/population_2020.csv @@ -0,0 +1,256 @@ +_geoid,Town,County,year1791,year1800,year1810,year1820,year1830,year1840,year1850,year1860,year1870,year1880,year1890,year1900,year1910,year1920,year1930,year1940,year1950,year1960,year1970,year1980,year1990,year2000,year2010,year2020 +5000100325,Addison,Addison,402,734,1100,1210,1306,1232,1279,1000,911,847,900,851,796,743,684,576,628,645,717,889,1023,1393,1371,1365 +5001900475,Albany,Orleans,0,0,101,253,683,920,1052,1224,1151,1138,995,1028,920,840,810,748,704,560,528,705,782,840,941,976 +5001300860,Alburgh,Grand Isle,446,750,1106,1172,1239,1344,1568,1793,1716,1614,1390,1474,1311,1491,1609,1623,1402,1123,1271,1352,1362,1952,1998,2106 +5002701300,Andover,Windsor,275,1016,957,1000,975,877,725,670,588,564,418,372,284,294,258,213,185,215,239,350,373,496,467,568 +5000301450,Arlington,Bennington,992,1597,1468,1354,1207,1038,1084,1146,1636,1532,1352,1193,1307,1370,1441,1418,1463,1605,1934,2184,2299,2397,2317,2457 +5002501900,Athens,Windham,450,459,478,507,415,358,359,382,295,284,205,180,201,123,132,136,139,142,159,250,313,340,442,380 +5000902125,Averill,Essex,0,0,0,0,1,11,7,12,14,48,43,18,15,4,9,12,20,16,8,15,7,8,24,21 +5000902162,Avery's Gore,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +5001102500,Bakersfield,Franklin,13,222,812,945,1087,1258,1523,1451,1043,1248,1162,1158,1079,980,889,827,779,664,635,852,977,1215,1322,1273 +5002702575,Baltimore,Windsor,0,174,207,204,179,155,124,116,83,71,64,55,54,62,69,85,89,90,170,181,190,250,244,229 +5002702725,Barnard,Windsor,673,1236,1648,1691,1889,1774,1647,1487,1208,1191,918,840,737,653,584,486,439,435,569,790,872,958,947,992 +5000502875,Barnet,Caledonia,477,858,1301,1488,1764,2030,2521,1994,1945,1907,1897,1763,1707,1685,2604,1596,1425,1445,1342,1338,1415,1690,1708,1663 +5002303175,Barre City,Washington,0,0,0,0,0,0,0,0,0,0,0,8448,10734,10008,11307,10909,10922,10387,10209,9824,9482,9291,9052,8491 +5002303250,Barre Town,Washington,76,919,1669,1955,2012,2126,1845,1839,1882,2060,6812,3346,4194,3882,4280,4052,4145,4580,6509,7090,7411,7602,7924,7923 +5001903550,Barton,Orleans,0,128,447,372,729,892,987,1590,1911,2364,2217,2790,3346,3506,3469,3371,3298,3066,2874,2990,2967,2780,2810,2872 +5001504375,Belvidere,Lamoille,0,0,217,198,185,207,256,366,369,400,571,428,429,363,258,220,207,155,189,218,228,294,348,358 +5000304825,Bennington,Bennington,2350,2243,2524,2485,3419,3429,3923,4369,5760,6335,6391,8033,8698,9982,10628,11257,12411,13002,14586,15815,16451,15737,15764,15333 +5002105200,Benson,Rutland,658,1164,1561,1481,1493,1403,1305,1256,1244,1104,880,844,813,807,630,572,573,549,583,739,847,1039,1056,974 +5001105425,Berkshire,Franklin,0,172,918,831,1308,1818,1955,1890,1609,1596,1421,1326,1286,1299,1234,1156,1063,965,931,1116,1190,1388,1692,1547 +5002305650,Berlin,Washington,134,684,1067,1455,1664,1598,1507,1545,1474,1380,1514,1021,1079,959,992,1111,1158,1306,2050,2454,2561,2864,2887,2849 +5002705800,Bethel,Windsor,473,913,1041,1318,1667,1886,1730,1804,1817,1693,1448,1611,1843,1782,1650,1477,1534,1356,1347,1715,1866,1968,2030,1942 +5000906325,Bloomfield,Essex,0,27,9,132,150,179,244,320,455,627,827,564,496,382,417,326,291,212,196,188,253,261,221,217 +5000706550,Bolton,Chittenden,88,219,249,306,452,470,602,632,711,674,547,486,469,390,325,287,301,237,427,715,971,971,1182,1301 +5001707375,Bradford,Orange,654,1062,1302,1411,1507,1655,1723,1689,1492,1520,1429,1338,1372,1422,1234,1507,1551,1619,1627,2191,2522,2619,2797,2790 +5001707600,Braintree,Orange,221,531,850,1037,1209,1232,1228,1225,1066,1051,854,776,760,707,635,648,626,536,751,1065,1174,1194,1246,1207 +5002107750,Brandon,Rutland,637,1076,1375,1495,1940,2194,2835,3077,3571,3280,3310,2759,2712,2874,2891,2979,3304,3329,3697,4194,4223,3917,3966,4129 +5002507900,Brattleboro,Windham,1589,1867,1891,2017,2141,2624,3816,3855,4933,5880,6862,6640,7541,8332,9816,10983,11522,11734,12239,11886,12241,12005,12046,12184 +5002708275,Bridgewater,Windsor,293,781,1154,1125,1311,1363,1311,1292,1141,1084,1124,972,874,808,741,695,903,776,783,867,895,980,936,903 +5000108575,Bridport,Addison,450,1124,1520,1511,1774,1233,1393,1298,1176,1167,1018,956,848,745,703,665,663,653,809,997,1137,1235,1218,1225 +5000908725,Brighton,Essex,0,0,0,144,105,157,193,945,1536,1691,2020,2023,2013,2280,2002,1754,1671,1545,1365,1557,1562,1260,1222,1157 +5000109025,Bristol,Addison,211,665,1179,1051,1247,1480,1334,1355,1365,1579,1828,2061,2005,1952,1832,1939,1988,2159,2744,3293,3762,3788,3894,3782 +5001709325,Brookfield,Orange,419,988,1384,1411,1677,1789,1672,1672,1269,1239,996,996,1008,860,761,808,762,597,606,959,1089,1222,1292,1244 +5002509475,Brookline,Windham,0,472,431,395,376,325,285,243,203,206,162,171,137,105,101,104,132,127,180,310,403,467,530,540 +5001909850,Brownington,Orleans,0,65,236,265,412,486,613,761,901,854,799,748,760,741,697,689,673,599,522,708,705,885,988,1042 +5000910075,Brunswick,Essex,66,86,143,124,160,130,119,212,221,193,160,106,82,89,73,86,73,62,45,82,92,107,112,88 +5000710300,Buels Gore,Chittenden,0,0,0,0,0,0,18,35,29,24,21,20,16,14,4,2,3,0,10,9,2,12,30,29 +5000510450,Burke,Caledonia,0,108,460,541,866,997,1103,1138,1162,1252,1198,1184,1183,1041,1016,988,1042,922,1053,1385,1406,1571,1753,1651 +5000710675,Burlington,Chittenden,332,816,1690,2111,3526,4271,7585,7713,14387,11365,14590,18640,20468,22779,24789,27686,33155,35531,38633,37712,39127,38889,42417,44743 +5002311125,Cabot,Washington,122,349,686,1032,1304,1440,1356,1318,1279,1242,1074,1126,1116,1036,1107,974,826,763,663,958,1043,1213,1433,1443 +5002311350,Calais,Washington,45,443,841,1111,1539,1709,1410,1409,1309,1253,1062,1101,1042,860,812,818,778,684,749,1207,1521,1529,1607,1661 +5001511500,Cambridge,Lamoille,359,733,990,1435,1613,1790,1849,1748,1651,1750,1689,1606,1696,1593,1402,1383,1435,1295,1528,2019,2667,3186,3659,3839 +5000911800,Canaan,Essex,19,44,143,227,373,378,471,408,417,637,829,934,869,982,906,872,969,1094,949,1196,1121,1078,972,896 +5002111950,Castleton,Rutland,809,1039,1420,1541,1783,1769,3016,2852,3243,2605,2396,2089,1886,1919,1794,1601,1748,1902,2837,3637,4278,4367,4717,4458 +5002712250,Cavendish,Windsor,491,922,1295,1551,1498,1427,1576,1509,1823,1276,1172,1352,1208,1319,1418,1598,1374,1223,1264,1355,1323,1470,1367,1392 +5001913150,Charleston,Orleans,0,0,56,90,564,731,1009,1160,1278,1204,1058,1025,993,921,895,834,764,668,654,851,844,895,1023,1021 +5000713300,Charlotte,Chittenden,635,1231,1679,1625,1702,1620,1634,1589,1430,1342,1240,1254,1163,1160,1089,1082,1215,1271,1802,2561,3148,3569,3754,3912 +5001713525,Chelsea,Orange,239,896,1327,1462,1958,1959,1958,1757,1526,1462,1230,1070,1074,1087,1004,1013,1025,957,983,1091,1166,1250,1238,1233 +5002713675,Chester,Windsor,981,1878,2370,2495,2320,2305,2001,2126,2052,1901,1787,1775,1784,1633,1666,1740,1981,2318,2371,2791,2832,3044,3154,3005 +5002114350,Chittenden,Rutland,159,327,446,528,610,644,675,763,802,1092,730,621,563,472,341,379,424,460,646,927,1102,1182,1258,1237 +5002114500,Clarendon,Rutland,1480,1789,1797,1712,1585,1549,1477,1237,1173,1105,928,915,857,826,883,868,1102,1091,1537,2372,2835,2811,2571,2412 +5000714875,Colchester,Chittenden,137,347,659,1192,1489,1739,2575,3041,3911,4421,5143,5352,6450,6627,2638,3031,3897,4718,8776,12629,14731,16986,17067,17524 +5000915250,Concord,Essex,49,322,232,806,1031,1024,1153,1291,1276,1612,1425,1129,1080,1102,1043,923,979,956,896,1125,1093,1196,1235,1141 +5001715700,Corinth,Orange,578,1410,1876,1907,1953,1970,1806,1627,1470,1627,1027,978,1005,936,817,822,786,775,683,904,1244,1461,1367,1455 +5000116000,Cornwall,Addison,825,1163,1279,1120,1264,1164,1155,977,969,1070,927,850,789,782,640,670,728,756,900,993,1101,1136,1185,1207 +5001916150,Coventry,Orleans,0,7,178,282,728,786,857,914,914,911,879,728,616,668,610,549,497,458,492,674,806,1014,1086,1100 +5001916300,Craftsbury,Orleans,0,229,566,605,982,1151,1223,1413,1330,1381,1271,1251,1119,1042,976,875,709,674,632,844,994,1136,1206,1343 +5002116825,Danby,Rutland,1206,1487,1730,1607,1362,1479,1535,1419,1319,1202,1084,964,1001,1007,1070,1112,990,891,910,992,1193,1292,1311,1284 +5000517125,Danville,Caledonia,574,1514,2240,2300,2631,2633,2577,2544,2216,2003,1784,1628,1564,1494,1600,1472,1312,1368,1405,1705,1917,2211,2196,2335 +5001917350,Derby,Orleans,0,178,714,925,1469,1681,1759,1908,2039,1957,2900,2361,2330,2201,2135,2118,2245,2506,3252,4222,4479,4604,4621,4579 +5000317725,Dorset,Bennington,957,1286,1294,1359,1507,1426,1700,2089,2195,2005,1696,1477,1472,1226,1120,1128,1150,1150,1293,1648,1918,2036,2031,2133 +5002517875,Dover,Windham,0,0,859,829,1216,729,709,650,635,621,524,503,377,385,278,244,252,370,555,666,994,1410,1124,1798 +5002518325,Dummerston,Windham,1490,1692,1704,1658,1592,1263,1645,1021,916,816,860,728,643,570,604,615,790,872,1295,1574,1863,1915,1864,1865 +5002318550,Duxbury,Washington,39,153,326,440,652,820,895,1000,893,894,912,778,648,631,553,854,489,546,621,877,976,1289,1337,1413 +5000921250,East Haven,Essex,0,0,0,0,33,79,94,136,191,225,236,171,194,148,99,92,85,164,197,280,269,301,290,270 +5002321925,East Montpelier,Washington,0,0,0,0,1156,1092,1447,1328,1130,972,953,1061,985,918,965,1025,1128,1200,1597,2205,2239,2578,2576,2598 +5001523500,Eden,Lamoille,0,29,224,201,461,703,668,919,958,934,851,738,751,619,568,498,496,430,513,612,840,1152,1323,1338 +5001523725,Elmore,Lamoille,12,45,157,164,442,476,504,602,637,682,593,550,563,468,382,300,312,237,292,421,573,849,855,886 +5001124050,Enosburgh,Franklin,0,143,704,932,1560,2022,2009,2066,2077,2213,2289,2054,2212,2231,2093,2082,2101,1966,1918,2070,2535,2788,2781,2810 +5000724175,Essex,Chittenden,354,729,957,1089,1664,1824,2052,1905,2022,2104,2013,2203,2714,2449,2876,3059,3931,7090,10951,14392,16498,18626,19587,22094 +5002125375,Fair Haven,Rutland,545,411,645,714,675,633,902,1378,2008,2211,2791,2999,3095,2540,2614,2245,2286,2378,2777,2819,2887,2928,2734,2736 +5001124925,Fairfax,Franklin,254,786,1301,1359,1729,1918,2111,1987,1956,1820,1523,1338,1318,1244,1249,1229,1129,1244,1366,1805,2486,3765,4285,5014 +5001125225,Fairfield,Franklin,126,901,1618,1573,2270,2448,2591,2497,2391,2172,1825,1830,1778,1632,1541,1444,1428,1225,1285,1493,1680,1800,1891,2044 +5001725675,Fairlee,Orange,463,386,983,1143,656,644,575,549,416,469,398,438,438,459,456,535,571,569,604,770,883,967,977,988 +5002325825,Fayston,Washington,0,18,149,263,458,635,684,800,694,638,533,466,452,424,318,284,172,158,292,657,846,1141,1353,1364 +5000925975,Ferdinand,Essex,0,0,0,0,0,0,0,34,33,40,73,41,213,106,18,17,10,16,14,12,23,33,32,16 +5000126300,Ferrisburgh,Addison,481,956,1647,1581,1822,1755,2075,1738,1768,1684,1601,1619,1433,1338,1285,1347,1387,1426,1875,2117,2317,2657,2775,2646 +5001126500,Fletcher,Franklin,47,200,382,497,793,1014,1084,916,865,868,793,750,737,656,667,549,485,399,456,626,941,1179,1277,1346 +5001127100,Franklin,Franklin,46,280,714,831,1129,1410,1646,1781,1612,1439,1300,1145,1108,994,1001,1021,878,796,821,1006,1068,1268,1405,1363 +5001127700,Georgia,Franklin,340,1068,1760,1703,1897,2106,2686,1547,1603,1504,1282,1280,1090,1075,1090,1008,1055,1079,1711,2818,3753,4375,4515,4845 +5000327962,Glastenbury,Bennington,34,48,76,48,59,53,52,47,119,241,181,48,29,40,7,4,1,0,0,3,7,16,8,9 +5001928075,Glover,Orleans,0,35,368,549,902,1119,1137,1244,1178,1055,970,891,932,826,860,788,727,683,649,843,820,966,1122,1114 +5000128600,Goshen,Addison,0,4,86,290,555,621,486,394,330,326,311,286,212,131,84,83,94,76,120,163,226,227,164,172 +5002528900,Grafton,Windham,0,1149,1365,1482,1439,1326,1241,1154,1008,929,817,804,729,476,453,393,422,426,465,604,602,649,679,645 +5000929125,Granby,Essex,0,69,30,49,97,105,127,132,174,194,361,182,95,70,69,76,74,56,52,70,85,86,88,81 +5001329275,Grand Isle,Grand Isle,0,0,338,698,642,724,666,708,682,749,793,851,839,808,857,791,735,624,809,1238,1642,1955,2067,2086 +5000129575,Granville,Addison,101,185,326,328,403,545,603,720,726,830,637,544,464,393,280,247,213,215,255,288,309,303,298,301 +5001930175,Greensboro,Orleans,19,280,566,625,784,883,1008,1065,1027,1061,918,874,931,906,831,768,737,600,593,677,717,770,762,811 +5000530550,Groton,Caledonia,45,248,449,595,836,928,893,939,811,1014,1040,1059,915,902,803,764,712,631,666,667,862,876,1022,984 +5000930775,Guildhall,Essex,158,296,120,529,481,470,501,552,483,558,511,455,445,376,351,313,270,248,169,202,270,268,261,262 +5002530925,Guilford,Windham,2422,2257,1872,1862,1760,1523,1389,1291,1277,1096,870,782,769,684,663,686,796,823,1108,1532,1941,2046,2121,2120 +5002531150,Halifax,Windham,1209,1600,1758,1567,1552,1399,1133,1125,1029,852,702,662,635,504,390,353,343,268,295,488,588,782,728,771 +5000131525,Hancock,Addison,56,149,311,548,472,465,430,448,430,382,283,253,287,300,303,371,391,323,283,334,340,382,323,359 +5000531825,Hardwick,Caledonia,3,260,735,867,1216,1354,1402,1369,1519,1484,1547,2466,3201,2641,2720,2605,2629,2349,2466,2613,2964,3174,3010,2920 +5002732275,Hartford,Windsor,988,1494,1831,2010,2044,2341,2159,2396,2480,2954,3740,3817,4179,4739,4888,4978,5827,6355,6477,7963,9404,10367,9952,10686 +5002732425,Hartland,Windsor,1552,1960,2352,2553,2503,2194,2063,1748,1710,1598,1393,1340,1316,1212,1266,1306,1559,1592,1806,2396,2988,3223,3393,3446 +5001133025,Highgate,Franklin,103,437,1374,1250,2038,2292,2653,2526,2260,2088,1853,1980,1758,1528,1574,1647,1681,1608,1936,2493,3020,3397,3535,3472 +5000733475,Hinesburg,Chittenden,454,933,1238,1332,1669,1682,1834,1702,1573,1330,1205,1216,1042,964,1019,1000,1120,1180,1775,2690,3780,4340,4396,4698 +5001933775,Holland,Orleans,0,0,126,100,422,605,669,748,881,913,878,838,722,714,560,533,406,376,383,473,423,588,629,632 +5002134450,Hubbardton,Rutland,410,541,734,810,865,719,701,606,606,533,506,488,455,328,307,346,332,238,228,490,576,752,706,735 +5000734600,Huntington,Chittenden,167,405,514,732,929,914,835,861,864,808,723,728,760,651,621,549,601,518,748,1161,1609,1861,1938,1934 +5001535050,Hyde Park,Lamoille,43,110,261,373,823,1080,1107,1409,1624,1716,1633,1472,1453,1323,1165,1178,1291,1219,1347,2021,2344,2847,2954,3020 +5002135425,Ira,Rutland,312,473,519,498,442,451,400,422,413,479,421,350,286,295,287,248,232,220,284,354,426,455,432,368 +5001935575,Irasburg,Orleans,0,15,392,432,860,971,1034,1131,1085,1054,999,939,983,999,924,852,711,711,775,870,907,1077,1163,1233 +5001335875,Isle La Motte,Grand Isle,47,135,623,312,459,435,476,564,497,505,551,608,510,385,352,335,295,238,262,393,408,488,471,488 +5002536175,Jamaica,Windham,263,582,996,1343,1523,1586,1606,1541,1223,1252,1074,800,716,686,570,567,597,496,590,681,754,946,1035,1005 +5001936325,Jay,Orleans,0,0,28,52,196,306,371,474,553,696,641,530,513,368,274,230,243,197,182,302,381,426,521,551 +5000736700,Jericho,Chittenden,381,728,1185,1219,1654,1685,1837,1669,1757,1687,1461,1373,1307,1138,1091,1077,1135,1425,2343,3575,4302,5015,5009,5104 +5001537075,Johnson,Lamoille,93,265,494,778,1079,1410,1381,1526,1558,1495,1462,1391,1526,1478,1378,1420,1527,1478,1927,2581,3156,3274,3446,3491 +5002137685,Killington,Rutland,32,90,116,154,432,698,578,525,462,450,451,402,409,336,298,266,283,266,558,891,738,1095,811,1407 +5000537900,Kirby,Caledonia,0,20,311,312,401,520,509,473,417,398,355,350,297,324,311,256,257,235,224,282,347,456,493,575 +5000339025,Landgrove,Bennington,31,147,299,341,385,344,357,320,302,246,220,225,160,143,104,64,80,49,104,121,134,144,158,177 +5000139325,Leicester,Addison,344,522,609,442,638,603,596,737,630,634,562,509,479,436,468,518,511,551,583,803,871,974,1100,990 +5000939700,Lemington,Essex,0,31,52,139,183,124,187,207,191,222,227,204,138,145,133,131,105,112,120,108,102,107,104,87 +5000939775,Lewis,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 +5000140075,Lincoln,Addison,0,97,225,278,639,770,1057,1070,1174,1368,1255,1152,980,841,800,745,577,481,599,870,974,1214,1271,1323 +5002540225,Londonderry,Windham,362,530,637,958,1302,1216,1274,1367,1252,1154,1010,961,962,911,799,859,953,898,1037,1510,1506,1709,1769,1919 +5001940525,Lowell,Orleans,0,0,0,144,314,431,637,813,942,1067,1178,983,1086,1005,725,615,643,617,515,573,594,738,879,887 +5002741275,Ludlow,Windsor,179,410,877,1140,1227,1363,1619,1568,1827,2005,1768,2042,2215,2421,2305,2458,2428,2386,2463,2414,2302,2449,1963,2172 +5000941425,Lunenburg,Essex,119,393,132,856,1054,1130,1123,1034,999,1038,1019,968,880,1048,1400,1374,1299,1237,1061,1138,1176,1328,1302,1246 +5000541725,Lyndon,Caledonia,59,542,1090,1296,1822,1753,1752,1695,2179,2434,2619,2956,3204,3558,3285,3144,3360,3425,3705,4924,5371,5448,5981,5491 +5000942475,Maidstone,Essex,125,152,714,165,236,271,237,259,254,286,198,206,175,171,123,96,81,78,94,100,131,105,208,211 +5000342850,Manchester,Bennington,1278,1397,1502,1508,1525,1594,1762,1688,1897,1928,1907,1955,2044,2057,2004,2139,2425,2470,2919,3261,3622,4180,4391,4484 +5002543375,Marlboro,Windham,629,1087,1245,1296,1218,1027,896,741,665,563,495,448,442,300,255,225,311,347,592,695,924,978,1078,1722 +5002343600,Marshfield,Washington,0,172,513,710,1271,1156,1102,1160,1072,1102,1121,1032,1011,898,872,901,830,891,1033,1267,1331,1496,1588,1583 +5002144125,Mendon,Rutland,34,39,111,174,432,545,504,633,612,629,570,392,321,264,251,313,334,461,743,1056,1049,1028,1059,1149 +5000144350,Middlebury,Addison,398,1263,2138,2635,3468,3162,3517,2879,3086,2995,2793,3045,2848,2914,2968,3175,4778,5305,6532,7574,8034,8183,8496,9152 +5002344500,Middlesex,Washington,60,262,401,726,1156,1270,1365,1254,1171,1087,889,883,858,762,751,817,887,770,857,1235,1514,1729,1731,1779 +5002144800,Middletown Springs,Rutland,699,1066,1207,1039,919,1057,875,712,777,823,786,746,716,567,583,493,496,381,426,603,686,823,745,794 +5000745250,Milton,Chittenden,283,786,1546,1746,2100,2134,2451,1983,2062,2006,1885,1804,1648,1523,1663,1750,1874,2022,4495,6829,8404,9479,10352,10723 +5000145550,Monkton,Addison,449,880,1248,1152,1384,1310,1246,1123,1006,1025,847,912,724,671,683,575,520,551,765,1201,1482,1759,1980,2079 +5001145850,Montgomery,Franklin,0,36,237,293,460,564,1001,1262,1423,1642,1734,1876,1721,1658,1386,1208,1091,876,651,681,823,992,1201,1184 +5002346000,Montpelier,Washington,118,890,1877,2308,1193,3725,2310,2411,3023,3219,4160,6266,7856,7125,7837,8006,8599,8782,8609,8241,8247,8035,7855,8074 +5002346225,Moretown,Washington,24,191,405,593,816,1128,1335,1410,1263,1180,956,902,686,930,889,675,883,788,904,1221,1415,1653,1658,1753 +5001946450,Morgan,Orleans,0,0,135,116,331,422,486,548,614,711,520,510,463,368,363,335,296,260,286,460,497,669,749,638 +5001546675,Morristown,Lamoille,10,144,550,726,1315,1502,1441,1751,1897,2099,2411,2583,2652,2813,2939,3130,3225,3347,4052,4448,4733,5139,5227,5434 +5002147200,Mount Holly,Rutland,0,668,922,1157,1318,1356,1534,1522,1582,1390,1214,999,871,856,727,656,567,517,687,938,1093,1241,1237,1385 +5002147425,Mount Tabor,Rutland,165,153,209,222,210,226,308,358,301,495,436,494,289,168,173,213,186,165,184,211,214,203,255,210 +5000148700,New Haven,Addison,717,1135,1688,1566,1834,1503,1663,1419,1355,1355,1224,1107,1161,1001,964,881,932,922,1039,1217,1375,1666,1727,1683 +5000547725,Newark,Caledonia,0,8,88,154,257,360,434,567,593,679,536,500,415,364,301,242,192,151,144,280,354,470,581,584 +5001748175,Newbury,Orange,873,1304,1363,1623,2252,2578,2984,2549,2241,2316,2080,2125,2036,1908,1744,1723,1667,1452,1440,1699,1985,1955,2216,2293 +5002548400,Newfane,Windham,860,1000,1276,1506,1441,1403,1304,1192,1115,1031,962,902,820,710,662,672,708,714,900,1129,1555,1680,1726,1645 +5001948850,Newport City,Orleans,0,0,0,0,0,0,0,0,0,0,0,2787,3657,4976,5094,4902,5217,5019,4664,4756,4434,5005,4589,4455 +5001948925,Newport Town,Orleans,0,50,28,52,284,591,748,1197,2050,2426,3047,1239,1236,1187,1193,1064,966,1010,1125,1319,1367,1511,1594,1526 +5001350650,North Hero,Grand Isle,125,324,552,503,638,716,730,594,601,637,550,712,496,494,485,442,407,328,364,442,502,810,803,939 +5002350275,Northfield,Washington,40,204,426,690,1412,2013,2922,4329,3410,2836,2628,2855,3226,3095,3436,3601,4314,4511,4870,5435,5610,5791,6207,5918 +5000952750,Norton,Essex,0,0,0,0,0,0,0,32,303,239,960,692,479,336,339,314,279,241,207,184,169,214,169,153 +5002752900,Norwich,Windsor,1158,1482,1812,1985,2316,2218,1978,1759,1639,1471,1304,1303,1252,1092,1371,1418,1532,1790,1966,2398,3093,3544,3414,3612 +5001753425,Orange,Orange,0,348,686,751,1016,984,1007,936,733,731,589,598,644,485,508,482,410,430,540,752,915,965,1072,1048 +5000153725,Orwell,Addison,778,1376,324,1730,1598,1688,1470,1341,1192,1351,1265,1150,1065,942,835,876,902,826,851,901,1114,1185,1250,1239 +5000153950,Panton,Addison,220,363,520,548,605,670,569,511,390,419,382,409,345,321,306,312,332,352,416,537,606,682,677,646 +5002154250,Pawlet,Rutland,1458,1938,2233,2155,1965,1748,1843,1539,1505,1696,1745,1731,1969,1413,1476,1192,1156,1112,1184,1244,1314,1394,1477,1424 +5000554400,Peacham,Caledonia,365,873,1301,1294,1351,1443,1377,1247,1141,1041,892,794,777,657,620,543,501,433,446,531,627,665,732,715 +5000355000,Peru,Bennington,239,314,445,578,0,0,567,543,500,556,445,373,242,216,156,142,197,194,243,312,324,416,375,531 +5002155450,Pittsfield,Rutland,49,164,338,453,505,615,512,493,482,555,468,435,402,343,256,259,225,254,249,396,389,427,546,504 +5002155600,Pittsford,Rutland,850,1413,1936,1916,2005,1927,2026,1839,2127,1982,1775,1866,2479,2098,2332,2093,2076,2225,2306,2590,2919,3140,2991,2862 +5002355825,Plainfield,Washington,0,256,543,660,874,880,808,822,726,729,745,716,785,781,766,832,945,966,1399,1249,1302,1286,1243,1236 +5002756050,Plymouth,Windsor,106,497,834,1112,1237,1417,1226,1252,1285,1075,765,646,482,449,331,432,348,308,283,405,440,555,619,641 +5002756350,Pomfret,Windsor,710,1105,1473,1835,1867,1774,1546,1375,1251,1139,865,777,703,732,728,686,586,600,620,856,874,997,904,916 +5002156875,Poultney,Rutland,1120,1694,1904,1956,1909,1878,2329,2278,2836,2717,3031,3108,3644,2868,3215,2781,2936,3009,3217,3196,3498,3633,3432,3020 +5000357025,Pownal,Bennington,1732,1692,1655,1812,1835,1615,1742,1731,1705,2019,1919,1976,1599,1396,1425,1402,1453,1509,2441,3269,3485,3560,3527,3258 +5002157250,Proctor,Rutland,0,0,0,0,0,0,0,0,0,0,1758,2136,2871,2789,2596,2292,1917,2102,2095,1998,1979,1877,1741,1763 +5002557700,Putney,Windham,1848,1574,1607,1547,1810,1585,1425,1163,1167,1124,1075,969,788,761,835,904,1019,1177,1727,1850,2352,2634,2702,2617 +5001758075,Randolph,Orange,893,1841,2255,2487,2743,2678,2666,2502,2829,2910,3232,3141,3191,3010,3166,3278,3499,3414,3882,4689,4764,4853,4778,4774 +5002758375,Reading,Windsor,747,1120,1565,1803,1409,1353,1171,1159,1012,953,749,649,530,483,474,437,470,472,564,647,614,707,666,687 +5000358600,Readsboro,Bennington,63,234,410,530,662,767,657,930,828,743,910,1139,1252,1173,1043,913,847,783,638,638,762,809,763,702 +5001159125,Richford,Franklin,0,113,442,440,704,914,1074,1338,1481,1818,2196,2421,2907,2842,2544,2646,2643,2316,2116,2206,2178,2321,2308,2346 +5000759275,Richmond,Chittenden,0,718,935,1014,1109,1064,1453,1400,1309,1264,1115,1057,1419,1447,1315,1225,1278,1303,2249,3159,3729,4090,4081,4167 +5000159650,Ripton,Addison,0,0,15,42,278,357,567,570,617,672,568,525,421,237,194,231,207,131,187,327,444,556,588,739 +5002760100,Rochester,Windsor,215,524,911,1148,1392,1396,1493,1607,1444,1362,1257,1250,1317,1397,1285,1129,937,879,884,1054,1181,1171,1139,1099 +5002560250,Rockingham,Windham,1235,1684,1954,2155,2272,2330,2837,2904,2854,3797,4579,5809,6207,6231,5302,5737,5499,5704,5501,5538,5484,5309,5282,4832 +5002360625,Roxbury,Washington,14,113,361,512,737,784,967,1060,916,938,768,712,618,609,594,554,465,364,354,452,575,576,691,678 +5002760850,Royalton,Windsor,748,1501,1758,1816,1893,1917,1850,1739,1679,1558,1433,1427,1452,1469,1491,1291,1331,1388,1399,2100,2389,2603,2773,2750 +5000361000,Rupert,Bennington,1034,1628,1630,1532,1318,1084,1101,1103,1017,957,861,863,825,674,691,678,713,603,582,605,654,704,714,698 +5002161225,Rutland City,Rutland,0,0,0,0,0,0,0,0,0,0,0,11499,13546,14954,17318,17082,17659,18325,19293,18436,18230,17292,16495,15807 +5002161300,Rutland Town,Rutland,1417,2125,2379,2369,2753,2708,3715,7577,9834,12149,11760,1109,1311,1270,1387,1350,1416,1542,2248,3300,3781,4038,4054,3924 +5000561525,Ryegate,Caledonia,187,415,812,994,1119,1223,1606,1098,935,1046,1126,995,1194,1188,1216,1105,996,894,830,1000,1058,1150,1174,1165 +5000162575,Salisbury,Addison,444,644,709,721,907,942,1027,853,902,775,740,692,693,635,632,581,573,575,649,881,1024,1090,1136,1221 +5000362875,Sandgate,Bennington,733,1020,1187,1185,933,776,850,805,705,681,587,482,401,283,189,187,158,93,127,234,278,353,405,387 +5000363175,Searsburg,Bennington,9,40,120,0,0,0,201,262,235,232,173,161,142,133,103,135,84,73,84,72,85,96,109,126 +5000363550,Shaftsbury,Bennington,1990,1896,1973,2022,2143,1885,1896,1936,2027,1887,1652,1857,1650,1534,1631,1577,1673,1939,2411,3001,3368,3767,3590,3598 +5002763775,Sharon,Windsor,569,1158,1363,1431,1459,1371,1240,1111,1013,1012,737,709,565,545,559,530,470,485,541,828,1211,1411,1502,1560 +5000564075,Sheffield,Caledonia,0,179,388,581,720,821,797,836,811,884,760,724,691,594,543,465,451,342,307,435,541,727,703,682 +5000764300,Shelburne,Chittenden,387,723,987,1168,1123,1098,1257,1178,1190,1095,1300,1292,1097,997,1006,1010,1365,1805,3728,5000,5871,6944,7144,7717 +5001164600,Sheldon,Franklin,0,408,883,927,2158,1734,1814,1655,1697,1529,1365,1341,1246,1473,1563,1471,1352,1281,1481,1618,1748,1990,2190,2136 +5000165050,Shoreham,Addison,701,1447,2033,1881,2137,1674,1601,1382,1225,1354,1240,1193,1098,925,948,865,829,786,790,972,1115,1222,1265,1260 +5002165275,Shrewsbury,Rutland,382,748,990,1151,1289,1218,1268,1175,1145,1235,974,935,751,620,540,537,464,445,570,866,1107,1108,1056,1096 +5002565762,Somerset,Windham,111,130,199,173,245,262,321,105,80,67,61,67,27,59,20,5,8,4,0,2,2,5,3,6 +5000766175,South Burlington,Chittenden,0,0,0,0,0,0,0,0,791,664,845,971,927,938,1023,1736,3279,6903,10032,10679,12809,15814,17904,20292 +5001367000,South Hero,Grand Isle,537,678,826,842,717,664,705,617,586,620,559,817,605,605,641,611,567,614,868,1188,1404,1696,1631,1674 +5002769550,Springfield,Windsor,1097,2032,2556,2702,2749,2625,2762,2968,2937,3144,2881,3432,4784,7202,6965,7720,9190,9934,10063,10190,9579,9078,9373,9062 +5001161675,St. Albans City,Franklin,0,0,0,0,0,0,0,0,0,0,0,6237,6381,7588,8020,8037,8552,8806,8082,7308,7339,7650,6918,6877 +5001161750,St. Albans Town,Franklin,256,901,1609,1636,2395,2702,3567,3637,7014,7193,7771,1715,1617,1583,1691,1733,1908,2303,3270,3555,4606,5086,5999,6988 +5000762050,St. George,Chittenden,57,65,68,120,135,121,127,121,111,93,106,90,109,100,84,87,117,108,477,677,705,698,674,794 +5000562200,St. Johnsbury,Caledonia,143,651,1334,1404,1592,1887,2758,3469,4665,5800,6567,7010,8098,8708,9696,9095,9292,8869,8409,7938,7608,7571,7603,7364 +5000369775,Stamford,Bennington,272,383,378,490,563,662,833,759,633,726,645,677,510,374,370,418,514,600,752,773,773,813,824,861 +5000569925,Stannard,Caledonia,0,0,0,0,0,0,0,0,228,252,239,222,206,173,154,140,116,113,88,142,148,185,216,208 +5000170075,Starksboro,Addison,0,359,726,914,1342,1263,1400,1437,1361,1249,1070,902,835,806,687,744,576,502,668,1336,1511,1898,1777,1756 +5002770375,Stockbridge,Windsor,100,432,700,964,1333,1418,1327,1264,1269,1124,894,822,737,618,460,490,427,392,389,508,618,674,736,718 +5001570525,Stowe,Lamoille,0,316,650,957,1570,1371,1771,2046,2049,1896,1886,1926,1991,1800,1654,1741,1720,1901,2388,2991,3433,4339,4314,5223 +5001770675,Strafford,Orange,844,1642,1805,1921,1935,1762,1640,1506,1290,1181,932,1000,776,601,615,598,680,548,536,731,902,1045,1098,1094 +5002570750,Stratton,Windham,95,271,265,272,312,341,286,365,294,302,222,271,86,90,55,117,54,24,104,122,121,136,216,440 +5002171050,Sudbury,Rutland,258,521,754,809,812,796,794,696,601,562,502,474,415,417,361,321,263,249,253,380,516,583,560,545 +5000371425,Sunderland,Bennington,414,557,575,496,463,438,479,667,553,655,633,518,494,409,375,442,493,566,601,768,872,850,956,1056 +5000571575,Sutton,Caledonia,0,0,433,697,1005,1068,1001,987,920,838,746,694,711,659,596,561,528,476,438,667,854,1001,1029,913 +5001171725,Swanton,Franklin,74,858,1657,607,2158,2312,2824,2678,2866,3079,3231,3745,3628,3342,3433,3543,3740,3946,4622,5141,5636,6203,6427,6701 +5001772400,Thetford,Orange,862,1478,1735,1915,2113,2065,2016,1876,1613,1529,1287,1249,1182,1089,1052,1043,1046,1049,1422,2188,2438,2617,2588,2775 +5002172925,Tinmouth,Rutland,935,973,1001,1069,1049,780,717,620,569,532,435,404,410,349,340,346,248,228,268,406,455,567,613,553 +5001773075,Topsham,Orange,126,344,814,1020,1384,1745,1668,1662,1418,1365,1187,1117,918,825,720,707,733,638,686,767,944,1142,1173,1199 +5002573300,Townshend,Windham,678,1083,1115,1406,1386,1345,1354,1376,1171,1099,865,833,817,786,633,694,584,643,668,849,1019,1149,1232,1291 +5001973525,Troy,Orleans,0,0,231,227,608,855,1008,1246,1355,1522,1673,1467,1685,1869,1098,1869,1786,1613,1457,1498,1609,1564,1662,1722 +5001773675,Tunbridge,Orange,487,1324,1640,2003,1920,1811,1786,1546,1405,1262,1011,885,918,907,903,882,774,743,791,925,1154,1309,1284,1337 +5000773975,Underhill,Chittenden,59,212,490,757,1052,1441,1599,1637,1655,1439,1301,1140,1004,896,781,760,698,730,1198,2172,2799,2980,3016,3129 +5000174650,Vergennes,Addison,201,516,835,817,999,1017,1378,1286,1670,1782,1773,1753,1463,1609,1705,1662,1736,1921,2242,2273,2578,2741,2588,2553 +5002574800,Vernon,Windham,482,480,521,627,681,705,821,726,764,652,567,578,606,656,609,559,712,865,1024,1175,1850,2141,2206,2192 +5001774950,Vershire,Orange,439,1031,1311,1290,1260,1188,1071,1053,1140,1875,954,641,446,410,368,367,284,236,299,442,560,629,730,672 +5000975175,Victory,Essex,0,0,0,6,53,140,168,212,263,321,554,321,206,125,80,104,49,46,42,56,50,97,62,70 +5002375325,Waitsfield,Washington,61,473,647,935,968,1048,1021,1005,948,938,815,760,709,682,723,706,661,658,837,1300,1422,1659,1719,1844 +5000575700,Walden,Caledonia,11,153,455,580,827,913,910,1001,992,931,810,764,739,674,664,547,481,427,442,575,703,782,935,956 +5002175925,Wallingford,Rutland,538,912,1386,1570,1740,1608,1688,1747,2023,1846,1733,1575,1719,1581,1564,1450,1482,1439,1676,1893,2184,2274,2079,2129 +5000176075,Waltham,Addison,247,247,244,264,330,283,270,263,249,248,255,264,202,204,175,184,193,186,265,394,454,479,486,446 +5002576225,Wardsboro,Windham,753,1484,2053,1010,1148,1102,1125,1005,866,766,704,637,559,380,355,401,377,322,391,505,654,854,900,869 +5000976337,Warner's Grant,Essex,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +5002376525,Warren,Washington,0,58,229,320,766,943,962,1041,1008,951,866,826,825,654,486,450,498,469,588,956,1172,1681,1705,1977 +5000976562,Warren's Gore,Washington,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,10,4,2 +5001776750,Washington,Orange,72,507,1040,1160,1374,1369,1348,1249,1113,922,820,820,762,660,697,730,650,565,667,855,937,1047,1039,1032 +5002376975,Waterbury,Washington,93,644,966,1269,1650,1992,2352,2198,2633,2297,2232,2810,3273,3542,4045,4118,4276,4303,4614,4465,4589,4915,5064,5331 +5000577125,Waterford,Caledonia,63,565,1289,1247,1358,1388,1412,1171,872,815,734,705,629,574,712,498,468,460,586,882,1190,1104,1280,1268 +5001577425,Waterville,Lamoille,0,0,0,0,488,610,753,747,574,547,577,529,485,469,307,386,409,332,397,470,532,697,673,686 +5002777500,Weathersfield,Windsor,1146,1944,2115,2301,2213,2081,1851,1766,1557,1354,1174,1089,1092,1087,1156,1075,1288,1254,2040,2534,2674,2788,2825,2842 +5002177950,Wells,Rutland,620,978,1040,986,880,740,804,642,483,665,621,606,569,521,515,420,487,419,560,815,902,1121,1150,1214 +5001779975,West Fairlee,Orange,0,391,983,1143,841,824,696,830,833,1038,561,531,446,387,405,428,363,333,337,427,633,726,652,621 +5002180875,West Haven,Rutland,0,430,679,684,724,774,718,580,713,492,412,355,363,343,280,302,232,220,240,253,273,278,264,239 +5002182300,West Rutland,Rutland,0,0,0,0,0,0,0,0,0,0,3680,2914,3427,3391,3421,2922,2487,2302,2381,2351,2448,2535,2326,2214 +5002783050,West Windsor,Windsor,0,0,0,0,0,0,1002,924,708,690,570,513,569,514,512,494,504,539,571,763,923,1067,1099,1344 +5001980200,Westfield,Orleans,0,16,149,225,353,370,502,618,721,698,763,646,613,490,448,354,358,347,375,418,422,503,536,534 +5000780350,Westford,Chittenden,63,648,1107,1218,1290,1352,1458,1231,1237,1133,1033,888,854,706,698,698,685,680,991,1413,1740,2086,2029,2062 +5002581400,Westminster,Windham,1599,1942,1925,1974,1737,1656,1721,1300,1238,1377,1265,1295,1327,1289,1324,1403,1400,1602,1875,2493,3026,3210,3178,3016 +5001981700,Westmore,Orleans,0,0,0,126,132,122,152,324,412,480,395,390,331,287,224,224,210,179,195,257,305,306,350,357 +5002782000,Weston,Windsor,0,0,629,890,972,1032,950,932,931,987,864,756,632,436,411,457,468,442,507,627,488,630,566,623 +5000183275,Weybridge,Addison,174,502,750,714,850,797,804,667,627,608,543,518,494,438,418,385,402,430,618,667,749,824,833,814 +5000583500,Wheelock,Caledonia,33,568,963,906,834,881,855,832,822,829,596,567,500,526,412,293,287,246,238,444,481,621,811,759 +5000183800,Whiting,Addison,249,404,565,609,653,659,629,542,430,455,355,361,348,302,358,312,282,304,359,379,407,380,419,405 +5002583950,Whitingham,Windham,442,868,1248,1397,1477,1391,1380,1372,1263,1240,1191,1042,969,811,734,789,816,838,1011,1043,1177,1298,1357,1344 +5001784175,Williamstown,Orange,146,839,1353,1481,1487,1620,1452,1377,1236,1038,1188,1610,1726,1526,1609,1477,1600,1553,1822,2284,2839,3225,3389,3515 +5000784475,Williston,Chittenden,469,836,1195,1538,1608,1554,1664,1472,1441,1342,1161,1176,1000,929,961,1021,1182,1484,3187,3843,4887,7650,8698,10103 +5002584700,Wilmington,Windham,645,1011,1193,1369,1367,1296,1372,1424,1246,1130,1106,1221,1229,1483,1171,1221,1169,1245,1586,1808,1968,2225,1876,2255 +5002584850,Windham,Windham,0,427,782,931,847,757,763,680,544,536,379,356,345,261,254,183,146,135,174,223,251,328,419,449 +5002784925,Windsor,Windsor,1542,2211,2757,2956,3134,2744,1928,1669,1699,2175,1846,2119,2407,3687,4359,4155,4402,4468,4158,4084,3714,3756,3553,3559 +5000385075,Winhall,Bennington,155,202,429,428,571,576,762,741,642,722,523,449,366,336,229,212,255,245,281,327,482,702,769,1182 +5000785150,Winooski,Chittenden,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5308,6036,6734,7420,7309,6318,6649,6561,7267,7997 +5001585375,Wolcott,Lamoille,32,47,124,123,492,910,909,1161,1132,1166,1158,1066,1049,932,831,772,766,633,676,986,1229,1456,1676,1670 +5002385525,Woodbury,Washington,23,23,254,432,824,1092,1070,999,902,856,810,862,824,686,529,463,449,317,399,573,766,809,906,928 +5000385675,Woodford,Bennington,60,138,254,212,395,487,423,379,371,487,353,279,187,231,137,170,198,207,286,314,331,414,424,355 +5002785975,Woodstock,Windsor,1597,2132,2672,2610,3044,3315,3041,3062,2910,2815,2545,2557,2545,2370,2469,2512,2613,2786,2608,3214,3212,3232,3048,3005 +5002386125,Worcester,Washington,25,25,41,44,432,567,702,684,775,802,725,636,694,463,471,396,445,417,505,727,906,902,998,964 diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index bb40f68a..fe0b2e04 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -259,14 +259,14 @@ not_stateless = [] ADDISON = ["ADDISON", "BRIDPORT", "BRISTOL", "CORNWALL", "FERRISBURGH", "GOSHEN", "GRANVILLE", "HANCOCK", "LEICESTER", "LINCOLN", "MIDDLEBURY", "MONKTON", "NEW HAVEN", "ORWELL", "PANTON", "RIPTON", "SALISBURY", "SHOREHAM", "STARKSBORO", "WALTHAM", "WEYBRIDGE", "WHITING", "VERGENNES"] BENNINGTON = ["ARLINGTON", "BENNINGTON", "DORSET", "GLASTENBURY", "LANDGROVE", "MANCHESTER", "PERU", "POWNAL", "READSBORO", "RUPERT", "SANDGATE", "SEARSBURG", "SHAFTSBURY", "STAMFORD", "SUNDERLAND", "WINHALL", "WOODFORD"] CALEDONIA = ["BARNET", "BURKE", "DANVILLE", "GROTON", "HARDWICK", "KIRBY", "LYNDON", "NEWARK", "PEACHAM", "RYEGATE", "SAINT JOHNSBURY", "SHEFFIELD", "STANNARD", "SUTTON", "WALDEN", "WATERFORD", "WHEELOCK"] -CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON"] -ESSEX = ["AVERILL", "BLOOMFIELD", "BRIGHTON", "BRUNSWICK", "CANAAN", "CONCORD", "EAST HAVEN", "FERDINAND", "GRANBY", "GUILDHALL", "LEMINGTON", "LEWIS", "LUNENBURG", "MAIDSTONE", "NORTON", "VICTORY"] -FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS CITY", "SHELDON", "SWANTON"] +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +ESSEX = ["AVERILL", "BLOOMFIELD", "BRIGHTON", "BRUNSWICK", "CANAAN", "CONCORD", "EAST HAVEN", "FERDINAND", "GRANBY", "GUILDHALL", "LEMINGTON", "LEWIS", "LUNENBURG", "MAIDSTONE", "NORTON", "VICTORY", "AVERYS GORE", "WARNERS GRANT", "WARREN GORE"] +FRANKLIN = ["BAKERSFIELD", "BERKSHIRE", "ENOSBURGH", "FAIRFAX", "FAIRFIELD", "FLETCHER", "FRANKLIN", "GEORGIA", "HIGHGATE", "MONTGOMERY", "RICHFORD", "SAINT ALBANS CITY", "SAINT ALBANS TOWN", "SHELDON", "SWANTON"] GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] LAMOILLE = ["BELVIDERE", "CAMBRIDGE", "EDEN", "ELMORE", "HYDE PARK", "JOHNSON", "MORRISTOWN", "STOWE", "WATERVILLE", "WOLCOTT"] ORANGE = ["BRADFORD", "BRAINTREE", "BROOKFIELD", "CHELSEA", "CORINTH", "FAIRLEE", "NEWBURY", "ORANGE", "RANDOLPH", "STRAFFORD", "THETFORD", "TOPSHAM", "TUNBRIDGE", "VERSHIRE", "WASHINGTON", "WEST FAIRLEE", "WILLIAMSTOWN"] -ORLEANS = ["ALBANY", "BARTON", "BROWNINGTON", "CHARLESTON", "COVENTRY", "CRAFTSBURY", "DERBY", "GLOVER", "GREENSBORO", "HOLLAND", "IRASBURG", "JAY", "LOWELL", "MORGAN", "NEWPORT", "TROY", "WESTFIELD", "WESTMORE"] -RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] +ORLEANS = ["ALBANY", "BARTON", "BROWNINGTON", "CHARLESTON", "COVENTRY", "CRAFTSBURY", "DERBY", "GLOVER", "GREENSBORO", "HOLLAND", "IRASBURG", "JAY", "LOWELL", "MORGAN", "NEWPORT CITY", "NEWPORT TOWN", "TROY", "WESTFIELD", "WESTMORE"] +RUTLAND = ["BENSON", "BRANDON", "CASTLETON", "CHITTENDEN", "CLARENDON", "DANBY", "FAIR HAVEN", "HUBBARDTON", "IRA", "KILLINGTON", "MENDON", "MIDDLETOWN SPRINGS", "MOUNT HOLLY", "MOUNT TABOR", "PAWLET", "PITTSFIELD", "PITTSFORD", "POULTNEY", "PROCTOR", "RUTLAND TOWN", "RUTLAND CITY", "SHREWSBURY", "SUDBURY", "TINMOUTH", "WALLINGFORD", "WELLS", "WEST HAVEN", "WEST RUTLAND"] WASHINGTON = ["BARRE TOWN", "BERLIN", "CABOT", "CALAIS", "DUXBURY", "EAST MONTPELIER", "FAYSTON", "MARSHFIELD", "MIDDLESEX", "MORETOWN", "NORTHFIELD", "PLAINFIELD", "ROXBURY", "WAITSFIELD", "WARREN", "WATERBURY", "WOODBURY", "WORCESTER", "BARRE CITY", "MONTPELIER"] WINDHAM = ["ATHENS", "BRATTLEBORO", "BROOKLINE", "DOVER", "DUMMERSTON", "GRAFTON", "GUILFORD", "HALIFAX", "JAMAICA", "LONDONDERRY", "MARLBORO", "NEWFANE", "PUTNEY", "ROCKINGHAM", "SOMERSET", "STRATTON", "TOWNSHEND", "VERNON", "WARDSBORO", "WESTMINSTER", "WHITINGHAM", "WILMINGTON", "WINDHAM"] WINDSOR = ["ANDOVER", "BALTIMORE", "BARNARD", "BETHEL", "BRIDGEWATER", "CAVENDISH", "CHESTER", "HARTFORD", "HARTLAND", "LUDLOW", "NORWICH", "PLYMOUTH", 'POMFRET', 'READING', 'ROCHESTER', 'ROYALTON', 'SHARON', 'SPRINGFIELD', 'STOCKBRIDGE', 'WEATHERSFIELD', 'WEST WINDSOR', 'WESTON', 'WINDSOR', 'WOODSTOCK'] @@ -317,24 +317,26 @@ print(towns_gdf.crs) towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() ``` ```{python} -#now, merging the town level merge i made earlier with the town_gdf geojson file +#now, merging the town level merge i made earlier with the town_gdf geojson file - changing to have null == 0 town_coverage = con.execute(""" SELECT c.COUNTY, i.TOWN, COUNT(*) AS parcels, SUM((i.REAL_FLV > 0)::INT) AS valued, - ROUND(100.0 * SUM((i.REAL_FLV > 0)::INT) / COUNT(*), 1) AS pct_valued + ROUND(100.0 * SUM(CASE WHEN i.REAL_FLV > 0 THEN 1 ELSE 0 END) / COUNT(*), 1) AS pct_valued FROM parcels_info i LEFT JOIN towns c USING (TOWN) GROUP BY c.COUNTY, i.TOWN """).df() +print(town_coverage.head(5)) + towns_map = towns_gdf.merge(town_coverage, on="TOWN", how="left") #making sure that all towns are matched print("unmatched towns:", towns_map["pct_valued"].isna().sum()) print(towns_map[towns_map["pct_valued"].isna()]["TOWN"].tolist()) ``` - +84 towns end up not represented in our coverage map, but that is alright! they don't have parcel rows sent into vgis - this just means that we will have to point that out in the chloropleth ```{python} #diagnosing the problem at hadn parcels_towns = set(town_coverage["TOWN"]) @@ -342,3 +344,535 @@ boundary_towns = set(towns_gdf["TOWN"]) print("in boundary, not parcels:", sorted(boundary_towns - parcels_towns)) print("in parcels, not boundary:", sorted(parcels_towns - boundary_towns)) ``` +making the cholorpleths - town level +```{python} +#making a valid towns list to dissolve for the county line +towns_valid = towns_map.copy() +towns_valid["geometry"] = towns_valid["geometry"].make_valid() +counties_outline = towns_valid.dissolve(by="COUNTY") + +import matplotlib.pyplot as plt +fig, ax = plt.subplots(figsize=(8, 11)) +towns_map.plot(column="pct_valued", cmap="RdYlGn", legend=True, + edgecolor="gray", linewidth=0.3, ax=ax, + missing_kwds={"color": "lightgray", "label": "No data"}) +#counties_outline.boundary.plot(ax=ax, color="black", linewidth=2) +ax.set_title("Parcel Availability Percentage by Town") +ax.axis("off") +plt.show() +``` + +```{python} +county_coverage = town_coverage.groupby("COUNTY").apply( + lambda g: 100.0 * g["valued"].sum() / g["parcels"].sum() +).reset_index(name="pct_valued") + +towns_map_clean = towns_map.dropna(subset=["COUNTY"]).copy() +towns_map_clean["geometry"] = towns_map_clean["geometry"].make_valid() +counties_map = towns_map_clean.dissolve(by="COUNTY").merge( + county_coverage, on="COUNTY", how="left", suffixes=("_town", "")) +counties_map.plot(column="pct_valued", cmap="RdYlGn", legend=True, edgecolor="black") +``` +```{python} +#step one: making a list of the parcels to investigate +import pandas as pd +import geopandas as gpd +population_csv_path = "backend/notebooks/parcels/local_data/population_2020.csv" + +import os + +print(os.getcwd()) + +# rel_path = "local_data/population_2020.csv" +abs_path = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv" + + +print("abosulte path exists?", os.path.exists(abs_path)) + +pop = pd.read_csv(abs_path) +pop = pop.loc[:, ["_geoid", "Town", "County", "year2020"]] +pop = pop.rename(columns={'Town': 'TOWN', 'County': 'COUNTY', "year2020": "2020_POP", }) +pop["TOWN"] = pop["TOWN"].str.upper() +pop = pop.sort_values(by = "2020_POP", ascending=False) + +#print(pop) + +#adding names to list +all_pop = [] + +print("getting all towns by population:\n") +for i in range(len(pop)): + town = pop.iloc[i]["TOWN"] + popu = pop.iloc[i]["2020_POP"] + #print(str(town) + " " + str(popu)) + all_pop.append(town) + +all_pop = set(all_pop) + + +#print(pop.iloc[23:]) + + +``` +comparing parcel counts between large pop centers +```{python} +info = con.execute("""SELECT TOWN, CITYGL, PROPTYPE, DESCPROP, CAT, RESCODE FROM parcels_info""").df() + +towns = info["TOWN"].unique() +towns = set(towns) + +excluded = (all_pop - towns) +print(excluded) + + + +# excluded_df = pop[pop["TOWN"].isin(excluded)].sort_values(by="2020_POP", ascending=False) +# for _, row in excluded_df.iterrows(): +# print(row["TOWN"], row["2020_POP"]) + + + +# current = [] +# for item in vermont_towns_from_parcels: +# item = item.upper() +# current.append(item) + +# set_of_pop_parcels = set(current) - set(pop["TOWN"]) +# print(set_of_pop_parcels) + +# for item in pop["TOWN"]: +# if item.startswith("WARNER"): +# print(item) + +# what I found - no essex junction in population csv +# however, all saints are there +# in population csv, saint is ST. +# warren's gore is warren gore +# avery's gore is averys gore +# warner's grant is warners grant +``` +#ingesting all cities - +```{python} +import requests, zipfile, io +import geopandas as gpd + +vermont_towns_from_parcels = [ + "Addison", "Albany", "Alburgh", "Andover", "Arlington", "Athens", + "Averill", "Averys Gore", "Bakersfield", "Baltimore", "Barnard", + "Barnet", "Barre City", "Barre Town", "Barton", "Belvidere", + "Bennington", "Benson", "Berkshire", "Berlin", "Bethel", + "Bloomfield", "Bolton", "Bradford", "Braintree", "Brandon", + "Brattleboro", "Bridgewater", "Bridport", "Brighton", "Bristol", + "Brookfield", "Brookline", "Brownington", "Brunswick", "Buels Gore", + "Burke", "Burlington", "Cabot", "Calais", "Cambridge", "Canaan", + "Castleton", "Cavendish", "Charleston", "Charlotte", "Chelsea", + "Chester", "Chittenden", "Clarendon", "Colchester", "Concord", + "Corinth", "Cornwall", "Coventry", "Craftsbury", "Danby", + "Danville", "Derby", "Dorset", "Dover", "Dummerston", "Duxbury", + "East Haven", "East Montpelier", "Eden", "Elmore", "Enosburgh", + "Essex Junction", "Essex", "Fair Haven", "Fairfax", "Fairfield", + "Fairlee", "Fayston", "Ferdinand", "Ferrisburgh", "Fletcher", + "Franklin", "Georgia", "Glastenbury", "Glover", "Goshen", + "Grafton", "Granby", "Grand Isle", "Granville", "Greensboro", + "Groton", "Guildhall", "Guilford", "Halifax", "Hancock", + "Hardwick", "Hartford", "Hartland", "Highgate", "Hinesburg", + "Holland", "Hubbardton", "Huntington", "Hyde Park", "Ira", + "Irasburg", "Isle La Motte", "Jamaica", "Jay", "Jericho", + "Johnson", "Killington", "Kirby", "Landgrove", "Leicester", + "Lemington", "Lewis", "Lincoln", "Londonderry", "Lowell", + "Ludlow", "Lunenburg", "Lyndon", "Maidstone", "Manchester", + "Marlboro","Marshfield", "Mendon", "Middlebury", "Middlesex", + "Middletown Springs", "Milton", "Monkton", "Montgomery", + "Montpelier", "Moretown", "Morgan", "Morristown", "Mount Holly", + "Mount Tabor", "New Haven", "Newark", "Newbury", "Newfane", + "Newport City", "Newport Town", "North Hero", "Northfield", + "Norton", "Norwich", "Orange", "Orwell", "Panton", "Pawlet", + "Peacham", "Peru", "Pittsfield", "Pittsford", "Plainfield", + "Plymouth", "Pomfret", "Poultney", "Pownal", "Proctor", "Putney", + "Randolph", "Reading", "Readsboro", "Richford", "Richmond", + "Ripton", "Rochester", "Rockingham", "Roxbury", "Royalton", + "Rupert", "Rutland City", "Rutland Town", "Ryegate", + "Saint Albans City", "Saint Albans Town", "Saint George", + "Saint Johnsbury", "Salisbury", "Sandgate", "Searsburg", + "Shaftsbury", "Sharon", "Sheffield", "Shelburne", "Sheldon", + "Shoreham", "Shrewsbury", "Somerset", "South Burlington", + "South Hero", "Springfield", "Stamford", "Stannard", "Starksboro", + "Stockbridge", "Stowe", "Strafford", "Stratton", "Sudbury", + "Sunderland", "Sutton", "Swanton", "Thetford", "Tinmouth", + "Topsham", "Townshend", "Troy", "Tunbridge", "Underhill", + "Vergennes", "Vernon", "Vershire", "Victory", "Waitsfield", + "Walden", "Wallingford", "Waltham", "Wardsboro", "Warners Grant", + "Warren Gore", "Warren", "Washington", "Waterbury", "Waterford", + "Waterville", "Weathersfield", "Wells", "West Fairlee", + "West Haven", "West Rutland", "West Windsor", "Westfield", + "Westford", "Westminster", "Westmore", "Weston", "Weybridge", + "Wheelock", "Whiting", "Whitingham", "Williamstown", "Williston", + "Wilmington", "Windham", "Windsor", "Winhall", "Winooski", + "Wolcott", "Woodbury", "Woodford", "Woodstock", "Worcester" +] + + + + +BASE = "https://maps.vcgi.vermont.gov/gisdata/vcgi/packaged_zips/CadastralParcels_VTPARCELS/" + +def fetch_town(town): + r = requests.get(f"{BASE}VTPARCELS_{town}.zip", timeout=60) + r.raise_for_status() + z = zipfile.ZipFile(io.BytesIO(r.content)) + z.extractall(f"Data/parcels/backfill/{town}") + return gpd.read_file(f"Data/parcels/backfill/{town}/VTPARCELS_{town}.shp") + +for town in vermont_towns_from_parcels: + fetch_town(town) + print(town + " Done") + +print("successful bro") +#SOMETHING OF NOTE! the names are capital letter for the first character of each name only, the saints are "Saint", and there are towns and cities for a select few, like barre + +#making a list of all those i need to get +``` +making the view for all of the joined parcel towns +```{python} +import geopandas as gpd +from pathlib import Path + +#KEEP_COLS = ["SPAN","GLIST_SPAN","MAPID","PARCID","PROPTYPE","YEAR","GLYEAR", + # "TOWN","SOURCENAME","SOURCETYPE","SOURCEDATE","EDITDATE","MATCHSTAT", + # "OWNER1","CITYGL","STGL","ZIPGL","DESCPROP","LOCAPROP","CAT","RESCODE", + # "ACRESGL","REAL_FLV","HSTED_FLV","NRES_FLV","LAND_LV","IMPRV_LV", + # "EQUIPVAL","INVENVAL","HSDECL","HSITEVAL","VETEXAMT", + # "EXAMT_HS","EXAMT_NR","UVREDUC_HS","UVREDUC_NR", + # "GLVAL_HS","GLVAL_NR","CRHOUSPCT","MUNGL1PCT","AOEGL_HS","AOEGL_NR", + # "E911ADDR","geometry"] + +#all_columns = ['OGC_FID', 'OBJECTID', 'SPAN', 'GLIST_SPAN', 'MAPID', 'PARCID', 'PROPTYPE', 'YEAR', 'GLYEAR', 'TOWN', 'TNAME', 'SOURCENAME', 'SOURCETYPE', 'SOURCEDATE', 'EDITMETHOD', 'EDITOR', 'EDITDATE', 'MATCHSTAT', 'EDITNOTE', 'OWNER1', 'OWNER2', 'ADDRGL1', 'ADDRGL2', 'CITYGL', 'STGL', 'ZIPGL', 'DESCPROP', 'LOCAPROP', 'CAT', 'RESCODE', 'ACRESGL', 'REAL_FLV', 'HSTED_FLV', 'NRES_FLV', 'LAND_LV', 'IMPRV_LV', 'EQUIPVAL', 'EQUIPCODE', 'INVENVAL', 'HSDECL', 'HSITEVAL', 'VETEXAMT', 'EXPDESC', 'ENDDATE', 'STATUTE', 'EXAMT_HS', 'EXAMT_NR', 'UVREDUC_HS', 'UVREDUC_NR', 'GLVAL_HS', 'GLVAL_NR', 'CRHOUSPCT', 'MUNGL1PCT', 'AOEGL_HS', 'AOEGL_NR', 'E911ADDR', 'geom'] + +def standardize_town(shp_path: Path) -> gpd.GeoDataFrame: + gdf = gpd.read_file(shp_path) + gdf = gdf.to_crs(4326) + gdf["TOWN"] = gdf["TOWN"].str.upper().str.strip() + # missing = [c for c in KEEP_COLS if c not in gdf.columns] + # if missing: + # print(f"{shp_path.name}: missing {missing}") + # gdf = gdf[[c for c in KEEP_COLS if c in gdf.columns]] + return gdf +``` +making the new join +Design philosophy: while it might be a more efficient idea to backfill only those towns missing with complementary data from the parcels that I just ingested, my thought is that if there are parcels missing for whole towns, there cannot be a guarantee that ALL parcels for all towns are there wit hno issue, so im deciding to follow the path of creating a new dataset +```{python} +import pandas as pd +import numpy as np + +frames = [] +for town in vermont_towns_from_parcels: + shp = Path(f"Data/parcels/backfill/{town}") / f"VTPARCELS_{town}.shp" + g = standardize_town(shp) + frames.append(g) + print(town + " Done") +print("frame created successfully") + +#concatenating all the frames from the town parcels into one +backfill = pd.concat(frames, ignore_index=True) +backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) +#making teh objectid and adding +backfill["OBJECTID"] = 1_000_000 + backfill.index +``` +investigating the backfill frame, and adding the county column +also, mapping new values to old columns that I deem valid - run this after having ran the dictionary definition cell +```{python} +#print((backfill.columns)) +# print(backfill.head(5)) + +#adding a counties column - filling the dictionary +counties_dict = {} +unknown = [] +for town in backfill["TOWN"].unique(): + for item in counties_key: + if town in item: + counties_dict[town] = names_key[(counties_key.index(item))] +#adding in sourcename as city and town +city_town_dict = {} + +city_source = backfill[ + (backfill["SOURCENAME"] == "CITY") | + (backfill["SOURCENAME"] == "TOWN") +] + + +#finding if the parcel was found from a local department or not +backfill["SOURCENAME"] = np.where( + backfill["SOURCENAME"].isin(["CITY", "TOWN", "City of Burlington"]), + "LOCAL DEPARTMENT", + "NOT LOCAL DEPARTMENT" +) + +#this code aids in the creation of a column that sees if the primary people live there, or if it is an investment property. +residential_codes = ['R1', 'R2', 'MHL', "MHU"] + +is_residential = backfill["CAT"].isin(residential_codes) +is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == 'N') + +vt_mask = backfill["STGL"].str.startswith("VT", na=False) | backfill["STGL"].str.startswith("Vt", na=False) | backfill["STGL"].str.startswith("05", na=False) | backfill["STGL"].str.contains("VT", na=False) | backfill["STGL"].str.contains("V T", na=False) | backfill["STGL"].str.startswith("vt", na=False) +backfill.loc[vt_mask, "STGL"] = "VERMONT" +canada_mask = backfill["STGL"].str.contains("CANADA", na=False) | backfill["STGL"].str.contains("Canada", na=False) | backfill["STGL"].str.contains("QC", na=False) +backfill.loc[canada_mask, "STGL"] = "CANADA" + +#mapping new items in columns +backfill["INVESTMENTPROP"] = is_residential & is_not_homestead + + + +backfill["VACANTLAND"] = (backfill["LAND_LV"] > 0) & (backfill["IMPRV_LV"].fillna(0) == 0) +equipcode_dict = {"E" : "ELECTRIC UTILITY", "C": "CABLE UTILITY"} +backfill["EXEMPT"] = backfill["STATUTE"].notna().map({True:"YES", False:"NO"}) +backfill["STATUTE"] = backfill["STATUTE"].map(statutes_dict).fillna("No Exemption") +backfill["EXPDESC"] = backfill["EXPDESC"].map(expdesc_dict).fillna("None") +backfill["EQUIPCODE"] = backfill["EQUIPCODE"].map(equipcode_dict).fillna("NOT A UTILITY") +#creating a foreign ownerpship column + + + + +backfill["COUNTY"] = backfill["TOWN"].map(counties_dict) +backfill["RESCODE"] = backfill["RESCODE"].map(rescode_dict) +backfill["CATEGORY"] = backfill["CAT"].map(cat_dict) +backfill["PURPOSE"] = backfill["CAT"].map(purpose_dict) +#adding state abbreivaitions and owner lcoations +backfill["STGL"] = backfill["STGL"].replace(oos_dict) +catch_all_mask = ( + ~backfill["STGL"].isin(['CANADA', 'FOREIGN', 'US TERRITORY']) & + backfill["STGL"].notna() & + ~backfill["STGL"].isin(states) +) +backfill.loc[catch_all_mask, "STGL"] = "UNNAMED AMERICA" +backfill["OOSOWNER"] = backfill["STGL"] != "VT" + +backfill["ADDRESS"] = backfill["E911ADDR"] +``` +RUN THIS CELL TO CHECK THE NULL COUNTS/percentage OF EACH column adn choosing to drop them. +```{python} +#inspeacting the data + +#backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "OBJECTID", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) + +print(backfill["geometry"]) + +# prof = pd.DataFrame({ +# "dtype": backfill.dtypes.astype(str), +# "null_pct": (backfill.isna().mean() * 100).round(1), +# "n_unique": backfill.nunique(dropna=True), +# "sample": [backfill[c].dropna().iloc[0] if backfill[c].notna().any() else None +# for c in backfill.columns], +# }) +# prof.sort_values("null_pct", ascending=False) + +``` +```{python} + +residential_codes = ['R1', 'R2', 'MH'] +is_residential = backfill["PROPTYPE"].isin(residential_codes) +is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == '0') + +states = [ + "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", + "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", + "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", + "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", + "SD", "TN", "TX", "UT", "VA", "WA", "WV", "WI", "WY", "VT", "DC" +] + + + +#print(backfill[~backfill["STGL"].isin(states)]["STGL"].value_counts()) + +#print(backfill[backfill["STGL"] == "MA"][["STGL", "ZIPGL"]].value_counts()) + +print((backfill["STGL"].value_counts())) + + +#print(len(states_except_vt)) + +#print(backfill[["TOWN", "REAL_FLV", "CAT", "INVESTMENTPROP"]]) +#print(backfill[["CAT", "DESCPROP"]].value_counts()) +#print(backfill[["CATEGORY", "TOWN", "ACRESGL", 'REAL_FLV']].value_counts().sort_index(level="REAL_FLV", ascending=False)) + +#print(backfill[backfill["STGL"] == "MA"][["CITYGL", "STGL"]].value_counts()) + +# mismatch_count = (backfill["E911ADDR"] != backfill["ADDR1"]).sum() +# print(f"Number of mismatched addresses: {mismatch_count}") + + + +#print(backfill["CAT"].value_counts()) +#print(backfill["CAT"].isin(residential_codes).value_counts()) +#print(backfill["VACANTLAND"].value_counts()) + +# foreign_investment = backfill[backfill["VACANTLAND"] == True] +# print(foreign_investment["TOWN"].value_counts()) + + +# data_directory = Path("Data") +# parcel_path = data_directory / "parcels" / "parcels_vermont.geojson" + +# parcels_parquet = Path("Data/parcels/parcels_p.parquet") +# parcels_fgb = Path("Data/parcels/parcels_f.fgb") + + +# con = duckdb.connect() +# con.execute("INSTALL spatial; LOAD spatial;") + +# if not parcels_fgb.exists(): +# con.execute(f""" +# COPY ( +# SELECT * FROM ST_Read('{parcel_path}') +# WHERE geom IS NOT NULL +# ) +# TO '{parcels_fgb}' (FORMAT GDAL, DRIVER 'FlatGeobuf') +# """) + + +# prof = pd.DataFrame({ +# "dtype": backfill.dtypes.astype(str), +# "null_pct": (backfill.isna().mean() * 100).round(1), +# "n_unique": backfill.nunique(dropna=True), +# "sample": [backfill[c].dropna().iloc[0] if backfill[c].notna().any() else None +# for c in backfill.columns], +# }) +# prof.sort_values("null_pct", ascending=False) +``` +RUN THIS TO DROP COLUMNS FROM BACKFILL +```{python} +backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) + +``` +SAVING THE BACKFILL df AS A FILE +```{python} +from pathlib import Path +import geopandas as gpd +from shapely.geometry import MultiPolygon + +out_path = Path("Data/parcels/all_parcels_vermont.fgb") +out_path.parent.mkdir(parents=True, exist_ok=True) + +gdf = backfill.loc[~bad_geom].copy() + +assert gdf.geom_type.isin(["Polygon", "MultiPolygon"]).all(), gdf.geom_type.value_counts() +gdf["geometry"] = gdf.geometry.apply( + lambda g: MultiPolygon([g]) if g.geom_type == "Polygon" else g +) + +gdf.to_file(out_path, driver="FlatGeobuf") +print("wrote", out_path, "|", gdf.shape) + + +#quick check +parcels = gpd.read_file("Data/parcels/all_parcels_vermont.fgb") +print(parcels.shape, "|", parcels.crs) +print(parcels.geom_type.value_counts()) + +# quick round-trip check — did the derived cols survive with sane values? +print(parcels[["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]].apply(lambda s: s.value_counts()).T) +parcels.head() +``` + + + + + +Bottom of the page cell where I am storing the dictionaries of transmuted columns +```{python} +statutes_dict = { + '3848:3849': 'Business Inventory & Equipment', + '3848:38:00': 'Business Inventory & Equipment', + '3840': 'Charitable, Fraternal, or Rescue', + '3840;5405a(a)(4)': 'Charitable/Rescue (inc. Education Tax)', + '3840;54': 'Charitable/Rescue (inc. Education Tax)', + '2741': 'Tax Stabilization Contract', + '24/2741': 'Tax Stabilization Contract', + '3832': 'Public, Pious, or Charitable', + '3832(1)': 'Out-of-Town Municipal Property', + '3832(7)': 'Health or Recreational Property', + '3832(7)(B)': 'Non-profit Ice Skating Rink', + '3832(7B': 'Non-profit Ice Skating Rink', + '5401': 'Statewide Education Tax Exception', + '3752(7)': 'Agricultural / Current Use' +} + +expdesc_dict = { + 'Statutory': 'State Law Exemption', + 'Solar Plant': 'Solar Energy Facility', + 'Non-Approved (Voted)': 'Local Town-Voted Exemption', + 'Qualified Housing Units': 'Affordable / Qualified Housing', + 'Grandfathered': 'Pre-existing Historical Exemption', + 'Partial-Statutory': 'Partial State Law Exemption', + 'Municipal Contract (Owner Pays)': 'Payment in Lieu of Taxes (PILOT)', + 'Ski Lifts / Snow Making Equip': 'Ski Resort Equipment', + 'Court Ordered': 'Judicially Mandated Exemption', + 'Wind Plant': 'Wind Energy Facility' +} + +rescode_dict = { + "T" : "TOWN RESIDENT", + "NS" : "OUT OF STATE RESIDENT", + "S" : "VERMONT RESIDENT", + "C" : "CORPORATION/ENTITY", + "c" : "CORPORATION/ENTITY" +} + +cat_dict = { + "R1": "Residential I (Under 6 Acres)", + "R2": "Residential II (6 Acres or More)", + "M": "Miscellaneous", + "O": "Other", + "C": "Commercial", + "MHL": "Mobile Home Landed (With Land)", + "S1": "Seasonal I (Under 6 Acres)", + "MHU": "Mobile Home Unlanded (Without Land)", + "W": "Woodland", + "S2": "Seasonal II (6 Acres or More)", + "F": "Farm", + "CA": "Commercial Apartments", + "I": "Industrial", + "UE": "Utility Electric", + "UO": "Utility Other" +} + +purpose_dict = { + "R1" : "PRIMARY RESIDENCE", + "R2" : "PRIMARY RESIDENCE", + "MHL" : "PRIMARY RESIDENCE", + "MHU": "PRIMARY RESIDENCE", + "S1": "SEASONAL PROPERTY", + "S2": "SEASONAL PROPERTY", + "W": "WOODLAND", + "F": "FARM", + "CA" : "COMMERCIAL APARTMENTS", + "M": "NOT LISTED", + "O" : "NOT LISTED", + "C": "COMMERCIAL/INDUSTRIAL/UTILITY", + "I": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UE" : "COMMERCIAL/INDUSTRIAL/UTILITY", + "UO": "COMMERCIAL/INDUSTRIAL/UTILITY" +} + + +oos_dict = { + "VERMONT" : "VT", +} +oos_dict.update(dict.fromkeys(['QC', 'QC CANADA', 'PQ', "QUEBEC", "ON", "ONTARIO", "QUE", "BC", "ONT", "CAN"], 'CANADA')) +oos_dict.update(dict.fromkeys(["MASS", "MA."], 'MA')) +oos_dict.update(dict.fromkeys(["MICHIGAN"], 'MI')) +oos_dict.update(dict.fromkeys(["OHIO"], 'OH')) +oos_dict.update(dict.fromkeys(["CT."], 'CT')) +oos_dict.update(dict.fromkeys(["R.I."], 'RI')) +oos_dict.update(dict.fromkeys(["W VA"], 'WV')) +oos_dict.update(dict.fromkeys(["MARYLAND"], 'MD')) +oos_dict.update(dict.fromkeys(["N CAROLINA"], 'NC')) +oos_dict.update(dict.fromkeys(["NEW YORK", "N.Y.", "12513", "N Y"], 'NY')) +oos_dict.update(dict.fromkeys(["FLORIDA", "FLA"], 'FL')) +oos_dict.update(dict.fromkeys(["ENGLAND", "AE", "UNK", "BERMUDA", "UK", "VY", "FRANCE", "IND", "ARUBA", "GERMANY", "IRELAND", "SWITZERLAN", "QLD AUS", "LIN", "0R", "BERLIN", "AUSTRALIA", "NS", "FWI", "BAHAMAS"], 'FOREIGN')) +oos_dict.update(dict.fromkeys(["VI", "PR", "GUAM"], 'US TERRITORY')) + + +``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build.qmd b/backend/notebooks/parcels/table_build.qmd index 16213a87..393a88c4 100644 --- a/backend/notebooks/parcels/table_build.qmd +++ b/backend/notebooks/parcels/table_build.qmd @@ -70,13 +70,24 @@ if not parcels_parquet.exists(): ```{python} #creating the parcels_raw view on which to query things to learn con.execute("CREATE OR REPLACE VIEW parcels_raw AS SELECT * FROM ST_Read('Data/Parcels/parcels_f.fgb')") + +col_list = con.execute("SELECT * FROM parcels_raw LIMIT 0").df().columns.to_list() +print(col_list) ``` Here, we print all the columns in the parcels_raw view, of which there are many. commented out is the code to yield the total row count, with is 191156. Now looking at the column types and their respective names (what they represent in real life), and making a reproducible code chunk to investigate what each means and what it looks like + +HERE ARE THE COLUMNS +OGC_FID: feature id +OBJECTID YEAR GLYEAR: specific tax year REALFLV: total assessed value of real estate HSTED_VAL:homestead value NRES_FLV: non residentialhomestead value LAND_LV:land value, IMPROVE_LV: improvement land value, EQUIPVAL_ equipment value, INVENVAL:inventory value, HSITEVAL: home value and two surrounding acres EXAMPT_HS: exemption amount homestead. EXAMPT_NR: exemption amount non-resdiential. UVREDUC_HS: use value reduction homestead. UVREDUC_NR: use value reduction non-residential + +SPAN: school property access number - property identifier. GLIST_SPAN: grand list property span number. MAPID: municipality tax identifier. PARCID: parcel identifier. PROPTYPE: what is on the parcel itself. TOWN: town name in code. TNAME: fully written out town name. SOURCENAME: name of infor source. SOURCETYPE: original data type. SOURCEDATE: date of original map publishing. EDITMETHOD: method to draw map. EDITOR initials. EDITDATE. MATCHSTAT: how well the parcel map shape attached to grand tax record. EDITNOTE. OWNER1. OWNER2. ADDRGL1. ADDRGL2. CITYGL. STATEGL. ZIPGL. DESCPROP: description of property. LOCAPROP. CAT: category of parcel use. RESCODE: where doesthe resident live - out of state?. EQUIPCODE: taxable equipcode. HSDECL: homestead declaration. ENDDATE: end of parcel data. STATUTE - reference ot vermont law if applicable to parcel. E911 ADRR: 911 address + +ACRESGL: taxable acreage. GLVAL_HS: 1% of homestead value. GLVAL_NR: non residential - 1 percent of non residential tax. CRHOUSPCT: percentage of property subject to housing statutes. MUNGL1PCT: total value on which to calculate 1% of value per municipality. AOEGL_HS: homestead value for calculating education taxation. AOEGL_NR: non residential taxation value calculated for education property. + +geom: The actual spatial map data (the points, lines, and polygons) that draws the shape and physical boundaries of the parcel. EPSG:4326 signifies that the coordinates are mapped using standard Latitude and Longitude (WGS 84), exactly like a GPS uses. ```{python} described_parcels_raw = con.execute("DESCRIBE parcels_raw").df() -#print(described_parcels_raw) -# print(con.execute("SELECT COUNT(*) AS row_count, COUNT (DISTINCT OBJECTID) AS distinct_ids, SUM((OBJECTID IS NULL)::INT) AS nulls FROM parcels_raw").df()) column_types = described_parcels_raw["column_type"].unique().tolist() columns_by_type = {} @@ -351,5 +362,5 @@ print(db.execute("SELECT COUNT(*) FROM parcels_geom").df()) ``` ```{python} -con.execute("SELECT DISTINCT TOWN FROM parcels_raw").df() +con.execute("SELECT DISTINCT RESCODE FROM parcels_raw").df() ``` \ No newline at end of file diff --git a/backend/notebooks/parcels/table_build_final.qmd b/backend/notebooks/parcels/table_build_final.qmd new file mode 100644 index 00000000..6f46f347 --- /dev/null +++ b/backend/notebooks/parcels/table_build_final.qmd @@ -0,0 +1,216 @@ +--- +title: "Vermont Parcels: making the new tables" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +```{python} +import geopandas as gpd, pandas as pd, numpy as np +import matplotlib.pyplot as plt +from matplotlib.colors import LogNorm +from pathlib import Path + +FGB = "Data/parcels/all_parcels_vermont.fgb" +parcels = gpd.read_file(FGB) +print("shape:", parcels.shape, "| CRS:", parcels.crs) +print(parcels.geom_type.value_counts()) + +# --- join key integrity (self-healing) --- +if "OBJECTID" not in parcels.columns: + print("!! OBJECTID not in file (dropped before save). Minting a fresh stable id.") + parcels = parcels.reset_index(drop=True) + parcels["OBJECTID"] = 1_000_000 + parcels.index +assert parcels["OBJECTID"].is_unique, "OBJECTID not unique — unusable as join key" +assert parcels["OBJECTID"].notna().all(), "OBJECTID has nulls" +print("OBJECTID ok | n =", parcels["OBJECTID"].nunique()) + +# --- the columns that kept silently breaking on you --- +for c in ["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]: + if c in parcels: + print(f"{c:16s} →", dict(parcels[c].value_counts(dropna=False))) + +# quick null scan on the fields the analysis leans on +key = ["TOWN","COUNTY","REAL_FLV","LAND_LV","IMPRV_LV","ACRESGL","CAT","PURPOSE"] +print((parcels[[c for c in key if c in parcels]].isna().mean()*100).round(1)) + +``` +making a reusable chloropleth filter +```{python} +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +towns_gdf = gpd.read_file(TOWN_URL) +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +towns_gdf = towns_gdf[["TOWN", "geometry"]].copy() + +def town_choropleth(values: pd.Series, title, log=False, cmap="viridis"): + """values: a Series indexed by TOWN.""" + v = values.rename("val").reset_index(); v.columns = ["TOWN", "val"] + m = towns_gdf.merge(v, on="TOWN", how="left") + if log: + m.loc[m["val"] <= 0, "val"] = np.nan + fig, ax = plt.subplots(figsize=(8, 11)) + kw = dict(column="val", cmap=cmap, legend=True, ax=ax, + edgecolor="gray", linewidth=0.3, + missing_kwds={"color": "lightgray", "label": "no data"}) + if log: + pos = m["val"].dropna() + kw["norm"] = LogNorm(vmin=pos.min(), vmax=pos.max()) + m.plot(**kw); ax.set_title(title); ax.axis("off"); plt.show() + print("unmatched towns:", m["val"].isna().sum()) + return m +``` +before vs after backfill +```{python} +import duckdb +con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) + +# AFTER — the rebuilt layer +new_n = len(parcels) +new_valued = int((parcels["REAL_FLV"] > 0).sum()) +new_sum = float(parcels.loc[parcels["REAL_FLV"] > 0, "REAL_FLV"].sum()) + +# BEFORE — the original statewide layer +old = con.execute(""" + SELECT TOWN, COUNT(*) n, + SUM((REAL_FLV > 0)::INT) valued, + SUM(CASE WHEN REAL_FLV > 0 THEN REAL_FLV ELSE 0 END) val_sum + FROM parcels_info GROUP BY TOWN +""").df() +old_n, old_valued, old_sum = int(old.n.sum()), int(old.valued.sum()), float(old.val_sum.sum()) + +print(f"BEFORE: {old_n:>8,} parcels | {old_valued:>8,} valued ({100*old_valued/old_n:4.1f}%) | ${old_sum:,.0f}") +print(f"AFTER : {new_n:>8,} parcels | {new_valued:>8,} valued ({100*new_valued/new_n:4.1f}%) | ${new_sum:,.0f}") +print(f"Δ : {new_n-old_n:>+8,} parcels | {new_valued-old_valued:>+8,} valued | ${new_sum-old_sum:+,.0f}") + +# --- tag each rebuilt parcel by how the OLD layer treated its town --- +old_all = set(old.TOWN) +old_broken = set(old.loc[old.valued == 0, "TOWN"]) +def origin(t): + if t not in old_all: return "absent_from_statewide" + if t in old_broken: return "restored_by_rebuild" + return "covered_in_statewide" +parcels["data_origin"] = parcels["TOWN"].map(origin) + +by_origin = (parcels.assign(valued=parcels["REAL_FLV"] > 0) + .groupby("data_origin") + .agg(parcels=("OBJECTID", "size"), + valued=("valued", "sum"), + grand_list=("REAL_FLV", lambda s: s[s > 0].sum()))) +print("\n", by_origin, sep="") + +restored = parcels[parcels.data_origin == "restored_by_rebuild"] +print(f"\nRestored {len(restored):,} parcels worth " + f"${restored.loc[restored.REAL_FLV>0,'REAL_FLV'].sum():,.0f}") +print(restored.groupby("TOWN") + .agg(n=("OBJECTID","size"), value=("REAL_FLV", lambda s: s[s>0].sum())) + .sort_values("n", ascending=False).head(10)) + +# spelling-drift guard between the two TOWN keys +print("\nin rebuilt, no old-layer match:", sorted(set(parcels.TOWN) - old_all)[:15]) +``` + +MEDIAN PARCEL VALUE +```{python} +med_val = parcels.loc[parcels["REAL_FLV"] > 0].groupby("TOWN")["REAL_FLV"].median() +town_choropleth(med_val, "Median parcel value (REAL_FLV) — log scale", log=True, cmap="viridis") +print("highest:\n", med_val.sort_values(ascending=False).head(8)) +print("lowest:\n", med_val.sort_values().head(8)) +``` + +```{python} +# EPSG:32145 = VT State Plane (meters); 1 acre = 4046.8564224 m² +parcels["AREA_ACRES_GEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 + +size = (parcels.groupby("TOWN") + .agg(deeded=("ACRESGL", "median"), geom=("AREA_ACRES_GEOM", "median")) + .dropna()) +print("largest lots:\n", size.sort_values("geom", ascending=False).head(8)) +print("smallest lots:\n", size.sort_values("geom").head(8)) +print("corr deeded vs geom:", + parcels[["ACRESGL","AREA_ACRES_GEOM"]].dropna().corr().iloc[0,1].round(3)) + +town_choropleth(size["geom"], "Median parcel size (geometry acres) — log scale", log=True, cmap="cividis") +``` + +LAND VS IMPROVEMENT VALUE +```{python} +val = parcels["REAL_FLV"] +parcels["IMPR_SHARE"] = np.where(val > 0, parcels["IMPRV_LV"].fillna(0) / val, np.nan) + +# alt denominator (assessed components) — often cleaner than REAL_FLV +den = parcels["LAND_LV"].fillna(0) + parcels["IMPRV_LV"].fillna(0) +parcels["IMPR_SHARE_ALT"] = np.where(den > 0, parcels["IMPRV_LV"].fillna(0) / den, np.nan) + +print(parcels["IMPR_SHARE"].describe()) +print("share > 1 (quirks where IMPRV_LV exceeds REAL_FLV):", int((parcels["IMPR_SHARE"] > 1).sum())) + +med_share = parcels.groupby("TOWN")["IMPR_SHARE"].median().clip(0, 1) +town_choropleth(med_share, "Median improvement share (building ÷ total value)", cmap="RdYlBu_r") +print("most built-up:\n", med_share.sort_values(ascending=False).head(8)) +print("most land-heavy:\n", med_share.sort_values().head(8)) +``` + + +```{python} +# PURPOSE already collapses CAT into the urban-rural-relevant buckets you engineered +mix = pd.crosstab(parcels["TOWN"], parcels["PURPOSE"], normalize="index").mul(100).round(1) +print(mix.head()) + +res = mix.get("PRIMARY RESIDENCE", pd.Series(0.0, index=mix.index)) +rural = mix.reindex(columns=["FARM","WOODLAND","SEASONAL PROPERTY"], fill_value=0).sum(axis=1) +print("\nmost residential:\n", res.sort_values(ascending=False).head(10)) +print("\nmost farm/woodland/seasonal:\n", rural.sort_values(ascending=False).head(10)) + +top = parcels["TOWN"].value_counts().head(12).index +mix.loc[top].plot(kind="barh", stacked=True, figsize=(9, 7)) +plt.xlabel("% of parcels"); plt.title("PURPOSE mix — 12 largest towns") +plt.legend(bbox_to_anchor=(1.02, 1)); plt.tight_layout(); plt.show() + +``` + + +```{python} +out = Path("Data/parcels/tables"); out.mkdir(parents=True, exist_ok=True) +def present(cols): return [c for c in cols if c in parcels.columns] # tolerate dropped cols + +geom_cols = present(["OBJECTID", "TOWN", "COUNTY", "geometry"]) +info_cols = present(["OBJECTID","TOWN","COUNTY","SPAN","PROPTYPE","CAT","CATEGORY","PURPOSE", + "DESCPROP","RESCODE","ACRESGL","AREA_ACRES_GEOM","CITYGL","STGL","ADDRESS", + "SOURCENAME","MATCHSTAT","TNAME","INVESTMENTPROP","VACANTLAND","OOSOWNER", + "data_origin","EDITOR","EDITDATE"]) +tax_cols = present(["OBJECTID","TOWN","REAL_FLV","HSTED_FLV","NRES_FLV","LAND_LV","IMPRV_LV", + "IMPR_SHARE","EQUIPVAL","EQUIPCODE","INVENVAL","HSDECL","VETEXAMT","EXPDESC", + "STATUTE","EXEMPT","EXAMT_HS","EXAMT_NR","UVREDUC_HS","UVREDUC_NR", + "GLVAL_HS","GLVAL_NR"]) + +geom = parcels[geom_cols].copy() # GeoDataFrame → GeoParquet +info = pd.DataFrame(parcels[info_cols]) # plain (no geometry) +tax = pd.DataFrame(parcels[tax_cols]) + +geom.to_parquet(out / "parcels_geom.parquet") +info.to_parquet(out / "parcels_info.parquet", index=False) +tax.to_parquet(out / "parcels_tax.parquet", index=False) +print("geom:", geom.shape, "| info:", info.shape, "| tax:", tax.shape) + +# --- prove the three still join cleanly on OBJECTID --- +g = gpd.read_parquet(out / "parcels_geom.parquet") +i = pd.read_parquet(out / "parcels_info.parquet") +t = pd.read_parquet(out / "parcels_tax.parquet") +assert g.OBJECTID.is_unique and i.OBJECTID.is_unique and t.OBJECTID.is_unique +assert set(g.OBJECTID) == set(i.OBJECTID) == set(t.OBJECTID), "OBJECTID sets differ!" +three = g.merge(i, on="OBJECTID").merge(t, on="OBJECTID", suffixes=("_info","_tax")) +print("3-way join rows:", len(three), "== parcel count?", len(three) == len(parcels)) +``` \ No newline at end of file From d4a701b75efea55010b9af4b6146727dee860ecb Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Mon, 24 Aug 2026 02:22:25 -0400 Subject: [PATCH 11/12] finished county and urban/rural exploration. still have yet to join with zoining or ship new tables to api --- .../notebooks/parcels/parcels_run_test.qmd | 79 ++- .../notebooks/parcels/table_build_final.qmd | 637 ++++++++++++++++-- backend/pyproject.toml | 1 + backend/uv.lock | 16 + 4 files changed, 652 insertions(+), 81 deletions(-) diff --git a/backend/notebooks/parcels/parcels_run_test.qmd b/backend/notebooks/parcels/parcels_run_test.qmd index fe0b2e04..6969a18a 100644 --- a/backend/notebooks/parcels/parcels_run_test.qmd +++ b/backend/notebooks/parcels/parcels_run_test.qmd @@ -307,11 +307,57 @@ now that we have the parcels and parcels' percentages based on county and town, ```{python} import geopandas as gpd -TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" -towns_gdf = gpd.read_file(TOWN_URL) -print(towns_gdf.shape) -print(towns_gdf.columns.tolist()) -print(towns_gdf.crs) +# TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" +# towns_gdf = gpd.read_file(TOWN_URL) +# print(towns_gdf.shape) +# print(towns_gdf.columns.tolist()) +# print(towns_gdf.crs) + +import geopandas as gpd +import requests + +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + + +base_url = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query" + +offset = 0 +chunk_size = 50 +all_features = [] + +# Fetch data in chunks of 50 to avoid server timeouts +while True: + params = { + "where": "1=1", + "outFields": "*", + "f": "geojson", + "resultOffset": offset, + "resultRecordCount": chunk_size + } + response = requests.get(base_url, params=params) + data = response.json() + + features = data.get("features", []) + if not features: + break + + all_features.extend(features) + + # If a chunk returns less than 50, we've reached the end + if len(features) < chunk_size: + break + + offset += chunk_size + +# Rebuild the full GeoJSON and load it into GeoPandas +full_geojson = {"type": "FeatureCollection", "features": all_features} +towns_gdf = gpd.GeoDataFrame.from_features(full_geojson, crs="EPSG:4326") + +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +#print(towns_gdf.shape) + +print(towns_gdf.columns) + #just certifying that the town namese are both uppercase and devoic of whitespace towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() @@ -646,17 +692,26 @@ catch_all_mask = ( ~backfill["STGL"].isin(states) ) backfill.loc[catch_all_mask, "STGL"] = "UNNAMED AMERICA" -backfill["OOSOWNER"] = backfill["STGL"] != "VT" +backfill["OOSOWNER"] = backfill["STGL"].fillna("VT") != "VT" backfill["ADDRESS"] = backfill["E911ADDR"] + +#adding an area in acres to geometry column, a +if "AREAACRESGEOM" not in backfill: + backfill["AREAACRESGEOM"] = backfill.to_crs(32145).geometry.area / 4046.8564224 +backfill["ACREVALUE"] = np.where( + (backfill.REAL_FLV > 0) & (backfill.AREAACRESGEOM > 0), + backfill.REAL_FLV / backfill.AREAACRESGEOM, np.nan) ``` RUN THIS CELL TO CHECK THE NULL COUNTS/percentage OF EACH column adn choosing to drop them. ```{python} #inspeacting the data -#backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "OBJECTID", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) +# backfill = backfill.drop(columns = ["LOCAPROP", "ADDRGL2", "ENDDATE", "OWNER1", "OWNER2", "EDITNOTE", "MAPID", "YEAR", "GLYEAR", "SOURCETYPE", "SOURCEDATE", "EDITMETHOD", "OBJECTID", "SHAPE_STAr", "SHAPE_STLe", "GLIST_SPAN", "PARCID", "CRHOUSPCT", "MUNGL1PCT", "AOEGL_HS", "AOEGL_NR", "HSITEVAL", "E911ADDR", "ADDRGL1", "ZIPGL"]) + +print(backfill["OOSOWNER"].value_counts()) -print(backfill["geometry"]) +# print(backfill[backfill["TOWN"] == "BUELS GORE"]["STGL"].value_counts()) # prof = pd.DataFrame({ # "dtype": backfill.dtypes.astype(str), @@ -755,6 +810,7 @@ from shapely.geometry import MultiPolygon out_path = Path("Data/parcels/all_parcels_vermont.fgb") out_path.parent.mkdir(parents=True, exist_ok=True) +bad_geom = backfill.geometry.isna() | backfill.geometry.is_empty gdf = backfill.loc[~bad_geom].copy() assert gdf.geom_type.isin(["Polygon", "MultiPolygon"]).all(), gdf.geom_type.value_counts() @@ -772,10 +828,13 @@ print(parcels.shape, "|", parcels.crs) print(parcels.geom_type.value_counts()) # quick round-trip check — did the derived cols survive with sane values? -print(parcels[["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]].apply(lambda s: s.value_counts()).T) -parcels.head() +# print(parcels[["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]].apply(lambda s: s.value_counts()).T) +# parcels.head() ``` +```{python} +print(gdf.columns) +``` diff --git a/backend/notebooks/parcels/table_build_final.qmd b/backend/notebooks/parcels/table_build_final.qmd index 6f46f347..2caae15e 100644 --- a/backend/notebooks/parcels/table_build_final.qmd +++ b/backend/notebooks/parcels/table_build_final.qmd @@ -18,127 +18,227 @@ execute: editor: render-on-save: true --- +#HELPING FIND THE DATA AFTER KERNEL RESTART +```{python} +import os +from pathlib import Path + +print("cwd (where the fresh kernel started):", Path.cwd()) + +# climb until we hit the folder containing api/ — identical anchor to your other notebooks +root = Path.cwd() +while root != root.parent and not (root / "api").exists(): + root = root.parent +assert (root / "api").exists(), f"never found an api/ folder climbing up from {Path.cwd()}" +os.chdir(root) +print("anchored to:", Path.cwd()) + +FGB = "Data/parcels/all_parcels_vermont.fgb" +print("exists now?:", os.path.exists(FGB)) +``` ```{python} import geopandas as gpd, pandas as pd, numpy as np import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm +from matplotlib.colors import LogNorm from pathlib import Path +#getting the FBG from file storage, making the parcels geo df, and then printing the shape and CRS, which should be EPSG 4326, then pirnting the type of all geom entries - hopefully it is multi polygon FGB = "Data/parcels/all_parcels_vermont.fgb" -parcels = gpd.read_file(FGB) -print("shape:", parcels.shape, "| CRS:", parcels.crs) -print(parcels.geom_type.value_counts()) -# --- join key integrity (self-healing) --- +parcels = gpd.read_file(FGB) +print("shape:", parcels.shape, "| CRS:", parcels.crs) +print(parcels.geom_type.value_counts()) + + +#makinga failsafe self healing join key in the case object id is missing - just so objectid is always present as a thing to join on if "OBJECTID" not in parcels.columns: - print("!! OBJECTID not in file (dropped before save). Minting a fresh stable id.") - parcels = parcels.reset_index(drop=True) - parcels["OBJECTID"] = 1_000_000 + parcels.index -assert parcels["OBJECTID"].is_unique, "OBJECTID not unique — unusable as join key" + print("!! OBJECTID not in file — minting a fresh stable id.") + parcels = parcels.reset_index(drop=True) + parcels["OBJECTID"] = 1_000_000 + parcels.index +assert parcels["OBJECTID"].is_unique, "OBJECTID not unique — unusable as a join key" assert parcels["OBJECTID"].notna().all(), "OBJECTID has nulls" +#objects count should be 343875 print("OBJECTID ok | n =", parcels["OBJECTID"].nunique()) -# --- the columns that kept silently breaking on you --- -for c in ["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]: - if c in parcels: - print(f"{c:16s} →", dict(parcels[c].value_counts(dropna=False))) - -# quick null scan on the fields the analysis leans on -key = ["TOWN","COUNTY","REAL_FLV","LAND_LV","IMPRV_LV","ACRESGL","CAT","PURPOSE"] -print((parcels[[c for c in key if c in parcels]].isna().mean()*100).round(1)) +# #checking if the columns that I created are present, and then printing the column and then checking both the true and false values of each item +# for c in ["OOSOWNER", "INVESTMENTPROP", "EXEMPT", "VACANTLAND"]: +# if c in parcels: +# print(f"{c:16s} →", dict(parcels[c].value_counts(dropna=False))) +#getting the null rates of each important column in the dataset for informative purposes +# key = ["TOWN","COUNTY","REAL_FLV","LAND_LV","IMPRV_LV","ACRESGL","CAT","PURPOSE"] +# print((parcels[[c for c in key if c in parcels]].isna().mean() * 100).round(1)) ``` making a reusable chloropleth filter ```{python} + +#getting the town boudnaries using rest api. well need this for hte chloropleths, and making the towns gdf from the town_url TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" -towns_gdf = gpd.read_file(TOWN_URL) + +# 1. Download the data robustly using requests +print("Downloading town boundaries...") +r = requests.get(TOWN_URL) +r.raise_for_status() # This will yell if the URL is broken or blocked + +# 2. Parse the JSON using Python's built-in dictionary parser +data = r.json() + +# 3. Read the features into a GeoDataFrame +towns_gdf = gpd.GeoDataFrame.from_features(data["features"]) + +# ArcGIS GeoJSON defaults to EPSG 4326 (WGS84 Lat/Lon). We must set it explicitly +# before you project it to 32145 later! +towns_gdf.set_crs(epsg=4326, inplace=True) + +print(towns_gdf.columns) + towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() towns_gdf = towns_gdf[["TOWN", "geometry"]].copy() +print(f"Successfully loaded {len(towns_gdf)} towns!") + + +#defining the functino for a chloropleth map. +"""values: a Series whose INDEX is TOWN and whose values are the metric to color by.""" # - so we can +#see by map +# Series → 2-col DataFrame so it can be merged onto the polygons by TOWN + + def town_choropleth(values: pd.Series, title, log=False, cmap="viridis"): - """values: a Series indexed by TOWN.""" + + #.values.rename takes the numbers column of the seies and names it "val". the columns are now names TOWN and val + #We now commit a left merge by town v = values.rename("val").reset_index(); v.columns = ["TOWN", "val"] - m = towns_gdf.merge(v, on="TOWN", how="left") + m = towns_gdf.merge(v, on="TOWN", how="left") + #this code is for if we pass in log as a true value if log: + #we cannot take 0 or negative numbers in our logarithmic m.loc[m["val"] <= 0, "val"] = np.nan - fig, ax = plt.subplots(figsize=(8, 11)) + fig, ax = plt.subplots(figsize=(8, 12)) + + #assigning data values - the keywords dictionary kw = dict(column="val", cmap=cmap, legend=True, ax=ax, edgecolor="gray", linewidth=0.3, - missing_kwds={"color": "lightgray", "label": "no data"}) + missing_kwds={"color": "lightgray", "label": "no data"}) + #if logarithmic - if log: - pos = m["val"].dropna() + #getting the true range after dropping nans + pos = m["val"].dropna() + #this line says that the colors should be added in a gradient based on the progression from lowest to highest value. we add a new keyword key of norm for that value kw["norm"] = LogNorm(vmin=pos.min(), vmax=pos.max()) + + #plotting m, towns_gdf, and allthe KW dictionary, which contain the instructions for logarithmic plotting if we send them in. we also output the towns with no values for the inputted considered column. m.plot(**kw); ax.set_title(title); ax.axis("off"); plt.show() - print("unmatched towns:", m["val"].isna().sum()) + print("unmatched towns:", m["val"].isna().sum()) return m ``` before vs after backfill ```{python} import duckdb -con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) +#connect access old table created before I updated the parcel collection method +con = duckdb.connect("Data/_Processed/all_data.duckdb", read_only=True) # the OLD layer lives here -# AFTER — the rebuilt layer +#parcels represents the updated dataset, so new_n, new_valued, and new_sum represent the updated parcels' length, parcel valuue, and sum of value new_n = len(parcels) -new_valued = int((parcels["REAL_FLV"] > 0).sum()) -new_sum = float(parcels.loc[parcels["REAL_FLV"] > 0, "REAL_FLV"].sum()) +new_valued = int((parcels["REAL_FLV"] > 0).sum()) +new_sum = float(parcels.loc[parcels["REAL_FLV"] > 0, "REAL_FLV"].sum()) -# BEFORE — the original statewide layer +#getting the old count, value per parcel, adn sum of total parcels from antiuated dataset old = con.execute(""" SELECT TOWN, COUNT(*) n, - SUM((REAL_FLV > 0)::INT) valued, + SUM((REAL_FLV > 0)::INT) valued, -- ::INT casts bool→0/1 to sum SUM(CASE WHEN REAL_FLV > 0 THEN REAL_FLV ELSE 0 END) val_sum FROM parcels_info GROUP BY TOWN """).df() +#getting the same old calculations old_n, old_valued, old_sum = int(old.n.sum()), int(old.valued.sum()), float(old.val_sum.sum()) +#printing the founds stats of the old and new datasets, with a change in row represented print(f"BEFORE: {old_n:>8,} parcels | {old_valued:>8,} valued ({100*old_valued/old_n:4.1f}%) | ${old_sum:,.0f}") print(f"AFTER : {new_n:>8,} parcels | {new_valued:>8,} valued ({100*new_valued/new_n:4.1f}%) | ${new_sum:,.0f}") print(f"Δ : {new_n-old_n:>+8,} parcels | {new_valued-old_valued:>+8,} valued | ${new_sum-old_sum:+,.0f}") -# --- tag each rebuilt parcel by how the OLD layer treated its town --- -old_all = set(old.TOWN) -old_broken = set(old.loc[old.valued == 0, "TOWN"]) +# --- tag each rebuilt parcel by how the OLD layer treated its town ---------- +# Every row here came from a town download, so "origin" really means: was this town +# already covered in the statewide layer, broken there, or absent entirely? + +#unique problem - we have the old and new datasets, but both also have broken parcels without valuation, like public property and stuff like that +old_all = set(old.TOWN) +old_broken = set(old.loc[old.valued == 0, "TOWN"]) +#quick wading function for all items inputted, if not in old dataset, the data must have been absent, if it was in the old broken set it was broken, and we return a list of what is covered in out statewide data analysis, as "covered in statewide" def origin(t): - if t not in old_all: return "absent_from_statewide" - if t in old_broken: return "restored_by_rebuild" + if t not in old_all: return "not_in_old_statewide" + if t in old_broken: return "found_in_rebuild" return "covered_in_statewide" -parcels["data_origin"] = parcels["TOWN"].map(origin) +#making a new column for the provenance of parcels, either restored or absent, by running our function on town +parcels["data_origin"] = parcels["TOWN"].map(origin) + +#grouping the parcels by data_origin, and also objectid - size is row count per group - grouped by data_origin +#we also sum the real_flv for all with values over 0, meaning informed parcels by_origin = (parcels.assign(valued=parcels["REAL_FLV"] > 0) .groupby("data_origin") - .agg(parcels=("OBJECTID", "size"), + .agg(parcels=("OBJECTID", "size"), valued=("valued", "sum"), - grand_list=("REAL_FLV", lambda s: s[s > 0].sum()))) + grand_list=("REAL_FLV", lambda s: s[s > 0].sum()))) print("\n", by_origin, sep="") -restored = parcels[parcels.data_origin == "restored_by_rebuild"] -print(f"\nRestored {len(restored):,} parcels worth " - f"${restored.loc[restored.REAL_FLV>0,'REAL_FLV'].sum():,.0f}") -print(restored.groupby("TOWN") - .agg(n=("OBJECTID","size"), value=("REAL_FLV", lambda s: s[s>0].sum())) - .sort_values("n", ascending=False).head(10)) - -# spelling-drift guard between the two TOWN keys +#getting the first 15 of the towns that weren't included in the old parcels dataset print("\nin rebuilt, no old-layer match:", sorted(set(parcels.TOWN) - old_all)[:15]) + +#making a plot for the provenance of each particular town's parcels +#merging the towns based on data_origin. im generalizing with a few select towns by dropping all duplicated (namely burlington), but I think its worth it +prov = towns_gdf.merge( + parcels[["TOWN","data_origin"]].drop_duplicates("TOWN"), + on="TOWN", how="left") +#being safe +prov["data_origin"] = prov["data_origin"].fillna("no parcels") + +#making the plot +fig, ax = plt.subplots(figsize=(8, 12)) + +prov.plot(column="data_origin", categorical=True, legend=True, + ax=ax, edgecolor="black", linewidth=0.3) +ax.set_title("Parcel Origin vs. the old statewide layer"); ax.axis("off"); plt.show() + +#outputting towns with no parcels - three are included +print("TOWNS WITH NO PARCELS:") +print(prov[prov["data_origin"] == "no parcels"]["TOWN"].value_counts().rename_axis(None).to_string()) ``` MEDIAN PARCEL VALUE + +#something to consider from these outcomes is that burlington has the most parcels, which inevitably will have some non-expensive ones, where avery's gore has 3. However, most of vermont's wealth/value can be assumed to reside there ```{python} + +#getting the median values of only the parcels where the median is positive AND PRESENT med_val = parcels.loc[parcels["REAL_FLV"] > 0].groupby("TOWN")["REAL_FLV"].median() + +#running the choropleth function with the aggregated medians list, and passing it as a logarithmic scale town_choropleth(med_val, "Median parcel value (REAL_FLV) — log scale", log=True, cmap="viridis") -print("highest:\n", med_val.sort_values(ascending=False).head(8)) -print("lowest:\n", med_val.sort_values().head(8)) +print("highest:\n", med_val.sort_values(ascending=False).head(5)) +print("lowest:\n", med_val.sort_values().head(5)) +print(f"AVERYS GORE parcel count {len(parcels[parcels["TOWN"]=="AVERYS GORE"])}") ``` ```{python} -# EPSG:32145 = VT State Plane (meters); 1 acre = 4046.8564224 m² -parcels["AREA_ACRES_GEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +# Deeded acreage (ACRESGL) is a legal figure; polygon area is what the shape actually covers. +# EPSG:32145 = VT State Plane in METERS — area is only meaningful in a projected CRS, never in 4326. + +#AGRESGL, the grand list acreage of a parcel is for taxation purposes, not actual size. we need to convert it to meters^2 per acre, and need to put it in crs(32145) to project it in meters +parcels["AREA_ACRES_GEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +#making the size aggregation - groupoig by town, and aggreagating by median, in this case size. dropping the nulls. size = (parcels.groupby("TOWN") - .agg(deeded=("ACRESGL", "median"), geom=("AREA_ACRES_GEOM", "median")) + .agg(deeded=("ACRESGL", "median"), + geom=("AREA_ACRES_GEOM", "median")) .dropna()) -print("largest lots:\n", size.sort_values("geom", ascending=False).head(8)) -print("smallest lots:\n", size.sort_values("geom").head(8)) + +#printing the largest and smallest lots of meters squared in vermont, median +print("largest lots:\n", size.sort_values("geom", ascending=False).head(5)) +print("smallest lots:\n", size.sort_values("geom").head(5)) + +#using the correaltion function to see the divergence or deeded acreage and true polygon plot size print("corr deeded vs geom:", parcels[["ACRESGL","AREA_ACRES_GEOM"]].dropna().corr().iloc[0,1].round(3)) @@ -148,44 +248,65 @@ town_choropleth(size["geom"], "Median parcel size (geometry acres) — log scale LAND VS IMPROVEMENT VALUE ```{python} val = parcels["REAL_FLV"] +#using np.where to avoid division by zero - this is the ratio of improvment to total parcels["IMPR_SHARE"] = np.where(val > 0, parcels["IMPRV_LV"].fillna(0) / val, np.nan) -# alt denominator (assessed components) — often cleaner than REAL_FLV +# Alternative denominator: assessed components (LAND+IMPRV). Often cleaner than REAL_FLV, +# which is sometimes listed independently and can disagree with LAND+IMPRV. + +#making a denominator value for land value + improvement value, with both nulls filled to 0. making a new improvement share column den = parcels["LAND_LV"].fillna(0) + parcels["IMPRV_LV"].fillna(0) parcels["IMPR_SHARE_ALT"] = np.where(den > 0, parcels["IMPRV_LV"].fillna(0) / den, np.nan) +#investigating the shape of the improvment share column print(parcels["IMPR_SHARE"].describe()) -print("share > 1 (quirks where IMPRV_LV exceeds REAL_FLV):", int((parcels["IMPR_SHARE"] > 1).sum())) +#share > 1 shouldn't happen if REAL = LAND + IMPRV — a nonzero count flags that assumption breaking +print("share > 1 (divergences where IMPRV_LV exceeds REAL_FLV):", int((parcels["IMPR_SHARE"] > 1).sum())) +#mad share with a min/max range for the sake of the map med_share = parcels.groupby("TOWN")["IMPR_SHARE"].median().clip(0, 1) + + town_choropleth(med_share, "Median improvement share (building ÷ total value)", cmap="RdYlBu_r") -print("most built-up:\n", med_share.sort_values(ascending=False).head(8)) -print("most land-heavy:\n", med_share.sort_values().head(8)) + +#"most built up" would represent the towns with highest improvement valuesm and the flip is the most land valued towns +print("most built-up:\n", med_share.sort_values(ascending=False).head(5)) +print("most land-heavy:\n", med_share.sort_values().head(5)) ``` ```{python} -# PURPOSE already collapses CAT into the urban-rural-relevant buckets you engineered +#using crosstab with normalize = index to make all towns have same 100% sum no matter their total purposes mix = pd.crosstab(parcels["TOWN"], parcels["PURPOSE"], normalize="index").mul(100).round(1) print(mix.head()) -res = mix.get("PRIMARY RESIDENCE", pd.Series(0.0, index=mix.index)) +#using .get to have returned to us a default value +res = mix.get("COMMERCIAL APARTMENTS", pd.Series(0.0, index=mix.index)) + + +#reindex forces these columns to exist (0 if missing), then sum across them per town rural = mix.reindex(columns=["FARM","WOODLAND","SEASONAL PROPERTY"], fill_value=0).sum(axis=1) -print("\nmost residential:\n", res.sort_values(ascending=False).head(10)) +print("\nmost city-like:\n", res.sort_values(ascending=False).head(10)) print("\nmost farm/woodland/seasonal:\n", rural.sort_values(ascending=False).head(10)) +#stacking horizontal bars for the top 12 towns with the most parcels top = parcels["TOWN"].value_counts().head(12).index mix.loc[top].plot(kind="barh", stacked=True, figsize=(9, 7)) -plt.xlabel("% of parcels"); plt.title("PURPOSE mix — 12 largest towns") +plt.xlabel("% of parcels"); plt.title("PURPOSE mix of 12 largest towns") plt.legend(bbox_to_anchor=(1.02, 1)); plt.tight_layout(); plt.show() ``` - +SAVING THE TABLES ```{python} + +#getting the path out for the tables out = Path("Data/parcels/tables"); out.mkdir(parents=True, exist_ok=True) -def present(cols): return [c for c in cols if c in parcels.columns] # tolerate dropped cols +#just making sure +def present(cols): + return [c for c in cols if c in parcels.columns] +#just need to ensure that objectid is in all three tables that make up parcels - the objectid is the join code geom_cols = present(["OBJECTID", "TOWN", "COUNTY", "geometry"]) info_cols = present(["OBJECTID","TOWN","COUNTY","SPAN","PROPTYPE","CAT","CATEGORY","PURPOSE", "DESCPROP","RESCODE","ACRESGL","AREA_ACRES_GEOM","CITYGL","STGL","ADDRESS", @@ -196,21 +317,395 @@ tax_cols = present(["OBJECTID","TOWN","REAL_FLV","HSTED_FLV","NRES_FLV","LAND_L "STATUTE","EXEMPT","EXAMT_HS","EXAMT_NR","UVREDUC_HS","UVREDUC_NR", "GLVAL_HS","GLVAL_NR"]) -geom = parcels[geom_cols].copy() # GeoDataFrame → GeoParquet -info = pd.DataFrame(parcels[info_cols]) # plain (no geometry) -tax = pd.DataFrame(parcels[tax_cols]) +#making a variable for the three tables that we will ship out later. we need to copy the geomcols to preserve active geometry +geom = parcels[geom_cols].copy() +info = parcels[info_cols].copy() +tax = parcels[tax_cols].copy() +# geometry side → GeoParquet via geopandas (this write already worked for you; it's the +# pandas .to_parquet calls that collided, so only those move to DuckDB below) geom.to_parquet(out / "parcels_geom.parquet") -info.to_parquet(out / "parcels_info.parquet", index=False) -tax.to_parquet(out / "parcels_tax.parquet", index=False) +info.to_parquet(out / "parcels_info.parquet") +tax.to_parquet(out / "parcels_tax.parquet") + +# info + tax → Parquet via DuckDB. DuckDB has its own pandas→arrow bridge and never triggers +# the "pandas.period already defined" extension registration that pandas.to_parquet hit. +# w = duckdb.connect() # scratch in-memory connection just for writing +# w.register("info_v", info) # expose the frames to SQL by name +# w.register("tax_v", tax) +# info_path, tax_path = str(out / "parcels_info.parquet"), str(out / "parcels_tax.parquet") +# w.execute(f"COPY (SELECT * FROM info_v) TO '{info_path}' (FORMAT PARQUET)") +# w.execute(f"COPY (SELECT * FROM tax_v) TO '{tax_path}' (FORMAT PARQUET)") print("geom:", geom.shape, "| info:", info.shape, "| tax:", tax.shape) -# --- prove the three still join cleanly on OBJECTID --- -g = gpd.read_parquet(out / "parcels_geom.parquet") +#must change info and tax tables to be read by pd, so that the system doesn't look for metadata that isn't there. +g = gpd.read_parquet(out / "parcels_geom.parquet") i = pd.read_parquet(out / "parcels_info.parquet") -t = pd.read_parquet(out / "parcels_tax.parquet") +t = pd.read_parquet(out / "parcels_tax.parquet") assert g.OBJECTID.is_unique and i.OBJECTID.is_unique and t.OBJECTID.is_unique assert set(g.OBJECTID) == set(i.OBJECTID) == set(t.OBJECTID), "OBJECTID sets differ!" three = g.merge(i, on="OBJECTID").merge(t, on="OBJECTID", suffixes=("_info","_tax")) print("3-way join rows:", len(three), "== parcel count?", len(three) == len(parcels)) +``` + + +#NOW SEEKING TO ADD COMPLEXITY TO PLOTS, STARTING WITH CHECKING IF BURLINGTON IS THERE +```{python} +import seaborn as sns +#making sure that they is a column for acresage in meters squared +#also, making a column for the value per acre of each parcel +if "AREAACRESGEOM" not in parcels: + parcels["AREAACRESGEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +parcels["ACREVALUE"] = np.where( + (parcels.REAL_FLV > 0) & (parcels.AREAACRESGEOM > 0), + parcels.REAL_FLV / parcels.AREAACRESGEOM, np.nan) + +#here are the jurisdictions in chittenden county +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +cc = parcels[parcels.TOWN.isin(CHITTENDEN)].copy() + +#making sure that we can reliably get the towns in chittenden county reliably - +#saving chittenden county and it's df as cc +check = cc.groupby("TOWN").agg( + n=("OBJECTID","size"), + valued=("REAL_FLV", lambda s: int((s>0).sum())), + pct_valued=("REAL_FLV", lambda s: round(100*(s>0).mean(),1)), + med_val=("REAL_FLV", lambda s: s[s>0].median()), +).sort_values("n", ascending=False) +print(check) +``` + +"Marquee 1 — distributions per town (the single biggest depth upgrade). Ordered by median so the urban-rural gradient reads top-to-bottom, and you'll see which towns are bimodal:" +```{python} +#checking for the logarithmic progression of all property value, and making sure that the towns are in order of lowest to highest median parcel size +cc["logv"] = np.log10(cc.REAL_FLV.where(cc.REAL_FLV > 0)) +order = cc.groupby("TOWN").logv.median().sort_values().index + +#making the violin plot +plt.figure(figsize=(9, 10)) + +#checking towns vs logarithmic scaling of median parcel value per town +sns.violinplot(data=cc, y="TOWN", x="logv", order=order, + density_norm="width", cut=0) +plt.xlabel("log10 parcel value"); plt.title("Value distribution by Chittenden town") +plt.tight_layout(); plt.show() + +#DOING THE EXACT SAME THING WITH PARCEL ACRE SIZE +cc["logv_acres"] = np.log10(cc.AREAACRESGEOM.where(cc.AREAACRESGEOM > 0)) +order = cc.groupby("TOWN").logv_acres.median().sort_values().index +plt.figure(figsize=(9, 10)) +sns.violinplot(data=cc, y="TOWN", x="logv_acres", order=order, + density_norm="width", cut=0, color = "orange") +plt.xlabel("log10 parcel value"); plt.title("PARCELS ACRE distribution by Chittenden town") +plt.tight_layout(); plt.show() +``` + +"Marquee 2 — parcel-level $/acre gradient (breaks out of town aggregation entirely). This is the "more than aggregated views" you asked for — Burlington→suburb→rural fringe shows as a continuous surface:" +```{python} +#DROPPING NULLS +cc_proj = cc.dropna(subset=["ACREVALUE"]).to_crs(32145) + +#getting rid of absurd outliers - first and 99th percentiles +lo, hi = cc_proj.ACREVALUE.quantile([0.01, 0.99]) + +#clipping, and making figure logarithmicly scaled +cc_proj["vpa_clip"] = cc_proj.ACREVALUE.clip(lo, hi) +fig, ax = plt.subplots(figsize=(11, 11)) +cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, + linewidth=0, norm=LogNorm(vmin=lo, vmax=hi)) +ax.set_title("Chittenden parcels — value per acre (log)"); ax.axis("off"); plt.show() +``` + +ideas for the following cell + +Absentee & seasonal geography: cc.groupby("TOWN").agg(pct_oos=("OOSOWNER","mean"), pct_seasonal=("PURPOSE", lambda s:(s=="SEASONAL PROPERTY").mean())) — Charlotte/Shelburne lakefront vs. the urban core is the contrast. +Development pressure: low IMPR_SHARE and high land value = underbuilt/speculative. Flag parcels where LAND_LV > IMPRV_LV in the core towns. +Urban/suburb/fringe tiers: hand-group the towns (core: Burlington/Winooski/S.Burlington; suburb: Williston/Essex/Colchester; fringe: Charlotte/Huntington/Bolton/Buels Gore) and compare distributions across tiers — a cleaner story than 19 separate towns. +```{python} +#cc_proj = cc.dropna(subset=["ACREVALUE"]).to_crs(32145) + +#printing the groupyby of out of state owners and purpose +check = cc.groupby("TOWN").agg( + parcel_count =("OBJECTID","size"), + pct_oos=("OOSOWNER","mean"), + pct_seasonal=("PURPOSE", lambda s:(s=="SEASONAL PROPERTY").mean()), +).sort_values("pct_seasonal", ascending=False) +#print(check) + +#checking where the sum of parcels wehre land_lv > improvement_value +land_based = cc[cc["IMPR_SHARE"]<0.50] +#print(cc["TOWN"].value_counts()) +#print(land_based["TOWN"].value_counts()) + + +#print(cc[["IMPR_SHARE", "TOWN"]].value_counts()) + +URBANCC = ["BURLINGTON", "WINOOSKI", "SOUTH BURLINGTON"] +SUBURBANCC = ["COLCHESTER", "WILLISTON", "ESSEX JUNCTION", "ESSEX"] +LAKEFRONT = ["CHARLOTTE", "SHELBURNE", "COLCHESTER"] +RURALCC = [] +for item in cc["TOWN"].unique(): + if item not in (URBANCC) and item not in (SUBURBANCC): + RURALCC.append(item) +print(RURALCC) + +#print(land_based.columns) +# a = len(land_based) + +# urb = land_based["TOWN"].isin(URBANCC).sum() +# sub = land_based["TOWN"].isin(SUBURBANCC).sum() +# rur = land_based["TOWN"].isin(RURALCC).sum() + +# print(f"{urb/a:.2f} {sub/a:.2f} {rur/a:.2f}") + +# print(land_based["TOWN"].isin(URBANCC).sum() / cc["TOWN"].isin(URBANCC).sum()) + + +# print(cc[cc["PURPOSE"]=="COMMERCIAL APARTMENTS"]["TOWN"].value_counts()) +# print(cc[cc["SOUTH BURLINGTON"] == "BUELS GORE"]["OOSOWNER"].value_counts()) + + +# fig, ax = plt.subplots(figsize=(11, 11)) +# cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, +# linewidth=0) +# ax.set_title("Chittenden parcels — seasonal vs not"); ax.axis("off"); plt.show() + +#makinga new dictionary for each town in urban, suburban, and rural, and adding those as keys to the towns int he dictionary +tier_map = {t: "urban" for t in URBANCC} +tier_map.update({t: "suburban" for t in SUBURBANCC}) +tier_map.update({t: "lakefront" for t in LAKEFRONT if t not in tier_map}) +#making rural the default of the dictionary +for t in cc["TOWN"].unique(): + tier_map.setdefault(t, "rural") + +#making a town groupby with size, the out of state owner percentage, on average, the seasonal property percentage, also on average, adn the percent underbuilt, where the ratio of improvement value to land value is less than 50%. Also adding the median value or tbe parcel at hand +town = cc.groupby("TOWN").agg( + n=("OBJECTID", "size"), + pct_oos=("OOSOWNER", "mean"), + pct_seasonal=("PURPOSE", lambda s: (s == "SEASONAL PROPERTY").mean()), + pct_underbuilt=("IMPR_SHARE", lambda s: (s < 0.50).mean()), + med_val=("REAL_FLV", lambda s: s[s > 0].median()), +) +#tier is urban, suburban, lakefront, and rural +town["tier"] = town.index.map(tier_map) + +#making the an aggregated groupby by tier, the newly created measuring system by town type, with the number of entries, the parcel count, the average of each out of state ownership, the average seasonal percentage, the average underbuilt (land is more valuable than improvement), and the mdian value of the parcel +tier_summary = (town.groupby("tier") + .apply(lambda g: pd.Series({ + "towns": len(g), + "parcels": int(g.n.sum()), + "pct_oos": np.average(g.pct_oos, weights=g.n), + "pct_seasonal": np.average(g.pct_seasonal, weights=g.n), + "pct_underbuilt": np.average(g.pct_underbuilt, weights=g.n), + "med_val": g.med_val.median(), + })) + .reindex(["urban", "suburban", "lakefront", "rural"])) +#getting a summary +print(tier_summary.round(3)) + +#making the figure +import matplotlib.pyplot as plt +colors = {"urban": "#d62728", "suburban": "#ff7f0e", "lakefront": "#1f77b4", "rural": "#2ca02c"} +fig, ax = plt.subplots(figsize=(8, 4)) + +#for item tier in ("urban", "suburban", etc) and g for twown, make the scatterplot and finding percentage +for tier, g in town.groupby("tier"): + ax.scatter(g.pct_underbuilt * 100, g.pct_oos * 100, + s=g.n / 8, alpha=0.7, color=colors[tier], label=tier, edgecolor="white", linewidth=0.5) + +#labelling every town +for name, r in town.iterrows(): + ax.annotate(name.title(), (r.pct_underbuilt * 100, r.pct_oos * 100), + fontsize=7, alpha=0.6, xytext=(3, 3), textcoords="offset points") + + +#getting the chittenden county average, to place with accuracy every town against the average +ax.axhline(town.pct_oos.mean() * 100, ls="--", color="gray", lw=0.8) +ax.axvline(town.pct_underbuilt.mean() * 100, ls="--", color="gray", lw=0.8) +ax.set_xlabel("% of parcels underbuilt (improvement < 50% of value)") +ax.set_ylabel("% out-of-state owned") +ax.set_title("Chittenden: absentee ownership vs. development pressure\n(dot size = parcel count)") +ax.legend(title="tier"); plt.tight_layout(); plt.show() + +``` + +```{python} +#making a gini function. +def gini(x): + #sorting the inputted array into order - getting rid of nan and negarive numbers + x = np.sort(np.asarray(x, float)); x = x[~np.isnan(x)]; x = x[x >= 0] + #making sure of the ability to handle an empty array or the sum of the array equalling 0 + if x.size == 0 or x.sum() == 0: return np.nan + #n is the size, and the index adds the correct index of the sorted list + n = x.size; idx = np.arange(1, n+1) + #returning the value of the gini coefficient + return (2*np.sum(idx*x)/(n*x.sum())) - (n+1)/n + + +#making a function that gets the county metrics, with a dataframe as an input +#first we get hte real value of the parcels that are over 0, then return a bunch of pertinent measuremtns of the dataframe +def county_metrics(df): + v = df.REAL_FLV[df.REAL_FLV > 0] + return pd.Series({ + "median_value": v.median(), + "median_lot_acres": df.AREAACRESGEOM.median(), + "median_val_per_acre": df.ACREVALUE.median(), + "impr_share": df.IMPR_SHARE.median(), + "pct_residential": (df.PURPOSE == "PRIMARY RESIDENCE").mean()*100, + "pct_farm_wood": df.PURPOSE.isin(["FARM","WOODLAND"]).mean()*100, + "pct_seasonal": (df.PURPOSE == "SEASONAL PROPERTY").mean()*100, + "pct_oos_owner": df.OOSOWNER.mean()*100, + "value_gini": gini(v), + }) +#applying the county metrics to the parcels dataset as it is grouped by county +cm = parcels.groupby("COUNTY").apply(county_metrics) +#standarsizing (z score) metrics so that they can be compared across counties +z = (cm - cm.mean()) / cm.std() +#order the z scores by median value per acre, which is a good metric of urban-level +z = z.loc[cm.median_val_per_acre.sort_values(ascending=False).index] +#plotting the figure, and heatmap to show standarsization +plt.figure(figsize=(10, 7)) +sns.heatmap(z, cmap="RdBu_r", center=0, cbar_kws={"label": "z-score"}) +plt.title("County profiles (standardized)"); plt.tight_layout(); plt.show() +#printing the values +print(cm.round(1)) +``` + + +```{python} +#getting the parcels where the value is over 0, and scaling it logarithmically +d = parcels.loc[parcels.REAL_FLV > 0, ["COUNTY","REAL_FLV"]].copy() +d["logv"] = np.log10(d.REAL_FLV) +#getting the average, and grouping by county +grand = d.logv.mean() +grp = d.groupby("COUNTY").logv + +#ss is sum of swaures. we are calculating the variance between the parcels within, with respect to property value. with this in mind, 93.2 percent of value is attributable to county, with 6.2 not due to county +ss_between = (grp.count() * (grp.mean() - grand)**2).sum() +ss_within = ((d.logv - grp.transform("mean"))**2).sum() +ss_total = ((d.logv - grand)**2).sum() +print(f"between-county: {ss_between/ss_total:.1%} | within-county: {ss_within/ss_total:.1%}") +``` + +conclusive urban rural divide +```{python} +#printing the towns by town acreage by -- look at what are light yellow +vpa = parcels.groupby("TOWN").ACREVALUE.median() +town_choropleth(vpa, "Median value per acre ($/acre) — log", log=True, cmap="inferno") +print(vpa.sort_values(ascending = False)) +print(vpa["CHARLOTTE"]) +``` + + + +performing machine learning on the data +```{python} +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA +from sklearn.cluster import KMeans + +#getting the town areas in kilometers squared +ta = towns_gdf.to_crs(32145); ta["area_km2"] = ta.geometry.area / 1e6 +town_area = ta.set_index("TOWN")["area_km2"] + +#making a group by for the median value, the median acreage lot size, the value per acre, the improvement share, the percent residence, and the number of parcels +feat = parcels.groupby("TOWN").agg( + med_val=("REAL_FLV", lambda s: s[s>0].median()), + med_lot=("AREAACRESGEOM", "median"), + val_per_acre=("ACREVALUE", "median"), + impr_share=("IMPR_SHARE", "median"), + pct_res=("PURPOSE", lambda s: (s=="PRIMARY RESIDENCE").mean()), + n_parcels=("OBJECTID", "size"), +) +#making a custom density column, made from the number of parcels over the area +feat["parcel_density"] = feat.n_parcels / town_area.reindex(feat.index) + + +feat_cols = ["med_val","med_lot","val_per_acre","impr_share","pct_res","parcel_density"] +# +F = feat[feat_cols].copy() +for c in ["med_val","med_lot","val_per_acre","parcel_density"]: + #applying a logarithnic value to the skewed values + F[c] = np.log10(F[c].replace(0, np.nan)) +#dropping null values +F = F.dropna() +#standardizing all values for future disvovery +X = StandardScaler().fit_transform(F) + +#running pca algorithm to try and discover what drives "urbanity" +pca = PCA(n_components=2).fit(X) +#getting the strongest indicator and saving it as "urban_rural" +F["urban_rural"] = pca.transform(X)[:, 0] +#for better understanding, making the urban share more positive; creating that correaltino +if np.corrcoef(F["urban_rural"], F["impr_share"])[0,1] < 0: + F["urban_rural"] *= -1 + +#printing findings +print("PC1 explains", f"{pca.explained_variance_ratio_[0]:.1%}", "of variance [in urbanity]") +print(pd.Series(pca.components_[0], index=feat_cols).sort_values()) # loadings = what "urban" means + +F["cluster"] = KMeans(n_clusters=5, n_init=10, random_state=0).fit_predict(X) +print(feat.loc[F.index].assign(cluster=F.cluster).groupby("cluster").median()) # read the typology + +town_choropleth(F["urban_rural"], "Urban–rural index (PC1)", cmap="Spectral_r") +``` + + +```{python} +#loading in the population dataset +pop = pd.read_csv("/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv") +pop["TOWN"] = pop["Town"].str.upper() +pop = pop.set_index("TOWN")["year2020"] + +#seeing what towns are missing - discrepancy between our parcels dataset and population dataset +missing = [t for t in F.index if t not in pop.index] +print(f"towns in F with no population match ({len(missing)}):", missing[:15]) + +#using reindex to keep whatever towns are overlapping (hopefully a lot) in the same order +popdens = np.log10((pop.reindex(F.index) / town_area.reindex(F.index)).replace(0, np.nan)) + +pop_sort = popdens.sort_values(ascending=False) + +# align both series and drop any NaN pairs so the r is computed on the same towns you plot +#making a variable for the valid towns. then, getting the correlation of the urban area towns and then checking the index vs population density, and getting the r value +valid = pd.DataFrame({"urban_rural": F["urban_rural"], "log_popdens": popdens}).dropna() +r = valid["urban_rural"].corr(valid["log_popdens"]) +print(f"index vs. log population density r = {r:.3f} (n = {len(valid)} towns)") + +#creating the scatterplot +import matplotlib.pyplot as plt +fig, ax = plt.subplots(figsize=(8, 7)) +#plotting the log population density vs the urban_rural found PCA +ax.scatter(valid["log_popdens"], valid["urban_rural"], alpha=0.6, edgecolor="white", linewidth=0.5) + +#finding the towns to label by finding hte ones that dont seem to conformwith the lien of best fit +resid = valid["urban_rural"] - np.poly1d(np.polyfit(valid["log_popdens"], valid["urban_rural"], 1))(valid["log_popdens"]) +to_label = valid.loc[resid.abs().sort_values(ascending=False).head(8).index] + +#adding the annotation to each item in the to_label +for name, row in to_label.iterrows(): + ax.annotate(name.title(), (row["log_popdens"], row["urban_rural"]), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") +for name, density_val in pop_sort.head(3).items(): + if name not in to_label.index: + urban_score = F.loc[name, "urban_rural"] + ax.annotate(name.title(), (density_val, urban_score), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") +for name, density_val in pop_sort.tail().items(): + if name not in to_label.index: + urban_score = F.loc[name, "urban_rural"] + ax.annotate(name.title(), (density_val, urban_score), + fontsize=8, alpha=0.7, xytext=(3, 3), textcoords="offset points") + +m, b = np.polyfit(valid["log_popdens"], valid["urban_rural"], 1) +xs = np.linspace(valid["log_popdens"].min(), valid["log_popdens"].max(), 50) +ax.plot(xs, m*xs + b, "--", color="gray", lw=1) +ax.set_xlabel("log₁₀ population density (people / km²)") +ax.set_ylabel("urban–rural index (PC1)") +ax.set_title(f"Parcel-derived urban index vs. census density (r = {r:.2f})") +plt.tight_layout(); plt.show() +``` + +IDEA FOR FUTURE Next-level (the portfolio standout you probably haven't done): spatial autocorrelation. libpysal + esda give you Moran's I (is value spatially clustered at all?) and LISA (statistically significant hot/cold spots that ignore town boundaries). uv pip install libpysal esda, build a spatial weights matrix on town or parcel centroids, and you can map "significant high-value clusters" as a defensible statistical object rather than an eyeballed choropleth. +```{python} ``` \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 5dfba9b3..06252e11 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "pyogrio>=0.12.1", "pyyaml>=6.0.3", "requests>=2.34.2", + "seaborn>=0.13.2", "shapely>=2.1.2", "uvicorn>=0.49.0", "xycmap>=1.0.1", diff --git a/backend/uv.lock b/backend/uv.lock index 3f877bcc..d6684aab 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -88,6 +88,7 @@ dependencies = [ { name = "pyogrio" }, { name = "pyyaml" }, { name = "requests" }, + { name = "seaborn" }, { name = "shapely" }, { name = "uvicorn" }, { name = "xycmap" }, @@ -120,6 +121,7 @@ requires-dist = [ { name = "pyogrio", specifier = ">=0.12.1" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.34.2" }, + { name = "seaborn", specifier = ">=0.13.2" }, { name = "shapely", specifier = ">=2.1.2" }, { name = "uvicorn", specifier = ">=0.49.0" }, { name = "xycmap", specifier = ">=1.0.1" }, @@ -1772,6 +1774,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, ] +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, +] + [[package]] name = "shapely" version = "2.1.2" From bcbb45b8a9d11025633383bb580f90cea1558046 Mon Sep 17 00:00:00 2001 From: Driedupisaac Date: Thu, 27 Aug 2026 13:00:23 -0400 Subject: [PATCH 12/12] add parcels collection, cleaning, and ML exploration notebook --- backend/data_cleaning/clean_parcels.py | 1048 +++++++++++++++++ backend/data_collection/parcels.py | 364 ++++++ backend/notebooks/parcels/parcels_ml.qmd | 270 +++++ .../notebooks/parcels/table_build_final.qmd | 77 +- backend/uv.lock | 160 +++ 5 files changed, 1913 insertions(+), 6 deletions(-) create mode 100644 backend/data_cleaning/clean_parcels.py create mode 100644 backend/data_collection/parcels.py create mode 100644 backend/notebooks/parcels/parcels_ml.qmd diff --git a/backend/data_cleaning/clean_parcels.py b/backend/data_cleaning/clean_parcels.py new file mode 100644 index 00000000..740e2b0d --- /dev/null +++ b/backend/data_cleaning/clean_parcels.py @@ -0,0 +1,1048 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-8-24 +**Description**: + Data cleaning script for the raw `parcels` table in the DuckLake + Run with: +python -m ETL.data_cleaning.clean_parcels +""" + +from pathlib import Path + +import geopandas as gpd +import numpy as np +import pandas as pd +from datastore.lake_build import con +from shapely.geometry import MultiPolygon + + +## LOAD SPATIAL EXTENSION FUNCTION -------------------- +def _load_spatial() -> None: + """ + Load the spatial extension, installing it first if necessary. + """ + try: + con.execute("""--sql LOAD spatial""") + except Exception: + con.execute("""--sql INSTALL spatial""") + con.execute("""--sql LOAD spatial""") + + +def make_tables(parcels): + out = Path("Data/parcels/tables") + out.mkdir(parents=True, exist_ok=True) + + def present(cols): + return [c for c in cols if c in parcels.columns] + + geom_cols = present(["OBJECTID", "TOWN", "COUNTY", "geometry"]) + info_cols = present( + [ + "OBJECTID", + "TOWN", + "COUNTY", + "SPAN", + "PROPTYPE", + "CAT", + "CATEGORY", + "PURPOSE", + "DESCPROP", + "RESCODE", + "ACRESGL", + "AREAACRESGEOM", + "CITYGL", + "STGL", + "ADDRESS", + "SOURCENAME", + "MATCHSTAT", + "TNAME", + "INVESTMENTPROP", + "VACANTLAND", + "OOSOWNER", + "data_origin", + "EDITOR", + "EDITDATE", + ] + ) + tax_cols = present( + [ + "OBJECTID", + "TOWN", + "REAL_FLV", + "HSTED_FLV", + "NRES_FLV", + "LAND_LV", + "IMPRV_LV", + "IMPR_SHARE", + "EQUIPVAL", + "EQUIPCODE", + "INVENVAL", + "HSDECL", + "VETEXAMT", + "EXPDESC", + "STATUTE", + "EXEMPT", + "EXAMT_HS", + "EXAMT_NR", + "UVREDUC_HS", + "UVREDUC_NR", + "GLVAL_HS", + "GLVAL_NR", + ] + ) + + geom = parcels[geom_cols].copy() + info = parcels[info_cols].copy() + tax = parcels[tax_cols].copy() + + geom.to_parquet(out / "parcels_geom.parquet") + info.to_parquet(out / "parcels_info.parquet") + tax.to_parquet(out / "parcels_tax.parquet") + # edit this for later - where should I store them + g = gpd.read_parquet(out / "parcels_geom.parquet") + i = pd.read_parquet(out / "parcels_info.parquet") + t = pd.read_parquet(out / "parcels_tax.parquet") + + +def register_parcels(gdf): + df = pd.DataFrame( + gdf.to_crs(4326).assign(geometry=gdf.to_crs(4326).geometry.to_wkb()) + ) + con.register("parcels_raw", df) + # returning the gdf for the make tables + + +def build_parcels(): + con.execute("""--sql + CREATE OR REPLACE VIEW parcels AS + SELECT ST_GeomFromWKB(geometry) AS geometry, OBJECTID, TOWN, COUNTY, SPAN, PROPTYPE, CAT, CATEGORY, PURPOSE, DESCPROP, RESCODE, ACRESGL, AREAACRESGEOM, CITYGL, STGL, ADDRESS, SOURCENAME, MATCHSTAT, TNAME, INVESTMENTPROP, VACANTLAND, OOSOWNER, data_origin, EDITOR, EDITDATE, REAL_FLV, HSTED_FLV, NRES_FLV, LAND_LV, IMPRV_LV, IMPR_SHARE, EQUIPVAL, EQUIPCODE, INVENVAL, HSDECL, VETEXAMT, EXPDESC, STATUTE, EXEMPT, EXAMT_HS, EXAMT_NR, UVREDUC_HS, UVREDUC_NR, GLVAL_HS, GLVAL_NR + FROM parcels_raw + """) + + +def add_to_lake(): + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_geom AS + SELECT * + FROM geom + """ + ) + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_info AS + SELECT * + FROM info + """ + ) + con.execute( + """--sql + CREATE OR REPLACE TABLE lake.CLEANED.cleaned_tax AS + SELECT * + FROM tax + """ + ) + + +def clean(): + _load_spatial() + gdf = load_helper_and_clean() + register_parcels(gdf) + build_parcels() + return gdf + + +def main(): + make_tables(clean()) + add_to_lake() + + +def load_helper_and_clean(backfill): + vermont_towns_from_parcels = [ + "Addison", + "Albany", + "Alburgh", + "Andover", + "Arlington", + "Athens", + "Averill", + "Averys Gore", + "Bakersfield", + "Baltimore", + "Barnard", + "Barnet", + "Barre City", + "Barre Town", + "Barton", + "Belvidere", + "Bennington", + "Benson", + "Berkshire", + "Berlin", + "Bethel", + "Bloomfield", + "Bolton", + "Bradford", + "Braintree", + "Brandon", + "Brattleboro", + "Bridgewater", + "Bridport", + "Brighton", + "Bristol", + "Brookfield", + "Brookline", + "Brownington", + "Brunswick", + "Buels Gore", + "Burke", + "Burlington", + "Cabot", + "Calais", + "Cambridge", + "Canaan", + "Castleton", + "Cavendish", + "Charleston", + "Charlotte", + "Chelsea", + "Chester", + "Chittenden", + "Clarendon", + "Colchester", + "Concord", + "Corinth", + "Cornwall", + "Coventry", + "Craftsbury", + "Danby", + "Danville", + "Derby", + "Dorset", + "Dover", + "Dummerston", + "Duxbury", + "East Haven", + "East Montpelier", + "Eden", + "Elmore", + "Enosburgh", + "Essex Junction", + "Essex", + "Fair Haven", + "Fairfax", + "Fairfield", + "Fairlee", + "Fayston", + "Ferdinand", + "Ferrisburgh", + "Fletcher", + "Franklin", + "Georgia", + "Glastenbury", + "Glover", + "Goshen", + "Grafton", + "Granby", + "Grand Isle", + "Granville", + "Greensboro", + "Groton", + "Guildhall", + "Guilford", + "Halifax", + "Hancock", + "Hardwick", + "Hartford", + "Hartland", + "Highgate", + "Hinesburg", + "Holland", + "Hubbardton", + "Huntington", + "Hyde Park", + "Ira", + "Irasburg", + "Isle La Motte", + "Jamaica", + "Jay", + "Jericho", + "Johnson", + "Killington", + "Kirby", + "Landgrove", + "Leicester", + "Lemington", + "Lewis", + "Lincoln", + "Londonderry", + "Lowell", + "Ludlow", + "Lunenburg", + "Lyndon", + "Maidstone", + "Manchester", + "Marlboro", + "Marshfield", + "Mendon", + "Middlebury", + "Middlesex", + "Middletown Springs", + "Milton", + "Monkton", + "Montgomery", + "Montpelier", + "Moretown", + "Morgan", + "Morristown", + "Mount Holly", + "Mount Tabor", + "New Haven", + "Newark", + "Newbury", + "Newfane", + "Newport City", + "Newport Town", + "North Hero", + "Northfield", + "Norton", + "Norwich", + "Orange", + "Orwell", + "Panton", + "Pawlet", + "Peacham", + "Peru", + "Pittsfield", + "Pittsford", + "Plainfield", + "Plymouth", + "Pomfret", + "Poultney", + "Pownal", + "Proctor", + "Putney", + "Randolph", + "Reading", + "Readsboro", + "Richford", + "Richmond", + "Ripton", + "Rochester", + "Rockingham", + "Roxbury", + "Royalton", + "Rupert", + "Rutland City", + "Rutland Town", + "Ryegate", + "Saint Albans City", + "Saint Albans Town", + "Saint George", + "Saint Johnsbury", + "Salisbury", + "Sandgate", + "Searsburg", + "Shaftsbury", + "Sharon", + "Sheffield", + "Shelburne", + "Sheldon", + "Shoreham", + "Shrewsbury", + "Somerset", + "South Burlington", + "South Hero", + "Springfield", + "Stamford", + "Stannard", + "Starksboro", + "Stockbridge", + "Stowe", + "Strafford", + "Stratton", + "Sudbury", + "Sunderland", + "Sutton", + "Swanton", + "Thetford", + "Tinmouth", + "Topsham", + "Townshend", + "Troy", + "Tunbridge", + "Underhill", + "Vergennes", + "Vernon", + "Vershire", + "Victory", + "Waitsfield", + "Walden", + "Wallingford", + "Waltham", + "Wardsboro", + "Warners Grant", + "Warren Gore", + "Warren", + "Washington", + "Waterbury", + "Waterford", + "Waterville", + "Weathersfield", + "Wells", + "West Fairlee", + "West Haven", + "West Rutland", + "West Windsor", + "Westfield", + "Westford", + "Westminster", + "Westmore", + "Weston", + "Weybridge", + "Wheelock", + "Whiting", + "Whitingham", + "Williamstown", + "Williston", + "Wilmington", + "Windham", + "Windsor", + "Winhall", + "Winooski", + "Wolcott", + "Woodbury", + "Woodford", + "Woodstock", + "Worcester", + ] + + ADDISON = [ + "ADDISON", + "BRIDPORT", + "BRISTOL", + "CORNWALL", + "FERRISBURGH", + "GOSHEN", + "GRANVILLE", + "HANCOCK", + "LEICESTER", + "LINCOLN", + "MIDDLEBURY", + "MONKTON", + "NEW HAVEN", + "ORWELL", + "PANTON", + "RIPTON", + "SALISBURY", + "SHOREHAM", + "STARKSBORO", + "WALTHAM", + "WEYBRIDGE", + "WHITING", + "VERGENNES", + ] + BENNINGTON = [ + "ARLINGTON", + "BENNINGTON", + "DORSET", + "GLASTENBURY", + "LANDGROVE", + "MANCHESTER", + "PERU", + "POWNAL", + "READSBORO", + "RUPERT", + "SANDGATE", + "SEARSBURG", + "SHAFTSBURY", + "STAMFORD", + "SUNDERLAND", + "WINHALL", + "WOODFORD", + ] + CALEDONIA = [ + "BARNET", + "BURKE", + "DANVILLE", + "GROTON", + "HARDWICK", + "KIRBY", + "LYNDON", + "NEWARK", + "PEACHAM", + "RYEGATE", + "SAINT JOHNSBURY", + "SHEFFIELD", + "STANNARD", + "SUTTON", + "WALDEN", + "WATERFORD", + "WHEELOCK", + ] + CHITTENDEN = [ + "BOLTON", + "CHARLOTTE", + "COLCHESTER", + "ESSEX", + "ESSEX JUNCTION", + "HINESBURG", + "HUNTINGTON", + "JERICHO", + "MILTON", + "RICHMOND", + "SAINT GEORGE", + "SHELBURNE", + "UNDERHILL", + "WESTFORD", + "WILLISTON", + "BURLINGTON", + "WINOOSKI", + "BUELS GORE", + "SOUTH BURLINGTON", + ] + ESSEX = [ + "AVERILL", + "BLOOMFIELD", + "BRIGHTON", + "BRUNSWICK", + "CANAAN", + "CONCORD", + "EAST HAVEN", + "FERDINAND", + "GRANBY", + "GUILDHALL", + "LEMINGTON", + "LEWIS", + "LUNENBURG", + "MAIDSTONE", + "NORTON", + "VICTORY", + "AVERYS GORE", + "WARNERS GRANT", + "WARREN GORE", + ] + FRANKLIN = [ + "BAKERSFIELD", + "BERKSHIRE", + "ENOSBURGH", + "FAIRFAX", + "FAIRFIELD", + "FLETCHER", + "FRANKLIN", + "GEORGIA", + "HIGHGATE", + "MONTGOMERY", + "RICHFORD", + "SAINT ALBANS CITY", + "SAINT ALBANS TOWN", + "SHELDON", + "SWANTON", + ] + GRAND_ISLE = ["ALBURGH", "GRAND ISLE", "ISLE LA MOTTE", "NORTH HERO", "SOUTH HERO"] + LAMOILLE = [ + "BELVIDERE", + "CAMBRIDGE", + "EDEN", + "ELMORE", + "HYDE PARK", + "JOHNSON", + "MORRISTOWN", + "STOWE", + "WATERVILLE", + "WOLCOTT", + ] + ORANGE = [ + "BRADFORD", + "BRAINTREE", + "BROOKFIELD", + "CHELSEA", + "CORINTH", + "FAIRLEE", + "NEWBURY", + "ORANGE", + "RANDOLPH", + "STRAFFORD", + "THETFORD", + "TOPSHAM", + "TUNBRIDGE", + "VERSHIRE", + "WASHINGTON", + "WEST FAIRLEE", + "WILLIAMSTOWN", + ] + ORLEANS = [ + "ALBANY", + "BARTON", + "BROWNINGTON", + "CHARLESTON", + "COVENTRY", + "CRAFTSBURY", + "DERBY", + "GLOVER", + "GREENSBORO", + "HOLLAND", + "IRASBURG", + "JAY", + "LOWELL", + "MORGAN", + "NEWPORT CITY", + "NEWPORT TOWN", + "TROY", + "WESTFIELD", + "WESTMORE", + ] + RUTLAND = [ + "BENSON", + "BRANDON", + "CASTLETON", + "CHITTENDEN", + "CLARENDON", + "DANBY", + "FAIR HAVEN", + "HUBBARDTON", + "IRA", + "KILLINGTON", + "MENDON", + "MIDDLETOWN SPRINGS", + "MOUNT HOLLY", + "MOUNT TABOR", + "PAWLET", + "PITTSFIELD", + "PITTSFORD", + "POULTNEY", + "PROCTOR", + "RUTLAND TOWN", + "RUTLAND CITY", + "SHREWSBURY", + "SUDBURY", + "TINMOUTH", + "WALLINGFORD", + "WELLS", + "WEST HAVEN", + "WEST RUTLAND", + ] + WASHINGTON = [ + "BARRE TOWN", + "BERLIN", + "CABOT", + "CALAIS", + "DUXBURY", + "EAST MONTPELIER", + "FAYSTON", + "MARSHFIELD", + "MIDDLESEX", + "MORETOWN", + "NORTHFIELD", + "PLAINFIELD", + "ROXBURY", + "WAITSFIELD", + "WARREN", + "WATERBURY", + "WOODBURY", + "WORCESTER", + "BARRE CITY", + "MONTPELIER", + ] + WINDHAM = [ + "ATHENS", + "BRATTLEBORO", + "BROOKLINE", + "DOVER", + "DUMMERSTON", + "GRAFTON", + "GUILFORD", + "HALIFAX", + "JAMAICA", + "LONDONDERRY", + "MARLBORO", + "NEWFANE", + "PUTNEY", + "ROCKINGHAM", + "SOMERSET", + "STRATTON", + "TOWNSHEND", + "VERNON", + "WARDSBORO", + "WESTMINSTER", + "WHITINGHAM", + "WILMINGTON", + "WINDHAM", + ] + WINDSOR = [ + "ANDOVER", + "BALTIMORE", + "BARNARD", + "BETHEL", + "BRIDGEWATER", + "CAVENDISH", + "CHESTER", + "HARTFORD", + "HARTLAND", + "LUDLOW", + "NORWICH", + "PLYMOUTH", + "POMFRET", + "READING", + "ROCHESTER", + "ROYALTON", + "SHARON", + "SPRINGFIELD", + "STOCKBRIDGE", + "WEATHERSFIELD", + "WEST WINDSOR", + "WESTON", + "WINDSOR", + "WOODSTOCK", + ] + counties_key = [ + ADDISON, + BENNINGTON, + CALEDONIA, + CHITTENDEN, + ESSEX, + FRANKLIN, + GRAND_ISLE, + LAMOILLE, + ORANGE, + ORLEANS, + RUTLAND, + WASHINGTON, + WINDHAM, + WINDSOR, + ] + names_key = [ + "ADDISON", + "BENNINGTON", + "CALEDONIA", + "CHITTENDEN", + "ESSEX", + "FRANKLIN", + "GRAND_ISLE", + "LAMOILLE", + "ORANGE", + "ORLEANS", + "RUTLAND", + "WASHINGTON", + "WINDHAM", + "WINDSOR", + ] + + statutes_dict = { + "3848:3849": "Business Inventory & Equipment", + "3848:38:00": "Business Inventory & Equipment", + "3840": "Charitable, Fraternal, or Rescue", + "3840;5405a(a)(4)": "Charitable/Rescue (inc. Education Tax)", + "3840;54": "Charitable/Rescue (inc. Education Tax)", + "2741": "Tax Stabilization Contract", + "24/2741": "Tax Stabilization Contract", + "3832": "Public, Pious, or Charitable", + "3832(1)": "Out-of-Town Municipal Property", + "3832(7)": "Health or Recreational Property", + "3832(7)(B)": "Non-profit Ice Skating Rink", + "3832(7B": "Non-profit Ice Skating Rink", + "5401": "Statewide Education Tax Exception", + "3752(7)": "Agricultural / Current Use", + } + + expdesc_dict = { + "Statutory": "State Law Exemption", + "Solar Plant": "Solar Energy Facility", + "Non-Approved (Voted)": "Local Town-Voted Exemption", + "Qualified Housing Units": "Affordable / Qualified Housing", + "Grandfathered": "Pre-existing Historical Exemption", + "Partial-Statutory": "Partial State Law Exemption", + "Municipal Contract (Owner Pays)": "Payment in Lieu of Taxes (PILOT)", + "Ski Lifts / Snow Making Equip": "Ski Resort Equipment", + "Court Ordered": "Judicially Mandated Exemption", + "Wind Plant": "Wind Energy Facility", + } + + rescode_dict = { + "T": "TOWN RESIDENT", + "NS": "OUT OF STATE RESIDENT", + "S": "VERMONT RESIDENT", + "C": "CORPORATION/ENTITY", + "c": "CORPORATION/ENTITY", + } + + cat_dict = { + "R1": "Residential I (Under 6 Acres)", + "R2": "Residential II (6 Acres or More)", + "M": "Miscellaneous", + "O": "Other", + "C": "Commercial", + "MHL": "Mobile Home Landed (With Land)", + "S1": "Seasonal I (Under 6 Acres)", + "MHU": "Mobile Home Unlanded (Without Land)", + "W": "Woodland", + "S2": "Seasonal II (6 Acres or More)", + "F": "Farm", + "CA": "Commercial Apartments", + "I": "Industrial", + "UE": "Utility Electric", + "UO": "Utility Other", + } + + purpose_dict = { + "R1": "PRIMARY RESIDENCE", + "R2": "PRIMARY RESIDENCE", + "MHL": "PRIMARY RESIDENCE", + "MHU": "PRIMARY RESIDENCE", + "S1": "SEASONAL PROPERTY", + "S2": "SEASONAL PROPERTY", + "W": "WOODLAND", + "F": "FARM", + "CA": "COMMERCIAL APARTMENTS", + "M": "NOT LISTED", + "O": "NOT LISTED", + "C": "COMMERCIAL/INDUSTRIAL/UTILITY", + "I": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UE": "COMMERCIAL/INDUSTRIAL/UTILITY", + "UO": "COMMERCIAL/INDUSTRIAL/UTILITY", + } + + states = [ + "AL", + "AK", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "HI", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VA", + "WA", + "WV", + "WI", + "WY", + "VT", + "DC", + ] + + oos_dict = { + "VERMONT": "VT", + } + oos_dict.update( + dict.fromkeys( + [ + "QC", + "QC CANADA", + "PQ", + "QUEBEC", + "ON", + "ONTARIO", + "QUE", + "BC", + "ONT", + "CAN", + ], + "CANADA", + ) + ) + oos_dict.update(dict.fromkeys(["MASS", "MA."], "MA")) + oos_dict.update(dict.fromkeys(["MICHIGAN"], "MI")) + oos_dict.update(dict.fromkeys(["OHIO"], "OH")) + oos_dict.update(dict.fromkeys(["CT."], "CT")) + oos_dict.update(dict.fromkeys(["R.I."], "RI")) + oos_dict.update(dict.fromkeys(["W VA"], "WV")) + oos_dict.update(dict.fromkeys(["MARYLAND"], "MD")) + oos_dict.update(dict.fromkeys(["N CAROLINA"], "NC")) + oos_dict.update(dict.fromkeys(["NEW YORK", "N.Y.", "12513", "N Y"], "NY")) + oos_dict.update(dict.fromkeys(["FLORIDA", "FLA"], "FL")) + oos_dict.update( + dict.fromkeys( + [ + "ENGLAND", + "AE", + "UNK", + "BERMUDA", + "UK", + "VY", + "FRANCE", + "IND", + "ARUBA", + "GERMANY", + "IRELAND", + "SWITZERLAN", + "QLD AUS", + "LIN", + "0R", + "BERLIN", + "AUSTRALIA", + "NS", + "FWI", + "BAHAMAS", + ], + "FOREIGN", + ) + ) + oos_dict.update(dict.fromkeys(["VI", "PR", "GUAM"], "US TERRITORY")) + + counties_dict = {} + unknown = [] + for town in backfill["TOWN"].unique(): + for item in counties_key: + if town in item: + counties_dict[town] = names_key[(counties_key.index(item))] + # adding in sourcename as city and town + city_town_dict = {} + + city_source = backfill[ + (backfill["SOURCENAME"] == "CITY") | (backfill["SOURCENAME"] == "TOWN") + ] + + # finding if the parcel was found from a local department or not + backfill["SOURCENAME"] = np.where( + backfill["SOURCENAME"].isin(["CITY", "TOWN", "City of Burlington"]), + "LOCAL DEPARTMENT", + "NOT LOCAL DEPARTMENT", + ) + + # this code aids in the creation of a column that sees if the primary people live there, or if it is an investment property. + residential_codes = ["R1", "R2", "MHL", "MHU"] + + is_residential = backfill["CAT"].isin(residential_codes) + is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == "N") + + vt_mask = ( + backfill["STGL"].str.startswith("VT", na=False) + | backfill["STGL"].str.startswith("Vt", na=False) + | backfill["STGL"].str.startswith("05", na=False) + | backfill["STGL"].str.contains("VT", na=False) + | backfill["STGL"].str.contains("V T", na=False) + | backfill["STGL"].str.startswith("vt", na=False) + ) + backfill.loc[vt_mask, "STGL"] = "VERMONT" + canada_mask = ( + backfill["STGL"].str.contains("CANADA", na=False) + | backfill["STGL"].str.contains("Canada", na=False) + | backfill["STGL"].str.contains("QC", na=False) + ) + backfill.loc[canada_mask, "STGL"] = "CANADA" + + # mapping new items in columns + backfill["INVESTMENTPROP"] = is_residential & is_not_homestead + + backfill["VACANTLAND"] = (backfill["LAND_LV"] > 0) & ( + backfill["IMPRV_LV"].fillna(0) == 0 + ) + equipcode_dict = {"E": "ELECTRIC UTILITY", "C": "CABLE UTILITY"} + backfill["EXEMPT"] = backfill["STATUTE"].notna().map({True: "YES", False: "NO"}) + backfill["STATUTE"] = backfill["STATUTE"].map(statutes_dict).fillna("No Exemption") + backfill["EXPDESC"] = backfill["EXPDESC"].map(expdesc_dict).fillna("None") + backfill["EQUIPCODE"] = ( + backfill["EQUIPCODE"].map(equipcode_dict).fillna("NOT A UTILITY") + ) + # creating a foreign ownerpship column + + backfill["COUNTY"] = backfill["TOWN"].map(counties_dict) + backfill["RESCODE"] = backfill["RESCODE"].map(rescode_dict) + backfill["CATEGORY"] = backfill["CAT"].map(cat_dict) + backfill["PURPOSE"] = backfill["CAT"].map(purpose_dict) + # adding state abbreivaitions and owner lcoations + backfill["STGL"] = backfill["STGL"].replace(oos_dict) + catch_all_mask = ( + ~backfill["STGL"].isin(["CANADA", "FOREIGN", "US TERRITORY"]) + & backfill["STGL"].notna() + & ~backfill["STGL"].isin(states) + ) + backfill.loc[catch_all_mask, "STGL"] = "UNNAMED AMERICA" + backfill["OOSOWNER"] = backfill["STGL"].fillna("VT") != "VT" + + backfill["ADDRESS"] = backfill["E911ADDR"] + + # adding an area in acres to geometry column, a + if "AREAACRESGEOM" not in backfill: + backfill["AREAACRESGEOM"] = backfill.to_crs(32145).geometry.area / 4046.8564224 + backfill["ACREVALUE"] = np.where( + (backfill.REAL_FLV > 0) & (backfill.AREAACRESGEOM > 0), + backfill.REAL_FLV / backfill.AREAACRESGEOM, + np.nan, + ) + + residential_codes = ["R1", "R2", "MH"] + is_residential = backfill["PROPTYPE"].isin(residential_codes) + is_not_homestead = backfill["HSDECL"].isna() | (backfill["HSDECL"] == "0") + + backfill = backfill.drop( + columns=[ + "LOCAPROP", + "ADDRGL2", + "ENDDATE", + "OWNER1", + "OWNER2", + "EDITNOTE", + "MAPID", + "YEAR", + "GLYEAR", + "SOURCETYPE", + "SOURCEDATE", + "EDITMETHOD", + "SHAPE_STAr", + "SHAPE_STLe", + "GLIST_SPAN", + "PARCID", + "CRHOUSPCT", + "MUNGL1PCT", + "AOEGL_HS", + "AOEGL_NR", + "HSITEVAL", + "E911ADDR", + "ADDRGL1", + "ZIPGL", + ] + ) + + bad_geom = backfill.geometry.isna() | backfill.geometry.is_empty + gdf = backfill.loc[~bad_geom].copy() + + assert gdf.geom_type.isin(["Polygon", "MultiPolygon"]).all(), ( + gdf.geom_type.value_counts() + ) + gdf["geometry"] = gdf.geometry.apply( + lambda g: MultiPolygon([g]) if g.geom_type == "Polygon" else g + ) + return gdf + + +if __name__ == "__main__": + main() diff --git a/backend/data_collection/parcels.py b/backend/data_collection/parcels.py new file mode 100644 index 00000000..95dde7cd --- /dev/null +++ b/backend/data_collection/parcels.py @@ -0,0 +1,364 @@ +""" +**Author**: + Isaac Wedaman +**Created**: + 2026-08-24 +**Description**: + fetches the parcel dataset +""" + +import zipfile +from io import BytesIO +from pathlib import Path + +import geopandas as gpd +import pandas as pd +import requests + +# list of towns from which to gather data +VT_TOWNS = [ + "Addison", + "Albany", + "Alburgh", + "Andover", + "Arlington", + "Athens", + "Averill", + "Averys Gore", + "Bakersfield", + "Baltimore", + "Barnard", + "Barnet", + "Barre City", + "Barre Town", + "Barton", + "Belvidere", + "Bennington", + "Benson", + "Berkshire", + "Berlin", + "Bethel", + "Bloomfield", + "Bolton", + "Bradford", + "Braintree", + "Brandon", + "Brattleboro", + "Bridgewater", + "Bridport", + "Brighton", + "Bristol", + "Brookfield", + "Brookline", + "Brownington", + "Brunswick", + "Buels Gore", + "Burke", + "Burlington", + "Cabot", + "Calais", + "Cambridge", + "Canaan", + "Castleton", + "Cavendish", + "Charleston", + "Charlotte", + "Chelsea", + "Chester", + "Chittenden", + "Clarendon", + "Colchester", + "Concord", + "Corinth", + "Cornwall", + "Coventry", + "Craftsbury", + "Danby", + "Danville", + "Derby", + "Dorset", + "Dover", + "Dummerston", + "Duxbury", + "East Haven", + "East Montpelier", + "Eden", + "Elmore", + "Enosburgh", + "Essex Junction", + "Essex", + "Fair Haven", + "Fairfax", + "Fairfield", + "Fairlee", + "Fayston", + "Ferdinand", + "Ferrisburgh", + "Fletcher", + "Franklin", + "Georgia", + "Glastenbury", + "Glover", + "Goshen", + "Grafton", + "Granby", + "Grand Isle", + "Granville", + "Greensboro", + "Groton", + "Guildhall", + "Guilford", + "Halifax", + "Hancock", + "Hardwick", + "Hartford", + "Hartland", + "Highgate", + "Hinesburg", + "Holland", + "Hubbardton", + "Huntington", + "Hyde Park", + "Ira", + "Irasburg", + "Isle La Motte", + "Jamaica", + "Jay", + "Jericho", + "Johnson", + "Killington", + "Kirby", + "Landgrove", + "Leicester", + "Lemington", + "Lewis", + "Lincoln", + "Londonderry", + "Lowell", + "Ludlow", + "Lunenburg", + "Lyndon", + "Maidstone", + "Manchester", + "Marlboro", + "Marshfield", + "Mendon", + "Middlebury", + "Middlesex", + "Middletown Springs", + "Milton", + "Monkton", + "Montgomery", + "Montpelier", + "Moretown", + "Morgan", + "Morristown", + "Mount Holly", + "Mount Tabor", + "New Haven", + "Newark", + "Newbury", + "Newfane", + "Newport City", + "Newport Town", + "North Hero", + "Northfield", + "Norton", + "Norwich", + "Orange", + "Orwell", + "Panton", + "Pawlet", + "Peacham", + "Peru", + "Pittsfield", + "Pittsford", + "Plainfield", + "Plymouth", + "Pomfret", + "Poultney", + "Pownal", + "Proctor", + "Putney", + "Randolph", + "Reading", + "Readsboro", + "Richford", + "Richmond", + "Ripton", + "Rochester", + "Rockingham", + "Roxbury", + "Royalton", + "Rupert", + "Rutland City", + "Rutland Town", + "Ryegate", + "Saint Albans City", + "Saint Albans Town", + "Saint George", + "Saint Johnsbury", + "Salisbury", + "Sandgate", + "Searsburg", + "Shaftsbury", + "Sharon", + "Sheffield", + "Shelburne", + "Sheldon", + "Shoreham", + "Shrewsbury", + "Somerset", + "South Burlington", + "South Hero", + "Springfield", + "Stamford", + "Stannard", + "Starksboro", + "Stockbridge", + "Stowe", + "Strafford", + "Stratton", + "Sudbury", + "Sunderland", + "Sutton", + "Swanton", + "Thetford", + "Tinmouth", + "Topsham", + "Townshend", + "Troy", + "Tunbridge", + "Underhill", + "Vergennes", + "Vernon", + "Vershire", + "Victory", + "Waitsfield", + "Walden", + "Wallingford", + "Waltham", + "Wardsboro", + "Warners Grant", + "Warren Gore", + "Warren", + "Washington", + "Waterbury", + "Waterford", + "Waterville", + "Weathersfield", + "Wells", + "West Fairlee", + "West Haven", + "West Rutland", + "West Windsor", + "Westfield", + "Westford", + "Westminster", + "Westmore", + "Weston", + "Weybridge", + "Wheelock", + "Whiting", + "Whitingham", + "Williamstown", + "Williston", + "Wilmington", + "Windham", + "Windsor", + "Winhall", + "Winooski", + "Wolcott", + "Woodbury", + "Woodford", + "Woodstock", + "Worcester", +] + +# base url from which to get the parcels +BASE_URL = BASE = ( + "https://maps.vcgi.vermont.gov/gisdata/vcgi/packaged_zips/CadastralParcels_VTPARCELS/" +) + + +# will figure this out later +STORAGE_LOCATION = "Data/parcels" + +# --------------------------------------------------------------------------- +# VERMONT PARCELS API fetch +# --------------------------------------------------------------------------- + + +def fetch_town(town: str): + # the destination is the backfill folder in the parcels folder, with one entrance per town + destination = Path(f"{STORAGE_LOCATION}/backfill/{town}") + # making a variable for the town's SHAPEFILE being there + shp = destination / f"VTPARCELS_{town}.shp" + # we can skip athe need to download if the file is already there - this will save time on multiple go arounds + if shp.exists(): + return + # getting the info from the api + r = requests.get(f"{BASE_URL}VTPARCELS_{town}.zip", timeout=60) + # throwing an exception if the https request fails + r.raise_for_status() + # extracting the data + zipfile.ZipFile(BytesIO(r.content)).extractall(destination) + + +def standardize_town(shp_path: Path) -> gpd.GeoDataFrame: + gdf = gpd.read_file(shp_path) + gdf = gdf.to_crs(4326) + gdf["TOWN"] = gdf["TOWN"].str.upper().str.strip() + return gdf + + +# --------------------------------------------------------------------------- +# Main scrape runner +# --------------------------------------------------------------------------- + + +# need to add a failsafe - all towns might not work +def collect(): + # failed = [] + # trying each town + # we collecdt each failure and add it to the failedd list, also citing it's failure essage + for town in VT_TOWNS: + fetch_town(town) + + gdfs = [ + standardize_town( + Path(f"{STORAGE_LOCATION}/backfill/{town}") / f"VTPARCELS_{town}.shp" + ) + for town in VT_TOWNS + ] + + # concatenating what we just grabbed to backfill gpd + backfill = pd.concat(gdfs, ignore_index=True) + # changing the geometry + backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) + # contriving a stable join key + backfill["OBJECTID"] = 1_000_000 + backfill.index + # returning + return backfill + + +# def combine(towns_list) -> gpd.GeoDataFrame: +# # running the loop for each town in town list: we standardize the town after having grabbed it from sotrage +# gdfs = [ +# standardize_town( +# Path(f"{STORAGE_LOCATION}/backfill/{town}") / f"VTPARCELS_{town}.shp" +# ) +# for town in towns_list +# ] +# # concatenating what we just grabbed to backfill gpd +# backfill = pd.concat(gdfs, ignore_index=True) +# # changing the geometry +# backfill = gpd.GeoDataFrame(backfill, geometry="geometry", crs=4326) +# # contriving a stable join key +# backfill["OBJECTID"] = 1_000_000 + backfill.index +# # returning +# return backfill + + +if __name__ == "__main__": + df = collect() diff --git a/backend/notebooks/parcels/parcels_ml.qmd b/backend/notebooks/parcels/parcels_ml.qmd new file mode 100644 index 00000000..581824d3 --- /dev/null +++ b/backend/notebooks/parcels/parcels_ml.qmd @@ -0,0 +1,270 @@ +--- +title: "Vermont Parcels: build.py [QMD]" +author: Isaac Wedaman +date: today +description: The second version of the vermotn parcels the exploratory motion +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +execute: + cache: true +editor: + render-on-save: true +--- +#root finder +```{python} +import os +from pathlib import Path +import pandas as pd +import duckdb + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) + +import logging + +from api.models import FilterSource +from query.processed_db import DB +``` + +```{python} +""" +Vermont Parcels — Spatial Statistics +Step 1 of the modeling pipeline: measure spatial autocorrelation to (a) establish +that value clustering is statistically real, (b) justify spatial cross-validation +downstream, and (c) build the weights matrix reused for spatial features. +""" +import numpy as np, pandas as pd, geopandas as gpd +import matplotlib.pyplot as plt +from libpysal.weights import Queen, KNN +from esda.moran import Moran, Moran_Local +import requests + +# --------------------------------------------------------------------------- +# TOWN LEVEL — coarse, fast, interpretable. Start here. +# --------------------------------------------------------------------------- + +FGB = "Data/parcels/all_parcels_vermont.fgb" + +parcels = gpd.read_file(FGB) +#adding IMPR_SHARE +parcels["IMPR_SHARE"] = np.where(parcels["REAL_FLV"] > 0, parcels["IMPRV_LV"].fillna(0) / parcels["REAL_FLV"], np.nan) +TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" + +print("Downloading town boundaries...") +r = requests.get(TOWN_URL) +r.raise_for_status() +data = r.json() + +towns_gdf = gpd.GeoDataFrame.from_features(data["features"]) + +towns_gdf.set_crs(epsg=4326, inplace=True) + +print(towns_gdf.columns) + +towns_gdf["TOWN"] = towns_gdf["TOWNNAME"].str.upper().str.strip() +towns_gdf = towns_gdf[["TOWN", "geometry"]].copy() + +# The metric: median parcel value per town. Log because Moran's I is a correlation +# statistic and raw value is heavy-tailed enough that a few towns would dominate it. +tv = parcels.loc[parcels.REAL_FLV > 0].groupby("TOWN")["REAL_FLV"].median() +tw = towns_gdf.merge(tv.rename("med_val").reset_index(), on="TOWN", how="left") +tw["log_val"] = np.log10(tw["med_val"]) + +# CRITICAL ORDERING: drop NaN BEFORE building weights. Weights encode who-borders-whom; +# dropping a town afterward would leave the matrix describing a map that no longer exists, +# and W's row order must align exactly with the value vector. +tw = tw.dropna(subset=["log_val"]).reset_index(drop=True) + +# Queen contiguity: neighbors share ANY boundary point (corners included). +w = Queen.from_dataframe(tw, use_index=True) + +# VERMONT-SPECIFIC TRAP: Lake Champlain islands (Grand Isle, Isle La Motte, North/South +# Hero, Alburgh) share no LAND border, so they get zero neighbors. Moran's I can't handle +# island rows. Drop them in a loop — removing one town can orphan another. +print("island towns:", [tw.TOWN.iloc[i] for i in w.islands]) +while w.islands: + tw = tw.drop(index=w.islands).reset_index(drop=True) + w = Queen.from_dataframe(tw, use_index=True) + +# Row-standardize: each row sums to 1, so the spatial lag is a neighbor AVERAGE. +# Without this, towns with more neighbors get mechanically more influence. +w.transform = "r" +y = tw["log_val"].values + +# GLOBAL MORAN'S I — one number, one p-value, for all of Vermont. +# 999 permutations: shuffle values across the map 999x, compare to the real I. +mi = Moran(y, w, permutations=999) +print(f"Moran's I = {mi.I:.3f} | E[I] = {mi.EI:.3f} | p = {mi.p_sim:.3f} | z = {mi.z_sim:.2f}") +#and now making the scatterplot +z = (y - y.mean()) / y.std() # standardize the values +lag_z = w.sparse @ z # spatial lag: neighbors' average, standardized +fig, ax = plt.subplots(figsize=(7, 7)) +ax.scatter(z, lag_z, alpha=0.6, edgecolor="white", linewidth=0.5) +slope = np.polyfit(z, lag_z, 1)[0] # this slope IS Moran's I +xs = np.array([z.min(), z.max()]) +ax.plot(xs, slope * xs, "r", lw=2, label=f"slope = I = {slope:.2f}") +ax.axvline(0, color="gray", ls="--", lw=0.8); ax.axhline(0, color="gray", ls="--", lw=0.8) +ax.set_xlabel("town value (standardized)"); ax.set_ylabel("neighbors' avg value (spatial lag)") +ax.set_title("Moran scatterplot"); ax.legend(); plt.show() +# quadrants preview LISA: upper-right=HH, lower-left=LL, upper-left=LH, lower-right=HL + + +``` + +```{python} +from esda.moran import Moran_Local +#getting the lisa cluster +lisa = Moran_Local(y, w, permutations=999, seed=42) +sig = lisa.p_sim < 0.05 +labels = {1: "High-High", 2: "Low-High", 3: "Low-Low", 4: "High-Low"} # verified encoding +tw["lisa"] = ["Not significant" if not s else labels[q] for q, s in zip(lisa.q, sig)] +print(tw["lisa"].value_counts()) + +cmap = {"High-High": "#d7191c", "Low-Low": "#2c7bb6", "Low-High": "#abd9e9", + "High-Low": "#fdae61", "Not significant": "#e8e8e8"} +fig, ax = plt.subplots(figsize=(8, 11)) +for cls, color in cmap.items(): + sub = tw[tw.lisa == cls] + if len(sub): + sub.plot(ax=ax, color=color, edgecolor="white", linewidth=0.3, label=f"{cls} ({len(sub)})") +ax.legend(loc="upper left", fontsize=9); ax.axis("off") +ax.set_title("LISA clusters — median parcel value (p < 0.05)"); plt.show() + +#high high means high value parcels in a hgih value area, low-low is low cost parcels surrounded by lower cost parcels, high low is high cost parcels surrounded by poorer areas, and low high is vice versa +for cls in ["High-High", "Low-Low", "High-Low", "Low-High"]: + towns_in = tw.loc[tw.lisa == cls, "TOWN"].tolist() + print(f"\n{cls} ({len(towns_in)}):", sorted(towns_in)) +print("\nHH spans counties:", sorted(tw.loc[tw.lisa=="High-High"].merge( + parcels[["TOWN","COUNTY"]].drop_duplicates(), on="TOWN")["COUNTY"].unique())) + + +CHITTENDEN = ["BOLTON", "CHARLOTTE", "COLCHESTER", "ESSEX", "ESSEX JUNCTION", "HINESBURG", "HUNTINGTON", "JERICHO", "MILTON", "RICHMOND", "SAINT GEORGE", "SHELBURNE", "UNDERHILL", "WESTFORD", "WILLISTON", "BURLINGTON", "WINOOSKI", "BUELS GORE", "SOUTH BURLINGTON"] +cc = parcels[parcels.TOWN.isin(CHITTENDEN)].copy() + +cc_v = cc.loc[cc.REAL_FLV > 0].to_crs(32145).copy() # PROJECTED CRS: distances must be metric +cc_v["logv"] = np.log10(cc_v["REAL_FLV"]) +cc_pts = cc_v.copy(); cc_pts["geometry"] = cc_v.geometry.centroid # KNN needs points + +# KNN, not contiguity: parcels are effectively points, and we WANT neighbor relationships +# that cross municipal boundaries — that's the whole finding. +# 1. Symmetrize: if A is B's neighbor, make B A's neighbor. Reduces fragmentation. +from libpysal.weights import KNN, W +wk = KNN.from_dataframe(cc_pts, k=8) +wk = W(wk.symmetrize().neighbors) # union of the directed edges +wk.transform = "r" + +# 2. Or use distance bands instead — every parcel within X meters is a neighbor. +from libpysal.weights import DistanceBand +wd = DistanceBand.from_dataframe(cc_pts, threshold=1000, binary=True) # 1km +wd.transform = "r" + +for k in [2, 5, 8, 12, 20]: + w_k = KNN.from_dataframe(cc_pts, k=k); w_k.transform = "r" + m = Moran(yk, w_k, permutations=99) + print(f"k={k:3d} → I = {m.I:.3f}, p = {m.p_sim:.3f}") + +# Fewer permutations at parcel scale — 99 shows the structure; this is the slow step. +mi_p = Moran(yk, wk, permutations=99) +print(f"parcel-level Moran's I = {mi_p.I:.3f} | p = {mi_p.p_sim:.3f}") + +lisa_p = Moran_Local(yk, wk, permutations=99, seed=42) +sigp = lisa_p.p_sim < 0.05 +cc_v["lisa"] = ["ns" if not s else labels[q] for q, s in zip(lisa_p.q, sigp)] +``` + +```{python} +fig, ax = plt.subplots(figsize=(11, 11)) +cc_v[cc_v.lisa == "ns"].plot(ax=ax, color="#ededed", linewidth=0) # gray base first +for cls, color in [("High-High","#d7191c"), ("Low-Low","#2c7bb6"), + ("Low-High","#abd9e9"), ("High-Low","#fdae61")]: + s = cc_v[cc_v.lisa == cls] + if len(s): + s.plot(ax=ax, color=color, linewidth=0, label=f"{cls} ({len(s)})") # linewidth=0: edges kill perf +ax.legend(loc="upper left"); ax.axis("off") +ax.set_title("Chittenden parcels — LISA hot/cold spots (KNN, k=8)"); plt.show() +``` + +#running some machine learning tweaks +```{python} +print(parcels.columns) +``` +#performing sklearn +#problem - it is super easy to predict population from parcel count, so first well run with only categorical value like median value and size values, and then we'll throw in the parcel count correlation that is destined to improve accuracy + +whats more, this is a small data problem (343,000 rows) predicting 255 towns +```{python} +import numpy as np, pandas as pd +from sklearn.linear_model import LassoCV, Lasso +from sklearn.preprocessing import StandardScaler +from sklearn.pipeline import make_pipeline +from sklearn.model_selection import KFold, cross_val_score +#here we group all the valid variables +feat = parcels.groupby("TOWN").agg( + #median parcel value + med_val=("REAL_FLV", lambda s: s[s > 0].median()), + #25, 50, 75th percentile + p25_val=("REAL_FLV", lambda s: s[s > 0].quantile(.25)), + p75_val=("REAL_FLV", lambda s: s[s > 0].quantile(.75)), + #median size + med_lot=("AREAACRESGEOM", "median"), + p25_lot=("AREAACRESGEOM", lambda s: s.quantile(.25)), + med_vpa=("ACREVALUE", "median"), + impr_share=("IMPR_SHARE", "median"), + pct_res=("PURPOSE", lambda s: (s == "PRIMARY RESIDENCE").mean()), + pct_seasonal=("PURPOSE", lambda s: (s == "SEASONAL PROPERTY").mean()), + pct_farmwood=("PURPOSE", lambda s: s.isin(["FARM","WOODLAND"]).mean()), + pct_commercial=("PURPOSE", lambda s: (s == "COMMERCIAL/INDUSTRIAL/UTILITY").mean()), + pct_apartments=("PURPOSE", lambda s: (s == "COMMERCIAL APARTMENTS").mean()), + pct_oos=("OOSOWNER", "mean"), + pct_investment=("INVESTMENTPROP", "mean"), + pct_vacant=("VACANTLAND", "mean"), + pct_exempt=("EXEMPT", lambda s: (s == "YES").mean()), + #THIS IS FOR SECOND PLOT ONLY + n_parcels=("OBJECTID", "size"), +) +#getting the town land area +ta = towns_gdf.to_crs(32145); ta["area_km2"] = ta.geometry.area / 1e6 +feat["area_km2"] = ta.set_index("TOWN")["area_km2"].reindex(feat.index) +#getting the density - this is for tier two only +feat["parcel_density"] = feat.n_parcels / feat.area_km2 + +# rel = "backend/notebooks/parcels/parcels_ml.qmd" +# abs = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv" +pop = pd.read_csv("/Users/isaacwedaman/local_computer_science/react-vt-data/backend/notebooks/parcels/local_data/population_2020.csv") +pop = pop[["Town", "year2020"]] +pop["Town"] = pop["Town"].str.upper().str.strip() +replacements = { + "ST. ALBANS CITY": "SAINT ALBANS CITY", + "ST. JOHNSBURY": "SAINT JOHNSBURY", + "ST. ALBANS TOWN": "SAINT ALBANS TOWN", + "ST. GEORGE": "SAINT GEORGE", + "WARNER'S GRANT" : "WARNERS GRANT", + "AVERY'S GORE": "AVERYS GORE", + "WARREN'S GORE" : "WARREN GORE", +} +pop["Town"] = pop["Town"].replace(replacements) +pop.loc[len(pop)] = {"Town": "ESSEX JUNCTION", "year2020": 10942} +pop.loc[pop["Town"] == "ESSEX", "year2020"] = (22094 - 10942) +stragglers = set(parcels["TOWN"]) - set(pop["Town"]) + +#making pop a series +pop_series = pop.set_index("Town")["year2020"].rename("pop") +df = feat.join(pop_series, how="inner") + +print(f"n towns matched: {len(df)} (of {len(feat)} parcel towns, {len(pop_series)} pop towns)") + + + +``` + diff --git a/backend/notebooks/parcels/table_build_final.qmd b/backend/notebooks/parcels/table_build_final.qmd index 2caae15e..b21cc93c 100644 --- a/backend/notebooks/parcels/table_build_final.qmd +++ b/backend/notebooks/parcels/table_build_final.qmd @@ -71,7 +71,7 @@ print("OBJECTID ok | n =", parcels["OBJECTID"].nunique()) ``` making a reusable chloropleth filter ```{python} - +import requests #getting the town boudnaries using rest api. well need this for hte chloropleths, and making the towns gdf from the town_url TOWN_URL = "https://services1.arcgis.com/BkFxaEFNwHqX3tAw/arcgis/rest/services/FS_VCGI_OPENDATA_Boundary_BNDHASH_poly_towns_SP_v1/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson" @@ -401,19 +401,49 @@ plt.tight_layout(); plt.show() ``` "Marquee 2 — parcel-level $/acre gradient (breaks out of town aggregation entirely). This is the "more than aggregated views" you asked for — Burlington→suburb→rural fringe shows as a continuous surface:" +```{python} +FGB = "/Users/isaacwedaman/local_computer_science/react-vt-data/backend/Data/parcels/all_parcels_vermont.fgb" +parcels = gpd.read_file(FGB) +print(parcels.columns) + +if "AREAACRESGEOM" not in parcels: + parcels["AREAACRESGEOM"] = parcels.to_crs(32145).geometry.area / 4046.8564224 +parcels["ACREVALUE"] = np.where( + (parcels.REAL_FLV > 0) & (parcels.AREAACRESGEOM > 0), + parcels.REAL_FLV / parcels.AREAACRESGEOM, np.nan) +``` + ```{python} #DROPPING NULLS +#ALSO DOING THIS FOR WHOLE STATE + +whole_summary = parcels.groupby("TOWN").agg( + n=("OBJECTID","size"), + valued=("REAL_FLV", lambda s: int((s>0).sum())), + pct_valued=("REAL_FLV", lambda s: round(100*(s>0).mean(),1)), + med_val=("REAL_FLV", lambda s: s[s>0].median()), +).sort_values("n", ascending=False) + cc_proj = cc.dropna(subset=["ACREVALUE"]).to_crs(32145) +whole_proj = parcels.dropna(subset=["ACREVALUE"]).to_crs(32145) #getting rid of absurd outliers - first and 99th percentiles lo, hi = cc_proj.ACREVALUE.quantile([0.01, 0.99]) +lo, hi = whole_proj.ACREVALUE.quantile([0.05, 0.99]) + #clipping, and making figure logarithmicly scaled -cc_proj["vpa_clip"] = cc_proj.ACREVALUE.clip(lo, hi) -fig, ax = plt.subplots(figsize=(11, 11)) -cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, +# cc_proj["vpa_clip"] = cc_proj.ACREVALUE.clip(lo, hi) +# fig, ax = plt.subplots(figsize=(11, 11)) +# cc_proj.plot(column="vpa_clip", cmap="magma", legend=True, ax=ax, +# linewidth=0, norm=LogNorm(vmin=lo, vmax=hi)) +# ax.set_title("Chittenden parcels — value per acre (log)"); ax.axis("off"); plt.show() + +whole_proj["vpa_clip"] = whole_proj.ACREVALUE.clip(lo, hi) +fig, ax = plt.subplots(figsize=(12, 18)) +whole_proj.plot(column="vpa_clip", cmap="inferno", legend=True, ax=ax, linewidth=0, norm=LogNorm(vmin=lo, vmax=hi)) -ax.set_title("Chittenden parcels — value per acre (log)"); ax.axis("off"); plt.show() +ax.set_title("Whole state of vermont — value per acre (log)"); ax.axis("off"); plt.show() ``` ideas for the following cell @@ -705,7 +735,42 @@ ax.set_ylabel("urban–rural index (PC1)") ax.set_title(f"Parcel-derived urban index vs. census density (r = {r:.2f})") plt.tight_layout(); plt.show() ``` +#an r value of 0.911 is very conclusively strong for being able to predict the urbanity level of a town based on everything but population IDEA FOR FUTURE Next-level (the portfolio standout you probably haven't done): spatial autocorrelation. libpysal + esda give you Moran's I (is value spatially clustered at all?) and LISA (statistically significant hot/cold spots that ignore town boundaries). uv pip install libpysal esda, build a spatial weights matrix on town or parcel centroids, and you can map "significant high-value clusters" as a defensible statistical object rather than an eyeballed choropleth. ```{python} -``` \ No newline at end of file +import numpy as np, matplotlib.pyplot as plt +from libpysal.weights import Queen, KNN +from esda.moran import Moran, Moran_Local + +# 1. attach the metric to town polygons — log because value is heavy-tailed and +# raw skew lets a few outliers dominate a correlation statistic like Moran's I +tv = parcels.loc[parcels.REAL_FLV > 0].groupby("TOWN")["REAL_FLV"].median() +tw = towns_gdf.merge(tv.rename("med_val").reset_index(), on="TOWN", how="left") +tw["log_val"] = np.log10(tw["med_val"]) + +# 2. Moran can't take NaN, and DROPPING a town changes who-borders-whom — so this +# must happen BEFORE weights are built, not after +tw = tw.dropna(subset=["log_val"]).reset_index(drop=True) + +# 3. Queen contiguity: two towns are neighbors if they share ANY boundary point +# (Queen includes corner-touches; Rook would require a shared edge) +w = Queen.from_dataframe(tw, use_index=True) +print("island towns (no land neighbor):", [tw.TOWN.iloc[i] for i in w.islands]) + +# 4. handle islands — drop them (looping, since dropping one can orphan another). +# This is the Champlain-islands fix. +while w.islands: + tw = tw.drop(index=w.islands).reset_index(drop=True) + w = Queen.from_dataframe(tw, use_index=True) + +# 5. row-standardize: each town's neighbors' weights sum to 1, so the "spatial lag" +# is a neighbor AVERAGE, not a neighbor SUM (fair across towns with 3 vs 8 neighbors) +w.transform = "r" +y = tw["log_val"].values + +# 6. global Moran's I with a 999-permutation pseudo p-value +mi = Moran(y, w, permutations=999) +print(f"Moran's I = {mi.I:.3f} | E[I] under randomness = {mi.EI:.3f}" + f" | p = {mi.p_sim:.3f} | z = {mi.z_sim:.2f}") +``` diff --git a/backend/uv.lock b/backend/uv.lock index 3d166877..c6795181 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -77,13 +77,19 @@ dependencies = [ { name = "fastapi" }, { name = "geopandas" }, { name = "jinja2" }, + { name = "jupyter-cache" }, { name = "matplotlib" }, + { name = "nbclient" }, + { name = "nbformat" }, { name = "numpy" }, { name = "pandas" }, + { name = "pathlib" }, { name = "pydantic" }, { name = "pyogrio" }, + { name = "pyyaml" }, { name = "requests" }, { name = "scikit-learn" }, + { name = "seaborn" }, { name = "shapely" }, { name = "uvicorn" }, { name = "xycmap" }, @@ -110,13 +116,19 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.137.2" }, { name = "geopandas", specifier = ">=1.1.3" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "jupyter-cache", specifier = ">=1.0.1" }, { name = "matplotlib", specifier = ">=3.11.0" }, + { name = "nbclient", specifier = ">=0.11.0" }, + { name = "nbformat", specifier = ">=5.10.4" }, { name = "numpy", specifier = ">=2.4.6" }, { name = "pandas", specifier = ">=3.0.3" }, + { name = "pathlib", specifier = ">=1.0.1" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pyogrio", specifier = ">=0.12.1" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.34.2" }, { name = "scikit-learn", specifier = ">=1.9.0" }, + { name = "seaborn", specifier = ">=0.13.2" }, { name = "shapely", specifier = ">=2.1.2" }, { name = "uvicorn", specifier = ">=0.49.0" }, { name = "xycmap", specifier = ">=1.0.1" }, @@ -526,6 +538,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/78/6a04792ace63a93e162f1305392d500ae8ddcb620e7eb88a22fd622b35bb/geopandas-1.1.3-py3-none-any.whl", hash = "sha256:90d62a64f95eaa3be2ccc115c5f3d6e24208bb11983b390fdc0621a3eccd0230", size = 342514, upload-time = "2026-03-09T21:49:07.973Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -572,6 +631,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -699,6 +770,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "jupyter-cache" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "importlib-metadata" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "pyyaml" }, + { name = "sqlalchemy" }, + { name = "tabulate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/f7/3627358075f183956e8c4974603232b03afd4ddc7baf72c2bc9fff522291/jupyter_cache-1.0.1.tar.gz", hash = "sha256:16e808eb19e3fb67a223db906e131ea6e01f03aa27f49a7214ce6a5fec186fb9", size = 32048, upload-time = "2024-11-15T16:03:55.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/6b/67b87da9d36bff9df7d0efbd1a325fa372a43be7158effaf43ed7b22341d/jupyter_cache-1.0.1-py3-none-any.whl", hash = "sha256:9c3cafd825ba7da8b5830485343091143dff903e4d8c69db9349b728b140abf6", size = 33907, upload-time = "2024-11-15T16:03:54.021Z" }, +] + [[package]] name = "jupyter-client" version = "8.9.1" @@ -916,6 +1006,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, ] +[[package]] +name = "nbclient" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a5/b3bae4b590c0cbcada2c63a34f7580024e834a8ba213e949a2f906705787/nbclient-0.11.0.tar.gz", hash = "sha256:04a134a5b087f2c5887f228aca155db50169b8cd9334dee6942c8e927e56081a", size = 62535, upload-time = "2026-06-05T07:52:41.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/c9/94d73e5a01c5b926c3fa2496e97d7a8dc28ed5a77c0b2ed712f1a62e6694/nbclient-0.11.0-py3-none-any.whl", hash = "sha256:ef7fa0d59d6e1d41103933d8a445a18d5de860ca6b613b87b8574accdb3c2895", size = 25288, upload-time = "2026-06-05T07:52:40.115Z" }, +] + [[package]] name = "nbformat" version = "5.10.4" @@ -1052,6 +1157,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] +[[package]] +name = "pathlib" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -1839,6 +1953,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" }, + { url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" }, + { url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" }, + { url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" }, + { url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, +] + [[package]] name = "sqlfluff" version = "4.2.2" @@ -1887,6 +2029,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + [[package]] name = "tblib" version = "3.2.2" @@ -2027,3 +2178,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/f7/d0/9ec6cd4913a726a19 wheels = [ { url = "https://files.pythonhosted.org/packages/1c/6c/6bbe66d9e40d8c50552a7df4014f557ae3aa67b990a4fa3f56fd92d07204/xycmap-1.0.1-py3-none-any.whl", hash = "sha256:669652bd2049f251d713d73ae18694a4b6ef92f0f9a5c8092fc5bbd1f5fb50b8", size = 6022, upload-time = "2021-03-02T11:00:32.961Z" }, ] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +]