From 8c6b725e1143d27c09ac59c99a2224bb9af091b7 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Fri, 24 Jul 2026 10:37:45 +0200 Subject: [PATCH 1/2] fix(week12): harden starter for Airflow 3 and cohort lessons Drop DagBag include_examples (Airflow 3), remove TODO from starter HTML comments, expand ASSIGNMENT_REPORT for backfill/deploy proof, require max_active_runs with catchup in the autograder, and clarify Gotcha #4 month-filter guidance in the DAG stub. Co-authored-by: Cursor --- .hyf/test.sh | 53 ++++++++++++++++++++++++++++++------- AI_ASSIST.md | 8 +++--- ASSIGNMENT_REPORT.md | 23 +++++++++++----- RUNBOOK.md | 14 +++++----- dags/taxi_pipeline.py | 11 +++++--- tests/test_dag_integrity.py | 6 +++-- 6 files changed, 81 insertions(+), 34 deletions(-) diff --git a/.hyf/test.sh b/.hyf/test.sh index eb1decf..9a4bb3f 100755 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -123,18 +123,30 @@ if [[ -f "$DAG" ]]; then if daggrep "datetime\.now\(|datetime\.today\("; then warn "dags/taxi_pipeline.py: datetime.now()/today() found — make sure the PARTITION comes from the logical date, not wall-clock time (Gotcha #1)" fi + # Remaining 5 pts require BOTH catchup=False and max_active_runs (Gotcha #6: + # set it on the @dag decorator, not only on the backfill CLI). + has_catchup=0 + has_max_active=0 if daggrep "catchup ?= ?False"; then - l5=$((l5 + 5)); pass "dags/taxi_pipeline.py: catchup=False set" + has_catchup=1 else fail "dags/taxi_pipeline.py: catchup=False not found — required for safe normal operation" fi + if daggrep "max_active_runs"; then + has_max_active=1 + else + fail "dags/taxi_pipeline.py: max_active_runs not found — set max_active_runs=1 on the @dag decorator (Gotcha #6); CLI --max-active-runs alone is not enough" + fi + if [[ "$has_catchup" -eq 1 && "$has_max_active" -eq 1 ]]; then + l5=$((l5 + 5)); pass "dags/taxi_pipeline.py: catchup=False and max_active_runs set" + fi fi score=$((score + l5)) pass "Level 5: parameterized runs ($l5/15 pts)" # ── Level 6 (10 pts): docs filled in ──────────────────────────────────────── -# Count TODO markers in visible markdown only. Starter HTML comments like -# must not fail a filled-in runbook/AI log. +# Count TODO markers in visible markdown only. Starter HTML comments must not +# contain the string TODO (use "fill in" / "REPLACE" instead). todo_count() { local f="$1" python3 - "$f" <<'PY' @@ -144,14 +156,24 @@ text = re.sub(r"", "", text, flags=re.S) print(len(re.findall(r"TODO", text))) PY } +visible_chars() { + local f="$1" + python3 - "$f" <<'PY' +import re, sys +text = open(sys.argv[1], encoding="utf-8").read() +text = re.sub(r"", "", text, flags=re.S) +print(len(text)) +PY +} l6=0 runbook="$REPO_ROOT/RUNBOOK.md" ai="$REPO_ROOT/AI_ASSIST.md" +report="$REPO_ROOT/ASSIGNMENT_REPORT.md" if file_has_content "$runbook"; then - rb_chars=$(wc -c < "$runbook" | tr -d ' ') + rb_chars=$(visible_chars "$runbook") rb_todo=$(todo_count "$runbook") if [[ "$rb_chars" -ge 400 && "$rb_todo" -eq 0 ]]; then - l6=$((l6 + 5)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)" + l6=$((l6 + 4)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)" else fail "RUNBOOK.md: still a template (${rb_chars} chars, ${rb_todo} TODO marker(s)) — fill in all four sections" fi @@ -159,16 +181,27 @@ else fail "RUNBOOK.md: empty" fi if file_has_content "$ai"; then - ai_chars=$(wc -c < "$ai" | tr -d ' ') + ai_chars=$(visible_chars "$ai") ai_todo=$(todo_count "$ai") if [[ "$ai_chars" -ge 400 && "$ai_todo" -eq 0 ]]; then - l6=$((l6 + 5)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)" + l6=$((l6 + 3)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)" else fail "AI_ASSIST.md: still a template (${ai_chars} chars, ${ai_todo} TODO marker(s))" fi else fail "AI_ASSIST.md: empty" fi +if file_has_content "$report"; then + rp_chars=$(visible_chars "$report") + rp_todo=$(todo_count "$report") + if [[ "$rp_chars" -ge 400 && "$rp_todo" -eq 0 ]]; then + l6=$((l6 + 3)); pass "ASSIGNMENT_REPORT.md: filled in (${rp_chars} chars, no TODO left)" + else + fail "ASSIGNMENT_REPORT.md: still a template (${rp_chars} chars, ${rp_todo} TODO marker(s)) — fill in schedule, deps, backfill, row counts, and shared deploy" + fi +else + fail "ASSIGNMENT_REPORT.md: empty" +fi score=$((score + l6)) pass "Level 6: documentation ($l6/10 pts)" @@ -176,6 +209,6 @@ pass "Level 6: documentation ($l6/10 pts)" print_results "Week 12 Autograder — Orchestrated Pipeline" write_score "$score" "$PASSING" "$SCRIPT_DIR/score.json" echo "" -echo "Reminder: the shared-Airflow deploy, the green run, and backfill" -echo "idempotency are Target-tier items a teacher reviews by hand — a high" -echo "static score here is necessary but not sufficient for Target." +echo "Reminder: screenshots, shared-Airflow deploy proof, and before/after" +echo "row counts are teacher-reviewed. Autograder green is not a pass — a" +echo "high static score is necessary but not sufficient." diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 8a161e2..171da98 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -1,12 +1,12 @@ # AI assistance log + Never paste connection strings, passwords, or real data. Fill in each field. --> ## Use 1 -**Prompt I sent:** TODO +**Prompt I sent:** _Replace this section._ -**What the model answered:** TODO +**What the model answered:** _Replace this section._ -**What I kept, changed, or discarded, and why:** TODO +**What I kept, changed, or discarded, and why:** _Replace this section._ diff --git a/ASSIGNMENT_REPORT.md b/ASSIGNMENT_REPORT.md index a4fbc34..97422f5 100644 --- a/ASSIGNMENT_REPORT.md +++ b/ASSIGNMENT_REPORT.md @@ -1,22 +1,31 @@ # Assignment report - + ## Schedule choice and reason -TODO +_Replace this section._ ## Task dependency graph -TODO — describe the chain (ingest -> dbt_run -> dbt_test) and why the order matters. +_Replace: describe ingest -> dbt_run -> dbt_test and why order matters._ ## dbt project used -TODO — your Week 10 project or the class reference? +_Replace: your Week 10 project or the class reference?_ ## One debugging case I resolved -TODO — what failed, how you found the cause in the logs, and the fix. +_Replace: what failed, how you found the cause in the logs, and the fix._ - +## Parameterized runs and backfill + +_Replace: how {{ ds }} / logical date drives the partition; the exact backfill create command you ran (with --max-active-runs 1)._ + +## Idempotency row counts (before / after re-run) + +_Replace: paste monthly counts before the re-run, then after. They must match._ + +## Shared Airflow deploy proof (if VM online) + +_Replace: merged c55-shared-airflow PR URL + path to your shared-UI screenshot in this repo._ diff --git a/RUNBOOK.md b/RUNBOOK.md index a305a54..84bca1c 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -1,22 +1,22 @@ # RUNBOOK - ## How to trigger the DAG manually -TODO +_Replace this section._ ## How to run a backfill -TODO +_Replace this section._ ## How to inspect task logs -TODO +_Replace this section._ ## Top 3 likely failures and first response -1. TODO — symptom, first check, fix -2. TODO -3. TODO +1. _Replace: symptom, first check, fix_ +2. _Replace this section._ +3. _Replace this section._ diff --git a/dags/taxi_pipeline.py b/dags/taxi_pipeline.py index 37f4614..1786d30 100644 --- a/dags/taxi_pipeline.py +++ b/dags/taxi_pipeline.py @@ -38,7 +38,8 @@ def find_dbt_dir() -> str: @dag( - # TODO Task 1 (see README): configure the decorator. + # Task 1 (see README): configure the decorator — schedule, start_date, + # catchup=False, max_active_runs=1, default_args retries, tags. start_date=datetime(2024, 1, 1), ) def taxi_pipeline(): @@ -47,13 +48,15 @@ def ingest_taxi_month() -> int: """Download one month of TLC green-taxi data and load it into ``{SCHEMA}.raw_trips`` idempotently. Return the number of rows. - TODO Task 2 and Task 3 (see README). + Task 2 and Task 3 (see README): derive the partition from the + logical date, DELETE-then-append that month, and filter the + parquet to the logical month before write (Gotcha #4). """ raise NotImplementedError - # TODO Task 2 (see README): add the two transform tasks, wire the full + # Task 2 (see README): add the two transform tasks, wire the full # chain, and run the transform through the Chapter 4 command so it works - # on the image's Python. TODO Task 4: add retry behaviour. + # on the image's Python. Task 4: add retry behaviour. ingest_taxi_month() diff --git a/tests/test_dag_integrity.py b/tests/test_dag_integrity.py index 7cc0698..83295ff 100644 --- a/tests/test_dag_integrity.py +++ b/tests/test_dag_integrity.py @@ -17,7 +17,8 @@ def test_no_import_errors(): """Every .py in dags/ must import cleanly.""" - dag_bag = DagBag(dag_folder="dags", include_examples=False) + # Airflow 3 DagBag no longer accepts include_examples. + dag_bag = DagBag(dag_folder="dags") assert dag_bag.import_errors == {}, ( f"DAG import errors: {dag_bag.import_errors}" ) @@ -25,6 +26,7 @@ def test_no_import_errors(): def test_every_dag_has_tags(): """Light convention check so DAGs are discoverable via the UI tag filter.""" - dag_bag = DagBag(dag_folder="dags", include_examples=False) + # Airflow 3 DagBag no longer accepts include_examples. + dag_bag = DagBag(dag_folder="dags") for dag_id, dag in dag_bag.dags.items(): assert dag.tags, f"DAG {dag_id} is missing tags" From 72276df42c4fcc9430f97ea3fc180e6d6d469622 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Fri, 24 Jul 2026 11:17:42 +0200 Subject: [PATCH 2/2] =?UTF-8?q?feat(autograder):=20require=20=E2=89=A53=20?= =?UTF-8?q?screenshot=20image=20files=20in=20Level=206?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Presence-only check (png/jpg/webp/gif); content and shared deploy stay teacher-reviewed. Rebalance doc points so Level 6 stays 10/100. Co-authored-by: Cursor --- .hyf/test.sh | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.hyf/test.sh b/.hyf/test.sh index 9a4bb3f..cdd04d3 100755 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -3,7 +3,8 @@ # The DAG needs a running Astro/Airflow stack and a live Azure PostgreSQL # connection that CI cannot reach, so this checks file presence and code # patterns in dags/taxi_pipeline.py and the docs. The actual green run, -# backfill idempotency, and shared-Airflow deploy are reviewed by a teacher. +# Screenshot files are presence-checked; content, backfill idempotency, and +# shared-Airflow deploy are reviewed by a teacher. # Total points: 100. Passing score: 60. set -euo pipefail @@ -173,7 +174,7 @@ if file_has_content "$runbook"; then rb_chars=$(visible_chars "$runbook") rb_todo=$(todo_count "$runbook") if [[ "$rb_chars" -ge 400 && "$rb_todo" -eq 0 ]]; then - l6=$((l6 + 4)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)" + l6=$((l6 + 3)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)" else fail "RUNBOOK.md: still a template (${rb_chars} chars, ${rb_todo} TODO marker(s)) — fill in all four sections" fi @@ -184,7 +185,7 @@ if file_has_content "$ai"; then ai_chars=$(visible_chars "$ai") ai_todo=$(todo_count "$ai") if [[ "$ai_chars" -ge 400 && "$ai_todo" -eq 0 ]]; then - l6=$((l6 + 3)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)" + l6=$((l6 + 2)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)" else fail "AI_ASSIST.md: still a template (${ai_chars} chars, ${ai_todo} TODO marker(s))" fi @@ -195,20 +196,39 @@ if file_has_content "$report"; then rp_chars=$(visible_chars "$report") rp_todo=$(todo_count "$report") if [[ "$rp_chars" -ge 400 && "$rp_todo" -eq 0 ]]; then - l6=$((l6 + 3)); pass "ASSIGNMENT_REPORT.md: filled in (${rp_chars} chars, no TODO left)" + l6=$((l6 + 2)); pass "ASSIGNMENT_REPORT.md: filled in (${rp_chars} chars, no TODO left)" else fail "ASSIGNMENT_REPORT.md: still a template (${rp_chars} chars, ${rp_todo} TODO marker(s)) — fill in schedule, deps, backfill, row counts, and shared deploy" fi else fail "ASSIGNMENT_REPORT.md: empty" fi +# Screenshots: presence only (3 pts). Content (Graph/Grid/log/shared UI) is teacher-reviewed. +# Ignore dbt package / tooling trees so vendored assets do not count. +mapfile -t _shot_files < <( + find "$REPO_ROOT" -type f \( -iname '*.png' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.webp' -o -iname '*.gif' \) \ + ! -path '*/.git/*' \ + ! -path '*/include/dbt_project/*' \ + ! -path '*/.venv/*' \ + ! -path '*/node_modules/*' \ + ! -path '*/__pycache__/*' \ + | sort +) +shot_count=${#_shot_files[@]} +if [[ "$shot_count" -ge 3 ]]; then + l6=$((l6 + 3)); pass "screenshots: found ${shot_count} image file(s) (need ≥3 for Graph + Grid/run + task log)" +elif [[ "$shot_count" -gt 0 ]]; then + fail "screenshots: only ${shot_count} image file(s) — commit at least 3 (local Graph, green Grid/run, one task log; add shared-UI shot when the VM is up)" +else + fail "screenshots: none found — commit Graph, Grid/run, and task-log images into the PR (any folder)" +fi score=$((score + l6)) -pass "Level 6: documentation ($l6/10 pts)" +pass "Level 6: documentation + screenshots ($l6/10 pts)" # ── Report ────────────────────────────────────────────────────────────────── print_results "Week 12 Autograder — Orchestrated Pipeline" write_score "$score" "$PASSING" "$SCRIPT_DIR/score.json" echo "" -echo "Reminder: screenshots, shared-Airflow deploy proof, and before/after" -echo "row counts are teacher-reviewed. Autograder green is not a pass — a" +echo "Reminder: screenshot *content*, shared-Airflow deploy proof, and before/after" +echo "row counts are still teacher-reviewed. Autograder green is not a pass — a" echo "high static score is necessary but not sufficient."