fix(week12): starter + autograder lessons from cohort grading

.hyf/test.sh

@@ -3,7 +3,8 @@
 # The DAG needs a running Astro/Airflow stack and a live Azure PostgreSQL
 # connection that CI cannot reach, so this checks file presence and code
 # patterns in dags/taxi_pipeline.py and the docs. The actual green run,
-# backfill idempotency, and shared-Airflow deploy are reviewed by a teacher.
+# Screenshot files are presence-checked; content, backfill idempotency, and
+# shared-Airflow deploy are reviewed by a teacher.
 # Total points: 100. Passing score: 60.
 set -euo pipefail
 
@@ -123,18 +124,30 @@ if [[ -f "$DAG" ]]; then
   if daggrep "datetime\.now\(|datetime\.today\("; then
     warn "dags/taxi_pipeline.py: datetime.now()/today() found — make sure the PARTITION comes from the logical date, not wall-clock time (Gotcha #1)"
   fi
+  # Remaining 5 pts require BOTH catchup=False and max_active_runs (Gotcha #6:
+  # set it on the @dag decorator, not only on the backfill CLI).
+  has_catchup=0
+  has_max_active=0
   if daggrep "catchup ?= ?False"; then
-    l5=$((l5 + 5)); pass "dags/taxi_pipeline.py: catchup=False set"
+    has_catchup=1
   else
     fail "dags/taxi_pipeline.py: catchup=False not found — required for safe normal operation"
   fi
+  if daggrep "max_active_runs"; then
+    has_max_active=1
+  else
+    fail "dags/taxi_pipeline.py: max_active_runs not found — set max_active_runs=1 on the @dag decorator (Gotcha #6); CLI --max-active-runs alone is not enough"
+  fi
+  if [[ "$has_catchup" -eq 1 && "$has_max_active" -eq 1 ]]; then
+    l5=$((l5 + 5)); pass "dags/taxi_pipeline.py: catchup=False and max_active_runs set"
+  fi
 fi
 score=$((score + l5))
 pass "Level 5: parameterized runs ($l5/15 pts)"
 
 # ── Level 6 (10 pts): docs filled in ────────────────────────────────────────
-# Count TODO markers in visible markdown only. Starter HTML comments like
-# <!-- Replace every TODO ... --> must not fail a filled-in runbook/AI log.
+# Count TODO markers in visible markdown only. Starter HTML comments must not
+# contain the string TODO (use "fill in" / "REPLACE" instead).
 todo_count() {
   local f="$1"
   python3 - "$f" <<'PY'
@@ -144,38 +157,78 @@ text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
 print(len(re.findall(r"TODO", text)))
 PY
 }
+visible_chars() {
+  local f="$1"
+  python3 - "$f" <<'PY'
+import re, sys
+text = open(sys.argv[1], encoding="utf-8").read()
+text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
+print(len(text))
+PY
+}
 l6=0
 runbook="$REPO_ROOT/RUNBOOK.md"
 ai="$REPO_ROOT/AI_ASSIST.md"
+report="$REPO_ROOT/ASSIGNMENT_REPORT.md"
 if file_has_content "$runbook"; then
-  rb_chars=$(wc -c < "$runbook" | tr -d ' ')
+  rb_chars=$(visible_chars "$runbook")
   rb_todo=$(todo_count "$runbook")
   if [[ "$rb_chars" -ge 400 && "$rb_todo" -eq 0 ]]; then
-    l6=$((l6 + 5)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)"
+    l6=$((l6 + 3)); pass "RUNBOOK.md: filled in (${rb_chars} chars, no TODO left)"
   else
     fail "RUNBOOK.md: still a template (${rb_chars} chars, ${rb_todo} TODO marker(s)) — fill in all four sections"
   fi
 else
   fail "RUNBOOK.md: empty"
 fi
 if file_has_content "$ai"; then
-  ai_chars=$(wc -c < "$ai" | tr -d ' ')
+  ai_chars=$(visible_chars "$ai")
   ai_todo=$(todo_count "$ai")
   if [[ "$ai_chars" -ge 400 && "$ai_todo" -eq 0 ]]; then
-    l6=$((l6 + 5)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)"
+    l6=$((l6 + 2)); pass "AI_ASSIST.md: filled in (${ai_chars} chars, no TODO left)"
   else
     fail "AI_ASSIST.md: still a template (${ai_chars} chars, ${ai_todo} TODO marker(s))"
   fi
 else
   fail "AI_ASSIST.md: empty"
 fi
+if file_has_content "$report"; then
+  rp_chars=$(visible_chars "$report")
+  rp_todo=$(todo_count "$report")
+  if [[ "$rp_chars" -ge 400 && "$rp_todo" -eq 0 ]]; then
+    l6=$((l6 + 2)); pass "ASSIGNMENT_REPORT.md: filled in (${rp_chars} chars, no TODO left)"
+  else
+    fail "ASSIGNMENT_REPORT.md: still a template (${rp_chars} chars, ${rp_todo} TODO marker(s)) — fill in schedule, deps, backfill, row counts, and shared deploy"
+  fi
+else
+  fail "ASSIGNMENT_REPORT.md: empty"
+fi
+# Screenshots: presence only (3 pts). Content (Graph/Grid/log/shared UI) is teacher-reviewed.
+# Ignore dbt package / tooling trees so vendored assets do not count.
+mapfile -t _shot_files < <(
+  find "$REPO_ROOT" -type f \( -iname '*.png' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.webp' -o -iname '*.gif' \) \
+    ! -path '*/.git/*' \
+    ! -path '*/include/dbt_project/*' \
+    ! -path '*/.venv/*' \
+    ! -path '*/node_modules/*' \
+    ! -path '*/__pycache__/*' \
+    | sort
+)
+shot_count=${#_shot_files[@]}
+if [[ "$shot_count" -ge 3 ]]; then
+  l6=$((l6 + 3)); pass "screenshots: found ${shot_count} image file(s) (need ≥3 for Graph + Grid/run + task log)"
+elif [[ "$shot_count" -gt 0 ]]; then
+  fail "screenshots: only ${shot_count} image file(s) — commit at least 3 (local Graph, green Grid/run, one task log; add shared-UI shot when the VM is up)"
+else
+  fail "screenshots: none found — commit Graph, Grid/run, and task-log images into the PR (any folder)"
+fi
 score=$((score + l6))
-pass "Level 6: documentation ($l6/10 pts)"
+pass "Level 6: documentation + screenshots ($l6/10 pts)"
 
 # ── Report ──────────────────────────────────────────────────────────────────
 print_results "Week 12 Autograder — Orchestrated Pipeline"
 write_score "$score" "$PASSING" "$SCRIPT_DIR/score.json"
 echo ""
-echo "Reminder: the shared-Airflow deploy, the green run, and backfill"
-echo "idempotency are Target-tier items a teacher reviews by hand — a high"
-echo "static score here is necessary but not sufficient for Target."
+echo "Reminder: screenshot *content*, shared-Airflow deploy proof, and before/after"
+echo "row counts are still teacher-reviewed. Autograder green is not a pass — a"
+echo "high static score is necessary but not sufficient."

AI_ASSIST.md

@@ -1,12 +1,12 @@
 # AI assistance log
 
 <!-- Document at least one point where you used an LLM on this assignment.
-     Never paste connection strings, passwords, or real data. Replace TODO. -->
+     Never paste connection strings, passwords, or real data. Fill in each field. -->
 
 ## Use 1
 
-**Prompt I sent:** TODO
+**Prompt I sent:** _Replace this section._
 
-**What the model answered:** TODO
+**What the model answered:** _Replace this section._
 
-**What I kept, changed, or discarded, and why:** TODO
+**What I kept, changed, or discarded, and why:** _Replace this section._

ASSIGNMENT_REPORT.md

@@ -1,22 +1,31 @@
 # Assignment report
 
-<!-- Replace every TODO. Keep it short: a few sentences per section. -->
+<!-- Fill in every section below. Keep it short: a few sentences each. -->
 
 ## Schedule choice and reason
 
-TODO
+_Replace this section._
 
 ## Task dependency graph
 
-TODO — describe the chain (ingest -> dbt_run -> dbt_test) and why the order matters.
+_Replace: describe ingest -> dbt_run -> dbt_test and why order matters._
 
 ## dbt project used
 
-TODO — your Week 10 project or the class reference?
+_Replace: your Week 10 project or the class reference?_
 
 ## One debugging case I resolved
 
-TODO — what failed, how you found the cause in the logs, and the fix.
+_Replace: what failed, how you found the cause in the logs, and the fix._
 
-<!-- Target tier: also document your {{ ds }} parameter usage and the
-     backfill command(s) you ran, with before/after row counts. -->
+## Parameterized runs and backfill
+
+_Replace: how {{ ds }} / logical date drives the partition; the exact backfill create command you ran (with --max-active-runs 1)._
+
+## Idempotency row counts (before / after re-run)
+
+_Replace: paste monthly counts before the re-run, then after. They must match._
+
+## Shared Airflow deploy proof (if VM online)
+
+_Replace: merged c55-shared-airflow PR URL + path to your shared-UI screenshot in this repo._

RUNBOOK.md

@@ -1,22 +1,22 @@
 # RUNBOOK
 
-<!-- Replace every TODO with real content. Another student should be able to
+<!-- Fill in every section below. Another student should be able to
      operate your DAG from this file alone, without reading your Python. -->
 
 ## How to trigger the DAG manually
 
-TODO
+_Replace this section._
 
 ## How to run a backfill
 
-TODO
+_Replace this section._
 
 ## How to inspect task logs
 
-TODO
+_Replace this section._
 
 ## Top 3 likely failures and first response
 
-1. TODO — symptom, first check, fix
-2. TODO
-3. TODO
+1. _Replace: symptom, first check, fix_
+2. _Replace this section._
+3. _Replace this section._

dags/taxi_pipeline.py

@@ -38,7 +38,8 @@ def find_dbt_dir() -> str:
 
 
 @dag(
-    # TODO Task 1 (see README): configure the decorator.
+    # Task 1 (see README): configure the decorator — schedule, start_date,
+    # catchup=False, max_active_runs=1, default_args retries, tags.
     start_date=datetime(2024, 1, 1),
 )
 def taxi_pipeline():
@@ -47,13 +48,15 @@ def ingest_taxi_month() -> int:
         """Download one month of TLC green-taxi data and load it into
         ``{SCHEMA}.raw_trips`` idempotently. Return the number of rows.
 
-        TODO Task 2 and Task 3 (see README).
+        Task 2 and Task 3 (see README): derive the partition from the
+        logical date, DELETE-then-append that month, and filter the
+        parquet to the logical month before write (Gotcha #4).
         """
         raise NotImplementedError
 
-    # TODO Task 2 (see README): add the two transform tasks, wire the full
+    # Task 2 (see README): add the two transform tasks, wire the full
     # chain, and run the transform through the Chapter 4 command so it works
-    # on the image's Python. TODO Task 4: add retry behaviour.
+    # on the image's Python. Task 4: add retry behaviour.
 
     ingest_taxi_month()
 

tests/test_dag_integrity.py

@@ -17,14 +17,16 @@
 
 def test_no_import_errors():
     """Every .py in dags/ must import cleanly."""
-    dag_bag = DagBag(dag_folder="dags", include_examples=False)
+    # Airflow 3 DagBag no longer accepts include_examples.
+    dag_bag = DagBag(dag_folder="dags")
     assert dag_bag.import_errors == {}, (
         f"DAG import errors: {dag_bag.import_errors}"
     )
 
 
 def test_every_dag_has_tags():
     """Light convention check so DAGs are discoverable via the UI tag filter."""
-    dag_bag = DagBag(dag_folder="dags", include_examples=False)
+    # Airflow 3 DagBag no longer accepts include_examples.
+    dag_bag = DagBag(dag_folder="dags")
     for dag_id, dag in dag_bag.dags.items():
         assert dag.tags, f"DAG {dag_id} is missing tags"