Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 26 additions & 19 deletions benchmarks/python/benchmark_callaway.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import sys
from pathlib import Path


# IMPORTANT: Parse --backend and set environment variable BEFORE importing diff_diff
# This ensures the backend configuration is respected by all modules
def _get_backend_from_args():
Expand All @@ -21,6 +22,7 @@ def _get_backend_from_args():
args, _ = parser.parse_known_args()
return args.backend


_requested_backend = _get_backend_from_args()
if _requested_backend in ("python", "rust"):
os.environ["DIFF_DIFF_BACKEND"] = _requested_backend
Expand All @@ -44,9 +46,7 @@ def _get_backend_from_args():


def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark Callaway-Sant'Anna estimator"
)
parser = argparse.ArgumentParser(description="Benchmark Callaway-Sant'Anna estimator")
parser.add_argument("--data", required=True, help="Path to input CSV data")
parser.add_argument("--output", required=True, help="Path to output JSON results")
parser.add_argument(
Expand All @@ -62,12 +62,15 @@ def parse_args():
help="Control group definition",
)
parser.add_argument(
"--backend", default="auto", choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)"
"--backend",
default="auto",
choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)",
)
parser.add_argument(
"--warmup", action="store_true",
help="Run one untimed fit before the timed fit (JIT/cache warm-up)"
"--warmup",
action="store_true",
help="Run one untimed fit before the timed fit (JIT/cache warm-up)",
)
return parser.parse_args()

Expand Down Expand Up @@ -153,20 +156,24 @@ def main():
# run_refresh.py joins these records against R on `event_time`.
es_effects = []
for row in es.to_dataframe().sort_values("event_time").itertuples(index=False):
es_effects.append({
"event_time": int(row.event_time),
"att": float(row.att),
"se": float(row.se),
})
es_effects.append(
{
"event_time": int(row.event_time),
"att": float(row.att),
"se": float(row.se),
}
)

# Format group effects from the post-fit container (joined on `group`).
grp_effects = []
for row in grp.to_dataframe().sort_values("label").itertuples(index=False):
grp_effects.append({
"group": int(row.label),
"att": float(row.att),
"se": float(row.se),
})
grp_effects.append(
{
"group": int(row.label),
"att": float(row.att),
"se": float(row.se),
}
)

# Build output
output = {
Expand All @@ -175,8 +182,8 @@ def main():
"method": args.method,
"control_group": args.control_group,
# Overall ATT
"overall_att": float(results.overall_att),
"overall_se": float(results.overall_se),
"overall_att": float(results.att),
"overall_se": float(results.se),
# Group-time effects
"group_time_effects": gt_effects,
# Event study
Expand Down
30 changes: 17 additions & 13 deletions benchmarks/python/benchmark_imputation.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import sys
from pathlib import Path


# IMPORTANT: Parse --backend and set environment variable BEFORE importing diff_diff
# This ensures the backend configuration is respected by all modules
def _get_backend_from_args():
Expand All @@ -23,6 +24,7 @@ def _get_backend_from_args():
args, _ = parser.parse_known_args()
return args.backend


_requested_backend = _get_backend_from_args()
if _requested_backend in ("python", "rust"):
os.environ["DIFF_DIFF_BACKEND"] = _requested_backend
Expand All @@ -38,14 +40,14 @@ def _get_backend_from_args():


def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark Imputation DiD estimator"
)
parser = argparse.ArgumentParser(description="Benchmark Imputation DiD estimator")
parser.add_argument("--data", required=True, help="Path to input CSV data")
parser.add_argument("--output", required=True, help="Path to output JSON results")
parser.add_argument(
"--backend", default="auto", choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)"
"--backend",
default="auto",
choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)",
)
return parser.parse_args()

Expand Down Expand Up @@ -98,19 +100,21 @@ def main():
for row in es.to_dataframe().sort_values("event_time").itertuples(index=False):
if bool(row.is_reference):
continue
es_effects.append({
"event_time": int(row.event_time),
"att": float(row.att),
"se": float(row.se),
})
es_effects.append(
{
"event_time": int(row.event_time),
"att": float(row.att),
"se": float(row.se),
}
)

# Build output
output = {
"estimator": "diff_diff.ImputationDiD",
"backend": actual_backend,
# Overall ATT
"overall_att": float(results.overall_att),
"overall_se": float(results.overall_se),
"overall_att": float(results.att),
"overall_se": float(results.se),
# Event study
"event_study": es_effects,
# Timing
Expand All @@ -135,7 +139,7 @@ def main():
with open(output_path, "w") as f:
json.dump(output, f, indent=2)

print(f"Overall ATT: {results.overall_att:.6f} (SE: {results.overall_se:.6f})")
print(f"Overall ATT: {results.att:.6f} (SE: {results.se:.6f})")
print(f"Completed in {total_time:.3f} seconds")
return output

Expand Down
6 changes: 3 additions & 3 deletions benchmarks/python/benchmark_stacked_did.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ def main():
"estimator": "diff_diff.StackedDiD",
"backend": actual_backend,
# Overall ATT
"overall_att": float(results.overall_att),
"overall_se": float(results.overall_se),
"overall_att": float(results.att),
"overall_se": float(results.se),
# Event study
"event_study": es_effects,
# Timing
Expand Down Expand Up @@ -164,7 +164,7 @@ def main():
with open(output_path, "w") as f:
json.dump(output, f, indent=2)

print(f"Overall ATT: {results.overall_att:.6f} (SE: {results.overall_se:.6f})")
print(f"Overall ATT: {results.att:.6f} (SE: {results.se:.6f})")
if results.event_study_effects:
for h, eff in sorted(results.event_study_effects.items()):
if eff.get("n_obs", 1) > 0:
Expand Down
26 changes: 16 additions & 10 deletions benchmarks/python/benchmark_sun_abraham.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import sys
from pathlib import Path


# IMPORTANT: Parse --backend and set environment variable BEFORE importing diff_diff
def _get_backend_from_args():
"""Parse --backend argument without importing diff_diff."""
Expand All @@ -23,6 +24,7 @@ def _get_backend_from_args():
args, _ = parser.parse_known_args()
return args.backend


_requested_backend = _get_backend_from_args()
if _requested_backend in ("python", "rust"):
os.environ["DIFF_DIFF_BACKEND"] = _requested_backend
Expand All @@ -43,8 +45,10 @@ def parse_args():
parser.add_argument("--data", required=True, help="Path to input CSV data")
parser.add_argument("--output", required=True, help="Path to output JSON results")
parser.add_argument(
"--backend", default="auto", choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)"
"--backend",
default="auto",
choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)",
)
return parser.parse_args()

Expand Down Expand Up @@ -78,19 +82,21 @@ def main():
first_treat="first_treat",
)

overall_att = results.overall_att
overall_se = results.overall_se
overall_pvalue = results.overall_p_value
overall_att = results.att
overall_se = results.se
overall_pvalue = results.p_value

# Extract event study effects
event_study = []
for e in sorted(results.event_study_effects.keys()):
eff = results.event_study_effects[e]
event_study.append({
"event_time": int(e),
"att": float(eff["effect"]),
"se": float(eff["se"]),
})
event_study.append(
{
"event_time": int(e),
"att": float(eff["effect"]),
"se": float(eff["se"]),
}
)

total_time = timer.elapsed

Expand Down
44 changes: 25 additions & 19 deletions benchmarks/python/benchmark_wooldridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import sys
from pathlib import Path


# IMPORTANT: Parse --backend and set environment variable BEFORE importing diff_diff
def _get_backend_from_args():
"""Parse --backend argument without importing diff_diff."""
Expand All @@ -23,6 +24,7 @@ def _get_backend_from_args():
args, _ = parser.parse_known_args()
return args.backend


_requested_backend = _get_backend_from_args()
if _requested_backend in ("python", "rust"):
os.environ["DIFF_DIFF_BACKEND"] = _requested_backend
Expand All @@ -38,14 +40,14 @@ def _get_backend_from_args():


def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark WooldridgeDiD (ETWFE) estimator"
)
parser = argparse.ArgumentParser(description="Benchmark WooldridgeDiD (ETWFE) estimator")
parser.add_argument("--data", required=True, help="Path to input CSV data (mpdta format)")
parser.add_argument("--output", required=True, help="Path to output JSON results")
parser.add_argument(
"--backend", default="auto", choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)"
"--backend",
default="auto",
choices=["auto", "python", "rust"],
help="Backend to use: auto (default), python (pure Python), rust (Rust backend)",
)
return parser.parse_args()

Expand Down Expand Up @@ -91,31 +93,35 @@ def main():
# Format ATT(g,t) effects
gt_effects_out = []
for (g, t), cell in sorted(results.group_time_effects.items()):
gt_effects_out.append({
"cohort": int(g),
"time": int(t),
"att": float(cell["att"]),
"se": float(cell["se"]),
})
gt_effects_out.append(
{
"cohort": int(g),
"time": int(t),
"att": float(cell["att"]),
"se": float(cell["se"]),
}
)

# Format event study effects
es_effects = []
if results.event_study_effects:
for rel_t, effect_data in sorted(results.event_study_effects.items()):
es_effects.append({
"event_time": int(rel_t),
"att": float(effect_data["att"]),
"se": float(effect_data["se"]),
})
es_effects.append(
{
"event_time": int(rel_t),
"att": float(effect_data["att"]),
"se": float(effect_data["se"]),
}
)

output = {
"estimator": "diff_diff.WooldridgeDiD",
"method": "ols",
"control_group": "not_yet_treated",
"backend": actual_backend,
# Overall ATT
"overall_att": float(results.overall_att),
"overall_se": float(results.overall_se),
"overall_att": float(results.att),
"overall_se": float(results.se),
# Group-time ATT(g,t)
"group_time_effects": gt_effects_out,
# Event study
Expand All @@ -140,7 +146,7 @@ def main():
with open(output_path, "w") as f:
json.dump(output, f, indent=2)

print(f"Overall ATT: {results.overall_att:.6f} (SE: {results.overall_se:.6f})")
print(f"Overall ATT: {results.att:.6f} (SE: {results.se:.6f})")
print(f"Completed in {total_time:.3f} seconds")
return output

Expand Down
2 changes: 1 addition & 1 deletion benchmarks/speed_review/bench_solve_ols_fastpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def fit(d):
# fit(aggregate="all") route, removed at 4.0).
res.aggregate("event_study")
res.aggregate("group")
return float(res.overall_att), float(res.overall_se)
return float(res.att), float(res.se)

return df, fit
if scenario == "survey_absorb":
Expand Down
Loading
Loading