From e5930e643565f8946a0fa4b06d15e193e5deb44c Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 31 Aug 2026 09:20:31 +0800 Subject: [PATCH 01/13] test(twfeweights): R output-parity goldens for TWFE weight diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generator + committed goldens for the upcoming `attgt_weights` / `decompose_twfe_weights` surface, ported from Brantly Callaway's `twfeweights` R package (MIT). Three fixtures: `mpdta` (real; non-1..T time labels), `sim_staggered` (equal cohorts, real pre-trend so pretrend_bias != 0), and `unbalanced_cohorts` (120/70/60 — breaks the p_g == 1/3 degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). Pins `twfe_weights`/`attO_weights`/`att_simple_weights`, `implicit_twfe_weights` (no-cov, covariate, gmin1), `implicit_aipw_weights`, `twfe_cov_bal`/`aipw_cov_bal` + the summary roll-up, and the two two-period kernels that will have no public Python surface. The no-covariate decomposition is generated with a TIME-INVARIANT covariate rather than `xformula = ~1`: upstream builds an nT x 0 model matrix on that branch and `fixest::demean` segfaults on a zero-column matrix (reproduced in isolation, fixest 0.14.2 / R 4.6.1). Double-demeaning annihilates a time-invariant regressor exactly, so the call is numerically the `~1` branch — and the Python test will assert both `covariates=None` and `covariates=[]` against this one golden, proving the equivalence rather than assuming it. R is only needed to regenerate the JSON, never to run the tests. Co-Authored-By: Claude --- benchmarks/R/generate_twfeweights_golden.R | 401 +++ benchmarks/data/twfeweights_golden.json | 580 ++++ benchmarks/data/twfeweights_mpdta_panel.csv | 2501 +++++++++++++++++ benchmarks/data/twfeweights_sim_panel.csv | 1501 ++++++++++ .../data/twfeweights_unbalanced_panel.csv | 1501 ++++++++++ 5 files changed, 6484 insertions(+) create mode 100644 benchmarks/R/generate_twfeweights_golden.R create mode 100644 benchmarks/data/twfeweights_golden.json create mode 100644 benchmarks/data/twfeweights_mpdta_panel.csv create mode 100644 benchmarks/data/twfeweights_sim_panel.csv create mode 100644 benchmarks/data/twfeweights_unbalanced_panel.csv diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R new file mode 100644 index 000000000..360a13145 --- /dev/null +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -0,0 +1,401 @@ +#!/usr/bin/env Rscript +# Generate R `twfeweights` parity goldens for the diff-diff TWFE weight diagnostics. +# +# Requires: twfeweights (>= 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, +# DRDID, jsonlite +# Output: benchmarks/data/twfeweights_golden.json +# benchmarks/data/twfeweights_mpdta_panel.csv +# benchmarks/data/twfeweights_sim_panel.csv +# benchmarks/data/twfeweights_unbalanced_panel.csv +# +# Run from the repository root: +# Rscript benchmarks/R/generate_twfeweights_golden.R +# +# --------------------------------------------------------------------------- +# WHAT THIS PINS +# +# `diff_diff/twfe_weights.py` exposes two entry points, each folding several +# upstream R functions: +# +# attgt_weights(aggregation=) <- twfe_weights / attO_weights / +# att_simple_weights +# decompose_twfe_weights(method=) <- implicit_twfe_weights / +# implicit_aipw_weights +# +# plus covariate balance as a result-object method (<- twfe_cov_bal / +# aipw_cov_bal / mp_covariate_bal_summary_helper) and two private two-period +# kernels (<- two_period_reg_weights / two_period_aipw_weights) that are +# pinned here because they have no public Python surface of their own. +# +# --------------------------------------------------------------------------- +# NOTE (upstream bug — do NOT "simplify" the no-covariate calls below) +# +# twfeweights::implicit_twfe_weights() with xformula = ~1 (or NULL) builds +# model.matrix(BMisc::addCovToFormla("-1", ~1), data), an nT x 0 matrix, and +# fixest::demean() SEGFAULTS on a zero-column matrix. Reproduced in isolation +# on R 4.6.1 / fixest 0.14.2: +# +# fixest::demean(matrix(numeric(0), nrow = 10, ncol = 0), ids) +# *** caught segfault *** address ..., cause 'memory not mapped' +# +# This is a zero-column bug, NOT a conditioning problem with any particular +# fixture — every fixture hits it on the ~1 branch and no fixture hits it +# otherwise. +# +# The no-covariate decomposition is therefore generated by passing a +# TIME-INVARIANT covariate: double-demeaning annihilates such a regressor +# exactly, so the call is numerically identical to the ~1 branch. Verified on +# mpdta, where `lpop` is time-invariant within county: +# +# twfe_weights(att_gt(...)) aggregate = -0.03654894 +# implicit_twfe_weights(xformula = ~lpop) = -0.03654894 (exact) +# +# The Python parity test asserts BOTH covariates=None AND +# covariates=[] against this single golden, so the equivalence is +# proven by the test rather than assumed by the generator. +# --------------------------------------------------------------------------- + +suppressPackageStartupMessages({ + library(twfeweights) + library(did) + library(jsonlite) +}) + +stopifnot(packageVersion("twfeweights") == "0.9.0") + +# BMisc 1.4.9 emits .Deprecated warnings for makeBalancedPanel / addCovToFormla +# / getListElement on every internal call; they would otherwise flood the log. +quiet <- function(expr) suppressWarnings(suppressMessages(expr)) + +out_dir <- file.path("benchmarks", "data") +if (!dir.exists(out_dir)) { + stop("run this script from the repository root (", out_dir, " not found)") +} + +# --------------------------------------------------------------------------- +# Extraction helpers +# +# Field names below are read off the upstream S3 objects: +# mp_weights_obj $weights_df: group, time.period, weight, attgt, post +# decomposed_twfe $twfe_gt[[i]]: g, tp, weighted_outcome_diff (= ATT(g,t)), +# alpha_weight, ess, remainder, cov_bal_df +# decomposed_aipw $aipw_gt[[i]]: g, tp, est (= ATT(g,t)), att_weight, ess +# The scalar roll-ups mirror summary.decomposed_twfe / summary.decomposed_aipw +# exactly (twfeweights_mp.R:337 and :995). +# --------------------------------------------------------------------------- + +extract_mp_weights <- function(obj) { + df <- obj$weights_df + list( + group = as.numeric(df$group), + time = as.numeric(df$time.period), + post = as.integer(df$post), + weight = as.numeric(df$weight), + att = as.numeric(df$attgt), + implied_att = sum(df$weight * df$attgt) + ) +} + +extract_fwl <- function(obj) { + cells <- obj$twfe_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "weighted_outcome_diff")) + wt <- unlist(BMisc::getListElement(cells, "alpha_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + rem <- unlist(BMisc::getListElement(cells, "remainder")) + post <- 1 * (tp >= g) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), + ess = as.numeric(ess), remainder = as.numeric(rem) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pretrend_bias = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_twfe:351 + effective_sample_size = sum(post) * sum(wt[post == 1] * ess[post == 1]) + ) +} + +extract_aipw <- function(obj) { + cells <- obj$aipw_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "est")) + wt <- unlist(BMisc::getListElement(cells, "att_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + post <- 1 * (tp >= g) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), ess = as.numeric(ess) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pretrend_bias = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_aipw:1006 — note the inner sum is NOT post-filtered, + # unlike the twfe roll-up. Preserved verbatim. + effective_sample_size = sum(post) * sum(wt * ess) + ) +} + +# Per-cell balance tables, one row per (g, t) x covariate. +extract_balance_cells <- function(cells) { + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + dfs <- BMisc::getListElement(cells, "cov_bal_df") + rows <- do.call(rbind.data.frame, lapply(seq_along(dfs), function(i) { + d <- dfs[[i]] + cbind.data.frame( + group = g[i], time = tp[i], post = 1 * (tp[i] >= g[i]), + covariate = rownames(d), d, row.names = NULL + ) + })) + as.list(lapply(rows, function(col) if (is.character(col)) col else as.numeric(col))) +} + +extract_balance_summary <- function(cells) { + s <- quiet(mp_covariate_bal_summary_helper(cells)) + out <- as.list(lapply(s, as.numeric)) + out$covariate <- rownames(s) + out +} + +extract_two_period <- function(obj) { + list( + estimate = as.numeric(obj$est), + ess = if (is.null(obj$ess)) NA_real_ else as.numeric(obj$ess), + weights = as.numeric(obj$weights), + dy = as.numeric(obj$dy), + treatment = as.numeric(obj$D) + ) +} + +# --------------------------------------------------------------------------- +# Per-fixture golden bundle +# +# `invariant_cov` must be TIME-INVARIANT within unit — it drives the +# no-covariate branch (see the segfault note at the top). +# `varying_cov` must be genuinely time-varying, so the covariate-adjusted +# weights differ from the unadjusted ones. +# --------------------------------------------------------------------------- + +build_fixture <- function(df, data_file, outcome, unit, time, first_treat, + invariant_cov, varying_cov, two_period_g) { + stopifnot(all(tapply(df[[invariant_cov]], df[[unit]], + function(z) length(unique(z))) == 1)) + + # Slice the two-period sub-panel FIRST. Several upstream entry points + # (did::att_gt, and BMisc helpers reached from implicit_*) call + # data.table::setDT() on the frame they are handed, which converts it BY + # REFERENCE — after that, `df[cond, ]` silently takes data.table semantics + # and errors. Taking the subset up front sidesteps the whole problem. + tp_periods <- c(two_period_g - 1, two_period_g) + sub <- as.data.frame(df)[df[[time]] %in% tp_periods & + df[[first_treat]] %in% c(0, two_period_g), ] + + ag <- quiet(att_gt( + yname = outcome, tname = time, idname = unit, gname = first_treat, + xformla = ~1, data = df, control_group = "nevertreated", + base_period = "universal", bstrap = FALSE + )) + # did::att_gt calls data.table::setDT(data), which converts the caller's + # frame BY REFERENCE. Everything below assumes data.frame `[` semantics, so + # convert back explicitly rather than relying on what att_gt left behind. + df <- as.data.frame(df) + + inv_f <- as.formula(paste0("~", invariant_cov)) + var_f <- as.formula(paste0("~", varying_cov)) + bal_f <- as.formula(paste0("~", invariant_cov, "+", varying_cov)) + + common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = df) + + fwl_nocov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f)))) + fwl_cov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = var_f)))) + fwl_gmin1 <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f, + base_period = "gmin1")))) + aipw <- quiet(do.call(implicit_aipw_weights, + c(common, list(xformula = inv_f)))) + + # Balance is taken off the COVARIATE-ADJUSTED decomposition: on the + # no-covariate branch the implicit weights are constant within the treated + # and comparison groups, so weighted == unweighted and the table is + # degenerate (verified). fwl_cov gives a non-trivial reweighting. + bal_fwl <- quiet(twfe_cov_bal(fwl_cov, bal_f)) + bal_aipw <- quiet(aipw_cov_bal(aipw, bal_f)) + + # Two-period kernels: the (g = two_period_g) cohort against never-treated, + # over periods {g-1, g}. These are private in Python, so this is the only + # place they are pinned. (`sub` was sliced at the top of this function.) + sub_common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = sub) + tp_reg <- quiet(do.call(two_period_reg_weights, + c(sub_common, list(xformula = var_f)))) + tp_aipw <- quiet(do.call(two_period_aipw_weights, + c(sub_common, list(xformula = var_f)))) + + list( + data_file = data_file, + columns = list(outcome = outcome, unit = unit, time = time, + first_treat = first_treat, + invariant_cov = invariant_cov, varying_cov = varying_cov), + two_period_group = two_period_g, + attgt_weights = list( + twfe = extract_mp_weights(quiet(twfe_weights(ag))), + overall = extract_mp_weights(quiet(attO_weights(ag))), + simple = extract_mp_weights(quiet(att_simple_weights(ag))) + ), + decompose = list( + fwl_nocov = extract_fwl(fwl_nocov), + fwl_cov = extract_fwl(fwl_cov), + fwl_gmin1 = extract_fwl(fwl_gmin1), + aipw = extract_aipw(aipw) + ), + balance = list( + fwl = list(cells = extract_balance_cells(bal_fwl$twfe_gt), + summary = extract_balance_summary(bal_fwl$twfe_gt)), + aipw = list(cells = extract_balance_cells(bal_aipw$aipw_gt), + summary = extract_balance_summary(bal_aipw$aipw_gt)) + ), + two_period = list(reg = extract_two_period(tp_reg), + aipw = extract_two_period(tp_aipw)) + ) +} + +# --------------------------------------------------------------------------- +# Fixture 1 — mpdta (real data) +# +# did::mpdta: 500 US counties x 2003-2007, cohorts {2004, 2006, 2007} plus +# never-treated (first.treat == 0), time-invariant `lpop`. Also exercises +# non-1..T time labels, which the Python side handles by positional rescaling. +# --------------------------------------------------------------------------- + +data(mpdta, package = "did") +mpdta_df <- data.frame( + unit = as.numeric(mpdta$countyreal), + period = as.numeric(mpdta$year), + first_treat = as.numeric(mpdta$first.treat), + outcome = as.numeric(mpdta$lemp), + lpop = as.numeric(mpdta$lpop) +) +mpdta_df <- mpdta_df[order(mpdta_df$unit, mpdta_df$period), ] +# `lpop` is time-invariant; build a genuinely time-varying companion from it so +# the covariate-adjusted branch is non-degenerate on this fixture too. +mpdta_df$lpop_t <- mpdta_df$lpop * (mpdta_df$period - 2002) / 5 + +# --------------------------------------------------------------------------- +# Fixture 2 — sim_staggered (simulated) +# +# Well-conditioned by construction: 3 equal cohorts of 100 so no (g,t) cell is +# degenerate and 100 controls per cell keep the AIPW propensity score bounded +# away from 0/1; `0.3 * x1 * period` induces a REAL pre-trend so +# pretrend_bias != 0 and the diagnostic is not testing a trivial zero. +# --------------------------------------------------------------------------- + +make_sim <- function(seed, cohort_sizes, cohort_times, n_periods) { + set.seed(seed) + n <- sum(cohort_sizes) + g <- rep(cohort_times, times = cohort_sizes) + x1 <- rnorm(n) + unit_fe <- rnorm(n) + df <- do.call(rbind, lapply(seq_len(n_periods), function(t) { + data.frame(unit = seq_len(n), period = t, first_treat = g, + x1 = x1, unit_fe = unit_fe) + })) + df <- df[order(df$unit, df$period), ] + df$xtv <- 0.2 * df$period + 0.5 * df$x1 + rnorm(nrow(df), 0, 0.5) + treated <- (df$first_treat != 0) & (df$period >= df$first_treat) + df$outcome <- df$unit_fe + 0.5 * df$period + 0.3 * df$x1 * df$period + + 1.0 * treated * (df$period - df$first_treat + 1) + rnorm(nrow(df)) + df$unit_fe <- NULL + rownames(df) <- NULL + df +} + +sim_df <- make_sim(20260831, c(100, 100, 100), c(0, 3, 4), 5) + +# --------------------------------------------------------------------------- +# Fixture 3 — unbalanced_cohorts +# +# Fixture 2 has equal thirds, so p_g == 1/3 and several of the weight formulas +# coincide — a bug in the cohort-share computation would pass silently there. +# Unequal cohort masses (120 / 70 / 60) break that degeneracy. Do not drop this +# fixture. +# --------------------------------------------------------------------------- + +unb_df <- make_sim(20260901, c(120, 70, 60), c(0, 3, 5), 6) + +# --------------------------------------------------------------------------- +# Build + write +# --------------------------------------------------------------------------- + +write.csv(mpdta_df, file.path(out_dir, "twfeweights_mpdta_panel.csv"), + row.names = FALSE) +write.csv(sim_df, file.path(out_dir, "twfeweights_sim_panel.csv"), + row.names = FALSE) +write.csv(unb_df, file.path(out_dir, "twfeweights_unbalanced_panel.csv"), + row.names = FALSE) + +cat("building mpdta ...\n") +fx_mpdta <- build_fixture(mpdta_df, "twfeweights_mpdta_panel.csv", + "outcome", "unit", "period", "first_treat", + "lpop", "lpop_t", two_period_g = 2004) +cat("building sim_staggered ...\n") +fx_sim <- build_fixture(sim_df, "twfeweights_sim_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) +cat("building unbalanced_cohorts ...\n") +fx_unb <- build_fixture(unb_df, "twfeweights_unbalanced_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) + +payload <- list( + meta = list( + description = paste( + "R twfeweights parity goldens for diff_diff attgt_weights /", + "decompose_twfe_weights. Regenerate with:", + "Rscript benchmarks/R/generate_twfeweights_golden.R" + ), + upstream = paste( + "twfeweights (Brantly Callaway), MIT License,", + "Copyright (c) 2023 Brantly Callaway" + ), + r_version = paste(R.version$major, R.version$minor, sep = "."), + twfeweights_version = as.character(packageVersion("twfeweights")), + did_version = as.character(packageVersion("did")), + fixest_version = as.character(packageVersion("fixest")), + BMisc_version = as.character(packageVersion("BMisc")), + DRDID_version = as.character(packageVersion("DRDID")), + seeds = list(sim_staggered = 20260831L, unbalanced_cohorts = 20260901L), + no_covariate_note = paste( + "decompose.fwl_nocov is generated with xformula = ~,", + "which is numerically the ~1 branch (double-demeaning annihilates a", + "time-invariant regressor exactly). The ~1 branch itself cannot be", + "called: fixest::demean segfaults on the zero-column model matrix it", + "builds. See the comment block at the top of the generator." + ) + ), + fixtures = list( + mpdta = fx_mpdta, + sim_staggered = fx_sim, + unbalanced_cohorts = fx_unb + ) +) + +out_path <- file.path(out_dir, "twfeweights_golden.json") +write_json(payload, out_path, auto_unbox = TRUE, digits = NA, pretty = TRUE) +cat("wrote", out_path, "\n") +cat(" mpdta twfe implied_att =", fx_mpdta$attgt_weights$twfe$implied_att, "\n") +cat(" mpdta fwl_nocov estimate =", fx_mpdta$decompose$fwl_nocov$estimate, "\n") +cat(" sim fwl_nocov estimate =", fx_sim$decompose$fwl_nocov$estimate, "\n") +cat(" unb twfe implied_att =", fx_unb$attgt_weights$twfe$implied_att, "\n") diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json new file mode 100644 index 000000000..1c5b88b4d --- /dev/null +++ b/benchmarks/data/twfeweights_golden.json @@ -0,0 +1,580 @@ +{ + "meta": { + "description": "R twfeweights parity goldens for diff_diff attgt_weights / decompose_twfe_weights. Regenerate with: Rscript benchmarks/R/generate_twfeweights_golden.R", + "upstream": "twfeweights (Brantly Callaway), MIT License, Copyright (c) 2023 Brantly Callaway", + "r_version": "4.6.1", + "twfeweights_version": "0.9.0", + "did_version": "2.5.1", + "fixest_version": "0.14.2", + "BMisc_version": "1.4.9", + "DRDID_version": "1.3.0", + "seeds": { + "sim_staggered": 20260831, + "unbalanced_cohorts": 20260901 + }, + "no_covariate_note": "decompose.fwl_nocov is generated with xformula = ~, which is numerically the ~1 branch (double-demeaning annihilates a time-invariant regressor exactly). The ~1 branch itself cannot be called: fixest::demean segfaults on the zero-column model matrix it builds. See the comment block at the top of the generator." + }, + "fixtures": { + "mpdta": { + "data_file": "twfeweights_mpdta_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "lpop", + "varying_cov": "lpop_t" + }, + "two_period_group": 2004, + "attgt_weights": { + "twfe": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "weight": [-0.113075467453585, 0.0457198057404491, 0.0457198057404491, 0.0324868663076129, -0.0108510103349257, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110522, 0.578994031944316], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.0365489366740672 + }, + "overall": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "weight": [0, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0, 0, 0, 0.104712041884817, 0.104712041884817, 0, 0, 0, 0, 0.68586387434555], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.031018282228749 + }, + "simple": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "weight": [0, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0, 0, 0, 0.13745704467354, 0.13745704467354, 0, 0, 0, 0, 0.450171821305842], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.039951275155177 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, 0, 0.00652011242423291, 0.00376929367371423, -0.000825313279148546, -0.0374551778725037, 0, 0.0305066555832921, 0.0277807626971762, -0.00330635669251204, -0.0293607674117092], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0365489366740672, + "decomposition": -0.0365489366740672, + "remainder": 0, + "pretrend_bias": -0.00817768207864544, + "post_only": -0.0283712545954217, + "effective_sample_size": 2163 + }, + "fwl_cov": { + "cells": { + "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0133665432176392, -0.0704231581031491, -0.267917985756793, -0.0863164461874393, 0, 0.00207806356666779, 0.00376929367371422, -0.126820852395217, -0.0470847199292006, 0, 0.0267475453560011, 0.0277807626971762, -0.12743173909835, -0.0323931143733938], + "weight": [-0.1135230713667, 0.0464885411048002, 0.0461635736399947, 0.0324772245138304, -0.0116062678919253, -0.0912244608682691, -0.10633971008501, -0.108093577640393, 0.197464333015477, 0.108193415578195, -0.0876606039551204, -0.133316485481803, -0.135213842067535, -0.224628248535168, 0.580819180039627], + "ess": [288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0398490715024001, + "decomposition": -0.0398490715024001, + "remainder": 0, + "pretrend_bias": 0.0206741188438872, + "post_only": -0.0605231903462873, + "effective_sample_size": 1650.8229562172 + }, + "fwl_gmin1": { + "cells": { + "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286273, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [-5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.63889628205466, -5.63889628205466, -5.63889628205466, -5.63889628205467, -5.63889628205466] + }, + "estimate": -0.0365489366740662, + "decomposition": -0.0365489366740672, + "remainder": 9.57567358739198e-16, + "pretrend_bias": -0.00893136238322589, + "post_only": -0.0276175742908413, + "effective_sample_size": 2163 + }, + "aipw": { + "cells": { + "group": [2, 2, 2, 2, 4, 4, 5], + "time": [2, 3, 4, 5, 4, 5, 5], + "post": [1, 1, 1, 1, 1, 1, 1], + "att": [-0.0145296683111164, -0.0764218817440481, -0.140448336820238, -0.106903898121729, 0.000960573746698802, -0.0412938655881805, -0.0287813610394872], + "weight": [0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.104712041884817, 0.104712041884817, 0.68586387434555], + "ess": [293.350247392128, 293.350247392128, 293.350247392128, 293.350247392128, 253.063348228582, 253.063348228582, 295.099929127014] + }, + "estimate": -0.0328195971622414, + "decomposition": -0.0328195971622414, + "remainder": 0, + "pretrend_bias": 0, + "post_only": -0.0328195971622414, + "effective_sample_size": 2002.7927075831 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "post": [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.4193925119911, 2.05163550719466, 3.54599912772186, 2.12759947663312, 3.47639737864719, 2.08583842718831, 3.37676805929306, 2.02606083557584, 4.03397261350373, 2.42038356810224, 3.63829301967841, 2.18297581180704, 3.70441978730257, 2.22265187238154, 3.75408740697126, 2.25245244418276, 3.72734009420077, 2.23640405652046, 3.6564541037253, 2.19387246223518, 2.98671356447979, 1.79202813868788, 3.30348590139512, 1.98209154083708, 3.45864088333128, 2.07518452999877, 3.55072510262075, 2.13043506157245, 3.38741487149293, 2.03244892289576], + "weighted_covs_comparison": [3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429], + "weighted_diff": [-0.111067069225372, -0.066640241535223, 0.099909711777264, 0.059945827066358, 0.292963827444527, 0.175778296466716, -9.35563344744539, -5.61338006846723, 0.700742359063241, 0.420445415437945, 0.107833438461934, 0.0647000630771606, 0.258330371357971, 0.154998222814783, 0.570653855768603, 0.342392313461162, -9.00506141253767, -5.40303684752261, 0.32322384928481, 0.193934309570886, -0.54374601673668, -0.326247610042007, -0.142603514549474, -0.0855621087296847, 0.275207332128623, 0.165124399277174, -9.1816764041177, -5.50900584247062, 0.0541846170524449, 0.0325107702314671], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162], + "unweighted_log_ratio_sd_diff": [-1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315], + "weighted_log_ratio_sd_diff": [-2.16928559609177, -2.16928559609177, -1.69865356192303, -1.69865356192303, -1.40022970637138, -1.40022970637138, -4.92827389243409, -4.92827389243409, -0.738212307151284, -0.738212307151284, -2.24359876506552, -2.24359876506552, -1.8116712863662, -1.8116712863662, -1.14224374564749, -1.14224374564749, -4.44862281223905, -4.44862281223905, -1.84942446599253, -1.84942446599253, -1.54827828974358, -1.54827828974358, -1.33211844247205, -1.33211844247205, -0.450837075241316, -0.450837075241316, -3.45912678775289, -3.45912678775289, -0.922554990848926, -0.922554990848926], + "unweighted_frac_treated_extreme": [0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764], + "weighted_frac_treated_extreme": [0, 0, 0, 0, 0.05, 0.05, 0, 0, 0.2, 0.2, 0, 0, 0, 0, 0.075, 0.075, 0, 0, 0, 0, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0610687022900764, 0.0610687022900764, 0, 0, 0.0305343511450382, 0.0305343511450382] + }, + "summary": { + "unweighted_treat": [3.55096217446187, 2.13057730467712], + "unweighted_untreat": [3.18343355120266, 1.91006013072159], + "unweighted_diff": [0.367528623259216, 0.22051717395553], + "weighted_treat": [3.48727645229424, 2.09236587137654], + "weighted_untreat": [5.49282183830626, 3.29569310298376], + "weighted_diff": [-2.00554538601202, -1.20332723160721], + "sd": [1.27667557327224, 0.766005343963346], + "unweighted_log_ratio_sd_diff": [-0.769948848899112, -0.769948848899111], + "weighted_log_ratio_sd_diff": [-1.90947382058374, -1.90947382058374], + "unweighted_frac_treated_extreme": [0.0640703583030666, 0.0640703583030666], + "weighted_frac_treated_extreme": [0.0177218618987178, 0.0177218618987178], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + }, + "aipw": { + "cells": { + "group": [2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5], + "time": [2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 5, 5, 5, 5], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_covs_comparison": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_diff": [8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 8.88178419700125e-16], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162] + }, + "summary": { + "unweighted_treat": [3.52237381972876, 2.11342429183725], + "unweighted_untreat": [3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.338940268526096, 0.203364161115658], + "weighted_treat": [3.52237381972876, 2.11342429183725], + "weighted_untreat": [3.52237381972875, 2.11342429183725], + "weighted_diff": [8.88178419700125e-16, 7.4867395587288e-16], + "sd": [1.27727853948457, 0.766367123690741], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + } + }, + "two_period": { + "reg": { + "estimate": -0.0149216931463987, + "ess": 295.437073557126, + "weights": [0.914205378751222, 1.10807424931814, 0.933944555836912, 1.30885089779359, 0.77667566709387, 1.09886549432803, 0.857802290117748, 1.29533608776587, 1.21797362329684, 0.890296869666088, 0.876826211579155, 1.22341271500028, 0.861220240964523, 1.2217569875752, 1.23432024149811, 1.10053686950126, 1.53243455097736, 0.991986984922291, 0.852012353181866, 0.892799434382799, 1.15625878851813, 1.01070586338101, 0.797104804854826, 0.82150263374418, 0.925758568885049, 1.06901911216371, 1.15731335732235, 1.20326032975876, 0.991885473966669, 1.07626137455175, 0.958503061349015, 0.921310469219996, 0.781305672392859, 0.880672266280691, 0.868599070143875, 1.15374257218318, 0.978235389547607, 1.20698785760044, 0.864242279469781, 0.984348590136261, 0.677018937975777, 0.78566066241169, 1.07051467897138, 0.851002176458188, 0.800796993125892, 0.873966920756733, 0.922755161442552, 0.970155022871146, 0.906505733152204, 1.16334110821022, 1.00654509305044, 1.00714845007322, 1.00861119030535, 1.01139099432848, 1.01655775675883, 0.999377477471499, 1.00885020589396, 0.99839371957829, 0.972753520276552, 0.967719834724691, 0.983415466966356, 1.00756804487529, 1.00057938242599, 1.01802260153828, 0.983491689143243, 1.0162255160028, 1.00802635808135, 0.993215960705129, 0.993102181707199, 0.999004556093054, 1.18092367771598, 1.08808685196963, 1.23100981435726, 1.0159731214296, 1.05650610397299, 1.10794776362609, 1.08543904432851, 1.26567040981729, 1.18545393782311, 1.24368399209914, 1.26059743946738, 1.08112581656952, 1.25304541183838, 1.10727033756971, 1.10436293348918, 1.1694637092217, 1.10854681715011, 0.757196653021297, 0.98296939076698, 1.06678038285255, 1.09802731714199, 1.30342090571583, 1.02010825188693, 1.02240090966505, 1.0076789351107, 1.27816316617544, 0.870787994093949, 0.864507234249911, 0.874319684458889, 0.958503061349015, 0.971488264668903, 1.07593314326871, 1.34547694644207, 0.987118006851055, 0.867133874241953, 0.880346868666857, 0.92188543431963, 1.45715035175091, 0.958529642604906, 0.752567595243238, 0.97420384402187, 0.787153467781112, 1.08523202059903, 0.864818407544575, 0.989121047467235, 0.832273511291086, 0.833084041058782, 1.23693372233058, 0.675719067675065, 0.656205952440136, 0.89244421716068, 0.829434045397887, 1.17439425571453, 0.941590453075537, 0.766703290704483, 0.772482352776408, 0.799168669044801, 0.74157547066872, 0.721387168342452, 0.858610943520528, 0.994797353734057, 1.47335777302894, 1.18514252521156, 1.0803767751372, 1.50009361600496, 1.30150183654471, 0.846743269804263, 1.17080805501646, 0.861696060941864, 0.990574796380501, 1.12968407360812, 1.09171654630869, 0.804140956149301, 0.831231009547144, 0.772076141498604, 0.850308351294569, 0.880658131758654, 1.49267787652659, 0.784758519879128, 0.681653607455449, 0.596392417171735, 1.13283686776408, 0.8429053802999, 0.691259374141176, 0.833271972854951, 0.842390476649212, 1.38993252780947, 0.75168132143843, 0.806891915601875, 0.925099414970179, 0.65723510736642, 0.845159246065923, 0.90737500976245, 1.27095562512986, 0.967288076540284, 0.773426347208265, 0.958219263901373, 1.00982894404555, 1.32997931908278, 1.13888195173276, 0.74351757596311, 1.25854765822588, 0.628166041980897, 0.761122527803733, 0.606941516111651, 1.16800760300955, 0.64339210703374, 0.767652647794539, 0.784683120487471, 0.688103599457281, 0.764929768007258, 0.825623696972631, 0.888876693782396, 0.966707035387086, 0.770906152874736, 1.06914691812822, 1.03713164810658, 1.21511287945244, 0.660496935454807, 1.01335391953086, 0.91107858003953, 0.86150589590303, 0.856908216912444, 1.05679134782963, 1.17622845309429, 1.17317400146166, 1.0970151373967, 1.52515362690788, 1.1447241641711, 0.955589011684891, 1.32121999096743, 0.938232159752092, 1.2686858175866, 0.922799077792815, 1.42558003259381, 1.05749743636383, 1.09540926270813, 1.00318548722524, 1.34970798424194, 1.13005176807204, 1.17650664269292, 1.06420125597473, 1.16827467058984, 0.961741161438828, 1.03563770711794, 1.06417305269481, 0.565642036368166, 1.01178430075611, 0.80013382343569, 0.804163323876345, 0.93645145535438, 0.498752941366539, 0.873185246178276, 0.591658673083957, 0.671509856439639, 0.85343303820078, 0.804185688602419, 0.543035144130157, 0.836526939309612, 0.937016901846683, 0.888580417998956, 1.24643165992158, 1.0836459985709, 1.0550178960445, 1.02953012986436, 1.42446146773525, 1.13292714422746, 0.986819202405292, 1.01935952633555, 1.08394687794462, 1.28673644369391, 1.12770172727897, 1.60279560849242, 0.94803517304973, 0.94936028579014, 0.996230905422344, 0.947827464556498, 1.0489182301759, 0.824459438478706, 0.833102843775575, 1.04806606792118, 1.43863822710714, 1.10992438468713, 0.869145215288365, 1.75368577552688, 0.956263947765851, 1.48129133194083, 0.714177591294102, 0.526137284043039, 1.18516755063542, 1.08877777658533, 1.04685002787374, 1.39110059759066, 1.08142279773759, 0.672301055720514, 1.11637172339878, 1.17790730294754, 0.655607208827931, 0.782634337497015, 0.728931237851374, 1.1121882817121, 0.969716513585689, 0.921553965756354, 0.8357145095486, 1.01889756947789, 0.85343303820078, 0.669665695616983, 0.65506102391745, 1.09662092099898, 0.876391731353484, 1.17058226607269, 0.935233286711417, 0.668458481030845, 0.571884667288901, 0.673529835319247, 1.10335516966084, 1.03836004925978, 1.04917045987158, 0.950619215471048, 0.88606795324462, 0.87790748338468, 1.09531959221919, 1.38223356446573, 0.820856369845608, 0.914101320529017, 1.22008833913762, 1.19820046386571, 1.16675729242193, 0.739214676891295, 1.15287231341345, 1.0390337443305, 0.954983748994095, 1.3962143210842, 1.05844399017882, 1.00276474444757, 1.61570706617249, 1.13776110303408, 0.950377493029127, 0.92344550363475, 1.14599221194468, 1.01003722393432, 0.89876316093955, 1.20719528706948, 1.05242527002734, 1.10239361278195, 0.776120851588995, 0.94518023966827, 0.962590677452325, 1.30057892353915, 1.08619456613142, 1.16587595058709, 1.33511731721303, 1.37900024035138, 0.877418175728803, 1.22829047749297, 0.998079075301512, 1.16175961199361, 0.883530934105611, 0.996195569866188], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "aipw": { + "estimate": -0.0145296683111164, + "ess": 293.350247392128, + "weights": [0.894002251049714, 1.08912887056764, 0.912139448798106, 1.33663687068034, 0.777353982575372, 1.07895492287744, 0.844159798828147, 1.3183278848748, 1.21827258329935, 0.872521288429006, 0.86064891071784, 1.22505040897046, 0.847098610659438, 1.22298314584019, 1.23875720803122, 1.08079433857419, 1.67945989357214, 0.967653605894451, 0.839205021306279, 0.874745094998973, 1.14396143122207, 0.986274905763588, 0.793655426945849, 0.813577909783472, 0.904573126031373, 1.04663392636343, 1.14519210123462, 1.20012641825204, 0.967553597396161, 1.05438584858977, 0.935224694727841, 0.900488358508805, 0.781018532914758, 0.864021879677238, 0.853478380599116, 1.14103041278715, 0.954200123207928, 1.2046977512144, 0.849705627797744, 0.960157332889347, 0.702565586174883, 0.784481381193752, 1.04823001919817, 0.838343574206719, 0.796638337684592, 0.858149958258436, 0.901812997158555, 0.946383153027563, 0.887026666261882, 1.15225205545184, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.17309689736406, 1.06716813845226, 1.23458096954703, 0.99157960228583, 1.03337549477273, 1.08898847481201, 1.06429258825978, 1.27901832490392, 1.17852895390884, 1.25064709755962, 1.27241523475185, 1.05962508654199, 1.26264882346518, 1.08823686024391, 1.0850169829417, 1.15946797717608, 1.08965356969476, 0.762126536933417, 0.95881005552149, 1.04424929648179, 1.07803366329225, 1.32925013731868, 0.995764287470606, 0.998092093051924, 0.983239427591536, 1.29542652220053, 0.855380262678252, 0.849934582420623, 0.858457868237986, 0.935224694727841, 0.9476684674978, 1.05403326965604, 1.3875512537957, 0.962868424186321, 0.852207710912963, 0.863735990959885, 0.901015308497714, 1.55514398580636, 0.935249998401912, 0.758552494022138, 0.95029188025848, 0.785671952662053, 1.06406808753133, 0.850203555377142, 0.964834093590168, 0.822533389701864, 0.823211329274651, 1.24206426011762, 0.701639876047414, 0.687891345473986, 0.874429096523289, 0.820162869388286, 1.16531204434823, 0.919264300892949, 0.769520096761839, 0.774050093411802, 0.795321421756718, 0.750133494599614, 0.734916916052496, 0.844854160183868, 0.970426525099808, 1.58110223940849, 1.17815474202094, 1.05881662670873, 1.6248775532266, 1.32664934577575, 0.834721535320327, 1.16105846772014, 0.847508548356641, 0.966263255565178, 1.11338493606689, 1.07112275936809, 0.799349711866075, 0.821662255653284, 0.773730798373221, 0.837752419995297, 0.864009459343324, 1.61261501076916, 0.78376277257012, 0.705876292742996, 0.647428686090855, 1.11696902174642, 0.831471116458873, 0.712788525836175, 0.823368599043228, 0.831036004982153, 1.45197650533514, 0.757870151528899, 0.801587265560697, 0.903966634067484, 0.688609587838628, 0.833378425203411, 0.88781143556202, 1.2859343470801, 0.943625225641593, 0.774792618745101, 0.934954580596145, 0.985394537886997, 1.36577400696854, 1.12387359379345, 0.751614058902251, 1.26975693234241, 0.668612420905503, 0.765171028921774, 0.654385132001179, 1.1577477383766, 0.679012060093675, 0.77026241566466, 0.783702742793706, 0.710510112907514, 0.768135289274691, 0.816992668722927, 0.871261846989218, 0.94306726889844, 0.772811895241902, 1.04677022713451, 1.01318017562438, 1.21472287797626, 0.690891037043235, 0.988938195082353, 0.891162830885818, 0.847344689605216, 0.84339276238123, 1.03367583677142, 1.16749363935029, 1.16386294782544, 1.07692221020635, 1.66701335998637, 1.13058730375163, 0.932454891509071, 1.35361774608789, 0.916127968293349, 1.28295957022046, 0.901853294887475, 1.50580256277375, 1.0344196777086, 1.07516120251468, 0.978750662101694, 1.39355713356033, 1.11380233235004, 1.16782487713884, 1.04150887454439, 1.15806305850471, 0.938312253912984, 1.01163956811404, 1.04147894758405, 0.627576820467188, 0.987358672362671, 0.796101729352255, 0.799367879632361, 0.914469343531904, 0.586506346702584, 0.85746807099136, 0.644331572783776, 0.698650747799085, 0.840418052265205, 0.799386045379082, 0.613378159731018, 0.826097329094868, 0.914995695834427, 0.87099933523313, 1.25415777108945, 1.06234975602779, 1.03180994165805, 1.00536574424389, 1.50408349546532, 1.11707181872119, 0.962575541260206, 0.99500527053113, 1.06267552030451, 1.306809166267, 1.11113734646778, 1.80465227361454, 0.925313475141831, 0.926562233085535, 0.971844061197384, 0.925117889238856, 1.02541821473023, 0.816026483828058, 0.823227062864221, 1.02452843411696, 1.52601767939589, 1.0911845575514, 0.853952506965403, 2.10561116586107, 0.93309568004779, 1.59396704068207, 0.729558976902732, 0.602979072986013, 1.17818480963529, 1.06791977822604, 1.02326006063466, 1.45370921722291, 1.05994579825869, 0.69921162182759, 1.09837880259324, 1.16949406859886, 0.687473835164664, 0.782073370173473, 0.740566165391499, 1.09370529557565, 0.945960791989393, 0.900711482725538, 0.82541539067686, 0.994537255170615, 0.840418052265205, 0.697345210807313, 0.687093199004881, 1.07648964211119, 0.860268711765294, 1.16079118422479, 0.913336436309487, 0.696491928549023, 0.631556234278697, 0.700083598765149, 1.08390315180698, 1.01444872130449, 1.02568172844303, 0.927750198039274, 0.868776406041439, 0.8615958327451, 1.07506295514825, 1.44060768659023, 0.813043719924969, 0.893907609746538, 1.22090330227284, 1.19394906972845, 1.15627267688281, 0.748337722818574, 1.14001845186309, 1.01514511301218, 0.931880629247007, 1.46131939576755, 1.03541768771894, 0.97833141796237, 1.82862252736225, 1.12259014409163, 0.927521982019721, 0.902446667017561, 1.13204983708822, 0.985603565025875, 0.880067648926837, 1.20495265096482, 1.02908828203376, 1.0828414651844, 0.776916027928803, 0.922628819453102, 0.939123978418286, 1.32540040382158, 1.06511229655857, 1.15523404587142, 1.37295557236873, 1.43585985120983, 0.861167191951046, 1.23116101723212, 0.973674667516388, 1.15039551027658, 0.866537571207786, 0.971809095301766], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + } + }, + "sim_staggered": { + "data_file": "twfeweights_sim_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.38176729464315 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "weight": [0, 0, 0.166666666666667, 0.166666666666667, 0.166666666666667, 0, 0, 0, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.73623602393453 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "weight": [0, 0, 0.2, 0.2, 0.2, 0, 0, 0, 0.2, 0.2], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.76984085043211 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0839722781780438, 0.848539485953839, 1.76991196956896, 2.84209802207204, 0, 0.0713186971413766, -0.203874498662049, 0.823397571607694, 1.90590552823814], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38176729464315, + "remainder": 0, + "pretrend_bias": 0.0929885879714431, + "post_only": 1.28877870667171, + "effective_sample_size": 499.00962938647 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0821486884936649, 34.2972067369434, 1.76608384450794, 2.83842487370002, 0, 0.0730127750279008, 33.2436059932573, 0.823520882264303, 1.90613909549667], + "weight": [-0.250278200442825, -0.250043995598454, 0.375228054332711, 0.0628082321912258, 0.0622859095173414, -0.0622827984811969, -0.06218597232544, -0.375209033152085, 0.249640734110113, 0.250037069848608], + "ess": [99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125, 99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38188157642338, + "decomposition": 1.38188157642338, + "remainder": 0, + "pretrend_bias": -12.4573008473221, + "post_only": 13.8391824237455, + "effective_sample_size": 312.37838960871 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0.0839722781780438, 0, 0.944867949565343, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [-0.940505910590496, -0.940505910590496, -0.947616432141581, -0.940505910590496, -0.940505910590496, -1.62957956051008, -1.62957956051008, -1.62464805376604, -1.62957956051008, -1.62957956051008] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38628305525382, + "remainder": -0.00451576061067098, + "pretrend_bias": -0.0508955308063108, + "post_only": 1.43717858606014, + "effective_sample_size": 499.00962938647 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 4, 4], + "time": [3, 4, 5, 4, 5], + "post": [1, 1, 1, 1, 1], + "att": [0.950890455977506, 1.90549496406924, 3.01528915156681, 1.08514205063569, 2.23370942485274], + "weight": [0.166666666666667, 0.166666666666667, 0.166666666666667, 0.25, 0.25], + "ess": [99.1390613452281, 99.1390613452281, 99.1390613452281, 97.3682491639651, 97.3682491639651] + }, + "estimate": 1.80832529747437, + "decomposition": 1.80832529747437, + "remainder": 0, + "pretrend_bias": 0, + "post_only": 1.80832529747437, + "effective_sample_size": 491.268276272983 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.0322253630566276, 0.61354301308016, 0.0290807000435411, 0.611652284352772, 0.0314456717092465, 0.613125653540162, 0.0304194230673653, 0.611090717337329, 0.0261194212453405, 0.610939920369823, -0.0356287952936007, 0.583175368672645, -0.0455885285626065, 0.579867791877802, -0.0422942050601538, 0.580882621650816, -0.041129084294574, 0.581367403796091, -0.0426164850331386, 0.580717322776911], + "weighted_covs_comparison": [0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278, 0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278], + "weighted_diff": [-0.0966874835384971, -0.0539575205435364, -0.0984158030057429, -0.0546943627600621, 22.3117698469391, 14.2979563480237, -0.0988260354667672, -0.0564424039151755, -0.102409531050493, -0.0562485592554547, -0.164541641888725, -0.0843251649510506, -0.17308503161189, -0.0864788552350323, 22.2380299701697, 14.2657133161343, -0.170374542828706, -0.0861657174564133, -0.171145437328972, -0.0864711568483669], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979], + "unweighted_log_ratio_sd_diff": [-0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373], + "weighted_log_ratio_sd_diff": [-0.078923039541646, -0.0981028673500817, -0.0787851460177666, -0.0992789443199351, -4.79800027862989, -5.26202924360934, -0.0788460372935362, -0.0965119215695593, -0.0755098089684916, -0.104025654316807, 0.027732869455797, 0.0160140941570539, 0.0107487270326319, -0.00359213115639778, -4.70526255947979, -5.16554315809729, 0.0155949542480163, 3.59691177780341e-05, 0.0154199797107926, 0.000125226527590172], + "unweighted_frac_treated_extreme": [0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08], + "weighted_frac_treated_extreme": [0.05, 0.02, 0.04, 0.02, 0, 0, 0.05, 0.03, 0.04, 0.02, 0.11, 0.08, 0.1, 0.09, 0, 0, 0.11, 0.08, 0.11, 0.09] + }, + "summary": { + "unweighted_treat": [-0.00530837076689503, 0.597037457313671], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.13350937598507, -0.0698986457475872], + "weighted_treat": [-0.00558643553339065, 0.596830265573273], + "weighted_untreat": [-8.27967753493962, -4.71798406790115], + "weighted_diff": [8.27409109940623, 5.31481433347443], + "sd": [1.02583786880053, 0.576529078265404], + "unweighted_log_ratio_sd_diff": [-0.0319022468238235, -0.0490791536243857], + "weighted_log_ratio_sd_diff": [-1.80225098421397, -1.98696177996179], + "unweighted_frac_treated_extreme": [0.0849774462771105, 0.0499806682375233], + "weighted_frac_treated_extreme": [0.0605964064257143, 0.0456045601712674], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 4, 4, 4, 4], + "time": [3, 3, 4, 4, 5, 5, 4, 4, 5, 5], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "weighted_covs_comparison": [0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, -0.0420734072606479, 0.57732449107713, -0.0420734072606479, 0.57732449107713], + "weighted_diff": [0, -0.0028549349573127, 0, -0.0028549349573127, 0, -0.0028549349573127, 2.77555756156289e-17, 0.00362300201281618, 2.77555756156289e-17, 0.00362300201281618], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979] + }, + "summary": { + "unweighted_treat": [-0.00533204660880145, 0.59702709574503], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.133533051826977, -0.0699090073162288], + "weighted_treat": [-0.00533204660880145, 0.59702709574503], + "weighted_untreat": [-0.00533204660880146, 0.596643062217278], + "weighted_diff": [1.38777878078145e-17, 0.000384033527751737], + "sd": [1.0258527982419, 0.576537669697901], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 0.933749536861636, + "ess": 99.9926610233913, + "weights": [1.00722890988084, 0.992990992262401, 0.985696926864851, 1.00428546538914, 1.00255445781008, 0.99320823416596, 0.999615523785608, 1.00343083006454, 1.00137356889656, 0.986745204378563, 0.995094957332289, 0.986870840632856, 1.00528820819911, 0.998835332251396, 0.99700861697196, 0.997936725463451, 0.998846629895534, 0.99656967557782, 1.00045816004973, 0.990771166474457, 0.997338926459884, 0.998846092341373, 0.995772857502799, 1.01166869648943, 0.993003662361777, 1.00600474847494, 1.00405785167947, 1.00353809034622, 0.995126225338609, 1.00043085640342, 0.998206308876953, 0.996320942216609, 1.00237756422585, 0.985442690419153, 0.989720250034912, 0.995867683639298, 0.99111809890224, 1.00019084884885, 0.990912512219895, 1.00911662153634, 1.0192714498591, 1.01698591482046, 1.00242798608037, 0.991647428774127, 1.00134810289326, 1.00443878939999, 0.996778530830711, 1.00669024403109, 1.00001152211208, 1.01998818237323, 0.99264596683436, 0.9952948133176, 1.01360504654043, 1.009933572531, 1.01298877137443, 1.00106309713833, 0.99368962673189, 0.994645710006753, 1.00603342986714, 0.995879858693205, 1.01601455811524, 1.02010680280297, 0.989784340649968, 1.00825224525215, 1.00677285251964, 1.00871986746097, 1.00390290828464, 0.993629249244157, 1.00167286392168, 1.01886807947052, 1.01099154001983, 1.00201217904192, 0.99836526676243, 1.01002660972161, 0.996287759273861, 1.00250415790818, 0.983606011950821, 1.00384205736887, 0.991605834088229, 1.00537797220874, 1.01272476427711, 0.99235753997329, 0.987933977679291, 1.000650667459, 0.984180988085733, 0.996029922902563, 0.997939009434197, 0.990121709237807, 0.991953055467323, 1.00235884266265, 1.00125277134316, 0.992473987574461, 0.992917670060727, 0.998942314836784, 1.00485857501194, 1.00287939258367, 1.00418948560417, 0.985443467362189, 0.985970178418021, 1.01723852508477, 0.997466706557304, 0.992423956214259, 1.00991905980572, 0.992291387074752, 0.994820831943633, 0.989354780777259, 1.00319011306147, 1.00827468557464, 1.00431722998307, 0.996353080619044, 1.00439019192514, 0.999040966656546, 1.00231586960461, 0.978372337508447, 0.995019298342224, 0.992937322106751, 0.993590737901513, 1.00475685538413, 1.00449017738004, 1.01172939172658, 0.996227103707377, 0.998538789522835, 0.995737537791419, 0.99411674103665, 1.00138684721952, 1.00931125598526, 0.99549586869746, 1.00710500069773, 1.01347249983801, 0.996627686726185, 1.00244252373843, 1.00388526279484, 0.996225099533706, 1.01622034867275, 0.992126287936598, 0.998427079293577, 0.997025997515504, 0.994683358673275, 0.994260944805646, 1.01294005227182, 1.01510040197259, 1.00867105634545, 0.999364111810118, 0.988370930721448, 1.00754081330585, 1.01108908526376, 1.00082929965702, 0.999652563328744, 0.99115023644786, 0.993797952889325, 1.00062317593941, 0.996370067258762, 1.00862548801356, 0.992934433889504, 1.00625697060545, 1.01325255367648, 1.00669485794986, 0.994504199351021, 0.998979328323581, 0.989317777254636, 1.0079062968392, 1.00142985808414, 1.00927426598237, 0.988654526955168, 0.989280380585935, 1.00609361546911, 0.997618744628682, 1.00320239737719, 1.00752528142108, 1.00990809476446, 1.00300322465167, 0.991801531478728, 0.982098203337734, 0.993156779738883, 0.99999216518783, 1.01701536391709, 1.00655564623742, 0.987239057302495, 0.997613724292227, 0.996893085121885, 0.998388510818664, 1.00686703463208, 0.989947899004373, 0.99691959100035, 0.992555666528202, 1.02749634237056, 0.990975757502908, 0.998821219235584, 1.00181547622219, 0.982006002279984, 1.00349537276095, 1.01340976418696, 1.00161767714608, 0.998384149967381, 1.01077308854491, 0.993352591748675, 1.00138206112585, 1.00203727492767, 0.990541643636813, 0.988514062376352], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 0.932908107897949, + "ess": 99.999180206262, + "weights": [0.999672729706382, 0.998021972909027, 1.00246030815386, 0.998513055504986, 1.00585412572353, 0.997051177858498, 0.994461321822849, 0.996136791680998, 0.99802322685533, 1.00130465103858, 1.0036020329346, 1.00209547702384, 1.00124115667369, 0.996393987795192, 0.998049799389463, 1.00262259770517, 1.00030309128241, 1.00064739548438, 1.00291569113999, 1.00435106542457, 1.0011434294141, 1.00188564912852, 1.00179620123262, 0.995509793569397, 1.00003833654497, 1.00001014848312, 1.00212284054395, 1.00181323882764, 1.00114490591003, 0.999410260939198, 1.00368717706179, 1.00000109453662, 0.998485491222804, 1.00698466777593, 0.996625230168786, 1.00352109899527, 0.995024150944156, 0.998998191251484, 1.00554621969829, 0.997495901024524, 0.999565102450949, 0.99382696946915, 0.993461661221226, 0.999828271614111, 1.00045584463728, 0.996827971795766, 1.00081229366678, 0.998565528198841, 1.00389512128434, 0.995518787588971, 1.00440712427909, 0.999875209026417, 1.0032200133693, 0.999900835948388, 0.997523979887541, 1.00025940775218, 1.00175413984952, 1.0043515707889, 0.999784272657803, 0.998278347015222, 1.00199659550982, 0.998870210394531, 1.00435304482845, 1.00013634872456, 0.996946252359397, 0.996194426606418, 0.995532736428361, 0.998699972048666, 0.994346574614814, 0.997342357218583, 1.00141195618414, 0.998554195649132, 0.995582390998808, 1.00104190020894, 0.996355646424078, 0.999910936252083, 1.00118919151784, 0.994777646401869, 1.00165288081142, 1.00136294435556, 1.00030533982388, 0.999923527203744, 0.999932083612262, 1.00055286791905, 1.00310726818895, 1.0008836585536, 1.00106078377252, 1.00232921480015, 1.00148651721726, 0.999986839444585, 1.00007595480953, 1.0026103987571, 0.999783262359765, 1.00559848648777, 0.998470814142084, 0.996007891476203, 1.00066584402688, 1.00217882663762, 1.00022113959277, 1.00148293775449, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + }, + "unbalanced_cohorts": { + "data_file": "twfeweights_unbalanced_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121212, 0.212121212121212, 0.053030303030303, 0.053030303030303, -0.0378787878787879, -0.0378787878787879, -0.196969696969697, -0.196969696969697, 0.234848484848485, 0.234848484848485], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 1.88846922090824 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "weight": [0, 0, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0, 0, 0, 0, 0.230769230769231, 0.230769230769231], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.25856121537518 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "weight": [0, 0, 0.175, 0.175, 0.175, 0.175, 0, 0, 0, 0, 0.15, 0.15], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.43266114548473 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.14625813390341, 1.14471439240315, 2.1890605959757, 3.06376746663026, 4.04138335784424, 0, -0.0481726004625502, 0.331153365133827, 0.149892455219012, 1.19117629320198, 2.46506484725609], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": 0, + "pretrend_bias": -0.0541461639412249, + "post_only": 1.94261538484946, + "effective_sample_size": 719.999999999998 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.146200722768517, 1.17426164200888, 2.18125058212415, 3.08489273485064, 4.02603401811628, 0, -0.0607908732286701, 0.361790510905905, 0.134790399259838, 1.18902450338214, 2.46470121594505], + "weight": [-0.265232105650193, -0.265193408359637, 0.212690896337156, 0.211910908562023, 0.0525407450858316, 0.0532829640248196, -0.0383900377379283, -0.0374892499005717, -0.197107012378148, -0.196588185973519, 0.235333389228316, 0.234241096761852], + "ess": [119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033, 119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88897945007637, + "decomposition": 1.88897945007637, + "remainder": 0, + "pretrend_bias": -0.0567591745747959, + "post_only": 1.94573862465116, + "effective_sample_size": 630.228235498893 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914814, 0, 1.04128383798297, 2.31517239203707], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [-0.945688714374791, -0.945688714374791, -0.945688714374792, -0.945688714374792, -0.945688714374791, -0.945688714374791, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": -1.52655665885959e-16, + "pretrend_bias": -0.0613032635598606, + "post_only": 1.94977248446809, + "effective_sample_size": 719.999999999998 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 5, 5], + "time": [3, 4, 5, 6, 5, 6], + "post": [1, 1, 1, 1, 1, 1], + "att": [1.28404655298829, 2.32056347203679, 3.18519253119276, 4.15837159652383, 0.966820926227897, 2.20786930029771], + "weight": [0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.230769230769231, 0.230769230769231], + "ess": [119.903588727274, 119.903588727274, 119.903588727274, 119.903588727274, 115.072096311797, 115.072096311797] + }, + "estimate": 2.20641349591344, + "decomposition": 2.20641349591344, + "remainder": 0, + "pretrend_bias": 0, + "post_only": 2.20641349591344, + "effective_sample_size": 706.0420149054 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552006329079015, 0.638635766785036, -0.0535601307184772, 0.640404316629236, -0.0544923109469719, 0.6401601593518, -0.0577693387390503, 0.637312816179378, -0.032929972143668, 0.648304910559683, -0.0616076261294278, 0.637080053522665, 0.0835621488155271, 0.737738482174229, 0.0945036039710708, 0.744157273744997, 0.108279525337989, 0.748815310739362, 0.103457592255963, 0.746870083930084, 0.104726058647783, 0.747708304746342, 0.101546993276873, 0.745734043929024], + "weighted_covs_comparison": [-0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288, -0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288], + "weighted_diff": [0.0268204827060351, -0.0550449074129595, 0.0260612967444623, -0.0539478803609912, 0.0145012620558822, -0.0711066429526097, -0.00168517891858527, -0.0507069305744083, 0.0470413292042998, -0.0459943870073296, 0.0220279248407949, -0.0560366792736232, 0.165583264429464, 0.0440578079762343, 0.17412503143401, 0.0498050767547699, 0.177273098340843, 0.0375485084349517, 0.159541752076428, 0.0588503371762982, 0.184697359995751, 0.0534090071793297, 0.185182544247096, 0.0526173111327358], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283], + "unweighted_log_ratio_sd_diff": [-0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277], + "weighted_log_ratio_sd_diff": [-0.327379355480894, -0.388856622667763, -0.332728063219466, -0.393001836482863, -0.482246153471369, -0.760575572777059, -0.444382241712145, -0.729047181913841, -0.341186304526637, -0.365744546430006, -0.294845966896052, -0.373086629641763, -0.277042097435864, -0.27985283207957, -0.331236252179694, -0.345749079434454, -0.457886921184364, -0.693111339953135, -0.430872341074444, -0.671839969125508, -0.316957716605812, -0.335463807510608, -0.312906075652989, -0.333392440708849], + "unweighted_frac_treated_extreme": [0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333], + "weighted_frac_treated_extreme": [0.0285714285714285, 0.0142857142857143, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0142857142857143, 0.0142857142857142, 0.0142857142857143, 0.0428571428571429, 0.0285714285714285, 0.0428571428571428, 0.0285714285714285, 0.133333333333333, 0.05, 0.133333333333333, 0.0333333333333333, 0.05, 0, 0.05, 0, 0.0833333333333333, 0.0333333333333333, 0.1, 0.0333333333333333] + }, + "summary": { + "unweighted_treat": [0.0202808028752963, 0.690196397646397], + "unweighted_untreat": [-0.081027435542576, 0.693933252829235], + "unweighted_diff": [0.101308238417872, -0.00373685518283795], + "weighted_treat": [0.0195872436833426, 0.689860002475752], + "weighted_untreat": [-0.0736280525335835, 0.696237410214999], + "weighted_diff": [0.0932152962169262, -0.00637740773924623], + "sd": [0.904821714009258, 0.497941950253664], + "unweighted_log_ratio_sd_diff": [-0.323553909442524, -0.3621867923957], + "weighted_log_ratio_sd_diff": [-0.378261456962234, -0.512396249204556], + "unweighted_frac_treated_extreme": [0.0701874388083532, 0.030807497552334], + "weighted_frac_treated_extreme": [0.0536362673342276, 0.0247417574823458], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5], + "time": [3, 3, 4, 4, 5, 5, 6, 6, 5, 5, 6, 6], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "weighted_covs_comparison": [-0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, 0.105591944111321, 0.789549339556244, 0.105591944111321, 0.789549339556244], + "weighted_diff": [-1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -8.32667268468867e-17, -0.041379069285629, -8.32667268468867e-17, -0.041379069285629], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283] + }, + "summary": { + "unweighted_treat": [0.0189883265541762, 0.689318084960332], + "unweighted_untreat": [-0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.100015762096752, -0.00461516786890477], + "weighted_treat": [0.0189883265541762, 0.689318084960332], + "weighted_untreat": [0.0189883265541762, 0.745177657936526], + "weighted_diff": [-4.5903451979694e-17, -0.0558595729761943], + "sd": [0.904574085367837, 0.497745508663157], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 1.28957044213691, + "ess": 119.995700148386, + "weights": [1.00925477055982, 0.998723185675059, 0.996634725503275, 1.0062962595469, 1.0116846563981, 1.00452028093292, 1.00771486087647, 1.00204953103351, 1.0081898745094, 0.993138351267652, 1.00064950601217, 1.00958832842553, 1.00708633443042, 1.00143658864871, 0.999489519214752, 1.00911008327078, 1.00168297069056, 0.983050385189857, 1.00668374658622, 0.994367888846623, 1.00306411999543, 1.00581032890592, 0.995834544735198, 1.00135304942198, 0.997982911644572, 1.00549542164811, 0.992021816202564, 0.99946715234167, 0.994504899838761, 1.00336245204938, 0.99810636855178, 1.0042481533285, 1.00202880076963, 0.99377550716937, 1.00083681709583, 0.993840646573631, 0.995025443722193, 0.996441015885937, 1.00837717223639, 1.00287990494642, 1.00165231794312, 0.996029676232405, 0.990238585162026, 0.995938530509862, 0.998372849179898, 1.00217444576362, 1.0017515355913, 0.997885202550526, 1.00957796620665, 0.99454446944383, 1.00781508610486, 0.989676661130244, 1.00195850082524, 1.00228415784846, 1.0044654432561, 1.01551328546879, 0.993101453522634, 1.00042397201405, 1.00324727474383, 0.996107026201761, 1.00279636023904, 0.999681571328118, 1.00552645669481, 1.00510148957087, 0.991265296747816, 0.999928753044178, 0.994735345537259, 0.998610758079682, 0.999659826801045, 1.00695181771713, 1.00087111390321, 0.99239696826285, 0.997596950451456, 1.00167680363745, 0.998186813694359, 1.00314547233257, 0.989000472345918, 0.989810225580615, 1.00068860595832, 1.00045926654644, 1.00185508614782, 0.997098887679209, 0.995092885869272, 1.00882365715098, 0.995943321465413, 0.995345394281881, 1.00242170714711, 1.01131485449591, 1.01546992314392, 1.00473931229751, 1.00454884206903, 1.00120579562608, 0.991933253970624, 0.992552423967053, 1.00103232626908, 0.996073666367965, 1.00606469413823, 0.993935153780701, 0.993405482710502, 1.00576004097157, 1.00404133753852, 0.990720622466242, 0.9895886575647, 0.990789224140744, 0.998229047704058, 0.993249994137338, 0.997704430441061, 1.00139407975238, 0.997384073985491, 0.998716973738327, 0.996509691218583, 0.994121921942389, 1.00069492728128, 0.99361118386715, 1.01106122493825, 0.999614467420359, 0.99719516770885, 1.00761213299192, 1.00044650582276, 0.996040442935408, 0.995776410452928, 1.00014559518799, 1.00163036922118, 0.998107414758181, 0.998929265083554, 1.00394146389334, 0.994663562354314, 0.997264291076991, 0.999722078445559, 1.00127725705156, 0.999660594616231, 0.997495701681234, 0.999221017580553, 0.999202469015296, 1.00137763212731, 1.00356440217487, 0.998483204711311, 0.999264607253516, 1.00003520861784, 1.00231366955048, 0.996020219541034, 1.00107946453243, 1.00122012240469, 0.995993541722388, 0.9957999534808, 1.00358672328067, 1.00377596409354, 1.00283713463745, 0.998897288822817, 1.00140206334268, 0.994859786413371, 0.999617444016502, 1.00558977187743, 1.00151742123166, 0.999440356527955, 1.00348353086931, 1.00103531753353, 0.994836036333979, 0.996484335173441, 1.00690040871755, 1.00252882679405, 0.996057521027822, 1.00074181334411, 0.999624789407533, 0.997053878174417, 0.999466692467566, 0.998408862902966, 0.998885822158993, 0.996357520999377, 0.999797669251639, 0.99870100074395, 1.00039444814759, 1.00012122673967, 1.00333196253064, 0.997811178378975, 1.00012878664617, 1.00344301864085, 0.998071892714265, 1.00306218854884, 1.00194832910166, 0.998881145839971, 1.00128524589817, 0.999049662363083, 0.995286955499897, 1.00136819116663, 0.9990008180002, 1.00499457305456, 0.999427617457224, 1.00038272399958, 1.00793253859212], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 1.29320632882505, + "ess": 119.995175832593, + "weights": [0.988121343500857, 1.00319460592959, 0.999326474631137, 0.995659632023005, 0.998606433108666, 0.995983391028368, 0.997905454026944, 0.993067190848352, 0.987447676543771, 1.00722546202046, 0.996098999745148, 0.995760987841229, 0.992551110480417, 1.00946334325196, 1.00213904078528, 0.99656654815872, 0.997707286846142, 1.01031273245395, 0.992552032760831, 1.00246403828434, 0.995416775257639, 1.00074394014255, 0.993277350509223, 1.00046363134566, 0.999387638777348, 0.995436337298158, 1.01002540383716, 0.997641596157817, 1.00142964949572, 1.00306186221818, 0.987568975485235, 0.998901146783634, 0.994210429294235, 1.00267630862969, 0.999430296647091, 0.994417627787845, 1.0006388237355, 0.997913197441718, 1.00474394702358, 0.998439282442818, 0.99809325945128, 0.996577959626109, 1.00807177937616, 0.997758940856196, 0.996365194057743, 0.992843437496156, 0.992961566287008, 1.01184445051893, 0.991172499125968, 1.00607581691, 0.988090533687459, 1.01404534961072, 1.00099910277687, 0.99727551050779, 0.997430908996597, 0.995323694271336, 1.01079586728789, 1.00383782318859, 0.990728396959983, 0.999165942059289, 0.991442682725426, 1.00480350190228, 0.999686306593479, 0.996424768681225, 1.00526516565322, 0.994278461080079, 1.00117423547719, 0.999522222423479, 1.00388724440051, 0.994125269190137, 0.988914815060044, 1.00873014851802, 1.0008025087164, 0.993333496799252, 1.002362069121, 0.995094506867751, 1.01554227443789, 1.01149662395533, 1.00848620855313, 1.00208384713849, 1.0059243759256, 1.00602450217315, 1.00674544005413, 0.994182090996886, 0.998685827626876, 1.00574936778508, 0.994171531804436, 0.990220250456699, 0.994937460421399, 1.00221250585952, 1.00502295762636, 1.00314061240821, 1.01594165711587, 1.00263250272366, 1.00423439904399, 0.999478219836675, 0.9869427697955, 1.00847994595626, 0.999754343768729, 0.990426930175441, 1.00285881267497, 1.00224635046947, 0.999841222797544, 1.01201466214112, 0.996777185410859, 0.994290009671401, 1.00355719467655, 1.00143411749026, 1.00053710082179, 1.01403368944758, 1.00088018587467, 1.00333167103173, 0.998578495421041, 1.00604281551186, 0.995915049400813, 0.996641880507851, 1.00573881013259, 1.0049267688703, 0.998445151041532, 1.00210874012526, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + } + } +} diff --git a/benchmarks/data/twfeweights_mpdta_panel.csv b/benchmarks/data/twfeweights_mpdta_panel.csv new file mode 100644 index 000000000..b88831b79 --- /dev/null +++ b/benchmarks/data/twfeweights_mpdta_panel.csv @@ -0,0 +1,2501 @@ +"unit","period","first_treat","outcome","lpop","lpop_t" +8001,2003,2007,8.46146904264388,5.8967609333053,1.17935218666106 +8001,2004,2007,8.33686963728496,5.8967609333053,2.35870437332212 +8001,2005,2007,8.34021732094704,5.8967609333053,3.53805655998318 +8001,2006,2007,8.37816098272068,5.8967609333053,4.71740874664424 +8001,2007,2007,8.48735234940522,5.8967609333053,5.8967609333053 +8019,2003,2007,4.99721227376411,2.23237719795055,0.44647543959011 +8019,2004,2007,5.08140436498446,2.23237719795055,0.89295087918022 +8019,2005,2007,4.78749174278205,2.23237719795055,1.33942631877033 +8019,2006,2007,4.99043258677874,2.23237719795055,1.78590175836044 +8019,2007,2007,5.03695260241363,2.23237719795055,2.23237719795055 +8023,2003,2007,2.19722457733622,1.29828248379668,0.259656496759335 +8023,2004,2007,2.30258509299405,1.29828248379668,0.519312993518671 +8023,2005,2007,2.77258872223978,1.29828248379668,0.778969490278006 +8023,2006,2007,1.94591014905531,1.29828248379668,1.03862598703734 +8023,2007,2007,2.99573227355399,1.29828248379668,1.29828248379668 +8029,2003,2007,5.88332238848828,3.32625829499766,0.665251658999532 +8029,2004,2007,5.78996017089725,3.32625829499766,1.33050331799906 +8029,2005,2007,5.85793315448346,3.32625829499766,1.9957549769986 +8029,2006,2007,5.99893656194668,3.32625829499766,2.66100663599813 +8029,2007,2007,6.10479323241498,3.32625829499766,3.32625829499766 +8041,2003,2007,8.96865090337487,6.24790553432335,1.24958110686467 +8041,2004,2007,8.9157008189569,6.24790553432335,2.49916221372934 +8041,2005,2007,8.90095787254506,6.24790553432335,3.74874332059401 +8041,2006,2007,8.90272766403552,6.24790553432335,4.99832442745868 +8041,2007,2007,8.88100262425557,6.24790553432335,6.24790553432335 +8063,2003,2007,4.9416424226093,2.08081559723298,0.416163119446596 +8063,2004,2007,4.86753445045558,2.08081559723298,0.832326238893192 +8063,2005,2007,4.66343909411207,2.08081559723298,1.24848935833979 +8063,2006,2007,4.81218435537242,2.08081559723298,1.66465247778638 +8063,2007,2007,4.91998092582813,2.08081559723298,2.08081559723298 +8075,2003,2007,5.6970934865054,3.02061998906177,0.604123997812354 +8075,2004,2007,5.63121178182137,3.02061998906177,1.20824799562471 +8075,2005,2007,5.72031177660741,3.02061998906177,1.81237199343706 +8075,2006,2007,5.79605775076537,3.02061998906177,2.41649599124942 +8075,2007,2007,5.56068163101553,3.02061998906177,3.02061998906177 +8103,2003,2007,4.45434729625351,1.78942340943051,0.357884681886101 +8103,2004,2007,4.58496747867057,1.78942340943051,0.715769363772203 +8103,2005,2007,4.75359019110636,1.78942340943051,1.0736540456583 +8103,2006,2007,4.90527477843843,1.78942340943051,1.43153872754441 +8103,2007,2007,4.71849887129509,1.78942340943051,1.78942340943051 +8117,2003,2007,6.37842618365159,3.15904089116601,0.631808178233203 +8117,2004,2007,6.27476202124194,3.15904089116601,1.26361635646641 +8117,2005,2007,6.2841341610708,3.15904089116601,1.89542453469961 +8117,2006,2007,6.3818160174061,3.15904089116601,2.52723271293281 +8117,2007,2007,6.43133108193348,3.15904089116601,3.15904089116601 +8123,2003,2007,7.7553388128465,5.19814337757751,1.0396286755155 +8123,2004,2007,7.68799716639302,5.19814337757751,2.079257351031 +8123,2005,2007,7.75405263903576,5.19814337757751,3.11888602654651 +8123,2006,2007,7.78945456608667,5.19814337757751,4.15851470206201 +8123,2007,2007,7.80302664363222,5.19814337757751,5.19814337757751 +12007,2003,2006,5.50125821054473,3.26147543848702,0.652295087697404 +12007,2004,2006,5.59471137960184,3.26147543848702,1.30459017539481 +12007,2005,2006,5.24702407216049,3.26147543848702,1.95688526309221 +12007,2006,2006,5.43372200355424,3.26147543848702,2.60918035078961 +12007,2007,2006,5.44241771052179,3.26147543848702,3.26147543848702 +12019,2003,2006,7.77275271646874,4.94743987059911,0.989487974119822 +12019,2004,2006,7.81116338502528,4.94743987059911,1.97897594823964 +12019,2005,2006,7.80139132029149,4.94743987059911,2.96846392235947 +12019,2006,2006,7.86326672400957,4.94743987059911,3.95795189647929 +12019,2007,2006,7.87016594646984,4.94743987059911,4.94743987059911 +12023,2003,2006,6.99117688712121,4.03447070018167,0.806894140036335 +12023,2004,2006,6.53813982376767,4.03447070018167,1.61378828007267 +12023,2005,2006,6.59030104819669,4.03447070018167,2.420682420109 +12023,2006,2006,6.5510803350434,4.03447070018167,3.22757656014534 +12023,2007,2006,6.71780469502369,4.03447070018167,4.03447070018167 +12029,2003,2006,3.85014760171006,2.62662320240648,0.525324640481296 +12029,2004,2006,4.07753744390572,2.62662320240648,1.05064928096259 +12029,2005,2006,4.35670882668959,2.62662320240648,1.57597392144389 +12029,2006,2006,4.31748811353631,2.62662320240648,2.10129856192518 +12029,2007,2006,4.33073334028633,2.62662320240648,2.62662320240648 +12051,2003,2006,6.42648845745769,3.58933532377755,0.71786706475551 +12051,2004,2006,6.42324696353352,3.58933532377755,1.43573412951102 +12051,2005,2006,6.37331978957701,3.58933532377755,2.15360119426653 +12051,2006,2006,6.4118182677099,3.58933532377755,2.87146825902204 +12051,2007,2006,6.37331978957701,3.58933532377755,3.58933532377755 +12071,2003,2006,8.75872660784204,6.08879087493748,1.2177581749875 +12071,2004,2006,8.79845469606509,6.08879087493748,2.43551634997499 +12071,2005,2006,8.99118884193151,6.08879087493748,3.65327452496249 +12071,2006,2006,9.1516513756275,6.08879087493748,4.87103269994998 +12071,2007,2006,9.06843112579349,6.08879087493748,6.08879087493748 +12087,2003,2006,6.72383244082121,4.37687589234631,0.875375178469263 +12087,2004,2006,6.79234442747081,4.37687589234631,1.75075035693853 +12087,2005,2006,6.75809450442773,4.37687589234631,2.62612553540779 +12087,2006,2006,6.74641212857337,4.37687589234631,3.50150071387705 +12087,2007,2006,6.74758652682932,4.37687589234631,4.37687589234631 +12093,2003,2006,5.72358510195238,3.58101580823799,0.716203161647598 +12093,2004,2006,6.01126717440416,3.58101580823799,1.4324063232952 +12093,2005,2006,6.10255859461357,3.58101580823799,2.14860948494279 +12093,2006,2006,6.0913098820777,3.58101580823799,2.86481264659039 +12093,2007,2006,6.14418563412565,3.58101580823799,3.58101580823799 +12097,2003,2006,7.59085212368858,5.15035665593789,1.03007133118758 +12097,2004,2006,7.59387784460512,5.15035665593789,2.06014266237516 +12097,2005,2006,7.76556908109732,5.15035665593789,3.09021399356273 +12097,2006,2006,7.87321705486274,5.15035665593789,4.12028532475031 +12097,2007,2006,7.84737183615979,5.15035665593789,5.15035665593789 +12099,2003,2006,9.76864100976636,7.03102015077071,1.40620403015414 +12099,2004,2006,9.62918227712599,7.03102015077071,2.81240806030828 +12099,2005,2006,9.72202562615406,7.03102015077071,4.21861209046242 +12099,2006,2006,9.81623978397185,7.03102015077071,5.62481612061657 +12099,2007,2006,9.89363921648131,7.03102015077071,7.03102015077071 +12109,2003,2006,7.55066124310534,4.81328131446834,0.962656262893668 +12109,2004,2006,7.62705741701893,4.81328131446834,1.92531252578734 +12109,2005,2006,7.71199650704767,4.81328131446834,2.887968788681 +12109,2006,2006,7.86288203464149,4.81328131446834,3.85062505157467 +12109,2007,2006,7.8811822022271,4.81328131446834,4.81328131446834 +12113,2003,2006,7.09090982207998,4.76850428314641,0.953700856629282 +12113,2004,2006,7.16394668434255,4.76850428314641,1.90740171325856 +12113,2005,2006,7.36073990305828,4.76850428314641,2.86110256988785 +12113,2006,2006,7.40306109109009,4.76850428314641,3.81480342651713 +12113,2007,2006,7.33367639565768,4.76850428314641,4.76850428314641 +12121,2003,2006,5.67332326717149,3.55088095595665,0.710176191191329 +12121,2004,2006,5.6970934865054,3.55088095595665,1.42035238238266 +12121,2005,2006,5.70044357339069,3.55088095595665,2.13052857357399 +12121,2006,2006,5.77765232322266,3.55088095595665,2.84070476476532 +12121,2007,2006,5.78382518232974,3.55088095595665,3.55088095595665 +13011,2003,0,5.27299955856375,2.66875481849457,0.533750963698914 +13011,2004,0,5.31320597904179,2.66875481849457,1.06750192739783 +13011,2005,0,5.49306144334055,2.66875481849457,1.60125289109674 +13011,2006,0,5.59842195899838,2.66875481849457,2.13500385479565 +13011,2007,0,5.57215403217776,2.66875481849457,2.66875481849457 +13013,2003,0,6.18826412308259,3.83176694166248,0.766353388332497 +13013,2004,0,6.18208490671663,3.83176694166248,1.53270677666499 +13013,2005,0,6.25575004175337,3.83176694166248,2.29906016499749 +13013,2006,0,6.4377516497364,3.83176694166248,3.06541355332999 +13013,2007,0,6.40191719672719,3.83176694166248,3.83176694166248 +13019,2003,0,5.56452040732269,2.78716940555481,0.557433881110962 +13019,2004,0,5.55682806169954,2.78716940555481,1.11486776222192 +13019,2005,0,5.65599181081985,2.78716940555481,1.67230164333289 +13019,2006,0,5.62762111369064,2.78716940555481,2.22973552444385 +13019,2007,0,4.99721227376411,2.78716940555481,2.78716940555481 +13021,2003,0,7.73892375743946,5.03621856684166,1.00724371336833 +13021,2004,0,7.72797554210556,5.03621856684166,2.01448742673666 +13021,2005,0,7.73149202924568,5.03621856684166,3.02173114010499 +13021,2006,0,7.70975686445416,5.03621856684166,4.02897485347333 +13021,2007,0,7.60738142563979,5.03621856684166,5.03621856684166 +13037,2003,0,2.77258872223978,1.84371920815877,0.368743841631753 +13037,2004,0,3.04452243772342,1.84371920815877,0.737487683263506 +13037,2005,0,2.70805020110221,1.84371920815877,1.10623152489526 +13037,2006,0,3.17805383034795,1.84371920815877,1.47497536652701 +13037,2007,0,3.46573590279973,1.84371920815877,1.84371920815877 +13039,2003,0,6.22455842927536,3.77652396396598,0.755304792793197 +13039,2004,0,6.29156913955832,3.77652396396598,1.51060958558639 +13039,2005,0,6.35610766069589,3.77652396396598,2.26591437837959 +13039,2006,0,6.34212141872115,3.77652396396598,3.02121917117279 +13039,2007,0,6.29341927884648,3.77652396396598,3.77652396396598 +13049,2003,0,4.62497281328427,2.33039479363331,0.466078958726663 +13049,2004,0,4.46590811865458,2.33039479363331,0.932157917453325 +13049,2005,0,4.66343909411207,2.33039479363331,1.39823687617999 +13049,2006,0,4.70048036579242,2.33039479363331,1.86431583490665 +13049,2007,0,4.75359019110636,2.33039479363331,2.33039479363331 +13057,2003,0,7.42237370098682,4.95514372559156,0.991028745118313 +13057,2004,0,7.49997654095212,4.95514372559156,1.98205749023663 +13057,2005,0,7.62608275807238,4.95514372559156,2.97308623535494 +13057,2006,0,7.71378461659875,4.95514372559156,3.96411498047325 +13057,2007,0,7.80343505695217,4.95514372559156,4.95514372559156 +13077,2003,0,7.11314210870709,4.49104918688339,0.898209837376678 +13077,2004,0,7.07834157955767,4.49104918688339,1.79641967475336 +13077,2005,0,7.10414409298753,4.49104918688339,2.69462951213004 +13077,2006,0,7.16162200293919,4.49104918688339,3.59283934950671 +13077,2007,0,7.26332961747684,4.49104918688339,4.49104918688339 +13079,2003,0,4.02535169073515,2.52532856428692,0.505065712857383 +13079,2004,0,3.66356164612965,2.52532856428692,1.01013142571477 +13079,2005,0,3.43398720448515,2.52532856428692,1.51519713857215 +13079,2006,0,3.40119738166216,2.52532856428692,2.02026285142953 +13079,2007,0,3.13549421592915,2.52532856428692,2.52532856428692 +13093,2003,0,3.76120011569356,2.44451858888271,0.488903717776542 +13093,2004,0,3.73766961828337,2.44451858888271,0.977807435553085 +13093,2005,0,3.63758615972639,2.44451858888271,1.46671115332963 +13093,2006,0,4.0943445622221,2.44451858888271,1.95561487110617 +13093,2007,0,4.34380542185368,2.44451858888271,2.44451858888271 +13097,2003,0,7.42714413340862,4.52367809513269,0.904735619026538 +13097,2004,0,7.46794233228585,4.52367809513269,1.80947123805308 +13097,2005,0,7.57507169950756,4.52367809513269,2.71420685707961 +13097,2006,0,7.71512360363211,4.52367809513269,3.61894247610615 +13097,2007,0,7.82763954636642,4.52367809513269,4.52367809513269 +13109,2003,0,5.19849703126583,2.3508989532726,0.470179790654519 +13109,2004,0,5.15905529921453,2.3508989532726,0.940359581309039 +13109,2005,0,5.21493575760899,2.3508989532726,1.41053937196356 +13109,2006,0,5.13579843705026,2.3508989532726,1.88071916261808 +13109,2007,0,5.20948615284142,2.3508989532726,2.3508989532726 +13113,2003,0,7.42535788702715,4.51374544806874,0.902749089613749 +13113,2004,0,7.44366368311559,4.51374544806874,1.8054981792275 +13113,2005,0,7.45182223652793,4.51374544806874,2.70824726884125 +13113,2006,0,7.51752085060303,4.51374544806874,3.610996358455 +13113,2007,0,7.63723438878947,4.51374544806874,4.51374544806874 +13117,2003,0,7.33106030521863,4.58911193973939,0.917822387947878 +13117,2004,0,7.38894609761844,4.58911193973939,1.83564477589576 +13117,2005,0,7.52779398772144,4.58911193973939,2.75346716384363 +13117,2006,0,7.64730883235624,4.58911193973939,3.67128955179151 +13117,2007,0,7.63819824428578,4.58911193973939,4.58911193973939 +13129,2003,0,6.44571981938558,3.78655048128738,0.757310096257476 +13129,2004,0,6.37672694789863,3.78655048128738,1.51462019251495 +13129,2005,0,6.361302477573,3.78655048128738,2.27193028877243 +13129,2006,0,6.35437004079735,3.78655048128738,3.02924038502991 +13129,2007,0,6.39859493453521,3.78655048128738,3.78655048128738 +13135,2003,0,9.19482056031815,6.37748856255844,1.27549771251169 +13135,2004,0,9.14590851181679,6.37748856255844,2.55099542502338 +13135,2005,0,9.19482056031815,6.37748856255844,3.82649313753507 +13135,2006,0,9.29504909990314,6.37748856255844,5.10199085004675 +13135,2007,0,9.29569203907569,6.37748856255844,6.37748856255844 +13147,2003,0,5.5834963087817,3.13536377263918,0.627072754527837 +13147,2004,0,5.44241771052179,3.13536377263918,1.25414550905567 +13147,2005,0,5.5834963087817,3.13536377263918,1.88121826358351 +13147,2006,0,5.51342874616498,3.13536377263918,2.50829101811135 +13147,2007,0,5.57215403217776,3.13536377263918,3.13536377263918 +13155,2003,0,4.06044301054642,2.29566117792122,0.459132235584244 +13155,2004,0,3.8286413964891,2.29566117792122,0.918264471168489 +13155,2005,0,4.12713438504509,2.29566117792122,1.37739670675273 +13155,2006,0,3.8286413964891,2.29566117792122,1.83652894233698 +13155,2007,0,3.87120101090789,2.29566117792122,2.29566117792122 +13161,2003,0,5.01063529409626,2.5403413566761,0.50806827133522 +13161,2004,0,4.91265488573605,2.5403413566761,1.01613654267044 +13161,2005,0,5.07517381523383,2.5403413566761,1.52420481400566 +13161,2006,0,5.19849703126583,2.5403413566761,2.03227308534088 +13161,2007,0,5.23644196282995,2.5403413566761,2.5403413566761 +13179,2003,0,6.02344759296103,4.12082419502648,0.824164839005296 +13179,2004,0,5.91620206260743,4.12082419502648,1.64832967801059 +13179,2005,0,6.17586727010576,4.12082419502648,2.47249451701589 +13179,2006,0,6.13556489108174,4.12082419502648,3.29665935602118 +13179,2007,0,6.15060276844628,4.12082419502648,4.12082419502648 +13195,2003,0,4.75359019110636,3.24765762629365,0.649531525258731 +13195,2004,0,4.77912349311153,3.24765762629365,1.29906305051746 +13195,2005,0,4.66343909411207,3.24765762629365,1.94859457577619 +13195,2006,0,4.87519732320115,3.24765762629365,2.59812610103492 +13195,2007,0,5.00394630594546,3.24765762629365,3.24765762629365 +13197,2003,0,3.73766961828337,1.9662728435742,0.39325456871484 +13197,2004,0,3.63758615972639,1.9662728435742,0.786509137429679 +13197,2005,0,3.52636052461616,1.9662728435742,1.17976370614452 +13197,2006,0,3.89182029811063,1.9662728435742,1.57301827485936 +13197,2007,0,3.95124371858143,1.9662728435742,1.9662728435742 +13209,2003,0,3.8286413964891,2.1126345090356,0.42252690180712 +13209,2004,0,3.40119738166216,2.1126345090356,0.84505380361424 +13209,2005,0,3.55534806148941,2.1126345090356,1.26758070542136 +13209,2006,0,3.66356164612965,2.1126345090356,1.69010760722848 +13209,2007,0,3.58351893845611,2.1126345090356,2.1126345090356 +13211,2003,0,5.19295685089021,2.73806197517018,0.547612395034036 +13211,2004,0,5.18738580584075,2.73806197517018,1.09522479006807 +13211,2005,0,5.28320372873799,2.73806197517018,1.64283718510211 +13211,2006,0,5.605802066296,2.73806197517018,2.19044958013615 +13211,2007,0,5.59471137960184,2.73806197517018,2.73806197517018 +13213,2003,0,5.71042701737487,3.59747663064059,0.719495326128119 +13213,2004,0,5.68357976733868,3.59747663064059,1.43899065225624 +13213,2005,0,5.38449506278909,3.59747663064059,2.15848597838436 +13213,2006,0,5.72031177660741,3.59747663064059,2.87798130451247 +13213,2007,0,5.65248918026865,3.59747663064059,3.59747663064059 +13217,2003,0,6.45204895443723,4.12715051394728,0.825430102789456 +13217,2004,0,6.49677499018586,4.12715051394728,1.65086020557891 +13217,2005,0,6.51619307604296,4.12715051394728,2.47629030836837 +13217,2006,0,6.62804137617953,4.12715051394728,3.30172041115782 +13217,2007,0,6.62539236800796,4.12715051394728,4.12715051394728 +13223,2003,0,6.38519439899773,4.40278468775839,0.880556937551678 +13223,2004,0,6.48463523563525,4.40278468775839,1.76111387510336 +13223,2005,0,6.71295620067707,4.40278468775839,2.64167081265503 +13223,2006,0,6.80682936039218,4.40278468775839,3.52222775020671 +13223,2007,0,6.98471632011827,4.40278468775839,4.40278468775839 +13227,2003,0,5.32300997913841,3.13475481220279,0.626950962440559 +13227,2004,0,5.4380793089232,3.13475481220279,1.25390192488112 +13227,2005,0,5.37527840768417,3.13475481220279,1.88085288732168 +13227,2006,0,5.40717177146012,3.13475481220279,2.50780384976223 +13227,2007,0,5.4510384535657,3.13475481220279,3.13475481220279 +13233,2003,0,5.96614673912369,3.64092269256804,0.728184538513609 +13233,2004,0,5.92692602597041,3.64092269256804,1.45636907702722 +13233,2005,0,5.94803498918065,3.64092269256804,2.18455361554083 +13233,2006,0,5.90808293816893,3.64092269256804,2.91273815405443 +13233,2007,0,5.91079664404053,3.64092269256804,3.64092269256804 +13237,2003,0,5.00394630594546,2.9344949640828,0.58689899281656 +13237,2004,0,5.03695260241363,2.9344949640828,1.17379798563312 +13237,2005,0,5.08759633523238,2.9344949640828,1.76069697844968 +13237,2006,0,5.52545293913178,2.9344949640828,2.34759597126624 +13237,2007,0,5.4249500174814,2.9344949640828,2.9344949640828 +13241,2003,0,5.4510384535657,2.71137799119488,0.542275598238977 +13241,2004,0,5.26269018890489,2.71137799119488,1.08455119647795 +13241,2005,0,5.4249500174814,2.71137799119488,1.62682679471693 +13241,2006,0,5.37063802812766,2.71137799119488,2.16910239295591 +13241,2007,0,5.40717177146012,2.71137799119488,2.71137799119488 +13263,2003,0,2.39789527279837,1.87149443724691,0.374298887449382 +13263,2004,0,2.77258872223978,1.87149443724691,0.748597774898763 +13263,2005,0,1.94591014905531,1.87149443724691,1.12289666234815 +13263,2006,0,1.79175946922805,1.87149443724691,1.49719554979753 +13263,2007,0,2.30258509299405,1.87149443724691,1.87149443724691 +13271,2003,0,4.68213122712422,2.46759092757831,0.493518185515662 +13271,2004,0,4.86753445045558,2.46759092757831,0.987036371031325 +13271,2005,0,5.16478597392351,2.46759092757831,1.48055455654699 +13271,2006,0,5.28320372873799,2.46759092757831,1.97407274206265 +13271,2007,0,5.19295685089021,2.46759092757831,2.46759092757831 +13273,2003,0,3.98898404656427,2.39516427428714,0.479032854857428 +13273,2004,0,4.0943445622221,2.39516427428714,0.958065709714856 +13273,2005,0,4.17438726989564,2.39516427428714,1.43709856457228 +13273,2006,0,4.18965474202643,2.39516427428714,1.91613141942971 +13273,2007,0,4.0943445622221,2.39516427428714,2.39516427428714 +13297,2003,0,6.53233429222235,4.10572950708866,0.821145901417733 +13297,2004,0,6.4085287910595,4.10572950708866,1.64229180283547 +13297,2005,0,6.35437004079735,4.10572950708866,2.4634377042532 +13297,2006,0,6.52649485957079,4.10572950708866,3.28458360567093 +13297,2007,0,6.62936325343745,4.10572950708866,4.10572950708866 +13303,2003,0,5.11799381241676,3.0528684649248,0.61057369298496 +13303,2004,0,4.99043258677874,3.0528684649248,1.22114738596992 +13303,2005,0,4.969813299576,3.0528684649248,1.83172107895488 +13303,2006,0,4.93447393313069,3.0528684649248,2.44229477193984 +13303,2007,0,5.04342511691925,3.0528684649248,3.0528684649248 +13313,2003,0,7.15148546390474,4.42514598824282,0.885029197648564 +13313,2004,0,7.15539630189673,4.42514598824282,1.77005839529713 +13313,2005,0,7.20042489294496,4.42514598824282,2.65508759294569 +13313,2006,0,7.27586460054653,4.42514598824282,3.54011679059426 +13313,2007,0,7.18841273649695,4.42514598824282,4.42514598824282 +13317,2003,0,4.92725368515721,2.36902804954276,0.473805609908552 +13317,2004,0,4.74493212836325,2.36902804954276,0.947611219817104 +13317,2005,0,4.64439089914137,2.36902804954276,1.42141682972566 +13317,2006,0,4.59511985013459,2.36902804954276,1.89522243963421 +13317,2007,0,4.54329478227,2.36902804954276,2.36902804954276 +13321,2003,0,4.96284463025991,3.08954132723205,0.61790826544641 +13321,2004,0,4.97673374242057,3.08954132723205,1.23581653089282 +13321,2005,0,5.08759633523238,3.08954132723205,1.85372479633923 +13321,2006,0,5.15329159449778,3.08954132723205,2.47163306178564 +13321,2007,0,4.89034912822175,3.08954132723205,3.08954132723205 +16003,2003,0,3.25809653802148,1.24588220740315,0.249176441480629 +16003,2004,0,3.61091791264422,1.24588220740315,0.498352882961258 +16003,2005,0,3.98898404656427,1.24588220740315,0.747529324441887 +16003,2006,0,3.98898404656427,1.24588220740315,0.996705765922517 +16003,2007,0,3.89182029811063,1.24588220740315,1.24588220740315 +16015,2003,0,3.89182029811063,1.89761985992753,0.379523971985506 +16015,2004,0,3.91202300542815,1.89761985992753,0.759047943971013 +16015,2005,0,4.17438726989564,1.89761985992753,1.13857191595652 +16015,2006,0,4.02535169073515,1.89761985992753,1.51809588794203 +16015,2007,0,4.14313472639153,1.89761985992753,1.89761985992753 +16017,2003,0,6.29710931993394,3.60644848013478,0.721289696026955 +16017,2004,0,6.29894924685594,3.60644848013478,1.44257939205391 +16017,2005,0,6.4281052726846,3.60644848013478,2.16386908808087 +16017,2006,0,6.42971947803914,3.60644848013478,2.88515878410782 +16017,2007,0,6.40522845803084,3.60644848013478,3.60644848013478 +16021,2003,0,4.89783979995091,2.28960116543575,0.45792023308715 +16021,2004,0,4.83628190695148,2.28960116543575,0.9158404661743 +16021,2005,0,4.96284463025991,2.28960116543575,1.37376069926145 +16021,2006,0,4.99721227376411,2.28960116543575,1.8316809323486 +16021,2007,0,4.82028156560504,2.28960116543575,2.28960116543575 +16029,2003,0,4.9416424226093,1.98842214329267,0.397684428658533 +16029,2004,0,4.79579054559674,1.98842214329267,0.795368857317067 +16029,2005,0,4.98360662170834,1.98842214329267,1.1930532859756 +16029,2006,0,4.78749174278205,1.98842214329267,1.59073771463413 +16029,2007,0,5.03695260241363,1.98842214329267,1.98842214329267 +16041,2003,0,5.63835466933375,2.42736580989135,0.48547316197827 +16041,2004,0,5.53733426701854,2.42736580989135,0.970946323956541 +16041,2005,0,5.62762111369064,2.42736580989135,1.45641948593481 +16041,2006,0,5.6937321388027,2.42736580989135,1.94189264791308 +16041,2007,0,5.74939298590825,2.42736580989135,2.42736580989135 +16045,2003,0,5.01727983681492,2.72004464595058,0.544008929190116 +16045,2004,0,4.92725368515721,2.72004464595058,1.08801785838023 +16045,2005,0,5.12989871492307,2.72004464595058,1.63202678757035 +16045,2006,0,5.08759633523238,2.72004464595058,2.17603571676046 +16045,2007,0,5.19849703126583,2.72004464595058,2.72004464595058 +16067,2003,0,5.73334127689775,3.00439464663264,0.600878929326529 +16067,2004,0,5.73979291217923,3.00439464663264,1.20175785865306 +16067,2005,0,5.73334127689775,3.00439464663264,1.80263678797959 +16067,2006,0,5.77455154554441,3.00439464663264,2.40351571730611 +16067,2007,0,5.80814248998044,3.00439464663264,3.00439464663264 +16079,2003,0,5.09375020080676,2.62256493174499,0.524512986348998 +16079,2004,0,5.17614973257383,2.62256493174499,1.049025972698 +16079,2005,0,5.14749447681345,2.62256493174499,1.57353895904699 +16079,2006,0,5.24702407216049,2.62256493174499,2.09805194539599 +16079,2007,0,5.38449506278909,2.62256493174499,2.62256493174499 +16083,2003,0,7.20489251020467,4.16331076668682,0.832662153337365 +16083,2004,0,7.18311170174328,4.16331076668682,1.66532430667473 +16083,2005,0,7.14045304310116,4.16331076668682,2.49798646001209 +16083,2006,0,7.23489842031483,4.16331076668682,3.33064861334946 +16083,2007,0,7.28961052145117,4.16331076668682,4.16331076668682 +17005,2003,2004,5.18738580584075,2.86977214642579,0.573954429285157 +17005,2004,2004,5.06890420222023,2.86977214642579,1.14790885857031 +17005,2005,2004,4.93447393313069,2.86977214642579,1.72186328785547 +17005,2006,2004,4.89034912822175,2.86977214642579,2.29581771714063 +17005,2007,2004,4.99043258677874,2.86977214642579,2.86977214642579 +17015,2003,2004,5.42934562895444,2.81385061998842,0.562770123997684 +17015,2004,2004,5.42053499927229,2.81385061998842,1.12554024799537 +17015,2005,2004,5.50125821054473,2.81385061998842,1.68831037199305 +17015,2006,2004,5.46806014113513,2.81385061998842,2.25108049599074 +17015,2007,2004,5.4553211153577,2.81385061998842,2.81385061998842 +17025,2003,2004,5.08759633523238,2.67827804276854,0.535655608553708 +17025,2004,2004,5.00394630594546,2.67827804276854,1.07131121710742 +17025,2005,2004,4.85203026391962,2.67827804276854,1.60696682566112 +17025,2006,2004,4.98360662170834,2.67827804276854,2.14262243421483 +17025,2007,2004,4.97673374242057,2.67827804276854,2.67827804276854 +17035,2003,2004,4.92725368515721,2.42063475976786,0.484126951953572 +17035,2004,2004,4.59511985013459,2.42063475976786,0.968253903907144 +17035,2005,2004,4.44265125649032,2.42063475976786,1.45238085586072 +17035,2006,2004,4.30406509320417,2.42063475976786,1.93650780781429 +17035,2007,2004,4.46590811865458,2.42063475976786,2.42063475976786 +17047,2003,2004,4.35670882668959,1.94175868650426,0.388351737300852 +17047,2004,2004,4.39444915467244,1.94175868650426,0.776703474601705 +17047,2005,2004,4.20469261939097,1.94175868650426,1.16505521190256 +17047,2006,2004,4.00733318523247,1.94175868650426,1.55340694920341 +17047,2007,2004,4.12713438504509,1.94175868650426,1.94175868650426 +17049,2003,2004,6.99025650049388,3.53409524031307,0.706819048062614 +17049,2004,2004,6.90975328164481,3.53409524031307,1.41363809612523 +17049,2005,2004,6.87523208727658,3.53409524031307,2.12045714418784 +17049,2006,2004,6.8330317327862,3.53409524031307,2.82727619225046 +17049,2007,2004,6.87626461189077,3.53409524031307,3.53409524031307 +17053,2003,2004,5.61677109766657,2.65612512823729,0.531225025647458 +17053,2004,2004,5.48893772615669,2.65612512823729,1.06245005129492 +17053,2005,2004,5.48893772615669,2.65612512823729,1.59367507694237 +17053,2006,2004,5.51342874616498,2.65612512823729,2.12490010258983 +17053,2007,2004,5.31320597904179,2.65612512823729,2.65612512823729 +17063,2003,2004,6.58063913728495,3.62527383102497,0.725054766204993 +17063,2004,2004,6.47697236288968,3.62527383102497,1.45010953240999 +17063,2005,2004,6.49375383985169,3.62527383102497,2.17516429861498 +17063,2006,2004,6.4831073514572,3.62527383102497,2.90021906481997 +17063,2007,2004,6.5694814204143,3.62527383102497,3.62527383102497 +17089,2003,2004,9.0067543198775,6.00170938904395,1.20034187780879 +17089,2004,2004,8.96763166516713,6.00170938904395,2.40068375561758 +17089,2005,2004,8.99118884193151,6.00170938904395,3.60102563342637 +17089,2006,2004,8.97802993283716,6.00170938904395,4.80136751123516 +17089,2007,2004,9.02509454366498,6.00170938904395,6.00170938904395 +17097,2003,2004,9.27369068747694,6.46825136840052,1.2936502736801 +17097,2004,2004,9.22147811638664,6.46825136840052,2.58730054736021 +17097,2005,2004,9.2072355570227,6.46825136840052,3.88095082104031 +17097,2006,2004,9.24599704918422,6.46825136840052,5.17460109472042 +17097,2007,2004,9.23824732522919,6.46825136840052,6.46825136840052 +17113,2003,2004,7.89095671613892,5.01351780234143,1.00270356046829 +17113,2004,2004,7.77653502818524,5.01351780234143,2.00540712093657 +17113,2005,2004,7.755767170103,5.01351780234143,3.00811068140486 +17113,2006,2004,7.74022952476318,5.01351780234143,4.01081424187314 +17113,2007,2004,7.8188324438034,5.01351780234143,5.01351780234143 +17125,2003,2004,5.24702407216049,2.77496090638484,0.554992181276967 +17125,2004,2004,5.25227342804663,2.77496090638484,1.10998436255393 +17125,2005,2004,4.97673374242057,2.77496090638484,1.6649765438309 +17125,2006,2004,4.99043258677874,2.77496090638484,2.21996872510787 +17125,2007,2004,5.05624580534831,2.77496090638484,2.77496090638484 +17135,2003,2004,6.07764224334903,3.42269791310816,0.684539582621632 +17135,2004,2004,6.10255859461357,3.42269791310816,1.36907916524326 +17135,2005,2004,6.15485809401642,3.42269791310816,2.05361874786489 +17135,2006,2004,6.0137151560428,3.42269791310816,2.73815833048653 +17135,2007,2004,6.01615715969835,3.42269791310816,3.42269791310816 +17155,2003,2004,4.02535169073515,1.80599105147478,0.361198210294956 +17155,2004,2004,3.73766961828337,1.80599105147478,0.722396420589912 +17155,2005,2004,3.91202300542815,1.80599105147478,1.08359463088487 +17155,2006,2004,3.78418963391826,1.80599105147478,1.44479284117982 +17155,2007,2004,4.24849524204936,1.80599105147478,1.80599105147478 +17161,2003,2004,7.87663846097546,5.0064532281027,1.00129064562054 +17161,2004,2004,7.82404601085629,5.0064532281027,2.00258129124108 +17161,2005,2004,7.86557175768479,5.0064532281027,3.00387193686162 +17161,2006,2004,7.85166117788927,5.0064532281027,4.00516258248216 +17161,2007,2004,7.82923253754359,5.0064532281027,5.0064532281027 +17169,2003,2004,4.52178857704904,1.97255208000173,0.394510416000347 +17169,2004,2004,4.53259949315326,1.97255208000173,0.789020832000694 +17169,2005,2004,4.31748811353631,1.97255208000173,1.18353124800104 +17169,2006,2004,4.38202663467388,1.97255208000173,1.57804166400139 +17169,2007,2004,4.43081679884331,1.97255208000173,1.97255208000173 +17193,2003,2004,5.4510384535657,2.7324826172428,0.546496523448559 +17193,2004,2004,5.3890717298165,2.7324826172428,1.09299304689712 +17193,2005,2004,5.25227342804663,2.7324826172428,1.63948957034568 +17193,2006,2004,5.17614973257383,2.7324826172428,2.18598609379424 +17193,2007,2004,5.27299955856375,2.7324826172428,2.7324826172428 +17195,2003,2004,6.71780469502369,4.10516909830525,0.82103381966105 +17195,2004,2004,6.6682282484174,4.10516909830525,1.6420676393221 +17195,2005,2004,6.76849321164863,4.10516909830525,2.46310145898315 +17195,2006,2004,6.68959926917897,4.10516909830525,3.2841352786442 +17195,2007,2004,6.73221070646721,4.10516909830525,4.10516909830525 +17199,2003,2004,6.81014245011514,4.11571458795829,0.823142917591659 +17199,2004,2004,6.8596149036542,4.11571458795829,1.64628583518332 +17199,2005,2004,6.77764659363512,4.11571458795829,2.46942875277498 +17199,2006,2004,6.72982407048948,4.11571458795829,3.29257167036663 +17199,2007,2004,6.77878489768518,4.11571458795829,4.11571458795829 +17203,2003,2004,6.5191472879404,3.56865907554909,0.713731815109817 +17203,2004,2004,6.44094654063292,3.56865907554909,1.42746363021963 +17203,2005,2004,6.41673228251233,3.56865907554909,2.14119544532945 +17203,2006,2004,6.46925031679577,3.56865907554909,2.85492726043927 +17203,2007,2004,6.48616078894409,3.56865907554909,3.56865907554909 +18005,2003,0,7.15305163493748,4.26878794531712,0.853757589063424 +18005,2004,0,7.11720550316434,4.26878794531712,1.70751517812685 +18005,2005,0,7.16626597413364,4.26878794531712,2.56127276719027 +18005,2006,0,7.10002716662926,4.26878794531712,3.4150303562537 +18005,2007,0,7.08170858610557,4.26878794531712,4.26878794531712 +18017,2003,0,6.29156913955832,3.71186329050431,0.742372658100862 +18017,2004,0,6.20455776256869,3.71186329050431,1.48474531620172 +18017,2005,0,6.10255859461357,3.71186329050431,2.22711797430259 +18017,2006,0,6.0913098820777,3.71186329050431,2.96949063240345 +18017,2007,0,6.06145691892802,3.71186329050431,3.71186329050431 +18019,2003,0,7.40610338123702,4.56925281080129,0.913850562160258 +18019,2004,0,7.31388683163346,4.56925281080129,1.82770112432052 +18019,2005,0,7.27931883541462,4.56925281080129,2.74155168648077 +18019,2006,0,7.40974195408092,4.56925281080129,3.65540224864103 +18019,2007,0,7.36897040219479,4.56925281080129,4.56925281080129 +18021,2003,0,5.47227067367148,3.27925571085691,0.655851142171383 +18021,2004,0,5.73657229747919,3.27925571085691,1.31170228434277 +18021,2005,0,5.75574221358691,3.27925571085691,1.96755342651415 +18021,2006,0,5.60947179518496,3.27925571085691,2.62340456868553 +18021,2007,0,5.56068163101553,3.27925571085691,3.27925571085691 +18023,2003,0,6.12029741895095,3.52241156124315,0.70448231224863 +18023,2004,0,6.07073772800249,3.52241156124315,1.40896462449726 +18023,2005,0,6.04263283368238,3.52241156124315,2.11344693674589 +18023,2006,0,6.00141487796115,3.52241156124315,2.81792924899452 +18023,2007,0,6.02344759296103,3.52241156124315,3.52241156124315 +18029,2003,0,6.80239476332431,3.83100815871387,0.766201631742774 +18029,2004,0,6.74405918631135,3.83100815871387,1.53240326348555 +18029,2005,0,6.75693238924755,3.83100815871387,2.29860489522832 +18029,2006,0,6.64898455002478,3.83100815871387,3.0648065269711 +18029,2007,0,6.81014245011514,3.83100815871387,3.83100815871387 +18033,2003,0,6.50727771238501,3.69597919122916,0.739195838245832 +18033,2004,0,6.42162226780652,3.69597919122916,1.47839167649166 +18033,2005,0,6.43615036836943,3.69597919122916,2.2175875147375 +18033,2006,0,6.51174532964473,3.69597919122916,2.95678335298333 +18033,2007,0,6.37842618365159,3.69597919122916,3.69597919122916 +18035,2003,0,7.34729970074316,4.77718043011612,0.955436086023223 +18035,2004,0,7.22766249872865,4.77718043011612,1.91087217204645 +18035,2005,0,7.1929342212158,4.77718043011612,2.86630825806967 +18035,2006,0,7.18992217074581,4.77718043011612,3.82174434409289 +18035,2007,0,7.17472430983638,4.77718043011612,4.77718043011612 +18053,2003,0,6.87109129461055,4.29596480671984,0.859192961343968 +18053,2004,0,6.78105762593618,4.29596480671984,1.71838592268794 +18053,2005,0,6.75809450442773,4.29596480671984,2.5775788840319 +18053,2006,0,6.67076632084587,4.29596480671984,3.43677184537587 +18053,2007,0,6.66568371778241,4.29596480671984,4.29596480671984 +18063,2003,0,7.56060116276856,4.64528473032447,0.929056946064893 +18063,2004,0,7.59789795052178,4.64528473032447,1.85811389212979 +18063,2005,0,7.64444076155657,4.64528473032447,2.78717083819468 +18063,2006,0,7.70616297019958,4.64528473032447,3.71622778425957 +18063,2007,0,7.74066440191724,4.64528473032447,4.64528473032447 +18081,2003,0,7.74975340627444,4.74674787021019,0.949349574042038 +18081,2004,0,7.73805229768932,4.74674787021019,1.89869914808408 +18081,2005,0,7.71868549519847,4.74674787021019,2.84804872212611 +18081,2006,0,7.7106533235012,4.74674787021019,3.79739829616815 +18081,2007,0,7.71646080017636,4.74674787021019,4.74674787021019 +18083,2003,0,6.33150184989369,3.6701042987876,0.734020859757521 +18083,2004,0,6.24997524225948,3.6701042987876,1.46804171951504 +18083,2005,0,6.24804287450843,3.6701042987876,2.20206257927256 +18083,2006,0,6.29341927884648,3.6701042987876,2.93608343903008 +18083,2007,0,6.18208490671663,3.6701042987876,3.6701042987876 +18091,2003,0,7.43543801981455,4.70144353815659,0.940288707631318 +18091,2004,0,7.37337430991005,4.70144353815659,1.88057741526264 +18091,2005,0,7.39510754656249,4.70144353815659,2.82086612289396 +18091,2006,0,7.3790081276283,4.70144353815659,3.76115483052527 +18091,2007,0,7.37148929521428,4.70144353815659,4.70144353815659 +18093,2003,0,6.21060007702465,3.82694430506983,0.765388861013966 +18093,2004,0,6.15485809401642,3.82694430506983,1.53077772202793 +18093,2005,0,6.15697898558556,3.82694430506983,2.2961665830419 +18093,2006,0,6.15909538849193,3.82694430506983,3.06155544405587 +18093,2007,0,6.20253551718792,3.82694430506983,3.82694430506983 +18099,2003,0,6.57368016696065,3.80950289643767,0.761900579287535 +18099,2004,0,6.51471269087253,3.80950289643767,1.52380115857507 +18099,2005,0,6.48004456192665,3.80950289643767,2.28570173786261 +18099,2006,0,6.52356230614951,3.80950289643767,3.04760231715014 +18099,2007,0,6.45833828334479,3.80950289643767,3.80950289643768 +18109,2003,0,6.68085467879022,4.20004002177996,0.840008004355991 +18109,2004,0,6.63068338564237,4.20004002177996,1.68001600871198 +18109,2005,0,6.65929391968364,4.20004002177996,2.52002401306797 +18109,2006,0,6.61606518513282,4.20004002177996,3.36003201742396 +18109,2007,0,6.59714570188665,4.20004002177996,4.20004002177996 +18113,2003,0,6.49072353450251,3.83460185845956,0.766920371691912 +18113,2004,0,6.45676965557216,3.83460185845956,1.53384074338382 +18113,2005,0,6.3919171133926,3.83460185845956,2.30076111507574 +18113,2006,0,6.41999492814714,3.83460185845956,3.06768148676765 +18113,2007,0,6.39692965521615,3.83460185845956,3.83460185845956 +18115,2003,0,4.29045944114839,1.72686532931006,0.345373065862013 +18115,2004,0,4.26267987704132,1.72686532931006,0.690746131724026 +18115,2005,0,4.17438726989564,1.72686532931006,1.03611919758604 +18115,2006,0,4.04305126783455,1.72686532931006,1.38149226344805 +18115,2007,0,4.07753744390572,1.72686532931006,1.72686532931006 +18119,2003,0,5.28320372873799,3.08126756166025,0.616253512332051 +18119,2004,0,5.30330490805908,3.08126756166025,1.2325070246641 +18119,2005,0,5.35185813347607,3.08126756166025,1.84876053699615 +18119,2006,0,5.30826769740121,3.08126756166025,2.4650140493282 +18119,2007,0,5.27299955856375,3.08126756166025,3.08126756166025 +18133,2003,0,6.4425401664682,3.58404657700818,0.716809315401636 +18133,2004,0,6.47697236288968,3.58404657700818,1.43361863080327 +18133,2005,0,6.43294009273918,3.58404657700818,2.15042794620491 +18133,2006,0,6.28785856016178,3.58404657700818,2.86723726160654 +18133,2007,0,6.24610676548156,3.58404657700818,3.58404657700818 +18145,2003,0,6.48768401848461,3.77149577029117,0.754299154058234 +18145,2004,0,6.4723462945009,3.77149577029117,1.50859830811647 +18145,2005,0,6.31173480915291,3.77149577029117,2.2628974621747 +18145,2006,0,6.18826412308259,3.77149577029117,3.01719661623294 +18145,2007,0,6.13556489108174,3.77149577029117,3.77149577029117 +18157,2003,0,7.70616297019958,5.0036442469074,1.00072884938148 +18157,2004,0,7.6425241342329,5.0036442469074,2.00145769876296 +18157,2005,0,7.70751219460034,5.0036442469074,3.00218654814444 +18157,2006,0,7.73587031995257,5.0036442469074,4.00291539752592 +18157,2007,0,7.74196789982069,5.0036442469074,5.0036442469074 +18175,2003,0,5.35185813347607,3.30406220422892,0.660812440845785 +18175,2004,0,5.31811999384422,3.30406220422892,1.32162488169157 +18175,2005,0,5.22035582507832,3.30406220422892,1.98243732253735 +18175,2006,0,5.24702407216049,3.30406220422892,2.64324976338314 +18175,2007,0,5.27299955856375,3.30406220422892,3.30406220422892 +18179,2003,0,6.08449941307517,3.3178157727231,0.663563154544621 +18179,2004,0,5.94803498918065,3.3178157727231,1.32712630908924 +18179,2005,0,5.92958914338989,3.3178157727231,1.99068946363386 +18179,2006,0,6.05208916892442,3.3178157727231,2.65425261817848 +18179,2007,0,5.83188247728352,3.3178157727231,3.3178157727231 +18181,2003,0,5.53338948872752,3.22949919650533,0.645899839301066 +18181,2004,0,5.32300997913841,3.22949919650533,1.29179967860213 +18181,2005,0,5.28826703069454,3.22949919650533,1.9376995179032 +18181,2006,0,5.25749537202778,3.22949919650533,2.58359935720427 +18181,2007,0,5.23110861685459,3.22949919650533,3.22949919650533 +19013,2003,0,7.9043348420851,4.85212400952536,0.970424801905072 +19013,2004,0,7.83794891602528,4.85212400952536,1.94084960381014 +19013,2005,0,7.8407064517494,4.85212400952536,2.91127440571522 +19013,2006,0,7.85476918349913,4.85212400952536,3.88169920762029 +19013,2007,0,7.87929148508227,4.85212400952536,4.85212400952536 +19025,2003,0,5.46806014113513,2.40829554741616,0.481659109483232 +19025,2004,0,5.2257466737132,2.40829554741616,0.963318218966464 +19025,2005,0,5.18738580584075,2.40829554741616,1.4449773284497 +19025,2006,0,5.22035582507832,2.40829554741616,1.92663643793293 +19025,2007,0,5.15329159449778,2.40829554741616,2.40829554741616 +19069,2003,0,5.28320372873799,2.37061750338587,0.474123500677175 +19069,2004,0,5.21493575760899,2.37061750338587,0.948247001354349 +19069,2005,0,5.31320597904179,2.37061750338587,1.42237050203152 +19069,2006,0,5.19849703126583,2.37061750338587,1.8964940027087 +19069,2007,0,5.2040066870768,2.37061750338587,2.37061750338587 +19077,2003,0,5.05624580534831,2.4294820261824,0.485896405236479 +19077,2004,0,4.97673374242057,2.4294820261824,0.971792810472959 +19077,2005,0,4.9416424226093,2.4294820261824,1.45768921570944 +19077,2006,0,4.84418708645859,2.4294820261824,1.94358562094592 +19077,2007,0,4.87519732320115,2.4294820261824,2.4294820261824 +19083,2003,0,5.87493073085203,2.9344949640828,0.58689899281656 +19083,2004,0,5.78689738136671,2.9344949640828,1.17379798563312 +19083,2005,0,5.75257263882563,2.9344949640828,1.76069697844968 +19083,2006,0,5.77144112313002,2.9344949640828,2.34759597126624 +19083,2007,0,5.75257263882563,2.9344949640828,2.9344949640828 +19087,2003,0,6.20253551718792,3.0123927144471,0.60247854288942 +19087,2004,0,6.01126717440416,3.0123927144471,1.20495708577884 +19087,2005,0,6.0137151560428,3.0123927144471,1.80743562866826 +19087,2006,0,5.98896141688986,3.0123927144471,2.40991417155768 +19087,2007,0,5.92692602597041,3.0123927144471,3.0123927144471 +19111,2003,0,6.40687998606931,3.63895364534421,0.727790729068842 +19111,2004,0,6.32076829425058,3.63895364534421,1.45558145813768 +19111,2005,0,6.39359075395063,3.63895364534421,2.18337218720653 +19111,2006,0,6.43615036836943,3.63895364534421,2.91116291627537 +19111,2007,0,6.43294009273918,3.63895364534421,3.63895364534421 +19113,2003,0,8.30992298925832,5.25593686652208,1.05118737330442 +19113,2004,0,8.28702502516506,5.25593686652208,2.10237474660883 +19113,2005,0,8.25400859056484,5.25593686652208,3.15356211991325 +19113,2006,0,8.28526113406895,5.25593686652208,4.20474949321767 +19113,2007,0,8.36985260351753,5.25593686652208,5.25593686652208 +19123,2003,0,5.89989735358249,3.10615495480408,0.621230990960816 +19123,2004,0,5.8406416573734,3.10615495480408,1.24246198192163 +19123,2005,0,5.73657229747919,3.10615495480408,1.86369297288245 +19123,2006,0,5.80211837537706,3.10615495480408,2.48492396384327 +19123,2007,0,5.77455154554441,3.10615495480408,3.10615495480408 +19131,2003,0,5.26785815906333,2.38637461872373,0.477274923744746 +19131,2004,0,5.18738580584075,2.38637461872373,0.954549847489492 +19131,2005,0,5.27811465923052,2.38637461872373,1.43182477123424 +19131,2006,0,5.25749537202778,2.38637461872373,1.90909969497898 +19131,2007,0,5.31811999384422,2.38637461872373,2.38637461872373 +19137,2003,0,5.37527840768417,2.46563887943034,0.493127775886069 +19137,2004,0,5.44673737166631,2.46563887943034,0.986255551772137 +19137,2005,0,5.37989735354046,2.46563887943034,1.47938332765821 +19137,2006,0,5.4510384535657,2.46563887943034,1.97251110354427 +19137,2007,0,5.4553211153577,2.46563887943034,2.46563887943034 +19141,2003,0,6.08904487544685,2.71482718538123,0.542965437076247 +19141,2004,0,6.00141487796115,2.71482718538123,1.08593087415249 +19141,2005,0,6.0330862217988,2.71482718538123,1.62889631122874 +19141,2006,0,6.05208916892442,2.71482718538123,2.17186174830499 +19141,2007,0,5.88610403145016,2.71482718538123,2.71482718538123 +19153,2003,0,9.19136140537838,5.92586145952057,1.18517229190411 +19153,2004,0,9.12565356380899,5.92586145952057,2.37034458380823 +19153,2005,0,9.12956406194889,5.92586145952057,3.55551687571234 +19153,2006,0,9.14729407025855,5.92586145952057,4.74068916761646 +19153,2007,0,9.1557785839835,5.92586145952057,5.92586145952057 +19157,2003,0,6.3456363608286,2.9346544240454,0.58693088480908 +19157,2004,0,6.09582456243222,2.9346544240454,1.17386176961816 +19157,2005,0,6.06378520868761,2.9346544240454,1.76079265442724 +19157,2006,0,6.1463292576689,2.9346544240454,2.34772353923632 +19157,2007,0,6.15485809401642,2.9346544240454,2.9346544240454 +19159,2003,0,4.64439089914137,1.69909578436464,0.339819156872927 +19159,2004,0,4.53259949315326,1.69909578436464,0.679638313745854 +19159,2005,0,4.33073334028633,1.69909578436464,1.01945747061878 +19159,2006,0,4.27666611901606,1.69909578436464,1.35927662749171 +19159,2007,0,4.36944785246702,1.69909578436464,1.69909578436464 +19183,2003,0,6.09582456243222,3.02868337369368,0.605736674738735 +19183,2004,0,5.92692602597041,3.02868337369368,1.21147334947747 +19183,2005,0,5.88887795833288,3.02868337369368,1.81721002421621 +19183,2006,0,5.97888576490112,3.02868337369368,2.42294669895494 +19183,2007,0,6.12468339089421,3.02868337369368,3.02868337369368 +19185,2003,0,4.24849524204936,1.90657514365664,0.381315028731327 +19185,2004,0,3.93182563272433,1.90657514365664,0.762630057462655 +19185,2005,0,4.02535169073515,1.90657514365664,1.14394508619398 +19185,2006,0,4.06044301054642,1.90657514365664,1.52526011492531 +19185,2007,0,4.4188406077966,1.90657514365664,1.90657514365664 +19187,2003,0,6.73340189183736,3.6947372635981,0.738947452719619 +19187,2004,0,6.68336094576627,3.6947372635981,1.47789490543924 +19187,2005,0,6.64898455002478,3.6947372635981,2.21684235815886 +19187,2006,0,6.68336094576627,3.6947372635981,2.95578981087848 +19187,2007,0,6.67329796776765,3.6947372635981,3.6947372635981 +20013,2003,0,5.42934562895444,2.37248422037371,0.474496844074743 +20013,2004,0,5.45958551414416,2.37248422037371,0.948993688149485 +20013,2005,0,5.50125821054473,2.37248422037371,1.42349053222423 +20013,2006,0,5.31811999384422,2.37248422037371,1.89798737629897 +20013,2007,0,5.28320372873799,2.37248422037371,2.37248422037371 +20021,2003,0,5.59098698051086,3.11817112074657,0.623634224149314 +20021,2004,0,5.35658627467201,3.11817112074657,1.24726844829863 +20021,2005,0,5.54517744447956,3.11817112074657,1.87090267244794 +20021,2006,0,5.40267738187228,3.11817112074657,2.49453689659725 +20021,2007,0,5.31811999384422,3.11817112074657,3.11817112074657 +20027,2003,0,5.14166355650266,2.17724860168275,0.43544972033655 +20027,2004,0,5.23644196282995,2.17724860168275,0.870899440673099 +20027,2005,0,5.37063802812766,2.17724860168275,1.30634916100965 +20027,2006,0,5.39362754635236,2.17724860168275,1.7417988813462 +20027,2007,0,5.40267738187228,2.17724860168275,2.17724860168275 +20031,2003,0,5.28826703069454,2.18211093952617,0.436422187905234 +20031,2004,0,5.19849703126583,2.18211093952617,0.872844375810468 +20031,2005,0,4.94875989037817,2.18211093952617,1.3092665637157 +20031,2006,0,5.37527840768417,2.18211093952617,1.74568875162094 +20031,2007,0,5.39362754635236,2.18211093952617,2.18211093952617 +20045,2003,0,7.52240023138712,4.6047901137698,0.920958022753959 +20045,2004,0,7.3664451483276,4.6047901137698,1.84191604550792 +20045,2005,0,7.37525577800975,4.6047901137698,2.76287406826188 +20045,2006,0,7.40306109109009,4.6047901137698,3.68383209101584 +20045,2007,0,7.4265490723973,4.6047901137698,4.6047901137698 +20047,2003,0,3.66356164612965,1.2380843339547,0.247616866790941 +20047,2004,0,3.80666248977032,1.2380843339547,0.495233733581882 +20047,2005,0,3.63758615972639,1.2380843339547,0.742850600372822 +20047,2006,0,2.94443897916644,1.2380843339547,0.990467467163763 +20047,2007,0,3.8286413964891,1.2380843339547,1.2380843339547 +20063,2003,0,4.26267987704132,1.12102588350501,0.224205176701002 +20063,2004,0,4.18965474202643,1.12102588350501,0.448410353402004 +20063,2005,0,4.00733318523247,1.12102588350501,0.672615530103006 +20063,2006,0,4.04305126783455,1.12102588350501,0.896820706804008 +20063,2007,0,3.97029191355212,1.12102588350501,1.12102588350501 +20085,2003,0,5.4380793089232,2.53821042181031,0.507642084362062 +20085,2004,0,5.37527840768417,2.53821042181031,1.01528416872412 +20085,2005,0,5.3890717298165,2.53821042181031,1.52292625308619 +20085,2006,0,5.35185813347607,2.53821042181031,2.03056833744825 +20085,2007,0,5.40717177146012,2.53821042181031,2.53821042181031 +20095,2003,0,4.92725368515721,2.16021475170232,0.432042950340464 +20095,2004,0,4.91265488573605,2.16021475170232,0.864085900680928 +20095,2005,0,5.03695260241363,2.16021475170232,1.29612885102139 +20095,2006,0,4.969813299576,2.16021475170232,1.72817180136186 +20095,2007,0,5.05624580534831,2.16021475170232,2.16021475170232 +20103,2003,0,6.59578051396131,4.22961818627967,0.845923637255934 +20103,2004,0,6.58892647753352,4.22961818627967,1.69184727451187 +20103,2005,0,6.46458830368996,4.22961818627967,2.5377709117678 +20103,2006,0,6.49828214947643,4.22961818627967,3.38369454902374 +20103,2007,0,6.45676965557216,4.22961818627967,4.22961818627967 +20133,2003,0,6.11146733950268,2.83303685789521,0.566607371579043 +20133,2004,0,5.92157841964382,2.83303685789521,1.13321474315809 +20133,2005,0,5.82008293035236,2.83303685789521,1.69982211473713 +20133,2006,0,5.84354441703136,2.83303685789521,2.26642948631617 +20133,2007,0,5.72358510195238,2.83303685789521,2.83303685789521 +20137,2003,0,4.7361984483945,1.78389529417125,0.356779058834251 +20137,2004,0,4.70953020131233,1.78389529417125,0.713558117668501 +20137,2005,0,4.85203026391962,1.78389529417125,1.07033717650275 +20137,2006,0,4.79579054559674,1.78389529417125,1.427116235337 +20137,2007,0,4.64439089914137,1.78389529417125,1.78389529417125 +20143,2003,0,3.95124371858143,1.81856367198011,0.363712734396021 +20143,2004,0,3.98898404656427,1.81856367198011,0.727425468792043 +20143,2005,0,3.98898404656427,1.81856367198011,1.09113820318806 +20143,2006,0,4.18965474202643,1.81856367198011,1.45485093758409 +20143,2007,0,4.06044301054642,1.81856367198011,1.81856367198011 +20145,2003,0,4.49980967033027,1.97865388786714,0.395730777573429 +20145,2004,0,4.48863636973214,1.97865388786714,0.791461555146857 +20145,2005,0,4.63472898822964,1.97865388786714,1.18719233272029 +20145,2006,0,4.80402104473326,1.97865388786714,1.58292311029371 +20145,2007,0,4.70048036579242,1.97865388786714,1.97865388786714 +20171,2003,0,5.03043792139244,1.63315443905142,0.326630887810283 +20171,2004,0,4.91265488573605,1.63315443905142,0.653261775620567 +20171,2005,0,4.81218435537242,1.63315443905142,0.97989266343085 +20171,2006,0,4.8283137373023,1.63315443905142,1.30652355124113 +20171,2007,0,4.79579054559674,1.63315443905142,1.63315443905142 +20183,2003,0,3.97029191355212,1.51204556642545,0.30240911328509 +20183,2004,0,3.93182563272433,1.51204556642545,0.60481822657018 +20183,2005,0,3.63758615972639,1.51204556642545,0.90722733985527 +20183,2006,0,4.17438726989564,1.51204556642545,1.20963645314036 +20183,2007,0,4.38202663467388,1.51204556642545,1.51204556642545 +20205,2003,0,4.79579054559674,2.33524587523359,0.467049175046719 +20205,2004,0,4.79579054559674,2.33524587523359,0.934098350093438 +20205,2005,0,4.81218435537242,2.33524587523359,1.40114752514016 +20205,2006,0,4.94875989037817,2.33524587523359,1.86819670018688 +20205,2007,0,5.16478597392351,2.33524587523359,2.33524587523359 +22007,2003,0,4.77912349311153,3.15222307031342,0.630444614062685 +22007,2004,0,4.83628190695148,3.15222307031342,1.26088922812537 +22007,2005,0,4.57471097850338,3.15222307031342,1.89133384218805 +22007,2006,0,4.91998092582813,3.15222307031342,2.52177845625074 +22007,2007,0,5.07517381523383,3.15222307031342,3.15222307031342 +22033,2003,0,8.74957394808293,6.02308917522617,1.20461783504523 +22033,2004,0,8.68016201969438,6.02308917522617,2.40923567009047 +22033,2005,0,8.68440111040014,6.02308917522617,3.6138535051357 +22033,2006,0,8.78094111357239,6.02308917522617,4.81847134018094 +22033,2007,0,8.65782432115598,6.02308917522617,6.02308917522617 +22045,2003,0,6.65415252018322,4.29409665407894,0.858819330815788 +22045,2004,0,6.58063913728495,4.29409665407894,1.71763866163158 +22045,2005,0,6.58202513889283,4.29409665407894,2.57645799244736 +22045,2006,0,6.76272950693188,4.29409665407894,3.43527732326315 +22045,2007,0,6.77992190747225,4.29409665407894,4.29409665407894 +22069,2003,0,5.64190707093811,3.66561082717458,0.733122165434916 +22069,2004,0,5.53733426701854,3.66561082717458,1.46624433086983 +22069,2005,0,5.40717177146012,3.66561082717458,2.19936649630475 +22069,2006,0,5.53733426701854,3.66561082717458,2.93248866173967 +22069,2007,0,5.78689738136671,3.66561082717458,3.66561082717458 +22071,2003,0,8.31164394850298,6.18347649998489,1.23669529999698 +22071,2004,0,8.1371033896393,6.18347649998489,2.47339059999396 +22071,2005,0,8.14206328310415,6.18347649998489,3.71008589999093 +22071,2006,0,7.1785454837637,6.18347649998489,4.94678119998791 +22071,2007,0,7.75147531802146,6.18347649998489,6.18347649998489 +22073,2003,0,7.51043055637801,4.9921318225317,0.998426364506339 +22073,2004,0,7.35115822643069,4.9921318225317,1.99685272901268 +22073,2005,0,7.40367029001237,4.9921318225317,2.99527909351902 +22073,2006,0,7.45240245122364,4.9921318225317,3.99370545802536 +22073,2007,0,7.50988306115491,4.9921318225317,4.9921318225317 +22081,2003,0,4.02535169073515,2.26405214327726,0.452810428655452 +22081,2004,0,3.78418963391826,2.26405214327726,0.905620857310905 +22081,2005,0,3.36729582998647,2.26405214327726,1.35843128596636 +22081,2006,0,3.3322045101752,2.26405214327726,1.81124171462181 +22081,2007,0,3.3322045101752,2.26405214327726,2.26405214327726 +22087,2003,0,6.74051935960622,4.20810470205826,0.841620940411652 +22087,2004,0,6.58892647753352,4.20810470205826,1.6832418808233 +22087,2005,0,6.5792512120101,4.20810470205826,2.52486282123496 +22087,2006,0,4.45434729625351,4.20810470205826,3.36648376164661 +22087,2007,0,5.21493575760899,4.20810470205826,4.20810470205826 +22091,2003,0,3.2188758248682,2.35375337956845,0.470750675913689 +22091,2004,0,3.55534806148941,2.35375337956845,0.941501351827378 +22091,2005,0,3.17805383034795,2.35375337956845,1.41225202774107 +22091,2006,0,3.3322045101752,2.35375337956845,1.88300270365476 +22091,2007,0,3.36729582998647,2.35375337956845,2.35375337956845 +22111,2003,0,5.05624580534831,3.12689210625201,0.625378421250403 +22111,2004,0,4.69134788222914,3.12689210625201,1.25075684250081 +22111,2005,0,5.05624580534831,3.12689210625201,1.87613526375121 +22111,2006,0,4.82028156560504,3.12689210625201,2.50151368500161 +22111,2007,0,4.88280192258637,3.12689210625201,3.12689210625201 +22115,2003,0,5.605802066296,3.96140347152558,0.792280694305117 +22115,2004,0,5.64544689764324,3.96140347152558,1.58456138861023 +22115,2005,0,5.65599181081985,3.96140347152558,2.37684208291535 +22115,2006,0,5.74620319054015,3.96140347152558,3.16912277722047 +22115,2007,0,5.78382518232974,3.96140347152558,3.96140347152558 +22119,2003,0,5.6970934865054,3.73363769145571,0.746727538291142 +22119,2004,0,5.46806014113513,3.73363769145571,1.49345507658228 +22119,2005,0,5.61677109766657,3.73363769145571,2.24018261487343 +22119,2006,0,5.57594910314632,3.73363769145571,2.98691015316457 +22119,2007,0,5.81413053182507,3.73363769145571,3.73363769145571 +24009,2003,2007,7.02553831463852,4.31164440561939,0.862328881123878 +24009,2004,2007,7.14677217945264,4.31164440561939,1.72465776224776 +24009,2005,2007,7.16162200293919,4.31164440561939,2.58698664337163 +24009,2006,2007,7.1066061377273,4.31164440561939,3.44931552449551 +24009,2007,2007,7.1929342212158,4.31164440561939,4.31164440561939 +24015,2003,2007,7.04053639021596,4.4537773664325,0.890755473286499 +24015,2004,2007,7.15305163493748,4.4537773664325,1.781510946573 +24015,2005,2007,7.14440718032114,4.4537773664325,2.6722664198595 +24015,2006,2007,7.18841273649695,4.4537773664325,3.563021893146 +24015,2007,2007,7.11151211649616,4.4537773664325,4.4537773664325 +24017,2003,2007,7.68017564043659,4.79203142282408,0.958406284564816 +24017,2004,2007,7.69256964806791,4.79203142282408,1.91681256912963 +24017,2005,2007,7.69074316354187,4.79203142282408,2.87521885369445 +24017,2006,2007,7.75491027202143,4.79203142282408,3.83362513825926 +24017,2007,2007,7.75619534394812,4.79203142282408,4.79203142282408 +24021,2003,2007,8.31016902198191,5.27441906341037,1.05488381268207 +24021,2004,2007,8.36264243156764,5.27441906341037,2.10976762536415 +24021,2005,2007,8.30819906320645,5.27441906341037,3.16465143804622 +24021,2006,2007,8.32748441618826,5.27441906341037,4.2195352507283 +24021,2007,2007,8.28803156777646,5.27441906341037,5.27441906341037 +24027,2003,2007,8.76826314537129,5.5127914463584,1.10255828927168 +24027,2004,2007,8.74273386733017,5.5127914463584,2.20511657854336 +24027,2005,2007,8.67077227934454,5.5127914463584,3.30767486781504 +24027,2006,2007,8.71719093322313,5.5127914463584,4.41023315708672 +24027,2007,2007,8.75131624677346,5.5127914463584,5.5127914463584 +24033,2003,2007,9.10830764641103,6.68650368678403,1.33730073735681 +24033,2004,2007,9.14034700015565,6.68650368678403,2.67460147471361 +24033,2005,2007,9.17232692977797,6.68650368678403,4.01190221207042 +24033,2006,2007,9.17926241640532,6.68650368678403,5.34920294942722 +24033,2007,2007,9.15567297063387,6.68650368678403,6.68650368678403 +26005,2003,2007,7.1693500166706,4.66027371221412,0.932054742442823 +26005,2004,2007,7.15773548424991,4.66027371221412,1.86410948488565 +26005,2005,2007,7.10578612948127,4.66027371221412,2.79616422732847 +26005,2006,2007,7.05185562295589,4.66027371221412,3.72821896977129 +26005,2007,2007,6.98749024700099,4.66027371221412,4.66027371221412 +26015,2003,2007,6.34388043412633,4.0387437582073,0.807748751641461 +26015,2004,2007,6.25190388316589,4.0387437582073,1.61549750328292 +26015,2005,2007,6.28599809450886,4.0387437582073,2.42324625492438 +26015,2006,2007,6.18001665365257,4.0387437582073,3.23099500656584 +26015,2007,2007,6.19440539110467,4.0387437582073,4.0387437582073 +26027,2003,2007,5.42053499927229,3.9338627720326,0.78677255440652 +26027,2004,2007,5.62401750618734,3.9338627720326,1.57354510881304 +26027,2005,2007,5.72358510195238,3.9338627720326,2.36031766321956 +26027,2006,2007,5.58724865840025,3.9338627720326,3.14709021762608 +26027,2007,2007,5.27299955856375,3.9338627720326,3.9338627720326 +26031,2003,2007,5.27811465923052,3.27518054107867,0.655036108215734 +26031,2004,2007,5.12989871492307,3.27518054107867,1.31007221643147 +26031,2005,2007,5.12989871492307,3.27518054107867,1.9651083246472 +26031,2006,2007,5.11198778835654,3.27518054107867,2.62014443286293 +26031,2007,2007,4.99721227376411,3.27518054107867,3.27518054107867 +26041,2003,2007,6.31173480915291,3.65117758692992,0.730235517385985 +26041,2004,2007,6.32076829425058,3.65117758692992,1.46047103477197 +26041,2005,2007,6.25766758788264,3.65117758692992,2.19070655215795 +26041,2006,2007,6.25382881157547,3.65117758692992,2.92094206954394 +26041,2007,2007,6.13988455222626,3.65117758692992,3.65117758692992 +26057,2003,2007,6.13988455222626,3.74443241323419,0.748886482646837 +26057,2004,2007,6.09582456243222,3.74443241323419,1.49777296529367 +26057,2005,2007,6.25190388316589,3.74443241323419,2.24665944794051 +26057,2006,2007,6.12249280951439,3.74443241323419,2.99554593058735 +26057,2007,2007,5.92692602597041,3.74443241323419,3.74443241323419 +26061,2003,2007,6.42648845745769,3.58396328416438,0.716792656832875 +26061,2004,2007,6.40025744530882,3.58396328416438,1.43358531366575 +26061,2005,2007,6.33859407820318,3.58396328416438,2.15037797049863 +26061,2006,2007,6.22851100359118,3.58396328416438,2.8671706273315 +26061,2007,2007,6.13556489108174,3.58396328416438,3.58396328416438 +26065,2003,2007,8.4142741374084,5.63235807797783,1.12647161559557 +26065,2004,2007,8.33254893925264,5.63235807797783,2.25294323119113 +26065,2005,2007,8.24643378616036,5.63235807797783,3.3794148467867 +26065,2006,2007,8.21283958467648,5.63235807797783,4.50588646238227 +26065,2007,2007,8.16365617616843,5.63235807797783,5.63235807797783 +26073,2003,2007,6.73933662735717,4.14869069196221,0.829738138392443 +26073,2004,2007,6.59850902861452,4.14869069196221,1.65947627678489 +26073,2005,2007,6.68586094706836,4.14869069196221,2.48921441517733 +26073,2006,2007,6.53087762772588,4.14869069196221,3.31895255356977 +26073,2007,2007,6.51025834052315,4.14869069196221,4.14869069196221 +26085,2003,2007,3.66356164612965,2.42771882375081,0.485543764750162 +26085,2004,2007,3.40119738166216,2.42771882375081,0.971087529500325 +26085,2005,2007,3.63758615972639,2.42771882375081,1.45663129425049 +26085,2006,2007,3.2188758248682,2.42771882375081,1.94217505900065 +26085,2007,2007,3.36729582998647,2.42771882375081,2.42771882375081 +26089,2003,2007,5.64190707093811,3.05017310923224,0.610034621846448 +26089,2004,2007,5.52942908751142,3.05017310923224,1.2200692436929 +26089,2005,2007,5.47227067367148,3.05017310923224,1.83010386553934 +26089,2006,2007,5.28320372873799,3.05017310923224,2.44013848738579 +26089,2007,2007,5.34233425196481,3.05017310923224,3.05017310923224 +26105,2003,2007,5.75890177387728,3.34194265444484,0.668388530888968 +26105,2004,2007,5.59842195899838,3.34194265444484,1.33677706177794 +26105,2005,2007,5.75574221358691,3.34194265444484,2.0051655926669 +26105,2006,2007,5.61312810638807,3.34194265444484,2.67355412355587 +26105,2007,2007,5.52942908751142,3.34194265444484,3.34194265444484 +26127,2003,2007,5.52942908751142,3.29112206507397,0.658224413014795 +26127,2004,2007,5.51745289646471,3.29112206507397,1.31644882602959 +26127,2005,2007,5.54907608489522,3.29112206507397,1.97467323904438 +26127,2006,2007,5.39362754635236,3.29112206507397,2.63289765205918 +26127,2007,2007,5.37527840768417,3.29112206507397,3.29112206507397 +26129,2003,2007,5.74300318780948,3.07477448089394,0.614954896178789 +26129,2004,2007,5.75574221358691,3.07477448089394,1.22990979235758 +26129,2005,2007,5.75890177387728,3.07477448089394,1.84486468853637 +26129,2006,2007,5.5683445037611,3.07477448089394,2.45981958471515 +26129,2007,2007,5.52942908751142,3.07477448089394,3.07477448089394 +26137,2003,2007,6.2363695902037,3.14849627810562,0.629699255621124 +26137,2004,2007,6.22851100359118,3.14849627810562,1.25939851124225 +26137,2005,2007,6.18620862390049,3.14849627810562,1.88909776686337 +26137,2006,2007,6.10479323241498,3.14849627810562,2.5187970224845 +26137,2007,2007,5.94542060860658,3.14849627810562,3.14849627810562 +26149,2003,2007,6.62273632394984,4.13391777734183,0.826783555468365 +26149,2004,2007,6.45204895443723,4.13391777734183,1.65356711093673 +26149,2005,2007,6.50727771238501,4.13391777734183,2.4803506664051 +26149,2006,2007,6.41836493593621,4.13391777734183,3.30713422187346 +26149,2007,2007,6.36990098282823,4.13391777734183,4.13391777734183 +27005,2003,2006,6.84374994900622,3.40119738166216,0.680239476332431 +27005,2004,2006,6.76157276880406,3.40119738166216,1.36047895266486 +27005,2005,2006,6.78671695060508,3.40119738166216,2.04071842899729 +27005,2006,2006,6.96790920180188,3.40119738166216,2.72095790532972 +27005,2007,2006,6.91075078796194,3.40119738166216,3.40119738166216 +27009,2003,2006,6.52356230614951,3.53298558915526,0.706597117831052 +27009,2004,2006,6.45362499889269,3.53298558915526,1.4131942356621 +27009,2005,2006,6.47697236288968,3.53298558915526,2.11979135349316 +27009,2006,2006,6.51767127291227,3.53298558915526,2.82638847132421 +27009,2007,2006,6.41673228251233,3.53298558915526,3.53298558915526 +27023,2003,2006,5.79301360838414,2.57169577986039,0.514339155972078 +27023,2004,2006,5.65599181081985,2.57169577986039,1.02867831194416 +27023,2005,2006,5.79909265446053,2.57169577986039,1.54301746791623 +27023,2006,2006,5.6970934865054,2.57169577986039,2.05735662388831 +27023,2007,2006,5.72358510195238,2.57169577986039,2.57169577986039 +27035,2003,2006,7.4899708988348,4.00913156717385,0.80182631343477 +27035,2004,2006,7.22983877815125,4.00913156717385,1.60365262686954 +27035,2005,2006,7.30114780585603,4.00913156717385,2.40547894030431 +27035,2006,2006,7.39510754656249,4.00913156717385,3.20730525373908 +27035,2007,2006,7.22329567956231,4.00913156717385,4.00913156717385 +27043,2003,2006,5.57594910314632,2.78383771441767,0.556767542883534 +27043,2004,2006,5.39362754635236,2.78383771441767,1.11353508576707 +27043,2005,2006,5.54907608489522,2.78383771441767,1.6703026286506 +27043,2006,2006,5.46383180502561,2.78383771441767,2.22707017153414 +27043,2007,2006,5.41164605185504,2.78383771441767,2.78383771441767 +27049,2003,2006,7.2991214627108,3.78707184000532,0.757414368001063 +27049,2004,2006,7.03174125876313,3.78707184000532,1.51482873600213 +27049,2005,2006,6.93731408122368,3.78707184000532,2.27224310400319 +27049,2006,2006,6.96979066990159,3.78707184000532,3.02965747200425 +27049,2007,2006,6.95844839329766,3.78707184000532,3.78707184000532 +27051,2003,2006,4.54329478227,1.83880207556137,0.367760415112273 +27051,2004,2006,4.49980967033027,1.83880207556137,0.735520830224547 +27051,2005,2006,4.33073334028633,1.83880207556137,1.10328124533682 +27051,2006,2006,4.44265125649032,1.83880207556137,1.47104166044909 +27051,2007,2006,4.06044301054642,1.83880207556137,1.83880207556137 +27055,2003,2006,5.77765232322266,2.98153192415285,0.59630638483057 +27055,2004,2006,5.74300318780948,2.98153192415285,1.19261276966114 +27055,2005,2006,5.73979291217923,2.98153192415285,1.78891915449171 +27055,2006,2006,5.71373280550937,2.98153192415285,2.38522553932228 +27055,2007,2006,5.66296048013595,2.98153192415285,2.98153192415285 +27091,2003,2006,6.30991827822652,3.08200170870617,0.616400341741233 +27091,2004,2006,6.28785856016178,3.08200170870617,1.23280068348247 +27091,2005,2006,6.3578422665081,3.08200170870617,1.8492010252237 +27091,2006,2006,6.40522845803084,3.08200170870617,2.46560136696493 +27091,2007,2006,6.3456363608286,3.08200170870617,3.08200170870617 +27115,2003,2006,6.12686918411419,3.27827616814965,0.65565523362993 +27115,2004,2006,6.12468339089421,3.27827616814965,1.31131046725986 +27115,2005,2006,6.09582456243222,3.27827616814965,1.96696570088979 +27115,2006,2006,5.96614673912369,3.27827616814965,2.62262093451972 +27115,2007,2006,6.00388706710654,3.27827616814965,3.27827616814965 +27123,2003,2006,9.2617936535651,6.23643808100884,1.24728761620177 +27123,2004,2006,9.18450961194166,6.23643808100884,2.49457523240354 +27123,2005,2006,9.13841463240459,6.23643808100884,3.7418628486053 +27123,2006,2006,9.19146331620649,6.23643808100884,4.98915046480707 +27123,2007,2006,9.11294802596753,6.23643808100884,6.23643808100884 +27129,2003,2006,5.77455154554441,2.84223138256823,0.568446276513646 +27129,2004,2006,5.72358510195238,2.84223138256823,1.13689255302729 +27129,2005,2006,5.75257263882563,2.84223138256823,1.70533882954094 +27129,2006,2006,5.64190707093811,2.84223138256823,2.27378510605458 +27129,2007,2006,5.87773578177964,2.84223138256823,2.84223138256823 +27149,2003,2006,5.35185813347607,2.30787109742328,0.461574219484657 +27149,2004,2006,5.14166355650266,2.30787109742328,0.923148438969313 +27149,2005,2006,5.2040066870768,2.30787109742328,1.38472265845397 +27149,2006,2006,5.17614973257383,2.30787109742328,1.84629687793863 +27149,2007,2006,5.35185813347607,2.30787109742328,2.30787109742328 +27157,2003,2006,6.04737217904628,3.07315617051879,0.614631234103759 +27157,2004,2006,5.95324333428778,3.07315617051879,1.22926246820752 +27157,2005,2006,5.89164421182577,3.07315617051879,1.84389370231128 +27157,2006,2006,5.92157841964382,3.07315617051879,2.45852493641504 +27157,2007,2006,5.91620206260743,3.07315617051879,3.07315617051879 +27163,2003,2006,8.27231514795602,5.30395146516513,1.06079029303303 +27163,2004,2006,8.21419441485256,5.30395146516513,2.12158058606605 +27163,2005,2006,8.17075142375753,5.30395146516513,3.18237087909908 +27163,2006,2006,8.25035895147729,5.30395146516513,4.2431611721321 +27163,2007,2006,8.20303024171486,5.30395146516513,5.30395146516513 +27165,2003,2006,5.48479693349065,2.47451955690151,0.494903911380303 +27165,2004,2006,5.40267738187228,2.47451955690151,0.989807822760605 +27165,2005,2006,5.30330490805908,2.47451955690151,1.48471173414091 +27165,2006,2006,5.41610040220442,2.47451955690151,1.97961564552121 +27165,2007,2006,5.38449506278909,2.47451955690151,2.47451955690151 +29003,2003,2007,4.83628190695148,2.80287541484465,0.560575082968931 +29003,2004,2007,4.85981240436167,2.80287541484465,1.12115016593786 +29003,2005,2007,4.87519732320115,2.80287541484465,1.68172524890679 +29003,2006,2007,4.88280192258637,2.80287541484465,2.24230033187572 +29003,2007,2007,5.01063529409626,2.80287541484465,2.80287541484465 +29011,2003,2007,5.51342874616498,2.52900327684191,0.505800655368383 +29011,2004,2007,5.51745289646471,2.52900327684191,1.01160131073677 +29011,2005,2007,5.40267738187228,2.52900327684191,1.51740196610515 +29011,2006,2007,5.30330490805908,2.52900327684191,2.02320262147353 +29011,2007,2007,5.25749537202778,2.52900327684191,2.52900327684191 +29013,2003,2007,5.36129216570943,2.81259038037613,0.562518076075227 +29013,2004,2007,5.25227342804663,2.81259038037613,1.12503615215045 +29013,2005,2007,5.32787616878958,2.81259038037613,1.68755422822568 +29013,2006,2007,5.18738580584075,2.81259038037613,2.25007230430091 +29013,2007,2007,5.12396397940326,2.81259038037613,2.81259038037613 +29023,2003,2007,6.40025744530882,3.71032289140759,0.742064578281518 +29023,2004,2007,6.51174532964473,3.71032289140759,1.48412915656304 +29023,2005,2007,6.53813982376767,3.71032289140759,2.22619373484455 +29023,2006,2007,6.41345895716736,3.71032289140759,2.96825831312607 +29023,2007,2007,6.29341927884648,3.71032289140759,3.71032289140759 +29033,2003,2007,4.82028156560504,2.33068652310492,0.466137304620984 +29033,2004,2007,4.80402104473326,2.33068652310492,0.932274609241968 +29033,2005,2007,4.65396035015752,2.33068652310492,1.39841191386295 +29033,2006,2007,4.69134788222914,2.33068652310492,1.86454921848394 +29033,2007,2007,4.8283137373023,2.33068652310492,2.33068652310492 +29041,2003,2007,4.71849887129509,2.13274531370215,0.426549062740429 +29041,2004,2007,4.65396035015752,2.13274531370215,0.853098125480859 +29041,2005,2007,4.64439089914137,2.13274531370215,1.27964718822129 +29041,2006,2007,4.36944785246702,2.13274531370215,1.70619625096172 +29041,2007,2007,4.4188406077966,2.13274531370215,2.13274531370215 +29065,2003,2007,5.07517381523383,2.7031716536511,0.540634330730221 +29065,2004,2007,5.13579843705026,2.7031716536511,1.08126866146044 +29065,2005,2007,5.11799381241676,2.7031716536511,1.62190299219066 +29065,2006,2007,5.09375020080676,2.7031716536511,2.16253732292088 +29065,2007,2007,5.02388052084628,2.7031716536511,2.7031716536511 +29073,2003,2007,5.8111409929767,2.7305941655371,0.546118833107421 +29073,2004,2007,5.76205138278018,2.7305941655371,1.09223766621484 +29073,2005,2007,5.87211778947542,2.7305941655371,1.63835649932226 +29073,2006,2007,5.94542060860658,2.7305941655371,2.18447533242968 +29073,2007,2007,5.8805329864007,2.7305941655371,2.7305941655371 +29077,2003,2007,8.44827174594982,5.48226676435625,1.09645335287125 +29077,2004,2007,8.39117635083275,5.48226676435625,2.1929067057425 +29077,2005,2007,8.39976009452414,5.48226676435625,3.28936005861375 +29077,2006,2007,8.46779284112111,5.48226676435625,4.385813411485 +29077,2007,2007,8.44891435066294,5.48226676435625,5.48226676435625 +29079,2003,2007,4.94875989037817,2.3448780051843,0.468975601036859 +29079,2004,2007,4.84418708645859,2.3448780051843,0.937951202073719 +29079,2005,2007,4.969813299576,2.3448780051843,1.40692680311058 +29079,2006,2007,4.8283137373023,2.3448780051843,1.87590240414744 +29079,2007,2007,4.8283137373023,2.3448780051843,2.3448780051843 +29081,2003,2007,5.11198778835654,2.18041745901984,0.436083491803968 +29081,2004,2007,5.06259503302697,2.18041745901984,0.872166983607935 +29081,2005,2007,5.17048399503815,2.18041745901984,1.3082504754119 +29081,2006,2007,4.94875989037817,2.18041745901984,1.74433396721587 +29081,2007,2007,4.90527477843843,2.18041745901984,2.18041745901984 +29085,2003,2007,3.46573590279973,2.19053558918542,0.438107117837085 +29085,2004,2007,3.25809653802148,2.19053558918542,0.876214235674169 +29085,2005,2007,3.3322045101752,2.19053558918542,1.31432135351125 +29085,2006,2007,3.40119738166216,2.19053558918542,1.75242847134834 +29085,2007,2007,3.49650756146648,2.19053558918542,2.19053558918542 +29089,2003,2007,4.70048036579242,2.32356349937924,0.464712699875848 +29089,2004,2007,4.7361984483945,2.32356349937924,0.929425399751695 +29089,2005,2007,4.70953020131233,2.32356349937924,1.39413809962754 +29089,2006,2007,4.69134788222914,2.32356349937924,1.85885079950339 +29089,2007,2007,4.62497281328427,2.32356349937924,2.32356349937924 +29099,2003,2007,7.80628928926703,5.28876690573619,1.05775338114724 +29099,2004,2007,7.78572089653462,5.28876690573619,2.11550676229447 +29099,2005,2007,7.74283595543075,5.28876690573619,3.17326014344171 +29099,2006,2007,7.73848812249465,5.28876690573619,4.23101352458895 +29099,2007,2007,7.7367436824535,5.28876690573619,5.28876690573619 +29101,2003,2007,6.07993319509559,3.87656161714999,0.775312323429997 +29101,2004,2007,6.14203740558736,3.87656161714999,1.55062464685999 +29101,2005,2007,6.20050917404269,3.87656161714999,2.32593697028999 +29101,2006,2007,6.16120732169508,3.87656161714999,3.10124929371999 +29101,2007,2007,6.18414889093748,3.87656161714999,3.87656161714999 +29105,2003,2007,6.13556489108174,3.48164000935702,0.696328001871404 +29105,2004,2007,6.0591231955818,3.48164000935702,1.39265600374281 +29105,2005,2007,6.1463292576689,3.48164000935702,2.08898400561421 +29105,2006,2007,6.28226674689601,3.48164000935702,2.78531200748562 +29105,2007,2007,6.27476202124194,3.48164000935702,3.48164000935702 +29107,2003,2007,6.1527326947041,3.49529470504127,0.699058941008254 +29107,2004,2007,6.19236248947487,3.49529470504127,1.39811788201651 +29107,2005,2007,6.1463292576689,3.49529470504127,2.09717682302476 +29107,2006,2007,6.09582456243222,3.49529470504127,2.79623576403302 +29107,2007,2007,6.12029741895095,3.49529470504127,3.49529470504127 +29117,2003,2007,5.7268477475872,2.67814067069607,0.535628134139213 +29117,2004,2007,5.75574221358691,2.67814067069607,1.07125626827843 +29117,2005,2007,5.64544689764324,2.67814067069607,1.60688440241764 +29117,2006,2007,5.67675380226828,2.67814067069607,2.14251253655685 +29117,2007,2007,5.54517744447956,2.67814067069607,2.67814067069607 +29119,2003,2007,5.35658627467201,3.07643630096895,0.615287260193791 +29119,2004,2007,5.23110861685459,3.07643630096895,1.23057452038758 +29119,2005,2007,5.16478597392351,3.07643630096895,1.84586178058137 +29119,2006,2007,5.28320372873799,3.07643630096895,2.46114904077516 +29119,2007,2007,5.33753807970132,3.07643630096895,3.07643630096895 +29127,2003,2007,6.39359075395063,3.34247303650915,0.668494607301829 +29127,2004,2007,6.33150184989369,3.34247303650915,1.33698921460366 +29127,2005,2007,6.42324696353352,3.34247303650915,2.00548382190549 +29127,2006,2007,6.39692965521615,3.34247303650915,2.67397842920732 +29127,2007,2007,6.36818718635049,3.34247303650915,3.34247303650915 +29131,2003,2007,5.52942908751142,3.15972012365944,0.631944024731888 +29131,2004,2007,5.34233425196481,3.15972012365944,1.26388804946378 +29131,2005,2007,5.34710753071747,3.15972012365944,1.89583207419566 +29131,2006,2007,5.41610040220442,3.15972012365944,2.52777609892755 +29131,2007,2007,5.38449506278909,3.15972012365944,3.15972012365944 +29149,2003,2007,4.21950770517611,2.33640664146956,0.467281328293911 +29149,2004,2007,4.18965474202643,2.33640664146956,0.934562656587823 +29149,2005,2007,4.35670882668959,2.33640664146956,1.40184398488173 +29149,2006,2007,4.35670882668959,2.33640664146956,1.86912531317565 +29149,2007,2007,4.51085950651685,2.33640664146956,2.33640664146956 +29155,2003,2007,5.4971682252932,2.99807951662234,0.599615903324468 +29155,2004,2007,5.34233425196481,2.99807951662234,1.19923180664894 +29155,2005,2007,5.26269018890489,2.99807951662234,1.7988477099734 +29155,2006,2007,5.45958551414416,2.99807951662234,2.39846361329787 +29155,2007,2007,5.55295958492162,2.99807951662234,2.99807951662234 +29163,2003,2007,5.605802066296,2.90968406892854,0.581936813785707 +29163,2004,2007,5.69035945432406,2.90968406892854,1.16387362757141 +29163,2005,2007,5.72031177660741,2.90968406892854,1.74581044135712 +29163,2006,2007,5.7037824746562,2.90968406892854,2.32774725514283 +29163,2007,2007,5.63835466933375,2.90968406892854,2.90968406892854 +29177,2003,2007,5.48063892334199,3.15076827565352,0.630153655130705 +29177,2004,2007,5.26785815906333,3.15076827565352,1.26030731026141 +29177,2005,2007,5.28320372873799,3.15076827565352,1.89046096539211 +29177,2006,2007,5.18738580584075,3.15076827565352,2.52061462052282 +29177,2007,2007,5.22035582507832,3.15076827565352,3.15076827565352 +29201,2003,2007,6.08904487544685,3.69937419120658,0.739874838241316 +29201,2004,2007,6.19031540585315,3.69937419120658,1.47974967648263 +29201,2005,2007,6.15060276844628,3.69937419120658,2.21962451472395 +29201,2006,2007,6.09807428216624,3.69937419120658,2.95949935296526 +29201,2007,2007,6.00635315960173,3.69937419120658,3.69937419120658 +29209,2003,2007,5.00394630594546,3.35543263629441,0.671086527258882 +29209,2004,2007,5.19849703126583,3.35543263629441,1.34217305451776 +29209,2005,2007,5.19849703126583,3.35543263629441,2.01325958177665 +29209,2006,2007,5.23644196282995,3.35543263629441,2.68434610903553 +29209,2007,2007,5.32300997913841,3.35543263629441,3.35543263629441 +29211,2003,2007,4.26267987704132,1.97671643915702,0.395343287831405 +29211,2004,2007,4.24849524204936,1.97671643915702,0.79068657566281 +29211,2005,2007,3.93182563272433,1.97671643915702,1.18602986349421 +29211,2006,2007,4.02535169073515,1.97671643915702,1.58137315132562 +29211,2007,2007,4.12713438504509,1.97671643915702,1.97671643915702 +29215,2003,2007,5.37989735354046,3.13562464220588,0.627124928441176 +29215,2004,2007,5.29831736654804,3.13562464220588,1.25424985688235 +29215,2005,2007,5.43372200355424,3.13562464220588,1.88137478532353 +29215,2006,2007,5.41610040220442,3.13562464220588,2.50849971376471 +29215,2007,2007,5.40267738187228,3.13562464220588,3.13562464220588 +29229,2003,2007,5.52942908751142,2.88786862767805,0.577573725535609 +29229,2004,2007,5.54517744447956,2.88786862767805,1.15514745107122 +29229,2005,2007,5.51745289646471,2.88786862767805,1.73272117660683 +29229,2006,2007,5.42053499927229,2.88786862767805,2.31029490214244 +29229,2007,2007,5.26269018890489,2.88786862767805,2.88786862767805 +29510,2003,2007,8.40200678160712,5.85274543579545,1.17054908715909 +29510,2004,2007,8.34616759436413,5.85274543579545,2.34109817431818 +29510,2005,2007,8.29704514908183,5.85274543579545,3.51164726147727 +29510,2006,2007,8.29679586577005,5.85274543579545,4.68219634863636 +29510,2007,2007,8.25816336153762,5.85274543579545,5.85274543579545 +30009,2003,2007,5.08140436498446,2.25675055665025,0.451350111330049 +30009,2004,2007,4.92725368515721,2.25675055665025,0.902700222660098 +30009,2005,2007,4.77912349311153,2.25675055665025,1.35405033399015 +30009,2006,2007,4.87519732320115,2.25675055665025,1.8054004453202 +30009,2007,2007,4.97673374242057,2.25675055665025,2.25675055665025 +30011,2003,2007,2.484906649788,0.307484699747961,0.0614969399495921 +30011,2004,2007,1.94591014905531,0.307484699747961,0.122993879899184 +30011,2005,2007,1.6094379124341,0.307484699747961,0.184490819848776 +30011,2006,2007,1.94591014905531,0.307484699747961,0.245987759798369 +30011,2007,2007,2.07944154167984,0.307484699747961,0.307484699747961 +30013,2003,2007,7.27793857294566,4.3864792072439,0.877295841448781 +30013,2004,2007,7.16626597413364,4.3864792072439,1.75459168289756 +30013,2005,2007,7.19142933003638,4.3864792072439,2.63188752434634 +30013,2006,2007,7.25981961036319,4.3864792072439,3.50918336579512 +30013,2007,2007,7.28138566357028,4.3864792072439,4.3864792072439 +30021,2003,2007,5.37063802812766,2.20375873868715,0.44075174773743 +30021,2004,2007,5.30826769740121,2.20375873868715,0.88150349547486 +30021,2005,2007,5.16478597392351,2.20375873868715,1.32225524321229 +30021,2006,2007,5.44241771052179,2.20375873868715,1.76300699094972 +30021,2007,2007,5.25227342804663,2.20375873868715,2.20375873868715 +30033,2003,2007,2.39789527279837,0.246078522596706,0.0492157045193411 +30033,2004,2007,2.89037175789616,0.246078522596706,0.0984314090386822 +30033,2005,2007,2.484906649788,0.246078522596706,0.147647113558023 +30033,2006,2007,2.83321334405622,0.246078522596706,0.196862818077364 +30033,2007,2007,2.89037175789616,0.246078522596706,0.246078522596706 +30049,2003,2007,7.12929754892937,4.02026735882578,0.804053471765157 +30049,2004,2007,7.0604763659998,4.02026735882578,1.60810694353031 +30049,2005,2007,7.00397413672268,4.02026735882578,2.41216041529547 +30049,2006,2007,7.09506437728713,4.02026735882578,3.21621388706063 +30049,2007,2007,7.09589322109753,4.02026735882578,4.02026735882578 +30081,2003,2007,6.21260609575152,3.58546149491545,0.71709229898309 +30081,2004,2007,6.21860011969173,3.58546149491545,1.43418459796618 +30081,2005,2007,6.18208490671663,3.58546149491545,2.15127689694927 +30081,2006,2007,6.26909628370626,3.58546149491545,2.86836919593236 +30081,2007,2007,6.27852142416584,3.58546149491545,3.58546149491545 +30087,2003,2007,5.35185813347607,2.23889954130791,0.447779908261583 +30087,2004,2007,5.32300997913841,2.23889954130791,0.895559816523165 +30087,2005,2007,5.11198778835654,2.23889954130791,1.34333972478475 +30087,2006,2007,5.0998664278242,2.23889954130791,1.79111963304633 +30087,2007,2007,4.969813299576,2.23889954130791,2.23889954130791 +30095,2003,2007,4.56434819146784,2.10352421219579,0.420704842439159 +30095,2004,2007,4.38202663467388,2.10352421219579,0.841409684878317 +30095,2005,2007,4.65396035015752,2.10352421219579,1.26211452731748 +30095,2006,2007,4.65396035015752,2.10352421219579,1.68281936975663 +30095,2007,2007,4.63472898822964,2.10352421219579,2.10352421219579 +30101,2003,2007,4.24849524204936,1.66146094050125,0.33229218810025 +30101,2004,2007,4.00733318523247,1.66146094050125,0.664584376200501 +30101,2005,2007,4.26267987704132,1.66146094050125,0.996876564300752 +30101,2006,2007,4.12713438504509,1.66146094050125,1.329168752401 +30101,2007,2007,4.29045944114839,1.66146094050125,1.66146094050125 +30109,2003,2007,2.70805020110221,0.0657877405380032,0.0131575481076006 +30109,2004,2007,2.63905732961526,0.0657877405380032,0.0263150962152013 +30109,2005,2007,2.19722457733622,0.0657877405380032,0.0394726443228019 +30109,2006,2007,2.19722457733622,0.0657877405380032,0.0526301924304025 +30109,2007,2007,2.30258509299405,0.0657877405380032,0.0657877405380032 +31003,2003,0,4.62497281328427,2.00848245273934,0.401696490547868 +31003,2004,0,4.44265125649032,2.00848245273934,0.803392981095737 +31003,2005,0,4.4188406077966,2.00848245273934,1.20508947164361 +31003,2006,0,4.55387689160054,2.00848245273934,1.60678596219147 +31003,2007,0,4.51085950651685,2.00848245273934,2.00848245273934 +31023,2003,0,4.76217393479776,2.17099467260645,0.43419893452129 +31023,2004,0,4.51085950651685,2.17099467260645,0.868397869042579 +31023,2005,0,4.70953020131233,2.17099467260645,1.30259680356387 +31023,2006,0,4.62497281328427,2.17099467260645,1.73679573808516 +31023,2007,0,4.77068462446567,2.17099467260645,2.17099467260645 +31031,2003,0,4.88280192258637,1.81612682567635,0.36322536513527 +31031,2004,0,5.01063529409626,1.81612682567635,0.72645073027054 +31031,2005,0,5.07517381523383,1.81612682567635,1.08967609540581 +31031,2006,0,4.96284463025991,1.81612682567635,1.45290146054108 +31031,2007,0,4.87519732320115,1.81612682567635,1.81612682567635 +31033,2003,0,5.49306144334055,2.28543893415908,0.457087786831815 +31033,2004,0,5.5683445037611,2.28543893415908,0.91417557366363 +31033,2005,0,5.5834963087817,2.28543893415908,1.37126336049545 +31033,2006,0,5.54126354515843,2.28543893415908,1.82835114732726 +31033,2007,0,5.59842195899838,2.28543893415908,2.28543893415908 +31041,2003,0,5.11198778835654,2.46750613510783,0.493501227021567 +31041,2004,0,5.04342511691925,2.46750613510783,0.987002454043133 +31041,2005,0,5.06890420222023,2.46750613510783,1.4805036810647 +31041,2006,0,5.21493575760899,2.46750613510783,1.97400490808627 +31041,2007,0,5.28826703069454,2.46750613510783,2.46750613510783 +31055,2003,0,9.36546200884348,6.13898975546329,1.22779795109266 +31055,2004,0,9.3321154289021,6.13898975546329,2.45559590218531 +31055,2005,0,9.31118068690225,6.13898975546329,3.68339385327797 +31055,2006,0,9.32197118814737,6.13898975546329,4.91119180437063 +31055,2007,0,9.32027043134838,6.13898975546329,6.13898975546329 +31059,2003,0,4.97673374242057,1.89220794052486,0.378441588104972 +31059,2004,0,4.77912349311153,1.89220794052486,0.756883176209944 +31059,2005,0,4.74493212836325,1.89220794052486,1.13532476431492 +31059,2006,0,4.70048036579242,1.89220794052486,1.51376635241989 +31059,2007,0,4.64439089914137,1.89220794052486,1.89220794052486 +31061,2003,0,3.71357206670431,1.27368541673724,0.254737083347447 +31061,2004,0,3.61091791264422,1.27368541673724,0.509474166694895 +31061,2005,0,3.40119738166216,1.27368541673724,0.764211250042342 +31061,2006,0,3.3322045101752,1.27368541673724,1.01894833338979 +31061,2007,0,3.61091791264422,1.27368541673724,1.27368541673724 +31073,2003,0,2.70805020110221,0.76220671649144,0.152441343298288 +31073,2004,0,2.77258872223978,0.76220671649144,0.304882686596576 +31073,2005,0,2.83321334405622,0.76220671649144,0.457324029894864 +31073,2006,0,2.70805020110221,0.76220671649144,0.609765373193152 +31073,2007,0,2.63905732961526,0.76220671649144,0.76220671649144 +31079,2003,0,7.1708884785125,3.98031696606713,0.796063393213427 +31079,2004,0,7.09920174355309,3.98031696606713,1.59212678642685 +31079,2005,0,7.06646697013696,3.98031696606713,2.38819017964028 +31079,2006,0,7.16549347506085,3.98031696606713,3.18425357285371 +31079,2007,0,7.17548971362422,3.98031696606713,3.98031696606713 +31081,2003,0,5.21493575760899,2.24102878729494,0.448205757458988 +31081,2004,0,5.24174701505964,2.24102878729494,0.896411514917976 +31081,2005,0,5.20948615284142,2.24102878729494,1.34461727237696 +31081,2006,0,5.15329159449778,2.24102878729494,1.79282302983595 +31081,2007,0,5.14749447681345,2.24102878729494,2.24102878729494 +31083,2003,0,3.85014760171006,1.33131005278713,0.266262010557426 +31083,2004,0,3.73766961828337,1.33131005278713,0.532524021114852 +31083,2005,0,3.89182029811063,1.33131005278713,0.798786031672279 +31083,2006,0,3.98898404656427,1.33131005278713,1.0650480422297 +31083,2007,0,3.85014760171006,1.33131005278713,1.33131005278713 +31101,2003,0,5.4249500174814,2.18323833536148,0.436647667072296 +31101,2004,0,5.30826769740121,2.18323833536148,0.873295334144592 +31101,2005,0,5.30330490805908,2.18323833536148,1.30994300121689 +31101,2006,0,5.23110861685459,2.18323833536148,1.74659066828918 +31101,2007,0,5.26269018890489,2.18323833536148,2.18323833536148 +31107,2003,0,5.04985600724954,2.23793989950051,0.447587979900103 +31107,2004,0,5.19295685089021,2.23793989950051,0.895175959800206 +31107,2005,0,5.08140436498446,2.23793989950051,1.34276393970031 +31107,2006,0,5.03695260241363,2.23793989950051,1.79035191960041 +31107,2007,0,5.14166355650266,2.23793989950051,2.23793989950051 +31109,2003,0,8.57828829077605,5.52262424093949,1.1045248481879 +31109,2004,0,8.52694548285892,5.52262424093949,2.2090496963758 +31109,2005,0,8.53168763756669,5.52262424093949,3.31357454456369 +31109,2006,0,8.52038808231276,5.52262424093949,4.41809939275159 +31109,2007,0,8.5571828396324,5.52262424093949,5.52262424093949 +31123,2003,0,4.46590811865458,1.69377906086785,0.33875581217357 +31123,2004,0,4.40671924726425,1.69377906086785,0.677511624347141 +31123,2005,0,4.20469261939097,1.69377906086785,1.01626743652071 +31123,2006,0,4.24849524204936,1.69377906086785,1.35502324869428 +31123,2007,0,4.04305126783455,1.69377906086785,1.69377906086785 +31127,2003,0,4.92725368515721,2.02498535588378,0.404997071176755 +31127,2004,0,4.95582705760126,2.02498535588378,0.809994142353511 +31127,2005,0,4.71849887129509,2.02498535588378,1.21499121353027 +31127,2006,0,4.64439089914137,2.02498535588378,1.61998828470702 +31127,2007,0,4.67282883446191,2.02498535588378,2.02498535588378 +31131,2003,0,5.94542060860658,2.7341077354215,0.5468215470843 +31131,2004,0,5.70711026474888,2.7341077354215,1.0936430941686 +31131,2005,0,5.69035945432406,2.7341077354215,1.6404646412529 +31131,2006,0,5.74939298590825,2.7341077354215,2.1872861883372 +31131,2007,0,5.80814248998044,2.7341077354215,2.7341077354215 +31133,2003,0,3.13549421592915,1.12719974552002,0.225439949104004 +31133,2004,0,3.09104245335832,1.12719974552002,0.450879898208009 +31133,2005,0,3.13549421592915,1.12719974552002,0.676319847312013 +31133,2006,0,3.49650756146648,1.12719974552002,0.901759796416018 +31133,2007,0,3.55534806148941,1.12719974552002,1.12719974552002 +31147,2003,0,4.83628190695148,2.25454964395549,0.450909928791098 +31147,2004,0,4.92725368515721,2.25454964395549,0.901819857582196 +31147,2005,0,4.93447393313069,2.25454964395549,1.35272978637329 +31147,2006,0,4.72738781871234,2.25454964395549,1.80363971516439 +31147,2007,0,4.74493212836325,2.25454964395549,2.25454964395549 +31151,2003,0,5.66296048013595,2.6277796897056,0.52555593794112 +31151,2004,0,5.44241771052179,2.6277796897056,1.05111187588224 +31151,2005,0,5.50125821054473,2.6277796897056,1.57666781382336 +31151,2006,0,5.4971682252932,2.6277796897056,2.10222375176448 +31151,2007,0,5.40267738187228,2.6277796897056,2.6277796897056 +31153,2003,0,7.53315880745556,4.8088862396362,0.961777247927241 +31153,2004,0,7.49665243816828,4.8088862396362,1.92355449585448 +31153,2005,0,7.53529670244409,4.8088862396362,2.88533174378172 +31153,2006,0,7.53743003658651,4.8088862396362,3.84710899170896 +31153,2007,0,7.61283103040736,4.8088862396362,4.8088862396362 +31155,2003,0,5.60947179518496,2.9871959425317,0.597439188506341 +31155,2004,0,5.6021188208797,2.9871959425317,1.19487837701268 +31155,2005,0,5.46806014113513,2.9871959425317,1.79231756551902 +31155,2006,0,5.50533153593236,2.9871959425317,2.38975675402536 +31155,2007,0,5.63835466933375,2.9871959425317,2.9871959425317 +31161,2003,0,4.45434729625351,1.82422665936556,0.364845331873111 +31161,2004,0,4.31748811353631,1.82422665936556,0.729690663746223 +31161,2005,0,4.26267987704132,1.82422665936556,1.09453599561933 +31161,2006,0,4.27666611901606,1.82422665936556,1.45938132749245 +31161,2007,0,4.11087386417331,1.82422665936556,1.82422665936556 +31177,2003,0,5.84932477994686,2.93279247378012,0.586558494756023 +31177,2004,0,5.83773044716594,2.93279247378012,1.17311698951205 +31177,2005,0,5.73009978297357,2.93279247378012,1.75967548426807 +31177,2006,0,5.67332326717149,2.93279247378012,2.34623397902409 +31177,2007,0,5.59842195899838,2.93279247378012,2.93279247378012 +32007,2003,2007,6.51323011091231,3.81310833725371,0.762621667450742 +32007,2004,2007,6.67203294546107,3.81310833725371,1.52524333490148 +32007,2005,2007,6.75110146893676,3.81310833725371,2.28786500235223 +32007,2006,2007,6.81563999007433,3.81310833725371,3.05048666980297 +32007,2007,2007,6.88653164253051,3.81310833725371,3.81310833725371 +32011,2003,2007,2.56494935746154,0.501381164937991,0.100276232987598 +32011,2004,2007,3.13549421592915,0.501381164937991,0.200552465975196 +32011,2005,2007,2.83321334405622,0.501381164937991,0.300828698962795 +32011,2006,2007,3.25809653802148,0.501381164937991,0.401104931950393 +32011,2007,2007,3.3322045101752,0.501381164937991,0.501381164937991 +32033,2003,2007,4.98360662170834,2.21713613116156,0.443427226232313 +32033,2004,2007,4.91265488573605,2.21713613116156,0.886854452464625 +32033,2005,2007,5.07517381523383,2.21713613116156,1.33028167869694 +32033,2006,2007,5.18738580584075,2.21713613116156,1.77370890492925 +32033,2007,2007,5.18178355029209,2.21713613116156,2.21713613116156 +35006,2003,0,5.44241771052179,3.24239701990955,0.648479403981909 +35006,2004,0,5.13579843705026,3.24239701990955,1.29695880796382 +35006,2005,0,5.26269018890489,3.24239701990955,1.94543821194573 +35006,2006,0,5.47227067367148,3.24239701990955,2.59391761592764 +35006,2007,0,5.44241771052179,3.24239701990955,3.24239701990955 +35013,2003,0,7.57865685059476,5.16296717805949,1.0325934356119 +35013,2004,0,7.56060116276856,5.16296717805949,2.06518687122379 +35013,2005,0,7.62851762657506,5.16296717805949,3.09778030683569 +35013,2006,0,7.59689443814454,5.16296717805949,4.13037374244759 +35013,2007,0,7.65681009148038,5.16296717805949,5.16296717805949 +35025,2003,0,6.71901315438526,4.01658119931192,0.803316239862384 +35025,2004,0,6.70563909486,4.01658119931192,1.60663247972477 +35025,2005,0,6.79122146272619,4.01658119931192,2.40994871958715 +35025,2006,0,6.89162589705225,4.01658119931192,3.21326495944954 +35025,2007,0,7.03262426102801,4.01658119931192,4.01658119931192 +35033,2003,0,3.04452243772342,1.64480505627139,0.328961011254278 +35033,2004,0,2.19722457733622,1.64480505627139,0.657922022508557 +35033,2005,0,2.484906649788,1.64480505627139,0.986883033762835 +35033,2006,0,3.04452243772342,1.64480505627139,1.31584404501711 +35033,2007,0,2.83321334405622,1.64480505627139,1.64480505627139 +35045,2003,0,7.44658509915773,4.73445130899984,0.946890261799969 +35045,2004,0,7.37713371283395,4.73445130899984,1.89378052359994 +35045,2005,0,7.35308192051543,4.73445130899984,2.84067078539991 +35045,2006,0,7.36770857237437,4.73445130899984,3.78756104719987 +35045,2007,0,7.44775128004791,4.73445130899984,4.73445130899984 +37005,2003,2007,4.62497281328427,2.36809189519617,0.473618379039233 +37005,2004,2007,4.58496747867057,2.36809189519617,0.947236758078466 +37005,2005,2007,4.67282883446191,2.36809189519617,1.4208551371177 +37005,2006,2007,4.63472898822964,2.36809189519617,1.89447351615693 +37005,2007,2007,4.62497281328427,2.36809189519617,2.36809189519617 +37021,2003,2007,8.0487882835342,5.32947682879714,1.06589536575943 +37021,2004,2007,8.06589354696427,5.32947682879714,2.13179073151886 +37021,2005,2007,8.14206328310415,5.32947682879714,3.19768609727828 +37021,2006,2007,8.18395571730495,5.32947682879714,4.26358146303771 +37021,2007,2007,8.24800570160062,5.32947682879714,5.32947682879714 +37025,2003,2007,7.6231530684769,4.87567812362863,0.975135624725727 +37025,2004,2007,7.68570306123455,4.87567812362863,1.95027124945145 +37025,2005,2007,7.70571282389443,4.87567812362863,2.92540687417718 +37025,2006,2007,7.8087293067444,4.87567812362863,3.90054249890291 +37025,2007,2007,7.900636613018,4.87567812362863,4.87567812362863 +37029,2003,2007,3.29583686600433,1.92934513218062,0.385869026436124 +37029,2004,2007,2.94443897916644,1.92934513218062,0.771738052872247 +37029,2005,2007,3.46573590279973,1.92934513218062,1.15760707930837 +37029,2006,2007,3.36729582998647,1.92934513218062,1.54347610574449 +37029,2007,2007,3.25809653802148,1.92934513218062,1.92934513218062 +37041,2003,2007,5.06890420222023,2.67594014718398,0.535188029436795 +37041,2004,2007,5.18738580584075,2.67594014718398,1.07037605887359 +37041,2005,2007,4.93447393313069,2.67594014718398,1.60556408831039 +37041,2006,2007,5.08140436498446,2.67594014718398,2.14075211774718 +37041,2007,2007,5.08140436498446,2.67594014718398,2.67594014718398 +37051,2003,2007,8.03624994213212,5.71361068584184,1.14272213716837 +37051,2004,2007,8.04269949689764,5.71361068584184,2.28544427433674 +37051,2005,2007,8.08948247436075,5.71361068584184,3.42816641150511 +37051,2006,2007,8.12622252945853,5.71361068584184,4.57088854867348 +37051,2007,2007,8.13973227971767,5.71361068584184,5.71361068584184 +37057,2003,2007,7.2841348061952,4.99210465747682,0.998420931495364 +37057,2004,2007,7.2283884515736,4.99210465747682,1.99684186299073 +37057,2005,2007,7.22983877815125,4.99210465747682,2.99526279448609 +37057,2006,2007,7.24208235925696,4.99210465747682,3.99368372598146 +37057,2007,2007,7.24064969425547,4.99210465747682,4.99210465747682 +37059,2003,2007,5.95842469302978,3.55062262848208,0.710124525696416 +37059,2004,2007,5.9427993751267,3.55062262848208,1.42024905139283 +37059,2005,2007,6.13988455222626,3.55062262848208,2.13037357708925 +37059,2006,2007,6.13556489108174,3.55062262848208,2.84049810278567 +37059,2007,2007,6.10924758276437,3.55062262848208,3.55062262848208 +37063,2003,2007,7.96797317966293,5.40857885280557,1.08171577056111 +37063,2004,2007,7.96485088744731,5.40857885280557,2.16343154112223 +37063,2005,2007,7.95296679092313,5.40857885280557,3.24514731168334 +37063,2006,2007,8.02059914989697,5.40857885280557,4.32686308224446 +37063,2007,2007,8.02845516411425,5.40857885280557,5.40857885280557 +37075,2003,2007,4.02535169073515,2.07856615864388,0.415713231728776 +37075,2004,2007,4.24849524204936,2.07856615864388,0.831426463457553 +37075,2005,2007,4.17438726989564,2.07856615864388,1.24713969518633 +37075,2006,2007,4.47733681447821,2.07856615864388,1.66285292691511 +37075,2007,2007,4.75359019110636,2.07856615864388,2.07856615864388 +37079,2003,2007,4.61512051684126,2.94306962097069,0.588613924194137 +37079,2004,2007,4.70953020131233,2.94306962097069,1.17722784838827 +37079,2005,2007,4.51085950651685,2.94306962097069,1.76584177258241 +37079,2006,2007,4.51085950651685,2.94306962097069,2.35445569677655 +37079,2007,2007,4.51085950651685,2.94306962097069,2.94306962097069 +37085,2003,2007,6.84374994900622,4.5111341940615,0.902226838812299 +37085,2004,2007,6.85751406254539,4.5111341940615,1.8044536776246 +37085,2005,2007,6.93731408122368,4.5111341940615,2.7066805164369 +37085,2006,2007,7.0352685992811,4.5111341940615,3.6089073552492 +37085,2007,2007,6.99301512293296,4.5111341940615,4.5111341940615 +37093,2003,2007,5.23110861685459,3.51589417829697,0.703178835659394 +37093,2004,2007,5.14166355650266,3.51589417829697,1.40635767131879 +37093,2005,2007,5.28826703069454,3.51589417829697,2.10953650697818 +37093,2006,2007,5.25227342804663,3.51589417829697,2.81271534263758 +37093,2007,2007,5.30826769740121,3.51589417829697,3.51589417829697 +37099,2003,2007,5.79301360838414,3.50016752229797,0.700033504459593 +37099,2004,2007,5.75574221358691,3.50016752229797,1.40006700891919 +37099,2005,2007,5.82600010738045,3.50016752229797,2.10010051337878 +37099,2006,2007,5.84932477994686,3.50016752229797,2.80013401783837 +37099,2007,2007,5.97380961186926,3.50016752229797,3.50016752229797 +37103,2003,2007,3.8286413964891,2.33997721221111,0.467995442442222 +37103,2004,2007,3.66356164612965,2.33997721221111,0.935990884884443 +37103,2005,2007,3.76120011569356,2.33997721221111,1.40398632732667 +37103,2006,2007,4.11087386417331,2.33997721221111,1.87198176976889 +37103,2007,2007,3.13549421592915,2.33997721221111,2.33997721221111 +37105,2003,2007,6.59850902861452,3.89263629162796,0.778527258325591 +37105,2004,2007,6.58617165485467,3.89263629162796,1.55705451665118 +37105,2005,2007,6.67582322163485,3.89263629162796,2.33558177497677 +37105,2006,2007,6.59304453414244,3.89263629162796,3.11410903330236 +37105,2007,2007,6.65027904858742,3.89263629162796,3.89263629162796 +37117,2003,2007,5.35658627467201,3.24231887659467,0.648463775318934 +37117,2004,2007,5.40267738187228,3.24231887659467,1.29692755063787 +37117,2005,2007,5.48893772615669,3.24231887659467,1.9453913259568 +37117,2006,2007,5.44241771052179,3.24231887659467,2.59385510127573 +37117,2007,2007,5.41164605185504,3.24231887659467,3.24231887659467 +37123,2003,2007,5.53338948872752,3.28922244628875,0.65784448925775 +37123,2004,2007,5.4380793089232,3.28922244628875,1.3156889785155 +37123,2005,2007,5.55682806169954,3.28922244628875,1.97353346777325 +37123,2006,2007,5.605802066296,3.28922244628875,2.631377957031 +37123,2007,2007,5.57972982598622,3.28922244628875,3.28922244628875 +37127,2003,2007,7.1708884785125,4.47072408943517,0.894144817887034 +37127,2004,2007,7.11558212618445,4.47072408943517,1.78828963577407 +37127,2005,2007,7.11639414409346,4.47072408943517,2.6824344536611 +37127,2006,2007,7.22548147278229,4.47072408943517,3.57657927154813 +37127,2007,2007,7.35627987655075,4.47072408943517,4.47072408943517 +37155,2003,2007,6.85751406254539,4.81493666186062,0.962987332372125 +37155,2004,2007,6.85329909318608,4.81493666186062,1.92597466474425 +37155,2005,2007,7.05444965813294,4.81493666186062,2.88896199711637 +37155,2006,2007,6.99393297522319,4.81493666186062,3.8519493294885 +37155,2007,2007,7.03790596344718,4.81493666186062,4.81493666186062 +37161,2003,2007,6.30078579466324,4.14153026532882,0.828306053065765 +37161,2004,2007,6.24610676548156,4.14153026532882,1.65661210613153 +37161,2005,2007,6.25575004175337,4.14153026532882,2.48491815919729 +37161,2006,2007,6.29710931993394,4.14153026532882,3.31322421226306 +37161,2007,2007,6.4281052726846,4.14153026532882,4.14153026532882 +37163,2003,2007,6.31716468674728,4.09702430184386,0.819404860368772 +37163,2004,2007,6.38519439899773,4.09702430184386,1.63880972073754 +37163,2005,2007,6.32435896238131,4.09702430184386,2.45821458110632 +37163,2006,2007,6.34212141872115,4.09702430184386,3.27761944147509 +37163,2007,2007,6.44730586254121,4.09702430184386,4.09702430184386 +37165,2003,2007,6.15485809401642,3.58346338135729,0.716692676271457 +37165,2004,2007,6.20657592672493,3.58346338135729,1.43338535254291 +37165,2005,2007,6.14846829591765,3.58346338135729,2.15007802881437 +37165,2006,2007,6.18208490671663,3.58346338135729,2.86677070508583 +37165,2007,2007,6.16331480403464,3.58346338135729,3.58346338135729 +37167,2003,2007,6.62539236800796,4.06216566385787,0.812433132771573 +37167,2004,2007,6.54391184556479,4.06216566385787,1.62486626554315 +37167,2005,2007,6.61338421837956,4.06216566385787,2.43729939831472 +37167,2006,2007,6.62936325343745,4.06216566385787,3.24973253108629 +37167,2007,2007,6.61740297797448,4.06216566385787,4.06216566385787 +37175,2003,2007,5.67675380226828,3.37874725282456,0.675749450564913 +37175,2004,2007,5.70711026474888,3.37874725282456,1.35149890112983 +37175,2005,2007,5.75890177387728,3.37874725282456,2.02724835169474 +37175,2006,2007,5.78074351579233,3.37874725282456,2.70299780225965 +37175,2007,2007,5.71373280550937,3.37874725282456,3.37874725282456 +37189,2003,2007,6.3261494731551,3.75408181735957,0.750816363471914 +37189,2004,2007,6.35610766069589,3.75408181735957,1.50163272694383 +37189,2005,2007,6.35088571671474,3.75408181735957,2.25244909041574 +37189,2006,2007,6.44413125670044,3.75408181735957,3.00326545388766 +37189,2007,2007,6.43294009273918,3.75408181735957,3.75408181735957 +37191,2003,2007,7.24708058458576,4.73029509291699,0.946059018583398 +37191,2004,2007,7.2283884515736,4.73029509291699,1.8921180371668 +37191,2005,2007,7.21744343169653,4.73029509291699,2.8381770557502 +37191,2006,2007,7.30988148582479,4.73029509291699,3.78423607433359 +37191,2007,2007,7.30451594646016,4.73029509291699,4.73029509291699 +38001,2003,0,4.11087386417331,0.952815506556527,0.190563101311305 +38001,2004,0,4.0943445622221,0.952815506556527,0.381126202622611 +38001,2005,0,4.06044301054642,0.952815506556527,0.571689303933916 +38001,2006,0,3.89182029811063,0.952815506556527,0.762252405245222 +38001,2007,0,4.17438726989564,0.952815506556527,0.952815506556527 +38021,2003,0,4.68213122712422,1.75041650569367,0.350083301138734 +38021,2004,0,4.29045944114839,1.75041650569367,0.700166602277469 +38021,2005,0,4.51085950651685,1.75041650569367,1.0502499034162 +38021,2006,0,4.74493212836325,1.75041650569367,1.40033320455494 +38021,2007,0,4.62497281328427,1.75041650569367,1.75041650569367 +38023,2003,0,4.11087386417331,0.825490367547658,0.165098073509532 +38023,2004,0,4.11087386417331,0.825490367547658,0.330196147019063 +38023,2005,0,3.87120101090789,0.825490367547658,0.495294220528595 +38023,2006,0,3.89182029811063,0.825490367547658,0.660392294038127 +38023,2007,0,3.85014760171006,0.825490367547658,0.825490367547659 +38035,2003,0,7.28138566357028,4.19130489492644,0.838260978985288 +38035,2004,0,7.20637729147225,4.19130489492644,1.67652195797058 +38035,2005,0,7.29369772060144,4.19130489492644,2.51478293695586 +38035,2006,0,7.28207365809346,4.19130489492644,3.35304391594115 +38035,2007,0,7.28207365809346,4.19130489492644,4.19130489492644 +38037,2003,0,3.46573590279973,1.04415610287205,0.20883122057441 +38037,2004,0,3.2188758248682,1.04415610287205,0.41766244114882 +38037,2005,0,3.13549421592915,1.04415610287205,0.626493661723231 +38037,2006,0,3.29583686600433,1.04415610287205,0.835324882297641 +38037,2007,0,3.46573590279973,1.04415610287205,1.04415610287205 +38049,2003,0,3.98898404656427,1.78959045194322,0.357918090388643 +38049,2004,0,3.63758615972639,1.78959045194322,0.715836180777286 +38049,2005,0,3.89182029811063,1.78959045194322,1.07375427116593 +38049,2006,0,3.8286413964891,1.78959045194322,1.43167236155457 +38049,2007,0,4.02535169073515,1.78959045194322,1.78959045194322 +38061,2003,0,4.9416424226093,1.89175562238673,0.378351124477346 +38061,2004,0,4.70953020131233,1.89175562238673,0.756702248954692 +38061,2005,0,4.64439089914137,1.89175562238673,1.13505337343204 +38061,2006,0,4.59511985013459,1.89175562238673,1.51340449790938 +38061,2007,0,4.7361984483945,1.89175562238673,1.89175562238673 +38063,2003,0,3.91202300542815,1.31237867816972,0.262475735633944 +38063,2004,0,4.04305126783455,1.31237867816972,0.524951471267889 +38063,2005,0,4.04305126783455,1.31237867816972,0.787427206901833 +38063,2006,0,3.80666248977032,1.31237867816972,1.04990294253578 +38063,2007,0,3.71357206670431,1.31237867816972,1.31237867816972 +38073,2003,0,4.83628190695148,1.7732559976635,0.354651199532699 +38073,2004,0,4.87519732320115,1.7732559976635,0.709302399065398 +38073,2005,0,4.75359019110636,1.7732559976635,1.0639535985981 +38073,2006,0,4.76217393479776,1.7732559976635,1.4186047981308 +38073,2007,0,4.79579054559674,1.7732559976635,1.7732559976635 +38097,2003,0,5.16478597392351,2.13735661362627,0.427471322725254 +38097,2004,0,5.14166355650266,2.13735661362627,0.854942645450507 +38097,2005,0,5.01727983681492,2.13735661362627,1.28241396817576 +38097,2006,0,5.08759633523238,2.13735661362627,1.70988529090101 +38097,2007,0,5.05624580534831,2.13735661362627,2.13735661362627 +38099,2003,0,5.84643877505772,2.5168089821336,0.50336179642672 +38099,2004,0,5.77144112313002,2.5168089821336,1.00672359285344 +38099,2005,0,5.6970934865054,2.5168089821336,1.51008538928016 +38099,2006,0,5.69035945432406,2.5168089821336,2.01344718570688 +38099,2007,0,5.67332326717149,2.5168089821336,2.5168089821336 +38105,2003,0,6.16751649088834,2.98371029832667,0.596742059665333 +38105,2004,0,6.07073772800249,2.98371029832667,1.19348411933067 +38105,2005,0,6.12468339089421,2.98371029832667,1.790226178996 +38105,2006,0,6.16961073249146,2.98371029832667,2.38696823866133 +38105,2007,0,6.18826412308259,2.98371029832667,2.98371029832667 +39019,2003,2007,5.46383180502561,3.36162460654233,0.672324921308466 +39019,2004,2007,5.44241771052179,3.36162460654233,1.34464984261693 +39019,2005,2007,5.40267738187228,3.36162460654233,2.0169747639254 +39019,2006,2007,5.27811465923052,3.36162460654233,2.68929968523387 +39019,2007,2007,5.35185813347607,3.36162460654233,3.36162460654233 +39021,2003,2007,5.89989735358249,3.66073714816766,0.732147429633531 +39021,2004,2007,5.77455154554441,3.66073714816766,1.46429485926706 +39021,2005,2007,5.80211837537706,3.66073714816766,2.19644228890059 +39021,2006,2007,5.8636311755981,3.66073714816766,2.92858971853412 +39021,2007,2007,5.78382518232974,3.66073714816766,3.66073714816766 +39035,2003,2007,10.0020198086004,7.23991680927341,1.44798336185468 +39035,2004,2007,9.94755228369823,7.23991680927341,2.89596672370936 +39035,2005,2007,9.91467474192669,7.23991680927341,4.34395008556404 +39035,2006,2007,9.91595945403145,7.23991680927341,5.79193344741873 +39035,2007,2007,9.89374020108125,7.23991680927341,7.23991680927341 +39047,2003,2007,6.34212141872115,3.34755044243138,0.669510088486276 +39047,2004,2007,6.3297209055227,3.34755044243138,1.33902017697255 +39047,2005,2007,6.33859407820318,3.34755044243138,2.00853026545883 +39047,2006,2007,6.23244801655052,3.34755044243138,2.6780403539451 +39047,2007,2007,6.24416690066374,3.34755044243138,3.34755044243138 +39055,2003,2007,7.29437729928882,4.50970499417002,0.901940998834005 +39055,2004,2007,7.24208235925696,4.50970499417002,1.80388199766801 +39055,2005,2007,7.22037383672395,4.50970499417002,2.70582299650201 +39055,2006,2007,7.30921236569276,4.50970499417002,3.60776399533602 +39055,2007,2007,7.24279792279376,4.50970499417002,4.50970499417002 +39063,2003,2007,7.15461535691366,4.26682619873408,0.853365239746817 +39063,2004,2007,7.06219163228656,4.26682619873408,1.70673047949363 +39063,2005,2007,7.12929754892937,4.26682619873408,2.56009571924045 +39063,2006,2007,7.11395610956603,4.26682619873408,3.41346095898727 +39063,2007,2007,7.14677217945264,4.26682619873408,4.26682619873408 +39071,2003,2007,6.11368217983223,3.71051862921742,0.742103725843483 +39071,2004,2007,6.14418563412565,3.71051862921742,1.48420745168697 +39071,2005,2007,6.21060007702465,3.71051862921742,2.22631117753045 +39071,2006,2007,6.19236248947487,3.71051862921742,2.96841490337393 +39071,2007,2007,6.10702288774225,3.71051862921742,3.71051862921742 +39079,2003,2007,5.8348107370626,3.4855691668815,0.697113833376299 +39079,2004,2007,5.75574221358691,3.4855691668815,1.3942276667526 +39079,2005,2007,5.86929691313377,3.4855691668815,2.0913415001289 +39079,2006,2007,5.82600010738045,3.4855691668815,2.7884553335052 +39079,2007,2007,5.78996017089725,3.4855691668815,3.4855691668815 +39095,2003,2007,9.01054706927019,6.12041609322756,1.22408321864551 +39095,2004,2007,8.92718138825199,6.12041609322756,2.44816643729102 +39095,2005,2007,8.84202652949881,6.12041609322756,3.67224965593654 +39095,2006,2007,8.84029066908897,6.12041609322756,4.89633287458205 +39095,2007,2007,8.82173238093444,6.12041609322756,6.12041609322756 +39097,2003,2007,5.97635090929793,3.69419032643251,0.738838065286503 +39097,2004,2007,5.94017125272043,3.69419032643251,1.47767613057301 +39097,2005,2007,6.13339804299665,3.69419032643251,2.21651419585951 +39097,2006,2007,5.95064255258773,3.69419032643251,2.95535226114601 +39097,2007,2007,5.85507192220243,3.69419032643251,3.69419032643251 +39107,2003,2007,6.77650699237218,3.71171668801669,0.742343337603339 +39107,2004,2007,6.7696419768525,3.71171668801669,1.48468667520668 +39107,2005,2007,6.73459165997295,3.71171668801669,2.22703001281002 +39107,2006,2007,6.80572255341699,3.71171668801669,2.96937335041335 +39107,2007,2007,6.71901315438526,3.71171668801669,3.71171668801669 +39111,2003,2007,3.3322045101752,2.71997877196748,0.543995754393497 +39111,2004,2007,3.25809653802148,2.71997877196748,1.08799150878699 +39111,2005,2007,3.52636052461616,2.71997877196748,1.63198726318049 +39111,2006,2007,3.49650756146648,2.71997877196748,2.17598301757399 +39111,2007,2007,3.40119738166216,2.71997877196748,2.71997877196748 +39119,2003,2007,7.12044437239249,4.43775694592726,0.887551389185451 +39119,2004,2007,7.02553831463852,4.43775694592726,1.7751027783709 +39119,2005,2007,6.94601399109923,4.43775694592726,2.66265416755635 +39119,2006,2007,6.8596149036542,4.43775694592726,3.5502055567418 +39119,2007,2007,6.89568269774787,4.43775694592726,4.43775694592726 +39143,2003,2007,6.83195356556585,4.1237739062461,0.824754781249221 +39143,2004,2007,6.82871207164168,4.1237739062461,1.64950956249844 +39143,2005,2007,6.74523634948436,4.1237739062461,2.47426434374766 +39143,2006,2007,6.73340189183736,4.1237739062461,3.29901912499688 +39143,2007,2007,6.76503897678054,4.1237739062461,4.1237739062461 +39163,2003,2007,4.23410650459726,2.54991381109661,0.509982762219322 +39163,2004,2007,4.24849524204936,2.54991381109661,1.01996552443864 +39163,2005,2007,4.0943445622221,2.54991381109661,1.52994828665797 +39163,2006,2007,3.87120101090789,2.54991381109661,2.03993104887729 +39163,2007,2007,4.00733318523247,2.54991381109661,2.54991381109661 +39173,2003,2007,7.55799495853081,4.79632759144445,0.959265518288889 +39173,2004,2007,7.52131798019924,4.79632759144445,1.91853103657778 +39173,2005,2007,7.46508273639955,4.79632759144445,2.87779655486667 +39173,2006,2007,7.47363710849621,4.79632759144445,3.83706207315556 +39173,2007,2007,7.51261754467451,4.79632759144445,4.79632759144445 +40003,2003,0,4.02535169073515,1.80910810756267,0.361821621512534 +40003,2004,0,3.76120011569356,1.80910810756267,0.723643243025067 +40003,2005,0,3.91202300542815,1.80910810756267,1.0854648645376 +40003,2006,0,3.85014760171006,1.80910810756267,1.44728648605013 +40003,2007,0,3.98898404656427,1.80910810756267,1.80910810756267 +40013,2003,0,6.1779441140506,3.59824333385449,0.719648666770898 +40013,2004,0,6.24222326545517,3.59824333385449,1.4392973335418 +40013,2005,0,6.2803958389602,3.59824333385449,2.1589460003127 +40013,2006,0,6.13556489108174,3.59824333385449,2.87859466708359 +40013,2007,0,6.19236248947487,3.59824333385449,3.59824333385449 +40015,2003,0,5.45958551414416,3.40618492317319,0.681236984634639 +40015,2004,0,5.27811465923052,3.40618492317319,1.36247396926928 +40015,2005,0,5.32787616878958,3.40618492317319,2.04371095390392 +40015,2006,0,5.41164605185504,3.40618492317319,2.72494793853856 +40015,2007,0,5.3890717298165,3.40618492317319,3.40618492317319 +40017,2003,0,7.12125245324454,4.47388769126739,0.894777538253478 +40017,2004,0,7.08590146436561,4.47388769126739,1.78955507650696 +40017,2005,0,7.00033446027523,4.47388769126739,2.68433261476043 +40017,2006,0,7.07834157955767,4.47388769126739,3.57911015301391 +40017,2007,0,7.09090982207998,4.47388769126739,4.47388769126739 +40025,2003,0,3.80666248977032,1.14676733055516,0.229353466111031 +40025,2004,0,3.58351893845611,1.14676733055516,0.458706932222063 +40025,2005,0,3.55534806148941,1.14676733055516,0.688060398333094 +40025,2006,0,3.93182563272433,1.14676733055516,0.917413864444126 +40025,2007,0,3.55534806148941,1.14676733055516,1.14676733055516 +40039,2003,0,6.04500531403601,3.26354321636336,0.652708643272672 +40039,2004,0,5.98645200528444,3.26354321636336,1.30541728654534 +40039,2005,0,5.89989735358249,3.26354321636336,1.95812592981802 +40039,2006,0,5.98393628068719,3.26354321636336,2.61083457309069 +40039,2007,0,6.02344759296103,3.26354321636336,3.26354321636336 +40063,2003,0,4.62497281328427,2.64999726965359,0.529999453930719 +40063,2004,0,4.45434729625351,2.64999726965359,1.05999890786144 +40063,2005,0,4.27666611901606,2.64999726965359,1.58999836179216 +40063,2006,0,4.11087386417331,2.64999726965359,2.11999781572287 +40063,2007,0,4.52178857704904,2.64999726965359,2.64999726965359 +40069,2003,0,4.49980967033027,2.35261258659369,0.470522517318739 +40069,2004,0,4.38202663467388,2.35261258659369,0.941045034637477 +40069,2005,0,4.18965474202643,2.35261258659369,1.41156755195622 +40069,2006,0,4.18965474202643,2.35261258659369,1.88209006927495 +40069,2007,0,4.66343909411207,2.35261258659369,2.35261258659369 +40075,2003,0,4.60517018598809,2.32503128182388,0.465006256364775 +40075,2004,0,4.38202663467388,2.32503128182388,0.93001251272955 +40075,2005,0,4.26267987704132,2.32503128182388,1.39501876909433 +40075,2006,0,4.26267987704132,2.32503128182388,1.8600250254591 +40075,2007,0,4.11087386417331,2.32503128182388,2.32503128182388 +40083,2003,0,5.65599181081985,3.52412272849945,0.704824545699891 +40083,2004,0,5.38449506278909,3.52412272849945,1.40964909139978 +40083,2005,0,5.42934562895444,3.52412272849945,2.11447363709967 +40083,2006,0,5.4380793089232,3.52412272849945,2.81929818279956 +40083,2007,0,5.47227067367148,3.52412272849945,3.52412272849945 +40101,2003,0,7.02642680869964,4.24062146794478,0.848124293588956 +40101,2004,0,6.7945865808765,4.24062146794478,1.69624858717791 +40101,2005,0,6.91075078796194,4.24062146794478,2.54437288076687 +40101,2006,0,6.95177216439891,4.24062146794478,3.39249717435583 +40101,2007,0,6.78332520060396,4.24062146794478,4.24062146794478 +40119,2003,0,6.96034772910131,4.22229792653256,0.844459585306512 +40119,2004,0,6.87729607149743,4.22229792653256,1.68891917061302 +40119,2005,0,6.7202201551353,4.22229792653256,2.53337875591954 +40119,2006,0,6.81014245011514,4.22229792653256,3.37783834122605 +40119,2007,0,6.83625927727707,4.22229792653256,4.22229792653256 +40137,2003,0,6.20657592672493,3.76542374175372,0.753084748350744 +40137,2004,0,6.17170059741091,3.76542374175372,1.50616949670149 +40137,2005,0,6.19440539110467,3.76542374175372,2.25925424505223 +40137,2006,0,6.30261897574491,3.76542374175372,3.01233899340298 +40137,2007,0,6.30809844150953,3.76542374175372,3.76542374175372 +40143,2003,0,9.27556615689389,6.33381057064612,1.26676211412922 +40143,2004,0,9.18502256399189,6.33381057064612,2.53352422825845 +40143,2005,0,9.21303673352392,6.33381057064612,3.80028634238767 +40143,2006,0,9.31775876874345,6.33381057064612,5.0670484565169 +40143,2007,0,9.36108525866861,6.33381057064612,6.33381057064612 +40145,2003,0,5.89164421182577,4.05162841381337,0.810325682762674 +40145,2004,0,5.85220247977447,4.05162841381337,1.62065136552535 +40145,2005,0,5.92425579741453,4.05162841381337,2.43097704828802 +40145,2006,0,6.00635315960173,4.05162841381337,3.24130273105069 +40145,2007,0,6.02102334934953,4.05162841381337,4.05162841381337 +40153,2003,0,5.73334127689775,2.91701368884259,0.583402737768517 +40153,2004,0,5.64190707093811,2.91701368884259,1.16680547553703 +40153,2005,0,5.7268477475872,2.91701368884259,1.75020821330555 +40153,2006,0,5.86929691313377,2.91701368884259,2.33361095107407 +40153,2007,0,5.95064255258773,2.91701368884259,2.91701368884259 +45007,2003,0,7.6586995582683,5.1104202954209,1.02208405908418 +45007,2004,0,7.61184239958042,5.1104202954209,2.04416811816836 +45007,2005,0,7.62217459481762,5.1104202954209,3.06625217725254 +45007,2006,0,7.65822752616135,5.1104202954209,4.08833623633672 +45007,2007,0,7.63964228785801,5.1104202954209,5.1104202954209 +45009,2003,0,5.03043792139244,2.81289058151315,0.56257811630263 +45009,2004,0,4.93447393313069,2.81289058151315,1.12515623260526 +45009,2005,0,5.12989871492307,2.81289058151315,1.68773434890789 +45009,2006,0,5.05624580534831,2.81289058151315,2.25031246521052 +45009,2007,0,5.14166355650266,2.81289058151315,2.81289058151315 +45013,2003,0,7.58222919427646,4.79526974884863,0.959053949769726 +45013,2004,0,7.57044325205737,4.79526974884863,1.91810789953945 +45013,2005,0,7.6511201757027,4.79526974884863,2.87716184930918 +45013,2006,0,7.74586822979227,4.79526974884863,3.8362157990789 +45013,2007,0,7.71779621101358,4.79526974884863,4.79526974884863 +45017,2003,0,4.21950770517611,2.72030809849771,0.544061619699542 +45017,2004,0,4.12713438504509,2.72030809849771,1.08812323939908 +45017,2005,0,4.14313472639153,2.72030809849771,1.63218485909863 +45017,2006,0,4.17438726989564,2.72030809849771,2.17624647879817 +45017,2007,0,4.23410650459726,2.72030809849771,2.72030809849771 +45019,2003,0,8.52237971810354,5.73647229247886,1.14729445849577 +45019,2004,0,8.50936261230105,5.73647229247886,2.29458891699154 +45019,2005,0,8.60575336839572,5.73647229247886,3.44188337548732 +45019,2006,0,8.70018119275335,5.73647229247886,4.58917783398309 +45019,2007,0,8.71292443512011,5.73647229247886,5.73647229247886 +45023,2003,0,5.34710753071747,3.52835852727883,0.705671705455767 +45023,2004,0,5.35185813347607,3.52835852727883,1.41134341091153 +45023,2005,0,5.35658627467201,3.52835852727883,2.1170151163673 +45023,2006,0,5.47646355193151,3.52835852727883,2.82268682182307 +45023,2007,0,5.52545293913178,3.52835852727883,3.52835852727883 +45025,2003,0,5.80513496891649,3.75579015939656,0.751158031879313 +45025,2004,0,5.73334127689775,3.75579015939656,1.50231606375863 +45025,2005,0,5.85220247977447,3.75579015939656,2.25347409563794 +45025,2006,0,5.78382518232974,3.75579015939656,3.00463212751725 +45025,2007,0,5.89989735358249,3.75579015939656,3.75579015939656 +45037,2003,0,5.0998664278242,3.2025431702473,0.640508634049461 +45037,2004,0,4.969813299576,3.2025431702473,1.28101726809892 +45037,2005,0,5.03043792139244,3.2025431702473,1.92152590214838 +45037,2006,0,4.99721227376411,3.2025431702473,2.56203453619784 +45037,2007,0,5.27299955856375,3.2025431702473,3.2025431702473 +45051,2003,0,7.98786409608569,5.28131870446459,1.05626374089292 +45051,2004,0,7.97453284413023,5.28131870446459,2.11252748178584 +45051,2005,0,8.11282747875137,5.28131870446459,3.16879122267875 +45051,2006,0,8.19973896063079,5.28131870446459,4.22505496357167 +45051,2007,0,8.28652137368124,5.28131870446459,5.28131870446459 +45055,2003,0,6.1779441140506,3.96360925689958,0.792721851379916 +45055,2004,0,6.30261897574491,3.96360925689958,1.58544370275983 +45055,2005,0,6.4085287910595,3.96360925689958,2.37816555413975 +45055,2006,0,6.28599809450886,3.96360925689958,3.17088740551966 +45055,2007,0,6.3818160174061,3.96360925689958,3.96360925689958 +45059,2003,0,6.42162226780652,4.24229031697041,0.848458063394082 +45059,2004,0,6.25766758788264,4.24229031697041,1.69691612678816 +45059,2005,0,6.32435896238131,4.24229031697041,2.54537419018224 +45059,2006,0,6.35610766069589,4.24229031697041,3.39383225357633 +45059,2007,0,6.34212141872115,4.24229031697041,4.24229031697041 +45067,2003,0,5.69035945432406,3.56857449107029,0.713714898214058 +45067,2004,0,5.605802066296,3.56857449107029,1.42742979642812 +45067,2005,0,5.61312810638807,3.56857449107029,2.14114469464217 +45067,2006,0,5.64544689764324,3.56857449107029,2.85485959285623 +45067,2007,0,5.63835466933375,3.56857449107029,3.56857449107029 +45073,2003,0,6.56526497003536,4.19290702337893,0.838581404675786 +45073,2004,0,6.43294009273918,4.19290702337893,1.67716280935157 +45073,2005,0,6.39692965521615,4.19290702337893,2.51574421402736 +45073,2006,0,6.41836493593621,4.19290702337893,3.35432561870314 +45073,2007,0,6.44413125670044,4.19290702337893,4.19290702337893 +45081,2003,0,5.04985600724954,2.95392020573955,0.59078404114791 +45081,2004,0,5.10594547390058,2.95392020573955,1.18156808229582 +45081,2005,0,5.15329159449778,2.95392020573955,1.77235212344373 +45081,2006,0,5.16478597392351,2.95392020573955,2.36313616459164 +45081,2007,0,5.11198778835654,2.95392020573955,2.95392020573955 +45087,2003,0,5.2257466737132,3.39722282690674,0.679444565381349 +45087,2004,0,5.01727983681492,3.39722282690674,1.3588891307627 +45087,2005,0,5.09375020080676,3.39722282690674,2.03833369614405 +45087,2006,0,5.32787616878958,3.39722282690674,2.7177782615254 +45087,2007,0,5.40717177146012,3.39722282690674,3.39722282690674 +46013,2003,0,6.74993119378857,3.56840530064606,0.713681060129212 +46013,2004,0,6.68586094706836,3.56840530064606,1.42736212025842 +46013,2005,0,6.70196036600254,3.56840530064606,2.14104318038764 +46013,2006,0,6.78219205600679,3.56840530064606,2.85472424051685 +46013,2007,0,6.78671695060508,3.56840530064606,3.56840530064606 +46021,2003,0,3.04452243772342,0.577736329048618,0.115547265809724 +46021,2004,0,2.99573227355399,0.577736329048618,0.231094531619447 +46021,2005,0,2.70805020110221,0.577736329048618,0.346641797429171 +46021,2006,0,2.70805020110221,0.577736329048618,0.462189063238894 +46021,2007,0,2.56494935746154,0.577736329048618,0.577736329048618 +46029,2003,0,6.61606518513282,3.25412713188084,0.650825426376167 +46029,2004,0,6.56244409369372,3.25412713188084,1.30165085275233 +46029,2005,0,6.62406522779989,3.25412713188084,1.9524762791285 +46029,2006,0,6.59304453414244,3.25412713188084,2.60330170550467 +46029,2007,0,6.57228254269401,3.25412713188084,3.25412713188084 +46033,2003,0,4.23410650459726,1.98444381305756,0.396888762611511 +46033,2004,0,4.18965474202643,1.98444381305756,0.793777525223023 +46033,2005,0,4.4188406077966,1.98444381305756,1.19066628783453 +46033,2006,0,4.47733681447821,1.98444381305756,1.58755505044605 +46033,2007,0,4.39444915467244,1.98444381305756,1.98444381305756 +46047,2003,0,4.60517018598809,2.00861663589956,0.401723327179911 +46047,2004,0,4.69134788222914,2.00861663589956,0.803446654359823 +46047,2005,0,4.53259949315326,2.00861663589956,1.20516998153973 +46047,2006,0,4.8283137373023,2.00861663589956,1.60689330871965 +46047,2007,0,4.83628190695148,2.00861663589956,2.00861663589956 +46065,2003,0,6.13339804299665,2.80220820225204,0.560441640450409 +46065,2004,0,6.12686918411419,2.80220820225204,1.12088328090082 +46065,2005,0,6.16751649088834,2.80220820225204,1.68132492135123 +46065,2006,0,6.13556489108174,2.80220820225204,2.24176656180163 +46065,2007,0,6.10479323241498,2.80220820225204,2.80220820225204 +46075,2003,0,3.49650756146648,0.176471143115779,0.0352942286231558 +46075,2004,0,3.49650756146648,0.176471143115779,0.0705884572463116 +46075,2005,0,3.25809653802148,0.176471143115779,0.105882685869467 +46075,2006,0,2.94443897916644,0.176471143115779,0.141176914492623 +46075,2007,0,3.17805383034795,0.176471143115779,0.176471143115779 +46079,2003,0,5.45958551414416,2.42267657325187,0.484535314650373 +46079,2004,0,5.31320597904179,2.42267657325187,0.969070629300746 +46079,2005,0,5.4249500174814,2.42267657325187,1.45360594395112 +46079,2006,0,5.30330490805908,2.42267657325187,1.93814125860149 +46079,2007,0,5.32300997913841,2.42267657325187,2.42267657325187 +46095,2003,0,2.39789527279837,0.733809162278835,0.146761832455767 +46095,2004,0,2.70805020110221,0.733809162278835,0.293523664911534 +46095,2005,0,2.83321334405622,0.733809162278835,0.440285497367301 +46095,2006,0,2.94443897916644,0.733809162278835,0.587047329823068 +46095,2007,0,3.36729582998647,0.733809162278835,0.733809162278835 +46105,2003,0,4.04305126783455,1.21283343275813,0.242566686551626 +46105,2004,0,4.02535169073515,1.21283343275813,0.485133373103253 +46105,2005,0,4.20469261939097,1.21283343275813,0.727700059654879 +46105,2006,0,4.17438726989564,1.21283343275813,0.970266746206506 +46105,2007,0,4.07753744390572,1.21283343275813,1.21283343275813 +46109,2003,0,5.14749447681345,2.30418381435774,0.460836762871549 +46109,2004,0,5.15905529921453,2.30418381435774,0.921673525743097 +46109,2005,0,5.24702407216049,2.30418381435774,1.38251028861465 +46109,2006,0,5.21493575760899,2.30418381435774,1.84334705148619 +46109,2007,0,5.12989871492307,2.30418381435774,2.30418381435774 +46115,2003,0,4.81218435537242,2.00875080105707,0.401750160211413 +46115,2004,0,4.91265488573605,2.00875080105707,0.803500320422827 +46115,2005,0,5.02388052084628,2.00875080105707,1.20525048063424 +46115,2006,0,5.08759633523238,2.00875080105707,1.60700064084565 +46115,2007,0,5.05624580534831,2.00875080105707,2.00875080105707 +46119,2003,0,2.83321334405622,0.4421184257562,0.08842368515124 +46119,2004,0,2.99573227355399,0.4421184257562,0.17684737030248 +46119,2005,0,2.70805020110221,0.4421184257562,0.26527105545372 +46119,2006,0,2.99573227355399,0.4421184257562,0.35369474060496 +46119,2007,0,2.89037175789616,0.4421184257562,0.4421184257562 +46121,2003,0,3.49650756146648,2.20276475771183,0.440552951542367 +46121,2004,0,3.76120011569356,2.20276475771183,0.881105903084734 +46121,2005,0,3.46573590279973,2.20276475771183,1.3216588546271 +46121,2006,0,3.58351893845611,2.20276475771183,1.76221180616947 +46121,2007,0,3.17805383034795,2.20276475771183,2.20276475771183 +47005,2003,0,5.06890420222023,2.80560029466806,0.561120058933612 +47005,2004,0,4.95582705760126,2.80560029466806,1.12224011786722 +47005,2005,0,4.99043258677874,2.80560029466806,1.68336017680084 +47005,2006,0,5.00394630594546,2.80560029466806,2.24448023573445 +47005,2007,0,4.9416424226093,2.80560029466806,2.80560029466806 +47007,2003,0,3.97029191355212,2.51503163475992,0.503006326951984 +47007,2004,0,3.85014760171006,2.51503163475992,1.00601265390397 +47007,2005,0,4.07753744390572,2.51503163475992,1.50901898085595 +47007,2006,0,3.93182563272433,2.51503163475992,2.01202530780794 +47007,2007,0,4.15888308335967,2.51503163475992,2.51503163475992 +47009,2003,0,6.94312242281943,4.66176788710248,0.932353577420496 +47009,2004,0,7.00215595440362,4.66176788710248,1.86470715484099 +47009,2005,0,7.06817200038804,4.66176788710248,2.79706073226149 +47009,2006,0,7.12044437239249,4.66176788710248,3.72941430968199 +47009,2007,0,7.09672137849476,4.66176788710248,4.66176788710248 +47013,2003,0,5.53733426701854,3.68522277661039,0.737044555322078 +47013,2004,0,5.54126354515843,3.68522277661039,1.47408911064416 +47013,2005,0,5.68357976733868,3.68522277661039,2.21113366596624 +47013,2006,0,5.61312810638807,3.68522277661039,2.94817822128831 +47013,2007,0,5.65599181081985,3.68522277661039,3.68522277661039 +47029,2003,0,5.36129216570943,3.51348385739071,0.702696771478143 +47029,2004,0,5.23644196282995,3.51348385739071,1.40539354295629 +47029,2005,0,5.34710753071747,3.51348385739071,2.10809031443443 +47029,2006,0,5.32300997913841,3.51348385739071,2.81078708591257 +47029,2007,0,5.4380793089232,3.51348385739071,3.51348385739071 +47047,2003,0,5.06890420222023,3.36058369877686,0.672116739755372 +47047,2004,0,5.07517381523383,3.36058369877686,1.34423347951074 +47047,2005,0,5.14749447681345,3.36058369877686,2.01635021926611 +47047,2006,0,5.40717177146012,3.36058369877686,2.68846695902149 +47047,2007,0,5.48063892334199,3.36058369877686,3.36058369877686 +47065,2003,0,8.57319538153152,5.72976206361515,1.14595241272303 +47065,2004,0,8.60300384782935,5.72976206361515,2.29190482544606 +47065,2005,0,8.63194942871443,5.72976206361515,3.43785723816909 +47065,2006,0,8.5577591531629,5.72976206361515,4.58380965089212 +47065,2007,0,8.56369502506766,5.72976206361515,5.72976206361515 +47073,2003,0,6.12686918411419,3.98085853120358,0.796171706240715 +47073,2004,0,6.00881318544259,3.98085853120358,1.59234341248143 +47073,2005,0,5.93489419561959,3.98085853120358,2.38851511872215 +47073,2006,0,6.04263283368238,3.98085853120358,3.18468682496286 +47073,2007,0,5.93753620508243,3.98085853120358,3.98085853120358 +47081,2003,0,4.98360662170834,3.10436243807944,0.620872487615888 +47081,2004,0,4.85203026391962,3.10436243807944,1.24174497523178 +47081,2005,0,4.8283137373023,3.10436243807944,1.86261746284766 +47081,2006,0,4.89034912822175,3.10436243807944,2.48348995046355 +47081,2007,0,5.01063529409626,3.10436243807944,3.10436243807944 +47097,2003,0,4.7361984483945,3.29957062757386,0.659914125514771 +47097,2004,0,4.72738781871234,3.29957062757386,1.31982825102954 +47097,2005,0,4.52178857704904,3.29957062757386,1.97974237654431 +47097,2006,0,4.78749174278205,3.29957062757386,2.63965650205909 +47097,2007,0,4.88280192258637,3.29957062757386,3.29957062757386 +47099,2003,0,5.89715386763674,3.68702774075046,0.737405548150092 +47099,2004,0,5.91889385427315,3.68702774075046,1.47481109630018 +47099,2005,0,5.96614673912369,3.68702774075046,2.21221664445028 +47099,2006,0,5.96614673912369,3.68702774075046,2.94962219260037 +47099,2007,0,6.05208916892442,3.68702774075046,3.68702774075046 +47125,2003,0,7.31654817718298,4.90355478157301,0.980710956314602 +47125,2004,0,7.33498187887181,4.90355478157301,1.9614219126292 +47125,2005,0,7.38894609761844,4.90355478157301,2.9421328689438 +47125,2006,0,7.42595365707754,4.90355478157301,3.92284382525841 +47125,2007,0,7.51479976048867,4.90355478157301,4.90355478157301 +47145,2003,0,6.02344759296103,3.94951144983914,0.789902289967828 +47145,2004,0,6.10255859461357,3.94951144983914,1.57980457993566 +47145,2005,0,6.19236248947487,3.94951144983914,2.36970686990348 +47145,2006,0,6.07993319509559,3.94951144983914,3.15960915987131 +47145,2007,0,6.04973345523196,3.94951144983914,3.94951144983914 +47157,2003,0,9.48607637340956,6.79958192210419,1.35991638442084 +47157,2004,0,9.44770236052795,6.79958192210419,2.71983276884167 +47157,2005,0,9.40640045488065,6.79958192210419,4.07974915326251 +47157,2006,0,9.47031717333574,6.79958192210419,5.43966553768335 +47157,2007,0,9.52493210973482,6.79958192210419,6.79958192210419 +47171,2003,0,4.99721227376411,2.87169849263054,0.574339698526109 +47171,2004,0,4.69134788222914,2.87169849263054,1.14867939705222 +47171,2005,0,4.85981240436167,2.87169849263054,1.72301909557833 +47171,2006,0,4.89034912822175,2.87169849263054,2.29735879410443 +47171,2007,0,4.85981240436167,2.87169849263054,2.87169849263054 +47173,2003,0,4.51085950651685,2.87964779453319,0.575929558906638 +47173,2004,0,4.34380542185368,2.87964779453319,1.15185911781328 +47173,2005,0,4.33073334028633,2.87964779453319,1.72778867671991 +47173,2006,0,4.63472898822964,2.87964779453319,2.30371823562655 +47173,2007,0,4.45434729625351,2.87964779453319,2.87964779453319 +48015,2003,0,5.73979291217923,3.16082289341958,0.632164578683917 +48015,2004,0,5.8664680569333,3.16082289341958,1.26432915736783 +48015,2005,0,5.8664680569333,3.16082289341958,1.89649373605175 +48015,2006,0,5.89989735358249,3.16082289341958,2.52865831473567 +48015,2007,0,5.99893656194668,3.16082289341958,3.16082289341958 +48019,2003,0,4.75359019110636,2.87045245712723,0.574090491425447 +48019,2004,0,4.67282883446191,2.87045245712723,1.14818098285089 +48019,2005,0,4.83628190695148,2.87045245712723,1.72227147427634 +48019,2006,0,4.79579054559674,2.87045245712723,2.29636196570179 +48019,2007,0,4.79579054559674,2.87045245712723,2.87045245712723 +48025,2003,0,5.22035582507832,3.47689218936423,0.695378437872845 +48025,2004,0,5.24174701505964,3.47689218936423,1.39075687574569 +48025,2005,0,5.19295685089021,3.47689218936423,2.08613531361854 +48025,2006,0,5.36597601502185,3.47689218936423,2.78151375149138 +48025,2007,0,5.33271879326537,3.47689218936423,3.47689218936423 +48031,2003,0,4.57471097850338,2.13037227034838,0.426074454069676 +48031,2004,0,4.60517018598809,2.13037227034838,0.852148908139352 +48031,2005,0,4.4188406077966,2.13037227034838,1.27822336220903 +48031,2006,0,4.45434729625351,2.13037227034838,1.7042978162787 +48031,2007,0,4.59511985013459,2.13037227034838,2.13037227034838 +48043,2003,0,4.83628190695148,2.18222373632286,0.436444747264572 +48043,2004,0,4.89034912822175,2.18222373632286,0.872889494529145 +48043,2005,0,5.16478597392351,2.18222373632286,1.30933424179372 +48043,2006,0,5.32787616878958,2.18222373632286,1.74577898905829 +48043,2007,0,5.4971682252932,2.18222373632286,2.18222373632286 +48055,2003,0,5.35185813347607,3.47178009978404,0.694356019956808 +48055,2004,0,5.4971682252932,3.47178009978404,1.38871203991362 +48055,2005,0,5.37989735354046,3.47178009978404,2.08306805987042 +48055,2006,0,5.44673737166631,3.47178009978404,2.77742407982723 +48055,2007,0,5.43372200355424,3.47178009978404,3.47178009978404 +48061,2003,0,7.84149292446001,5.81480791429005,1.16296158285801 +48061,2004,0,7.8164169836918,5.81480791429005,2.32592316571602 +48061,2005,0,7.74240202181578,5.81480791429005,3.48888474857403 +48061,2006,0,7.81843027207066,5.81480791429005,4.65184633143204 +48061,2007,0,7.92768504561578,5.81480791429005,5.81480791429005 +48073,2003,0,6.35262939631957,3.84286583473053,0.768573166946107 +48073,2004,0,6.07764224334903,3.84286583473053,1.53714633389221 +48073,2005,0,6.19440539110467,3.84286583473053,2.30571950083832 +48073,2006,0,6.21060007702465,3.84286583473053,3.07429266778443 +48073,2007,0,6.22653666928747,3.84286583473053,3.84286583473053 +48077,2003,0,4.34380542185368,2.39844057863757,0.479688115727513 +48077,2004,0,4.11087386417331,2.39844057863757,0.959376231455027 +48077,2005,0,4.15888308335967,2.39844057863757,1.43906434718254 +48077,2006,0,3.91202300542815,2.39844057863757,1.91875246291005 +48077,2007,0,3.80666248977032,2.39844057863757,2.39844057863757 +48113,2003,0,10.4426965306939,7.70476640589851,1.5409532811797 +48113,2004,0,10.3391261606518,7.70476640589851,3.0819065623594 +48113,2005,0,10.3371189079706,7.70476640589851,4.62285984353911 +48113,2006,0,10.3784787232929,7.70476640589851,6.16381312471881 +48113,2007,0,10.4352622050525,7.70476640589851,7.70476640589851 +48117,2003,0,5.4249500174814,2.92106260521696,0.584212521043392 +48117,2004,0,5.28320372873799,2.92106260521696,1.16842504208678 +48117,2005,0,5.26785815906333,2.92106260521696,1.75263756313018 +48117,2006,0,5.46383180502561,2.92106260521696,2.33685008417357 +48117,2007,0,5.4971682252932,2.92106260521696,2.92106260521696 +48121,2003,0,8.62353322718756,6.07068229921461,1.21413645984292 +48121,2004,0,8.63034328934889,6.07068229921461,2.42827291968584 +48121,2005,0,8.68253812400308,6.07068229921461,3.64240937952877 +48121,2006,0,8.75731184693641,6.07068229921461,4.85654583937169 +48121,2007,0,8.80956425335415,6.07068229921461,6.07068229921461 +48151,2003,0,2.94443897916644,1.46879558263163,0.293759116526327 +48151,2004,0,2.99573227355399,1.46879558263163,0.587518233052654 +48151,2005,0,2.56494935746154,1.46879558263163,0.881277349578981 +48151,2006,0,2.99573227355399,1.46879558263163,1.17503646610531 +48151,2007,0,2.484906649788,1.46879558263163,1.46879558263163 +48173,2003,0,2.39789527279837,0.340748793388473,0.0681497586776946 +48173,2004,0,1.79175946922805,0.340748793388473,0.136299517355389 +48173,2005,0,1.94591014905531,0.340748793388473,0.204449276033084 +48173,2006,0,2.07944154167984,0.340748793388473,0.272599034710778 +48173,2007,0,1.38629436111989,0.340748793388473,0.340748793388473 +48213,2003,0,6.57088296233958,4.29424678066323,0.858849356132646 +48213,2004,0,6.47389069635227,4.29424678066323,1.71769871226529 +48213,2005,0,6.39692965521615,4.29424678066323,2.57654806839794 +48213,2006,0,6.49526555593701,4.29424678066323,3.43539742453058 +48213,2007,0,6.46458830368996,4.29424678066323,4.29424678066323 +48221,2003,0,6.2363695902037,3.71600812150219,0.743201624300438 +48221,2004,0,6.13772705408623,3.71600812150219,1.48640324860088 +48221,2005,0,6.05678401322862,3.71600812150219,2.22960487290131 +48221,2006,0,6.25382881157547,3.71600812150219,2.97280649720175 +48221,2007,0,6.36302810354046,3.71600812150219,3.71600812150219 +48223,2003,0,5.95583736946483,3.46448512089807,0.692897024179615 +48223,2004,0,5.91889385427315,3.46448512089807,1.38579404835923 +48223,2005,0,5.85793315448346,3.46448512089807,2.07869107253884 +48223,2006,0,5.95064255258773,3.46448512089807,2.77158809671846 +48223,2007,0,5.90536184805457,3.46448512089807,3.46448512089807 +48245,2003,0,8.01763715990848,5.52963144798754,1.10592628959751 +48245,2004,0,7.92334821193015,5.52963144798754,2.21185257919502 +48245,2005,0,8.09162741160107,5.52963144798754,3.31777886879253 +48245,2006,0,8.18618599422608,5.52963144798754,4.42370515839003 +48245,2007,0,8.17301131172497,5.52963144798754,5.52963144798754 +48249,2003,0,5.62040086571715,3.67188587773737,0.734377175547473 +48249,2004,0,5.76205138278018,3.67188587773737,1.46875435109495 +48249,2005,0,5.73979291217923,3.67188587773737,2.20313152664242 +48249,2006,0,5.78689738136671,3.67188587773737,2.93750870218989 +48249,2007,0,5.86929691313377,3.67188587773737,3.67188587773737 +48271,2003,0,2.63905732961526,1.21757980773215,0.243515961546431 +48271,2004,0,1.6094379124341,1.21757980773215,0.487031923092861 +48271,2005,0,1.6094379124341,1.21757980773215,0.730547884639292 +48271,2006,0,2.19722457733622,1.21757980773215,0.974063846185722 +48271,2007,0,2.30258509299405,1.21757980773215,1.21757980773215 +48277,2003,0,6.32256523992728,3.88154317917417,0.776308635834834 +48277,2004,0,6.24027584517077,3.88154317917417,1.55261727166967 +48277,2005,0,6.15060276844628,3.88154317917417,2.3289259075045 +48277,2006,0,6.37672694789863,3.88154317917417,3.10523454333934 +48277,2007,0,6.45676965557216,3.88154317917417,3.88154317917417 +48291,2003,0,6.26530121273771,4.25069282559285,0.850138565118569 +48291,2004,0,6.24610676548156,4.25069282559285,1.70027713023714 +48291,2005,0,6.19644412779452,4.25069282559285,2.55041569535571 +48291,2006,0,6.27664348934164,4.25069282559285,3.40055426047428 +48291,2007,0,6.289715570909,4.25069282559285,4.25069282559285 +48295,2003,0,3.58351893845611,1.1174340429087,0.223486808581739 +48295,2004,0,3.91202300542815,1.1174340429087,0.446973617163479 +48295,2005,0,4.04305126783455,1.1174340429087,0.670460425745219 +48295,2006,0,3.98898404656427,1.1174340429087,0.893947234326958 +48295,2007,0,4.20469261939097,1.1174340429087,1.1174340429087 +48305,2003,0,3.78418963391826,1.87946504964716,0.375893009929432 +48305,2004,0,3.71357206670431,1.87946504964716,0.751786019858864 +48305,2005,0,3.80666248977032,1.87946504964716,1.1276790297883 +48305,2006,0,3.87120101090789,1.87946504964716,1.50357203971773 +48305,2007,0,2.99573227355399,1.87946504964716,1.87946504964716 +48317,2003,0,2.56494935746154,1.55730215801357,0.311460431602714 +48317,2004,0,1.09861228866811,1.55730215801357,0.622920863205429 +48317,2005,0,2.07944154167984,1.55730215801357,0.934381294808143 +48317,2006,0,1.38629436111989,1.55730215801357,1.24584172641086 +48317,2007,0,1.94591014905531,1.55730215801357,1.55730215801357 +48323,2003,0,5.78996017089725,3.85644686853929,0.771289373707859 +48323,2004,0,5.4553211153577,3.85644686853929,1.54257874741572 +48323,2005,0,5.50125821054473,3.85644686853929,2.31386812112358 +48323,2006,0,5.71702770140622,3.85644686853929,3.08515749483144 +48323,2007,0,5.91620206260743,3.85644686853929,3.85644686853929 +48341,2003,0,5.74939298590825,3.00176404578571,0.600352809157142 +48341,2004,0,5.59471137960184,3.00176404578571,1.20070561831428 +48341,2005,0,5.58724865840025,3.00176404578571,1.80105842747143 +48341,2006,0,5.6937321388027,3.00176404578571,2.40141123662857 +48341,2007,0,5.6970934865054,3.00176404578571,3.00176404578571 +48351,2003,0,3.3322045101752,2.71283871783401,0.542567743566801 +48351,2004,0,3.46573590279973,2.71283871783401,1.0851354871336 +48351,2005,0,3.80666248977032,2.71283871783401,1.6277032307004 +48351,2006,0,3.61091791264422,2.71283871783401,2.17027097426721 +48351,2007,0,3.8286413964891,2.71283871783401,2.71283871783401 +48357,2003,0,5.30330490805908,2.19789102187938,0.439578204375876 +48357,2004,0,5.23644196282995,2.19789102187938,0.879156408751752 +48357,2005,0,5.18178355029209,2.19789102187938,1.31873461312763 +48357,2006,0,5.25227342804663,2.19789102187938,1.7583128175035 +48357,2007,0,5.25749537202778,2.19789102187938,2.19789102187938 +48363,2003,0,5.61312810638807,3.29679936561589,0.659359873123179 +48363,2004,0,5.62762111369064,3.29679936561589,1.31871974624636 +48363,2005,0,5.67332326717149,3.29679936561589,1.97807961936954 +48363,2006,0,5.78689738136671,3.29679936561589,2.63743949249272 +48363,2007,0,5.78996017089725,3.29679936561589,3.29679936561589 +48369,2003,0,4.76217393479776,2.30418381435774,0.460836762871549 +48369,2004,0,4.88280192258637,2.30418381435774,0.921673525743097 +48369,2005,0,4.87519732320115,2.30418381435774,1.38251028861465 +48369,2006,0,4.77912349311153,2.30418381435774,1.84334705148619 +48369,2007,0,4.79579054559674,2.30418381435774,2.30418381435774 +48383,2003,0,3.43398720448515,1.20177038077074,0.240354076154147 +48383,2004,0,3.61091791264422,1.20177038077074,0.480708152308294 +48383,2005,0,3.29583686600433,1.20177038077074,0.721062228462442 +48383,2006,0,3.61091791264422,1.20177038077074,0.961416304616589 +48383,2007,0,3.76120011569356,1.20177038077074,1.20177038077074 +48385,2003,0,3.09104245335832,1.11415750000357,0.222831500000714 +48385,2004,0,2.70805020110221,1.11415750000357,0.445663000001429 +48385,2005,0,2.39789527279837,1.11415750000357,0.668494500002143 +48385,2006,0,2.70805020110221,1.11415750000357,0.891326000002858 +48385,2007,0,2.94443897916644,1.11415750000357,1.11415750000357 +48397,2003,0,6.55535689181067,3.76305885228819,0.752611770457637 +48397,2004,0,6.6052979209482,3.76305885228819,1.50522354091527 +48397,2005,0,6.68835471394676,3.76305885228819,2.25783531137291 +48397,2006,0,6.78332520060396,3.76305885228819,3.01044708183055 +48397,2007,0,6.93828448401696,3.76305885228819,3.76305885228819 +48399,2003,0,4.86753445045558,2.44191215821514,0.488382431643029 +48399,2004,0,4.75359019110636,2.44191215821514,0.976764863286058 +48399,2005,0,4.65396035015752,2.44191215821514,1.46514729492909 +48399,2006,0,4.62497281328427,2.44191215821514,1.95352972657212 +48399,2007,0,4.62497281328427,2.44191215821514,2.44191215821514 +48409,2003,0,6.12249280951439,4.20675020260656,0.841350040521313 +48409,2004,0,6.13988455222626,4.20675020260656,1.68270008104263 +48409,2005,0,6.19644412779452,4.20675020260656,2.52405012156394 +48409,2006,0,6.30627528694802,4.20675020260656,3.36540016208525 +48409,2007,0,6.38856140554563,4.20675020260656,4.20675020260656 +48415,2003,0,5.19849703126583,2.79490045400094,0.558980090800188 +48415,2004,0,5.03043792139244,2.79490045400094,1.11796018160038 +48415,2005,0,4.969813299576,2.79490045400094,1.67694027240056 +48415,2006,0,5.07517381523383,2.79490045400094,2.23592036320075 +48415,2007,0,5.19849703126583,2.79490045400094,2.79490045400094 +48417,2003,0,3.58351893845611,1.19452834549794,0.238905669099587 +48417,2004,0,3.55534806148941,1.19452834549794,0.477811338199175 +48417,2005,0,3.66356164612965,1.19452834549794,0.716717007298762 +48417,2006,0,4.15888308335967,1.19452834549794,0.955622676398349 +48417,2007,0,3.93182563272433,1.19452834549794,1.19452834549794 +48447,2003,0,2.56494935746154,0.615185639090233,0.123037127818047 +48447,2004,0,2.19722457733622,0.615185639090233,0.246074255636093 +48447,2005,0,2.484906649788,0.615185639090233,0.36911138345414 +48447,2006,0,2.484906649788,0.615185639090233,0.492148511272187 +48447,2007,0,2.484906649788,0.615185639090233,0.615185639090233 +48461,2003,0,2.77258872223978,1.22495121071113,0.244990242142226 +48461,2004,0,2.30258509299405,1.22495121071113,0.489980484284451 +48461,2005,0,2.77258872223978,1.22495121071113,0.734970726426677 +48461,2006,0,2.83321334405622,1.22495121071113,0.979960968568902 +48461,2007,0,2.56494935746154,1.22495121071113,1.22495121071113 +48465,2003,0,6.01126717440416,3.80345735882137,0.760691471764274 +48465,2004,0,5.82304589548302,3.80345735882137,1.52138294352855 +48465,2005,0,5.78382518232974,3.80345735882137,2.28207441529282 +48465,2006,0,5.80513496891649,3.80345735882137,3.0427658870571 +48465,2007,0,5.92692602597041,3.80345735882137,3.80345735882137 +48477,2003,0,6.32435896238131,3.41355405587153,0.682710811174305 +48477,2004,0,6.32435896238131,3.41355405587153,1.36542162234861 +48477,2005,0,6.23244801655052,3.41355405587153,2.04813243352292 +48477,2006,0,6.22059017009974,3.41355405587153,2.73084324469722 +48477,2007,0,6.39692965521615,3.41355405587153,3.41355405587153 +48493,2003,0,5.55295958492162,3.47840530590039,0.695681061180078 +48493,2004,0,5.50938833662798,3.47840530590039,1.39136212236016 +48493,2005,0,5.46806014113513,3.47840530590039,2.08704318354023 +48493,2006,0,5.73334127689775,3.47840530590039,2.78272424472031 +48493,2007,0,5.69035945432406,3.47840530590039,3.47840530590039 +48503,2003,0,5.48063892334199,2.88720006673053,0.577440013346106 +48503,2004,0,5.4971682252932,2.88720006673053,1.15488002669221 +48503,2005,0,5.51342874616498,2.88720006673053,1.73232004003832 +48503,2006,0,5.54517744447956,2.88720006673053,2.30976005338442 +48503,2007,0,5.51342874616498,2.88720006673053,2.88720006673053 +48505,2003,0,3.87120101090789,2.49995945241429,0.499991890482859 +48505,2004,0,4.12713438504509,2.49995945241429,0.999983780965717 +48505,2005,0,4.26267987704132,2.49995945241429,1.49997567144858 +48505,2006,0,4.39444915467244,2.49995945241429,1.99996756193143 +48505,2007,0,4.33073334028633,2.49995945241429,2.49995945241429 +48507,2003,0,3.40119738166216,2.45100509811232,0.490201019622464 +48507,2004,0,3.43398720448515,2.45100509811232,0.980402039244928 +48507,2005,0,3.3322045101752,2.45100509811232,1.47060305886739 +48507,2006,0,3.40119738166216,2.45100509811232,1.96080407848986 +48507,2007,0,3.55534806148941,2.45100509811232,2.45100509811232 +49003,2003,0,6.84481547920826,3.7552522294782,0.751050445895639 +49003,2004,0,6.79234442747081,3.7552522294782,1.50210089179128 +49003,2005,0,6.79682371827486,3.7552522294782,2.25315133768692 +49003,2006,0,6.82654522355659,3.7552522294782,3.00420178358256 +49003,2007,0,6.93049476595163,3.7552522294782,3.7552522294782 +49011,2003,0,8.52257866369258,5.47643844701387,1.09528768940277 +49011,2004,0,8.49412925181769,5.47643844701387,2.19057537880555 +49011,2005,0,8.51959031601596,5.47643844701387,3.28586306820832 +49011,2006,0,8.59192953753026,5.47643844701387,4.3811507576111 +49011,2007,0,8.6909784171879,5.47643844701387,5.47643844701387 +49023,2003,0,5.29330482472449,2.10875759601389,0.421751519202778 +49023,2004,0,5.22035582507832,2.10875759601389,0.843503038405556 +49023,2005,0,5.27811465923052,2.10875759601389,1.26525455760833 +49023,2006,0,5.46383180502561,2.10875759601389,1.68700607681111 +49023,2007,0,5.48893772615669,2.10875759601389,2.10875759601389 +49037,2003,0,5.01727983681492,2.66813057710097,0.533626115420193 +49037,2004,0,5.12989871492307,2.66813057710097,1.06725223084039 +49037,2005,0,5.07517381523383,2.66813057710097,1.60087834626058 +49037,2006,0,5.23644196282995,2.66813057710097,2.13450446168077 +49037,2007,0,5.01063529409626,2.66813057710097,2.66813057710097 +49053,2003,0,7.84109976542212,4.50373528833274,0.900747057666547 +49053,2004,0,7.82763954636642,4.50373528833274,1.80149411533309 +49053,2005,0,7.92840602618053,4.50373528833274,2.70224117299964 +49053,2006,0,8.06180227453835,4.50373528833274,3.60298823066619 +49053,2007,0,8.08978917578932,4.50373528833274,4.50373528833274 +51003,2003,0,7.08086789669078,4.37243074100043,0.874486148200087 +51003,2004,0,7.12044437239249,4.37243074100043,1.74897229640017 +51003,2005,0,7.06987412845857,4.37243074100043,2.62345844460026 +51003,2006,0,7.18916773842032,4.37243074100043,3.49794459280035 +51003,2007,0,7.27931883541462,4.37243074100043,4.37243074100043 +51015,2003,0,6.46925031679577,4.18380432834825,0.83676086566965 +51015,2004,0,6.39359075395063,4.18380432834825,1.6735217313393 +51015,2005,0,6.46458830368996,4.18380432834825,2.51028259700895 +51015,2006,0,6.49677499018586,4.18380432834825,3.3470434626786 +51015,2007,0,6.48463523563525,4.18380432834825,4.18380432834825 +51017,2003,0,2.89037175789616,1.61899212523891,0.323798425047782 +51017,2004,0,2.39789527279837,1.61899212523891,0.647596850095565 +51017,2005,0,2.77258872223978,1.61899212523891,0.971395275143347 +51017,2006,0,2.56494935746154,1.61899212523891,1.29519370019113 +51017,2007,0,2.30258509299405,1.61899212523891,1.61899212523891 +51019,2003,0,5.80513496891649,4.10050885718995,0.820101771437989 +51019,2004,0,5.82304589548302,4.10050885718995,1.64020354287598 +51019,2005,0,5.89715386763674,4.10050885718995,2.46030531431397 +51019,2006,0,5.94542060860658,4.10050885718995,3.28040708575196 +51019,2007,0,5.88332238848828,4.10050885718995,4.10050885718995 +51023,2003,0,5.56452040732269,3.41759552747179,0.683519105494358 +51023,2004,0,5.50938833662798,3.41759552747179,1.36703821098872 +51023,2005,0,5.65599181081985,3.41759552747179,2.05055731648307 +51023,2006,0,5.89440283426485,3.41759552747179,2.73407642197743 +51023,2007,0,5.98393628068719,3.41759552747179,3.41759552747179 +51025,2003,0,4.64439089914137,2.91338274053697,0.582676548107393 +51025,2004,0,4.54329478227,2.91338274053697,1.16535309621479 +51025,2005,0,4.56434819146784,2.91338274053697,1.74802964432218 +51025,2006,0,4.54329478227,2.91338274053697,2.33070619242957 +51025,2007,0,4.61512051684126,2.91338274053697,2.91338274053697 +51041,2003,0,8.49801037199946,5.56030848448194,1.11206169689639 +51041,2004,0,8.50875771259514,5.56030848448194,2.22412339379278 +51041,2005,0,8.53129331579502,5.56030848448194,3.33618509068916 +51041,2006,0,8.57395152523485,5.56030848448194,4.44824678758555 +51041,2007,0,8.56254889313703,5.56030848448194,5.56030848448194 +51047,2003,0,6.04500531403601,3.53403686830827,0.706807373661653 +51047,2004,0,6.13772705408623,3.53403686830827,1.41361474732331 +51047,2005,0,6.1779441140506,3.53403686830827,2.12042212098496 +51047,2006,0,6.24027584517077,3.53403686830827,2.82722949464661 +51047,2007,0,6.28226674689601,3.53403686830827,3.53403686830827 +51053,2003,0,5.45958551414416,3.2000191500179,0.640003830003579 +51053,2004,0,5.4249500174814,3.2000191500179,1.28000766000716 +51053,2005,0,5.26269018890489,3.2000191500179,1.92001149001074 +51053,2006,0,5.39816270151775,3.2000191500179,2.56001532001432 +51053,2007,0,5.46806014113513,3.2000191500179,3.2000191500179 +51059,2003,0,9.39499254410842,6.87703727512594,1.37540745502519 +51059,2004,0,9.38681151509196,6.87703727512594,2.75081491005038 +51059,2005,0,9.45297277587424,6.87703727512594,4.12622236507556 +51059,2006,0,9.4922055591064,6.87703727512594,5.50162982010075 +51059,2007,0,9.4753933263676,6.87703727512594,6.87703727512594 +51061,2003,0,6.71174039505618,4.0098572697765,0.8019714539553 +51061,2004,0,6.77078942390898,4.0098572697765,1.6039429079106 +51061,2005,0,6.74758652682932,4.0098572697765,2.4059143618659 +51061,2006,0,6.76041469108343,4.0098572697765,3.2078858158212 +51061,2007,0,6.86693328446188,4.0098572697765,4.0098572697765 +51077,2003,0,4.06044301054642,2.88574998281776,0.577149996563552 +51077,2004,0,4.26267987704132,2.88574998281776,1.1542999931271 +51077,2005,0,4.07753744390572,2.88574998281776,1.73144998969066 +51077,2006,0,4.26267987704132,2.88574998281776,2.30859998625421 +51077,2007,0,4.14313472639153,2.88574998281776,2.88574998281776 +51079,2003,0,5.30826769740121,2.72418598301161,0.544837196602323 +51079,2004,0,4.90527477843843,2.72418598301161,1.08967439320465 +51079,2005,0,4.8283137373023,2.72418598301161,1.63451158980697 +51079,2006,0,4.78749174278205,2.72418598301161,2.17934878640929 +51079,2007,0,4.70953020131233,2.72418598301161,2.72418598301161 +51089,2003,0,6.15909538849193,4.05923538510853,0.811847077021707 +51089,2004,0,5.96357934361845,4.05923538510853,1.62369415404341 +51089,2005,0,6.06610809010375,4.05923538510853,2.43554123106512 +51089,2006,0,5.98141421125448,4.05923538510853,3.24738830808683 +51089,2007,0,6.0591231955818,4.05923538510853,4.05923538510853 +51109,2003,0,5.45958551414416,3.24364648319341,0.648729296638683 +51109,2004,0,5.40267738187228,3.24364648319341,1.29745859327737 +51109,2005,0,5.55682806169954,3.24364648319341,1.94618788991605 +51109,2006,0,5.52942908751142,3.24364648319341,2.59491718655473 +51109,2007,0,5.71042701737487,3.24364648319341,3.24364648319341 +51111,2003,0,4.39444915467244,2.57611752984138,0.515223505968277 +51111,2004,0,4.35670882668959,2.57611752984138,1.03044701193655 +51111,2005,0,4.38202663467388,2.57611752984138,1.54567051790483 +51111,2006,0,4.38202663467388,2.57611752984138,2.06089402387311 +51111,2007,0,4.20469261939097,2.57611752984138,2.57611752984138 +51121,2003,0,6.96885037834195,4.4263903498904,0.88527806997808 +51121,2004,0,6.89972310728487,4.4263903498904,1.77055613995616 +51121,2005,0,6.95749737087695,4.4263903498904,2.65583420993424 +51121,2006,0,7.0057890192535,4.4263903498904,3.54111227991232 +51121,2007,0,7.05703698169789,4.4263903498904,4.4263903498904 +51149,2003,0,5.97888576490112,3.49793079061946,0.699586158123893 +51149,2004,0,5.8664680569333,3.49793079061946,1.39917231624779 +51149,2005,0,5.78074351579233,3.49793079061946,2.09875847437168 +51149,2006,0,5.71373280550937,3.49793079061946,2.79834463249557 +51149,2007,0,5.73009978297357,3.49793079061946,3.49793079061946 +51185,2003,0,6.21460809842219,3.79768901497121,0.759537802994242 +51185,2004,0,6.02586597382531,3.79768901497121,1.51907560598848 +51185,2005,0,6.19440539110467,3.79768901497121,2.27861340898273 +51185,2006,0,6.26530121273771,3.79768901497121,3.03815121197697 +51185,2007,0,6.14846829591765,3.79768901497121,3.79768901497121 +51515,2003,0,5.07517381523383,1.84039089063979,0.368078178127958 +51515,2004,0,5.11799381241676,1.84039089063979,0.736156356255917 +51515,2005,0,5.03695260241363,1.84039089063979,1.10423453438388 +51515,2006,0,5.20948615284142,1.84039089063979,1.47231271251183 +51515,2007,0,5.21493575760899,1.84039089063979,1.84039089063979 +51520,2003,0,6.10255859461357,2.85457185376498,0.570914370752995 +51520,2004,0,6.16331480403464,2.85457185376498,1.14182874150599 +51520,2005,0,6.20253551718792,2.85457185376498,1.71274311225899 +51520,2006,0,6.22455842927536,2.85457185376498,2.28365748301198 +51520,2007,0,6.289715570909,2.85457185376498,2.85457185376498 +51630,2003,0,6.81892406527552,2.95901642061733,0.591803284123466 +51630,2004,0,6.92067150424868,2.95901642061733,1.18360656824693 +51630,2005,0,7.02197642307216,2.95901642061733,1.7754098523704 +51630,2006,0,6.99301512293296,2.95901642061733,2.36721313649387 +51630,2007,0,7.00306545878646,2.95901642061733,2.95901642061733 +51650,2003,0,7.62657020629066,4.98659530183624,0.997319060367247 +51650,2004,0,7.6275443904885,4.98659530183624,1.99463812073449 +51650,2005,0,7.70616297019958,4.98659530183624,2.99195718110174 +51650,2006,0,7.65207074611648,4.98659530183624,3.98927624146899 +51650,2007,0,7.57814547241947,4.98659530183624,4.98659530183624 +51660,2003,0,6.86380339145295,3.70051153834364,0.740102307668729 +51660,2004,0,6.85118492749374,3.70051153834364,1.48020461533746 +51660,2005,0,6.90975328164481,3.70051153834364,2.22030692300619 +51660,2006,0,6.9555926083963,3.70051153834364,2.96040923067492 +51660,2007,0,7.01571242048723,3.70051153834364,3.70051153834364 +51680,2003,0,7.21744343169653,4.17851719155535,0.835703438311069 +51680,2004,0,7.1785454837637,4.17851719155535,1.67140687662214 +51680,2005,0,7.2211050981825,4.17851719155535,2.50711031493321 +51680,2006,0,7.27031288607902,4.17851719155535,3.34281375324428 +51680,2007,0,7.37713371283395,4.17851719155535,4.17851719155535 +51700,2003,0,7.6685611080159,5.1937898371941,1.03875796743882 +51700,2004,0,7.73236922228439,5.1937898371941,2.07751593487764 +51700,2005,0,7.67229245562876,5.1937898371941,3.11627390231646 +51700,2006,0,7.73061406606374,5.1937898371941,4.15503186975528 +51700,2007,0,7.78113850984502,5.1937898371941,5.1937898371941 +51710,2003,0,7.98309894071089,5.45704185625577,1.09140837125115 +51710,2004,0,7.95962530509811,5.45704185625577,2.18281674250231 +51710,2005,0,7.95331834656043,5.45704185625577,3.27422511375346 +51710,2006,0,8.00869818298853,5.45704185625577,4.36563348500461 +51710,2007,0,8.0149968943483,5.45704185625577,5.45704185625577 +51735,2003,0,4.85981240436167,2.44806975973603,0.489613951947205 +51735,2004,0,4.91998092582813,2.44806975973603,0.979227903894411 +51735,2005,0,4.96284463025991,2.44806975973603,1.46884185584162 +51735,2006,0,4.99721227376411,2.44806975973603,1.95845580778882 +51735,2007,0,4.98360662170834,2.44806975973603,2.44806975973603 +51770,2003,0,7.6685611080159,4.55293961038444,0.910587922076888 +51770,2004,0,7.59438124255182,4.55293961038444,1.82117584415378 +51770,2005,0,7.60837447438078,4.55293961038444,2.73176376623066 +51770,2006,0,7.66152708135852,4.55293961038444,3.64235168830755 +51770,2007,0,7.73543335249969,4.55293961038444,4.55293961038444 +51790,2003,0,6.21260609575152,3.17190999558746,0.634381999117491 +51790,2004,0,6.19236248947487,3.17190999558746,1.26876399823498 +51790,2005,0,6.1779441140506,3.17190999558746,1.90314599735247 +51790,2006,0,6.19847871649231,3.17190999558746,2.53752799646997 +51790,2007,0,6.31173480915291,3.17190999558746,3.17190999558746 +51800,2003,0,6.60800062529609,4.15382342987363,0.830764685974725 +51800,2004,0,6.70441435496411,4.15382342987363,1.66152937194945 +51800,2005,0,6.71659477352098,4.15382342987363,2.49229405792418 +51800,2006,0,6.77992190747225,4.15382342987363,3.3230587438989 +51800,2007,0,6.83195356556585,4.15382342987363,4.15382342987363 +51830,2003,0,6.45833828334479,2.4847399692309,0.49694799384618 +51830,2004,0,6.4281052726846,2.4847399692309,0.99389598769236 +51830,2005,0,6.35437004079735,2.4847399692309,1.49084398153854 +51830,2006,0,6.42648845745769,2.4847399692309,1.98779197538472 +51830,2007,0,6.35610766069589,2.4847399692309,2.4847399692309 +51840,2003,0,6.96508034560141,3.16061091673622,0.632122183347245 +51840,2004,0,7.02908756414966,3.16061091673622,1.26424436669449 +51840,2005,0,7.00669522683704,3.16061091673622,1.89636655004173 +51840,2006,0,6.98933526597456,3.16061091673622,2.52848873338898 +51840,2007,0,7.02286808608264,3.16061091673622,3.16061091673622 +54005,2003,2007,5.04342511691925,3.24005006009961,0.648010012019921 +54005,2004,2007,4.79579054559674,3.24005006009961,1.29602002403984 +54005,2005,2007,4.72738781871234,3.24005006009961,1.94403003605976 +54005,2006,2007,4.36944785246702,3.24005006009961,2.59204004807969 +54005,2007,2007,4.4188406077966,3.24005006009961,3.24005006009961 +54017,2003,2007,3.36729582998647,2.00188532346096,0.400377064692192 +54017,2004,2007,3.17805383034795,2.00188532346096,0.800754129384385 +54017,2005,2007,2.484906649788,2.00188532346096,1.20113119407658 +54017,2006,2007,2.19722457733622,2.00188532346096,1.60150825876877 +54017,2007,2007,2.07944154167984,2.00188532346096,2.00188532346096 +54021,2003,2007,2.77258872223978,1.96850998097255,0.393701996194511 +54021,2004,2007,2.94443897916644,1.96850998097255,0.787403992389022 +54021,2005,2007,3.52636052461616,1.96850998097255,1.18110598858353 +54021,2006,2007,3.46573590279973,1.96850998097255,1.57480798477804 +54021,2007,2007,3.09104245335832,1.96850998097255,1.96850998097255 +54027,2003,2007,4.89783979995091,3.00583110823141,0.601166221646281 +54027,2004,2007,4.72738781871234,3.00583110823141,1.20233244329256 +54027,2005,2007,4.80402104473326,3.00583110823141,1.80349866493884 +54027,2006,2007,4.96284463025991,3.00583110823141,2.40466488658512 +54027,2007,2007,4.85981240436167,3.00583110823141,3.00583110823141 +54029,2003,2007,5.78382518232974,3.48636539403203,0.697273078806407 +54029,2004,2007,5.73334127689775,3.48636539403203,1.39454615761281 +54029,2005,2007,5.67332326717149,3.48636539403203,2.09181923641922 +54029,2006,2007,5.65948221575962,3.48636539403203,2.78909231522563 +54029,2007,2007,5.60947179518496,3.48636539403203,3.48636539403203 +54037,2003,2007,6.17170059741091,3.74218322613625,0.74843664522725 +54037,2004,2007,6.32435896238131,3.74218322613625,1.4968732904545 +54037,2005,2007,6.36647044773144,3.74218322613625,2.24530993568175 +54037,2006,2007,6.30444880242198,3.74218322613625,2.993746580909 +54037,2007,2007,6.27852142416584,3.74218322613625,3.74218322613625 +54047,2003,2007,4.07753744390572,3.30794840944252,0.661589681888505 +54047,2004,2007,3.73766961828337,3.30794840944252,1.32317936377701 +54047,2005,2007,3.80666248977032,3.30794840944252,1.98476904566552 +54047,2006,2007,4.18965474202643,3.30794840944252,2.64635872755402 +54047,2007,2007,4.06044301054642,3.30794840944252,3.30794840944252 +54051,2003,2007,5.21493575760899,3.57006776457451,0.714013552914902 +54051,2004,2007,5.18738580584075,3.57006776457451,1.4280271058298 +54051,2005,2007,5.37527840768417,3.57006776457451,2.14204065874471 +54051,2006,2007,5.33753807970132,3.57006776457451,2.85605421165961 +54051,2007,2007,5.4249500174814,3.57006776457451,3.57006776457451 +54085,2003,2007,3.71357206670431,2.33630996239567,0.467261992479133 +54085,2004,2007,3.52636052461616,2.33630996239567,0.934523984958267 +54085,2005,2007,3.66356164612965,2.33630996239567,1.4017859774374 +54085,2006,2007,4.20469261939097,2.33630996239567,1.86904796991653 +54085,2007,2007,4.26267987704132,2.33630996239567,2.33630996239567 +54087,2003,2007,3.93182563272433,2.73735007014707,0.547470014029414 +54087,2004,2007,3.8286413964891,2.73735007014707,1.09494002805883 +54087,2005,2007,4.02535169073515,2.73735007014707,1.64241004208824 +54087,2006,2007,4.14313472639153,2.73735007014707,2.18988005611766 +54087,2007,2007,4.38202663467388,2.73735007014707,2.73735007014707 +54099,2003,2007,5.23110861685459,3.75894175355891,0.751788350711782 +54099,2004,2007,5.27811465923052,3.75894175355891,1.50357670142356 +54099,2005,2007,5.15329159449778,3.75894175355891,2.25536505213534 +54099,2006,2007,5.05624580534831,3.75894175355891,3.00715340284713 +54099,2007,2007,5.14166355650266,3.75894175355891,3.75894175355891 +55001,2003,2006,5.08140436498446,2.92547074051332,0.585094148102665 +55001,2004,2006,4.7361984483945,2.92547074051332,1.17018829620533 +55001,2005,2006,4.59511985013459,2.92547074051332,1.75528244430799 +55001,2006,2006,4.8283137373023,2.92547074051332,2.34037659241066 +55001,2007,2006,4.79579054559674,2.92547074051332,2.92547074051332 +55051,2003,2006,4.76217393479776,1.92585320370875,0.385170640741751 +55051,2004,2006,4.70048036579242,1.92585320370875,0.770341281483501 +55051,2005,2006,4.64439089914137,1.92585320370875,1.15551192222525 +55051,2006,2006,4.63472898822964,1.92585320370875,1.540682562967 +55051,2007,2006,4.4188406077966,1.92585320370875,1.92585320370875 +55059,2003,2006,8.00235954625271,5.00781131040515,1.00156226208103 +55059,2004,2006,7.91644286012226,5.00781131040515,2.00312452416206 +55059,2005,2006,7.95050243480885,5.00781131040515,3.00468678624309 +55059,2006,2006,7.9208096792886,5.00781131040515,4.00624904832412 +55059,2007,2006,7.91205688817901,5.00781131040515,5.00781131040515 +55069,2003,2006,6.59578051396131,3.3891585380504,0.67783170761008 +55069,2004,2006,6.55535689181067,3.3891585380504,1.35566341522016 +55069,2005,2006,6.5366915975913,3.3891585380504,2.03349512283024 +55069,2006,2006,6.4150969591716,3.3891585380504,2.71132683044032 +55069,2007,2006,6.45204895443723,3.3891585380504,3.3891585380504 +55081,2003,2006,6.65801104587075,3.7111056128721,0.74222112257442 +55081,2004,2006,6.68710860786651,3.7111056128721,1.48444224514884 +55081,2005,2006,6.62406522779989,3.7111056128721,2.22666336772326 +55081,2006,2006,6.68210859744981,3.7111056128721,2.96888449029768 +55081,2007,2006,6.71901315438526,3.7111056128721,3.7111056128721 +55085,2003,2006,6.72503364216684,3.60484545850393,0.720969091700785 +55085,2004,2006,6.59850902861452,3.60484545850393,1.44193818340157 +55085,2005,2006,6.63463335786169,3.60484545850393,2.16290727510236 +55085,2006,2006,6.58479139238572,3.60484545850393,2.88387636680314 +55085,2007,2006,6.56526497003536,3.60484545850393,3.60484545850393 +55103,2003,2006,5.6937321388027,2.8861405969239,0.577228119384779 +55103,2004,2006,5.54907608489522,2.8861405969239,1.15445623876956 +55103,2005,2006,5.48063892334199,2.8861405969239,1.73168435815434 +55103,2006,2006,5.53338948872752,2.8861405969239,2.30891247753912 +55103,2007,2006,5.55682806169954,2.8861405969239,2.8861405969239 +55109,2003,2006,7.45760928971561,4.14559202222594,0.829118404445188 +55109,2004,2006,7.37713371283395,4.14559202222594,1.65823680889038 +55109,2005,2006,7.35755620091035,4.14559202222594,2.48735521333556 +55109,2006,2006,7.35627987655075,4.14559202222594,3.31647361778075 +55109,2007,2006,7.31121838441963,4.14559202222594,4.14559202222594 +55123,2003,2006,6.04263283368238,3.33420251283788,0.666840502567575 +55123,2004,2006,6.01859321449623,3.33420251283788,1.33368100513515 +55123,2005,2006,5.91350300563827,3.33420251283788,2.00052150770273 +55123,2006,2006,5.89440283426485,3.33420251283788,2.6673620102703 +55123,2007,2006,5.91079664404053,3.33420251283788,3.33420251283788 +55131,2003,2006,8.0870254706677,4.7663787573415,0.9532757514683 +55131,2004,2006,8.0925452638913,4.7663787573415,1.9065515029366 +55131,2005,2006,8.12118324207883,4.7663787573415,2.8598272544049 +55131,2006,2006,8.07340296898641,4.7663787573415,3.8131030058732 +55131,2007,2006,8.11820704940578,4.7663787573415,4.7663787573415 +55137,2003,2006,5.95583736946483,3.14216755178354,0.628433510356707 +55137,2004,2006,5.92425579741453,3.14216755178354,1.25686702071341 +55137,2005,2006,5.9427993751267,3.14216755178354,1.88530053107012 +55137,2006,2006,5.93753620508243,3.14216755178354,2.51373404142683 +55137,2007,2006,5.96357934361845,3.14216755178354,3.14216755178354 diff --git a/benchmarks/data/twfeweights_sim_panel.csv b/benchmarks/data/twfeweights_sim_panel.csv new file mode 100644 index 000000000..48fbc4d96 --- /dev/null +++ b/benchmarks/data/twfeweights_sim_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,-0.44174194168612,0.962638158757321,0.250725238240098 +1,2,0,-0.44174194168612,0.480873514413711,0.302633430899334 +1,3,0,-0.44174194168612,0.114120072263393,-0.099400179401941 +1,4,0,-0.44174194168612,0.448153732387612,1.47267278381102 +1,5,0,-0.44174194168612,0.0450381423543704,-0.304624682447771 +2,1,0,1.00712662877707,0.809674316987755,1.40448682517955 +2,2,0,1.00712662877707,0.875226098004157,1.11498919693233 +2,3,0,1.00712662877707,1.73308918012076,3.30043194865778 +2,4,0,1.00712662877707,1.94159375407109,3.73624251678568 +2,5,0,1.00712662877707,1.49919266041645,5.00490117694144 +3,1,0,0.179611286037529,0.143495801148179,-0.619592378770517 +3,2,0,0.179611286037529,-0.183975411244885,1.59825802459073 +3,3,0,0.179611286037529,1.30125701337682,2.00613564154948 +3,4,0,0.179611286037529,2.14545926587906,3.02931552915487 +3,5,0,0.179611286037529,1.270347859372,0.626648784956356 +4,1,0,1.13465562548833,0.98082184944363,0.789054631222788 +4,2,0,1.13465562548833,0.757860539178009,0.885976737984897 +4,3,0,1.13465562548833,0.644275492814336,2.72497858093159 +4,4,0,1.13465562548833,0.804994143245251,2.7470035085833 +4,5,0,1.13465562548833,3.1402591500037,5.6594454100412 +5,1,0,-2.56473120432832,-1.42045200997001,-2.47773503860188 +5,2,0,-2.56473120432832,-0.99158787828575,-1.91595275185634 +5,3,0,-2.56473120432832,-0.956287354318694,-1.57386277107317 +5,4,0,-2.56473120432832,-0.173688498361684,-2.32481328982363 +5,5,0,-2.56473120432832,-0.220122926796195,-4.77426084425192 +6,1,0,2.03726526019619,0.906852984645204,3.09145390139369 +6,2,0,2.03726526019619,1.10736421765238,1.28301946741103 +6,3,0,2.03726526019619,1.94654212178223,6.4558523593605 +6,4,0,2.03726526019619,2.29731461571635,3.98797124370253 +6,5,0,2.03726526019619,1.90892019581178,8.63257346295759 +7,1,0,0.982893295690819,0.925972608667814,2.56477125464081 +7,2,0,0.982893295690819,1.72746462392985,1.37542121177932 +7,3,0,0.982893295690819,2.01554559420596,3.15083265246945 +7,4,0,0.982893295690819,1.17634719987377,4.35800530188081 +7,5,0,0.982893295690819,0.760671887747645,4.47518570225275 +8,1,0,1.80195765072669,1.92370507752248,2.46720402294067 +8,2,0,1.80195765072669,1.32616500548667,2.7227903293348 +8,3,0,1.80195765072669,1.28608793752108,2.90070675615349 +8,4,0,1.80195765072669,1.81009744339314,5.80136124967764 +8,5,0,1.80195765072669,1.15760990674598,6.55560198907432 +9,1,0,0.225424255156626,0.763669731908457,1.54288624086341 +9,2,0,0.225424255156626,0.874926359148595,0.836845360548911 +9,3,0,0.225424255156626,1.0117962358522,1.82503853096451 +9,4,0,0.225424255156626,0.603697174472571,-0.416276008333528 +9,5,0,0.225424255156626,1.18445658923149,2.87016682298805 +10,1,0,0.0609610990580548,-0.529661334550992,1.66925745533143 +10,2,0,0.0609610990580548,0.0914889871378843,4.08422527417878 +10,3,0,0.0609610990580548,1.48655808874342,2.35118727045354 +10,4,0,0.0609610990580548,0.00203530191935619,2.33692220382611 +10,5,0,0.0609610990580548,1.28112960064862,3.21562126389664 +11,1,0,0.211622353786371,0.677732585269737,1.01402669670136 +11,2,0,0.211622353786371,-0.455890381301695,1.09790190289759 +11,3,0,0.211622353786371,0.221008710144804,0.642633318493822 +11,4,0,0.211622353786371,0.855358439590161,1.02060294398976 +11,5,0,0.211622353786371,1.46959772659548,2.37526538283031 +12,1,0,-0.106312592206072,-9.23288872419215e-05,-0.658080268881745 +12,2,0,-0.106312592206072,-0.0970387920280995,0.736568103159182 +12,3,0,-0.106312592206072,1.28722421860427,1.36381391065831 +12,4,0,-0.106312592206072,0.493486811724755,0.441330823328568 +12,5,0,-0.106312592206072,0.954292388879426,0.728253826747307 +13,1,0,-0.775220530410582,-0.0485173410173907,0.428116160890274 +13,2,0,-0.775220530410582,0.10663036250203,-0.120754966975773 +13,3,0,-0.775220530410582,-0.0932015246981229,0.830881466674987 +13,4,0,-0.775220530410582,0.645640369635815,2.78067278573257 +13,5,0,-0.775220530410582,0.950987026453864,1.73479528567685 +14,1,0,2.04614622265237,0.509538313746868,-0.195487168960969 +14,2,0,2.04614622265237,1.26460645944908,2.49104556632605 +14,3,0,2.04614622265237,1.61979242853853,3.22054459345334 +14,4,0,2.04614622265237,1.96446721321234,2.94031366637701 +14,5,0,2.04614622265237,1.6819638784622,6.3045203371182 +15,1,0,1.91047856851638,1.41809400206497,0.352933179214503 +15,2,0,1.91047856851638,0.868574620082492,2.14983395101162 +15,3,0,1.91047856851638,1.38087806774412,1.4332857406434 +15,4,0,1.91047856851638,2.7376571119788,3.36504006381984 +15,5,0,1.91047856851638,2.50140408640708,5.0339321437845 +16,1,0,0.0904267516471942,-0.127925449618051,-0.629070962146693 +16,2,0,0.0904267516471942,-0.222640384632004,-0.60311595170131 +16,3,0,0.0904267516471942,0.209835589276016,0.386063603258975 +16,4,0,0.0904267516471942,0.518050979618486,1.34554957607334 +16,5,0,0.0904267516471942,1.35263633341545,1.76253705516832 +17,1,0,-0.978997126733862,-0.430729084042156,1.11006747787362 +17,2,0,-0.978997126733862,0.330410967028457,0.806951193076084 +17,3,0,-0.978997126733862,0.684625215247789,1.422241220599 +17,4,0,-0.978997126733862,0.436699992122361,1.05823352541785 +17,5,0,-0.978997126733862,-0.0965033949600655,-0.28861652804708 +18,1,0,-0.305160444674541,-0.916075051557808,-0.542992489238567 +18,2,0,-0.305160444674541,0.248257475817807,-0.860451526400478 +18,3,0,-0.305160444674541,0.798314680654779,-1.7886253887699 +18,4,0,-0.305160444674541,0.521228762410946,0.221495447624939 +18,5,0,-0.305160444674541,0.603880966224836,0.268782385230717 +19,1,0,-1.29347091631414,-0.440720995998017,-0.0917243454597562 +19,2,0,-1.29347091631414,-0.292457346868644,1.69390714797117 +19,3,0,-1.29347091631414,-0.076852304649137,-1.4267554614469 +19,4,0,-1.29347091631414,0.523168449121407,0.836559020778829 +19,5,0,-1.29347091631414,-0.126845315020416,-0.748902007493232 +20,1,0,-0.86946128396502,-0.731640889278664,0.897492140407103 +20,2,0,-0.86946128396502,-0.634157933584901,-0.356647943745405 +20,3,0,-0.86946128396502,0.414634427219905,-0.102122625271403 +20,4,0,-0.86946128396502,0.735617051894694,-0.493531099100508 +20,5,0,-0.86946128396502,0.0783122168879679,2.04744316684707 +21,1,0,0.193364867469568,0.527132325030444,2.83509885985534 +21,2,0,0.193364867469568,0.129936563195705,0.377414995366493 +21,3,0,0.193364867469568,0.613829784876638,2.64691385949551 +21,4,0,0.193364867469568,-0.120119586049108,2.99668502488452 +21,5,0,0.193364867469568,0.508258887874505,4.53816843932383 +22,1,0,-0.285006996929422,-0.544273797377208,0.61950479499134 +22,2,0,-0.285006996929422,-0.0470278141175047,3.0416910941977 +22,3,0,-0.285006996929422,0.307232669634666,3.87159719910662 +22,4,0,-0.285006996929422,1.01711272566063,2.77876159208761 +22,5,0,-0.285006996929422,1.31564072955683,3.62013111033041 +23,1,0,-0.200733998184601,0.406863652048468,0.0824709662570302 +23,2,0,-0.200733998184601,-0.0257062143537835,0.394833336515454 +23,3,0,-0.200733998184601,0.592886053520876,1.23455541173059 +23,4,0,-0.200733998184601,0.110099171857438,2.54019055351788 +23,5,0,-0.200733998184601,0.963691513380404,2.35389561030956 +24,1,0,1.24328569249421,0.92191360816656,1.14586532408122 +24,2,0,1.24328569249421,1.47628244052515,1.97289272773888 +24,3,0,1.24328569249421,0.727658826710546,2.37572049221841 +24,4,0,1.24328569249421,1.32540192428514,6.1798378307657 +24,5,0,1.24328569249421,1.99608430464066,7.76758515003412 +25,1,0,0.472298422553046,0.451130953214354,2.3849436399372 +25,2,0,0.472298422553046,0.393597463425591,-0.0898224276117208 +25,3,0,0.472298422553046,1.25037077852183,2.52589921950728 +25,4,0,0.472298422553046,0.523272852971633,2.37917100316457 +25,5,0,0.472298422553046,0.962139498902349,4.36517881139554 +26,1,0,0.416224904614732,-0.101584055903843,-0.579744769633871 +26,2,0,0.416224904614732,0.400325561772782,-0.972234260476899 +26,3,0,0.416224904614732,0.138863379638057,1.59661513491138 +26,4,0,0.416224904614732,0.397912352674014,0.742603700730964 +26,5,0,0.416224904614732,0.920278133420186,0.713813878237303 +27,1,0,-1.20810463188673,-0.656731791542808,0.112586405323568 +27,2,0,-1.20810463188673,-0.103560125599836,1.12355202824385 +27,3,0,-1.20810463188673,-0.197567905296184,1.53340035807994 +27,4,0,-1.20810463188673,0.974601656086776,-0.432564528542204 +27,5,0,-1.20810463188673,-0.0897357035583219,-0.828997632235555 +28,1,0,-1.43723667380504,0.0565108804189347,-1.84738581883577 +28,2,0,-1.43723667380504,-0.0297675553081633,-0.894498560080979 +28,3,0,-1.43723667380504,-0.0790701797677419,0.0220988662434747 +28,4,0,-1.43723667380504,-0.224757725463843,-0.847105774844605 +28,5,0,-1.43723667380504,0.166110286931856,-2.68630843144862 +29,1,0,1.08387441716567,0.401051829495605,2.54055593664914 +29,2,0,1.08387441716567,0.129584432966412,-0.290371476862269 +29,3,0,1.08387441716567,0.803794134135084,3.27778845957416 +29,4,0,1.08387441716567,0.930893075272324,4.61666902997821 +29,5,0,1.08387441716567,1.66326112137049,4.37994629191061 +30,1,0,1.14107632772678,-0.130216125775587,1.42153467508241 +30,2,0,1.14107632772678,0.543543313638396,1.82708108795342 +30,3,0,1.14107632772678,0.761496767847219,5.94565127190796 +30,4,0,1.14107632772678,1.11210754956847,3.8284928975795 +30,5,0,1.14107632772678,2.12456034096161,4.35152164443134 +31,1,0,-1.68363239672603,-0.394676649711449,1.23436313536159 +31,2,0,-1.68363239672603,-0.476159350468402,-1.2717016524923 +31,3,0,-1.68363239672603,-0.066870496453248,-1.85565601122068 +31,4,0,-1.68363239672603,0.69531782960049,0.800860195139755 +31,5,0,-1.68363239672603,0.252916979891308,0.5339463200228 +32,1,0,0.35862773442386,-0.254987375618001,-0.0878826690809986 +32,2,0,0.35862773442386,0.402486642358358,1.00755257360309 +32,3,0,0.35862773442386,0.973937634716594,1.63709582600101 +32,4,0,0.35862773442386,1.73454911285787,-1.18345432941171 +32,5,0,0.35862773442386,1.95248503861603,3.25719711630834 +33,1,0,1.03062394978694,1.1300648931605,0.251863182842662 +33,2,0,1.03062394978694,0.764447104885131,2.35818680820894 +33,3,0,1.03062394978694,0.81496241040219,-0.240462101427006 +33,4,0,1.03062394978694,1.23953989524317,2.39602149767805 +33,5,0,1.03062394978694,0.868796603908667,4.07416204116012 +34,1,0,-1.36021422621808,-0.037663969198275,0.281343353835558 +34,2,0,-1.36021422621808,-1.26017420369277,0.802690839736971 +34,3,0,-1.36021422621808,0.246925333860566,-0.703061697563873 +34,4,0,-1.36021422621808,-0.16975234477317,1.32797213567783 +34,5,0,-1.36021422621808,0.00740586453438835,0.44762238971308 +35,1,0,2.07462580052836,1.45675082321133,-0.57933493901855 +35,2,0,2.07462580052836,1.20926972518549,1.55463054174886 +35,3,0,2.07462580052836,2.34845238546939,1.6631742348656 +35,4,0,2.07462580052836,2.52239895186696,2.51391502849078 +35,5,0,2.07462580052836,1.52963756522155,4.43956443092784 +36,1,0,-0.624898820481009,0.322071656054509,-1.51999549840851 +36,2,0,-0.624898820481009,-0.436622498509552,-0.401234013390455 +36,3,0,-0.624898820481009,0.173813685252127,-0.0621678249348833 +36,4,0,-0.624898820481009,0.679651321432316,-0.737748502907965 +36,5,0,-0.624898820481009,0.716835039913072,0.736328659584627 +37,1,0,2.30895941653656,1.9742116653891,-2.10332094044947 +37,2,0,2.30895941653656,1.59260370469019,0.821201875196453 +37,3,0,2.30895941653656,2.61155608501098,1.66217100280885 +37,4,0,2.30895941653656,2.52595052377694,2.47477397356914 +37,5,0,2.30895941653656,2.54572354575135,2.57388225810638 +38,1,0,0.468838681888286,0.343993769445637,2.30521872621672 +38,2,0,0.468838681888286,0.641957713818,2.87394510695166 +38,3,0,0.468838681888286,0.880554440811486,4.3703467296499 +38,4,0,0.468838681888286,0.706193181423138,5.39737846119355 +38,5,0,0.468838681888286,1.30558403199082,5.23379128440331 +39,1,0,-0.427218215500197,0.566084390858194,2.13097968656818 +39,2,0,-0.427218215500197,-0.918399310828481,0.413215180305786 +39,3,0,-0.427218215500197,0.118235771072415,2.57020147610967 +39,4,0,-0.427218215500197,0.340995712937832,2.74807377011147 +39,5,0,-0.427218215500197,0.496211049768828,2.9555621794116 +40,1,0,0.777782496536268,0.300747632479431,-0.368684990628421 +40,2,0,0.777782496536268,1.0010007622223,0.595206586920402 +40,3,0,0.777782496536268,0.47188348651664,1.30591749207004 +40,4,0,0.777782496536268,1.86755346941996,3.31554734968523 +40,5,0,0.777782496536268,0.858892873747311,2.75457375011982 +41,1,0,-0.989113034685617,-0.134240789368439,-1.36994283379942 +41,2,0,-0.989113034685617,0.506570270785846,1.0534061137458 +41,3,0,-0.989113034685617,-0.895973224849036,0.27274725887401 +41,4,0,-0.989113034685617,1.31802792818807,0.26632977281089 +41,5,0,-0.989113034685617,0.454944454611298,2.40853446851008 +42,1,0,1.15840793985001,0.124868418660003,0.259801217583987 +42,2,0,1.15840793985001,1.87953027894752,2.17974265446549 +42,3,0,1.15840793985001,0.673567775709603,2.42638252821395 +42,4,0,1.15840793985001,0.63133400895871,2.89643454446268 +42,5,0,1.15840793985001,0.951863725906572,4.47572451802154 +43,1,0,2.67054324703151,2.31680906226424,1.05064037303325 +43,2,0,2.67054324703151,1.96713340733527,4.01260734361365 +43,3,0,2.67054324703151,2.01331188229246,1.02095053233103 +43,4,0,2.67054324703151,2.61485776901755,4.79120104206803 +43,5,0,2.67054324703151,2.06056399005803,7.36148928041125 +44,1,0,0.515574279047854,0.121900480753256,0.12827299244454 +44,2,0,0.515574279047854,0.443740380381531,0.669446613371002 +44,3,0,0.515574279047854,1.41716460656681,2.88575234491894 +44,4,0,0.515574279047854,1.06078400139549,-0.328837067278666 +44,5,0,0.515574279047854,0.846696081449088,1.42436157265577 +45,1,0,0.354081082960897,0.671167680765342,2.13948449234028 +45,2,0,0.354081082960897,0.293960347886397,1.20288799131482 +45,3,0,0.354081082960897,0.433020579196941,2.56751459819996 +45,4,0,0.354081082960897,0.845873361595047,3.54626002487861 +45,5,0,0.354081082960897,0.672652897873653,4.50017565808538 +46,1,0,1.52337503986612,1.32146148533203,3.34298891732902 +46,2,0,1.52337503986612,1.16076099809997,4.24667620804936 +46,3,0,1.52337503986612,1.03398841109356,8.15898555074743 +46,4,0,1.52337503986612,0.994797262587139,8.0779901676523 +46,5,0,1.52337503986612,1.1249651792548,5.68266893304037 +47,1,0,-0.483989426883416,0.574105356443331,-3.20613712871425 +47,2,0,-0.483989426883416,0.208918900884307,0.00649019809915752 +47,3,0,-0.483989426883416,0.741012271351296,0.757311787415 +47,4,0,-0.483989426883416,0.288505807018783,-0.62067301933059 +47,5,0,-0.483989426883416,0.278450260206523,0.549664430944102 +48,1,0,-0.241224470517284,0.538518235254474,0.562488211839541 +48,2,0,-0.241224470517284,0.745322403561392,-0.207881642487272 +48,3,0,-0.241224470517284,0.424900111677857,1.17191392468433 +48,4,0,-0.241224470517284,0.31091296192471,0.890812850085176 +48,5,0,-0.241224470517284,0.603363988372063,0.194681618635391 +49,1,0,-0.502923176484982,-0.845613766548061,-0.0280330536253708 +49,2,0,-0.502923176484982,-0.525656173107788,0.471650194793933 +49,3,0,-0.502923176484982,-0.27163538685412,1.77038444487592 +49,4,0,-0.502923176484982,0.748563048156958,1.06206817243644 +49,5,0,-0.502923176484982,0.862683732520299,1.87497630545043 +50,1,0,0.331707056486104,1.43310608470543,-1.03629025822675 +50,2,0,0.331707056486104,1.47412858157122,0.590271437555205 +50,3,0,0.331707056486104,0.00993825640601731,0.576019539322155 +50,4,0,0.331707056486104,1.18446092385485,1.06859243230635 +50,5,0,0.331707056486104,1.36296624104844,0.823833814241365 +51,1,0,-1.28376827728359,-0.054772706538596,-0.112182614259992 +51,2,0,-1.28376827728359,-0.647495854943373,-0.94712105801857 +51,3,0,-1.28376827728359,0.240043184837449,-0.00441638364042768 +51,4,0,-1.28376827728359,0.0606767301153908,0.875661792221071 +51,5,0,-1.28376827728359,0.379524142127041,1.22049460268803 +52,1,0,0.0268264038406972,0.0813711067876824,0.796399841450881 +52,2,0,0.0268264038406972,0.43253565739872,1.347067415838 +52,3,0,0.0268264038406972,1.09224494984207,1.34411702360282 +52,4,0,0.0268264038406972,0.775188384647136,2.98262612542836 +52,5,0,0.0268264038406972,0.413154429761364,2.67115624971039 +53,1,0,-1.88861627689539,-0.98294737042299,0.975402024308385 +53,2,0,-1.88861627689539,-0.364933260882153,-1.36155509419646 +53,3,0,-1.88861627689539,-1.28010414184404,-0.0953612032279289 +53,4,0,-1.88861627689539,-0.26285694011793,0.645459537255967 +53,5,0,-1.88861627689539,0.320342118934647,-1.34617214380709 +54,1,0,-0.758160785176121,-0.904859165411716,-1.92964585465319 +54,2,0,-0.758160785176121,0.426418257280298,-1.49702195088084 +54,3,0,-0.758160785176121,-0.172965732589018,0.573031919060233 +54,4,0,-0.758160785176121,0.501441761447204,0.60555176976647 +54,5,0,-0.758160785176121,0.748841788556021,0.386565229532384 +55,1,0,0.772657673186417,1.26108129853558,0.848326823659983 +55,2,0,0.772657673186417,0.994286367108801,0.533785646432795 +55,3,0,0.772657673186417,0.132121885819294,1.8393490008326 +55,4,0,0.772657673186417,1.4440198335426,4.31073338025935 +55,5,0,0.772657673186417,1.0421026037683,3.91593091869044 +56,1,0,0.0711352172241156,1.10908756660978,-1.62655633290739 +56,2,0,0.0711352172241156,0.34083565612227,-0.4916460672834 +56,3,0,0.0711352172241156,0.504409497242438,0.466038589769887 +56,4,0,0.0711352172241156,0.204265125308877,-0.613721113111909 +56,5,0,0.0711352172241156,1.43138380470924,0.955380118911139 +57,1,0,-0.794980135199814,-1.0945752615964,-0.419352393006358 +57,2,0,-0.794980135199814,-0.015679604949227,-2.37572615892288 +57,3,0,-0.794980135199814,0.782093277273994,-0.2996905359297 +57,4,0,-0.794980135199814,0.491006375103466,2.1447829543285 +57,5,0,-0.794980135199814,1.00082663497421,-0.477142427035938 +58,1,0,-1.02181782285825,-0.489529600000614,1.87489097842735 +58,2,0,-1.02181782285825,-0.634278175900665,-0.00496069455539061 +58,3,0,-1.02181782285825,0.0812610954568617,3.09374637476232 +58,4,0,-1.02181782285825,1.76062570141884,0.47159533276173 +58,5,0,-1.02181782285825,-0.118990402253873,2.51810462575589 +59,1,0,-0.798010454873102,0.0609546958435624,-2.36229581308972 +59,2,0,-0.798010454873102,0.454243995946293,0.0438796181667978 +59,3,0,-0.798010454873102,0.190314900617629,-1.03078925055765 +59,4,0,-0.798010454873102,0.449234077715798,-0.817729703997447 +59,5,0,-0.798010454873102,0.84368503725751,0.705517798639742 +60,1,0,-0.0306085669312452,0.452616374413926,-0.102673703149613 +60,2,0,-0.0306085669312452,0.813949076299101,-0.434575720306834 +60,3,0,-0.0306085669312452,1.4233380723612,1.29161760939601 +60,4,0,-0.0306085669312452,0.355689837861843,1.56195409402315 +60,5,0,-0.0306085669312452,0.965951055191722,0.909229841469014 +61,1,0,0.067078462093333,-1.09774855868318,0.0382140984881273 +61,2,0,0.067078462093333,-0.0734720467411606,0.173142949330493 +61,3,0,0.067078462093333,-1.19588725717607,0.359042696049106 +61,4,0,0.067078462093333,1.3856053140362,0.427910828219115 +61,5,0,0.067078462093333,1.44251835963329,3.56206647898682 +62,1,0,-1.52731617743477,-1.47707966252442,0.52878696784946 +62,2,0,-1.52731617743477,0.6725293667928,-0.713013683979517 +62,3,0,-1.52731617743477,-0.801863611756564,0.166830610735084 +62,4,0,-1.52731617743477,0.104227361969105,0.458581632509362 +62,5,0,-1.52731617743477,-0.0939253723866919,2.61021076307 +63,1,0,-0.576873399474489,0.568469415283613,-0.0885467512739461 +63,2,0,-0.576873399474489,-0.63462889673578,0.841368442972864 +63,3,0,-0.576873399474489,0.499041270194032,0.730812734245097 +63,4,0,-0.576873399474489,0.46564016095323,0.379790557893517 +63,5,0,-0.576873399474489,0.727842048483113,0.879392177634746 +64,1,0,-0.46221481649607,0.128333789084384,-0.810771268931039 +64,2,0,-0.46221481649607,0.370204400190163,-0.442277924307933 +64,3,0,-0.46221481649607,-0.0845670681314973,0.425221145845222 +64,4,0,-0.46221481649607,0.0289259292010756,1.92641758452772 +64,5,0,-0.46221481649607,0.832360426764675,1.76116093310025 +65,1,0,0.315703776453431,0.185119261555453,2.44859995246444 +65,2,0,0.315703776453431,1.132464076799,2.36264388338644 +65,3,0,0.315703776453431,0.804936552055527,4.33952964926613 +65,4,0,0.315703776453431,1.63738405965253,3.45633927678726 +65,5,0,0.315703776453431,0.654315798187792,2.29243512074624 +66,1,0,1.37658808505767,0.937250197886316,2.81324292738336 +66,2,0,1.37658808505767,1.31236937691446,1.43950882543872 +66,3,0,1.37658808505767,0.817377287895898,3.55004984527523 +66,4,0,1.37658808505767,0.697002957276908,5.44502481471952 +66,5,0,1.37658808505767,1.36246318909377,6.13221143906437 +67,1,0,1.42681697200005,0.929878026545164,1.29641812279912 +67,2,0,1.42681697200005,1.47078818640231,2.49949074805398 +67,3,0,1.42681697200005,1.39010723206942,2.28621088692472 +67,4,0,1.42681697200005,1.4994344602743,3.12588315590834 +67,5,0,1.42681697200005,2.07835117603224,5.93525294343137 +68,1,0,1.35308419041603,0.513814763134145,2.6808262083581 +68,2,0,1.35308419041603,0.71319979591278,3.11853804665789 +68,3,0,1.35308419041603,1.51616580195878,3.70939790832736 +68,4,0,1.35308419041603,0.303962610691678,5.32893930278764 +68,5,0,1.35308419041603,0.61382915360482,4.76798985640524 +69,1,0,1.79859786884555,0.714140771413365,-1.9749036071355 +69,2,0,1.79859786884555,1.75496633936581,1.15033972954931 +69,3,0,1.79859786884555,1.86609357285324,1.43700928442477 +69,4,0,1.79859786884555,1.03796845123607,4.69802297627656 +69,5,0,1.79859786884555,2.80748043992152,4.19758682907294 +70,1,0,-0.333836240206274,-0.271022425491758,1.63473212396469 +70,2,0,-0.333836240206274,1.03771957276743,2.02484454029549 +70,3,0,-0.333836240206274,-0.330129660933743,0.691713233135543 +70,4,0,-0.333836240206274,0.920392687785187,4.37086187320388 +70,5,0,-0.333836240206274,0.555092886752288,2.30949232147289 +71,1,0,-1.63895398153686,-1.18730170194834,2.09459510693227 +71,2,0,-1.63895398153686,0.0659017407261382,0.30490883430922 +71,3,0,-1.63895398153686,-0.62447901592315,0.471122244746123 +71,4,0,-1.63895398153686,-0.915053299758479,-0.865034452672139 +71,5,0,-1.63895398153686,1.14578444726516,0.986950199835889 +72,1,0,0.642543716487307,1.36176968092239,-1.81422208681334 +72,2,0,0.642543716487307,0.748030229144181,1.01388214037359 +72,3,0,0.642543716487307,0.82997265384777,0.0323834944231848 +72,4,0,0.642543716487307,0.880074784091964,1.82932376746001 +72,5,0,0.642543716487307,2.12456495392237,1.75625293742539 +73,1,0,1.8104991643694,0.841870657072021,2.72974425521786 +73,2,0,1.8104991643694,1.45889744701685,3.88174699419143 +73,3,0,1.8104991643694,1.85451418559813,2.80083656747205 +73,4,0,1.8104991643694,1.71168475350052,5.04857246953356 +73,5,0,1.8104991643694,2.09495458053712,6.84980828621333 +74,1,0,-0.932126399374389,-0.568022636104557,2.76205407969921 +74,2,0,-0.932126399374389,0.154151225212912,2.56941894156864 +74,3,0,-0.932126399374389,-0.45323497987067,0.131049516259385 +74,4,0,-0.932126399374389,0.707366959033985,2.23782040811992 +74,5,0,-0.932126399374389,1.2179979556303,3.53842169670308 +75,1,0,0.666231891524512,1.08694657400816,1.87974063555088 +75,2,0,0.666231891524512,1.27378252924364,2.53844953486072 +75,3,0,0.666231891524512,1.84808761734133,2.06146755721079 +75,4,0,0.666231891524512,1.2250029565835,4.65112423855464 +75,5,0,0.666231891524512,2.16385717182326,5.81266126562459 +76,1,0,0.341678631484337,1.06347467961005,0.883641144348533 +76,2,0,0.341678631484337,0.424007246284149,0.927200899437488 +76,3,0,0.341678631484337,0.463634111552505,1.09297461415004 +76,4,0,0.341678631484337,0.883946377784091,1.35138421699445 +76,5,0,0.341678631484337,0.846805066898023,4.35549048057971 +77,1,0,0.637620706206215,1.13509768737945,1.43068721208062 +77,2,0,0.637620706206215,0.119022913744544,4.02884996915181 +77,3,0,0.637620706206215,1.78409687300482,5.37165267440474 +77,4,0,0.637620706206215,0.572587375028069,4.54884339641776 +77,5,0,0.637620706206215,1.30157744988052,4.66081396519702 +78,1,0,0.916740129944944,0.527388555742362,1.67885626210288 +78,2,0,0.916740129944944,1.6516624227088,2.88404956264771 +78,3,0,0.916740129944944,1.57621531218402,5.17729353626615 +78,4,0,0.916740129944944,1.12169573790953,6.39677493288079 +78,5,0,0.916740129944944,1.56442333890756,3.51684585995395 +79,1,0,0.678393345349387,0.981084168977522,-0.303125990400053 +79,2,0,0.678393345349387,0.00845983067858702,3.77063304832269 +79,3,0,0.678393345349387,0.985461654447826,1.28792599622788 +79,4,0,0.678393345349387,1.37335705414109,3.93901911601086 +79,5,0,0.678393345349387,1.59918286286575,4.20477704101249 +80,1,0,-0.614505923587853,-0.074124901025198,0.557946043511742 +80,2,0,-0.614505923587853,0.0775885147622684,3.20589794298296 +80,3,0,-0.614505923587853,-0.129964058314692,1.43190426473233 +80,4,0,-0.614505923587853,1.12112075034697,3.91574997314387 +80,5,0,-0.614505923587853,0.349774794978119,2.92434398626071 +81,1,0,-0.827776110905095,0.0779920496627954,2.20724033209597 +81,2,0,-0.827776110905095,0.329874381465543,-0.343896532607536 +81,3,0,-0.827776110905095,-0.509582604067608,0.184330306231341 +81,4,0,-0.827776110905095,0.425631993327435,-0.677124424118725 +81,5,0,-0.827776110905095,0.576793045090816,5.25417091189951 +82,1,0,0.861142171302151,0.534905942272975,-0.559829415854437 +82,2,0,0.861142171302151,0.421001725699289,-0.640432437832771 +82,3,0,0.861142171302151,1.33334862780767,1.15221010623295 +82,4,0,0.861142171302151,1.51902787603164,1.84609545714379 +82,5,0,0.861142171302151,2.27137340338279,1.20605568714721 +83,1,0,0.616000967810865,1.56647356845175,0.856810537009867 +83,2,0,0.616000967810865,0.418959285740418,3.0293435997056 +83,3,0,0.616000967810865,1.71178089113956,4.1807485972266 +83,4,0,0.616000967810865,1.75291581545166,3.61295843048961 +83,5,0,0.616000967810865,1.95198829511699,5.40410230770897 +84,1,0,-0.364563738233122,0.729912191174741,-1.54643580105865 +84,2,0,-0.364563738233122,0.270810379458042,0.662855640387282 +84,3,0,-0.364563738233122,0.46985768052808,3.08624127314465 +84,4,0,-0.364563738233122,0.471373212800752,0.515624799993429 +84,5,0,-0.364563738233122,0.781470542382374,1.70977184890727 +85,1,0,0.143653481491035,0.940535155473577,-0.0595363250531367 +85,2,0,0.143653481491035,-0.338084292631134,-0.805588718694135 +85,3,0,0.143653481491035,1.27753543499624,0.00832597427848673 +85,4,0,0.143653481491035,0.89192092464685,2.03135738913338 +85,5,0,0.143653481491035,0.703431750258417,0.599027040421859 +86,1,0,-0.131220650539475,-0.0817446182852218,-0.396935575601526 +86,2,0,-0.131220650539475,0.191895364526733,-0.194650738734955 +86,3,0,-0.131220650539475,0.788377198501869,3.73853028280585 +86,4,0,-0.131220650539475,1.02415048291942,1.91396930977068 +86,5,0,-0.131220650539475,1.03878595672618,2.1859552336763 +87,1,0,-0.333066256749381,1.06674969434541,-0.0657657799846833 +87,2,0,-0.333066256749381,0.149647369032651,0.498161299881925 +87,3,0,-0.333066256749381,0.581926896494485,1.13646345708566 +87,4,0,-0.333066256749381,-0.247133860668094,-0.175924284009217 +87,5,0,-0.333066256749381,1.46732523994061,1.91561016686449 +88,1,0,-0.434370693446027,-0.437377299668023,-0.772574889112886 +88,2,0,-0.434370693446027,-0.152739483160924,-0.568834744275269 +88,3,0,-0.434370693446027,0.951913298045205,0.385469624381133 +88,4,0,-0.434370693446027,1.1189528375249,-0.057927566466312 +88,5,0,-0.434370693446027,0.568042594475733,0.443220743369422 +89,1,0,-0.141474869850029,0.0801484231476475,0.0270269788568515 +89,2,0,-0.141474869850029,0.0481236129066036,0.205532603073162 +89,3,0,-0.141474869850029,0.995260603221087,0.651653657935192 +89,4,0,-0.141474869850029,1.32639060956188,1.59469437463446 +89,5,0,-0.141474869850029,0.645906816771665,2.15442315953645 +90,1,0,1.03227036316416,0.63763451526355,-1.25422979344628 +90,2,0,1.03227036316416,0.405889210017968,-0.110219289419371 +90,3,0,1.03227036316416,0.458014774582056,0.982330971708697 +90,4,0,1.03227036316416,0.852538382878488,3.76935892335802 +90,5,0,1.03227036316416,0.962265711370071,2.56238146466164 +91,1,0,-0.486491594070801,-0.172045064925041,1.83038665225263 +91,2,0,-0.486491594070801,0.384618723183606,0.0474591275790941 +91,3,0,-0.486491594070801,0.531878509700311,1.7092334560639 +91,4,0,-0.486491594070801,0.793638805747236,2.5679967220646 +91,5,0,-0.486491594070801,0.722297893770697,3.12470764883272 +92,1,0,0.219980494850729,0.0734058917248482,0.384234866874619 +92,2,0,0.219980494850729,-0.219734183232657,0.903091028630454 +92,3,0,0.219980494850729,0.682596952495566,2.12751213821996 +92,4,0,0.219980494850729,1.23869541320448,3.5612634595244 +92,5,0,0.219980494850729,0.578532061197268,3.27222425245696 +93,1,0,0.288608812761447,0.421231189913603,-1.40212126529636 +93,2,0,0.288608812761447,0.454485182510659,0.422698666792128 +93,3,0,0.288608812761447,1.31865477532447,0.115353686002623 +93,4,0,0.288608812761447,1.08220992063066,2.67936764198457 +93,5,0,0.288608812761447,0.959458903140813,2.13821853667061 +94,1,0,-1.61902954772177,-1.20480995091084,1.82409496936011 +94,2,0,-1.61902954772177,-0.930824167709811,1.42101239978627 +94,3,0,-1.61902954772177,-0.584839870198029,1.78332374092068 +94,4,0,-1.61902954772177,-0.346404263307608,-0.227553617442285 +94,5,0,-1.61902954772177,1.1729349153929,1.9498809084647 +95,1,0,0.222271128189547,-0.0387473417629938,0.858725090646077 +95,2,0,0.222271128189547,0.767954290471442,2.69923913505218 +95,3,0,0.222271128189547,0.605075552915976,1.99193702670537 +95,4,0,0.222271128189547,1.3558238746813,2.73108721331929 +95,5,0,0.222271128189547,1.63139262575401,0.783913778717177 +96,1,0,1.54600822113029,2.02001061311009,2.62936331979604 +96,2,0,1.54600822113029,1.35702096802835,1.96438966667089 +96,3,0,1.54600822113029,1.36437355018916,1.82028484401593 +96,4,0,1.54600822113029,1.54806112741881,2.22526558385425 +96,5,0,1.54600822113029,2.88653595933768,5.29452516245512 +97,1,0,0.133697246677124,-0.0718599103232635,1.93759954133085 +97,2,0,0.133697246677124,0.243856103638007,1.83522434020185 +97,3,0,0.133697246677124,0.138526367764456,4.6449042058556 +97,4,0,0.133697246677124,1.8769167656904,3.54647214833668 +97,5,0,0.133697246677124,0.962401521139283,2.9247955599588 +98,1,0,0.513461762623951,0.550111450173374,1.13638082120044 +98,2,0,0.513461762623951,-0.116902446416303,2.0047933773136 +98,3,0,0.513461762623951,1.39013026554457,2.89151767905503 +98,4,0,0.513461762623951,1.5481840147936,2.39962626326609 +98,5,0,0.513461762623951,0.988855636857315,2.77565780113198 +99,1,0,0.0284391113388029,0.689085116598258,0.254359114140023 +99,2,0,0.0284391113388029,0.349968288749272,0.878606062697145 +99,3,0,0.0284391113388029,1.81169809337691,0.440914916417645 +99,4,0,0.0284391113388029,0.983652616962293,2.17316792107343 +99,5,0,0.0284391113388029,1.20691015927357,3.06483425373133 +100,1,0,-1.7202950012414,-0.115879309040799,-1.39743104846185 +100,2,0,-1.7202950012414,0.0489770680107625,-1.04291605039209 +100,3,0,-1.7202950012414,-1.17871267878631,-0.81502457454413 +100,4,0,-1.7202950012414,-0.105520702288695,0.334070265764476 +100,5,0,-1.7202950012414,-0.467384608422146,-0.890554303485205 +101,1,3,-0.657717766276882,-0.94206308645325,0.232488558205459 +101,2,3,-0.657717766276882,0.0705885988157602,0.953684678547186 +101,3,3,-0.657717766276882,0.0953022991421917,4.99112586140753 +101,4,3,-0.657717766276882,0.271992657738819,3.73290731784579 +101,5,3,-0.657717766276882,0.170440318698168,5.85181562906916 +102,1,3,0.606628948683484,-0.13131505600894,2.91452541834592 +102,2,3,0.606628948683484,1.4022497729769,0.261042740854546 +102,3,3,0.606628948683484,0.993231830563238,2.71853186863986 +102,4,3,0.606628948683484,0.884332783542463,4.56588455342279 +102,5,3,0.606628948683484,1.56713473862668,7.09247867741218 +103,1,3,-0.332641963407837,0.154534274202655,0.602279850487755 +103,2,3,-0.332641963407837,0.072432790726706,-0.309540706057066 +103,3,3,-0.332641963407837,1.16818496470408,-0.165286394085044 +103,4,3,-0.332641963407837,0.702511034762439,1.34374356615043 +103,5,3,-0.332641963407837,1.01447715390258,5.3160097148392 +104,1,3,0.469706705466534,0.279153249372299,-0.383474155857604 +104,2,3,0.469706705466534,1.26056420037322,0.450636566145736 +104,3,3,0.469706705466534,0.840143863086957,3.06714331395692 +104,4,3,0.469706705466534,1.5062897407947,5.65655038983017 +104,5,3,0.469706705466534,1.1964368519802,5.11830997961023 +105,1,3,1.66748637673741,1.12973770628659,-1.99633809999194 +105,2,3,1.66748637673741,1.71587553353966,-1.18150760955215 +105,3,3,1.66748637673741,1.51301509979169,3.13175716039968 +105,4,3,1.66748637673741,1.52614573553539,5.51260657016865 +105,5,3,1.66748637673741,1.80360251228863,6.86533385915753 +106,1,3,1.27268901841241,0.410841768659794,0.678968405380034 +106,2,3,1.27268901841241,2.42608384357659,0.996016318495919 +106,3,3,1.27268901841241,1.75308326994184,2.48578861598163 +106,4,3,1.27268901841241,1.50527027515069,6.60521083807514 +106,5,3,1.27268901841241,0.851577609230435,9.15879471695537 +107,1,3,-0.113521083630058,-0.617931847269183,0.389795760489174 +107,2,3,-0.113521083630058,0.00578001996492189,1.24529091081873 +107,3,3,-0.113521083630058,0.522769231917015,3.59142446549231 +107,4,3,-0.113521083630058,0.55896664892714,4.08609378496072 +107,5,3,-0.113521083630058,0.624181425631188,6.65868685406829 +108,1,3,1.42915725643045,1.03946831567814,-1.49277112713376 +108,2,3,1.42915725643045,0.355936899218436,-0.993582726915977 +108,3,3,1.42915725643045,1.3102549176241,0.706937887500567 +108,4,3,1.42915725643045,1.91255925354156,2.4057349385667 +108,5,3,1.42915725643045,1.11706892387658,5.42943529075469 +109,1,3,-0.559679157783233,0.240146108567419,1.85721232954093 +109,2,3,-0.559679157783233,-1.18164851474772,0.598496577056571 +109,3,3,-0.559679157783233,-0.567714929049523,3.8190582793659 +109,4,3,-0.559679157783233,1.58499831755575,3.61974345956697 +109,5,3,-0.559679157783233,0.281509011231452,5.64271736926589 +110,1,3,0.187728838960483,-0.0990282930604425,0.636491412067755 +110,2,3,0.187728838960483,0.522934521191726,-1.27512878217836 +110,3,3,0.187728838960483,0.451864219784705,0.505624681678577 +110,4,3,0.187728838960483,0.778209878446357,2.64560002072363 +110,5,3,0.187728838960483,1.85942863847523,5.79668804951045 +111,1,3,1.68213095275821,0.251934065681341,3.91605951374998 +111,2,3,1.68213095275821,1.35011115331486,3.79188977773365 +111,3,3,1.68213095275821,1.97032026344658,3.08046167492404 +111,4,3,1.68213095275821,2.2589275661647,8.27001943564112 +111,5,3,1.68213095275821,1.81228175371279,7.85436781063261 +112,1,3,-0.0788326245762765,-0.0886876014313262,0.741167712744016 +112,2,3,-0.0788326245762765,0.057345760642664,-0.289974470099434 +112,3,3,-0.0788326245762765,0.217463034910929,1.05732757506079 +112,4,3,-0.0788326245762765,1.61994832884064,5.16748675022915 +112,5,3,-0.0788326245762765,1.10512689283478,6.32704551373198 +113,1,3,-1.11820261456398,-0.326553621508285,4.59619133661404 +113,2,3,-1.11820261456398,0.138424198638174,3.08762952083826 +113,3,3,-1.11820261456398,0.580218918516405,4.11792205698977 +113,4,3,-1.11820261456398,0.180859352511177,5.12177393132146 +113,5,3,-1.11820261456398,0.466131001972245,6.08972404155704 +114,1,3,0.00617399378076931,0.655178842061438,-2.62808216419414 +114,2,3,0.00617399378076931,1.54562520211198,-0.333313638255983 +114,3,3,0.00617399378076931,-0.0719855219840838,0.0862359331011819 +114,4,3,0.00617399378076931,0.444363543616229,2.84552412987694 +114,5,3,0.00617399378076931,0.531577221298498,3.34103605859625 +115,1,3,-0.466490512919715,-0.70628149463559,-1.45817530327387 +115,2,3,-0.466490512919715,0.729198541085584,-1.93903353127595 +115,3,3,-0.466490512919715,0.543408386695873,-0.351268598390575 +115,4,3,-0.466490512919715,1.02986330956025,1.1754453578064 +115,5,3,-0.466490512919715,0.412796845409179,3.51229890894027 +116,1,3,-2.26408332836368,-1.30773888985853,-2.10077614733638 +116,2,3,-2.26408332836368,-1.37763784721948,0.236161311201754 +116,3,3,-2.26408332836368,-1.7425007121992,1.23135889576542 +116,4,3,-2.26408332836368,-0.708621021967853,1.87035942321069 +116,5,3,-2.26408332836368,-1.03635888476242,-0.645611331548352 +117,1,3,0.552125571623184,1.32365285117336,0.779376482838142 +117,2,3,0.552125571623184,0.930668266663576,3.04427235955234 +117,3,3,0.552125571623184,0.622006301346775,3.68654705941641 +117,4,3,0.552125571623184,1.77537177199262,5.84936566524921 +117,5,3,0.552125571623184,1.4463562521855,7.26081776390063 +118,1,3,-0.294573500984481,-0.226854558548949,0.624381599018543 +118,2,3,-0.294573500984481,0.451975050319793,-0.50960478842452 +118,3,3,-0.294573500984481,1.10372122526393,1.56675527682092 +118,4,3,-0.294573500984481,0.820290560391889,3.70060353900024 +118,5,3,-0.294573500984481,0.0496382963428972,4.85596894365002 +119,1,3,-0.723593527799914,-0.505152373633315,2.0676374826663 +119,2,3,-0.723593527799914,0.101750802662789,1.03136110282366 +119,3,3,-0.723593527799914,0.730559754677908,3.41623122468754 +119,4,3,-0.723593527799914,0.00221535064169642,3.78216320855324 +119,5,3,-0.723593527799914,0.636998000740745,4.93833031585068 +120,1,3,0.858310574796443,1.08316871624732,5.23943573915105 +120,2,3,0.858310574796443,0.863887874687908,5.35534446922412 +120,3,3,0.858310574796443,2.11534837934243,6.58543264119611 +120,4,3,0.858310574796443,1.88024199878906,9.8528988127391 +120,5,3,0.858310574796443,1.27582322704727,11.2611305249796 +121,1,3,0.220462861432781,-0.309440648082669,-0.755306621580593 +121,2,3,0.220462861432781,0.627055418074356,-0.023684948685408 +121,3,3,0.220462861432781,0.545149725436819,1.36183809596365 +121,4,3,0.220462861432781,0.792007240646273,3.18699848795418 +121,5,3,0.220462861432781,1.69726445585479,3.95436649310404 +122,1,3,-0.140752859229319,0.177062299482488,-1.76683361804301 +122,2,3,-0.140752859229319,0.304636966807162,0.993079818624433 +122,3,3,-0.140752859229319,0.421561519180845,0.332450286328496 +122,4,3,-0.140752859229319,1.36949693276948,3.85954344361671 +122,5,3,-0.140752859229319,0.568440407673914,3.85447057933448 +123,1,3,1.08154108301988,0.535266128351368,0.779969396251111 +123,2,3,1.08154108301988,0.880289324345213,1.59969760053008 +123,3,3,1.08154108301988,0.756275612370415,4.49990130269742 +123,4,3,1.08154108301988,1.05475695661148,5.66132166010569 +123,5,3,1.08154108301988,0.991482360834123,7.76732143736291 +124,1,3,0.000773525396488833,0.462844594613568,0.841288741599271 +124,2,3,0.000773525396488833,0.602602082712852,0.312361825429711 +124,3,3,0.000773525396488833,0.339182135812421,2.46117172121027 +124,4,3,0.000773525396488833,0.560962815672359,3.19918261277187 +124,5,3,0.000773525396488833,0.445130224054067,6.56037285845182 +125,1,3,-0.212450189809644,0.948086214476399,3.10996825066564 +125,2,3,-0.212450189809644,0.318713140332635,3.16644442473342 +125,3,3,-0.212450189809644,0.680601781620023,3.20618608330843 +125,4,3,-0.212450189809644,1.62789839447148,4.76744325623378 +125,5,3,-0.212450189809644,1.3134135459942,5.12920413147696 +126,1,3,-0.0749218641308017,-0.250989637603912,-0.34160343652017 +126,2,3,-0.0749218641308017,-0.107579672268752,0.194139114567561 +126,3,3,-0.0749218641308017,0.93589473033804,3.58262564440672 +126,4,3,-0.0749218641308017,0.683688042722152,1.72966565599836 +126,5,3,-0.0749218641308017,1.71491065530947,4.19726792441863 +127,1,3,0.627289566110117,1.05729793945288,1.62128596144571 +127,2,3,0.627289566110117,0.973758617212842,2.34129685541675 +127,3,3,0.627289566110117,0.828958721907215,3.88047996241525 +127,4,3,0.627289566110117,1.54879032336665,5.42297645981372 +127,5,3,0.627289566110117,1.48328874793323,8.63035389347065 +128,1,3,0.410545949407018,0.625141759106629,1.94363260911011 +128,2,3,0.410545949407018,0.840869756849677,2.63277909520217 +128,3,3,0.410545949407018,1.69458209211863,6.08657628694377 +128,4,3,0.410545949407018,0.638430284893267,6.45369861408724 +128,5,3,0.410545949407018,2.5733341846153,7.72778285671548 +129,1,3,1.33028430702108,2.13073984612338,1.69376984087629 +129,2,3,1.33028430702108,0.646345800359394,3.21444010789856 +129,3,3,1.33028430702108,2.04773265337806,3.61285827791554 +129,4,3,1.33028430702108,1.67761633423117,5.69764116082381 +129,5,3,1.33028430702108,0.912577561277432,9.44538478230292 +130,1,3,-1.30395806956874,0.140532026865759,-0.905148818564164 +130,2,3,-1.30395806956874,0.1334871419863,0.394705286548389 +130,3,3,-1.30395806956874,0.0860359670190863,3.32062284302592 +130,4,3,-1.30395806956874,0.30340861859709,3.28255261756849 +130,5,3,-1.30395806956874,0.658912821802418,2.56331948055225 +131,1,3,-0.357820487255056,0.604743403188498,-0.951886500117099 +131,2,3,-0.357820487255056,0.647902518744151,-0.167819315846137 +131,3,3,-0.357820487255056,1.10059087835971,0.101181804228307 +131,4,3,-0.357820487255056,0.308431637907341,3.75349531284482 +131,5,3,-0.357820487255056,0.718013535203741,2.89114217148319 +132,1,3,0.48766605796259,0.0140713891864764,-2.21162281394707 +132,2,3,0.48766605796259,0.156165054524314,-0.679269124727664 +132,3,3,0.48766605796259,0.732944740978031,2.81641237256502 +132,4,3,0.48766605796259,1.28017422519403,2.46764691181743 +132,5,3,0.48766605796259,0.938788379132021,4.56142902922779 +133,1,3,0.0965095252581635,-0.424257311714683,-0.906138381171024 +133,2,3,0.0965095252581635,0.682038652664297,0.467549662669626 +133,3,3,0.0965095252581635,0.599960579187045,1.56808656793963 +133,4,3,0.0965095252581635,0.670053117047675,5.99640467483164 +133,5,3,0.0965095252581635,0.00290729768220532,5.61187286084779 +134,1,3,0.470813281504669,-0.0237977219345792,-1.17861719509669 +134,2,3,0.470813281504669,0.0210231463695877,2.26535855456937 +134,3,3,0.470813281504669,1.65875503097192,2.79972362019488 +134,4,3,0.470813281504669,1.4522899241199,6.96850396868235 +134,5,3,0.470813281504669,1.19697537953027,5.53220254058808 +135,1,3,1.67155479620156,1.63003289483879,1.13465670732127 +135,2,3,1.67155479620156,1.66566175054474,2.16928327099793 +135,3,3,1.67155479620156,1.23104108297534,4.62609919171196 +135,4,3,1.67155479620156,2.51547061628803,6.42366307314904 +135,5,3,1.67155479620156,0.773175668804269,6.06302343062228 +136,1,3,-0.0332394453434365,0.0760360749606722,0.0208825919783583 +136,2,3,-0.0332394453434365,0.758090599491503,0.568066216720766 +136,3,3,-0.0332394453434365,0.865406851245269,2.16385713696812 +136,4,3,-0.0332394453434365,0.287846254413097,4.10855697907143 +136,5,3,-0.0332394453434365,0.735172436002937,4.53398714072422 +137,1,3,-0.688225331751058,0.0016693549665899,-0.605796558059025 +137,2,3,-0.688225331751058,0.31564905198807,2.09364296940547 +137,3,3,-0.688225331751058,0.302456958121517,2.91353634798101 +137,4,3,-0.688225331751058,1.4601427357091,3.55511507145684 +137,5,3,-0.688225331751058,0.52377242178836,6.28678207632846 +138,1,3,-0.500122627493091,0.396683358364411,1.06354027076539 +138,2,3,-0.500122627493091,0.147266283149743,-0.416532302563743 +138,3,3,-0.500122627493091,-0.0674183543222071,2.39971802018502 +138,4,3,-0.500122627493091,0.444002742154018,4.72359154487664 +138,5,3,-0.500122627493091,-0.00639513954455206,6.2797345135803 +139,1,3,0.18107731763868,0.358754646027512,2.56815917737501 +139,2,3,0.18107731763868,0.473843315161791,5.09625792076298 +139,3,3,0.18107731763868,0.222826468427173,5.09301261569359 +139,4,3,0.18107731763868,0.998960955021969,5.19457217305691 +139,5,3,0.18107731763868,0.94396796417614,7.6238515902996 +140,1,3,-1.37211800780397,-0.397986200765386,-0.486212418062664 +140,2,3,-1.37211800780397,-1.14777684605006,-0.587349940190411 +140,3,3,-1.37211800780397,0.207813697049092,0.0179085891028778 +140,4,3,-1.37211800780397,-0.0157145368531961,0.718055885197066 +140,5,3,-1.37211800780397,0.141532328048656,3.84922898075712 +141,1,3,-1.19897389691436,0.360461248314332,-1.8152035620548 +141,2,3,-1.19897389691436,-0.542266936350596,-2.95909668834627 +141,3,3,-1.19897389691436,0.99913729191338,0.895701803570273 +141,4,3,-1.19897389691436,0.397516705219247,1.30946775962812 +141,5,3,-1.19897389691436,0.557238702745939,2.26160192220899 +142,1,3,-0.998319924353911,-0.617579580251617,-1.74650801638042 +142,2,3,-0.998319924353911,-0.949218896378564,-0.530887740124697 +142,3,3,-0.998319924353911,0.039191340332905,0.0526537324003162 +142,4,3,-0.998319924353911,1.06166107117482,1.17556748272577 +142,5,3,-0.998319924353911,0.388788578221689,3.55709306215575 +143,1,3,1.20119683314219,0.774166948615702,1.59415308736676 +143,2,3,1.20119683314219,1.60847913451569,1.16604946065365 +143,3,3,1.20119683314219,1.79639042370985,3.97103994157813 +143,4,3,1.20119683314219,0.749734928571424,4.99089995233893 +143,5,3,1.20119683314219,1.4255599566404,9.34890063805875 +144,1,3,1.00628694430582,1.08167301522945,-0.555804553785998 +144,2,3,1.00628694430582,1.28955773107959,-0.794982544165143 +144,3,3,1.00628694430582,0.531935299948853,2.6417560919309 +144,4,3,1.00628694430582,1.96009703859714,3.43572503800884 +144,5,3,1.00628694430582,1.20637425198866,4.34655255306734 +145,1,3,1.67091120380766,1.25583677189642,1.88484533573143 +145,2,3,1.67091120380766,0.697055921239634,0.705532024610885 +145,3,3,1.67091120380766,1.58825290389066,3.24141239581406 +145,4,3,1.67091120380766,1.83616400511528,6.05043141490311 +145,5,3,1.67091120380766,1.28185088085329,5.02523739500611 +146,1,3,-0.379316649007007,-0.0834809234825081,-0.604888670214619 +146,2,3,-0.379316649007007,-0.795085926968904,-0.439042960407943 +146,3,3,-0.379316649007007,0.401301223844121,0.614602642472997 +146,4,3,-0.379316649007007,0.476915753436164,2.40152270032433 +146,5,3,-0.379316649007007,0.130152214243919,3.90321730111349 +147,1,3,0.420754458915134,0.466207147814328,-1.67292060140418 +147,2,3,0.420754458915134,0.569134175687715,-0.828083532691714 +147,3,3,0.420754458915134,0.883067633049473,0.919896016564556 +147,4,3,0.420754458915134,0.531632319632921,3.08518200522015 +147,5,3,0.420754458915134,0.635630282770971,4.59162567793378 +148,1,3,0.0311732664506885,-0.0978878456004359,0.293726126730062 +148,2,3,0.0311732664506885,0.565589374149626,0.569298807450127 +148,3,3,0.0311732664506885,0.778310646490644,1.69791627492246 +148,4,3,0.0311732664506885,0.962303021614927,2.33824377902591 +148,5,3,0.0311732664506885,1.19136094560224,4.05174848811233 +149,1,3,0.512760193053312,0.0915936923166048,0.00633837792554687 +149,2,3,0.512760193053312,0.581983898348367,-0.436480189053884 +149,3,3,0.512760193053312,0.0634121352834593,2.40477462650618 +149,4,3,0.512760193053312,0.569371944296203,4.32343142524583 +149,5,3,0.512760193053312,1.21441640506989,4.73370495644068 +150,1,3,1.87868646637167,0.69125924639008,3.38649600786785 +150,2,3,1.87868646637167,1.31653504546201,3.27059369523601 +150,3,3,1.87868646637167,1.02569583379149,6.39971209529027 +150,4,3,1.87868646637167,1.44661621609673,6.51984488748056 +150,5,3,1.87868646637167,2.02528698187357,9.61899939982145 +151,1,3,-1.84278530939641,-0.644029715762125,-2.22769544227137 +151,2,3,-1.84278530939641,-0.441329046218621,-1.45924645134561 +151,3,3,-1.84278530939641,-0.145124481339132,-1.15662859328522 +151,4,3,-1.84278530939641,-0.168275634333566,-1.17671356983013 +151,5,3,-1.84278530939641,1.1584080631215,-0.828004130089602 +152,1,3,1.72965756391273,0.255517813411765,1.53812815985889 +152,2,3,1.72965756391273,2.03646206475824,3.02640293459076 +152,3,3,1.72965756391273,1.96685280001739,5.0807862101301 +152,4,3,1.72965756391273,1.78778264470661,7.07323562820844 +152,5,3,1.72965756391273,2.63307633311382,9.20377058468654 +153,1,3,-0.29666031962568,-0.286682880554559,-0.905739111548258 +153,2,3,-0.29666031962568,0.0730518169668248,1.83539397771319 +153,3,3,-0.29666031962568,1.05754267911064,2.24979895590196 +153,4,3,-0.29666031962568,0.543774543760072,4.15543805256443 +153,5,3,-0.29666031962568,1.52983625197896,4.26308726987872 +154,1,3,-0.629413689190318,0.727509671424609,3.71401857261263 +154,2,3,-0.629413689190318,1.29760725996956,1.12434826453582 +154,3,3,-0.629413689190318,0.932495976740686,3.01651148831642 +154,4,3,-0.629413689190318,0.864584440262066,3.13699893765196 +154,5,3,-0.629413689190318,0.712000503501498,5.4012578917997 +155,1,3,1.26483805731984,0.101373228065778,0.373539829402359 +155,2,3,1.26483805731984,0.212809068946509,1.58504815689564 +155,3,3,1.26483805731984,0.99358154798726,2.71644229480947 +155,4,3,1.26483805731984,2.25517833481268,4.34058665417517 +155,5,3,1.26483805731984,1.97118903746078,7.74877923012855 +156,1,3,-0.883753614415126,-0.399535191884741,0.0960186166312004 +156,2,3,-0.883753614415126,-0.320901569906869,0.244002366570801 +156,3,3,-0.883753614415126,1.06156750938429,2.24533452927691 +156,4,3,-0.883753614415126,0.461016513134735,1.85726192341166 +156,5,3,-0.883753614415126,0.510702258754348,4.97094755371067 +157,1,3,-0.920994725090723,-0.892479349459743,-0.998800959364927 +157,2,3,-0.920994725090723,-1.15880706003958,0.367783265190597 +157,3,3,-0.920994725090723,-0.34037148362012,1.01156615038342 +157,4,3,-0.920994725090723,0.835538390428648,2.31233387560194 +157,5,3,-0.920994725090723,-0.25552330880026,3.01427008481776 +158,1,3,-0.34893368979522,-0.267555864749471,0.391980485124665 +158,2,3,-0.34893368979522,0.631473827249395,0.118968890864104 +158,3,3,-0.34893368979522,0.401379529995111,2.00487956507939 +158,4,3,-0.34893368979522,0.253640013407772,2.70578554698554 +158,5,3,-0.34893368979522,0.824772396694025,5.63766780336331 +159,1,3,-0.304255532851446,-0.675035468671625,0.975756093026101 +159,2,3,-0.304255532851446,0.0759771386422023,1.40147646679613 +159,3,3,-0.304255532851446,0.230792842617687,1.65368279369012 +159,4,3,-0.304255532851446,0.811272643636225,2.10771822600086 +159,5,3,-0.304255532851446,1.28756246587767,6.54277097690431 +160,1,3,-1.01219093638919,0.127507399755032,-2.35564452433788 +160,2,3,-1.01219093638919,0.268959034351851,-1.51029850831283 +160,3,3,-1.01219093638919,-0.407224246647464,-0.822751398045535 +160,4,3,-1.01219093638919,-0.0518131717045414,2.25240396176116 +160,5,3,-1.01219093638919,0.605259863963078,1.95588826400922 +161,1,3,-0.757717772717969,0.605568531426284,1.07474760513235 +161,2,3,-0.757717772717969,0.0731360455976896,-0.615467370613254 +161,3,3,-0.757717772717969,0.995768606560662,1.33512778948212 +161,4,3,-0.757717772717969,-0.542214426349538,3.73414229763899 +161,5,3,-0.757717772717969,0.376089632978491,4.32273833332243 +162,1,3,-0.633058606184867,0.444474326199027,-0.157864196813987 +162,2,3,-0.633058606184867,-0.358409061943328,1.36588036876563 +162,3,3,-0.633058606184867,0.00717898377558673,1.5208683399469 +162,4,3,-0.633058606184867,0.928888472555571,3.91583595246959 +162,5,3,-0.633058606184867,0.573737524170066,3.11203053101271 +163,1,3,-1.18786324515284,-0.340443408233712,1.14317643363866 +163,2,3,-1.18786324515284,-0.768032906626293,1.95833915002506 +163,3,3,-1.18786324515284,0.272259951460627,0.114323384349211 +163,4,3,-1.18786324515284,-0.298978178385548,0.61384379102693 +163,5,3,-1.18786324515284,0.637413178921166,5.50132822693627 +164,1,3,0.899797869140011,1.05596549859405,-0.21122143226474 +164,2,3,0.899797869140011,2.17406704550966,1.24046961464129 +164,3,3,0.899797869140011,1.44083698995944,2.66189961002648 +164,4,3,0.899797869140011,0.915916560712705,2.36672985820162 +164,5,3,0.899797869140011,1.38588314168802,5.02702750512009 +165,1,3,-0.301129927044137,0.752158340030402,-1.50907794920551 +165,2,3,-0.301129927044137,1.06436019651387,-0.62448442577378 +165,3,3,-0.301129927044137,0.384960393290526,0.832916152884812 +165,4,3,-0.301129927044137,0.949586844959261,1.88928233188054 +165,5,3,-0.301129927044137,0.943744899483462,2.01036225355556 +166,1,3,-0.549765513879748,0.651497636134973,0.935561103491787 +166,2,3,-0.549765513879748,-0.129357145730795,2.34756202467047 +166,3,3,-0.549765513879748,0.637365006242933,2.98981842673778 +166,4,3,-0.549765513879748,0.966660731480892,5.43125892856486 +166,5,3,-0.549765513879748,0.133694598822538,5.07759223972584 +167,1,3,-0.78547554309208,-0.134836724965572,-1.36799223380912 +167,2,3,-0.78547554309208,0.184023568082734,-0.0674588789252142 +167,3,3,-0.78547554309208,0.221814204202511,2.78941494261896 +167,4,3,-0.78547554309208,-0.0907598902618565,1.28630104303229 +167,5,3,-0.78547554309208,-0.490928636693926,3.28879673375415 +168,1,3,-1.16481614736776,-1.05549899084636,2.30664287448066 +168,2,3,-1.16481614736776,-0.109049141845168,0.403074535883857 +168,3,3,-1.16481614736776,0.408996655515945,2.96852326627862 +168,4,3,-1.16481614736776,1.13286892015195,3.26151734891858 +168,5,3,-1.16481614736776,0.25309773918563,4.60609542882273 +169,1,3,-0.100473245130853,-0.286277055576258,0.816176439613865 +169,2,3,-0.100473245130853,0.148238163475768,1.89100786414498 +169,3,3,-0.100473245130853,1.03809923428597,3.45061203241681 +169,4,3,-0.100473245130853,0.439631515666702,4.97362624547598 +169,5,3,-0.100473245130853,0.676649197592102,6.785372217849 +170,1,3,1.52370285962906,0.928353518230501,-1.74430422590474 +170,2,3,1.52370285962906,0.857441727058135,-1.77069380443786 +170,3,3,1.52370285962906,1.95225078764784,3.5723941518199 +170,4,3,1.52370285962906,0.174810994139092,3.22889090694693 +170,5,3,1.52370285962906,1.45240210098787,5.61292746676696 +171,1,3,1.70021766980079,0.845187574622548,2.88850505078726 +171,2,3,1.70021766980079,1.60306723055171,3.48529952975244 +171,3,3,1.70021766980079,2.10398199671952,2.99813281338445 +171,4,3,1.70021766980079,1.73711534632248,5.2631280122806 +171,5,3,1.70021766980079,0.948911760532366,8.4889015355624 +172,1,3,-0.382449184151253,0.601160041563296,-0.218836631756927 +172,2,3,-0.382449184151253,0.594478731981541,0.107666949516889 +172,3,3,-0.382449184151253,0.131925459706348,1.22736418934748 +172,4,3,-0.382449184151253,0.27205811111869,4.22513044903411 +172,5,3,-0.382449184151253,0.481932581280621,4.48764564425334 +173,1,3,-0.321864791232853,-0.526916540913429,-0.919112507738525 +173,2,3,-0.321864791232853,0.82545643954509,0.575624239613491 +173,3,3,-0.321864791232853,-0.471689102491135,1.89058232409286 +173,4,3,-0.321864791232853,1.13588994941228,3.41459357670557 +173,5,3,-0.321864791232853,0.36386567246704,5.03955541787637 +174,1,3,-0.976473495882479,0.711165565116071,0.282395370482445 +174,2,3,-0.976473495882479,-0.173011423957099,-0.569514663383517 +174,3,3,-0.976473495882479,-0.518998534076677,1.38191187101233 +174,4,3,-0.976473495882479,-0.107407599484349,2.48935378518072 +174,5,3,-0.976473495882479,-0.222395331439423,4.14571722953055 +175,1,3,0.206596723904414,-0.207059321698249,-0.0363719683260104 +175,2,3,0.206596723904414,0.481350779890947,-0.334257864561706 +175,3,3,0.206596723904414,0.723281523690698,3.17349996250998 +175,4,3,0.206596723904414,1.43357721571009,2.39430459844192 +175,5,3,0.206596723904414,0.591094458526448,4.36295883976816 +176,1,3,0.0577034421995391,0.709794129416285,-0.28756468129535 +176,2,3,0.0577034421995391,-0.143545223443117,2.92356539775918 +176,3,3,0.0577034421995391,1.56256666105461,3.9682478053146 +176,4,3,0.0577034421995391,0.606339899387379,4.5254491122782 +176,5,3,0.0577034421995391,0.76336680665267,8.17539594185283 +177,1,3,0.97012801697565,0.804576615031986,0.53560539768589 +177,2,3,0.97012801697565,0.196553211150512,1.36387006814793 +177,3,3,0.97012801697565,1.00301505882641,3.51400322417612 +177,4,3,0.97012801697565,0.271445191644487,5.04448566201533 +177,5,3,0.97012801697565,1.55312861154355,6.70569729949341 +178,1,3,-0.357064593741126,-0.0607403759752695,1.01233438166498 +178,2,3,-0.357064593741126,0.490630427379729,-0.312090709939029 +178,3,3,-0.357064593741126,-0.364345488254412,2.05738386473734 +178,4,3,-0.357064593741126,0.764552669846505,1.8568801615876 +178,5,3,-0.357064593741126,1.49628986724636,4.22922510038847 +179,1,3,1.83082274441214,1.10323167318757,1.2584912282798 +179,2,3,1.83082274441214,0.995153151371863,0.976468061563311 +179,3,3,1.83082274441214,1.03251198368829,2.88884149892202 +179,4,3,1.83082274441214,1.26989041677338,4.53978057520108 +179,5,3,1.83082274441214,2.27557816245241,9.01599947501831 +180,1,3,0.155573560993865,-0.192908369096136,1.27981156464175 +180,2,3,0.155573560993865,-0.157534180273984,2.81031818521695 +180,3,3,0.155573560993865,-0.182158192618231,3.42405164561696 +180,4,3,0.155573560993865,1.35094942132599,3.96796596950746 +180,5,3,0.155573560993865,1.90194981967979,8.03537055637734 +181,1,3,-0.561729182385448,-0.716364698575066,0.0258495576976853 +181,2,3,-0.561729182385448,0.0517589100032271,-1.9223798367341 +181,3,3,-0.561729182385448,0.155757851391865,2.14638687839844 +181,4,3,-0.561729182385448,1.47783449114265,2.58274092963544 +181,5,3,-0.561729182385448,0.870450644156802,3.24356808948639 +182,1,3,0.26725208660956,-0.0491357653714034,2.49956058511792 +182,2,3,0.26725208660956,-0.210909802338806,0.886149232902366 +182,3,3,0.26725208660956,0.622334850505215,3.8931379580875 +182,4,3,0.26725208660956,1.46665374250886,5.198204176303 +182,5,3,0.26725208660956,0.627533310561888,6.03883577577174 +183,1,3,-1.58651614698834,0.0078323858732432,2.93167490126052 +183,2,3,-1.58651614698834,-0.309478667711119,0.850162920976363 +183,3,3,-1.58651614698834,-0.931464591505026,3.28271352889749 +183,4,3,-1.58651614698834,0.331128687743081,2.80293524183476 +183,5,3,-1.58651614698834,0.235317940303293,1.89763913483971 +184,1,3,0.964008293737022,0.862304156601825,1.07052054416651 +184,2,3,0.964008293737022,0.78332774517736,2.54544572781402 +184,3,3,0.964008293737022,0.760983528069229,3.13566633991018 +184,4,3,0.964008293737022,1.86056654613075,5.340943408593 +184,5,3,0.964008293737022,1.53754550750828,7.19655087170372 +185,1,3,-0.136501349380018,-0.714009827475327,0.784769499960354 +185,2,3,-0.136501349380018,0.133897812852903,0.171777702203639 +185,3,3,-0.136501349380018,-0.263791603073048,0.962395384355854 +185,4,3,-0.136501349380018,1.19789754598816,2.59010782342184 +185,5,3,-0.136501349380018,0.268244919773508,3.92974278467078 +186,1,3,-1.28961941968184,-0.918430047518358,-1.9686296021114 +186,2,3,-1.28961941968184,-1.4415577155884,-0.148603099732199 +186,3,3,-1.28961941968184,1.16603286464104,0.160275393261301 +186,4,3,-1.28961941968184,-0.420341395456616,2.6351542975303 +186,5,3,-1.28961941968184,0.219569883299303,5.01919880111458 +187,1,3,-0.50487449632609,-0.397170453608418,1.91968990836625 +187,2,3,-0.50487449632609,0.310110929892259,1.93900182773006 +187,3,3,-0.50487449632609,-0.223467929354194,1.54844471705867 +187,4,3,-0.50487449632609,0.632636963181234,2.64265112445697 +187,5,3,-0.50487449632609,1.36338592016433,5.02622184660742 +188,1,3,0.346591684139401,0.147175470379114,-0.466845097919033 +188,2,3,0.346591684139401,1.03169374286638,0.659536188793482 +188,3,3,0.346591684139401,1.17291033726791,1.44307857044969 +188,4,3,0.346591684139401,0.9003299952002,2.15240892070531 +188,5,3,0.346591684139401,0.903593984244808,5.02830703671822 +189,1,3,0.576308110605711,-0.122536463758614,-1.57705029269863 +189,2,3,0.576308110605711,0.291477322026575,-0.306041213307675 +189,3,3,0.576308110605711,0.690232742116615,2.2422184539491 +189,4,3,0.576308110605711,1.01847578790693,1.08223612517947 +189,5,3,0.576308110605711,0.711335171990724,4.07221565330838 +190,1,3,2.08095198548021,1.33707232095089,1.3473200599755 +190,2,3,2.08095198548021,1.83156228593153,2.73102211717648 +190,3,3,2.08095198548021,0.526486445244764,3.26264599133572 +190,4,3,2.08095198548021,1.87485237391011,7.60163197735244 +190,5,3,2.08095198548021,1.4436691449854,8.69741127785079 +191,1,3,0.541741765731616,0.75677092222344,-0.17741464798554 +191,2,3,0.541741765731616,0.580456407073188,-0.352770440207899 +191,3,3,0.541741765731616,1.1237012894253,0.401484430278701 +191,4,3,0.541741765731616,0.771952065397894,5.02746098105613 +191,5,3,0.541741765731616,1.86655557642009,6.30572793910101 +192,1,3,0.374552810372194,0.606532770511559,2.05554214836543 +192,2,3,0.374552810372194,-0.514822382316804,2.18945756616643 +192,3,3,0.374552810372194,0.881168519061104,4.45925069297196 +192,4,3,0.374552810372194,0.812796538462421,6.0401274280228 +192,5,3,0.374552810372194,0.790236464746359,8.97534411362399 +193,1,3,-0.547755672644997,0.195225650839263,2.65504759756614 +193,2,3,-0.547755672644997,0.0367267682650421,-0.280690334987242 +193,3,3,-0.547755672644997,0.418469306023656,0.601330235148127 +193,4,3,-0.547755672644997,0.488693531216109,5.44972918275153 +193,5,3,-0.547755672644997,0.921971026069666,5.69678613783221 +194,1,3,0.823375237959988,0.794874722936144,-0.778832178556864 +194,2,3,0.823375237959988,1.12542250133637,-0.934186219579191 +194,3,3,0.823375237959988,1.22904636185405,2.74314586857224 +194,4,3,0.823375237959988,0.528522318252197,5.13677706748988 +194,5,3,0.823375237959988,1.4961081216668,6.19996985429195 +195,1,3,-2.28077468001876,-0.895318594048709,-1.24407433196296 +195,2,3,-2.28077468001876,-1.63435720128489,-0.86850217910119 +195,3,3,-2.28077468001876,-0.465149221884744,0.311909085828801 +195,4,3,-2.28077468001876,-0.0153283011291445,-0.250171464401362 +195,5,3,-2.28077468001876,0.402100989150701,0.679870171029218 +196,1,3,-0.877192455164409,0.0199183920553034,1.12042339769811 +196,2,3,-0.877192455164409,0.309966868712776,-0.870717661006155 +196,3,3,-0.877192455164409,-0.0191782684956241,0.804081512299063 +196,4,3,-0.877192455164409,1.184332013726,2.94402306952057 +196,5,3,-0.877192455164409,1.22883637027276,4.02831372207414 +197,1,3,-1.79625296210919,-0.361051529214706,-0.343292683318492 +197,2,3,-1.79625296210919,-0.384109190615947,1.42603749350341 +197,3,3,-1.79625296210919,-0.0226322056923884,2.55701916614874 +197,4,3,-1.79625296210919,-0.279036736874818,2.26910569930322 +197,5,3,-1.79625296210919,0.909628022790928,4.30698920325096 +198,1,3,0.294599695261137,0.89036589334075,1.69884316076478 +198,2,3,0.294599695261137,0.0796835929948345,2.69317132702638 +198,3,3,0.294599695261137,0.497516125839013,6.6276099186752 +198,4,3,0.294599695261137,0.354299212915527,7.44367073594493 +198,5,3,0.294599695261137,0.387027339411681,6.31279079228627 +199,1,3,1.03393476511463,1.05710791200614,-0.973952817405197 +199,2,3,1.03393476511463,1.5307145467393,0.234826291447174 +199,3,3,1.03393476511463,0.95979715049037,0.779861823865699 +199,4,3,1.03393476511463,1.57566721611891,0.355580473898519 +199,5,3,1.03393476511463,1.95098305906965,4.97771532813293 +200,1,3,0.67406226635374,0.275277717694543,0.843334384136715 +200,2,3,0.67406226635374,1.30090857740661,1.93264815873112 +200,3,3,0.67406226635374,0.555597032912415,1.0968307169484 +200,4,3,0.67406226635374,2.22816439869584,5.46923755530594 +200,5,3,0.67406226635374,1.61344654035566,7.23989637649208 +201,1,4,-0.145619005764009,0.425062464970386,0.261616790409442 +201,2,4,-0.145619005764009,0.868817131430003,-0.445132841776166 +201,3,4,-0.145619005764009,0.492502278198704,1.83029648571645 +201,4,4,-0.145619005764009,0.630758469484849,3.30405105563127 +201,5,4,-0.145619005764009,0.776041722729689,5.63701109236231 +202,1,4,0.20848748143,-0.194711403480027,0.179745580798258 +202,2,4,0.20848748143,0.492994024084207,1.13854396901761 +202,3,4,0.20848748143,0.783119039986885,2.92293105342569 +202,4,4,0.20848748143,0.299329049970995,4.60042133367869 +202,5,4,0.20848748143,1.26220961181419,5.89206406267787 +203,1,4,-0.0189341751210595,0.882938170134449,-2.05139873048146 +203,2,4,-0.0189341751210595,1.22759454414411,-0.0627311545998119 +203,3,4,-0.0189341751210595,0.408892095900875,-0.568041233080671 +203,4,4,-0.0189341751210595,0.436565159144488,0.272986052753453 +203,5,4,-0.0189341751210595,1.0238662931341,4.38776490089903 +204,1,4,0.527436038711728,0.697663366868311,1.93523689027915 +204,2,4,0.527436038711728,0.427325742523677,0.473254757727205 +204,3,4,0.527436038711728,2.00923508548696,0.622493693881969 +204,4,4,0.527436038711728,0.971970604102201,2.53441331906684 +204,5,4,0.527436038711728,0.778763771105946,5.68427368611851 +205,1,4,1.27844865328638,0.687467032299747,-0.380514602850353 +205,2,4,1.27844865328638,0.849429647247574,-0.681233299781682 +205,3,4,1.27844865328638,1.70010462355952,1.11108413627184 +205,4,4,1.27844865328638,0.655787819387356,3.39764474726244 +205,5,4,1.27844865328638,1.88135604224227,5.64744922059397 +206,1,4,-1.18246074137794,0.170036724255962,1.13871929843077 +206,2,4,-1.18246074137794,-0.12748806549754,-0.0532917062726999 +206,3,4,-1.18246074137794,-0.370786134857948,-0.896112659770101 +206,4,4,-1.18246074137794,-0.00441465296565557,3.02106930082714 +206,5,4,-1.18246074137794,1.42894983162112,4.63504222287801 +207,1,4,-0.157505532820991,0.716516078170618,0.509385682877153 +207,2,4,-0.157505532820991,0.000137832348927613,1.72928345426769 +207,3,4,-0.157505532820991,0.544829211905254,3.3758879541191 +207,4,4,-0.157505532820991,1.06084126726245,2.44803531226303 +207,5,4,-0.157505532820991,1.39705579795745,4.77175746613998 +208,1,4,0.397900614221911,1.00807136568749,-0.252075928949645 +208,2,4,0.397900614221911,0.838451756495337,-0.268883833866314 +208,3,4,0.397900614221911,0.826190093558104,3.20343190402967 +208,4,4,0.397900614221911,0.766221557988312,3.90633552973664 +208,5,4,0.397900614221911,1.24218805277057,6.43545801727386 +209,1,4,3.08818771202226,1.18057364412943,2.18396135142987 +209,2,4,3.08818771202226,2.27702919591625,3.55491931574783 +209,3,4,3.08818771202226,1.85263702901336,5.08918598891455 +209,4,4,3.08818771202226,2.78921862657327,6.17571524563542 +209,5,4,3.08818771202226,3.2225773233352,8.73776270096561 +210,1,4,1.68702207569785,1.7993545395904,0.532838840080434 +210,2,4,1.68702207569785,0.869440033595608,2.2410028229324 +210,3,4,1.68702207569785,1.01883836380461,2.13221062747574 +210,4,4,1.68702207569785,2.00583631958716,5.87886551140786 +210,5,4,1.68702207569785,1.70231546818835,8.79615064905242 +211,1,4,-0.875883372538378,-0.978958973674517,0.581621421483876 +211,2,4,-0.875883372538378,-0.786653411189003,1.41121195690622 +211,3,4,-0.875883372538378,-0.190563082665021,2.20404293055772 +211,4,4,-0.875883372538378,0.260808890938771,2.21556675699285 +211,5,4,-0.875883372538378,0.834906311643272,3.43606019304585 +212,1,4,1.30400386151833,-0.0739311314895021,-0.27259878117807 +212,2,4,1.30400386151833,1.81479648564854,2.13416997391053 +212,3,4,1.30400386151833,0.995640877361736,1.45154720287005 +212,4,4,1.30400386151833,1.30443724412722,3.63731216528277 +212,5,4,1.30400386151833,1.13207952550597,5.59816451674992 +213,1,4,-0.23259357883221,0.543204811836342,-1.01449534225791 +213,2,4,-0.23259357883221,0.12693829181683,-1.00590686596478 +213,3,4,-0.23259357883221,-0.550437546679332,-0.857902657994927 +213,4,4,-0.23259357883221,1.08009092657631,1.48411443232596 +213,5,4,-0.23259357883221,1.28748975003943,2.58372978563663 +214,1,4,0.934509368328794,0.283756358537217,3.21736850838587 +214,2,4,0.934509368328794,0.997296492121042,3.83994823577147 +214,3,4,0.934509368328794,0.404522074604913,3.87328670453522 +214,4,4,0.934509368328794,1.75594042974006,4.3940685244154 +214,5,4,0.934509368328794,1.15095624853172,5.93523237524799 +215,1,4,-2.02633774528942,-0.627275698118266,-1.29360704342256 +215,2,4,-2.02633774528942,-0.984718100457219,-0.77486879653039 +215,3,4,-2.02633774528942,-0.797652009536068,-3.57862337525148 +215,4,4,-2.02633774528942,-0.696679896135425,-0.940107496082131 +215,5,4,-2.02633774528942,-0.600475556059633,-0.057816328034735 +216,1,4,-1.1345723514721,-0.999801486681439,-1.99926954853096 +216,2,4,-1.1345723514721,0.051632320309278,-1.30478035939822 +216,3,4,-1.1345723514721,-0.218461416657432,-0.639274587840224 +216,4,4,-1.1345723514721,0.477947371104333,0.953246898586203 +216,5,4,-1.1345723514721,0.248014439178224,2.70084584246067 +217,1,4,0.29136391143389,0.183245908076356,1.68485761265153 +217,2,4,0.29136391143389,0.395426152640086,1.93259002639756 +217,3,4,0.29136391143389,0.872056192556017,0.908055760001004 +217,4,4,0.29136391143389,0.48010831096284,2.72864700809539 +217,5,4,0.29136391143389,0.766822417416036,4.90222656954324 +218,1,4,1.02025953266783,1.18979863184446,4.13767206136758 +218,2,4,1.02025953266783,1.31261695412382,3.3885702756416 +218,3,4,1.02025953266783,1.62271221719155,3.51247762596368 +218,4,4,1.02025953266783,2.30279001427239,6.44275765130315 +218,5,4,1.02025953266783,2.15392921059779,6.075212576793 +219,1,4,0.0658993928206432,0.790306891360803,-0.482614087946492 +219,2,4,0.0658993928206432,-0.349717346583336,1.52429755080546 +219,3,4,0.0658993928206432,0.56426858103447,3.24615220616319 +219,4,4,0.0658993928206432,1.20758231685881,2.95719739567898 +219,5,4,0.0658993928206432,0.327607937927172,7.82263208692572 +220,1,4,-1.02912673591929,-1.16919501498074,-3.13433236697156 +220,2,4,-1.02912673591929,0.434877627773687,-1.23305344836268 +220,3,4,-1.02912673591929,-0.797303788823903,-0.284062193305323 +220,4,4,-1.02912673591929,0.116308946916564,1.35017095093358 +220,5,4,-1.02912673591929,0.94393107694499,1.91875502569586 +221,1,4,-0.382443187726708,0.389597842296713,0.382879340799262 +221,2,4,-0.382443187726708,0.374049508941051,0.114914620326604 +221,3,4,-0.382443187726708,-0.127663034111751,1.20459217867908 +221,4,4,-0.382443187726708,0.293992048559745,2.99629254268107 +221,5,4,-0.382443187726708,1.36640484160858,5.19669505614587 +222,1,4,-2.07378709910601,-1.2515950543621,0.958666353461258 +222,2,4,-2.07378709910601,-0.827366544533211,0.931687279406236 +222,3,4,-2.07378709910601,0.2149092035418,1.61719600463625 +222,4,4,-2.07378709910601,-0.182078896920811,1.44021079569763 +222,5,4,-2.07378709910601,-1.05195888378581,1.57467042273008 +223,1,4,0.0571434861440279,0.947769560761134,1.0358412759774 +223,2,4,0.0571434861440279,0.41621436563672,0.292865445061844 +223,3,4,0.0571434861440279,1.48618542216699,3.40291389631824 +223,4,4,0.0571434861440279,0.762338360571163,1.96188222901659 +223,5,4,0.0571434861440279,0.177984948090528,4.52859209908139 +224,1,4,-0.190737942505499,-0.738255736922229,0.127569868577206 +224,2,4,-0.190737942505499,0.372240080607862,-0.12501447773974 +224,3,4,-0.190737942505499,0.619002253283965,0.887761699211636 +224,4,4,-0.190737942505499,-0.221048847835207,2.18896719978384 +224,5,4,-0.190737942505499,1.03986148135431,5.75670165761339 +225,1,4,0.504750370322883,-0.0984297837436904,1.67458725198859 +225,2,4,0.504750370322883,1.27231230071316,2.08663858390143 +225,3,4,0.504750370322883,0.322652963868896,2.95092850709882 +225,4,4,0.504750370322883,0.478947178775107,5.43233008713638 +225,5,4,0.504750370322883,0.825625553003256,8.74656660654164 +226,1,4,0.566646966396415,0.385509168954166,0.789949604386638 +226,2,4,0.566646966396415,0.761102906211001,0.77574412438031 +226,3,4,0.566646966396415,0.864967381215401,1.58861176824269 +226,4,4,0.566646966396415,1.51851886429129,3.61972237667015 +226,5,4,0.566646966396415,1.76621774098752,4.61367138253264 +227,1,4,0.416298605248852,0.241969846324854,0.461314108917958 +227,2,4,0.416298605248852,0.307258447403572,1.02689317889856 +227,3,4,0.416298605248852,0.389077651776476,0.410468139553837 +227,4,4,0.416298605248852,1.17249774667565,3.58318359838724 +227,5,4,0.416298605248852,1.09203011758049,3.99468544614685 +228,1,4,-1.8216741221751,-0.49545657763507,-0.918010485280072 +228,2,4,-1.8216741221751,0.027308409221532,1.68188149042672 +228,3,4,-1.8216741221751,-0.594474408950622,1.80605678566885 +228,4,4,-1.8216741221751,-0.541947413609883,1.61611304815113 +228,5,4,-1.8216741221751,0.627253636608009,2.48045304876612 +229,1,4,1.95837805386369,1.21634091921974,-0.462827312292844 +229,2,4,1.95837805386369,1.74988569899565,0.945110855282057 +229,3,4,1.95837805386369,1.52757147585732,2.50011393068116 +229,4,4,1.95837805386369,1.99132635448695,4.01187377755918 +229,5,4,1.95837805386369,1.31915164171884,5.35579248615526 +230,1,4,-0.854064145631843,-1.41460473600476,0.144806847230399 +230,2,4,-0.854064145631843,0.662825868088806,1.89877543200894 +230,3,4,-0.854064145631843,-0.652182540520576,0.308075121258965 +230,4,4,-0.854064145631843,0.68245012625534,1.76009392748307 +230,5,4,-0.854064145631843,1.61509807716483,2.02400699601978 +231,1,4,-1.83315433652527,-0.751082840375815,-0.520185148961388 +231,2,4,-1.83315433652527,-0.88514688326197,1.65944375180379 +231,3,4,-1.83315433652527,-0.295660470843824,-0.907336878398132 +231,4,4,-1.83315433652527,-0.495792414559182,-0.225431070997175 +231,5,4,-1.83315433652527,0.0418227109791764,2.81883558740669 +232,1,4,1.24837686380501,0.838224876289186,-0.621704789994096 +232,2,4,1.24837686380501,2.25510434548755,0.328480973937103 +232,3,4,1.24837686380501,2.10481475648221,2.88086825041376 +232,4,4,1.24837686380501,1.42429405303522,3.36866842489618 +232,5,4,1.24837686380501,1.05793232096196,6.97140678219277 +233,1,4,2.85114953488483,1.29139647396778,0.99238604179257 +233,2,4,2.85114953488483,1.8705653471451,-0.134331908602972 +233,3,4,2.85114953488483,2.20740279929195,3.07499052105055 +233,4,4,2.85114953488483,2.15171879921461,5.39633768858733 +233,5,4,2.85114953488483,2.80830718066296,8.80251164018261 +234,1,4,-0.936743330351871,-0.0945872932981996,-0.141865793981232 +234,2,4,-0.936743330351871,0.574562882075806,0.403537407215633 +234,3,4,-0.936743330351871,0.0997238820429076,-1.1903911691781 +234,4,4,-0.936743330351871,0.361027224824763,0.762696042186893 +234,5,4,-0.936743330351871,0.557639499328201,3.13537313460174 +235,1,4,1.47580185605354,0.00281937707094781,2.6165660705684 +235,2,4,1.47580185605354,1.34854747593014,2.41643446713607 +235,3,4,1.47580185605354,1.99294917196747,2.43476543091642 +235,4,4,1.47580185605354,1.6967718166645,5.85878317716718 +235,5,4,1.47580185605354,1.95520672221328,8.07589311734043 +236,1,4,0.208747033908035,0.0745711285520664,1.31236043050198 +236,2,4,0.208747033908035,1.20431328156382,1.33371052800362 +236,3,4,0.208747033908035,-0.0523967515138719,2.31728577207176 +236,4,4,0.208747033908035,0.307677941881186,3.09254540576663 +236,5,4,0.208747033908035,0.894146130541882,2.88336258223656 +237,1,4,0.896059204631448,1.18810550586458,1.30092916076486 +237,2,4,0.896059204631448,0.454227458333857,1.52099651693863 +237,3,4,0.896059204631448,1.35758023985095,1.35051819874402 +237,4,4,0.896059204631448,0.331892873266859,5.76870679312247 +237,5,4,0.896059204631448,1.41000502507945,6.23106759672651 +238,1,4,0.113951054067604,-0.220492161787358,0.481424354940191 +238,2,4,0.113951054067604,0.811282363216797,0.725121018412195 +238,3,4,0.113951054067604,-0.34398335882458,-1.56392926823064 +238,4,4,0.113951054067604,1.53352069000688,1.92056368213871 +238,5,4,0.113951054067604,1.65603526197291,3.2588267214286 +239,1,4,0.332304485256041,1.15620286359608,-1.6681412354686 +239,2,4,0.332304485256041,0.747202621907737,-0.314622856456936 +239,3,4,0.332304485256041,0.334322005229968,-0.308578372252669 +239,4,4,0.332304485256041,1.42756911312914,2.47095762222018 +239,5,4,0.332304485256041,0.5879033667627,3.50822725475215 +240,1,4,0.00970594683895151,-0.0704452216851235,3.04122955188286 +240,2,4,0.00970594683895151,0.76395694446961,2.95416927363478 +240,3,4,0.00970594683895151,-0.046594574847339,2.45553375362412 +240,4,4,0.00970594683895151,0.769993965782161,3.94403688734056 +240,5,4,0.00970594683895151,-0.443491592376686,5.34311300206848 +241,1,4,-0.223773244924263,-0.65278284105429,-1.48828544189116 +241,2,4,-0.223773244924263,0.593489908952763,0.370330080113892 +241,3,4,-0.223773244924263,0.545104944921187,0.891140750941535 +241,4,4,-0.223773244924263,1.64379566963664,3.74751639708699 +241,5,4,-0.223773244924263,0.846644395352223,4.99132096127366 +242,1,4,0.311444780258236,1.48178803288983,-0.380767459864371 +242,2,4,0.311444780258236,1.2734006735771,0.0259808435259069 +242,3,4,0.311444780258236,-0.109772227045998,1.50016359137237 +242,4,4,0.311444780258236,0.593862883300123,2.71532805813609 +242,5,4,0.311444780258236,1.72865594619768,3.97334193486166 +243,1,4,0.341619072387833,-0.207498945762728,-0.820909073821433 +243,2,4,0.341619072387833,0.385145039448137,-0.841004612792712 +243,3,4,0.341619072387833,0.368659884620509,0.0844219376525748 +243,4,4,0.341619072387833,1.22635035866907,0.952682324592303 +243,5,4,0.341619072387833,1.76880167951173,4.90943609337553 +244,1,4,-2.04126212237833,-1.60140313192775,0.663827237102985 +244,2,4,-2.04126212237833,-0.930248302703456,1.58493805703856 +244,3,4,-2.04126212237833,0.346794972361608,0.410523261284975 +244,4,4,-2.04126212237833,-0.401524382975107,1.79390821724782 +244,5,4,-2.04126212237833,0.273694848325163,1.61289090300176 +245,1,4,-0.36739874796181,0.384491144940372,-0.570341416585646 +245,2,4,-0.36739874796181,-0.0554327743052686,1.77983314229872 +245,3,4,-0.36739874796181,1.17310166519544,1.2281890200107 +245,4,4,-0.36739874796181,1.24012454957959,2.79092718189971 +245,5,4,-0.36739874796181,0.779841451650018,4.30924212933595 +246,1,4,-0.231984127628662,0.476599317012363,0.43657726339478 +246,2,4,-0.231984127628662,0.248843711953803,0.863012870447415 +246,3,4,-0.231984127628662,0.274345464327848,1.55119128297576 +246,4,4,-0.231984127628662,0.473713731438912,3.09769523693271 +246,5,4,-0.231984127628662,0.222520430314378,3.86862426455256 +247,1,4,-0.0956398371392947,0.149491999232215,0.598097954992508 +247,2,4,-0.0956398371392947,0.022604092587618,-0.0664439657673578 +247,3,4,-0.0956398371392947,0.220212399721201,0.524357762567856 +247,4,4,-0.0956398371392947,1.00365977468176,2.46031605710149 +247,5,4,-0.0956398371392947,1.28279241725496,4.85795540887373 +248,1,4,-1.70619282588905,-0.645361101889452,1.9916344801831 +248,2,4,-1.70619282588905,-0.862927516651366,3.66791717271056 +248,3,4,-1.70619282588905,0.0127762391683118,0.609025656290565 +248,4,4,-1.70619282588905,-0.658014594679568,3.18183436799844 +248,5,4,-1.70619282588905,-0.072577383389055,2.07262299675295 +249,1,4,-0.927336330190315,0.0705125883849166,2.91410539149122 +249,2,4,-0.927336330190315,-0.151899489836396,2.18979215037809 +249,3,4,-0.927336330190315,-0.42190651916858,1.65888100186867 +249,4,4,-0.927336330190315,0.460398446536725,4.08669056464587 +249,5,4,-0.927336330190315,0.600612524859274,5.90915200490956 +250,1,4,-1.14162801235656,0.206590383641819,0.28352920602639 +250,2,4,-1.14162801235656,-0.47704255454694,1.449317489013 +250,3,4,-1.14162801235656,-0.263536574659807,-0.4116398920302 +250,4,4,-1.14162801235656,0.292921465867566,0.36941772834878 +250,5,4,-1.14162801235656,0.326599649673672,3.25456142230352 +251,1,4,-0.533133577076554,0.331363034011417,-1.68870149736653 +251,2,4,-0.533133577076554,-0.117821643463392,0.857656318646477 +251,3,4,-0.533133577076554,1.30481063021976,0.226720234297566 +251,4,4,-0.533133577076554,-0.0456515009391002,1.62908564662487 +251,5,4,-0.533133577076554,0.336734673783287,2.6538556184386 +252,1,4,-1.4242396017973,-0.616487643209825,-1.16197382550618 +252,2,4,-1.4242396017973,0.809377735235426,0.0307873435046787 +252,3,4,-1.4242396017973,0.105432878183446,-0.0608605853037751 +252,4,4,-1.4242396017973,-0.0919608429902865,2.1255165629107 +252,5,4,-1.4242396017973,1.14213687390222,1.39823536766718 +253,1,4,0.0421130196327412,1.03467783928002,-1.11463016065861 +253,2,4,0.0421130196327412,0.426069218333878,-0.766654289126376 +253,3,4,0.0421130196327412,0.434427914234819,1.22879798728331 +253,4,4,0.0421130196327412,0.956344900243424,1.95082296497428 +253,5,4,0.0421130196327412,0.995569508719923,6.3550670944955 +254,1,4,1.26480281838255,1.44835886628602,1.17559904360065 +254,2,4,1.26480281838255,0.882706330710806,1.49438786415408 +254,3,4,1.26480281838255,1.32459160552475,2.15149912950138 +254,4,4,1.26480281838255,0.784870571407219,2.86896633076918 +254,5,4,1.26480281838255,1.15274927647445,6.9490189531146 +255,1,4,-1.90012999787815,0.673194082270379,-0.096756989707168 +255,2,4,-1.90012999787815,-1.02133154445786,0.000225348399189085 +255,3,4,-1.90012999787815,-0.50678712210121,-1.02312348114859 +255,4,4,-1.90012999787815,0.542192852658531,0.997428462853665 +255,5,4,-1.90012999787815,0.273626936566653,0.92625656443509 +256,1,4,-0.252346344034527,1.03251230536201,0.683415454947388 +256,2,4,-0.252346344034527,0.161490907558304,0.168102835634015 +256,3,4,-0.252346344034527,0.498605839954082,1.01539631672537 +256,4,4,-0.252346344034527,0.110756718531093,2.9865347314332 +256,5,4,-0.252346344034527,0.798434708656316,5.63224386538034 +257,1,4,0.295978757655662,0.216684978603956,-1.77045143244746 +257,2,4,0.295978757655662,0.0586862830669549,1.44714523155237 +257,3,4,0.295978757655662,0.523459956256675,0.538358101141446 +257,4,4,0.295978757655662,1.56990095241408,2.09305803603714 +257,5,4,0.295978757655662,1.22199253431524,3.45003271456258 +258,1,4,1.45826398970798,1.73386027058223,-0.407308799164423 +258,2,4,1.45826398970798,1.61805486454142,-0.604501564966646 +258,3,4,1.45826398970798,2.15750962591565,-0.0310382347093021 +258,4,4,1.45826398970798,1.2065993232119,2.57988120880643 +258,5,4,1.45826398970798,2.3927575283945,4.378101946541 +259,1,4,-1.74927828242113,-0.9582717131597,-0.291874873747854 +259,2,4,-1.74927828242113,-0.0841825243742936,2.55602592722196 +259,3,4,-1.74927828242113,-0.66186210802338,1.08527973151208 +259,4,4,-1.74927828242113,-1.08468288544549,0.00776297528025038 +259,5,4,-1.74927828242113,0.0597058834678753,2.38968504577223 +260,1,4,-0.137485383994025,0.792215203761203,-1.10257299306267 +260,2,4,-0.137485383994025,0.0812225692570001,0.457900169155973 +260,3,4,-0.137485383994025,0.0805787039345165,1.08921496380089 +260,4,4,-0.137485383994025,0.443699607906665,3.65248584537344 +260,5,4,-0.137485383994025,0.574308585361864,2.1595938602977 +261,1,4,0.499633628174919,1.13992947289452,-1.70794926705912 +261,2,4,0.499633628174919,0.522811863197591,1.86087969706738 +261,3,4,0.499633628174919,0.56806718260539,2.17218766492301 +261,4,4,0.499633628174919,1.01730076322439,1.34763443391338 +261,5,4,0.499633628174919,1.47679969319364,2.79623267918103 +262,1,4,1.08067198678011,1.17717605152764,1.20733511818297 +262,2,4,1.08067198678011,1.1751219890645,2.17608523804642 +262,3,4,1.08067198678011,1.701187483165,3.52833996545476 +262,4,4,1.08067198678011,1.41892071374161,4.73777070820504 +262,5,4,1.08067198678011,1.7323885465477,7.84386251335008 +263,1,4,-0.56060719046659,0.105163095142194,1.2071490768394 +263,2,4,-0.56060719046659,0.0198785644593,0.831932700091331 +263,3,4,-0.56060719046659,0.449070786540603,1.66734205810187 +263,4,4,-0.56060719046659,-0.0394458634330443,2.59047103292563 +263,5,4,-0.56060719046659,0.678285785599231,3.15533257937058 +264,1,4,0.0907057898496138,-0.47278090245472,1.58095980850717 +264,2,4,0.0907057898496138,0.446766877366091,-1.38279888216449 +264,3,4,0.0907057898496138,0.544943667505921,2.71674869734128 +264,4,4,0.0907057898496138,0.76573621541547,2.96574195549868 +264,5,4,0.0907057898496138,1.41895561912996,1.95434593015983 +265,1,4,1.08467352421369,1.3456816273122,-0.708797764954616 +265,2,4,1.08467352421369,1.08450782081211,-1.78097508051042 +265,3,4,1.08467352421369,1.39961924915943,-1.79699087295665 +265,4,4,1.08467352421369,1.17295798482399,4.10392366788344 +265,5,4,1.08467352421369,1.25389946236374,3.77002388463445 +266,1,4,-0.663235486302357,-0.156636189295917,-0.913206940327043 +266,2,4,-0.663235486302357,-0.157832130555018,-1.0445091310822 +266,3,4,-0.663235486302357,1.16715836241289,0.447228752399067 +266,4,4,-0.663235486302357,0.216898343439123,1.26839928522657 +266,5,4,-0.663235486302357,0.953672436944933,2.48130106007881 +267,1,4,0.507544490493503,0.417972511745019,0.941617354075974 +267,2,4,0.507544490493503,1.18950643981523,1.9862507863618 +267,3,4,0.507544490493503,0.371223098149939,2.95823701428986 +267,4,4,0.507544490493503,1.54379395353631,4.3867244894278 +267,5,4,0.507544490493503,1.78172598951428,7.97799074265316 +268,1,4,-1.58517361075372,-0.449522195882874,0.360124536502925 +268,2,4,-1.58517361075372,-0.636906574393382,-0.0564251355860901 +268,3,4,-1.58517361075372,-0.498536311791312,-0.784646559316506 +268,4,4,-1.58517361075372,0.215602396572638,-0.0850944418558538 +268,5,4,-1.58517361075372,-0.190835436734482,1.64915421342684 +269,1,4,-0.451581849130411,0.788061274122034,0.443360546254768 +269,2,4,-0.451581849130411,-0.255544152496929,0.0991548889815735 +269,3,4,-0.451581849130411,-0.0456743309929889,-0.0154119586940922 +269,4,4,-0.451581849130411,0.863064056517142,1.38119858118668 +269,5,4,-0.451581849130411,0.375685752463198,3.70360413505207 +270,1,4,0.291875229266022,1.16560069998419,-0.0623894341601451 +270,2,4,0.291875229266022,0.478483492965743,1.60919552841901 +270,3,4,0.291875229266022,0.893078366696526,2.0977066590772 +270,4,4,0.291875229266022,1.55374186798575,2.24487890667299 +270,5,4,0.291875229266022,0.608647637970957,3.53143962307269 +271,1,4,0.0779558094282166,0.341207463349526,1.23452468056018 +271,2,4,0.0779558094282166,0.952023617349011,0.641523269725003 +271,3,4,0.0779558094282166,0.809678561125316,1.4314431131351 +271,4,4,0.0779558094282166,0.998192867745152,1.96103023715827 +271,5,4,0.0779558094282166,0.727489746127947,4.93574553674648 +272,1,4,0.302325606233118,-0.313005897459611,2.22744223449703 +272,2,4,0.302325606233118,1.26633743489874,0.947006120888524 +272,3,4,0.302325606233118,0.456127163622586,0.915729413013545 +272,4,4,0.302325606233118,0.613026738962035,2.78755041110836 +272,5,4,0.302325606233118,1.51074045322368,4.97690784001007 +273,1,4,-0.598886389459515,-1.23214633546578,2.85034802997392 +273,2,4,-0.598886389459515,0.469231509795185,2.28936950149414 +273,3,4,-0.598886389459515,0.0653801138614925,5.32178136606866 +273,4,4,-0.598886389459515,1.01017905748285,3.99470307015833 +273,5,4,-0.598886389459515,1.47174857219337,5.94702329102721 +274,1,4,-0.67930601368561,0.636870921277184,1.62418182853493 +274,2,4,-0.67930601368561,0.10978632365971,-0.016826567941179 +274,3,4,-0.67930601368561,1.28079905770633,-0.540404005824146 +274,4,4,-0.67930601368561,0.580204977086341,3.20791975425798 +274,5,4,-0.67930601368561,0.429730028989504,4.02565224395567 +275,1,4,-0.11883036286453,0.0987066122506285,0.341097128148806 +275,2,4,-0.11883036286453,0.333640883169436,0.775998225006508 +275,3,4,-0.11883036286453,0.118838622926395,1.93875270343894 +275,4,4,-0.11883036286453,0.663691666488021,2.02147912609745 +275,5,4,-0.11883036286453,0.776990997101618,3.70891660244939 +276,1,4,-1.05250893889619,0.207706706604471,1.55724848283629 +276,2,4,-1.05250893889619,-0.331494328078079,1.9362374432695 +276,3,4,-1.05250893889619,-0.319789048714261,2.05616909833095 +276,4,4,-1.05250893889619,-0.143506203445625,2.9038522125567 +276,5,4,-1.05250893889619,0.799605265472782,4.45063561169586 +277,1,4,-0.795844037443362,0.19773315538162,-1.8378169583471 +277,2,4,-0.795844037443362,0.270395158483434,0.6880310785137 +277,3,4,-0.795844037443362,0.472347160923808,1.47613775270217 +277,4,4,-0.795844037443362,-0.768088395024826,2.51168159079353 +277,5,4,-0.795844037443362,0.547199254652822,4.15777050391864 +278,1,4,0.173615046018517,0.965776968467884,1.13520464623337 +278,2,4,0.173615046018517,0.588715357259776,1.60995251889149 +278,3,4,0.173615046018517,0.503368361124506,1.16655823323351 +278,4,4,0.173615046018517,0.299150390000537,4.05486954578688 +278,5,4,0.173615046018517,0.827024303091238,4.9639610608935 +279,1,4,0.0416951645261407,0.866967622081922,1.18077004074956 +279,2,4,0.0416951645261407,0.0551413833702407,-0.0854470487913126 +279,3,4,0.0416951645261407,1.53638709831634,1.26794253836204 +279,4,4,0.0416951645261407,1.13716982550896,2.83178478428392 +279,5,4,0.0416951645261407,1.12145927780378,5.62157478688342 +280,1,4,1.7584858414703,0.539505404090267,1.42838736347038 +280,2,4,1.7584858414703,1.99518281921814,0.520484304504163 +280,3,4,1.7584858414703,1.29540090344877,3.68015674163707 +280,4,4,1.7584858414703,2.616567296328,5.95731680505034 +280,5,4,1.7584858414703,1.86573904906666,8.38399941725952 +281,1,4,0.294307873629322,0.367016285926457,0.290612600985405 +281,2,4,0.294307873629322,0.255139421662298,1.35557946058075 +281,3,4,0.294307873629322,1.89254428009209,0.47947229670969 +281,4,4,0.294307873629322,0.756565102470336,5.02955675119337 +281,5,4,0.294307873629322,0.59436204587968,5.65770391957791 +282,1,4,-0.668920427496635,-0.16090516774787,-0.0220861603980491 +282,2,4,-0.668920427496635,-0.293932941387419,-0.860657413226595 +282,3,4,-0.668920427496635,-0.617888639151153,-1.15166725712085 +282,4,4,-0.668920427496635,0.888558954552449,2.92589475136101 +282,5,4,-0.668920427496635,0.966788957178054,2.38434469294651 +283,1,4,-0.147887549805199,1.21883984593654,0.201493397033268 +283,2,4,-0.147887549805199,-0.104507144604202,1.05560905284092 +283,3,4,-0.147887549805199,0.644480146848447,2.5893567926083 +283,4,4,-0.147887549805199,0.689421659305855,2.97529728089224 +283,5,4,-0.147887549805199,0.0146694646590244,2.7385080198275 +284,1,4,1.44872204277006,1.52566993336519,-1.09833942670499 +284,2,4,1.44872204277006,1.13121617916432,-0.763523549678274 +284,3,4,1.44872204277006,1.02618553930428,2.14434055999589 +284,4,4,1.44872204277006,2.32644315791059,4.70517765580124 +284,5,4,1.44872204277006,1.6632694223249,4.67184316918207 +285,1,4,-1.73133492647527,-1.1333333954352,0.530923214738404 +285,2,4,-1.73133492647527,-0.0797624768335474,2.04554158053777 +285,3,4,-1.73133492647527,0.279531137402414,1.22563072312665 +285,4,4,-1.73133492647527,0.324317655202742,1.53333743270017 +285,5,4,-1.73133492647527,-0.0842300049235476,3.21185361312661 +286,1,4,0.573428126344241,0.794394353051903,4.01969753025436 +286,2,4,0.573428126344241,0.283988223371164,5.20447479100824 +286,3,4,0.573428126344241,0.822028322194829,4.55993273559606 +286,4,4,0.573428126344241,0.36957610586572,4.07308490588507 +286,5,4,0.573428126344241,0.461021582718634,8.45361684973381 +287,1,4,1.49761969936007,0.38082553869154,3.75268269573446 +287,2,4,1.49761969936007,1.89226501243119,2.33915199737279 +287,3,4,1.49761969936007,2.11120221547793,3.87823256197682 +287,4,4,1.49761969936007,0.575220036330622,7.54075988529905 +287,5,4,1.49761969936007,1.56730512370106,10.87439030553 +288,1,4,0.120198304783001,0.221838332294035,-0.0343789585175994 +288,2,4,0.120198304783001,0.912273335999318,0.486857046923631 +288,3,4,0.120198304783001,0.46617645674522,-0.324957673735775 +288,4,4,0.120198304783001,0.833832779999747,1.28965171934442 +288,5,4,0.120198304783001,1.97307799051121,2.86679713845684 +289,1,4,-1.21503732454083,0.242954873846912,2.83881937447877 +289,2,4,-1.21503732454083,-0.0432160852840555,0.078823737728758 +289,3,4,-1.21503732454083,-0.427710871312259,0.70638669292398 +289,4,4,-1.21503732454083,-0.556300707912749,0.928782733937543 +289,5,4,-1.21503732454083,-0.135667200525247,2.68144692998541 +290,1,4,-1.1571502810733,0.210540410190643,0.338661393530131 +290,2,4,-1.1571502810733,-0.539918911846132,1.72072318693133 +290,3,4,-1.1571502810733,0.484706650698575,1.59241521439714 +290,4,4,-1.1571502810733,-0.405062434886484,4.11562329467502 +290,5,4,-1.1571502810733,0.981369158137991,3.20289878600007 +291,1,4,-0.373521121162934,0.0230948219244378,1.26027391139345 +291,2,4,-0.373521121162934,-0.256186240139986,2.11891529051296 +291,3,4,-0.373521121162934,-0.441530085576679,1.62632162393161 +291,4,4,-0.373521121162934,0.838881189609289,2.97479271232015 +291,5,4,-0.373521121162934,0.445423620665624,5.739502917094 +292,1,4,0.893632838246333,0.649527073294073,-0.0466265337101809 +292,2,4,0.893632838246333,1.23151636406837,2.32570939009992 +292,3,4,0.893632838246333,1.67383795568239,2.78685583010959 +292,4,4,0.893632838246333,1.18427970593881,2.58308217320773 +292,5,4,0.893632838246333,1.10217795369724,4.17545088830462 +293,1,4,-0.17329984570749,-0.495703964986854,0.426598483323649 +293,2,4,-0.17329984570749,-0.20694697270213,1.43844736703838 +293,3,4,-0.17329984570749,0.545163558474171,3.17427049488256 +293,4,4,-0.17329984570749,0.401595214074252,3.6781335251007 +293,5,4,-0.17329984570749,0.177816834539147,3.23938527767826 +294,1,4,0.326497399280871,-0.13472193785092,1.25809685074472 +294,2,4,0.326497399280871,0.034022762598704,2.84557217537418 +294,3,4,0.326497399280871,0.598162799541413,1.46014716927973 +294,4,4,0.326497399280871,0.126200469466238,3.65678460464985 +294,5,4,0.326497399280871,0.164798225218632,5.61297214487013 +295,1,4,-1.69765547653391,-0.154280889366906,2.75895647414194 +295,2,4,-1.69765547653391,0.688766738833268,2.00287035833896 +295,3,4,-1.69765547653391,-0.476175631328628,2.83643637564895 +295,4,4,-1.69765547653391,-0.427259753858562,3.00758562895917 +295,5,4,-1.69765547653391,0.0358480577355416,4.42414645411934 +296,1,4,0.633650933641627,-0.821113587145336,-0.195984269286924 +296,2,4,0.633650933641627,-0.0816294051250152,0.955935691449413 +296,3,4,0.633650933641627,0.46272800410014,3.04460414246167 +296,4,4,0.633650933641627,1.24145014749022,2.25349804152533 +296,5,4,0.633650933641627,0.91046353957579,4.10649106176556 +297,1,4,0.573362320259365,0.211954671830595,-1.22089638368291 +297,2,4,0.573362320259365,0.531850561306936,-1.36805062084361 +297,3,4,0.573362320259365,0.294502809667766,-0.368782377037484 +297,4,4,0.573362320259365,1.6726363990039,3.37390164758102 +297,5,4,0.573362320259365,0.904425320734592,4.48803997278305 +298,1,4,0.777482844206898,0.532731775655578,-0.69093420369047 +298,2,4,0.777482844206898,1.23200658222154,2.17322151719419 +298,3,4,0.777482844206898,1.11553239844212,1.00522514103251 +298,4,4,0.777482844206898,1.50951732571487,2.41323059679681 +298,5,4,0.777482844206898,1.70040147775544,3.75537833523442 +299,1,4,0.662331180984266,0.822473966761673,-0.573524004218574 +299,2,4,0.662331180984266,-0.150138919613316,0.704459464206402 +299,3,4,0.662331180984266,0.912328456619516,2.77438102760412 +299,4,4,0.662331180984266,0.677139268370734,3.18581770685075 +299,5,4,0.662331180984266,1.45162176721901,3.32255926170751 +300,1,4,-1.7765642769854,0.0682655892671309,-2.41606241019692 +300,2,4,-1.7765642769854,-0.596264264781815,-2.58896938493558 +300,3,4,-1.7765642769854,0.244954817789035,-2.86925495415941 +300,4,4,-1.7765642769854,0.716148966232101,0.454508967483654 +300,5,4,-1.7765642769854,-0.508267542335298,0.917821961357774 diff --git a/benchmarks/data/twfeweights_unbalanced_panel.csv b/benchmarks/data/twfeweights_unbalanced_panel.csv new file mode 100644 index 000000000..4283b98c8 --- /dev/null +++ b/benchmarks/data/twfeweights_unbalanced_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,0.762280032796771,0.22170929102064,0.32572261663086 +1,2,0,0.762280032796771,1.68545437574437,0.32042267370171 +1,3,0,0.762280032796771,0.783974881767155,1.70068782526904 +1,4,0,0.762280032796771,0.151811746643227,2.97713297459989 +1,5,0,0.762280032796771,1.2007958635123,2.34523214422804 +1,6,0,0.762280032796771,1.78117297800612,3.02378983652724 +2,1,0,-0.503334604664531,0.003264242668709,1.68714543541815 +2,2,0,-0.503334604664531,-0.01160465898369,2.11519135601787 +2,3,0,-0.503334604664531,0.431326801314968,2.52345814716303 +2,4,0,-0.503334604664531,0.770960476173647,1.13187660893405 +2,5,0,-0.503334604664531,0.557599780535241,3.50475895194589 +2,6,0,-0.503334604664531,0.882451795015903,2.56076108897434 +3,1,0,0.639896047927709,-0.0821545846502019,0.575850883278317 +3,2,0,0.639896047927709,0.420510283615455,1.46869634270462 +3,3,0,0.639896047927709,1.13004442178635,1.12926487305912 +3,4,0,0.639896047927709,1.54970030934366,3.75188178767779 +3,5,0,0.639896047927709,1.33314318152428,3.63591530837704 +3,6,0,0.639896047927709,2.22273056340426,4.58792223153239 +4,1,0,0.550362540546485,0.365760271855874,1.09027190546647 +4,2,0,0.550362540546485,0.832282303943132,3.26795580759371 +4,3,0,0.550362540546485,0.308471950710698,2.58526381483487 +4,4,0,0.550362540546485,1.56094152897056,2.56528027385053 +4,5,0,0.550362540546485,0.975725496813423,4.77071046622743 +4,6,0,0.550362540546485,2.03906796710312,2.64882999446538 +5,1,0,-1.29901420371325,-0.460999451990144,-0.354880543831845 +5,2,0,-1.29901420371325,0.501202768546227,1.69427801668797 +5,3,0,-1.29901420371325,-0.71046414492934,-0.134441277863296 +5,4,0,-1.29901420371325,-0.404803176271545,0.955977315050681 +5,5,0,-1.29901420371325,1.19643256869914,0.94227680755701 +5,6,0,-1.29901420371325,0.302674962248639,0.661806868491084 +6,1,0,-0.28270754060892,0.509984546718241,-1.60615679868968 +6,2,0,-0.28270754060892,0.795840872958493,1.42057064874697 +6,3,0,-0.28270754060892,0.498743320252149,2.56986086833164 +6,4,0,-0.28270754060892,0.444741791228927,2.43785042499056 +6,5,0,-0.28270754060892,0.0659811927562748,2.03911739958511 +6,6,0,-0.28270754060892,0.434119468978059,2.66364554745197 +7,1,0,-0.228457094854756,-0.125665405042128,-1.50029270012919 +7,2,0,-0.228457094854756,0.579836444928065,-0.271374116596818 +7,3,0,-0.228457094854756,-0.125065661168389,0.134719536136669 +7,4,0,-0.228457094854756,0.540215036454454,1.27719558483555 +7,5,0,-0.228457094854756,0.78024096522406,1.91860134504533 +7,6,0,-0.228457094854756,1.08320687793625,2.14194113990727 +8,1,0,0.904243934680486,1.63054948591081,2.44848344757429 +8,2,0,0.904243934680486,1.12467480171044,1.73465701246783 +8,3,0,0.904243934680486,1.14298117599048,3.13058617549245 +8,4,0,0.904243934680486,1.34982399747455,5.14687438054722 +8,5,0,0.904243934680486,1.24058911320076,5.20947125423497 +8,6,0,0.904243934680486,1.91183461909463,6.66336910996244 +9,1,0,1.0703229218455,0.74531880575199,-1.01317106446202 +9,2,0,1.0703229218455,1.76213986405735,0.295535989411215 +9,3,0,1.0703229218455,0.996599825632608,3.73789484050547 +9,4,0,1.0703229218455,1.5717667357908,3.12292622990361 +9,5,0,1.0703229218455,1.53030746151368,3.66172079419978 +9,6,0,1.0703229218455,2.28600306767279,4.01915059847461 +10,1,0,-0.143983755489064,-0.231058747953471,0.202498691570403 +10,2,0,-0.143983755489064,-0.459455998164224,2.02623199895792 +10,3,0,-0.143983755489064,0.696408286328998,1.20596875375505 +10,4,0,-0.143983755489064,-0.00560727561101249,2.99415578809309 +10,5,0,-0.143983755489064,1.05054222823483,1.7483927621027 +10,6,0,-0.143983755489064,2.11038914740653,1.80106981071404 +11,1,0,0.586058897064396,0.547033869273745,-0.104895161161322 +11,2,0,0.586058897064396,0.782832252459021,-0.858059059928484 +11,3,0,0.586058897064396,0.979859018356124,-0.889891395153455 +11,4,0,0.586058897064396,1.13205981238157,0.418028183041052 +11,5,0,0.586058897064396,1.94312488363535,2.62673794895776 +11,6,0,0.586058897064396,1.34556107555346,1.69235327979137 +12,1,0,-0.0328133420909616,-0.718664318284347,1.01256054018714 +12,2,0,-0.0328133420909616,0.820872201858118,-0.951074969175408 +12,3,0,-0.0328133420909616,-0.123187668525215,0.899700408876431 +12,4,0,-0.0328133420909616,0.78432993049315,-0.16904574962847 +12,5,0,-0.0328133420909616,1.5789293870534,2.50565159085796 +12,6,0,-0.0328133420909616,1.13043900063265,2.18474286754862 +13,1,0,-0.261074291075106,-0.886304628526536,0.190059587043397 +13,2,0,-0.261074291075106,1.18300828926824,1.73607216665735 +13,3,0,-0.261074291075106,0.558340815315284,2.39851236830549 +13,4,0,-0.261074291075106,0.677849256703997,1.93832673654187 +13,5,0,-0.261074291075106,1.24279144956035,4.16803650110529 +13,6,0,-0.261074291075106,1.60879506475197,1.84949612465038 +14,1,0,-2.01934334017198,-0.823487619952887,0.965002660772585 +14,2,0,-2.01934334017198,-0.707032294505178,0.205238668207667 +14,3,0,-2.01934334017198,-0.61048067361937,-0.481964402432323 +14,4,0,-2.01934334017198,-0.471995841450106,0.539239702125543 +14,5,0,-2.01934334017198,0.20463530567174,0.600661309672454 +14,6,0,-2.01934334017198,0.563781281949436,0.603161656459746 +15,1,0,0.361957454931056,0.0189565591765361,-0.129555386494278 +15,2,0,0.361957454931056,0.106085329301318,0.425642861619322 +15,3,0,0.361957454931056,0.451190373474882,2.23576139017725 +15,4,0,0.361957454931056,1.28538019707699,4.29088314743058 +15,5,0,0.361957454931056,1.20237818127509,3.214356355133 +15,6,0,0.361957454931056,1.4677696966727,4.14629939829314 +16,1,0,-0.599632942668391,-0.0865504039587467,-2.49861640755116 +16,2,0,-0.599632942668391,0.730243732022659,0.509031417314836 +16,3,0,-0.599632942668391,-0.152765685660319,0.369550346158571 +16,4,0,-0.599632942668391,0.437564273403713,0.0251070431731277 +16,5,0,-0.599632942668391,1.25673745360795,-1.26356621386106 +16,6,0,-0.599632942668391,0.396948169632596,-1.19812926828942 +17,1,0,0.198694952835121,1.48026675231997,-0.0914373018488306 +17,2,0,0.198694952835121,0.602080106111401,-1.13650828313566 +17,3,0,0.198694952835121,0.66717979268474,1.28036383890753 +17,4,0,0.198694952835121,0.744336945605951,0.31249515532039 +17,5,0,0.198694952835121,0.734442176603276,3.41280747785275 +17,6,0,0.198694952835121,1.14792728063429,0.520393061796443 +18,1,0,-0.560769887816829,0.740566418840177,-1.13242281217491 +18,2,0,-0.560769887816829,-0.800802303021818,-1.39936169368382 +18,3,0,-0.560769887816829,1.64284271435801,-1.02033967144853 +18,4,0,-0.560769887816829,-0.33064398708322,0.436977600914218 +18,5,0,-0.560769887816829,1.52038443046078,1.20604277293126 +18,6,0,-0.560769887816829,1.07184454667575,0.846352514375142 +19,1,0,0.426318371985953,0.615622372338839,1.52561631350584 +19,2,0,0.426318371985953,1.1829040046622,3.60645868325585 +19,3,0,0.426318371985953,0.60962893875821,2.6921711792421 +19,4,0,0.426318371985953,1.43857945487693,3.86330852828638 +19,5,0,0.426318371985953,1.66282398015716,4.83728207603734 +19,6,0,0.426318371985953,1.40236230094667,4.44066588875079 +20,1,0,0.0661747368407101,-0.468916572476273,0.944687218273834 +20,2,0,0.0661747368407101,0.0698315654648725,2.09812105308828 +20,3,0,0.0661747368407101,1.0687390568495,3.04216485731978 +20,4,0,0.0661747368407101,1.15786295583289,3.54263125509707 +20,5,0,0.0661747368407101,1.27377826018422,1.96382561369836 +20,6,0,0.0661747368407101,0.499986760723536,3.63781888104593 +21,1,0,0.836932519114268,1.19554747601385,0.744833504737609 +21,2,0,0.836932519114268,0.859628392812328,2.36872962935447 +21,3,0,0.836932519114268,0.748417270018536,2.15303566865993 +21,4,0,0.836932519114268,1.59274446404155,3.29582632329775 +21,5,0,0.836932519114268,1.4555910021538,4.40482843122191 +21,6,0,0.836932519114268,1.58763855557056,4.8251150631344 +22,1,0,0.0941874351853353,0.466497866418425,2.31236487898151 +22,2,0,0.0941874351853353,0.261894899289606,1.20207512086356 +22,3,0,0.0941874351853353,-0.199883909412536,2.61279927072359 +22,4,0,0.0941874351853353,1.06060738068883,2.92500959946553 +22,5,0,0.0941874351853353,0.83115437138983,2.86255384244105 +22,6,0,0.0941874351853353,0.547611602018412,3.86553389086576 +23,1,0,2.24292067852269,1.65272226128883,0.512316531737125 +23,2,0,2.24292067852269,1.10093214898091,0.278294515654655 +23,3,0,2.24292067852269,1.91261347599136,1.64154746116044 +23,4,0,2.24292067852269,1.70894499719591,3.9889543067309 +23,5,0,2.24292067852269,1.88316975411293,5.14776444387616 +23,6,0,2.24292067852269,2.22298347240385,5.60218491495817 +24,1,0,-1.35767302263554,-0.532569922147083,0.854035384193373 +24,2,0,-1.35767302263554,0.293236980944925,0.859939323644371 +24,3,0,-1.35767302263554,0.400452813605349,-0.601293901404389 +24,4,0,-1.35767302263554,-0.0464606324392122,0.377478681201744 +24,5,0,-1.35767302263554,1.30355241324811,0.0307348313527933 +24,6,0,-1.35767302263554,0.640401721123196,-0.291861720912245 +25,1,0,0.536762929066239,0.494542956395155,1.13128088027847 +25,2,0,0.536762929066239,0.413659545685529,0.175996686768765 +25,3,0,0.536762929066239,0.951090791785006,0.89142621783185 +25,4,0,0.536762929066239,1.42138214987661,1.83478859337311 +25,5,0,0.536762929066239,0.197687553568442,3.82746389246305 +25,6,0,0.536762929066239,0.997008528348205,3.29347248061781 +26,1,0,0.080424492077766,0.55057496426079,-1.57794939371139 +26,2,0,0.080424492077766,0.857425330645063,0.204173346072951 +26,3,0,0.080424492077766,0.435846052366296,0.467273164232521 +26,4,0,0.080424492077766,0.984228608534135,-0.358590158898051 +26,5,0,0.080424492077766,1.72601324978517,-0.249224951584627 +26,6,0,0.080424492077766,1.09283259694636,2.05876710749068 +27,1,0,-0.661809943463178,-0.351151472617987,1.3332812259713 +27,2,0,-0.661809943463178,-0.769094233439663,1.02937579325852 +27,3,0,-0.661809943463178,0.529301492309873,0.670624009578854 +27,4,0,-0.661809943463178,0.340211950424195,2.83276083639353 +27,5,0,-0.661809943463178,0.672804544960685,1.33367085251827 +27,6,0,-0.661809943463178,0.836997883685823,1.64463392166496 +28,1,0,1.18763780487985,0.857112953615082,0.918371137986335 +28,2,0,1.18763780487985,0.609455036547718,-0.1953916519845 +28,3,0,1.18763780487985,0.957415327060119,1.64710330460386 +28,4,0,1.18763780487985,1.70678167693069,2.82206969876212 +28,5,0,1.18763780487985,2.28994975704781,2.81441799704668 +28,6,0,1.18763780487985,2.70532991473682,6.64412995317486 +29,1,0,0.171569793047149,-0.398831901085214,0.103682763935853 +29,2,0,0.171569793047149,0.185275047950682,0.183298060409325 +29,3,0,0.171569793047149,1.16669238180421,1.92277150882252 +29,4,0,0.171569793047149,1.12694852976321,1.54493766913653 +29,5,0,0.171569793047149,1.04381631290082,4.12170013580083 +29,6,0,0.171569793047149,1.43494779620057,2.35016615646919 +30,1,0,0.214899638814364,0.990047580435093,0.0329317907015836 +30,2,0,0.214899638814364,0.00318616298013463,3.1538826374806 +30,3,0,0.214899638814364,-0.146108580250217,1.40058443711373 +30,4,0,0.214899638814364,0.538522428274927,3.45129621770057 +30,5,0,0.214899638814364,1.52956640764869,4.19427236133207 +30,6,0,0.214899638814364,1.99778295611785,3.56034464827219 +31,1,0,2.32254390100494,1.15644468484223,2.77259843088733 +31,2,0,2.32254390100494,1.7483266658964,2.02564757997795 +31,3,0,2.32254390100494,2.2699980030887,1.64606771516423 +31,4,0,2.32254390100494,2.71526529576935,3.37656661650927 +31,5,0,2.32254390100494,2.46194873478394,6.10188976491978 +31,6,0,2.32254390100494,2.97510578028662,8.52554433662228 +32,1,0,-0.169502937020473,0.693215490975842,2.05141406053865 +32,2,0,-0.169502937020473,0.468165521466207,4.61437618451743 +32,3,0,-0.169502937020473,0.205806456493878,1.11339736229636 +32,4,0,-0.169502937020473,1.25812023117349,4.1166099867286 +32,5,0,-0.169502937020473,1.44654158298815,3.40523553515562 +32,6,0,-0.169502937020473,1.31004230358703,1.01835012296617 +33,1,0,0.552770969821111,-0.306006044577907,1.25412066983468 +33,2,0,0.552770969821111,0.995602390305268,1.80917798576773 +33,3,0,0.552770969821111,1.01655508934542,0.757291376613842 +33,4,0,0.552770969821111,1.1036479557666,2.96417691332605 +33,5,0,0.552770969821111,1.45864948387186,2.46197479844314 +33,6,0,0.552770969821111,0.859567156764491,5.97071719266024 +34,1,0,0.481846461393583,0.269302303860423,0.526376540360734 +34,2,0,0.481846461393583,0.0461613847557073,2.19364928878128 +34,3,0,0.481846461393583,1.12068944272412,2.44731864429638 +34,4,0,0.481846461393583,0.783270153094354,2.52781800340864 +34,5,0,0.481846461393583,1.97211005145252,3.13973621130369 +34,6,0,0.481846461393583,1.03839429696509,2.94271775797344 +35,1,0,0.180785291726702,0.590217475976332,2.01882790659905 +35,2,0,0.180785291726702,0.408881961264579,4.08885751899202 +35,3,0,0.180785291726702,0.581997504350181,3.11714635871518 +35,4,0,0.180785291726702,1.49965041737788,5.38773518005293 +35,5,0,0.180785291726702,2.02563496208288,5.9748136314136 +35,6,0,0.180785291726702,2.28603451086318,2.7474872302793 +36,1,0,0.937530562434575,0.211835239755195,0.264811770320571 +36,2,0,0.937530562434575,0.97223160253963,3.72358171039763 +36,3,0,0.937530562434575,2.03844428066716,2.52837086994306 +36,4,0,0.937530562434575,1.35584019846206,3.46668468173737 +36,5,0,0.937530562434575,2.31452132526835,4.30663345338415 +36,6,0,0.937530562434575,1.52220257385175,5.90295382804056 +37,1,0,0.605297706819014,1.33501016334775,-1.22155851591522 +37,2,0,0.605297706819014,0.273646824543883,0.94685268474313 +37,3,0,0.605297706819014,1.18861405551878,-1.06749360477619 +37,4,0,0.605297706819014,0.430022662342312,2.49683112338337 +37,5,0,0.605297706819014,0.980782441057555,2.87859515158806 +37,6,0,0.605297706819014,1.68491715613623,3.18312049166695 +38,1,0,-0.150046075996628,0.907283121370997,-0.269990850232008 +38,2,0,-0.150046075996628,0.578967394515049,-0.886845866605914 +38,3,0,-0.150046075996628,1.31322956121607,2.17397969319074 +38,4,0,-0.150046075996628,0.770060624548639,0.629084998485956 +38,5,0,-0.150046075996628,0.858759768922271,0.248192249762817 +38,6,0,-0.150046075996628,1.31157900399226,3.13842264488913 +39,1,0,-1.53940169474244,0.386233806928156,-0.523153781143094 +39,2,0,-1.53940169474244,-0.184038667457276,-1.58742257093027 +39,3,0,-1.53940169474244,-0.973488223644379,-0.749417744231022 +39,4,0,-1.53940169474244,0.510965070488413,-0.735701532265713 +39,5,0,-1.53940169474244,0.766370381872722,-1.75342467837839 +39,6,0,-1.53940169474244,0.687218453403146,0.252169837110935 +40,1,0,0.391843330512968,0.704055957164002,0.409086287780538 +40,2,0,0.391843330512968,0.519946271805668,-1.89191667178992 +40,3,0,0.391843330512968,0.432251147082645,0.648330099168622 +40,4,0,0.391843330512968,1.21713541238615,1.31189584887626 +40,5,0,0.391843330512968,1.27472438671965,0.103667887130483 +40,6,0,0.391843330512968,1.66786350472797,0.685042322957091 +41,1,0,-0.0373260431491352,0.0317324594708311,-0.747942068188531 +41,2,0,-0.0373260431491352,0.558761507456241,-1.19204731827254 +41,3,0,-0.0373260431491352,0.627774174828559,-0.00444322772313305 +41,4,0,-0.0373260431491352,1.22057537184321,0.0157895549428625 +41,5,0,-0.0373260431491352,-0.13463140571047,2.03834883872972 +41,6,0,-0.0373260431491352,1.26326673119073,2.38013129655104 +42,1,0,0.827626116517122,0.619582666001477,2.05284651791961 +42,2,0,0.827626116517122,0.728960630288116,2.63692144330927 +42,3,0,0.827626116517122,1.51573241849325,2.82734402300804 +42,4,0,0.827626116517122,2.32134629984684,4.70066367856648 +42,5,0,0.827626116517122,1.26080310077716,2.71057083119057 +42,6,0,0.827626116517122,1.56560512998824,4.83856397972047 +43,1,0,-0.665331979033663,-0.225102401789033,2.56456608500982 +43,2,0,-0.665331979033663,-0.553172893008135,4.61457088107317 +43,3,0,-0.665331979033663,0.972861442802118,1.82443898633191 +43,4,0,-0.665331979033663,0.32326929160598,2.64950447327976 +43,5,0,-0.665331979033663,0.306489791796068,2.60561890553988 +43,6,0,-0.665331979033663,1.03909552234328,3.64564011990368 +44,1,0,0.942288464073336,0.00874783812863711,0.056977549744632 +44,2,0,0.942288464073336,0.596281511258369,1.12468821275525 +44,3,0,0.942288464073336,1.39468851953371,2.90909190594441 +44,4,0,0.942288464073336,2.11675576968934,4.23166721482138 +44,5,0,0.942288464073336,1.58330641741654,5.43308550177595 +44,6,0,0.942288464073336,1.16350891633872,4.71261506783713 +45,1,0,-0.0809416298757052,0.807692153394734,1.0789586456094 +45,2,0,-0.0809416298757052,0.752887274713188,0.785720766407489 +45,3,0,-0.0809416298757052,1.24054098972764,2.55331843603172 +45,4,0,-0.0809416298757052,1.34412295664021,2.48536763140985 +45,5,0,-0.0809416298757052,-0.0307181568992582,1.1879236197245 +45,6,0,-0.0809416298757052,1.78022693303965,3.71848890245285 +46,1,0,0.521163361059873,-0.723702502547912,1.09867938696003 +46,2,0,0.521163361059873,1.1499608686645,3.55842561079306 +46,3,0,0.521163361059873,1.15232123544905,3.17634759665484 +46,4,0,0.521163361059873,1.42926586195998,3.39003436513758 +46,5,0,0.521163361059873,0.879129584097901,2.80180340599266 +46,6,0,0.521163361059873,0.914156746191509,2.46863004623085 +47,1,0,1.61466175286344,0.643006559192123,0.836204718894284 +47,2,0,1.61466175286344,1.13661030532442,2.30379657813442 +47,3,0,1.61466175286344,1.19295732982162,2.64086502103959 +47,4,0,1.61466175286344,0.239328559267526,3.65153083011424 +47,5,0,1.61466175286344,1.68375842695897,5.33901616867935 +47,6,0,1.61466175286344,2.02785308242104,4.85439287872028 +48,1,0,-1.7565659395326,-0.218238640820971,1.78646195020932 +48,2,0,-1.7565659395326,-0.969625592296388,0.784269865689745 +48,3,0,-1.7565659395326,-0.419721278011531,2.20671186654619 +48,4,0,-1.7565659395326,-0.0179174885255089,0.521674280505722 +48,5,0,-1.7565659395326,0.0262037737083689,0.883000983935286 +48,6,0,-1.7565659395326,0.434997896953442,1.76393693136589 +49,1,0,0.0236704073602128,0.342406228293342,-0.0627973687877335 +49,2,0,0.0236704073602128,1.339042197954,0.510408069568147 +49,3,0,0.0236704073602128,0.396305118083889,0.128070410293366 +49,4,0,0.0236704073602128,0.301229123953579,2.34822787611627 +49,5,0,0.0236704073602128,1.84984351148937,2.02095085612302 +49,6,0,0.0236704073602128,1.14237872441368,3.88976774422889 +50,1,0,-1.05243771447958,0.290206750660849,-0.27631290661132 +50,2,0,-1.05243771447958,-0.331976147509505,0.0784093174185179 +50,3,0,-1.05243771447958,0.644389922826399,1.85558157163888 +50,4,0,-1.05243771447958,-0.0363544318974775,1.14349827947337 +50,5,0,-1.05243771447958,-0.63902084988463,0.0282478533308865 +50,6,0,-1.05243771447958,0.277763327171488,1.60530873660652 +51,1,0,1.41893961966826,1.72807036319162,1.23790039735326 +51,2,0,1.41893961966826,1.68895995675209,2.07276778859 +51,3,0,1.41893961966826,0.971263584985605,3.6394918544787 +51,4,0,1.41893961966826,1.11899195431195,3.41900141304584 +51,5,0,1.41893961966826,1.87654473407966,4.68940054213404 +51,6,0,1.41893961966826,3.04105530464514,4.8924758822659 +52,1,0,-1.1910633756242,0.200524510461652,-2.7265059697081 +52,2,0,-1.1910633756242,-1.2115909345573,-1.40963894581555 +52,3,0,-1.1910633756242,0.386175892807225,-1.47451140247512 +52,4,0,-1.1910633756242,1.02789828620209,0.146871306849964 +52,5,0,-1.1910633756242,-0.273264466564166,-0.0141430110384286 +52,6,0,-1.1910633756242,0.578879852223559,-1.17416016414042 +53,1,0,0.195514781381017,-0.742554152921851,2.23579400276791 +53,2,0,0.195514781381017,0.23337503458647,1.10762600155431 +53,3,0,0.195514781381017,0.263301882945963,2.51597948959044 +53,4,0,0.195514781381017,-0.199791624518697,3.42840858712109 +53,5,0,0.195514781381017,1.60088371927206,1.87400311570008 +53,6,0,0.195514781381017,1.23132242768948,2.10936229576561 +54,1,0,0.111145488608917,0.41248587564804,-1.23962257171445 +54,2,0,0.111145488608917,0.650566893014723,0.757366835792272 +54,3,0,0.111145488608917,0.638921948130227,-0.317803478051985 +54,4,0,0.111145488608917,-0.29731490602435,0.682731490142251 +54,5,0,0.111145488608917,1.38608083061868,1.22724299967546 +54,6,0,0.111145488608917,1.38422425480489,3.12874530862489 +55,1,0,0.666862478709229,1.66603184966515,-0.96199867245643 +55,2,0,0.666862478709229,0.633112902252682,-1.17037332152456 +55,3,0,0.666862478709229,0.343015660916893,0.874503842048745 +55,4,0,0.666862478709229,0.770817827624448,-0.0109943396516543 +55,5,0,0.666862478709229,0.964533553442544,1.71046636135371 +55,6,0,0.666862478709229,1.76201281537328,1.8951412953794 +56,1,0,-1.77404829500802,-1.40580752868787,1.39863275680412 +56,2,0,-1.77404829500802,0.870111926030201,-0.982599170611262 +56,3,0,-1.77404829500802,-0.830299171612467,-0.778719888508792 +56,4,0,-1.77404829500802,0.334381341446428,2.10682625136074 +56,5,0,-1.77404829500802,0.0402025096737488,0.21272995862292 +56,6,0,-1.77404829500802,-0.102134961440965,-0.482035354350842 +57,1,0,-1.69162901504091,-0.937011635623388,-1.822960278522 +57,2,0,-1.69162901504091,-0.854090558628001,0.246599401891903 +57,3,0,-1.69162901504091,0.306483912711003,-1.10550309471832 +57,4,0,-1.69162901504091,-0.375186713716243,-0.403217811110663 +57,5,0,-1.69162901504091,-0.0372694866470827,-1.2397278955316 +57,6,0,-1.69162901504091,0.042822212170315,-1.72546983425334 +58,1,0,-0.849031394079241,-0.365462147353985,-0.312601656922766 +58,2,0,-0.849031394079241,-0.0832361702771065,0.103741778870931 +58,3,0,-0.849031394079241,0.142581169967471,-0.482286945873843 +58,4,0,-0.849031394079241,0.460384860409781,1.40366770538517 +58,5,0,-0.849031394079241,0.996609986700584,0.871624823978852 +58,6,0,-0.849031394079241,0.85264265006985,1.89123646594172 +59,1,0,1.21025798910521,1.04847972204707,-0.575583913121781 +59,2,0,1.21025798910521,1.38937084327584,1.37186521680745 +59,3,0,1.21025798910521,1.2547790752355,0.529851722231607 +59,4,0,1.21025798910521,1.17327232495747,1.47840784888763 +59,5,0,1.21025798910521,1.172880927698,3.91882955565966 +59,6,0,1.21025798910521,1.39574007441178,2.28500196266698 +60,1,0,0.789386089516239,0.179951915237443,0.435733939422799 +60,2,0,0.789386089516239,0.438493621227776,1.47709316141718 +60,3,0,0.789386089516239,1.21539128818204,3.37967447648071 +60,4,0,0.789386089516239,1.76191512346368,2.94762787653769 +60,5,0,0.789386089516239,1.35758016131045,2.48842314882934 +60,6,0,0.789386089516239,1.50244938572149,4.71258184189825 +61,1,0,1.15707226166531,0.695992482122723,0.366873356023154 +61,2,0,1.15707226166531,1.30843852180442,0.25928295987725 +61,3,0,1.15707226166531,1.23140830848896,0.670889744195337 +61,4,0,1.15707226166531,1.52680662397394,3.10031216794104 +61,5,0,1.15707226166531,1.97553314383926,4.52201393669106 +61,6,0,1.15707226166531,1.89587096185583,4.17000041602687 +62,1,0,-0.344242039937823,-0.012607129644549,1.53095259638453 +62,2,0,-0.344242039937823,-0.190659504558242,2.99016158987811 +62,3,0,-0.344242039937823,0.129929099988597,2.55026017406343 +62,4,0,-0.344242039937823,0.75025985280716,0.871172605987694 +62,5,0,-0.344242039937823,-0.088414077084213,2.66185824077775 +62,6,0,-0.344242039937823,1.68941056098916,2.3342515589607 +63,1,0,-0.520156154475325,0.115284773651995,-0.554998777313392 +63,2,0,-0.520156154475325,0.380215358847244,0.198926509461979 +63,3,0,-0.520156154475325,-0.045325702695873,0.599340066190899 +63,4,0,-0.520156154475325,0.403372511379866,3.05428355693047 +63,5,0,-0.520156154475325,0.339830867502976,2.03433756350707 +63,6,0,-0.520156154475325,0.952544521227367,1.17519783476071 +64,1,0,0.203435284900822,0.515493855780151,-0.749085263516488 +64,2,0,0.203435284900822,0.746187070355182,2.1052938488016 +64,3,0,0.203435284900822,0.374895246973636,1.58265151593793 +64,4,0,0.203435284900822,0.506751948936316,2.14495255559874 +64,5,0,0.203435284900822,1.51739942863017,0.95825975606708 +64,6,0,0.203435284900822,1.09147631899005,3.41111423514602 +65,1,0,-0.55341970038927,-0.131380062378949,-1.32195592808742 +65,2,0,-0.55341970038927,-0.241965241577032,-1.02423916263361 +65,3,0,-0.55341970038927,1.15300408189915,-0.535841679982353 +65,4,0,-0.55341970038927,0.0532746291132736,-0.968231722746471 +65,5,0,-0.55341970038927,0.745558003100886,0.832777050424564 +65,6,0,-0.55341970038927,-0.00751406091164042,1.96077688545181 +66,1,0,1.16897479134018,0.184878831174988,-1.28620870245401 +66,2,0,1.16897479134018,0.987928053916087,0.166753297701763 +66,3,0,1.16897479134018,1.27696264162646,1.68247659314794 +66,4,0,1.16897479134018,1.55536067948273,3.87746819914394 +66,5,0,1.16897479134018,2.15354648101136,4.79993387221023 +66,6,0,1.16897479134018,1.71085429487955,4.04950544043591 +67,1,0,-0.311033874726052,0.0263159579110574,0.871500540829715 +67,2,0,-0.311033874726052,0.213806039758833,1.10566090191848 +67,3,0,-0.311033874726052,1.16580579545896,2.62828302528987 +67,4,0,-0.311033874726052,0.941962746311597,1.57046531053 +67,5,0,-0.311033874726052,0.185785078920236,1.0287988657371 +67,6,0,-0.311033874726052,0.0881200987592246,4.48340110608756 +68,1,0,0.609445260914666,0.160720456451015,-0.875045477717211 +68,2,0,0.609445260914666,0.398587442880102,1.97447918279283 +68,3,0,0.609445260914666,0.855870863727989,1.76870737136006 +68,4,0,0.609445260914666,1.43803622432617,3.07264275168721 +68,5,0,0.609445260914666,1.53294248674142,3.1268484428726 +68,6,0,0.609445260914666,2.30216153797432,4.90626806484531 +69,1,0,-1.70140877068258,-0.772137510934144,-3.67033050406739 +69,2,0,-1.70140877068258,-0.0887373122839057,0.00741645039112426 +69,3,0,-1.70140877068258,0.23462709293423,-0.35143639642592 +69,4,0,-1.70140877068258,-0.0133450385366955,-3.11261503664916 +69,5,0,-1.70140877068258,0.647212644285179,-1.22067385183629 +69,6,0,-1.70140877068258,-0.318090934527394,-1.7703701960703 +70,1,0,-0.701528235373902,0.419756517766687,-0.392170361021321 +70,2,0,-0.701528235373902,1.00520991718909,-1.24465679664118 +70,3,0,-0.701528235373902,0.397714193248576,1.73346811162478 +70,4,0,-0.701528235373902,0.141784792120088,2.62165510171308 +70,5,0,-0.701528235373902,-0.229555991095407,1.66547134257899 +70,6,0,-0.701528235373902,0.457270807196878,3.5583092350656 +71,1,0,0.505874716504533,0.0328685484582639,1.22528702490137 +71,2,0,0.505874716504533,1.59522458519428,2.23505630653046 +71,3,0,0.505874716504533,1.76396196449465,4.50976634967245 +71,4,0,0.505874716504533,2.02663440047412,5.34020271673294 +71,5,0,0.505874716504533,1.16929077274353,5.67948718050403 +71,6,0,0.505874716504533,0.987516672556885,4.01479530785995 +72,1,0,-0.844325706369891,-0.160769095993306,1.77828123515083 +72,2,0,-0.844325706369891,-0.626002441128783,2.86943080366521 +72,3,0,-0.844325706369891,0.624503195437875,1.13819663453126 +72,4,0,-0.844325706369891,0.4128826402263,2.34116236266462 +72,5,0,-0.844325706369891,0.64321904824842,6.00376292914209 +72,6,0,-0.844325706369891,1.30388473771754,4.39817205983043 +73,1,0,0.35463969852429,0.175853581251931,1.54901516888919 +73,2,0,0.35463969852429,0.255347723083929,3.75759086722241 +73,3,0,0.35463969852429,0.842048899742575,2.74674519300349 +73,4,0,0.35463969852429,0.843858032144758,3.48463769028479 +73,5,0,0.35463969852429,2.02272986936328,2.63274428096982 +73,6,0,0.35463969852429,1.756515220362,3.36491283175246 +74,1,0,0.634049900150801,1.06843050846366,0.631806643103645 +74,2,0,0.634049900150801,1.09459023941769,1.10185630635582 +74,3,0,0.634049900150801,1.16047718202649,2.21277078081482 +74,4,0,0.634049900150801,0.194209434370495,4.49679293334212 +74,5,0,0.634049900150801,1.57111541788119,3.78301141707104 +74,6,0,0.634049900150801,1.8709185053691,3.83801701206847 +75,1,0,-0.144614154426945,-0.1812136320715,-1.50303099629943 +75,2,0,-0.144614154426945,0.0812045686177193,-0.30813252539654 +75,3,0,-0.144614154426945,0.592606669784812,-0.87606327177109 +75,4,0,-0.144614154426945,2.21284582539953,1.49034525451696 +75,5,0,-0.144614154426945,0.89070743973123,1.99305042503676 +75,6,0,-0.144614154426945,1.22531924408363,1.04904749337575 +76,1,0,-0.965620194716236,-0.416127905137339,-0.767970743384194 +76,2,0,-0.965620194716236,0.895930664804138,-0.185344950505385 +76,3,0,-0.965620194716236,0.77433449793566,0.860449806370312 +76,4,0,-0.965620194716236,0.70085446817956,1.07172526057056 +76,5,0,-0.965620194716236,1.05852067016297,0.51916850372457 +76,6,0,-0.965620194716236,0.638859520759945,1.33787453176867 +77,1,0,-1.86580715673843,-0.391278486536145,-0.0920440713171948 +77,2,0,-1.86580715673843,-1.3757502672685,-0.560366699602578 +77,3,0,-1.86580715673843,0.308335534756901,-0.371286061286313 +77,4,0,-1.86580715673843,0.345233743814379,-0.20407441671565 +77,5,0,-1.86580715673843,0.381411993188402,-1.14332176885771 +77,6,0,-1.86580715673843,-0.584252643983715,-1.27669748966723 +78,1,0,-1.36945985986056,-0.317388985722178,0.757961675843857 +78,2,0,-1.36945985986056,-0.931319605141151,-1.47231495687832 +78,3,0,-1.36945985986056,0.64939703346783,0.947760079117256 +78,4,0,-1.36945985986056,0.141848675286346,0.427700713693576 +78,5,0,-1.36945985986056,0.645979505797502,0.123611381550555 +78,6,0,-1.36945985986056,0.853461304724385,-1.30724575909746 +79,1,0,-1.4085132336069,-1.4450148783583,0.727494818634948 +79,2,0,-1.4085132336069,-0.599025158604699,0.291135574546722 +79,3,0,-1.4085132336069,-0.406989701850758,0.882944383939482 +79,4,0,-1.4085132336069,-0.0202306741507156,0.290743072184474 +79,5,0,-1.4085132336069,0.418122082147065,-0.554654948017118 +79,6,0,-1.4085132336069,0.161492862233551,-0.104305091084399 +80,1,0,1.0465570019994,0.83467245617602,1.79474289967135 +80,2,0,1.0465570019994,0.112243844479774,2.42789304208095 +80,3,0,1.0465570019994,0.333555656213314,4.11924338896377 +80,4,0,1.0465570019994,1.01788286394161,3.88357495448787 +80,5,0,1.0465570019994,1.53660910957585,4.5182271100794 +80,6,0,1.0465570019994,1.62573779201925,6.09413820742505 +81,1,0,-1.41453829223088,-0.422101284770018,0.769101887017708 +81,2,0,-1.41453829223088,-0.315168451954516,0.729566856970148 +81,3,0,-1.41453829223088,-0.272040188359851,-0.671347616711595 +81,4,0,-1.41453829223088,0.183840000922275,-0.614516066633574 +81,5,0,-1.41453829223088,0.299215308065219,0.0802272383475152 +81,6,0,-1.41453829223088,0.732800101297085,0.510417085488192 +82,1,0,0.160539536166133,0.959530186512375,-0.215930107181071 +82,2,0,0.160539536166133,-0.326281366208948,1.07948246934508 +82,3,0,0.160539536166133,0.324000083908395,1.52302152579562 +82,4,0,0.160539536166133,-0.258160129826018,1.74745221913454 +82,5,0,0.160539536166133,1.46023105551091,2.33039670539431 +82,6,0,0.160539536166133,0.788817140506763,2.32858808738527 +83,1,0,-0.567801186208016,0.231833577654445,1.9093919950695 +83,2,0,-0.567801186208016,-0.406252558339809,1.71838187145903 +83,3,0,-0.567801186208016,0.500105335834885,0.424872683579041 +83,4,0,-0.567801186208016,1.6055462707676,1.14871256154913 +83,5,0,-0.567801186208016,1.27321097698752,2.60872052340111 +83,6,0,-0.567801186208016,0.495575788605087,1.73434907598337 +84,1,0,-0.782213005252933,-1.24210551325893,0.62824879952858 +84,2,0,-0.782213005252933,0.998799311582334,-0.285355853362829 +84,3,0,-0.782213005252933,0.152353660632812,1.08574270218729 +84,4,0,-0.782213005252933,0.437770170026559,0.992180904182235 +84,5,0,-0.782213005252933,0.555952746039179,2.04941499620541 +84,6,0,-0.782213005252933,0.890162464146768,-0.894620656851764 +85,1,0,-0.487130227305875,0.47292213649988,-0.82059178734108 +85,2,0,-0.487130227305875,0.492301356302156,0.262528389117469 +85,3,0,-0.487130227305875,1.29009677447065,1.17301382240356 +85,4,0,-0.487130227305875,1.43591257033689,2.60956505836633 +85,5,0,-0.487130227305875,1.82677210590248,0.693530436043082 +85,6,0,-0.487130227305875,1.32378375915425,1.21715970534733 +86,1,0,-0.384373119535855,-0.170904424548637,0.498232930318139 +86,2,0,-0.384373119535855,-0.295740805748663,0.301854807254213 +86,3,0,-0.384373119535855,0.578383091390104,0.695014709575832 +86,4,0,-0.384373119535855,0.528019648180887,-0.737125962869525 +86,5,0,-0.384373119535855,1.01985265304516,-0.0830034763200127 +86,6,0,-0.384373119535855,1.67834611857395,2.49504179213634 +87,1,0,0.408020963230945,0.962156203251732,1.94439889343524 +87,2,0,0.408020963230945,0.99999055549027,1.16647176165158 +87,3,0,0.408020963230945,0.970786735481066,1.69236048882497 +87,4,0,0.408020963230945,1.41338326575805,4.12905636501844 +87,5,0,0.408020963230945,1.49273145808118,2.17402526741817 +87,6,0,0.408020963230945,1.37163125752385,5.37627582349371 +88,1,0,0.599584079099597,0.730924181364043,-1.31376569334669 +88,2,0,0.599584079099597,1.44699595117378,-0.623281047649964 +88,3,0,0.599584079099597,0.28253615169462,-0.693242466503484 +88,4,0,0.599584079099597,1.16021618973925,1.01881668516363 +88,5,0,0.599584079099597,0.964519264352473,1.21574324307197 +88,6,0,0.599584079099597,1.84858002344773,2.57058311503664 +89,1,0,-0.311048261021212,0.172929318481353,2.7900643892079 +89,2,0,-0.311048261021212,0.913627245775031,1.24443702713629 +89,3,0,-0.311048261021212,-0.781248428169652,1.03938341222443 +89,4,0,-0.311048261021212,0.602717775024287,0.461064451496916 +89,5,0,-0.311048261021212,0.77172145901647,2.32861716047523 +89,6,0,-0.311048261021212,0.875048468268877,4.46406247718196 +90,1,0,-0.812370112719702,0.508688758563729,0.663812620865317 +90,2,0,-0.812370112719702,0.0978888128213179,0.604934747365124 +90,3,0,-0.812370112719702,-0.227169219633528,0.483064206170982 +90,4,0,-0.812370112719702,-0.0789620142453338,1.30016784410719 +90,5,0,-0.812370112719702,0.474031207045202,-0.35852658877042 +90,6,0,-0.812370112719702,0.362037836323042,2.62002321031834 +91,1,0,-0.835160216707722,0.0821439607244709,-1.31297017293814 +91,2,0,-0.835160216707722,-0.215052170622971,-0.00952574873257295 +91,3,0,-0.835160216707722,-0.515795699189277,0.93911380476138 +91,4,0,-0.835160216707722,0.0994435769261413,0.29534073274764 +91,5,0,-0.835160216707722,0.640632783084201,0.631494983316459 +91,6,0,-0.835160216707722,0.444237884051304,-0.614904284648697 +92,1,0,-0.0347979035615357,-0.425071253260572,-0.276129691013229 +92,2,0,-0.0347979035615357,-0.00558881648247656,2.6914081615219 +92,3,0,-0.0347979035615357,0.12042472026752,0.139596914798406 +92,4,0,-0.0347979035615357,0.19294717753885,-0.0358857413034133 +92,5,0,-0.0347979035615357,0.961964380434135,2.00578635972552 +92,6,0,-0.0347979035615357,1.17135420091513,2.59638281957783 +93,1,0,-0.875486806182028,0.130202709056072,0.0515320316326158 +93,2,0,-0.875486806182028,-1.41949223413023,0.503122856462025 +93,3,0,-0.875486806182028,-0.109791084167903,-1.1180468841845 +93,4,0,-0.875486806182028,0.659965080718873,-0.368352880340779 +93,5,0,-0.875486806182028,0.067354181000394,0.0520547024496831 +93,6,0,-0.875486806182028,1.4060340511004,-1.52569833771551 +94,1,0,-0.257821470178811,0.103041140124536,3.00578229260879 +94,2,0,-0.257821470178811,0.0510455966337183,2.81600739821049 +94,3,0,-0.257821470178811,1.28170651334834,1.88866826165356 +94,4,0,-0.257821470178811,0.524450670730671,4.24222926203413 +94,5,0,-0.257821470178811,0.543757953082137,3.89014943444177 +94,6,0,-0.257821470178811,0.837694358790485,5.33137502521585 +95,1,0,-1.06522102729796,0.338860131607811,1.32692462722341 +95,2,0,-1.06522102729796,-0.127369030427613,-0.0832637940592554 +95,3,0,-1.06522102729796,0.0207887615637769,-0.11264669981747 +95,4,0,-1.06522102729796,0.226974834769246,2.89905849310711 +95,5,0,-1.06522102729796,0.495476577660107,2.54073657860863 +95,6,0,-1.06522102729796,0.231418798298941,0.847408625748489 +96,1,0,0.55656405447025,1.84356052443496,2.97044415315513 +96,2,0,0.55656405447025,0.403515009694641,4.91746159042525 +96,3,0,0.55656405447025,1.18467123093859,3.82888905681828 +96,4,0,0.55656405447025,1.45829670995655,4.519569208739 +96,5,0,0.55656405447025,2.31876334410251,2.96806867089888 +96,6,0,0.55656405447025,1.45175015135933,4.95375559492303 +97,1,0,0.968155189610845,0.731474808343252,2.00819632359671 +97,2,0,0.968155189610845,1.81966272467269,2.41871187476564 +97,3,0,0.968155189610845,1.32541288635434,2.85560582347779 +97,4,0,0.968155189610845,1.02840238957306,1.99543376514698 +97,5,0,0.968155189610845,0.72402655805433,6.36901987614095 +97,6,0,0.968155189610845,1.62812732719515,5.45740806337183 +98,1,0,-0.642161084239234,-0.285123013459829,-0.000642853350642403 +98,2,0,-0.642161084239234,-0.598332460697446,2.16895969756833 +98,3,0,-0.642161084239234,0.455815886578303,0.909661380366459 +98,4,0,-0.642161084239234,0.799759144823868,2.97501355379955 +98,5,0,-0.642161084239234,0.742981106632073,2.0103010060523 +98,6,0,-0.642161084239234,1.28403169128433,3.27781589866413 +99,1,0,0.489663886549812,1.58049905561598,0.63224613149953 +99,2,0,0.489663886549812,0.372598634840632,0.153880335628493 +99,3,0,0.489663886549812,1.4943622173095,1.81815428213934 +99,4,0,0.489663886549812,1.5378597808116,0.935298801828264 +99,5,0,0.489663886549812,1.56518687462083,2.92415904318258 +99,6,0,0.489663886549812,0.602576741552375,2.44369000347583 +100,1,0,0.83152065605674,0.478913216287586,0.746784055135776 +100,2,0,0.83152065605674,1.4235529798745,2.04540124845096 +100,3,0,0.83152065605674,0.968193684526933,1.32348962919775 +100,4,0,0.83152065605674,1.07833225792295,0.547613792696509 +100,5,0,0.83152065605674,1.06746551895251,3.31761291520659 +100,6,0,0.83152065605674,1.85523532671524,3.08316773932352 +101,1,0,-0.919726134671162,-0.0153717261852472,2.7788719289477 +101,2,0,-0.919726134671162,0.0258159670995393,2.62781854772703 +101,3,0,-0.919726134671162,-0.210141999024792,2.44073176765192 +101,4,0,-0.919726134671162,0.524799330002078,1.78160512517281 +101,5,0,-0.919726134671162,-0.0437030303186893,3.04165380941417 +101,6,0,-0.919726134671162,1.3026199543081,5.32428649347593 +102,1,0,0.641255824240951,0.532093223730807,-2.09251354544218 +102,2,0,0.641255824240951,0.0941130407259139,0.733006859014464 +102,3,0,0.641255824240951,1.55861283645998,-0.23990034173479 +102,4,0,0.641255824240951,1.21632707133294,-0.176646402386964 +102,5,0,0.641255824240951,1.77130479077965,4.52178543711811 +102,6,0,0.641255824240951,2.00779025241614,1.91109596784127 +103,1,0,1.14326851073882,0.0119837643154777,2.22043326561435 +103,2,0,1.14326851073882,0.362873618700088,4.6975800148237 +103,3,0,1.14326851073882,1.97187455363987,2.20278367481176 +103,4,0,1.14326851073882,0.549403280336012,4.53412190050463 +103,5,0,1.14326851073882,0.74357105113021,4.04691725702911 +103,6,0,1.14326851073882,1.41192305654462,5.38945740422014 +104,1,0,-1.08072100327974,-0.36568086508946,-0.464035936891244 +104,2,0,-1.08072100327974,-0.98836432896696,-0.356166326023515 +104,3,0,-1.08072100327974,0.467378110330098,-0.0636497459678805 +104,4,0,-1.08072100327974,-0.935126213484022,-0.716266458809519 +104,5,0,-1.08072100327974,0.502717496775458,-0.660129046615434 +104,6,0,-1.08072100327974,1.47173852447056,2.01181621362982 +105,1,0,-0.308977011536411,-0.179345642251318,-0.562705733283704 +105,2,0,-0.308977011536411,0.70656302910021,0.18167481062825 +105,3,0,-0.308977011536411,1.21257374180067,0.813478648356054 +105,4,0,-0.308977011536411,0.381854939551222,0.207736765764304 +105,5,0,-0.308977011536411,1.2106178506653,0.0126906713691238 +105,6,0,-0.308977011536411,0.964448644390534,1.03945464476557 +106,1,0,1.50226005393992,1.34925401718096,2.64319435904704 +106,2,0,1.50226005393992,0.98662538563711,1.3596364327739 +106,3,0,1.50226005393992,2.12823788384978,3.59777809891119 +106,4,0,1.50226005393992,2.30362907733559,5.55472033802837 +106,5,0,1.50226005393992,1.37909145501961,4.84064460409266 +106,6,0,1.50226005393992,1.71825363617274,4.87571321763657 +107,1,0,-0.89457220946846,-0.463955575441369,-3.48745734772936 +107,2,0,-0.89457220946846,-0.0519919382569897,0.125749218439857 +107,3,0,-0.89457220946846,0.520988865165158,-1.70695509814757 +107,4,0,-0.89457220946846,0.33236786268323,0.725662474427429 +107,5,0,-0.89457220946846,1.03760437559963,1.39038624668232 +107,6,0,-0.89457220946846,0.256795459568391,-0.335002534800443 +108,1,0,-1.22324299509621,-0.526114758338154,-0.692133068311805 +108,2,0,-1.22324299509621,0.18477604071721,0.429974089819244 +108,3,0,-1.22324299509621,0.286754140761167,0.372277811842254 +108,4,0,-1.22324299509621,0.542204942504875,0.697878652044274 +108,5,0,-1.22324299509621,0.399442852524572,1.14482729349123 +108,6,0,-1.22324299509621,1.27799739480744,0.760227606864043 +109,1,0,-0.758669466253882,-0.783003297910145,0.0513878206820995 +109,2,0,-0.758669466253882,0.285020986239263,-0.434316127626099 +109,3,0,-0.758669466253882,0.898896938193642,-1.10042010645776 +109,4,0,-0.758669466253882,0.828259841590877,0.643677661471694 +109,5,0,-0.758669466253882,0.380786238406674,1.65897088961278 +109,6,0,-0.758669466253882,0.456410360436729,1.16614884094835 +110,1,0,-2.28836186963232,-1.42240905596012,1.56363050881366 +110,2,0,-2.28836186963232,-1.21031092491361,2.01509164036705 +110,3,0,-2.28836186963232,-0.766586478955116,0.996018770236634 +110,4,0,-2.28836186963232,-0.412915805901137,1.21019829707552 +110,5,0,-2.28836186963232,-1.10526766855183,1.11050770085309 +110,6,0,-2.28836186963232,-0.0558900611664474,0.75444310848228 +111,1,0,0.79771570665335,-0.553133989340675,1.36541391115453 +111,2,0,0.79771570665335,0.246665282521328,2.08614423172338 +111,3,0,0.79771570665335,0.972160689944622,2.5652568519032 +111,4,0,0.79771570665335,0.376996285841898,2.30036239478882 +111,5,0,0.79771570665335,1.4268445074441,3.26085042603327 +111,6,0,0.79771570665335,1.94231661484459,5.31180379360411 +112,1,0,0.12870078897994,0.307506803447317,0.692955762744788 +112,2,0,0.12870078897994,-0.0268741505941875,2.4379863824427 +112,3,0,0.12870078897994,1.00343228065177,1.30208928752089 +112,4,0,0.12870078897994,0.131765893134764,2.86354301083545 +112,5,0,0.12870078897994,0.933523097093359,2.94208138077174 +112,6,0,0.12870078897994,1.76925438035231,3.72716043227665 +113,1,0,-0.00931074971757802,-0.397545128543512,-0.304337726872656 +113,2,0,-0.00931074971757802,0.504335269263596,-1.47696043309091 +113,3,0,-0.00931074971757802,0.69556377664362,0.706831865285902 +113,4,0,-0.00931074971757802,0.952311641049705,1.58895831015197 +113,5,0,-0.00931074971757802,1.72201418175824,1.80348015736981 +113,6,0,-0.00931074971757802,1.00346429177833,2.07938285516792 +114,1,0,-1.33814559514493,-0.13084645095529,0.258563622937842 +114,2,0,-1.33814559514493,-0.328313780755942,-0.755357255404722 +114,3,0,-1.33814559514493,0.767190991540338,0.547314979192388 +114,4,0,-1.33814559514493,0.551545429130956,1.51813317281849 +114,5,0,-1.33814559514493,0.280707403536271,0.585742471844865 +114,6,0,-1.33814559514493,1.87073018189421,1.79129333628737 +115,1,0,-0.0307450017978104,-0.203832382523738,-0.547529905930902 +115,2,0,-0.0307450017978104,0.803531851748503,-1.94762176066094 +115,3,0,-0.0307450017978104,-0.328550830765483,1.15802858200512 +115,4,0,-0.0307450017978104,1.08894515645791,0.479216783688697 +115,5,0,-0.0307450017978104,1.00890444877836,2.146119939363 +115,6,0,-0.0307450017978104,2.54984322597835,2.60041936200757 +116,1,0,0.416210502177554,0.800227902503501,-1.13776949509889 +116,2,0,0.416210502177554,0.72177376351145,-1.29095759397637 +116,3,0,0.416210502177554,1.05092852687046,0.163616534216729 +116,4,0,0.416210502177554,1.07841177778874,0.318569963738997 +116,5,0,0.416210502177554,1.27216409348855,2.62571443309508 +116,6,0,0.416210502177554,2.20291413903546,1.40992567038327 +117,1,0,0.185682053987045,0.0154632265573412,0.126843635097071 +117,2,0,0.185682053987045,-0.294568652078405,-1.59502600932928 +117,3,0,0.185682053987045,0.3434221572718,-0.0954442306696356 +117,4,0,0.185682053987045,0.736971931659641,0.410252145148292 +117,5,0,0.185682053987045,1.5296434146358,1.91722421838383 +117,6,0,0.185682053987045,1.67782346096464,1.01747266376817 +118,1,0,-1.59532950598604,-0.798373104920996,-1.00968400645432 +118,2,0,-1.59532950598604,-0.204361624072532,0.370830252251386 +118,3,0,-1.59532950598604,-0.896149987574084,0.484139473997054 +118,4,0,-1.59532950598604,0.141700472907281,-0.466405186376612 +118,5,0,-1.59532950598604,1.00839653260119,-0.026721952059122 +118,6,0,-1.59532950598604,0.332064509443496,0.821121599465879 +119,1,0,0.121268709978484,0.181681164145453,0.570191327984123 +119,2,0,0.121268709978484,0.519288120645323,2.03742277716536 +119,3,0,0.121268709978484,0.742228904360963,0.281887395792038 +119,4,0,0.121268709978484,0.580678820252124,2.26926858657859 +119,5,0,0.121268709978484,0.646598927456894,0.783493583393684 +119,6,0,0.121268709978484,1.40112832662989,0.82104317069335 +120,1,0,-0.523538252561795,-0.179295568123839,0.22212232449333 +120,2,0,-0.523538252561795,0.10946622333482,0.335398721679965 +120,3,0,-0.523538252561795,0.89486358654903,0.578696943291453 +120,4,0,-0.523538252561795,0.176361255514506,1.50645843591311 +120,5,0,-0.523538252561795,0.706148111155747,4.27936001732894 +120,6,0,-0.523538252561795,1.38773488273727,3.45033864189556 +121,1,3,-1.3209898980621,-0.659030256624808,-1.11457919929402 +121,2,3,-1.3209898980621,-0.141217731860885,1.28435903211288 +121,3,3,-1.3209898980621,-0.791920375267903,0.685288874745403 +121,4,3,-1.3209898980621,-0.0734820400267869,3.25785366872356 +121,5,3,-1.3209898980621,-0.49656205388661,2.02276889139347 +121,6,3,-1.3209898980621,-0.965589275967642,5.21429798519067 +122,1,3,0.65557245598294,0.507227828874015,-0.250570912893117 +122,2,3,0.65557245598294,1.03929914437131,-0.420264983423434 +122,3,3,0.65557245598294,1.34473746170499,1.76685337912262 +122,4,3,0.65557245598294,0.87359591944603,3.4820602234097 +122,5,3,0.65557245598294,0.892485396451848,5.95175331176178 +122,6,3,0.65557245598294,2.52111655846667,7.40612320147572 +123,1,3,0.0144327369651312,0.897110333973617,-0.887945988958214 +123,2,3,0.0144327369651312,0.562597501739654,0.177689505327719 +123,3,3,0.0144327369651312,1.19295988538516,2.78876965030089 +123,4,3,0.0144327369651312,-0.152912404575872,2.14857714673905 +123,5,3,0.0144327369651312,0.354572848563497,5.03305075612627 +123,6,3,0.0144327369651312,0.11038766791842,6.34960584861583 +124,1,3,0.06591379512311,0.0534203651908569,-0.843874058964572 +124,2,3,0.06591379512311,0.270394743140451,2.86834595033898 +124,3,3,0.06591379512311,0.129802985645562,3.91738873706921 +124,4,3,0.06591379512311,0.873770307631855,4.42769196153781 +124,5,3,0.06591379512311,0.983498849130909,7.0964325924724 +124,6,3,0.06591379512311,1.20719003484248,9.48929410346392 +125,1,3,-0.275046326426422,0.293249514156622,1.81714098956309 +125,2,3,-0.275046326426422,0.229476930121458,0.185804954844934 +125,3,3,-0.275046326426422,0.268736748455175,2.76347481395949 +125,4,3,-0.275046326426422,0.466683387349963,4.5530580706989 +125,5,3,-0.275046326426422,1.15612694018968,7.60432928703051 +125,6,3,-0.275046326426422,0.396102765747029,5.50714869110561 +126,1,3,-0.556089165281445,-0.204073561439021,0.917827508857894 +126,2,3,-0.556089165281445,0.19698200850372,2.39608998202929 +126,3,3,-0.556089165281445,1.33309830602468,2.51649382846799 +126,4,3,-0.556089165281445,0.813016580393121,3.75983978920604 +126,5,3,-0.556089165281445,0.279270401241487,5.81598295015132 +126,6,3,-0.556089165281445,0.767015417251097,6.57061869757607 +127,1,3,-0.481761250465059,-0.507168035910364,-0.907680621931876 +127,2,3,-0.481761250465059,0.782178870451505,-1.068489152912 +127,3,3,-0.481761250465059,-0.112056540437024,1.85024307065664 +127,4,3,-0.481761250465059,0.398656788397188,5.22249311314911 +127,5,3,-0.481761250465059,0.423006794578998,3.11951784142225 +127,6,3,-0.481761250465059,0.671986212079901,5.44040257022184 +128,1,3,-1.32740904200675,-1.65144181693399,1.33668575620608 +128,2,3,-1.32740904200675,0.13845682943084,-0.164424994709493 +128,3,3,-1.32740904200675,-0.186641909443694,1.74404216922443 +128,4,3,-1.32740904200675,-0.0193098809394121,1.16926685493817 +128,5,3,-1.32740904200675,0.260278867551969,3.41050128420705 +128,6,3,-1.32740904200675,0.426968699743162,5.81478718328758 +129,1,3,-0.835635156209953,0.365077037473928,1.17563749280077 +129,2,3,-0.835635156209953,0.0255047424585454,0.463487723132613 +129,3,3,-0.835635156209953,0.238261763497732,1.81748991235753 +129,4,3,-0.835635156209953,1.14995136482981,3.02643141094252 +129,5,3,-0.835635156209953,-0.368831270681908,4.53077069457151 +129,6,3,-0.835635156209953,0.494855790328605,3.7369366925089 +130,1,3,0.014617056865741,-0.920193437062664,-0.468859478425729 +130,2,3,0.014617056865741,-0.409771247075564,-0.195779473995372 +130,3,3,0.014617056865741,0.143316992860131,3.35793046974238 +130,4,3,0.014617056865741,1.16344796814541,2.90092316112646 +130,5,3,0.014617056865741,1.92474818134048,7.05539446585689 +130,6,3,0.014617056865741,1.23561893888835,7.4371222065699 +131,1,3,0.438622701805894,1.00256898424452,0.535588551592362 +131,2,3,0.438622701805894,0.891298196316596,-0.51181417381938 +131,3,3,0.438622701805894,1.09060025694346,2.97164644917054 +131,4,3,0.438622701805894,1.34453810826578,2.25079123994776 +131,5,3,0.438622701805894,1.40699884669438,3.59911808574875 +131,6,3,0.438622701805894,0.939875156111874,7.29199956126919 +132,1,3,-1.76243810418353,-1.55642713172473,-0.805043158381241 +132,2,3,-1.76243810418353,-0.391682064940616,-2.0263867595638 +132,3,3,-1.76243810418353,-0.666139512388104,-1.05765094965184 +132,4,3,-1.76243810418353,0.0238185660916042,0.817677818881005 +132,5,3,-1.76243810418353,0.400946115070112,2.91380467653736 +132,6,3,-1.76243810418353,0.913005144984613,2.69049089180893 +133,1,3,1.36534824167319,0.763831528913434,0.577023159081942 +133,2,3,1.36534824167319,0.844309096416079,0.042038282212332 +133,3,3,1.36534824167319,0.947415268176042,2.18378635404389 +133,4,3,1.36534824167319,1.43995238461224,4.92965591378078 +133,5,3,1.36534824167319,2.89808347087941,8.12380754151815 +133,6,3,1.36534824167319,1.10219087008148,9.40004516409809 +134,1,3,1.29446825743501,0.608144869107727,1.08618105043624 +134,2,3,1.29446825743501,1.41973826830158,2.32331581819127 +134,3,3,1.29446825743501,1.51878532055069,3.65140165693938 +134,4,3,1.29446825743501,1.69646592321119,5.36297031018112 +134,5,3,1.29446825743501,1.56911161772488,8.48595978983258 +134,6,3,1.29446825743501,1.90002513778021,9.35068411797129 +135,1,3,-0.178549336156913,0.151158553945093,0.828144325264999 +135,2,3,-0.178549336156913,-0.471060546327792,3.9089119030933 +135,3,3,-0.178549336156913,0.103993512674667,6.07766753358876 +135,4,3,-0.178549336156913,1.08664644183412,5.71413666854425 +135,5,3,-0.178549336156913,1.68404926041005,5.99317910927949 +135,6,3,-0.178549336156913,0.577078077823259,9.483960218053 +136,1,3,-1.07692090882478,-0.401929333738954,1.04747553448351 +136,2,3,-1.07692090882478,-0.545038593804227,0.890555345254023 +136,3,3,-1.07692090882478,0.508562503370516,0.983523883778991 +136,4,3,-1.07692090882478,0.00140170282829649,2.45797787912278 +136,5,3,-1.07692090882478,0.13965802945591,4.85004362804588 +136,6,3,-1.07692090882478,0.935084069218439,5.46362892185911 +137,1,3,0.728454193443001,-0.0114977260864277,0.974364596857209 +137,2,3,0.728454193443001,0.7135343982297,0.67155879830354 +137,3,3,0.728454193443001,0.655179531278547,2.90146090914518 +137,4,3,0.728454193443001,1.38622403146752,5.33426163164905 +137,5,3,0.728454193443001,1.12156269309693,7.6727252728244 +137,6,3,0.728454193443001,1.15693441030068,8.86462680988762 +138,1,3,-1.46938141572406,-0.0883249129282256,0.722072099170389 +138,2,3,-1.46938141572406,-0.142449306634193,0.823877896167457 +138,3,3,-1.46938141572406,-0.0298040848424808,2.30322550088994 +138,4,3,-1.46938141572406,-0.322736256121953,2.31264370539726 +138,5,3,-1.46938141572406,0.124282773210025,6.14394845899594 +138,6,3,-1.46938141572406,1.1023978140493,5.08916101818471 +139,1,3,-0.549298690203426,0.301171733824676,-0.227928347441122 +139,2,3,-0.549298690203426,0.402237345797786,-0.737524166727403 +139,3,3,-0.549298690203426,0.683518954814073,-1.0638203854028 +139,4,3,-0.549298690203426,0.403594704560704,2.16485431597404 +139,5,3,-0.549298690203426,0.55999493763121,4.51625222775326 +139,6,3,-0.549298690203426,1.06915987676031,4.39776263951282 +140,1,3,-1.07513120640263,-0.218917779907026,0.741651877086614 +140,2,3,-1.07513120640263,-0.412691091490353,-0.171855503087313 +140,3,3,-1.07513120640263,0.367202949190019,0.58558380183808 +140,4,3,-1.07513120640263,1.24324583274143,2.70757401857652 +140,5,3,-1.07513120640263,0.424084621174276,1.96616566887664 +140,6,3,-1.07513120640263,0.534559115396839,4.59790385921341 +141,1,3,0.589903379971464,0.518152792811317,0.532857161426118 +141,2,3,0.589903379971464,1.74119554926273,0.583690811233439 +141,3,3,0.589903379971464,1.14384744906574,1.11047142482919 +141,4,3,0.589903379971464,1.43423263831571,5.00913061015248 +141,5,3,0.589903379971464,1.74498549823101,6.56368416729218 +141,6,3,0.589903379971464,1.67289590250084,9.02370809506211 +142,1,3,0.657929099074857,-0.527630501829893,2.23509371293706 +142,2,3,0.657929099074857,0.790313265176156,3.28376796139343 +142,3,3,0.657929099074857,1.30011710750872,1.89310004655243 +142,4,3,0.657929099074857,1.27724881504038,5.63969392962316 +142,5,3,0.657929099074857,0.767272841138719,6.94525677505373 +142,6,3,0.657929099074857,2.32556023319219,9.74629505340234 +143,1,3,0.918209578132046,0.389990741964451,1.73002264959394 +143,2,3,0.918209578132046,0.324124967116553,2.51685777627825 +143,3,3,0.918209578132046,0.864710010354607,4.10998227389613 +143,4,3,0.918209578132046,0.721106454137181,5.74373385047243 +143,5,3,0.918209578132046,2.35220290569798,6.31425548414344 +143,6,3,0.918209578132046,1.06086818047735,9.11988048390896 +144,1,3,1.03723233079857,0.780710632652874,0.212208576948754 +144,2,3,1.03723233079857,1.05604424585774,-0.667835264144556 +144,3,3,1.03723233079857,0.452858041598415,2.84834674243207 +144,4,3,1.03723233079857,1.974918457812,5.49179716277523 +144,5,3,1.03723233079857,1.04103576092729,6.15510734290007 +144,6,3,1.03723233079857,2.13124097931571,10.0580109007073 +145,1,3,0.432864420882128,0.0546511643881509,2.06577211453651 +145,2,3,0.432864420882128,0.542564453524272,0.826589014872826 +145,3,3,0.432864420882128,-0.102986095234928,3.30224330441748 +145,4,3,0.432864420882128,0.818817860108457,5.56089893194988 +145,5,3,0.432864420882128,0.961458251664878,7.7240973086671 +145,6,3,0.432864420882128,1.27895634770374,8.02085741371424 +146,1,3,0.494887951274981,0.906621933718628,1.35833730554146 +146,2,3,0.494887951274981,0.609584098447873,1.50672641372537 +146,3,3,0.494887951274981,1.66806988803927,2.68420111093815 +146,4,3,0.494887951274981,1.17243386474274,5.77226847745143 +146,5,3,0.494887951274981,1.7367792550661,6.86343483473947 +146,6,3,0.494887951274981,1.32962006663717,7.39856300627146 +147,1,3,0.337514991797093,0.714474564182813,3.77502417711785 +147,2,3,0.337514991797093,-0.120717345754449,2.39855290111283 +147,3,3,0.337514991797093,0.979181408407107,3.3951013798209 +147,4,3,0.337514991797093,0.765822372463936,4.51187239398326 +147,5,3,0.337514991797093,1.1636592509228,7.32871256559361 +147,6,3,0.337514991797093,1.69073725736773,7.89889121515618 +148,1,3,0.940914420491235,0.32599988959407,-2.68432419188846 +148,2,3,0.940914420491235,0.902667702594575,-0.502223665400725 +148,3,3,0.940914420491235,1.79711547389592,2.72616983157362 +148,4,3,0.940914420491235,1.89564536968462,3.39439552886371 +148,5,3,0.940914420491235,1.86546638275212,4.46411002977733 +148,6,3,0.940914420491235,1.85126607219131,5.8921851033331 +149,1,3,0.457967792328404,0.378698935479082,4.12572103429067 +149,2,3,0.457967792328404,0.625145122806749,4.12170173700891 +149,3,3,0.457967792328404,0.657407339867929,5.67039906683291 +149,4,3,0.457967792328404,0.894496831842379,8.9524316705688 +149,5,3,0.457967792328404,1.27815558368229,9.34940165660425 +149,6,3,0.457967792328404,1.21757039960575,9.72894783793935 +150,1,3,1.28822951042595,0.929280238078882,-0.681541858459146 +150,2,3,1.28822951042595,0.37556044800664,-2.14029291717815 +150,3,3,1.28822951042595,0.955960970286821,1.49994481205458 +150,4,3,1.28822951042595,0.666797039509379,3.18273482219049 +150,5,3,1.28822951042595,1.93863479694882,6.10370652383497 +150,6,3,1.28822951042595,1.3261231323096,5.09851390581681 +151,1,3,0.799327933544311,0.752383684129809,0.998835172417796 +151,2,3,0.799327933544311,1.48794166496513,2.50739666840068 +151,3,3,0.799327933544311,0.636647413978102,3.39248141555062 +151,4,3,0.799327933544311,0.43183050965539,6.2241784706405 +151,5,3,0.799327933544311,1.60186143581385,6.57526813277581 +151,6,3,0.799327933544311,1.48009902563213,9.68047322868392 +152,1,3,0.641075436191527,0.410381761209829,0.181288378872355 +152,2,3,0.641075436191527,0.833387068122976,0.440335510368963 +152,3,3,0.641075436191527,1.02324616435653,1.92203052266619 +152,4,3,0.641075436191527,1.77222798638308,3.95065028029737 +152,5,3,0.641075436191527,1.50187251866824,2.58165781357217 +152,6,3,0.641075436191527,0.913951107889827,6.90369945886804 +153,1,3,0.647106275420512,-0.237400237086652,-2.02164808965544 +153,2,3,0.647106275420512,0.257978734055656,0.228139264140656 +153,3,3,0.647106275420512,1.75480644001202,-0.169015192170175 +153,4,3,0.647106275420512,0.41350545765392,3.79498507213573 +153,5,3,0.647106275420512,1.07445417959469,4.38245420741927 +153,6,3,0.647106275420512,0.98341038645438,6.61511100339346 +154,1,3,1.04666451517136,0.341354827336876,3.42913227448808 +154,2,3,1.04666451517136,0.672471846348521,0.616635516261048 +154,3,3,1.04666451517136,1.27811698735899,2.7748662991019 +154,4,3,1.04666451517136,1.48649127087623,6.01209090483621 +154,5,3,1.04666451517136,1.61543543912908,7.37520919143932 +154,6,3,1.04666451517136,2.02536607455159,9.48209414318584 +155,1,3,0.576975845608666,0.0914320774480271,0.722851227802534 +155,2,3,0.576975845608666,0.108522327794117,1.6890512266516 +155,3,3,0.576975845608666,0.259628061168946,2.58597206970974 +155,4,3,0.576975845608666,1.01811928925711,6.21498745277975 +155,5,3,0.576975845608666,0.719060589759526,7.07453482145899 +155,6,3,0.576975845608666,1.92334185803099,10.009696002852 +156,1,3,0.17037629608571,0.784349410920146,2.11235841463052 +156,2,3,0.17037629608571,0.214232082289887,-0.697328657084354 +156,3,3,0.17037629608571,1.25013551445887,3.29808927709344 +156,4,3,0.17037629608571,1.7086115578765,4.77325299010961 +156,5,3,0.17037629608571,1.11415545041536,5.70327263182313 +156,6,3,0.17037629608571,0.847156272453002,6.739097993206 +157,1,3,0.396115427083706,0.923645518828156,0.864477515155358 +157,2,3,0.396115427083706,0.648033309117745,3.31731508957503 +157,3,3,0.396115427083706,1.14817613766954,3.60719373954189 +157,4,3,0.396115427083706,0.420299068553708,5.09071471525354 +157,5,3,0.396115427083706,1.14833117298122,6.75113263123557 +157,6,3,0.396115427083706,1.07770858221266,8.92370742505083 +158,1,3,-0.38503981699399,0.249091044450303,-0.455481254742036 +158,2,3,-0.38503981699399,0.652965111326567,0.513194049834224 +158,3,3,-0.38503981699399,-0.203526544921249,3.97850435486181 +158,4,3,-0.38503981699399,0.178102153152252,3.50022698865287 +158,5,3,-0.38503981699399,0.238824839896899,5.90492745057116 +158,6,3,-0.38503981699399,1.41010656792478,6.139197871099 +159,1,3,-0.606966899370249,-0.235070624391252,1.67459208020634 +159,2,3,-0.606966899370249,0.397058569815924,-0.983876672431654 +159,3,3,-0.606966899370249,-0.0987236791816803,2.11597559238572 +159,4,3,-0.606966899370249,0.0175133414764335,4.93858440091871 +159,5,3,-0.606966899370249,1.07180450901133,6.65025467972839 +159,6,3,-0.606966899370249,1.03878178919866,4.64039122565767 +160,1,3,0.183609061947842,0.922644868366452,1.06263247541716 +160,2,3,0.183609061947842,-0.521822880184819,3.84919810086592 +160,3,3,0.183609061947842,1.26182116417092,2.87398711588164 +160,4,3,0.183609061947842,1.1296518783099,4.97402442219597 +160,5,3,0.183609061947842,1.16806292711789,6.59628616328911 +160,6,3,0.183609061947842,1.54910874984912,7.78160398847949 +161,1,3,-1.5739802437226,-0.738750875893334,0.646798997035059 +161,2,3,-1.5739802437226,-0.817190355326895,-0.199794544446061 +161,3,3,-1.5739802437226,0.00978813383769062,1.53768978518043 +161,4,3,-1.5739802437226,-0.0692027187562883,4.22559029952303 +161,5,3,-1.5739802437226,-0.321648430686517,2.49489898430561 +161,6,3,-1.5739802437226,1.80291081637126,2.97534946673223 +162,1,3,-1.5135182536791,-1.21415498016749,1.08344375432323 +162,2,3,-1.5135182536791,0.00862446836171632,0.366096436545889 +162,3,3,-1.5135182536791,-0.580560672020834,0.386396282908581 +162,4,3,-1.5135182536791,0.284802496374087,2.65431055942517 +162,5,3,-1.5135182536791,0.481985729465867,4.08004569199787 +162,6,3,-1.5135182536791,-0.164457779819998,5.44486509744182 +163,1,3,0.17709443103689,0.282609663048505,1.2187601244373 +163,2,3,0.17709443103689,0.411582509889598,3.46647644603637 +163,3,3,0.17709443103689,0.847495649227131,6.24284448978847 +163,4,3,0.17709443103689,1.00780264364244,6.26054223704011 +163,5,3,0.17709443103689,0.567458999918712,6.75896719170673 +163,6,3,0.17709443103689,1.73124741621038,8.55153680922603 +164,1,3,0.403250756295859,0.492850083886365,2.48897701813397 +164,2,3,0.403250756295859,0.540083950359147,2.10861656799587 +164,3,3,0.403250756295859,0.731550492755462,5.78483021182663 +164,4,3,0.403250756295859,0.458474142754207,6.25304959523288 +164,5,3,0.403250756295859,1.4995702617515,7.24373626356283 +164,6,3,0.403250756295859,0.984707497276815,8.31693696597542 +165,1,3,-1.75890966008881,-0.492384152667725,-3.0165269004572 +165,2,3,-1.75890966008881,-0.114806693432703,1.51062699564072 +165,3,3,-1.75890966008881,-0.48595164161601,-1.00828460304626 +165,4,3,-1.75890966008881,0.617535901530216,3.78881684855537 +165,5,3,-1.75890966008881,-0.477578070519953,4.53617118871654 +165,6,3,-1.75890966008881,0.0165545486318738,2.41304871152758 +166,1,3,0.550444344652174,0.720896381917308,-0.0902337840097769 +166,2,3,0.550444344652174,1.07643595531667,1.67174317972133 +166,3,3,0.550444344652174,1.23330497683997,3.30095700316899 +166,4,3,0.550444344652174,0.239620344875785,5.01348139892779 +166,5,3,0.550444344652174,0.572685885770811,5.04458155047439 +166,6,3,0.550444344652174,1.90034510518233,8.66633628083652 +167,1,3,-0.177518472281448,0.559127011742701,-2.84585414560063 +167,2,3,-0.177518472281448,0.671231625413232,0.97648769056059 +167,3,3,-0.177518472281448,0.596607991549095,0.67623314360395 +167,4,3,-0.177518472281448,0.28242453789717,2.26063148836001 +167,5,3,-0.177518472281448,0.772759320165847,4.26137543656067 +167,6,3,-0.177518472281448,0.0529782047803846,4.03342527507047 +168,1,3,-0.283378689715887,0.56880477901632,0.566393501073422 +168,2,3,-0.283378689715887,0.687228894918578,1.33767774653906 +168,3,3,-0.283378689715887,0.716981777257136,3.20528354672777 +168,4,3,-0.283378689715887,0.589459989413421,4.6418581200693 +168,5,3,-0.283378689715887,0.896581550535394,5.35234524586164 +168,6,3,-0.283378689715887,1.16641332038933,5.89859615929218 +169,1,3,0.0949255069380952,-0.138844826813786,0.880456211503483 +169,2,3,0.0949255069380952,0.899054228996397,1.70093245545876 +169,3,3,0.0949255069380952,0.375520297802488,2.34651154150143 +169,4,3,0.0949255069380952,0.765847242429938,7.0065917773773 +169,5,3,0.0949255069380952,0.770509647956809,4.98968023938993 +169,6,3,0.0949255069380952,1.34483793919838,6.9101710957784 +170,1,3,0.109708934474219,0.386120730420973,2.85160717995104 +170,2,3,0.109708934474219,0.407879747082585,1.33752020856814 +170,3,3,0.109708934474219,0.637178862432442,4.20098703126136 +170,4,3,0.109708934474219,0.207565281405001,4.77374671451531 +170,5,3,0.109708934474219,1.85419902965847,6.3939200558063 +170,6,3,0.109708934474219,1.61088867029117,6.59919416619873 +171,1,3,-1.02075201769416,-0.356310807707324,1.02592651163283 +171,2,3,-1.02075201769416,0.0424193927151826,0.741657135044775 +171,3,3,-1.02075201769416,0.0317264399471442,1.71370818238931 +171,4,3,-1.02075201769416,0.467235778780831,5.47811263281214 +171,5,3,-1.02075201769416,0.0791128775261391,4.12329110395588 +171,6,3,-1.02075201769416,0.895303915215752,5.00292845449822 +172,1,3,-0.214471204357173,-0.727646095799363,1.35131925370431 +172,2,3,-0.214471204357173,0.210944497892413,2.37175026318938 +172,3,3,-0.214471204357173,0.570841146082983,3.65951143912713 +172,4,3,-0.214471204357173,0.461650424820328,6.3022531708993 +172,5,3,-0.214471204357173,0.342228491003671,8.22419778472385 +172,6,3,-0.214471204357173,1.28117396857179,7.84810036822843 +173,1,3,0.95312723300276,0.587136635676131,2.27117731500969 +173,2,3,0.95312723300276,0.467985411032795,2.69182871995691 +173,3,3,0.95312723300276,0.768091000867892,6.44965476196657 +173,4,3,0.95312723300276,1.53518612893103,6.20379049823067 +173,5,3,0.95312723300276,0.734032956366248,8.37840205107447 +173,6,3,0.95312723300276,0.880826456225791,10.6071742329299 +174,1,3,-0.299503059426975,-0.0143918053073009,0.165378654819905 +174,2,3,-0.299503059426975,0.351702004390223,1.14652818851001 +174,3,3,-0.299503059426975,1.35443661771852,1.34930878825417 +174,4,3,-0.299503059426975,0.641451213769214,2.97573642323419 +174,5,3,-0.299503059426975,0.792151951680742,3.2784114112508 +174,6,3,-0.299503059426975,0.559389848134574,7.10103746423127 +175,1,3,-1.35656841055337,-0.68822256639622,1.09383253065236 +175,2,3,-1.35656841055337,0.165950349733725,2.32705481637459 +175,3,3,-1.35656841055337,-0.0394690022428881,1.58889766941899 +175,4,3,-1.35656841055337,-0.304479039922878,0.857436014714444 +175,5,3,-1.35656841055337,1.07142210914254,2.26750720166773 +175,6,3,-1.35656841055337,1.22342980686212,3.62877035996801 +176,1,3,-0.722906834501834,0.163769122541646,-0.632232933093176 +176,2,3,-0.722906834501834,0.0157880302844722,0.938801879860888 +176,3,3,-0.722906834501834,0.317548010285299,3.67834365574177 +176,4,3,-0.722906834501834,0.320655034651504,2.99198807090912 +176,5,3,-0.722906834501834,1.20799091004943,3.5296721551273 +176,6,3,-0.722906834501834,1.4715576177269,3.80599815112298 +177,1,3,-0.873027113678424,-0.134710889146149,1.20554182330847 +177,2,3,-0.873027113678424,-0.343203700912485,1.08472943367274 +177,3,3,-0.873027113678424,0.68383414113027,4.85152313687422 +177,4,3,-0.873027113678424,-0.0307043288901434,4.54226192906532 +177,5,3,-0.873027113678424,0.476565413251091,8.21586862915938 +177,6,3,-0.873027113678424,0.692592828471272,7.78871269859934 +178,1,3,0.04325400319841,0.483812986689953,2.23748077948003 +178,2,3,0.04325400319841,0.124723740750307,0.0257740658283268 +178,3,3,0.04325400319841,-0.0236415679346836,2.07893121569823 +178,4,3,0.04325400319841,0.38645207413081,4.73747920429682 +178,5,3,0.04325400319841,0.911079572477391,6.13334250277362 +178,6,3,0.04325400319841,1.56328403641471,5.10634606436181 +179,1,3,-0.705597138651361,-0.203512657961043,0.0705300276670152 +179,2,3,-0.705597138651361,-0.500641996603977,1.80939716425173 +179,3,3,-0.705597138651361,0.443055984083823,1.22782580215329 +179,4,3,-0.705597138651361,0.785627941108739,3.54952364932024 +179,5,3,-0.705597138651361,0.370592747588609,5.44620669723755 +179,6,3,-0.705597138651361,0.784335634233162,6.91807435557 +180,1,3,2.06398866391933,1.05930399943233,-0.00767017810784587 +180,2,3,2.06398866391933,0.974937384643123,1.3889518162926 +180,3,3,2.06398866391933,1.67488127697509,4.941828354601 +180,4,3,2.06398866391933,1.54166970733265,6.37226663940419 +180,5,3,2.06398866391933,1.55908201581147,7.63019049127558 +180,6,3,2.06398866391933,3.13182760994635,11.0824508424514 +181,1,3,0.360194044770426,0.137219648791359,-0.120984723180791 +181,2,3,0.360194044770426,0.38551064785073,1.30910385832044 +181,3,3,0.360194044770426,0.414240176764444,2.40758725231587 +181,4,3,0.360194044770426,1.03798805335224,4.04969508727083 +181,5,3,0.360194044770426,1.26482943472362,7.05018989645842 +181,6,3,0.360194044770426,0.674817592128119,8.06113032044475 +182,1,3,-1.68147033404543,-0.529992351263284,-0.468300262616531 +182,2,3,-1.68147033404543,-0.361074846598351,-0.82128675411022 +182,3,3,-1.68147033404543,0.193761651635151,2.11230702234137 +182,4,3,-1.68147033404543,0.217264661905855,1.47293064025627 +182,5,3,-1.68147033404543,-0.887821017937981,2.24634296880867 +182,6,3,-1.68147033404543,0.741792955070734,3.71110597293242 +183,1,3,-0.196778992696935,-0.646729030890644,-1.00205754402073 +183,2,3,-0.196778992696935,0.0139864871831406,0.953921068284142 +183,3,3,-0.196778992696935,0.0795937312953826,0.860583458955781 +183,4,3,-0.196778992696935,0.837209243653747,4.51956682189724 +183,5,3,-0.196778992696935,0.601170803412534,4.97723339242763 +183,6,3,-0.196778992696935,1.47422885794644,5.32121969953623 +184,1,3,0.730421144749429,-0.139311224235713,0.171081394747952 +184,2,3,0.730421144749429,1.53996280134507,1.94344486906202 +184,3,3,0.730421144749429,0.782149116082784,3.23383839820865 +184,4,3,0.730421144749429,0.609823463790785,2.62356551558351 +184,5,3,0.730421144749429,1.89101976789857,6.35341741228886 +184,6,3,0.730421144749429,1.72754545713741,7.05230497645335 +185,1,3,-0.0863000134300777,0.953213880450527,0.0349463777858372 +185,2,3,-0.0863000134300777,0.24448596475191,0.745217744798797 +185,3,3,-0.0863000134300777,0.817473988609581,3.84902626581413 +185,4,3,-0.0863000134300777,0.341876847218706,3.75460775000166 +185,5,3,-0.0863000134300777,0.648765871009181,5.82819842016675 +185,6,3,-0.0863000134300777,1.0361709300895,6.16150290917458 +186,1,3,0.506421572168642,0.367175153198149,-0.150215409953131 +186,2,3,0.506421572168642,0.567547043778752,1.16468293926085 +186,3,3,0.506421572168642,0.622465315259915,2.3988406787079 +186,4,3,0.506421572168642,0.394663508169827,6.48536024802799 +186,5,3,0.506421572168642,0.884648088357487,5.87931879381252 +186,6,3,0.506421572168642,1.2150386952591,7.23309433364219 +187,1,3,0.241411363100991,0.926007128421722,0.230728131241308 +187,2,3,0.241411363100991,0.613940209317918,0.0577344374340867 +187,3,3,0.241411363100991,1.98051616149487,2.11188576701728 +187,4,3,0.241411363100991,-0.0482207260230793,1.6364886842494 +187,5,3,0.241411363100991,1.13745180089868,3.05389500156471 +187,6,3,0.241411363100991,1.43270154624331,4.83798493222586 +188,1,3,-0.291937535972332,-0.432249249948712,-3.09229985520743 +188,2,3,-0.291937535972332,0.267479708595359,-1.57889714548199 +188,3,3,-0.291937535972332,0.415797657047746,0.287257962183195 +188,4,3,-0.291937535972332,-0.473326451583243,1.62858283902274 +188,5,3,-0.291937535972332,0.250647366273356,1.21906452860634 +188,6,3,-0.291937535972332,1.39466252891878,4.89754797997687 +189,1,3,-1.42673622400026,0.0874786130907167,-0.492174801618482 +189,2,3,-1.42673622400026,-0.360288142109093,-0.590640207208217 +189,3,3,-1.42673622400026,-0.00295717580758077,0.594372840951305 +189,4,3,-1.42673622400026,0.250174512466053,4.34575202958559 +189,5,3,-1.42673622400026,0.331734399880543,2.33611571216873 +189,6,3,-1.42673622400026,0.496807974220238,4.54737715581265 +190,1,3,-0.2115984836605,0.60223614723348,0.833538926670778 +190,2,3,-0.2115984836605,-0.731387152145866,2.22483251339626 +190,3,3,-0.2115984836605,1.278125494643,2.18564298307356 +190,4,3,-0.2115984836605,0.78864245668234,5.51233312764889 +190,5,3,-0.2115984836605,1.26236600590622,6.24849207856658 +190,6,3,-0.2115984836605,1.05785986019084,8.69015711295789 +191,1,5,0.983592225005866,0.685269042736118,1.38335066496562 +191,2,5,0.983592225005866,0.544502809451078,2.63622648994195 +191,3,5,0.983592225005866,1.27722602650698,1.7975669580347 +191,4,5,0.983592225005866,0.921971281461675,4.00241556657792 +191,5,5,0.983592225005866,1.16159622356324,3.40768352363552 +191,6,5,0.983592225005866,1.87368858096347,6.01936776577239 +192,1,5,-0.283157026746315,0.0104512285659374,1.27997348572735 +192,2,5,-0.283157026746315,-0.0834672788684822,-1.10599779071778 +192,3,5,-0.283157026746315,-0.232655886244044,-0.454134990455851 +192,4,5,-0.283157026746315,0.723728064985206,1.68155470824088 +192,5,5,-0.283157026746315,1.6857236853256,1.87155358218702 +192,6,5,-0.283157026746315,1.44827549669874,5.95064220847003 +193,1,5,0.00523970150119662,0.276436364797033,-1.93364739079216 +193,2,5,0.00523970150119662,-0.286764607187908,-1.76658683663304 +193,3,5,0.00523970150119662,0.830021732791269,-1.31917039846352 +193,4,5,0.00523970150119662,0.0683583315476364,-0.585846233099063 +193,5,5,0.00523970150119662,1.41795184080921,1.07536146125153 +193,6,5,0.00523970150119662,0.820462889803177,4.78603679649615 +194,1,5,0.92545213318825,1.31461163819231,2.92951584215972 +194,2,5,0.92545213318825,1.90310839616577,0.226299788014934 +194,3,5,0.92545213318825,1.87138107688812,3.63924728317667 +194,4,5,0.92545213318825,1.47734438773685,4.09859230112481 +194,5,5,0.92545213318825,1.61762449653127,5.5875297297966 +194,6,5,0.92545213318825,1.53505759071181,5.73026764736178 +195,1,5,-0.635670393667737,0.92906119673608,0.727083910123489 +195,2,5,-0.635670393667737,0.945527892927173,1.89213182433102 +195,3,5,-0.635670393667737,-0.0488719634014442,2.509396606672 +195,4,5,-0.635670393667737,0.589292270566448,1.32663557253803 +195,5,5,-0.635670393667737,0.481316722409301,3.19923450497982 +195,6,5,-0.635670393667737,1.08897508741881,4.12014467427658 +196,1,5,1.15670868958139,0.524265461334005,1.14361307382569 +196,2,5,1.15670868958139,1.09018652226291,1.7777630785721 +196,3,5,1.15670868958139,1.17365751121989,2.80845429477206 +196,4,5,1.15670868958139,0.519026773875079,4.76364255763518 +196,5,5,1.15670868958139,1.52994359394921,4.67118251896483 +196,6,5,1.15670868958139,1.00861805373324,7.97521550687341 +197,1,5,1.78969021425029,0.4434355361539,1.75105876560352 +197,2,5,1.78969021425029,2.26138915469364,3.18412757427007 +197,3,5,1.78969021425029,2.03441329772634,4.87704223939755 +197,4,5,1.78969021425029,2.19733416987764,5.07213816231237 +197,5,5,1.78969021425029,1.86882625281979,7.36445273426155 +197,6,5,1.78969021425029,0.869774157574492,7.89045853098738 +198,1,5,-0.149019232380438,0.906656488536891,0.617582157062771 +198,2,5,-0.149019232380438,-0.303441296388933,1.74706459113415 +198,3,5,-0.149019232380438,-0.0386216673435962,1.41188339797885 +198,4,5,-0.149019232380438,0.227646993112056,2.47893997311591 +198,5,5,-0.149019232380438,1.44183369734344,3.68740351555302 +198,6,5,-0.149019232380438,0.337064359767777,4.93171424134225 +199,1,5,-0.261067529046921,1.10193513549883,4.07397809127892 +199,2,5,-0.261067529046921,0.831821005947784,2.64115271413386 +199,3,5,-0.261067529046921,1.0507459873153,2.5984635365937 +199,4,5,-0.261067529046921,0.954457862577376,3.15005778434285 +199,5,5,-0.261067529046921,0.726355188233059,6.09140004175636 +199,6,5,-0.261067529046921,0.925631026478668,6.26570200917389 +200,1,5,-0.279422558021935,-1.14039912066845,-3.71994209961769 +200,2,5,-0.279422558021935,-1.06989083106309,-1.03919624114515 +200,3,5,-0.279422558021935,1.36983043403129,-0.00638631184268384 +200,4,5,-0.279422558021935,1.08189923332103,0.228665527854604 +200,5,5,-0.279422558021935,0.902665178547284,2.26697554552666 +200,6,5,-0.279422558021935,0.731355545786545,6.36441088594964 +201,1,5,-0.867508001059406,-0.233675811236789,-1.49758007707428 +201,2,5,-0.867508001059406,-0.550883931428525,0.374386856211282 +201,3,5,-0.867508001059406,0.0857728512645791,0.0294364273898407 +201,4,5,-0.867508001059406,-0.232389250145715,1.42191855779375 +201,5,5,-0.867508001059406,1.33590509096511,2.55688670984156 +201,6,5,-0.867508001059406,0.534449929648672,1.42124688873966 +202,1,5,-0.266028106354418,-0.128148387025209,0.975074215504778 +202,2,5,-0.266028106354418,0.446467815751043,-0.0510663436929477 +202,3,5,-0.266028106354418,0.0897105442552565,1.67391335328299 +202,4,5,-0.266028106354418,0.268223994946335,2.19974815286627 +202,5,5,-0.266028106354418,0.298937620984014,3.08651582214762 +202,6,5,-0.266028106354418,1.19175101086346,5.35184933489553 +203,1,5,0.0257161972054362,-0.00777182454554659,0.887316769481998 +203,2,5,0.0257161972054362,0.195171140233948,-2.34645529619854 +203,3,5,0.0257161972054362,0.943185858944576,-0.886172854562971 +203,4,5,0.0257161972054362,0.97297438051478,0.999551256331918 +203,5,5,0.0257161972054362,1.5413637280623,1.46967773439603 +203,6,5,0.0257161972054362,1.09237538286599,4.74829328190536 +204,1,5,0.116551434634878,0.778053422766819,0.923901269733919 +204,2,5,0.116551434634878,0.283905891981172,0.912512995323872 +204,3,5,0.116551434634878,0.939725254375677,1.49793081521857 +204,4,5,0.116551434634878,-0.105034571045226,1.4170606846868 +204,5,5,0.116551434634878,0.0742100832343512,2.88003343428499 +204,6,5,0.116551434634878,2.19181543302645,4.89563372429911 +205,1,5,-0.553125937781561,0.0859988450814071,-0.126962865847536 +205,2,5,-0.553125937781561,-0.113794244285939,1.04790901416167 +205,3,5,-0.553125937781561,1.50265709885282,-0.0121647533712959 +205,4,5,-0.553125937781561,0.753642118076197,-1.87801353698332 +205,5,5,-0.553125937781561,0.798974629882657,1.81444078247793 +205,6,5,-0.553125937781561,1.22940991518573,2.72413883940338 +206,1,5,0.756255260174997,1.37286170808534,-0.0618669650195494 +206,2,5,0.756255260174997,0.0270268243862444,3.87004440747301 +206,3,5,0.756255260174997,0.100304660700929,2.62002685969236 +206,4,5,0.756255260174997,1.50297292169296,2.10325982384873 +206,5,5,0.756255260174997,1.47481298806093,5.08851039988119 +206,6,5,0.756255260174997,1.49517633071853,5.329207300942 +207,1,5,0.402919292696054,0.448184026029819,1.6851353088623 +207,2,5,0.402919292696054,-0.309345001830238,-0.0480430976597863 +207,3,5,0.402919292696054,1.22136648402251,1.95683410550332 +207,4,5,0.402919292696054,1.38816029165409,1.56554848066498 +207,5,5,0.402919292696054,1.12448389959517,4.80323957292905 +207,6,5,0.402919292696054,0.670591444723001,7.61710463034625 +208,1,5,-0.269406763993592,0.206667648141733,2.22207660012001 +208,2,5,-0.269406763993592,0.0288576177863964,1.86463949430405 +208,3,5,-0.269406763993592,1.3635139605955,2.09011945100763 +208,4,5,-0.269406763993592,1.17538271934532,1.67784711763015 +208,5,5,-0.269406763993592,0.615716880287763,4.45742510031184 +208,6,5,-0.269406763993592,0.974785266523716,3.3847857749142 +209,1,5,1.37322747966558,1.51005515038624,-1.95142063469521 +209,2,5,1.37322747966558,0.66548558946035,-0.00933320143897454 +209,3,5,1.37322747966558,0.368067356457127,1.92149373004633 +209,4,5,1.37322747966558,2.2044979582161,2.22552051395123 +209,5,5,1.37322747966558,1.32532189498429,4.42699163117127 +209,6,5,1.37322747966558,2.29794574303181,6.54842437979203 +210,1,5,-1.21546340892904,-1.09510776302794,2.16313335716541 +210,2,5,-1.21546340892904,-0.0869249651887154,-0.113012776498633 +210,3,5,-1.21546340892904,-0.279230661394171,1.56240828061579 +210,4,5,-1.21546340892904,-0.049521274309301,4.51715372142149 +210,5,5,-1.21546340892904,0.00657525039390539,2.86563277524601 +210,6,5,-1.21546340892904,-0.0645347156398759,2.47481350145028 +211,1,5,-0.0701117965547371,-0.715929786867697,0.0255299412037402 +211,2,5,-0.0701117965547371,1.36168105909399,0.998667609709119 +211,3,5,-0.0701117965547371,-0.135807675651325,1.56559791547375 +211,4,5,-0.0701117965547371,1.51851847154815,1.02220638575337 +211,5,5,-0.0701117965547371,0.165993948214305,4.03246492986913 +211,6,5,-0.0701117965547371,0.959425763838153,3.69187032029544 +212,1,5,1.52424799429573,0.754050279225861,1.82652570977404 +212,2,5,1.52424799429573,1.36008329509545,1.87592392729932 +212,3,5,1.52424799429573,1.75505071496496,4.67880928526713 +212,4,5,1.52424799429573,1.22602677201853,6.36476797191923 +212,5,5,1.52424799429573,1.44265784000527,6.45907084431469 +212,6,5,1.52424799429573,1.81542507698388,8.88774350019465 +213,1,5,-1.13898322177537,-0.704850530259102,0.989954312743611 +213,2,5,-1.13898322177537,0.269910398419923,2.64079703302344 +213,3,5,-1.13898322177537,-0.30661493225157,0.234400241959169 +213,4,5,-1.13898322177537,0.173200709565968,0.115256042285581 +213,5,5,-1.13898322177537,-0.174767967493852,2.37243004034063 +213,6,5,-1.13898322177537,0.062705181813378,3.15196239199807 +214,1,5,0.0136635721568709,0.281286858491064,0.308737894502187 +214,2,5,0.0136635721568709,0.709632912619912,2.86447518064835 +214,3,5,0.0136635721568709,0.885759818762133,1.60433385861157 +214,4,5,0.0136635721568709,0.508252732894141,2.9767702147517 +214,5,5,0.0136635721568709,0.995181587793084,4.04309942636804 +214,6,5,0.0136635721568709,0.952559089661326,4.772002205928 +215,1,5,-0.524976137743338,-0.608745128024079,-0.623987422658077 +215,2,5,-0.524976137743338,0.0441390976086862,0.3386677182339 +215,3,5,-0.524976137743338,1.013040638404,0.859440864419736 +215,4,5,-0.524976137743338,0.368850033899774,-0.818130100579811 +215,5,5,-0.524976137743338,1.28996274747004,0.747054019336613 +215,6,5,-0.524976137743338,0.521169572898015,1.66670573365348 +216,1,5,0.528119708420632,0.651477084437435,-0.227441920101093 +216,2,5,0.528119708420632,0.581493185777966,0.0857339576458841 +216,3,5,0.528119708420632,1.74913980629871,0.254230180328475 +216,4,5,0.528119708420632,0.897716621442405,2.42565796191354 +216,5,5,0.528119708420632,1.71655243592702,2.22339067686188 +216,6,5,0.528119708420632,1.34115667416258,4.07122551146554 +217,1,5,-0.278139356660236,-0.432553538863004,-0.781011633085371 +217,2,5,-0.278139356660236,0.422378930946401,-0.567678688198085 +217,3,5,-0.278139356660236,0.562491377327001,-0.965369032587661 +217,4,5,-0.278139356660236,0.500297950693926,0.934629473125308 +217,5,5,-0.278139356660236,1.31421861315264,0.758231973389853 +217,6,5,-0.278139356660236,1.35139823865584,2.03912728069027 +218,1,5,0.49043594073852,0.321110560676538,0.510384585688465 +218,2,5,0.49043594073852,0.706729675579629,-1.04299998018294 +218,3,5,0.49043594073852,0.880495568887448,1.15469874941526 +218,4,5,0.49043594073852,0.865795526845137,2.38447041558231 +218,5,5,0.49043594073852,1.36781795204138,3.44280884506035 +218,6,5,0.49043594073852,2.36944881552513,8.16161080730789 +219,1,5,-1.98958128344518,-0.534903390163236,1.42588349306103 +219,2,5,-1.98958128344518,-0.373290925434039,1.36822515439796 +219,3,5,-1.98958128344518,0.158331617880881,-0.926249260297009 +219,4,5,-1.98958128344518,-0.607765725191721,-1.20272573134459 +219,5,5,-1.98958128344518,0.499900456385467,-0.65847289715537 +219,6,5,-1.98958128344518,-0.128902846011836,2.52464534710552 +220,1,5,-0.213493340618975,0.185391211106394,1.26700247541726 +220,2,5,-0.213493340618975,0.0140852828787207,-1.4427375813286 +220,3,5,-0.213493340618975,0.0174615772931458,-0.306210357001648 +220,4,5,-0.213493340618975,0.539954347150334,1.81353252436983 +220,5,5,-0.213493340618975,0.663050221600149,3.42996813535437 +220,6,5,-0.213493340618975,0.611350423560622,4.1990652063131 +221,1,5,-0.877521252652733,-0.558494533889764,-1.24506994830942 +221,2,5,-0.877521252652733,-0.054670778540151,-1.05245973258249 +221,3,5,-0.877521252652733,-0.434196160948057,-0.49282114431209 +221,4,5,-0.877521252652733,0.178911659606747,-2.59280555529188 +221,5,5,-0.877521252652733,0.68317178405947,1.53212805684875 +221,6,5,-0.877521252652733,1.00294622561415,2.4801642646124 +222,1,5,1.81905705063474,1.37293696497615,1.80371980454759 +222,2,5,1.81905705063474,1.72673008537909,2.32717660429447 +222,3,5,1.81905705063474,1.37469608885288,2.83327951835427 +222,4,5,1.81905705063474,1.44072915243679,5.45277832067784 +222,5,5,1.81905705063474,1.94592738842887,8.26489174542936 +222,6,5,1.81905705063474,2.65622088626748,8.25819620853794 +223,1,5,0.852925428022894,1.67333324392399,0.922675753730593 +223,2,5,0.852925428022894,1.16539499869153,3.04908307330675 +223,3,5,0.852925428022894,1.73698490260616,1.69916384611494 +223,4,5,0.852925428022894,1.13029237034851,4.50861305072888 +223,5,5,0.852925428022894,0.725190204392934,4.0963264949155 +223,6,5,0.852925428022894,1.49173024477212,7.68586493395678 +224,1,5,-0.517878257327904,-1.35732639663923,-1.23119186049687 +224,2,5,-0.517878257327904,1.11491866568095,-0.315709039321787 +224,3,5,-0.517878257327904,0.587780945805903,-0.369763432693986 +224,4,5,-0.517878257327904,0.927163157395854,-0.506417691026338 +224,5,5,-0.517878257327904,0.488711607226898,0.686305714138264 +224,6,5,-0.517878257327904,0.385312701261675,3.62404644990348 +225,1,5,-1.11257146628431,-0.475426417311899,-1.04280843910047 +225,2,5,-1.11257146628431,-1.18634048699766,-2.30358967343468 +225,3,5,-1.11257146628431,0.091173974222244,0.945624995795265 +225,4,5,-1.11257146628431,0.238514872098877,-1.99598194201174 +225,5,5,-1.11257146628431,0.425818445029469,1.50873654886708 +225,6,5,-1.11257146628431,1.05295883081395,2.07497420733641 +226,1,5,-1.81489141645341,-1.24259664095849,1.51719448330531 +226,2,5,-1.81489141645341,-0.554428904590342,1.41446259490764 +226,3,5,-1.81489141645341,-0.299740806185905,2.18468422027469 +226,4,5,-1.81489141645341,0.542734837538821,0.163871949464976 +226,5,5,-1.81489141645341,-0.298329304331962,0.435908366646806 +226,6,5,-1.81489141645341,0.414989726238207,1.96641824618135 +227,1,5,-0.260227815842348,-1.03005150290773,-0.019407951568893 +227,2,5,-0.260227815842348,0.468446628646404,0.0285719838885757 +227,3,5,-0.260227815842348,0.470855524402432,-0.174855338773838 +227,4,5,-0.260227815842348,1.44176617213929,0.69418426946141 +227,5,5,-0.260227815842348,0.226607556826976,4.60969384532652 +227,6,5,-0.260227815842348,0.906000249868074,3.70417043721014 +228,1,5,-0.248887440272954,-0.418765638490002,-0.0863136410068941 +228,2,5,-0.248887440272954,0.314102203471253,2.38681030771552 +228,3,5,-0.248887440272954,0.362571171005034,2.76383374351119 +228,4,5,-0.248887440272954,0.47788363062241,3.55624065831369 +228,5,5,-0.248887440272954,0.520531301129979,3.77951398822541 +228,6,5,-0.248887440272954,1.24345219848178,4.81422068100672 +229,1,5,-0.300211489798693,-0.603458799036538,0.129483577367022 +229,2,5,-0.300211489798693,0.379696846594982,2.39422380550729 +229,3,5,-0.300211489798693,0.285898323584938,0.56453671209308 +229,4,5,-0.300211489798693,-0.343957165096895,1.81928926476601 +229,5,5,-0.300211489798693,1.34665793798712,1.60250849670397 +229,6,5,-0.300211489798693,1.38600178614644,4.44490888146164 +230,1,5,2.18079606684413,0.0328589209121795,1.38593922837778 +230,2,5,2.18079606684413,2.20428194064908,3.38086181845178 +230,3,5,2.18079606684413,1.69611948536251,3.41464459135443 +230,4,5,2.18079606684413,2.64727843019363,4.4219208488337 +230,5,5,2.18079606684413,2.91539831678562,6.44465284207725 +230,6,5,2.18079606684413,2.02577390016517,10.3074908553895 +231,1,5,0.682258613138577,0.232553052940103,0.575192037116431 +231,2,5,0.682258613138577,0.934575833052082,2.93725402467604 +231,3,5,0.682258613138577,1.79765272992026,4.02869272214191 +231,4,5,0.682258613138577,0.700136142980352,5.18621671240476 +231,5,5,0.682258613138577,1.29305205341976,7.67676120198773 +231,6,5,0.682258613138577,1.47254977346884,6.40465083371395 +232,1,5,0.738564646474746,0.255708737415919,2.17066071793656 +232,2,5,0.738564646474746,1.30570948732827,1.61187659853227 +232,3,5,0.738564646474746,1.3389155585084,3.07698051626951 +232,4,5,0.738564646474746,2.51777450060082,3.29898038024719 +232,5,5,0.738564646474746,2.01416388727751,4.04999085321885 +232,6,5,0.738564646474746,1.14429575274794,6.22284218610352 +233,1,5,1.31891652792444,0.943432331705388,-0.0289136497162035 +233,2,5,1.31891652792444,0.978239063144288,2.06170299473121 +233,3,5,1.31891652792444,0.310622694521734,3.69752660270523 +233,4,5,1.31891652792444,0.68052796604515,1.5204378915292 +233,5,5,1.31891652792444,1.70099796725061,5.71280736379098 +233,6,5,1.31891652792444,2.00471022400003,5.58278554448763 +234,1,5,1.28907990310246,1.65412984713345,-0.602892382604196 +234,2,5,1.28907990310246,0.853258601018456,0.907141408866408 +234,3,5,1.28907990310246,1.60150771388762,3.24796159869254 +234,4,5,1.28907990310246,1.96933258012763,4.19436697567741 +234,5,5,1.28907990310246,1.27632561755083,4.99253953190101 +234,6,5,1.28907990310246,1.77582680830315,4.8548769273247 +235,1,5,0.978070722904636,0.948150182775719,-1.03874117061971 +235,2,5,0.978070722904636,0.543444277286829,0.925766870985002 +235,3,5,0.978070722904636,1.49477068531528,2.21839953506068 +235,4,5,0.978070722904636,1.56819489132473,1.93251425438558 +235,5,5,0.978070722904636,1.59527502367316,3.20318666486669 +235,6,5,0.978070722904636,1.35543245600248,6.86424987830461 +236,1,5,0.00643884641450284,-0.641212010033781,-0.575014663395219 +236,2,5,0.00643884641450284,0.937855659547919,2.06384914461294 +236,3,5,0.00643884641450284,0.739732482846854,1.41647034926797 +236,4,5,0.00643884641450284,1.10190774770209,3.79348717413345 +236,5,5,0.00643884641450284,1.02193117870911,3.04259873537341 +236,6,5,0.00643884641450284,0.0204587745911464,7.00891684792682 +237,1,5,0.0165814801249611,-0.305408606389655,1.76392967553631 +237,2,5,0.0165814801249611,0.267051028226902,-1.10004204212255 +237,3,5,0.0165814801249611,0.319994248367331,2.40546891854285 +237,4,5,0.0165814801249611,1.43776249361238,2.12131653600467 +237,5,5,0.0165814801249611,1.14569000055294,2.5595375324857 +237,6,5,0.0165814801249611,0.435442855090185,4.89078063646968 +238,1,5,1.91587682170855,1.28061275956914,-0.647304703918855 +238,2,5,1.91587682170855,0.682339854322889,1.91715709450631 +238,3,5,1.91587682170855,0.682233140309582,2.42845110380224 +238,4,5,1.91587682170855,1.19451909853685,5.00427468249585 +238,5,5,1.91587682170855,1.36885315652453,5.70056465024374 +238,6,5,1.91587682170855,1.52792382399838,8.49809489229398 +239,1,5,-1.76542576063516,-1.30871766053306,0.221759257176214 +239,2,5,-1.76542576063516,-0.706967060937396,-0.283031516205847 +239,3,5,-1.76542576063516,-0.0826534122809553,-0.199081921554736 +239,4,5,-1.76542576063516,0.553569425457922,0.804541456652704 +239,5,5,-1.76542576063516,-0.425046079826805,0.893639807836897 +239,6,5,-1.76542576063516,1.118581133231,2.48466150451466 +240,1,5,-1.47447432420743,-0.590776716929888,-2.04062301930976 +240,2,5,-1.47447432420743,-0.0386169438484504,-1.89717081094193 +240,3,5,-1.47447432420743,0.210964261447659,1.17208473689677 +240,4,5,-1.47447432420743,-0.640970403235413,-1.29107091503318 +240,5,5,-1.47447432420743,0.758534883730153,0.323916789750266 +240,6,5,-1.47447432420743,1.2629442004812,1.9383167523482 +241,1,5,0.444480192099237,0.618633887994417,-2.68240112037008 +241,2,5,0.444480192099237,0.386435895591268,-2.42533638325667 +241,3,5,0.444480192099237,0.0306556682985822,-0.208110794073889 +241,4,5,0.444480192099237,1.65754806459742,1.05287475605699 +241,5,5,0.444480192099237,1.65965765467233,3.85979072939338 +241,6,5,0.444480192099237,0.390040101044708,5.13972097810125 +242,1,5,-0.109370786918,0.180835200749669,0.469693708300177 +242,2,5,-0.109370786918,-0.241649823954673,1.68019556392288 +242,3,5,-0.109370786918,1.63054915289421,2.99946048149086 +242,4,5,-0.109370786918,1.79876724716694,1.70281395029541 +242,5,5,-0.109370786918,1.50933658861389,1.92243530107267 +242,6,5,-0.109370786918,1.44996003366407,3.62636572590989 +243,1,5,0.336040969026486,0.395984087004319,-0.510470501592253 +243,2,5,0.336040969026486,1.49213107911934,0.341057260343169 +243,3,5,0.336040969026486,1.0322243877222,1.5963528291938 +243,4,5,0.336040969026486,1.13980145115043,3.22875490267963 +243,5,5,0.336040969026486,1.02657502168611,2.99297209111054 +243,6,5,0.336040969026486,1.51889009798494,5.66833238619657 +244,1,5,0.0246226254591995,-0.310292434088861,0.138069972547055 +244,2,5,0.0246226254591995,0.141199815446702,1.29756197057001 +244,3,5,0.0246226254591995,0.643356826099255,1.84884603919037 +244,4,5,0.0246226254591995,1.15478197762161,1.39675030805328 +244,5,5,0.0246226254591995,0.339075524589951,4.5868265068948 +244,6,5,0.0246226254591995,1.31437861623794,5.24668297480788 +245,1,5,0.885781948931482,0.806314046758624,2.2654923874529 +245,2,5,0.885781948931482,1.18408803633258,3.43352381181165 +245,3,5,0.885781948931482,0.227934592981508,3.16745054969171 +245,4,5,0.885781948931482,1.55756174226992,3.35340671191667 +245,5,5,0.885781948931482,2.33813525718331,4.78860640648161 +245,6,5,0.885781948931482,1.99813377457044,6.51661354997833 +246,1,5,-0.722659484881553,-0.384792945271069,-1.98735622308168 +246,2,5,-0.722659484881553,0.314472242298869,-0.285445896415748 +246,3,5,-0.722659484881553,1.11495488518433,-1.08780307763077 +246,4,5,-0.722659484881553,0.467747700320704,0.240682059810906 +246,5,5,-0.722659484881553,1.33148113439858,0.926984623451972 +246,6,5,-0.722659484881553,1.01374841804555,2.50113616748474 +247,1,5,1.59162124543521,0.801382985950288,0.157749357181468 +247,2,5,1.59162124543521,1.08447653818498,0.966600766348528 +247,3,5,1.59162124543521,1.64504914121795,2.97737955065785 +247,4,5,1.59162124543521,1.66859567663211,4.27374442535286 +247,5,5,1.59162124543521,1.20129706760012,3.93965864036669 +247,6,5,1.59162124543521,1.85582140799441,7.47447291305973 +248,1,5,0.00628203166510098,0.967574078834003,-0.277598332529191 +248,2,5,0.00628203166510098,0.251219158257299,-1.35116285419275 +248,3,5,0.00628203166510098,0.37795289944178,0.394806803517492 +248,4,5,0.00628203166510098,1.38110864533361,0.390223429962069 +248,5,5,0.00628203166510098,2.14074652648849,3.0551896743306 +248,6,5,0.00628203166510098,1.2842173284638,5.41393151214885 +249,1,5,-0.157830993412902,0.225604784343245,-0.63926524186835 +249,2,5,-0.157830993412902,0.815653032758968,1.08592736946653 +249,3,5,-0.157830993412902,-0.926936488629974,0.70719821540352 +249,4,5,-0.157830993412902,0.99159892459773,3.42146939341312 +249,5,5,-0.157830993412902,1.55574470790068,5.93041240162439 +249,6,5,-0.157830993412902,1.14913138457702,3.7143475809707 +250,1,5,-0.486593733281175,0.496828444822754,0.484715409441276 +250,2,5,-0.486593733281175,0.050098683873394,3.15320629989067 +250,3,5,-0.486593733281175,0.250532784667647,1.5334139004638 +250,4,5,-0.486593733281175,-1.07335766406325,-0.164378577396767 +250,5,5,-0.486593733281175,0.0773300025131354,3.30694573947018 +250,6,5,-0.486593733281175,1.12906863829978,4.46490543466803 From 2817248ec6ecf8c1fc5e85a7061ee9b2fd988065 Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 31 Aug 2026 09:30:51 +0800 Subject: [PATCH 02/13] feat(twfeweights): ATTGTWeightsResult / TWFEDecompositionResult containers Result containers for the incoming TWFE implicit-weight diagnostics, landed ahead of the compute module so they pin the output schema and the Diagnostic contract before any math depends on them. Both subclass Diagnostic: they assess what a regression implicitly weights rather than estimating an effect, so neither carries the estimator quintet. The headline scalars are deliberately named `implied_att` and `estimate` rather than `att` so they do not read as inference-bearing. Output columns are diff-diff's (`group`, `time`, `post`, `weight`, `att`), not R's (`time.period`, `attgt`). Covariate balance is a result-object method rather than a mutate-in-place second pass as in R: `covariate_balance(level="summary"|"cell")` reads a table computed at construction time, so the result never retains the raw panel. It raises with the fix inlined when balance was not requested. Names are clearly separated from the existing dCDH surface (`twowayfeweights` / `TWFEWeightsResult`), which weights (unit, time) cells; these weight ATT(g,t) parameters. Roster (M-091) and the shared construction fixture updated; the roster test auto-enrolls both classes. Co-Authored-By: Claude --- diff_diff/__init__.py | 9 + diff_diff/twfe_weights_results.py | 432 ++++++++++++++++++++++++++++ tests/helpers/results_foundation.py | 68 +++++ tests/test_diagnostic_marker.py | 2 + 4 files changed, 511 insertions(+) create mode 100644 diff_diff/twfe_weights_results.py diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 1ef3b75f6..25cd832f0 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -298,6 +298,10 @@ TROPResults, trop, ) +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) from diff_diff.two_stage import ( TwoStageBootstrapResults, TwoStageDiD, @@ -459,6 +463,11 @@ def __getattr__(name: str) -> _Any: "TWFEWeightsResult", "chaisemartin_dhaultfoeuille", "twowayfeweights", + # TWFE weight diagnostics (Callaway `twfeweights` port) - distinct from + # the dCDH `twowayfeweights` surface above: these weight ATT(g,t) + # parameters, not (unit, time) cells. + "ATTGTWeightsResult", + "TWFEDecompositionResult", # WooldridgeDiD (ETWFE) "WooldridgeDiD", "WooldridgeDiDResults", diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py new file mode 100644 index 000000000..bdf4831b3 --- /dev/null +++ b/diff_diff/twfe_weights_results.py @@ -0,0 +1,432 @@ +"""Result containers for the TWFE implicit-weight diagnostics. + +See :mod:`diff_diff.twfe_weights` for the entry points that build these, and +for the upstream MIT attribution. + +Both containers subclass :class:`diff_diff.Diagnostic`: they assess a design +(what a regression implicitly weights) rather than estimating a causal effect, +so neither carries the estimator quintet ``att``/``se``/``t_stat``/``p_value``/ +``conf_int``. The headline scalars are named ``implied_att`` and ``estimate`` +precisely so they do not read as inference-bearing point estimates - the +decomposition is an algebraic identity, exactly like +:class:`~diff_diff.BaconDecompositionResults`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Tuple + +import numpy as np +import pandas as pd + +from diff_diff.results_base import Diagnostic + +__all__ = ["ATTGTWeightsResult", "TWFEDecompositionResult"] + +_AGGREGATION_LABELS = { + "twfe": "TWFE regression", + "overall": "ATT^O (Callaway & Sant'Anna overall)", + "simple": "ATT^simple (Callaway & Sant'Anna simple)", +} + +# Per-cell balance columns, in report order. The three ``_`` -prefixed groups +# mirror R's ``cov_bal_df`` under diff-diff naming; see the mapping table in +# the REGISTRY entry. +_BALANCE_STATS = ( + "unweighted_treated", + "unweighted_control", + "unweighted_diff", + "weighted_treated", + "weighted_control", + "weighted_diff", + "sd", + "unweighted_log_ratio_sd", + "weighted_log_ratio_sd", + "unweighted_frac_extreme", + "weighted_frac_extreme", +) + + +def _fmt(value: float, width: int = 12, digits: int = 4) -> str: + """Right-aligned float that renders NaN without blowing up the layout.""" + if value is None or (isinstance(value, float) and not np.isfinite(value)): + return f"{'n/a':>{width}}" + return f"{value:>{width}.{digits}f}" + + +@dataclass +class ATTGTWeightsResult(Diagnostic): + """Weights that an estimand places on each group-time effect ATT(g, t). + + Returned by :func:`diff_diff.attgt_weights`. One row per ``(g, t)`` cell. + + Attributes + ---------- + weights : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``weight``, ``att``. ``post`` + is ``1`` when ``time >= group`` (the cells the estimand targets), + ``0`` for pre-treatment cells. ``att`` is the ATT(g, t) the weight + multiplies, carried through from the source so that + ``(weight * att).sum()`` reproduces ``implied_att``. + aggregation : str + Which estimand's weights these are: ``"twfe"``, ``"overall"`` + (ATT^O), or ``"simple"`` (ATT^simple). + implied_att : float + ``sum(weight * att)`` - what the estimand delivers given these + ATT(g, t). For ``aggregation="twfe"`` this is the TWFE coefficient. + n_negative : int + Number of cells receiving a negative weight. Non-zero is the + classic staggered-adoption pathology: the regression is subtracting + treatment effects it should be adding. + negative_weight_share : float + ``sum(|w| : w < 0) / sum(|w|)`` - how much of the total weight mass + points the wrong way. ``0.0`` when no weight is negative. + n_cells : int + Number of ``(g, t)`` cells contributing. + source : str or None + ``"CallawaySantAnnaResults"`` when built from a fitted result, + ``"DataFrame"`` on the fallback path. + control_group, base_period : str or None + Design metadata carried from the source fit, when available. + n_dropped_cells : int + Cells excluded because their ATT(g, t) was non-estimable (NaN). + """ + + weights: pd.DataFrame + aggregation: str + implied_att: float + n_negative: int + negative_weight_share: float + n_cells: int + source: Optional[str] = None + control_group: Optional[str] = None + base_period: Optional[str] = None + n_dropped_cells: int = 0 + + def __repr__(self) -> str: + return ( + f"ATTGTWeightsResult(aggregation={self.aggregation!r}, " + f"implied_att={self.implied_att:.4f}, " + f"n_cells={self.n_cells}, n_negative={self.n_negative})" + ) + + def summary(self) -> str: + """Formatted per-cell weight table with the negative-weight roll-up.""" + width = 72 + label = _AGGREGATION_LABELS.get(self.aggregation, self.aggregation) + lines = [ + "=" * width, + "Implicit Weights on ATT(g, t)".center(width), + "=" * width, + "", + f"{'Estimand:':<28} {label}", + f"{'Group-time cells:':<28} {self.n_cells:>10}", + ] + if self.n_dropped_cells: + lines.append(f"{'Non-estimable cells dropped:':<28} {self.n_dropped_cells:>10}") + if self.source is not None: + lines.append(f"{'Source:':<28} {self.source}") + if self.control_group is not None: + lines.append(f"{'Control group:':<28} {self.control_group}") + if self.base_period is not None: + lines.append(f"{'Base period:':<28} {self.base_period}") + lines += [ + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} {'ATT(g,t)':>14}", + "-" * width, + ] + for row in self.weights.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Implied estimate:':<28} {_fmt(self.implied_att)}", + f"{'Negative-weight cells:':<28} {self.n_negative:>12}", + f"{'Negative-weight share:':<28} {_fmt(self.negative_weight_share)}", + "", + ] + if self.n_negative: + lines += [ + "Note: negative weights mean this estimand subtracts some ATT(g, t).", + " Under heterogeneous effects the estimate need not lie in the", + " convex hull of the underlying group-time effects.", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell weight table (a copy).""" + return self.weights.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + return { + "aggregation": self.aggregation, + "implied_att": self.implied_att, + "n_cells": self.n_cells, + "n_negative": self.n_negative, + "negative_weight_share": self.negative_weight_share, + "n_dropped_cells": self.n_dropped_cells, + "source": self.source, + "control_group": self.control_group, + "base_period": self.base_period, + "weights": self.weights.to_dict(orient="list"), + } + + +@dataclass +class TWFEDecompositionResult(Diagnostic): + """Decomposition of a TWFE (or AIPW) estimate into weighted ATT(g, t). + + Returned by :func:`diff_diff.decompose_twfe_weights`. + + Attributes + ---------- + cells : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``att``, ``weight``, ``ess``, + and (``method="fwl"`` only) ``remainder``. ``weight`` is the implicit + weight the regression places on that cell's ATT(g, t) - R's + ``alpha_weight`` under ``method="fwl"`` and ``att_weight`` under + ``method="aipw"``. + method : str + ``"fwl"`` (Frisch-Waugh-Lovell residual weights from the TWFE + regression) or ``"aipw"`` (per-cell doubly-robust weights). + estimate : float + The estimate being decomposed - ``decomposition + remainder``. + decomposition : float + ``sum(weight * att)`` over all cells, pre and post. + remainder : float + Part of ``estimate`` not attributable to any ATT(g, t) cell. + Identically ``0.0`` except under ``method="fwl"`` with + ``base_period="gmin1"``. + pretrend_bias : float + ``sum(weight * att)`` over PRE-treatment cells only. Under parallel + trends every pre-treatment ATT(g, t) is zero and this vanishes; a + non-zero value is the contribution of parallel-trends violations to + ``estimate``. + post_only : float + ``sum(weight * att)`` over post-treatment cells only. + base_period : str or None + ``"first_period"`` or ``"gmin1"`` (``method="fwl"`` only). + covariates : tuple of str + Covariates the regression adjusted for. Empty tuple when none. + effective_sample_size : float + Weight-concentration roll-up. Small values relative to ``n_units`` + mean the estimate leans on few observations. + n_units, n_periods : int + Panel dimensions. + balance : pd.DataFrame or None + Per-cell implicit covariate balance, populated when + ``balance_covariates=`` was requested. Read it via + :meth:`covariate_balance`. + """ + + cells: pd.DataFrame + method: str + estimate: float + decomposition: float + remainder: float + pretrend_bias: float + post_only: float + base_period: Optional[str] + covariates: Tuple[str, ...] + effective_sample_size: float + n_units: int + n_periods: int + balance: Optional[pd.DataFrame] = field(default=None) + + def __repr__(self) -> str: + return ( + f"TWFEDecompositionResult(method={self.method!r}, " + f"estimate={self.estimate:.4f}, " + f"pretrend_bias={self.pretrend_bias:.4f}, " + f"n_cells={len(self.cells)})" + ) + + def summary(self) -> str: + """Formatted decomposition table with the pre-trend contribution.""" + width = 78 + method_label = { + "fwl": "TWFE regression (Frisch-Waugh-Lovell implicit weights)", + "aipw": "AIPW (doubly-robust per-cell weights)", + }.get(self.method, self.method) + covs = ", ".join(self.covariates) if self.covariates else "(none)" + lines = [ + "=" * width, + "Decomposition into Group-Time Effects".center(width), + "=" * width, + "", + f"{'Method:':<30} {method_label}", + f"{'Covariates:':<30} {covs}", + ] + if self.base_period is not None: + lines.append(f"{'Base period:':<30} {self.base_period}") + lines += [ + f"{'Units / periods:':<30} {self.n_units} / {self.n_periods}", + f"{'Group-time cells:':<30} {len(self.cells)}", + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} " + f"{'ATT(g,t)':>14} {'Contribution':>14}", + "-" * width, + ] + for row in self.cells.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)} " + f"{_fmt(row.weight * row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Estimate:':<30} {_fmt(self.estimate)}", + f"{' from ATT(g,t) cells:':<30} {_fmt(self.decomposition)}", + f"{' post-treatment only:':<30} {_fmt(self.post_only)}", + f"{' pre-trend violations:':<30} {_fmt(self.pretrend_bias)}", + f"{' remainder:':<30} {_fmt(self.remainder)}", + "", + f"{'Effective sample size:':<30} {_fmt(self.effective_sample_size)}", + "", + ] + if abs(self.pretrend_bias) > 1e-10: + lines += [ + "Note: a non-zero pre-trend contribution means pre-treatment", + " ATT(g, t) are not zero, so part of the estimate reflects", + " parallel-trends violations rather than treatment effects.", + "", + ] + if self.balance is not None: + lines += [ + "Covariate balance available via .covariate_balance().", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell decomposition table (a copy).""" + return self.cells.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + out: Dict[str, Any] = { + "method": self.method, + "estimate": self.estimate, + "decomposition": self.decomposition, + "remainder": self.remainder, + "pretrend_bias": self.pretrend_bias, + "post_only": self.post_only, + "base_period": self.base_period, + "covariates": list(self.covariates), + "effective_sample_size": self.effective_sample_size, + "n_units": self.n_units, + "n_periods": self.n_periods, + "cells": self.cells.to_dict(orient="list"), + } + if self.balance is not None: + out["balance"] = self.balance.to_dict(orient="list") + return out + + def covariate_balance( + self, + *, + level: str = "summary", + standardize: bool = True, + post_only: bool = True, + ) -> pd.DataFrame: + """Implicit-weight covariate balance. + + Asks whether the weights the regression implicitly applies actually + balance the covariates across the treated and comparison groups. If + ``weighted_diff`` is no closer to zero than ``unweighted_diff``, the + covariate adjustment is not buying what it appears to. + + Parameters + ---------- + level : {"summary", "cell"}, default "summary" + ``"summary"`` aggregates across ``(g, t)`` cells to one row per + covariate, weighting each cell by its implicit weight. + ``"cell"`` returns the unaggregated per-``(g, t)`` rows. + standardize : bool, default True + Append ``unweighted_std_diff`` / ``weighted_std_diff``, the + differences divided by the pooled standard deviation. These are + a diff-diff addition; R reports the raw differences only. + post_only : bool, default True + Restrict the summary roll-up to post-treatment cells, matching + R's ``mp_covariate_bal_summary_helper``. Ignored when + ``level="cell"``. + + Returns + ------- + pd.DataFrame + One row per covariate (``level="summary"``) or per + ``(group, time, covariate)`` (``level="cell"``). + + Raises + ------ + ValueError + If balance was not requested at compute time, or ``level`` is + not one of the two accepted values. + """ + if self.balance is None: + raise ValueError( + "Covariate balance was not computed for this decomposition. " + "Re-run with balance_covariates=, e.g.\n" + " decompose_twfe_weights(..., balance_covariates=['x1', 'x2'])" + ) + if level not in ("summary", "cell"): + raise ValueError(f"level must be 'summary' or 'cell', got {level!r}") + + table = self.balance.copy() + if level == "cell": + if standardize: + table = _append_standardized(table) + return table + + weights = self.cells.set_index(["group", "time"])["weight"] + keys = pd.MultiIndex.from_arrays([table["group"], table["time"]]) + cell_weight = weights.reindex(keys).to_numpy() + if post_only: + cell_weight = cell_weight * table["post"].to_numpy() + + table["_w"] = cell_weight + rolled = ( + table[list(_BALANCE_STATS)] + .mul(table["_w"], axis=0) + .groupby(table["covariate"].to_numpy(), sort=False) + .sum() + ) + rolled.index.name = "covariate" + out = rolled.reset_index() + if standardize: + out = _append_standardized(out) + return out + + +def _append_standardized(table: pd.DataFrame) -> pd.DataFrame: + """Add ``*_std_diff`` columns (difference / pooled SD). + + A diff-diff addition on top of R's columns - additive, so parity is + asserted on the R columns only. ``sd == 0`` yields NaN rather than an + infinity, so a degenerate covariate does not poison a summary table. + """ + out = table.copy() + sd = out["sd"].to_numpy(dtype=float) + safe = np.where(sd == 0, np.nan, sd) + out["unweighted_std_diff"] = out["unweighted_diff"].to_numpy(dtype=float) / safe + out["weighted_std_diff"] = out["weighted_diff"].to_numpy(dtype=float) / safe + return out diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index 09b3dbff7..ce81e9e78 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -43,6 +43,48 @@ def make_constructed_diagnostics() -> Dict[str, Any]: poly = pd.DataFrame({"rdplot_x": [-1.0, 0.0, 1.0], "rdplot_y": [0.9, 1.4, 2.1]}) coef = pd.DataFrame({"side": ["left", "right"], "coef_0": [1.0, 2.0]}) + # TWFE weight diagnostics: a 2-cohort x 2-period grid with one negative + # weight, so summary() exercises the negative-weight branch. + attgt_weight_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "weight": [0.6, 0.5, -0.2, 0.1], + "att": [1.0, 1.2, 0.0, 0.8], + } + ) + decomposition_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "att": [1.0, 1.2, 0.0, 0.8], + "weight": [0.4, 0.3, 0.1, 0.2], + "ess": [8.0, 8.0, 6.0, 6.0], + "remainder": [0.0, 0.0, 0.0, 0.0], + } + ) + decomposition_balance = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "covariate": ["x1", "x1", "x1", "x1"], + "unweighted_treated": [0.5, 0.5, 0.4, 0.4], + "unweighted_control": [0.3, 0.3, 0.2, 0.2], + "unweighted_diff": [0.2, 0.2, 0.2, 0.2], + "weighted_treated": [0.5, 0.5, 0.4, 0.4], + "weighted_control": [0.45, 0.45, 0.38, 0.38], + "weighted_diff": [0.05, 0.05, 0.02, 0.02], + "sd": [1.0, 1.0, 1.0, 1.0], + "unweighted_log_ratio_sd": [0.01, 0.01, 0.02, 0.02], + "weighted_log_ratio_sd": [0.005, 0.005, 0.01, 0.01], + "unweighted_frac_extreme": [0.05, 0.05, 0.06, 0.06], + "weighted_frac_extreme": [0.04, 0.04, 0.05, 0.05], + } + ) + qug = diff_diff.QUGTestResults( t_stat=1.2, p_value=0.23, @@ -290,5 +332,31 @@ def make_constructed_diagnostics() -> Dict[str, Any]: interpretation="All applicable checks passed.", applicable_checks=("parallel_trends",), ), + "ATTGTWeightsResult": diff_diff.ATTGTWeightsResult( + weights=attgt_weight_cells, + aggregation="twfe", + implied_att=float((attgt_weight_cells["weight"] * attgt_weight_cells["att"]).sum()), + n_negative=1, + negative_weight_share=0.25, + n_cells=len(attgt_weight_cells), + source="CallawaySantAnnaResults", + control_group="never_treated", + base_period="universal", + ), + "TWFEDecompositionResult": diff_diff.TWFEDecompositionResult( + cells=decomposition_cells, + method="fwl", + estimate=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + decomposition=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + remainder=0.0, + pretrend_bias=0.0, + post_only=0.5, + base_period="first_period", + covariates=("x1",), + effective_sample_size=42.0, + n_units=12, + n_periods=4, + balance=decomposition_balance, + ), } return instances diff --git a/tests/test_diagnostic_marker.py b/tests/test_diagnostic_marker.py index c390abcec..df758e1a5 100644 --- a/tests/test_diagnostic_marker.py +++ b/tests/test_diagnostic_marker.py @@ -49,6 +49,8 @@ "HADPretestReport", "DiagnosticReportResults", "DurationDiDPretestResults", + "ATTGTWeightsResult", + "TWFEDecompositionResult", ] # Representative ESTIMATOR results: marked with BaseResults, never Diagnostic. From 39e33c19e497c0c798d07a5b799a57601bb80283 Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 31 Aug 2026 09:39:46 +0800 Subject: [PATCH 03/13] feat(twfeweights): attgt_weights() - implicit weights on ATT(g,t) One entry point folding R's three separate weight functions behind `aggregation=`: "twfe" (twfe_weights), "overall" (attO_weights, ATT^O), and "simple" (att_simple_weights, ATT^simple). Reports what each estimand implicitly puts on every group-time effect, plus the negative-weight share that makes the staggered-TWFE pathology legible. Takes a fitted CallawaySantAnnaResults as the primary input, reading cohort masses off the aggregation bookkeeping so no raw panel is needed; a (gt_frame, data=, unit=, time=, first_treat=) fallback consumes `result.to_dataframe("group_time")` verbatim. Design restrictions are hard errors, not warnings, each naming its fix: aggregation="twfe" needs base_period="universal" and control_group="never_treated" (matching R's own stop()s), and no aggregation accepts a repeated-cross-section or unbalanced-fallback fit, whose cohort shares are not comparable across periods. Deviation from R: cohorts and periods are mapped to positional time before the (maxT - g + 1)/T arithmetic. R evaluates that on raw labels, which is only correct on consecutive integers; positional time is bit-identical there (mpdta 2003..2007 -> 1..5 both give 4/5 at g=2004) and correct on gapped grids. Pinned by a test that remaps periods to 10,20,30,40,50. R's keep_untreated= is not exposed: it synthesizes G=0 rows that are excluded from every normalization and contribute exactly zero. Parity: machine precision (max |dw| = 4.7e-16) against R twfeweights 0.9.0 on 3 fixtures x 3 aggregations. Primary assertions feed R's own ATT(g,t) back in, isolating this module from CallawaySantAnna-vs-`did` parity; a separate, deliberately looser class covers the composed end-to-end path. Also fixes a generator bug: R stores `post` as a FACTOR, so as.integer() emitted level codes 1/2 rather than 0/1. Known-red until the docs commit: test_doc_deps_integrity.py wants a docs/doc-deps.yaml entry, which lands with the API page. Co-Authored-By: Claude --- benchmarks/R/generate_twfeweights_golden.R | 7 +- benchmarks/data/twfeweights_golden.json | 18 +- diff_diff/__init__.py | 4 + diff_diff/twfe_weights.py | 570 +++++++++++++++++++++ tests/test_twfe_weights.py | 359 +++++++++++++ tests/test_twfe_weights_parity.py | 240 +++++++++ 6 files changed, 1188 insertions(+), 10 deletions(-) create mode 100644 diff_diff/twfe_weights.py create mode 100644 tests/test_twfe_weights.py create mode 100644 tests/test_twfe_weights_parity.py diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R index 360a13145..8e4541f24 100644 --- a/benchmarks/R/generate_twfeweights_golden.R +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -86,10 +86,15 @@ if (!dir.exists(out_dir)) { extract_mp_weights <- function(obj) { df <- obj$weights_df + # mp_weights_obj stores `post` as a FACTOR (for ggplot colouring), so + # as.integer() would emit level codes 1/2 rather than the values 0/1. + # Round-trip through character. + post <- as.integer(as.character(df$post)) + stopifnot(all(post %in% c(0L, 1L))) list( group = as.numeric(df$group), time = as.numeric(df$time.period), - post = as.integer(df$post), + post = post, weight = as.numeric(df$weight), att = as.numeric(df$attgt), implied_att = sum(df$weight * df$attgt) diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json index 1c5b88b4d..771a0d15d 100644 --- a/benchmarks/data/twfeweights_golden.json +++ b/benchmarks/data/twfeweights_golden.json @@ -30,7 +30,7 @@ "twfe": { "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], - "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "weight": [-0.113075467453585, 0.0457198057404491, 0.0457198057404491, 0.0324868663076129, -0.0108510103349257, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110522, 0.578994031944316], "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], "implied_att": -0.0365489366740672 @@ -38,7 +38,7 @@ "overall": { "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], - "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "weight": [0, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0, 0, 0, 0.104712041884817, 0.104712041884817, 0, 0, 0, 0, 0.68586387434555], "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], "implied_att": -0.031018282228749 @@ -46,7 +46,7 @@ "simple": { "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], - "post": [1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "weight": [0, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0, 0, 0, 0.13745704467354, 0.13745704467354, 0, 0, 0, 0, 0.450171821305842], "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], "implied_att": -0.039951275155177 @@ -217,7 +217,7 @@ "twfe": { "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], - "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], "implied_att": 1.38176729464315 @@ -225,7 +225,7 @@ "overall": { "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], - "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], "weight": [0, 0, 0.166666666666667, 0.166666666666667, 0.166666666666667, 0, 0, 0, 0.25, 0.25], "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], "implied_att": 1.73623602393453 @@ -233,7 +233,7 @@ "simple": { "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], - "post": [1, 1, 2, 2, 2, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], "weight": [0, 0, 0.2, 0.2, 0.2, 0, 0, 0, 0.2, 0.2], "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], "implied_att": 1.76984085043211 @@ -404,7 +404,7 @@ "twfe": { "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], - "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121212, 0.212121212121212, 0.053030303030303, 0.053030303030303, -0.0378787878787879, -0.0378787878787879, -0.196969696969697, -0.196969696969697, 0.234848484848485, 0.234848484848485], "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], "implied_att": 1.88846922090824 @@ -412,7 +412,7 @@ "overall": { "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], - "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], "weight": [0, 0, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0, 0, 0, 0, 0.230769230769231, 0.230769230769231], "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], "implied_att": 2.25856121537518 @@ -420,7 +420,7 @@ "simple": { "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], - "post": [1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], "weight": [0, 0, 0.175, 0.175, 0.175, 0.175, 0, 0, 0, 0, 0.15, 0.15], "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], "implied_att": 2.43266114548473 diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 25cd832f0..62914158d 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -298,6 +298,9 @@ TROPResults, trop, ) +from diff_diff.twfe_weights import ( + attgt_weights, +) from diff_diff.twfe_weights_results import ( ATTGTWeightsResult, TWFEDecompositionResult, @@ -468,6 +471,7 @@ def __getattr__(name: str) -> _Any: # parameters, not (unit, time) cells. "ATTGTWeightsResult", "TWFEDecompositionResult", + "attgt_weights", # WooldridgeDiD (ETWFE) "WooldridgeDiD", "WooldridgeDiDResults", diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py new file mode 100644 index 000000000..a1357b636 --- /dev/null +++ b/diff_diff/twfe_weights.py @@ -0,0 +1,570 @@ +"""Implicit TWFE weights on group-time average treatment effects. + +A two-way fixed effects regression run on staggered-adoption data does not +estimate a simple average of the underlying ATT(g, t). It estimates a +*weighted* average, and some of those weights can be negative - so the +coefficient need not lie in the convex hull of the effects it summarizes. +:func:`attgt_weights` reports those weights, next to the weights the target +estimands ATT^O and ATT^simple would use. :func:`decompose_twfe_weights` +re-derives the regression from its building blocks and separates the part +driven by pre-treatment parallel-trends violations. + +Distinct from :func:`diff_diff.twowayfeweights`, which implements the de +Chaisemartin & D'Haultfoeuille (2020) Theorem 1 decomposition: that one +weights ``(unit, time)`` cells, this one weights ATT(g, t) *parameters*. +Distinct also from :class:`diff_diff.BaconDecomposition`, which decomposes +TWFE into 2x2 DiD comparisons rather than into group-time effects. + +Ported from the R package ``twfeweights`` (version 0.9.0) by Brantly +Callaway, released under the MIT License. The upstream notice is reproduced +in full, as its terms require:: + + MIT License + + Copyright (c) 2023 Brantly Callaway + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Methodology: Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna (2025), +"Difference-in-Differences Designs: A Practitioner's Guide" +(arXiv:2503.13323); Callaway & Sant'Anna (2021) for the ATT^O / ATT^simple +weights. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union + +import numpy as np +import pandas as pd + +from diff_diff.twfe_weights_results import ATTGTWeightsResult + +if TYPE_CHECKING: # pragma: no cover - typing only + from diff_diff.staggered_results import CallawaySantAnnaResults + +__all__ = ["attgt_weights"] + +_AGGREGATIONS = ("twfe", "overall", "simple") + + +def _is_never(values: np.ndarray) -> np.ndarray: + """Boolean mask for never-treated cohort labels. + + diff-diff and R ``did`` have both used ``0`` and ``+inf`` as the + never-treated sentinel over time; accept either and normalize to ``0``. + """ + arr = np.asarray(values, dtype=float) + return ~np.isfinite(arr) | (arr == 0) + + +def _positional_grid( + time_periods: Sequence[Any], +) -> Dict[float, int]: + """Map ordered period labels onto ``1..T``. + + R computes ``(maxT - g + 1) / length(tlist)`` directly on the raw period + labels, which is only correct when those labels are consecutive integers. + Working in positional time makes the same expression correct on gapped or + non-integer grids, and is bit-identical when the grid IS consecutive + (mpdta's 2003..2007 maps to 1..5 and both give 4/5 for g = 2004). + Recorded as a deviation in the methodology registry. + """ + ordered = sorted({float(t) for t in time_periods}) + return {t: i + 1 for i, t in enumerate(ordered)} + + +def _to_positional_cohort(cohorts: np.ndarray, grid: Dict[float, int]) -> np.ndarray: + """Cohort labels -> positional time; never-treated stays 0. + + Mirrors ``BMisc::orig2t``, which leaves the never-treated sentinel alone + under positional rescaling. + """ + out = np.zeros(len(cohorts), dtype=float) + never = _is_never(cohorts) + for i, (g, is_never) in enumerate(zip(cohorts, never)): + if is_never: + continue + key = float(g) + if key not in grid: + raise ValueError( + f"cohort label {g!r} is not one of the observed time periods " + f"{sorted(grid)!r}; cannot place it on the period grid" + ) + out[i] = grid[key] + return out + + +def _cohort_masses( + unit_cohorts: np.ndarray, + grid: Dict[float, int], + weights: Optional[np.ndarray], +) -> Tuple[Dict[int, float], Dict[int, float], Dict[int, float], float]: + """Cohort shares and treated-share-by-period, all in positional time. + + Returns + ------- + p_all : {positional g: share of ALL units in cohort g} + R's ``pg2`` - the denominator is every unit, never-treated included. + Used by the TWFE weights. + p_treated : {positional g: share of EVER-TREATED units in cohort g} + R's ``pg``. Used by the ATT^O / ATT^simple weights. + e_dt : {positional t: weighted share of units treated by t} + R's ``Edt(t)``. + mean_e_dt : float + R's ``mEdt`` - the average of ``e_dt`` over the period grid. + """ + g_pos = _to_positional_cohort(unit_cohorts, grid) + w = np.ones(len(g_pos)) if weights is None else np.asarray(weights, dtype=float) + if len(w) != len(g_pos): + raise ValueError(f"weights has length {len(w)} but there are {len(g_pos)} units") + total = w.sum() + if total <= 0: + raise ValueError("unit weights sum to zero; cannot form cohort shares") + + treated = g_pos != 0 + treated_mass = w[treated].sum() + + cohorts = sorted({int(g) for g in g_pos if g != 0}) + p_all = {g: float(w[g_pos == g].sum() / total) for g in cohorts} + if treated_mass > 0: + p_treated = {g: float(w[g_pos == g].sum() / treated_mass) for g in cohorts} + else: # pragma: no cover - guarded upstream by the never-treated check + p_treated = {g: 0.0 for g in cohorts} + + periods = sorted(grid.values()) + e_dt = {t: float(w[treated & (g_pos <= t)].sum() / total) for t in periods} + mean_e_dt = float(np.mean([e_dt[t] for t in periods])) + return p_all, p_treated, e_dt, mean_e_dt + + +def _twfe_weight_vector( + groups: np.ndarray, + times: np.ndarray, + n_periods: int, + p_all: Dict[int, float], + e_dt: Dict[int, float], + mean_e_dt: float, +) -> np.ndarray: + """Weights a static TWFE regression places on each ATT(g, t). + + ``h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D]`` + ``num(g,t) = h(g,t) * p_g``, normalized by the sum over post cells. + + All arguments are in positional time, so ``maxT == n_periods``. + """ + h = ( + (times >= groups).astype(float) + - (n_periods - groups + 1.0) / n_periods + - np.array([e_dt[int(t)] for t in times]) + + mean_e_dt + ) + num = h * np.array([p_all[int(g)] for g in groups]) + post = times >= groups + denom = num[post].sum() + if denom == 0: + raise ValueError( + "TWFE weight normalization is degenerate (post-treatment weights " + "sum to zero); the regression has no identifying variation" + ) + return num / denom + + +def _overall_weight_vector( + groups: np.ndarray, + times: np.ndarray, + n_periods: int, + p_treated: Dict[int, float], +) -> np.ndarray: + """ATT^O weights: ``1[t >= g] * pbar_g / (maxT - g + 1)``. + + Not renormalized - the ``(maxT - g + 1)`` divisor already makes them sum + to one over a complete post-treatment grid. + """ + return ( + (times >= groups).astype(float) + * np.array([p_treated[int(g)] for g in groups]) + / (n_periods - groups + 1.0) + ) + + +def _simple_weight_vector( + groups: np.ndarray, + times: np.ndarray, + p_treated: Dict[int, float], +) -> np.ndarray: + """ATT^simple weights: ``1[t >= g] * pbar_g``, normalized to sum to one.""" + raw = (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) + total = raw.sum() + if total == 0: + raise ValueError( + "ATT^simple weight normalization is degenerate (no post-treatment " + "cells carry weight)" + ) + return raw / total + + +def _attgt_from_cs( + results: "CallawaySantAnnaResults", +) -> Tuple[pd.DataFrame, int]: + """Extract the ``(g, t, att)`` table from a fitted CS result. + + Non-estimable cells (``skip_reason`` set, NaN effect) are dropped and + counted, so a partially-estimable fit still produces weights over the + cells that exist rather than propagating NaN through every aggregate. + """ + rows: List[Dict[str, Any]] = [] + dropped = 0 + for (g, t), cell in results.group_time_effects.items(): + effect = cell.get("effect", np.nan) + if cell.get("skip_reason") is not None or not np.isfinite(effect): + dropped += 1 + continue + rows.append({"group": g, "time": t, "att": float(effect)}) + if not rows: + raise ValueError( + "the fitted result has no estimable group-time cells; there is " "nothing to weight" + ) + table = pd.DataFrame(rows).sort_values(["group", "time"]).reset_index(drop=True) + return table, dropped + + +def _attgt_from_frame(frame: pd.DataFrame) -> Tuple[pd.DataFrame, int]: + """Extract ``(g, t, att)`` from a user-supplied ATT(g, t) frame. + + ``effect`` is preferred over ``att`` because that is the column + ``CallawaySantAnnaResults.to_dataframe("group_time")`` emits - so the + fallback consumes our own frame verbatim. + """ + missing = {"group", "time"} - set(frame.columns) + if missing: + raise ValueError( + f"ATT(g,t) frame is missing required column(s) {sorted(missing)!r}; " + "expected 'group', 'time', and one of 'effect' / 'att'" + ) + for candidate in ("effect", "att"): + if candidate in frame.columns: + value_col = candidate + break + else: + raise ValueError( + "ATT(g,t) frame must carry an 'effect' or 'att' column; got " f"{list(frame.columns)!r}" + ) + table = pd.DataFrame( + { + "group": frame["group"].to_numpy(), + "time": frame["time"].to_numpy(), + "att": pd.to_numeric(frame[value_col], errors="coerce").to_numpy(), + } + ) + dropped = int(table["att"].isna().sum()) + table = table.dropna(subset=["att"]) + if table.empty: + raise ValueError("ATT(g,t) frame has no finite effects to weight") + return table.sort_values(["group", "time"]).reset_index(drop=True), dropped + + +def _unit_cohorts_from_frame( + data: pd.DataFrame, unit: str, time: str, first_treat: str +) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: + """Collapse a long panel to one cohort label per unit.""" + for col in (unit, time, first_treat): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + per_unit = data.groupby(unit, sort=True)[first_treat].nunique() + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + cohorts = data.groupby(unit, sort=True)[first_treat].first().to_numpy() + periods = np.asarray(sorted(data[time].unique())) + return cohorts, periods, None + + +def _resolve_cs_inputs( + results: "CallawaySantAnnaResults", +) -> Tuple[np.ndarray, Optional[np.ndarray]]: + """Read cohort labels (and survey weights) off a fitted CS result. + + The aggregation kit is package-internal, but it is the same channel + ``CallawaySantAnnaResults._aggregate_compute`` already uses - so this is + an established in-package coupling rather than a new one. When the kit is + absent (an old pickle), the caller is pointed at the ``data=`` fallback. + """ + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + raise ValueError( + "this CallawaySantAnnaResults carries no aggregation bookkeeping " + "(it may have been unpickled from an older version), so cohort " + "shares cannot be recovered from it. Pass the panel explicitly:\n" + " attgt_weights(result.to_dataframe('group_time'), data=panel,\n" + " unit=..., time=..., first_treat=...)" + ) + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + cohorts = bookkeeping.get("unit_cohorts") + if cohorts is None: + raise ValueError( + "aggregation bookkeeping does not carry 'unit_cohorts'; pass the " + "panel explicitly via data=/unit=/time=/first_treat=" + ) + weights = bookkeeping.get("survey_weights") + return np.asarray(cohorts), (None if weights is None else np.asarray(weights, dtype=float)) + + +def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> None: + """Reject fits whose design breaks the weight formulas. + + These are hard errors rather than warnings: a silently wrong weight table + is worse than no weight table, and every one of these has a concrete fix. + """ + if not getattr(results, "panel", True): + raise ValueError( + "attgt_weights requires a panel fit: E_t[D] and the cohort shares " + "average over a fixed set of units, which repeated cross-sections " + "do not provide. Refit with panel=True." + ) + if getattr(results, "used_rc_on_unbalanced_panel", False): + raise ValueError( + "this fit fell back to repeated-cross-section estimation on an " + "unbalanced panel, so the cohort shares are not comparable across " + "periods. Balance the panel (diff_diff.balance_panel) and refit." + ) + if aggregation != "twfe": + return + control_group = getattr(results, "control_group", None) + if control_group not in (None, "never_treated"): + raise ValueError( + f"aggregation='twfe' requires control_group='never_treated', got " + f"{control_group!r}. The TWFE weight formula is derived against a " + "never-treated comparison group (matching R's twfe_weights, which " + "raises the same restriction)." + ) + base_period = getattr(results, "base_period", None) + if base_period not in (None, "universal"): + raise ValueError( + f"aggregation='twfe' requires base_period='universal', got " + f"{base_period!r}. The formula needs the complete cohort x period " + "grid, including the pre-treatment cells that a varying base does " + "not report. Refit with base_period='universal'." + ) + + +def attgt_weights( + results: Union["CallawaySantAnnaResults", pd.DataFrame], + *, + aggregation: str = "twfe", + data: Optional[pd.DataFrame] = None, + unit: Optional[str] = None, + time: Optional[str] = None, + first_treat: Optional[str] = None, + weights: Optional[Union[str, np.ndarray]] = None, +) -> ATTGTWeightsResult: + """Weights an estimand places on each group-time effect ATT(g, t). + + Three estimands are available. ``"twfe"`` gives the weights implied by a + static two-way fixed effects regression - the ones that can go negative. + ``"overall"`` and ``"simple"`` give the weights of the Callaway & + Sant'Anna (2021) target parameters ATT^O and ATT^simple, which are + non-negative by construction. Comparing them shows how far the regression + is from the estimand you meant to report. + + Parameters + ---------- + results : CallawaySantAnnaResults or pd.DataFrame + A fitted Callaway & Sant'Anna result (preferred), or a frame with + ``group`` / ``time`` / ``effect`` (or ``att``) columns. On the frame + path, ``data``, ``unit``, ``time`` and ``first_treat`` are required + so cohort shares can be formed. + aggregation : {"twfe", "overall", "simple"}, default "twfe" + Which estimand's weights to report. + data : pd.DataFrame, optional + Balanced panel backing the ATT(g, t) frame. Only for the fallback + path; passing it alongside a fitted result raises. + unit, time, first_treat : str, optional + Column names in ``data``. Required together with ``data``. + weights : str or array-like, optional + Unit-level sampling weights (R's ``w=``): a column name in ``data``, + or one value per unit. Rejected when the fit already carries survey + weights, which take precedence. + + Returns + ------- + ATTGTWeightsResult + Per-cell weights plus the negative-weight roll-up. + + Raises + ------ + ValueError + On an unknown ``aggregation``; on a design the formula does not + support (repeated cross-sections, unbalanced fallback, and - for + ``aggregation="twfe"`` - a non-never-treated control group or a + non-universal base period); or on an incomplete fallback spec. + + Notes + ----- + R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows + with ``attgt = 0`` to mirror an internal vector layout; those rows are + excluded from every normalization and contribute exactly zero, so the + argument does not affect any number. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> cs = diff_diff.CallawaySantAnna(base_period="universal") # doctest: +SKIP + >>> res = cs.fit(df, outcome="y", unit="id", time="t", + ... first_treat="g") # doctest: +SKIP + >>> w = diff_diff.attgt_weights(res, aggregation="twfe") # doctest: +SKIP + >>> print(w.summary()) # doctest: +SKIP + """ + if aggregation not in _AGGREGATIONS: + raise ValueError( + f"aggregation must be one of {list(_AGGREGATIONS)!r}, got " f"{aggregation!r}" + ) + + frame_path = isinstance(results, pd.DataFrame) + fallback_args = {"data": data, "unit": unit, "time": time, "first_treat": first_treat} + supplied = {k: v for k, v in fallback_args.items() if v is not None} + + if frame_path: + if len(supplied) != 4: + missing = sorted(set(fallback_args) - set(supplied)) + raise ValueError( + "the DataFrame path needs the panel too, so cohort shares can " + f"be formed; missing {missing!r}. Call it as:\n" + " attgt_weights(gt_frame, data=panel, unit='id', " + "time='t', first_treat='g')" + ) + assert data is not None and unit is not None + assert time is not None and first_treat is not None + table, dropped = _attgt_from_frame(results) + cohorts, periods, _ = _unit_cohorts_from_frame(data, unit, time, first_treat) + unit_weights = _resolve_frame_weights(weights, data, unit) + source = "DataFrame" + control_group = None + base_period = None + else: + if supplied: + raise ValueError( + f"{sorted(supplied)!r} are only for the DataFrame fallback. A " + "fitted CallawaySantAnnaResults already carries the cohort " + "bookkeeping - drop them, or pass " + "result.to_dataframe('group_time') as the first argument." + ) + _guard_cs_design(results, aggregation) + table, dropped = _attgt_from_cs(results) + cohorts, survey_weights = _resolve_cs_inputs(results) + if survey_weights is not None and weights is not None: + raise ValueError( + "this fit already carries survey weights; passing weights= as " + "well is ambiguous. Drop weights= to use the fit's own." + ) + if weights is not None and not isinstance(weights, str): + unit_weights = np.asarray(weights, dtype=float) + elif isinstance(weights, str): + raise ValueError( + "weights= may only name a column on the DataFrame path; pass " + "an array of per-unit weights instead" + ) + else: + unit_weights = survey_weights + periods = np.asarray(results.time_periods) + source = "CallawaySantAnnaResults" + control_group = getattr(results, "control_group", None) + base_period = getattr(results, "base_period", None) + + if dropped: + warnings.warn( + f"{dropped} group-time cell(s) had no estimable ATT(g,t) and were " + "excluded from the weight table; the reported weights renormalize " + "over the remaining cells", + UserWarning, + stacklevel=2, + ) + + grid = _positional_grid(periods) + n_periods = len(grid) + p_all, p_treated, e_dt, mean_e_dt = _cohort_masses(cohorts, grid, unit_weights) + if not p_treated: + raise ValueError( + "no ever-treated units found; cohort labels are all never-treated " + "sentinels (0 or inf)" + ) + + g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) + t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + + if aggregation == "twfe": + weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) + elif aggregation == "overall": + weight_vec = _overall_weight_vector(g_pos, t_pos, n_periods, p_treated) + else: + weight_vec = _simple_weight_vector(g_pos, t_pos, p_treated) + + out = pd.DataFrame( + { + "group": table["group"].to_numpy(), + "time": table["time"].to_numpy(), + "post": (t_pos >= g_pos).astype(int), + "weight": weight_vec, + "att": table["att"].to_numpy(), + } + ) + + negative = weight_vec < 0 + abs_total = float(np.abs(weight_vec).sum()) + return ATTGTWeightsResult( + weights=out, + aggregation=aggregation, + implied_att=float((weight_vec * table["att"].to_numpy()).sum()), + n_negative=int(negative.sum()), + negative_weight_share=( + float(np.abs(weight_vec[negative]).sum() / abs_total) if abs_total > 0 else 0.0 + ), + n_cells=len(out), + source=source, + control_group=control_group, + base_period=base_period, + n_dropped_cells=dropped, + ) + + +def _resolve_frame_weights( + weights: Optional[Union[str, np.ndarray]], + data: pd.DataFrame, + unit: str, +) -> Optional[np.ndarray]: + """Turn ``weights=`` into one value per unit, or None.""" + if weights is None: + return None + if isinstance(weights, str): + if weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + per_unit = data.groupby(unit, sort=True)[weights].nunique() + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"weights column {weights!r} varies within unit(s) " + f"{offenders!r}; sampling weights must be time-invariant" + ) + return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) + return np.asarray(weights, dtype=float) diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py new file mode 100644 index 000000000..d55d567f3 --- /dev/null +++ b/tests/test_twfe_weights.py @@ -0,0 +1,359 @@ +"""Contract, guard and edge-case tests for the TWFE weight diagnostics. + +R output parity lives in ``tests/test_twfe_weights_parity.py``; this module +covers the behaviour that is ours rather than R's - the input guards, the +result-object surface, and the design restrictions we enforce as errors. +""" + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + + +def _panel(seed=11, n_per_cohort=40, n_periods=5, cohorts=(0, 3, 4)): + """Balanced staggered panel with a never-treated group.""" + rng = np.random.RandomState(seed) + first_treat = np.repeat(np.array(cohorts), n_per_cohort) + n_units = len(first_treat) + unit_fe = rng.normal(size=n_units) + rows = [] + for t in range(1, n_periods + 1): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(n_units), + "period": t, + "first_treat": first_treat, + "outcome": ( + unit_fe + + 0.5 * t + + 1.0 * treated * (t - first_treat + 1) + + rng.normal(scale=0.3, size=n_units) + ), + } + ) + ) + return pd.concat(rows, ignore_index=True).sort_values(["unit", "period"]) + + +def _fit(df, **kwargs): + params = {"control_group": "never_treated", "base_period": "universal"} + params.update(kwargs) + return diff_diff.CallawaySantAnna(**params).fit( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +@pytest.fixture(scope="module") +def panel(): + return _panel() + + +@pytest.fixture(scope="module") +def fitted(panel): + return _fit(panel) + + +class TestPublicSurface: + def test_exported_from_package_root(self): + assert diff_diff.attgt_weights is attgt_weights + for name in ("attgt_weights", "ATTGTWeightsResult", "TWFEDecompositionResult"): + assert name in diff_diff.__all__ + + def test_name_is_distinct_from_the_dcdh_surface(self): + """The two weight surfaces must stay separately addressable.""" + assert diff_diff.attgt_weights is not diff_diff.twowayfeweights + assert diff_diff.ATTGTWeightsResult is not diff_diff.TWFEWeightsResult + + def test_result_is_a_diagnostic_without_the_quintet(self, fitted): + result = attgt_weights(fitted) + assert isinstance(result, diff_diff.Diagnostic) + for banned in ("att", "se", "t_stat", "p_value", "conf_int"): + assert not hasattr(result, banned) + + def test_result_renders(self, fitted): + result = attgt_weights(fitted) + text = result.summary() + assert "Implicit Weights on ATT(g, t)" in text + assert "TWFE regression" in text + frame = result.to_dataframe() + assert list(frame.columns) == ["group", "time", "post", "weight", "att"] + # to_dataframe hands back a copy, not the live table + frame.loc[0, "weight"] = 999.0 + assert result.weights.loc[0, "weight"] != 999.0 + assert set(result.to_dict()) >= {"aggregation", "implied_att", "weights"} + assert "aggregation='twfe'" in repr(result) + + +class TestAggregationBehaviour: + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_implied_att_is_the_weighted_sum(self, fitted, aggregation): + result = attgt_weights(fitted, aggregation=aggregation) + expected = (result.weights["weight"] * result.weights["att"]).sum() + assert result.implied_att == pytest.approx(expected, abs=1e-14) + + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_target_estimands_are_convex(self, fitted, aggregation): + """ATT^O / ATT^simple weights are non-negative and sum to one.""" + weights = attgt_weights(fitted, aggregation=aggregation).weights["weight"] + assert (weights >= 0).all() + assert weights.sum() == pytest.approx(1.0, abs=1e-12) + + def test_twfe_weights_can_be_negative(self, fitted): + """The whole point of the diagnostic: staggered TWFE is not convex.""" + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative > 0 + assert 0.0 < result.negative_weight_share < 1.0 + assert "Negative-weight cells:" in result.summary() + + def test_pre_treatment_cells_carry_weight_under_twfe(self, fitted): + """TWFE loads on pre-treatment cells; the CS estimands do not.""" + twfe = attgt_weights(fitted, aggregation="twfe").weights + assert (twfe.loc[twfe["post"] == 0, "weight"].abs() > 0).any() + for aggregation in ("overall", "simple"): + benign = attgt_weights(fitted, aggregation=aggregation).weights + assert (benign.loc[benign["post"] == 0, "weight"] == 0).all() + + def test_rejects_unknown_aggregation(self, fitted): + with pytest.raises(ValueError, match="aggregation must be one of"): + attgt_weights(fitted, aggregation="everything") + + +class TestDesignGuards: + def test_rejects_non_universal_base_period_for_twfe(self, panel): + fit = _fit(panel, base_period="varying") + with pytest.raises(ValueError, match="base_period='universal'"): + attgt_weights(fit, aggregation="twfe") + + def test_varying_base_is_fine_for_the_cs_estimands(self, panel): + """Only the TWFE formula needs the complete grid.""" + fit = _fit(panel, base_period="varying") + for aggregation in ("overall", "simple"): + result = attgt_weights(fit, aggregation=aggregation) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_not_yet_treated_control_for_twfe(self, panel): + fit = _fit(panel, control_group="not_yet_treated") + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + def test_rejects_repeated_cross_sections(self, panel): + # A true RCS needs one observation per unit id, so re-key the rows + # rather than just flipping the flag (panel=False rejects duplicates). + rcs = panel.copy().reset_index(drop=True) + rcs["unit"] = np.arange(len(rcs)) + fit = _fit(rcs, panel=False) + with pytest.raises(ValueError, match="requires a panel fit"): + attgt_weights(fit) + + +class TestDataFrameFallback: + def test_requires_the_full_panel_spec(self, fitted, panel): + frame = fitted.to_dataframe("group_time") + with pytest.raises(ValueError, match="missing"): + attgt_weights(frame, data=panel, unit="unit") + + def test_rejects_panel_args_alongside_a_fitted_result(self, fitted, panel): + with pytest.raises(ValueError, match="only for the DataFrame fallback"): + attgt_weights( + fitted, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_accepts_an_att_column_as_well_as_effect(self, fitted, panel): + frame = fitted.to_dataframe("group_time")[["group", "time", "effect"]] + via_effect = attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat" + ) + via_att = attgt_weights( + frame.rename(columns={"effect": "att"}), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + np.testing.assert_allclose( + via_effect.weights["weight"], via_att.weights["weight"], atol=1e-15 + ) + + def test_rejects_a_frame_without_an_effect_column(self, panel): + frame = pd.DataFrame({"group": [3], "time": [3]}) + with pytest.raises(ValueError, match="'effect' or 'att'"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_time_varying_cohort_labels(self, fitted, panel): + broken = panel.copy() + broken.loc[broken.index[0], "first_treat"] = 99 + with pytest.raises(ValueError, match="varies within unit"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_source_is_recorded(self, fitted, panel): + assert attgt_weights(fitted).source == "CallawaySantAnnaResults" + from_frame = attgt_weights( + fitted.to_dataframe("group_time"), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert from_frame.source == "DataFrame" + + +class TestNonConsecutiveTimeLabels: + """Positional rescaling: gapped period labels must not change the weights.""" + + def test_gapped_periods_match_consecutive_ones(self, panel): + consecutive = attgt_weights(_fit(panel), aggregation="twfe") + + gapped = panel.copy() + remap = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50} + gapped["period"] = gapped["period"].map(remap) + gapped["first_treat"] = gapped["first_treat"].map(lambda g: remap.get(g, 0)) + result = attgt_weights(_fit(gapped), aggregation="twfe") + + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), + consecutive.weights["weight"].to_numpy(), + atol=1e-14, + ) + assert result.implied_att == pytest.approx(consecutive.implied_att, abs=1e-14) + + +class TestSamplingWeights: + def test_uniform_weights_are_a_no_op(self, fitted, panel): + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + weighted_panel = panel.assign(w=1.0) + weighted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=weighted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + baseline.weights["weight"], weighted.weights["weight"], atol=1e-15 + ) + + def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): + """Doubling a cohort's sampling weight raises its share of ATT^O.""" + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + tilted_panel = panel.assign(w=np.where(panel["first_treat"] == 3, 2.0, 1.0)) + tilted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=tilted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + mass_3_before = baseline.weights.query("group == 3")["weight"].sum() + mass_3_after = tilted.weights.query("group == 3")["weight"].sum() + assert mass_3_after > mass_3_before + assert tilted.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_time_varying_sampling_weights(self, fitted, panel): + broken = panel.copy() + broken["w"] = np.arange(len(broken), dtype=float) + with pytest.raises(ValueError, match="must be time-invariant"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + def test_rejects_a_weights_column_name_on_the_fitted_path(self, fitted): + with pytest.raises(ValueError, match="only name a column"): + attgt_weights(fitted, weights="w") + + +class TestDegenerateInputs: + def test_rejects_a_panel_with_no_treated_units(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + never = panel.assign(first_treat=0) + with pytest.raises(ValueError, match="no ever-treated units"): + attgt_weights( + frame, + data=never, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_a_frame_with_no_finite_effects(self, panel): + frame = pd.DataFrame({"group": [3.0, 3.0], "time": [3.0, 4.0], "effect": [np.nan, np.nan]}) + with pytest.raises(ValueError, match="no finite effects"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_warns_and_renormalizes_when_cells_are_dropped(self, fitted, panel): + frame = fitted.to_dataframe("group_time").copy() + frame.loc[frame.index[0], "effect"] = np.nan + with pytest.warns(UserWarning, match="had no estimable ATT"): + result = attgt_weights( + frame, + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert result.n_dropped_cells == 1 + assert len(result.weights) == len(frame) - 1 + assert "Non-estimable cells dropped:" in result.summary() + + def test_rejects_a_cohort_label_off_the_period_grid(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + broken = panel.copy() + broken.loc[broken["first_treat"] == 4, "first_treat"] = 99 + with pytest.raises(ValueError, match="not one of the observed time periods"): + attgt_weights( + frame, + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py new file mode 100644 index 000000000..5eec10f20 --- /dev/null +++ b/tests/test_twfe_weights_parity.py @@ -0,0 +1,240 @@ +"""R ``twfeweights`` output-parity tests for the TWFE weight diagnostics. + +Loads pre-computed golden values from +``benchmarks/data/twfeweights_golden.json`` (generated by +``benchmarks/R/generate_twfeweights_golden.R``) and asserts that the Python +implementation matches R ``twfeweights`` 0.9.0. + +**R is only needed to regenerate the JSON**, never to run these tests. The +committed JSON plus its sibling panel CSVs are the source of truth and the +assertions run on any Python-only environment. Tests skip ONLY if a fixture +file is absent. + +Tolerances are module constants with a stated rationale; see the tolerance +table in ``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". +""" + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + +DATA_DIR = Path(__file__).parents[1] / "benchmarks" / "data" +GOLDEN_PATH = DATA_DIR / "twfeweights_golden.json" +REGENERATE = "Rscript benchmarks/R/generate_twfeweights_golden.R" + +FIXTURES = ("mpdta", "sim_staggered", "unbalanced_cohorts") +AGGREGATIONS = ("twfe", "overall", "simple") + +# Closed-form weights: both sides evaluate the same rational expression in +# cohort masses in double precision, so only representation error separates +# them. Observed max deviation across all 3 fixtures x 3 aggregations is +# 4.7e-16 - two orders of margin below this gate. +WEIGHT_ATOL = 1e-12 +WEIGHT_RTOL = 0.0 + +# Composed check: our CallawaySantAnna ATT(g,t) vs R did::att_gt, then the +# weights on top. Bounded by the pre-existing CS parity band, not by anything +# this module introduces. +CS_COMPOSED_RTOL = 1e-6 + + +@pytest.fixture(scope="module") +def golden(): + """Load the committed R goldens; skip when absent.""" + if not GOLDEN_PATH.exists(): + pytest.skip(f"golden file not found at {GOLDEN_PATH}; run: {REGENERATE}") + with open(GOLDEN_PATH) as fh: + return json.load(fh) + + +def _fixture(golden, name): + payload = golden["fixtures"][name] + path = DATA_DIR / payload["data_file"] + if not path.exists(): + pytest.skip(f"panel {path} not found; run: {REGENERATE}") + return payload, pd.read_csv(path) + + +def _sorted_golden_weights(block): + """Golden weight table, sorted to the same key order the API emits.""" + return ( + pd.DataFrame( + { + "group": block["group"], + "time": block["time"], + "post": block["post"], + "weight": block["weight"], + "att": block["att"], + } + ) + .sort_values(["group", "time"]) + .reset_index(drop=True) + ) + + +def _fit_cs(df, cols): + return diff_diff.CallawaySantAnna(control_group="never_treated", base_period="universal").fit( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + + +class TestATTGTWeightsParity: + """Weights asserted against R using R's OWN ATT(g,t) values. + + Feeding the golden ``att`` column back in isolates the weight arithmetic + from CallawaySantAnna-vs-``did`` parity, which is covered separately by + ``csdid_golden_values.json``. A regression here is a regression in THIS + module. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_weight_column(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + expected = _sorted_golden_weights(payload["attgt_weights"][aggregation]) + + gt_frame = expected[["group", "time", "att"]].rename(columns={"att": "effect"}) + cols = payload["columns"] + result = attgt_weights( + gt_frame, + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + + assert len(actual) == len(expected) + np.testing.assert_array_equal(actual["group"].to_numpy(), expected["group"].to_numpy()) + np.testing.assert_array_equal(actual["time"].to_numpy(), expected["time"].to_numpy()) + np.testing.assert_array_equal(actual["post"].to_numpy(), expected["post"].to_numpy()) + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_implied_att(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + expected = _sorted_golden_weights(block) + cols = payload["columns"] + + result = attgt_weights( + expected[["group", "time", "att"]].rename(columns={"att": "effect"}), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + +class TestATTGTWeightsFromCSFit: + """End-to-end: fit CallawaySantAnna, then weight its own ATT(g,t). + + This is a COMPOSED check - it multiplies this module's parity by + CallawaySantAnna-vs-``did`` parity. It is deliberately looser than + :class:`TestATTGTWeightsParity`, and a failure here with that class green + points at CS, not at the weights. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_end_to_end(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + result = attgt_weights(_fit_cs(df, payload["columns"]), aggregation=aggregation) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + expected = _sorted_golden_weights(block) + + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + # ATT(g,t) come from our own fit here, so this leg carries the CS band. + np.testing.assert_allclose( + actual["att"].to_numpy(), + expected["att"].to_numpy(), + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): + """TWFE puts negative weight on some cells; the CS estimands never do.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + + twfe = attgt_weights(fit, aggregation="twfe") + assert twfe.n_negative > 0 + assert twfe.negative_weight_share > 0 + + for aggregation in ("overall", "simple"): + benign = attgt_weights(fit, aggregation=aggregation) + assert benign.n_negative == 0 + assert benign.negative_weight_share == 0.0 + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_target_estimand_weights_sum_to_one(self, golden, fixture): + """ATT^O and ATT^simple are proper averages of the post cells.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + for aggregation in ("overall", "simple"): + weights = attgt_weights(fit, aggregation=aggregation).weights + np.testing.assert_allclose(weights["weight"].sum(), 1.0, atol=1e-12) + + +class TestCSFitAndFrameAgree: + """The DataFrame fallback reproduces the fitted-result path exactly.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_paths_agree(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + fit = _fit_cs(df, cols) + + from_fit = attgt_weights(fit, aggregation=aggregation) + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + left = from_fit.weights.sort_values(["group", "time"]).reset_index(drop=True) + right = from_frame.weights.sort_values(["group", "time"]).reset_index(drop=True) + np.testing.assert_allclose( + left["weight"].to_numpy(), right["weight"].to_numpy(), atol=1e-15 + ) + np.testing.assert_allclose(from_fit.implied_att, from_frame.implied_att, atol=1e-15) From ce209a0fac2655dc8a711e639ae801dc33280348 Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 31 Aug 2026 10:08:59 +0800 Subject: [PATCH 04/13] feat(twfeweights): decompose_twfe_weights() - FWL decomposition + balance Re-derives a TWFE estimate from its ATT(g,t) building blocks, reporting the implicit weight on each cell, the contribution of PRE-treatment cells (`pretrend_bias` - parallel-trends violations rather than treatment), and implicit-weight covariate balance via `result.covariate_balance()`. Takes the raw panel rather than a fitted CS result because it re-estimates: it double-demeans treatment and covariates and forms its own group-time contrasts, so there is no ATT(g,t) table it could consume, and a CS result carries no panel by design. The two surfaces are tied by an identity that the suite pins: attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate Two numerical points, both found by disagreeing with the goldens and then proving which side was right: 1. Covariates that double-demeaning ANNIHILATES are now dropped before the projection, judged against each column's own PRE-demeaning norm. A time-invariant regressor leaves a column of pure rounding noise (~1e-16 against a raw scale of ~1); regressing on it amplifies that by ~1e16 and silently corrupted the per-cell weights. A rank test on the demeaned matrix alone cannot see this - there, 1e-16 is simply the largest pivot. With the fix, covariates=None and covariates=[] agree to 1e-15 on every fixture, which is exactly the equivalence the no-covariate golden relies on. 2. Cells whose comparison-group implicit weights are constant AND average to zero make `resid / mean(resid)` a 0/0. On sim_staggered (equal cohorts at g in {0,3,4}, T=5) this happens exactly at t=3, where -E_3[D] + mean_t E_t[D] = -1/3 + 1/3. We take the limit (a constant over its own mean is one); R divides the rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that needs none of this module: ours is exact to 4.4e-16. The weights on such cells cancel exactly in the aggregate, so `estimate` is unaffected - the suite gates `estimate` tightly on every fixture and relaxes only the per-cell and decomposition/remainder-split assertions, on cells DETECTED as degenerate rather than on a hard-coded fixture. Parity vs R twfeweights 0.9.0: estimate and per-cell weights at machine precision on all 3 fixtures x 4 configurations; all 11 balance statistics at machine precision, including `frac_treated_extreme`, which required reproducing BMisc's weighted-ECDF plus `stats:::quantile.ecdf`'s pseudo-sample reconstruction rather than a plain quantile. method="aipw" is not implemented yet and raises listing the accepted values. Co-Authored-By: Claude --- diff_diff/__init__.py | 2 + diff_diff/twfe_weights.py | 767 +++++++++++++++++++++++++++++- tests/test_twfe_weights_parity.py | 277 +++++++++++ 3 files changed, 1044 insertions(+), 2 deletions(-) diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 62914158d..9a5efd1c6 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -300,6 +300,7 @@ ) from diff_diff.twfe_weights import ( attgt_weights, + decompose_twfe_weights, ) from diff_diff.twfe_weights_results import ( ATTGTWeightsResult, @@ -472,6 +473,7 @@ def __getattr__(name: str) -> _Any: "ATTGTWeightsResult", "TWFEDecompositionResult", "attgt_weights", + "decompose_twfe_weights", # WooldridgeDiD (ETWFE) "WooldridgeDiD", "WooldridgeDiDResults", diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index a1357b636..7464c6012 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -55,13 +55,17 @@ import numpy as np import pandas as pd +from scipy.linalg import qr as scipy_qr -from diff_diff.twfe_weights_results import ATTGTWeightsResult +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) if TYPE_CHECKING: # pragma: no cover - typing only from diff_diff.staggered_results import CallawaySantAnnaResults -__all__ = ["attgt_weights"] +__all__ = ["attgt_weights", "decompose_twfe_weights"] _AGGREGATIONS = ("twfe", "overall", "simple") @@ -568,3 +572,762 @@ def _resolve_frame_weights( ) return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) return np.asarray(weights, dtype=float) + + +# --------------------------------------------------------------------------- +# Panel plumbing for the decomposition +# --------------------------------------------------------------------------- + + +def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: + """``stats::weighted.mean`` on flat arrays.""" + total = weights.sum() + if total == 0: + return float("nan") + return float((values * weights).sum() / total) + + +def _demean_two_way( + values: np.ndarray, + weights: np.ndarray, + *, + tol: float = 1e-12, + max_iter: int = 100, +) -> np.ndarray: + """Two-way (unit and period) demeaning of a ``(n_units, n_periods, k)`` block. + + Alternating projections, matching what ``fixest::demean`` does. On a + balanced panel with uniform weights this converges after a single sweep + to the closed form ``x - xbar_i - xbar_t + xbar``; the loop exists so + sampling weights (which break that identity) are still handled exactly + rather than approximately. + + ``weights`` is ``(n_units, n_periods)`` and broadcasts over the trailing + covariate axis. + """ + out = np.array(values, dtype=float, copy=True) + if out.size == 0: + return out + w = weights[:, :, None] + for _ in range(max_iter): + unit_mass = w.sum(axis=1, keepdims=True) + out -= np.divide( + (out * w).sum(axis=1, keepdims=True), + unit_mass, + out=np.zeros_like(unit_mass), + where=unit_mass > 0, + ) + time_mass = w.sum(axis=0, keepdims=True) + shift = np.divide( + (out * w).sum(axis=0, keepdims=True), + time_mass, + out=np.zeros_like(time_mass), + where=time_mass > 0, + ) + out -= shift + if np.max(np.abs(shift)) < tol: + break + return out + + +def _drop_collinear(matrix: np.ndarray) -> Tuple[np.ndarray, List[int]]: + """Drop linearly dependent columns via a pivoted QR. + + Mirrors ``BMisc::drop_collinear`` (which delegates to + ``caret::findLinearCombos``) in effect: keep a maximal independent set, + dropping later columns first. + """ + if matrix.shape[1] == 0: + return matrix, [] + _, r_mat, piv = scipy_qr(matrix, mode="economic", pivoting=True) + diag = np.abs(np.diag(r_mat)) + if diag.size == 0: + return matrix[:, :0], list(range(matrix.shape[1])) + tol = diag.max() * max(matrix.shape) * np.finfo(float).eps + rank = int((diag > tol).sum()) + keep = sorted(piv[:rank].tolist()) + dropped = [j for j in range(matrix.shape[1]) if j not in keep] + return matrix[:, keep], dropped + + +def _wls_coefficients(design: np.ndarray, target: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Weighted least squares through the origin (R's ``lm(y ~ -1 + X, w)``).""" + if design.shape[1] == 0: + return np.zeros(0) + root_w = np.sqrt(weights) + coef, *_ = np.linalg.lstsq(design * root_w[:, None], target * root_w, rcond=None) + return coef + + +def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray) -> float: + """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors.""" + sw = sampling_weights / sampling_weights.mean() + ew = est_weights / _weighted_mean(est_weights, sw) + denom = float((ew**2).sum()) + if denom == 0: + return float("nan") + return float(ew.sum() ** 2 / denom) + + +class _Panel: + """Balanced panel reshaped to ``(n_units, n_periods)`` with positional time. + + Sorting by ``(unit, period)`` and reshaping means every ``(g, t)`` slice + is a plain boolean row mask plus a column index, instead of repeated + boolean scans over the long frame. + """ + + def __init__( + self, + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + covariates: Sequence[str], + weights: Optional[str], + ) -> None: + for col in (outcome, unit, time, first_treat, *covariates): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + if weights is not None and weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + + frame = data.sort_values([unit, time]).reset_index(drop=True) + units = frame[unit].to_numpy() + periods = frame[time].to_numpy() + self.unit_ids = np.asarray(sorted(pd.unique(units))) + self.period_labels = np.asarray(sorted(pd.unique(periods))) + n_units = len(self.unit_ids) + n_periods = len(self.period_labels) + if len(frame) != n_units * n_periods: + raise ValueError( + f"decompose_twfe_weights requires a balanced panel: got " + f"{len(frame)} rows for {n_units} units x {n_periods} periods. " + "Balance it first, e.g. diff_diff.balance_panel(data, unit=..., " + "time=...)." + ) + counts = frame.groupby(unit, sort=True)[time].nunique().to_numpy() + if not np.all(counts == n_periods): + raise ValueError( + "decompose_twfe_weights requires a balanced panel: some units " + "are missing periods" + ) + + self.grid = _positional_grid(self.period_labels) + self.n_units = n_units + self.n_periods = n_periods + + cohort_long = frame[first_treat].to_numpy() + per_unit = frame.groupby(unit, sort=True)[first_treat].nunique() + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + self.cohorts = _to_positional_cohort( + cohort_long.reshape(n_units, n_periods)[:, 0], self.grid + ) + self.outcome = frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods) + if weights is None: + self.weights = np.ones((n_units, n_periods)) + else: + self.weights = frame[weights].to_numpy(dtype=float).reshape(n_units, n_periods) + if not np.allclose(self.weights, self.weights[:, :1]): + raise ValueError( + f"weights column {weights!r} varies within unit; sampling " + "weights must be time-invariant" + ) + self.covariates = tuple(covariates) + if covariates: + self.design = ( + frame[list(covariates)] + .to_numpy(dtype=float) + .reshape(n_units, n_periods, len(covariates)) + ) + else: + self.design = np.zeros((n_units, n_periods, 0)) + + periods_positional = np.arange(1, n_periods + 1) + self.treated = ( + (periods_positional[None, :] >= self.cohorts[:, None]) & (self.cohorts[:, None] != 0) + ).astype(float) + + def covariate_block( + self, names: Sequence[str], data: pd.DataFrame, unit: str, time: str + ) -> np.ndarray: + """Unit-mean-collapsed covariates, one column per name. + + R's ``twfe_cov_bal`` averages each balance covariate over ALL periods + within a unit before comparing groups, so a time-varying covariate is + summarized by its unit mean. + """ + frame = data.sort_values([unit, time]).reset_index(drop=True) + block = ( + frame[list(names)] + .to_numpy(dtype=float) + .reshape(self.n_units, self.n_periods, len(names)) + ) + return block.mean(axis=1) + + +def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: + """Frisch-Waugh-Lovell residual of treatment on covariates, plus its scale. + + Double-demeans ``D`` and ``X``, projects the demeaned treatment on the + demeaned covariates, and returns the residual. That residual IS the + implicit weight the regression applies to each observation; ``alpha_den`` + is the normalization ``E[resid * Ddot]`` from R's + ``combine_twfe_weights_gt``. + + With no covariates the projection is empty and the residual is just the + double-demeaned treatment - which is exactly the branch R cannot run, + because ``fixest::demean`` segfaults on the zero-column model matrix it + builds for ``xformula = ~1``. + """ + weights = panel.weights + d_dot = _demean_two_way(panel.treated[:, :, None], weights)[:, :, 0] + x_dot = _demean_two_way(panel.design, weights) + + flat_d = d_dot.reshape(-1) + flat_w = weights.reshape(-1) + # Explicit row count: with zero covariates the trailing axis is 0 and + # numpy cannot infer a -1 against it. This is the same no-covariate branch + # on which fixest::demean segfaults; here it simply has to be spelled out. + flat_x = x_dot.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]) + + # Drop covariates that double-demeaning ANNIHILATED before anything is + # projected on them. A time-invariant regressor leaves a column of pure + # rounding noise (~1e-16 against a raw scale of ~1), and regressing on + # that amplifies the noise by ~1e16 - which silently corrupts the per-cell + # weights. The test is scale-relative: a column counts as having no + # within-variation when its demeaned norm is negligible NEXT TO ITS OWN + # raw norm, which a rank test on the demeaned matrix alone cannot see + # (there, 1e-16 is simply the largest pivot). + raw_scale = np.linalg.norm( + panel.design.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]), + axis=0, + ) + demeaned_scale = np.linalg.norm(flat_x, axis=0) + annihilated = demeaned_scale <= 1e-10 * np.maximum(raw_scale, 1.0) + if annihilated.any(): + names = [panel.covariates[j] for j in np.flatnonzero(annihilated)] + warnings.warn( + f"covariate(s) {names!r} have no within-unit-and-period variation " + "and were dropped: two-way demeaning annihilates them, so they " + "cannot affect a two-way fixed effects regression", + UserWarning, + stacklevel=3, + ) + flat_x = flat_x[:, ~annihilated] + surviving = [name for name, drop in zip(panel.covariates, annihilated) if not drop] + + kept, dropped = _drop_collinear(flat_x) + if dropped: + names = [surviving[j] for j in dropped] + warnings.warn( + f"dropped collinear covariate column(s) {names!r} after " + "double-demeaning; they carry no within-variation independent of " + "the others", + UserWarning, + stacklevel=3, + ) + gamma = _wls_coefficients(kept, flat_d, flat_w) + resid = flat_d - (kept @ gamma if kept.shape[1] else 0.0) + alpha_den = _weighted_mean(resid * flat_d, flat_w) + if not np.isfinite(alpha_den) or alpha_den == 0: + raise ValueError( + "the treatment indicator has no within-variation left after " + "double-demeaning and covariate adjustment, so the TWFE " + "coefficient is not identified" + ) + return resid.reshape(panel.n_units, panel.n_periods), alpha_den + + +def _normalize_cell_weights( + resid: np.ndarray, sampling_weights: np.ndarray, scale: float +) -> Tuple[np.ndarray, bool]: + """Scale a cell's residuals to mean one, handling the 0/0 case. + + The implicit weights within a cell are ``resid / mean(resid)``. For the + never-treated comparison group the residual is CONSTANT within a period + (their treatment indicator is identically zero, so the double-demeaned + value is ``-E_t[D] + mean_t E_t[D]``, the same for every control unit) - + and for some cohort structures that constant is analytically ZERO. On + sim_staggered (three equal cohorts at g in {0,3,4}, T=5) it vanishes + exactly at t=3: ``-1/3 + 1/3``. + + That makes the ratio 0/0. The limit is unambiguous - a constant divided + by its own mean is one - so return exactly one rather than dividing two + rounding errors. R divides anyway, which is why its per-cell ATT(g,t) at + such a cell carries ~1e-4 of noise; the aggregate is unaffected because + the weights on the affected cells cancel exactly. + + Returns the weights and whether the degenerate branch was taken. + """ + mean = _weighted_mean(resid, sampling_weights) + spread = float(np.max(resid) - np.min(resid)) if resid.size else 0.0 + tol = 1e-12 * max(scale, 1.0) + if abs(mean) <= tol: + if spread <= tol: + return np.ones_like(resid), True + raise ValueError( + "a group-time cell has comparison-group implicit weights that " + "average to zero but are not constant, so the cell's ATT(g,t) is " + "not identified. This usually means the panel has too little " + "variation in treatment timing." + ) + return resid / mean, False + + +def _decompose_fwl( + panel: _Panel, + base_period: str, + balance_covariates: Sequence[str], + balance_block: Optional[np.ndarray], +) -> Dict[str, Any]: + """R ``implicit_twfe_weights``: TWFE as weighted ATT(g, t) + a remainder.""" + resid, alpha_den = _fwl_residuals(panel) + weights = panel.weights + flat_w = weights.reshape(-1) + cohorts = panel.cohorts + treated_cohorts = sorted({int(g) for g in cohorts if g != 0}) + if not treated_cohorts: + raise ValueError("no ever-treated units found; nothing to decompose") + control_mask = cohorts == 0 + if not control_mask.any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) + if base_period == "gmin1" and 1 in treated_cohorts: + raise ValueError( + "base_period='gmin1' needs a period before each cohort's " + "treatment, but a cohort is treated in the first period. Use " + "base_period='first_period', or drop that cohort." + ) + + resid_scale = float(np.abs(resid).max()) + cells: List[Dict[str, Any]] = [] + balance_rows: List[Dict[str, Any]] = [] + degenerate_cells: List[Tuple[Any, Any]] = [] + for g in treated_cohorts: + treated_mask = cohorts == g + for t_pos in range(1, panel.n_periods + 1): + col = t_pos - 1 + w_treated = weights[treated_mask, col] + w_control = weights[control_mask, col] + + r_treated = resid[treated_mask, col] + r_control = resid[control_mask, col] + gpart_w, _ = _normalize_cell_weights(r_treated, w_treated, resid_scale) + upart_w, degenerate = _normalize_cell_weights(r_control, w_control, resid_scale) + if degenerate: + degenerate_cells.append((panel.period_labels[g - 1], panel.period_labels[col])) + + y_t = panel.outcome[:, col] + if base_period == "first_period": + base = panel.outcome[:, 0] + else: + base = panel.outcome[:, g - 2] + adjusted = y_t - base + + gpart = _weighted_mean(gpart_w * adjusted[treated_mask], w_treated) + upart = _weighted_mean(upart_w * adjusted[control_mask], w_control) + + p_g = _weighted_mean( + (cohorts == g).astype(float)[:, None].repeat(panel.n_periods, axis=1).reshape(-1), + flat_w, + ) + alpha_weight = ( + _weighted_mean(r_treated, w_treated) * p_g / (alpha_den * panel.n_periods) + ) + + remainder = 0.0 + if base_period == "gmin1": + y_gmin1 = panel.outcome[:, g - 2] + remainder = -_weighted_mean(upart_w * y_gmin1[control_mask], w_control) + + cells.append( + { + "group": panel.period_labels[g - 1], + "time": panel.period_labels[col], + "post": int(t_pos >= g), + "att": gpart - upart, + "weight": alpha_weight, + "ess": _effective_sample_size(upart_w, w_control), + "remainder": remainder, + } + ) + if balance_block is not None: + balance_rows.extend( + _balance_cell( + balance_block, + balance_covariates, + treated_mask, + control_mask, + gpart_w, + upart_w, + w_treated, + w_control, + group=panel.period_labels[g - 1], + time=panel.period_labels[col], + post=int(t_pos >= g), + ) + ) + + if degenerate_cells: + warnings.warn( + f"{len(degenerate_cells)} group-time cell(s) {degenerate_cells[:4]!r}" + " have comparison-group implicit weights that are constant and " + "average to zero, so their ATT(g,t) is a 0/0 limit (taken as the " + "unweighted contrast). The weights on these cells cancel in the " + "aggregate, so `estimate` is unaffected; read the individual " + "ATT(g,t) there with caution", + UserWarning, + stacklevel=3, + ) + + frame = pd.DataFrame(cells) + weight_col = frame["weight"].to_numpy() + att_col = frame["att"].to_numpy() + post_col = frame["post"].to_numpy().astype(bool) + decomposition = float((weight_col * att_col).sum()) + remainder_total = float((frame["remainder"].to_numpy() * weight_col).sum()) + ess_col = frame["ess"].to_numpy() + return { + "cells": frame, + "estimate": decomposition + remainder_total, + "decomposition": decomposition, + "remainder": remainder_total, + "pretrend_bias": float((weight_col[~post_col] * att_col[~post_col]).sum()), + "post_only": float((weight_col[post_col] * att_col[post_col]).sum()), + # summary.decomposed_twfe: post cells only, on both factors + "effective_sample_size": float( + post_col.sum() * (weight_col[post_col] * ess_col[post_col]).sum() + ), + "balance": pd.DataFrame(balance_rows) if balance_block is not None else None, + } + + +# --------------------------------------------------------------------------- +# Balance statistics (Imbens & Rubin 2015, as implemented upstream) +# --------------------------------------------------------------------------- + + +def _weighted_ecdf(values: np.ndarray, weights: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """``BMisc::weighted_ecdf``: knots and CDF heights. + + ``weights`` are normalized by their mean, the knots are the sorted unique + values, and ``F(knot_j) = mean(w * (y <= knot_j))``. + """ + w = weights / weights.mean() + knots = np.unique(values) + heights = np.array([float((w * (values <= knot)).mean()) for knot in knots]) + return knots, heights + + +def _ecdf_eval(knots: np.ndarray, heights: np.ndarray, at: float) -> float: + """Evaluate the step function from ``BMisc::make_dist``. + + ``approxfun(method="constant", yleft=0, yright=1, f=0)``: the value on + ``[knot_i, knot_{i+1})`` is ``heights[i]``, zero below the first knot and + one above the last. + """ + if at < knots[0]: + return 0.0 + if at > knots[-1]: + return 1.0 + idx = int(np.searchsorted(knots, at, side="right") - 1) + return float(heights[idx]) + + +def _ecdf_quantile(knots: np.ndarray, heights: np.ndarray, prob: float) -> float: + """``stats:::quantile.ecdf``: type-7 quantile of a reconstructed sample. + + R does NOT invert the step function directly. It rebuilds a pseudo-sample + by repeating each knot ``diff(c(0, round(nobs * F)))`` times - where + ``nobs`` is the number of KNOTS, not the number of observations - and then + takes an ordinary type-7 quantile of that. Reproduced exactly, because the + rounding makes the result differ from a direct inversion. + """ + nobs = len(knots) + counts = np.diff(np.concatenate([[0.0], np.round(nobs * heights)])) + counts = np.maximum(counts, 0).astype(int) + sample = np.repeat(knots, counts) + if sample.size == 0: + return float("nan") + # R's default type-7 quantile. + sample = np.sort(sample) + h = (len(sample) - 1) * prob + lo = int(np.floor(h)) + hi = min(lo + 1, len(sample) - 1) + return float(sample[lo] + (h - lo) * (sample[hi] - sample[lo])) + + +def _pooled_sd(x: np.ndarray, treated: np.ndarray, sampling_weights: np.ndarray) -> float: + """Pooled standard deviation across the treated and comparison groups.""" + sw = sampling_weights / sampling_weights.mean() + + def wvar(values: np.ndarray, w: np.ndarray) -> float: + return _weighted_mean((values - _weighted_mean(values, w)) ** 2, w) + + var1 = wvar(x[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + if n1 + n0 - 2 <= 0: + return float("nan") + return float(np.sqrt(((n1 - 1) * var1 + (n0 - 1) * var0) / (n1 + n0 - 2))) + + +def _normalize_est_weights( + est_weights: np.ndarray, treated: np.ndarray, sw: np.ndarray +) -> np.ndarray: + """Scale estimation weights to mean one WITHIN each group, as R does.""" + out = np.array(est_weights, dtype=float, copy=True) + for group in (0, 1): + mask = treated == group + if mask.any(): + out[mask] = out[mask] / _weighted_mean(out[mask], sw[mask]) + return out + + +def _log_ratio_sd( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, +) -> float: + """Log ratio of treated to comparison spread. + + Note: upstream scales each group's SD by ``sqrt(n - 1)`` before taking + the ratio, which is not a conventional standard deviation. Preserved + verbatim for parity - the quantity is only ever read as a relative + balance statistic, and the extra factor largely cancels in the ratio. + """ + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + def wvar(values: np.ndarray, e: np.ndarray, w: np.ndarray) -> float: + scaled = values * e + return _weighted_mean((scaled - _weighted_mean(scaled, w)) ** 2, w) + + var1 = wvar(x[treated == 1], ew[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], ew[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + sd1 = np.sqrt(max(n1 - 1, 0)) * np.sqrt(var1) + sd0 = np.sqrt(max(n0 - 1, 0)) * np.sqrt(var0) + if sd1 <= 0 or sd0 <= 0: + return float("nan") + return float(np.log(sd1) - np.log(sd0)) + + +def _frac_treated_extreme( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, + alpha: float = 0.05, +) -> float: + """Share of treated mass outside the comparison group's central range. + + A step function of a weighted empirical CDF, so a perturbation of order + 1e-12 can move one unit across a knot and shift the value by 1/n. Tests + gate it with an absolute tolerance of ``1 / n_control`` rather than a + relative one. + """ + if len(np.unique(x)) < 3: + return float("nan") + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + control = treated == 0 + treat = treated == 1 + knots_u, heights_u = _weighted_ecdf(ew[control] * x[control], sw[control]) + upper = _ecdf_quantile(knots_u, heights_u, 1 - alpha / 2) + lower = _ecdf_quantile(knots_u, heights_u, alpha / 2) + knots_t, heights_t = _weighted_ecdf(ew[treat] * x[treat], sw[treat]) + return float( + 1.0 - _ecdf_eval(knots_t, heights_t, upper) + _ecdf_eval(knots_t, heights_t, lower) + ) + + +def _balance_cell( + block: np.ndarray, + names: Sequence[str], + treated_mask: np.ndarray, + control_mask: np.ndarray, + weights_treated: np.ndarray, + weights_control: np.ndarray, + sw_treated: np.ndarray, + sw_control: np.ndarray, + *, + group: Any, + time: Any, + post: int, +) -> List[Dict[str, Any]]: + """Per-covariate implicit-weight balance for one ``(g, t)`` cell.""" + both = treated_mask | control_mask + indicator = np.where(treated_mask[both], 1, 0) + est = np.empty(int(both.sum())) + est[indicator == 1] = weights_treated + est[indicator == 0] = weights_control + sw_both = np.empty_like(est) + sw_both[indicator == 1] = sw_treated + sw_both[indicator == 0] = sw_control + ones = np.ones_like(est) + + rows: List[Dict[str, Any]] = [] + for j, name in enumerate(names): + col = block[:, j] + x_t = col[treated_mask] + x_c = col[control_mask] + x_both = col[both] + unweighted_treated = _weighted_mean(x_t, sw_treated) + unweighted_control = _weighted_mean(x_c, sw_control) + weighted_treated = _weighted_mean(x_t * weights_treated, sw_treated) + weighted_control = _weighted_mean(x_c * weights_control, sw_control) + rows.append( + { + "group": group, + "time": time, + "post": post, + "covariate": name, + "unweighted_treated": unweighted_treated, + "unweighted_control": unweighted_control, + "unweighted_diff": unweighted_treated - unweighted_control, + "weighted_treated": weighted_treated, + "weighted_control": weighted_control, + "weighted_diff": weighted_treated - weighted_control, + "sd": _pooled_sd(x_both, indicator, sw_both), + "unweighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, ones, sw_both), + "weighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, est, sw_both), + "unweighted_frac_extreme": _frac_treated_extreme(x_both, indicator, ones, sw_both), + "weighted_frac_extreme": _frac_treated_extreme(x_both, indicator, est, sw_both), + } + ) + return rows + + +_METHODS = ("fwl",) +_BASE_PERIODS = ("first_period", "gmin1") + + +def decompose_twfe_weights( + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + method: str = "fwl", + covariates: Optional[Sequence[str]] = None, + base_period: str = "first_period", + balance_covariates: Optional[Sequence[str]] = None, + weights: Optional[str] = None, +) -> TWFEDecompositionResult: + """Decompose a TWFE estimate into weighted group-time effects. + + Runs the regression, recovers the implicit weight it places on each + ATT(g, t), and separates the part of the estimate that comes from + PRE-treatment cells - i.e. from parallel-trends violations rather than + from treatment. + + Takes the raw panel rather than a fitted result, because it re-estimates: + it double-demeans treatment and covariates and forms its own group-time + contrasts, so there is no ATT(g, t) table it could consume. Its companion + :func:`attgt_weights` is the fitted-result surface, and the two are tied + by an identity that holds when the fit used ``base_period="universal"``, + ``control_group="never_treated"`` and no covariates:: + + sum(attgt_weights(cs, aggregation="twfe").weights.eval("weight * att")) + == decompose_twfe_weights(panel, ...).estimate + + Parameters + ---------- + data : pd.DataFrame + Balanced panel in long form. + outcome, unit, time, first_treat : str + Column names, matching :meth:`CallawaySantAnna.fit`. Never-treated + units carry ``first_treat`` of ``0`` (or ``inf``). + method : {"fwl"}, default "fwl" + ``"fwl"`` recovers the Frisch-Waugh-Lovell implicit weights from the + TWFE regression. + covariates : sequence of str, optional + Covariates the regression adjusts for. ``None`` runs the + no-covariate decomposition. + base_period : {"first_period", "gmin1"}, default "first_period" + Which pre-period each cell is measured against. ``"gmin1"`` (the + period before treatment) generates a non-zero ``remainder``. + balance_covariates : sequence of str, optional + Covariates to report implicit-weight balance for, readable afterwards + via :meth:`TWFEDecompositionResult.covariate_balance`. Each is + averaged over periods within unit before groups are compared, as + upstream does. + weights : str, optional + Time-invariant sampling-weight column. + + Returns + ------- + TWFEDecompositionResult + + Raises + ------ + ValueError + On an unknown ``method`` or ``base_period``; on an unbalanced panel, + a missing never-treated group, or time-varying cohort labels; or when + the treatment has no within-variation left after demeaning. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> dec = diff_diff.decompose_twfe_weights( # doctest: +SKIP + ... panel, outcome="y", unit="id", time="t", first_treat="g", + ... covariates=["x"], balance_covariates=["x"], + ... ) + >>> dec.pretrend_bias # doctest: +SKIP + >>> dec.covariate_balance() # doctest: +SKIP + """ + if method not in _METHODS: + raise ValueError(f"method must be one of {list(_METHODS)!r}, got {method!r}") + if base_period not in _BASE_PERIODS: + raise ValueError( + f"base_period must be one of {list(_BASE_PERIODS)!r}, got " f"{base_period!r}" + ) + + covariate_names = tuple(covariates or ()) + balance_names = tuple(balance_covariates or ()) + panel = _Panel( + data, + outcome=outcome, + unit=unit, + time=time, + first_treat=first_treat, + covariates=covariate_names, + weights=weights, + ) + balance_block = ( + panel.covariate_block(balance_names, data, unit, time) if balance_names else None + ) + + payload = _decompose_fwl(panel, base_period, balance_names, balance_block) + return TWFEDecompositionResult( + cells=payload["cells"], + method=method, + estimate=payload["estimate"], + decomposition=payload["decomposition"], + remainder=payload["remainder"], + pretrend_bias=payload["pretrend_bias"], + post_only=payload["post_only"], + base_period=base_period, + covariates=covariate_names, + effective_sample_size=payload["effective_sample_size"], + n_units=panel.n_units, + n_periods=panel.n_periods, + balance=payload["balance"], + ) diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py index 5eec10f20..a668573a6 100644 --- a/tests/test_twfe_weights_parity.py +++ b/tests/test_twfe_weights_parity.py @@ -238,3 +238,280 @@ def test_paths_agree(self, golden, fixture, aggregation): left["weight"].to_numpy(), right["weight"].to_numpy(), atol=1e-15 ) np.testing.assert_allclose(from_fit.implied_att, from_frame.implied_att, atol=1e-15) + + +# FWL decomposition. R double-demeans with `fixest::demean`, an iterative +# alternating-projections solver with a 1e-8 fixed-point tolerance; ours is +# the exact closed form on a balanced panel. The gap is fixest's convergence +# slack, which then propagates through the OLS projection of Ddot on Xdot. +DEMEAN_ATOL = 1e-10 +DEMEAN_COV_ATOL = 1e-8 +BALANCE_ATOL = 1e-9 + +# Cells whose comparison-group implicit weights are constant AND average to +# zero: ATT(g,t) there is a 0/0 limit. We return the limit (the unweighted +# contrast, exact); R divides the rounding errors and lands ~1e-4 away. The +# weights on such cells cancel exactly in the aggregate, so `estimate` is +# unaffected - which is why the scalar assertions below stay at 1e-10 while +# the per-cell gate is relaxed only where the degeneracy is DETECTED, never +# by hard-coding a fixture or period. +DEGENERATE_CELL_ATOL = 5e-2 + +# R names balance rows `mean_` (it averages each covariate over +# periods within unit first); we keep the covariate's own name. +R_BALANCE_COLUMNS = { + "unweighted_covs_treated": "unweighted_treated", + "unweighted_covs_comparison": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_covs_treated": "weighted_treated", + "weighted_covs_comparison": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", +} + + +def _decompose(df, cols, **kwargs): + return diff_diff.decompose_twfe_weights( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + **kwargs, + ) + + +def _degenerate_mask(cells, golden_cells): + """Rows where R's ATT(g,t) is a 0/0 artifact rather than a disagreement. + + Detected from the DATA: a degenerate cell is one whose weight is exactly + offset by another cell in the same period (they cancel in the aggregate), + which is the signature of a vanishing comparison-group normalizer. + """ + weights = np.asarray(golden_cells["weight"], dtype=float) + times = np.asarray(golden_cells["time"], dtype=float) + mask = np.zeros(len(weights), dtype=bool) + for t in np.unique(times): + in_period = times == t + if in_period.sum() > 1 and abs(weights[in_period].sum()) < 1e-12: + mask |= in_period + return mask + + +class TestDecompositionParityFWL: + """R ``implicit_twfe_weights`` parity, including the no-covariate branch.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_no_covariate_branch_two_ways(self, golden, fixture): + """``covariates=None`` and a time-invariant covariate must agree. + + The golden was generated with ``xformula = ~`` + because upstream cannot run ``~1`` (``fixest::demean`` segfaults on + the zero-column model matrix). Asserting BOTH Python calls against + that single golden proves the equivalence instead of assuming it. + """ + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"]["fwl_nocov"] + + without = _decompose(df, cols, covariates=None) + with pytest.warns(UserWarning, match="no within-unit-and-period variation"): + with_invariant = _decompose(df, cols, covariates=[cols["invariant_cov"]]) + + np.testing.assert_allclose( + without.cells["weight"].to_numpy(), + with_invariant.cells["weight"].to_numpy(), + atol=1e-15, + ) + assert without.estimate == pytest.approx(with_invariant.estimate, abs=1e-15) + + for result in (without, with_invariant): + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=DEMEAN_ATOL) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=DEMEAN_ATOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) + def test_scalars(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + + # `estimate` is invariant to the 0/0 cells - the weights on them + # cancel - so it is gated tightly on EVERY fixture. The + # decomposition/remainder SPLIT is not invariant: under gmin1 the + # remainder is itself built from the degenerate comparison-group + # weights, so R's noise moves mass between the two halves while + # leaving their sum exact. + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=atol) + + split_atol = atol + if _degenerate_mask(result.cells, expected["cells"]).any(): + split_atol = DEGENERATE_CELL_ATOL + for field in ("decomposition", "remainder"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=split_atol) + + # estimate == decomposition + remainder is an identity, not a fit + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov"]) + def test_cells(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=atol, + ) + + actual_att = result.cells["att"].to_numpy() + golden_att = np.asarray(expected["cells"]["att"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + np.testing.assert_allclose(actual_att[~degenerate], golden_att[~degenerate], atol=atol) + if degenerate.any(): + np.testing.assert_allclose( + actual_att[degenerate], + golden_att[degenerate], + atol=DEGENERATE_CELL_ATOL, + ) + + +class TestDecompositionIsExactAtDegenerateCells: + """Where R reports 0/0 noise, we report the analytic limit.""" + + def test_limit_equals_the_unweighted_contrast(self, golden): + """sim_staggered has three equal cohorts, so E_3[D] == mean_t E_t[D]. + + The comparison-group implicit weights are then constant and average to + zero. The limit of ``resid / mean(resid)`` for a constant vector is + one, so ATT(g, 3) is the plain difference of mean outcome changes - + computable here without any of the module's machinery. + """ + payload, df = _fixture(golden, "sim_staggered") + cols = payload["columns"] + result = _decompose(df, cols, covariates=None) + + wide = df.pivot(index=cols["unit"], columns=cols["time"], values=cols["outcome"]).to_numpy() + cohorts = df.groupby(cols["unit"])[cols["first_treat"]].first().to_numpy() + change = wide[:, 2] - wide[:, 0] # base_period="first_period" + control_mean = change[cohorts == 0].mean() + + for cohort in (3, 4): + expected = change[cohorts == cohort].mean() - control_mean + actual = result.cells.query("group == @cohort and time == 3")["att"] + assert actual.iloc[0] == pytest.approx(expected, abs=1e-12) + + def test_warns_about_the_degenerate_cells(self, golden): + payload, df = _fixture(golden, "sim_staggered") + with pytest.warns(UserWarning, match="0/0 limit"): + _decompose(df, payload["columns"], covariates=None) + + +class TestBalanceParity: + """R ``twfe_cov_bal`` + ``mp_covariate_bal_summary_helper`` parity.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_cell_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["cells"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="cell", standardize=False) + + key = ["group", "time", "covariate"] + expected = expected.sort_values(key).reset_index(drop=True) + actual = actual.sort_values(key).reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + + for r_name, our_name in R_BALANCE_COLUMNS.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance column {our_name!r} ({fixture})", + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_summary_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["summary"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="summary", standardize=False) + + expected = expected.sort_values("covariate").reset_index(drop=True) + actual = actual.sort_values("covariate").reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + + r_summary = { + "unweighted_treat": "unweighted_treated", + "unweighted_untreat": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_treat": "weighted_treated", + "weighted_untreat": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", + } + for r_name, our_name in r_summary.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance summary {our_name!r} ({fixture})", + ) + + +class TestCrossSurfaceIdentity: + """attgt_weights and decompose_twfe_weights describe the same regression.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_twfe_weights_reproduce_the_decomposition(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + + weighted = attgt_weights(_fit_cs(df, cols), aggregation="twfe") + decomposed = _decompose(df, cols, covariates=None) + + assert weighted.implied_att == pytest.approx(decomposed.estimate, abs=1e-6) From c4efb850503b6c9b61b5f0ea3eb531e213e4fe24 Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 31 Aug 2026 11:42:10 +0800 Subject: [PATCH 05/13] feat(twfeweights): plot_twfe_weights() + full docs surface Plotting (replacing upstream's ggtwfeweights S3 methods) and every documentation surface the new API owes. plot_twfe_weights(result, kind="auto"|"weights"|"balance") lives beside plot_bacon in visualization/_diagnostic.py and dispatches on either result type. The weights view puts weight on x and ATT(g,t) on y with zero lines, so negative-weight cells sit visibly left of the axis; the balance view plots unweighted against implicitly-weighted covariate differences with a no-improvement diagonal. "auto" picks balance when a balance table is present. Docs: a REGISTRY.md section carrying the weight equations, the cross-surface identity, the tolerance table with per-gate rationale, and eleven explicit Note/Deviation-from-R entries - including the fixest zero-column segfault and its root cause, the annihilated-covariate drop, and the 0/0-cell limit, so the two places we deliberately differ from R are recorded rather than discovered later by a reviewer. Two paragraphs separate this surface from `twowayfeweights` (dCDH, weights (unit, time) cells) and from BaconDecomposition (decomposes into 2x2 comparisons), since all three are "TWFE weight" diagnostics and the distinction is the thing a reader most needs. Also: docs/api/twfe_weights.rst with runnable examples, four api/index.rst registrations (2 result classes, the plot, 2 functions, toctree), doc-deps.yaml group + sources entries, a README one-liner in Diagnostics & Sensitivity, llms.txt catalog entry, llms-full.txt API + result blocks, a references.rst sub-entry naming the upstream package and its MIT copyright, and a changelog.d fragment. This closes the doc-deps gate the attgt_weights commit left red. Verified: 14378 tests collect clean; docs IA, doc-deps integrity, diagnostic roster, guides, changelog-fragment, serialization and all visualization suites green (903 passed, 43 skipped). Co-Authored-By: Claude --- README.md | 1 + .../20260831-twfe-weight-diagnostics.md | 33 ++++ diff_diff/__init__.py | 2 + diff_diff/guides/llms-full.txt | 65 +++++++ diff_diff/guides/llms.txt | 1 + diff_diff/visualization/__init__.py | 2 + diff_diff/visualization/_diagnostic.py | 173 ++++++++++++++++++ docs/api/index.rst | 6 + docs/api/twfe_weights.rst | 145 +++++++++++++++ docs/doc-deps.yaml | 30 +++ docs/methodology/REGISTRY.md | 118 ++++++++++++ docs/references.rst | 5 + 12 files changed, 581 insertions(+) create mode 100644 changelog.d/20260831-twfe-weight-diagnostics.md create mode 100644 docs/api/twfe_weights.rst diff --git a/README.md b/README.md index bd7496f84..e1879cc80 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html) - Cattaneo, Jansson & Ma (2020) density-discontinuity test (`RDDensityTest`): rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna (2025) implicit weights on ATT(g,t): `attgt_weights(cs_result, aggregation='twfe'|'overall'|'simple')` shows what a TWFE regression (vs ATT^O / ATT^simple) implicitly puts on each group-time effect, including negative weights; `decompose_twfe_weights(panel, method='fwl')` re-derives the estimate from its building blocks with the pre-trend-violation contribution and implicit covariate balance. Ported from Brantly Callaway's `twfeweights` R package (MIT) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md new file mode 100644 index 000000000..07d7420b3 --- /dev/null +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -0,0 +1,33 @@ +### Added +- **TWFE weight diagnostics** (port of Brantly Callaway's `twfeweights` R + package, MIT): what a two-way fixed effects regression *implicitly* weights + on staggered-adoption data. + - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the + weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), + plus the negative-weight share. Takes a fitted `CallawaySantAnnaResults` + (reading cohort masses off its aggregation bookkeeping, so no raw panel is + needed); a `(gt_frame, data=, unit=, time=, first_treat=)` fallback + consumes `result.to_dataframe("group_time")` verbatim. Returns + `ATTGTWeightsResult`. `aggregation="twfe"` requires + `base_period="universal"` and `control_group="never_treated"`, matching + the restrictions R enforces. + - `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, + method="fwl", covariates=, base_period="first_period"|"gmin1")` re-derives + the estimate from its ATT(g,t) building blocks and returns + `TWFEDecompositionResult` with `pretrend_bias` — the contribution of + pre-treatment cells, i.e. of parallel-trends violations rather than of + treatment. With `balance_covariates=`, `result.covariate_balance()` + reports whether the implicit weights actually balance those covariates. + - `plot_twfe_weights()` renders either view. + + Names are deliberately separate from the existing `twowayfeweights` / + `TWFEWeightsResult` (de Chaisemartin & D'Haultfoeuille) surface, which + weights (unit, time) cells rather than ATT(g,t) parameters. + + Validated against R `twfeweights` 0.9.0 output on three fixtures (`mpdta` + plus two simulated panels); goldens at + `benchmarks/data/twfeweights_golden.json`, regenerated by + `benchmarks/R/generate_twfeweights_golden.R`. R is never needed to run the + test suite. Methodology: Baker, Callaway, Cunningham, Goodman-Bacon & + Sant'Anna (2025); Callaway & Sant'Anna (2021) for the ATT^O / ATT^simple + weights. diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 9a5efd1c6..dcbb3ea77 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -331,6 +331,7 @@ plot_sensitivity, plot_staircase, plot_synth_weights, + plot_twfe_weights, ) from diff_diff.wooldridge import WooldridgeDiD from diff_diff.wooldridge_results import WooldridgeDiDResults @@ -494,6 +495,7 @@ def __getattr__(name: str) -> _Any: "DurationDiDPretestResults", # Visualization "plot_bacon", + "plot_twfe_weights", "plot_event_study", "plot_group_effects", "plot_sensitivity", diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index b84b3f03d..8a51fedb6 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1534,6 +1534,44 @@ results.print_summary() plot_bacon(results) ``` +### TWFE Weight Diagnostics + +What a TWFE regression implicitly weights on staggered data. Distinct from +`twowayfeweights` (dCDH), which weights (unit, time) cells: these weight +ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package +(MIT); methodology Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna +(2025). + +```python +attgt_weights( + results, # CallawaySantAnnaResults, or a (g,t) frame + aggregation="twfe", # "twfe" | "overall" (ATT^O) | "simple" + data=None, unit=None, time=None, first_treat=None, # frame path only + weights=None, # unit-level sampling weights +) -> ATTGTWeightsResult + +decompose_twfe_weights( + data, # balanced long panel (it re-estimates) + outcome=, unit=, time=, first_treat=, + method="fwl", + covariates=None, + base_period="first_period", # or "gmin1" + balance_covariates=None, # enables result.covariate_balance() + weights=None, +) -> TWFEDecompositionResult + +plot_twfe_weights(result, kind="auto") # "weights" | "balance" +``` + +`aggregation="twfe"` requires a fit with `base_period="universal"` and +`control_group="never_treated"`; it raises otherwise. ATT^O and ATT^simple +weights are non-negative and sum to one, so comparing `implied_att` across +the three aggregations shows what the TWFE specification costs. + +`decompose_twfe_weights` takes the raw panel rather than a fitted result +because it re-estimates. It is tied to `attgt_weights` by an identity: +`attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. + ### StaggeredTripleDifference DEPRECATED in 3.9, removed in 4.0 (ledger row M-013). Use @@ -2017,6 +2055,33 @@ Returned by `BaconDecomposition.fit()` (and the deprecated `bacon_decompose()` w **Methods:** `summary()`, `print_summary()`, `to_dataframe()` +### ATTGTWeightsResult + +Diagnostic result from `attgt_weights`. No inference quintet - the +decomposition is an algebraic identity. + +- `weights`: DataFrame with `group`, `time`, `post`, `weight`, `att` +- `implied_att`: `sum(weight * att)` - the TWFE coefficient when + `aggregation="twfe"` +- `n_negative`, `negative_weight_share`: the staggered-TWFE pathology +- `aggregation`, `source`, `control_group`, `base_period`, `n_cells` +- `summary()`, `to_dataframe()`, `to_dict()` + +### TWFEDecompositionResult + +Diagnostic result from `decompose_twfe_weights`. + +- `cells`: DataFrame with `group`, `time`, `post`, `att`, `weight`, `ess`, + `remainder` +- `estimate` == `decomposition` + `remainder` +- `pretrend_bias`: contribution of PRE-treatment cells, i.e. of + parallel-trends violations rather than of treatment +- `post_only`, `effective_sample_size`, `covariates`, `base_period` +- `covariate_balance(level="summary"|"cell", standardize=True, + post_only=True)`: implicit-weight covariate balance; raises when + `balance_covariates=` was not requested +- `summary()`, `to_dataframe()`, `to_dict()` + ### Comparison2x2 Individual 2x2 DiD comparison (used in BaconDecompositionResults). diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index a9df7dc97..784fe8536 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -92,6 +92,7 @@ The site is organized into 5 sections, each with a landing page: - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html): Cattaneo, Jansson & Ma (2020) density-discontinuity manipulation test (`RDDensityTest`), parity with R rddensity 3.0 - boundary-adaptive local polynomial density estimation at the cutoff, robust bias-corrected inference, unrestricted/restricted models, jackknife/plugin variances, data-driven bandwidths, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pretrend_bias`, and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/diff_diff/visualization/__init__.py b/diff_diff/visualization/__init__.py index 4c06bb751..8d255f67c 100644 --- a/diff_diff/visualization/__init__.py +++ b/diff_diff/visualization/__init__.py @@ -15,6 +15,7 @@ from diff_diff.visualization._diagnostic import ( plot_bacon, plot_sensitivity, + plot_twfe_weights, ) from diff_diff.visualization._event_study import ( PlottableResults, @@ -48,6 +49,7 @@ "plot_group_effects", "plot_sensitivity", "plot_bacon", + "plot_twfe_weights", "plot_power_curve", "plot_pretrends_power", # New public functions diff --git a/diff_diff/visualization/_diagnostic.py b/diff_diff/visualization/_diagnostic.py index ef2dd05b2..599a6216a 100644 --- a/diff_diff/visualization/_diagnostic.py +++ b/diff_diff/visualization/_diagnostic.py @@ -817,3 +817,176 @@ def _render_bacon_plotly( fig.show() return fig + + +def plot_twfe_weights( + results: Any, + *, + kind: str = "auto", + standardize: bool = True, + absolute_value: bool = True, + figsize: Tuple[float, float] = (10, 6), + title: Optional[str] = None, + xlabel: Optional[str] = None, + ylabel: Optional[str] = None, + post_color: str = "#2563eb", + pre_color: str = "#dc2626", + markersize: int = 80, + alpha: float = 0.8, + annotate: bool = False, + ax: Optional[Any] = None, + show: bool = True, +) -> Any: + """Visualize implicit TWFE weights on ATT(g, t), or their covariate balance. + + Two views, matching upstream's ``ggtwfeweights`` methods: + + - ``kind="weights"`` plots weight against ATT(g, t), one point per + group-time cell, coloured by pre/post. Points to the LEFT of the + vertical zero line carry negative weight - the staggered-TWFE + pathology. + - ``kind="balance"`` plots unweighted against implicitly-weighted + covariate differences. Points near zero on the vertical axis are + covariates the implicit weights balance. + + Parameters + ---------- + results : ATTGTWeightsResult or TWFEDecompositionResult + Output of :func:`diff_diff.attgt_weights` or + :func:`diff_diff.decompose_twfe_weights`. + kind : {"auto", "weights", "balance"}, default "auto" + ``"auto"`` picks ``"balance"`` when the result carries a balance + table and ``"weights"`` otherwise. + standardize : bool, default True + Balance view: divide differences by the pooled standard deviation. + absolute_value : bool, default True + Balance view: plot absolute differences, so "closer to zero is + better" reads the same for every covariate. + figsize : tuple, default (10, 6) + Figure size in inches. Ignored when ``ax`` is supplied. + title, xlabel, ylabel : str, optional + Overrides for the defaults chosen per ``kind``. + post_color, pre_color : str + Colors for post- and pre-treatment cells (weights view). + markersize : int, default 80 + Scatter marker area. + alpha : float, default 0.8 + Marker opacity. + annotate : bool, default False + Label each point with its ``(group, time)`` or covariate name. + ax : matplotlib Axes, optional + Axes to draw on. A new figure is created when omitted. + show : bool, default True + Call ``plt.show()`` before returning. + + Returns + ------- + matplotlib.axes.Axes + + Raises + ------ + ValueError + On an unknown ``kind``, or when ``kind="balance"`` is requested for a + result that carries no balance table. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> w = diff_diff.attgt_weights(cs_result) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w) # doctest: +SKIP + """ + if kind not in ("auto", "weights", "balance"): + raise ValueError(f"kind must be one of ['auto', 'weights', 'balance'], got {kind!r}") + has_balance = getattr(results, "balance", None) is not None + if kind == "auto": + kind = "balance" if has_balance else "weights" + if kind == "balance" and not has_balance: + raise ValueError( + "this result carries no covariate balance table, so kind='balance' " + "has nothing to plot. Recompute with " + "decompose_twfe_weights(..., balance_covariates=[...])." + ) + + from diff_diff.visualization._common import _require_matplotlib + + plt = _require_matplotlib() + if ax is None: + _, ax = plt.subplots(figsize=figsize) + + if kind == "weights": + table = getattr(results, "weights", None) + if table is None: + table = results.cells + post = table["post"].to_numpy().astype(bool) + weight = table["weight"].to_numpy() + att = table["att"].to_numpy() + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.axvline(0, color="0.4", linewidth=1.2, zorder=1) + for mask, color, label in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + ax.scatter( + weight[mask], + att[mask], + s=markersize, + alpha=alpha, + color=color, + label=label, + zorder=3, + ) + if annotate: + for w, a, g, t in zip(weight, att, table["group"], table["time"]): + ax.annotate( + f"({g}, {t})", (w, a), fontsize=8, xytext=(4, 4), textcoords="offset points" + ) + ax.set_xlabel(xlabel or "Implicit weight") + ax.set_ylabel(ylabel or "ATT(g, t)") + default_title = "Implicit weights on group-time effects" + n_negative = int((weight < 0).sum()) + if n_negative: + default_title += f" ({n_negative} negative)" + ax.set_title(title or default_title) + ax.legend(frameon=False) + else: + balance = results.covariate_balance(level="summary", standardize=standardize) + suffix = "_std_diff" if standardize else "_diff" + unweighted = balance["unweighted" + suffix].to_numpy(dtype=float) + weighted = balance["weighted" + suffix].to_numpy(dtype=float) + if absolute_value: + unweighted = np.abs(unweighted) + weighted = np.abs(weighted) + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.scatter( + unweighted, + weighted, + s=markersize, + alpha=alpha, + color=post_color, + zorder=3, + ) + limit = float(np.nanmax(np.abs(np.concatenate([unweighted, weighted]))) or 1.0) + ax.plot( + [0, limit], + [0, limit], + color="0.6", + linestyle="--", + linewidth=1.0, + zorder=2, + label="no improvement", + ) + if annotate: + for x, y, name in zip(unweighted, weighted, balance["covariate"]): + ax.annotate( + str(name), (x, y), fontsize=8, xytext=(4, 4), textcoords="offset points" + ) + kindword = "standardized " if standardize else "" + ax.set_xlabel(xlabel or f"Unweighted {kindword}difference") + ax.set_ylabel(ylabel or f"Implicitly-weighted {kindword}difference") + ax.set_title(title or "Covariate balance under the implicit weights") + ax.legend(frameon=False) + + if show: + plt.show() + return ax diff --git a/docs/api/index.rst b/docs/api/index.rst index ad05dee06..5c02bb7ec 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -77,6 +77,8 @@ Result containers returned by estimators: diff_diff.TwoStageBootstrapResults diff_diff.SpilloverDiDResults diff_diff.BaconDecompositionResults + diff_diff.ATTGTWeightsResult + diff_diff.TWFEDecompositionResult diff_diff.wooldridge_results.WooldridgeDiDResults diff_diff.lpdid_results.LPDiDResults diff_diff.changes_in_changes_results.ChangesInChangesResults @@ -122,6 +124,7 @@ Plotting functions and plot builders: diff_diff.plot_honest_event_study diff_diff.RDPlot diff_diff.plot_bacon + diff_diff.plot_twfe_weights diff_diff.plot_power_curve diff_diff.plot_pretrends_power @@ -141,6 +144,8 @@ Placebo tests and model diagnostics: diff_diff.leave_one_out_test diff_diff.run_all_placebo_tests diff_diff.PlaceboTestResults + diff_diff.attgt_weights + diff_diff.decompose_twfe_weights diff_diff.RDDensityTest Panel Profiling @@ -404,6 +409,7 @@ Diagnostics & Inference honest_did power pretrends + twfe_weights Reporting ~~~~~~~~~ diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst new file mode 100644 index 000000000..cfd3cae6e --- /dev/null +++ b/docs/api/twfe_weights.rst @@ -0,0 +1,145 @@ +TWFE Weight Diagnostics (Callaway ``twfeweights``) +=================================================== + +What a two-way fixed effects regression *implicitly* weights. + +Run on staggered-adoption data, a TWFE regression does not estimate a simple +average of the underlying group-time effects ATT(g, t). It estimates a +weighted average, and some of those weights can be **negative** -- so the +coefficient need not lie in the convex hull of the effects it summarizes. +This module reports those weights, next to the weights the target estimands +ATT\ :sup:`O` and ATT\ :sup:`simple` would use, and decomposes the regression +back into its building blocks. + +**When to use these diagnostics:** + +- You have a staggered design and want to see, cell by cell, what your TWFE + specification is actually averaging +- You want to quantify how much of a TWFE estimate comes from *pre-treatment* + cells -- i.e. from parallel-trends violations rather than from treatment +- Your TWFE and :class:`~diff_diff.CallawaySantAnna` estimates disagree and + you want to see which cells drive the gap +- You adjusted for covariates and want to check whether the regression's + implicit weights actually *balance* them + +**How this differs from the neighbouring surfaces:** + +- :func:`diff_diff.twowayfeweights` implements de Chaisemartin & + D'Haultfoeuille (2020) Theorem 1 and weights **(unit, time) cells**. The + functions here weight **ATT(g, t) parameters**. +- :class:`diff_diff.BaconDecomposition` decomposes TWFE into **2x2 DiD + comparisons**. :func:`diff_diff.decompose_twfe_weights` decomposes it into + **group-time effects**, plus a pre-trend-violation term. + +**Reference:** Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & +Sant'Anna, P. H. C. (2025). Difference-in-Differences Designs: A +Practitioner's Guide. arXiv:2503.13323. Callaway, B., & Sant'Anna, P. H. C. +(2021) for the ATT\ :sup:`O` / ATT\ :sup:`simple` weights. + +Ported from the ``twfeweights`` R package (v0.9.0) by Brantly Callaway, MIT +License, Copyright (c) 2023 Brantly Callaway. + +.. module:: diff_diff.twfe_weights + +attgt_weights +------------- + +Weights an estimand places on each group-time effect. + +.. autofunction:: diff_diff.attgt_weights + +decompose_twfe_weights +---------------------- + +Decomposition of a TWFE estimate into weighted group-time effects. + +.. autofunction:: diff_diff.decompose_twfe_weights + +plot_twfe_weights +----------------- + +.. autofunction:: diff_diff.plot_twfe_weights + +Result Objects +-------------- + +.. autoclass:: diff_diff.ATTGTWeightsResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +.. autoclass:: diff_diff.TWFEDecompositionResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +Example Usage +------------- + +Inspecting what a TWFE regression weights +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", + base_period="universal", # required for aggregation="twfe" + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first.treat", + ) + + weights = diff_diff.attgt_weights(cs, aggregation="twfe") + print(weights.summary()) + print(weights.n_negative, "cells carry negative weight") + +Comparing against the estimand you meant to report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``aggregation="overall"`` and ``"simple"`` give the Callaway & Sant'Anna +target-parameter weights, which are non-negative and sum to one. The gap +between ``implied_att`` values is the cost of the TWFE specification: + +.. code-block:: python + + for aggregation in ("twfe", "overall", "simple"): + w = diff_diff.attgt_weights(cs, aggregation=aggregation) + print(f"{aggregation:8s} {w.implied_att: .4f} " + f"({w.n_negative} negative weights)") + +Separating treatment effects from pre-trend violations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + decomposition = diff_diff.decompose_twfe_weights( + panel, + outcome="lemp", unit="countyreal", time="year", + first_treat="first.treat", + covariates=["lpop"], + balance_covariates=["lpop"], + ) + + print(decomposition.summary()) + print("from pre-treatment cells:", decomposition.pretrend_bias) + + # Do the implicit weights balance the covariates? + print(decomposition.covariate_balance()) + + diff_diff.plot_twfe_weights(decomposition) + +Validation +---------- + +Validated against R ``twfeweights`` 0.9.0 output on three fixtures (``mpdta`` +plus two simulated panels). Goldens live at +``benchmarks/data/twfeweights_golden.json`` and are regenerated with +``Rscript benchmarks/R/generate_twfeweights_golden.R``; R is never needed to +run the test suite. Tolerances and their rationale are in +``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index eac062e9c..61120b816 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -21,6 +21,9 @@ # Members resolve to the first entry (the primary module) for doc lookup. # ────────────���───────────────────────────────────────────────────────── groups: + twfe_weights: + - diff_diff/twfe_weights.py + - diff_diff/twfe_weights_results.py staggered: - diff_diff/staggered.py - diff_diff/staggered_aggregation.py @@ -1203,6 +1206,33 @@ sources: # ── BaconDecomposition ───���───────────────────────────────────────── + diff_diff/twfe_weights.py: + drift_risk: low + docs: + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology + - path: docs/api/twfe_weights.rst + type: api_reference + - path: README.md + section: "Diagnostics & Sensitivity (one-line catalog entry)" + type: user_guide + - path: docs/references.rst + type: user_guide + - path: diff_diff/guides/llms.txt + section: "Diagnostics and Sensitivity Analysis" + type: user_guide + - path: diff_diff/guides/llms-full.txt + section: "TWFE Weight Diagnostics" + type: user_guide + diff_diff/twfe_weights_results.py: + drift_risk: low + docs: + - path: docs/api/twfe_weights.rst + type: api_reference + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology diff_diff/bacon.py: drift_risk: low docs: diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 7d4886fff..495888e74 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -39,6 +39,7 @@ This document provides the academic foundations and key implementation requireme 5. [Diagnostics and Sensitivity](#diagnostics-and-sensitivity) - [PlaceboTests](#placebotests) - [BaconDecomposition](#bacondecomposition) + - [TWFE Weight Diagnostics](#twfe-weight-diagnostics) - [HonestDiD](#honestdid) - [PreTrendsPower](#pretrendspower) - [PowerAnalysis](#poweranalysis) @@ -6211,6 +6212,123 @@ Where `n_k` is the sample share of timing group `k`, `n_{kℓ} = n_k / (n_k + n_ --- +## TWFE Weight Diagnostics + +**Primary source:** [Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025). "Difference-in-Differences Designs: A Practitioner's Guide." arXiv:2503.13323](https://arxiv.org/abs/2503.13323) + +**Secondary source:** [Callaway, B., & Sant'Anna, P. H. C. (2021). Difference-in-Differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.](https://doi.org/10.1016/j.jeconom.2020.12.001) — for the ATT^O / ATT^simple target-parameter weights. + +**Reference implementation:** the `twfeweights` R package (v0.9.0) by Brantly Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. The upstream notice is reproduced in the module docstring of `diff_diff/twfe_weights.py`, as its terms require. + +**Scope:** these are DIAGNOSTICS, not estimators. Both result containers subclass `Diagnostic` and carry no inference quintet — the decomposition is an algebraic identity, so there is nothing to attach a standard error to. The headline scalars are named `implied_att` and `estimate` rather than `att` for the same reason. + +### Relationship to neighbouring surfaces + +- **vs `twowayfeweights` (de Chaisemartin & D'Haultfoeuille 2020, Theorem 1):** that surface weights **(unit, time) cells**; these functions weight **ATT(g,t) parameters** — the cohort-by-period building blocks. Both detect negative weighting in staggered TWFE, but they decompose along different axes and their weight tables are not comparable row-for-row. The names are deliberately disjoint (`attgt_weights` / `ATTGTWeightsResult` vs `twowayfeweights` / `TWFEWeightsResult`). +- **vs `BaconDecomposition` (Goodman-Bacon 2021):** Bacon decomposes TWFE into **2x2 DiD comparisons** and asks which comparisons drive the estimate. `decompose_twfe_weights` decomposes it into **group-time effects** and additionally isolates a pre-trend-violation term. Use Bacon to see the forbidden comparisons; use this to see the per-`(g,t)` weights and how much of the estimate is not a treatment effect at all. + +### Estimator equations (as implemented) + +All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T`. + +*ATT(g,t) weights — `attgt_weights(aggregation=...)`:* + +`aggregation="twfe"` (R `twfe_weights`), with `p_g` the share of ALL units in cohort `g` and `E_t[D]` the share of units treated by `t`: + +``` +h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D] +num(g,t) = h(g,t) * p_g +w(g,t) = num(g,t) / sum over {t >= g, g != 0} of num(g,t) +``` + +`aggregation="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: + +``` +w(g,t) = 1[t >= g] * pbar_g / (maxT - g + 1) +``` + +`aggregation="simple"` (ATT^simple, R `att_simple_weights`): + +``` +w(g,t) = 1[t >= g] * pbar_g, then normalized to sum to one +``` + +*FWL decomposition — `decompose_twfe_weights(method="fwl")` (R `implicit_twfe_weights`):* + +Double-demean the treatment indicator `D` and the covariates `X` over unit and period, project the demeaned treatment on the demeaned covariates, and take the residual: + +``` +gamma = argmin_b || Ddot - Xdot b ||_w +resid = Ddot - Xdot gamma +alpha_den = E_w[resid * Ddot] +``` + +The residual IS the implicit weight the regression applies to each observation. Per `(g,t)` cell, with the treated and comparison weights each normalized to mean one: + +``` +alpha_weight(g,t) = E_w[resid | G=g, T=t] * p_g / (alpha_den * T) +ATT(g,t) = E_w[wtreated * Ytilde | G=g] - E_w[wcontrol * Ytilde | G=0] +``` + +where `Ytilde` is the outcome measured against the base period (`Y_t - Y_1` under `base_period="first_period"`, `Y_t - Y_{g-1}` under `"gmin1"`). Roll-ups: + +``` +decomposition = sum over all cells of alpha_weight * ATT +remainder = sum of alpha_weight * cell remainder (0 unless base_period="gmin1") +estimate = decomposition + remainder +pretrend_bias = sum over PRE cells (t < g) of alpha_weight * ATT +``` + +Under parallel trends every pre-treatment ATT(g,t) is zero and `pretrend_bias` vanishes; a non-zero value is the contribution of parallel-trends violations to the TWFE coefficient. + +*Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, + +``` +attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate +``` + +Verified on `mpdta` at `-0.03654894` from both directions. + +### Edge cases + +- Non-estimable `(g,t)` cells (NaN ATT) are dropped from `attgt_weights` with a `UserWarning` and counted in `n_dropped_cells`; the remaining weights renormalize. +- `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. +- `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. +- `attgt_weights` rejects repeated-cross-section fits and unbalanced-panel fallbacks: `E_t[D]` and the cohort shares average over a fixed unit set. + +### Notes and deviations + +- **Note (upstream `fixest::demean` segfault on the no-covariate branch):** `twfeweights::implicit_twfe_weights(xformula = ~1)` builds `model.matrix(~-1, data)`, an `nT x 0` matrix, and `fixest::demean()` SEGFAULTS on a zero-column matrix (reproduced in isolation on R 4.6.1 / fixest 0.14.2: `fixest::demean(matrix(numeric(0), 10, 0), ids)` → `*** caught segfault *** memory not mapped`). This is a zero-column bug, not a property of any fixture. The no-covariate golden is therefore generated with a TIME-INVARIANT covariate, which double-demeaning annihilates exactly, making the call numerically the `~1` branch; the parity test asserts BOTH `covariates=None` and `covariates=[]` against that single golden, so the equivalence is proven rather than assumed. Verified on `mpdta`: `twfe_weights(att_gt(...))` aggregate and `implicit_twfe_weights(xformula = ~lpop)$est` both equal `-0.03654894`. +- **Note (annihilated covariates are dropped before the projection):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Regressing on it amplifies that noise by ~1e16 and corrupts the per-cell weights. diff-diff drops such columns, judged against each column's own PRE-demeaning norm — a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. A `UserWarning` names the dropped covariates. This is what makes `covariates=None` and `covariates=[]` agree to 1e-15. +- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **The aggregate is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), which is why `estimate` matches R to 1e-15 while the individual `ATT(g,t)` do not. +- **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. +- **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. +- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The two-period kernels, per-cell helpers and balance statistics are private; they are pinned directly by the parity suite since they have no public surface. +- **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. +- **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. +- **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. +- **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. +- **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. +- **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. + +### R output parity + +Goldens: `benchmarks/data/twfeweights_golden.json` (+ three sibling panel CSVs), regenerated by `benchmarks/R/generate_twfeweights_golden.R`. R is needed only to regenerate them, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. + +Three fixtures: `mpdta` (real; non-`1..T` period labels), `sim_staggered` (equal cohorts, a real pre-trend so `pretrend_bias != 0`, and the degenerate `t=3` cells above), and `unbalanced_cohorts` (120/70/60 — breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). + +| Surface | Gate | Rationale | +|---------|------|-----------| +| ATT(g,t) weights, all three aggregations | `atol=1e-12` | Closed-form rational expression in cohort masses; only double-precision representation error separates the two sides. Observed max deviation 4.7e-16. | +| `implied_att` (R's own ATT(g,t) fed back in) | `atol=1e-12` | Isolates the weight arithmetic from CallawaySantAnna-vs-`did` parity. | +| End-to-end from a CS fit | `rtol=1e-6` | COMPOSED check — carries the pre-existing CS parity band, not this module's. | +| FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | +| FWL with covariates | `atol=1e-8` | The demeaning slack propagates through the OLS projection of `Ddot` on `Xdot`. | +| Covariate balance (11 statistics) | `atol=1e-9` | Smooth functions of the weights above. Observed max deviation 7.3e-11. | +| Per-cell ATT and the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. | + +--- + ## HonestDiD **Primary source:** [Rambachan, A., & Roth, J. (2023). A More Credible Approach to Parallel Trends. *Review of Economic Studies*, 90(5), 2555-2591.](https://doi.org/10.1093/restud/rdad018) diff --git a/docs/references.rst b/docs/references.rst index 29c162e93..e9f76999d 100644 --- a/docs/references.rst +++ b/docs/references.rst @@ -292,6 +292,11 @@ Multi-Period and Staggered Adoption - **Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025).** "Difference-in-Differences Designs: A Practitioner's Guide." *arXiv preprint* arXiv:2503.13323. https://arxiv.org/abs/2503.13323 + Primary source for the implicit-TWFE-weight diagnostics + (:func:`diff_diff.attgt_weights`, :func:`diff_diff.decompose_twfe_weights`). + Reference implementation: the ``twfeweights`` R package (v0.9.0) by Brantly + Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. + Source for the 8-step practitioner workflow surfaced via ``diff_diff.get_llm_guide("practitioner")`` and the README ``## Practitioner Workflow`` section. See ``docs/methodology/REGISTRY.md`` for the diff-diff renumbering and per-step deviations. Double/Debiased Machine Learning From 930f44988744aca1fa439c53b8f6b729f43af9e4 Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 7 Sep 2026 14:07:00 +0800 Subject: [PATCH 06/13] fix(twfeweights): address review - input validation, house helpers, parity gates Addresses all 21 items in igerber's review of #812. The port's R-parity was accepted; almost everything here is about the paths that go BEYOND the R reference (the ATT(g,t)-frame input, user weights, non-standard cohort labels), where R never faced the input so the port had no validation. Correctness (items 1-5), all reproduced by the reviewer: - Cohort labels: never-treated is exactly 0 or +inf. NaN / -inf raise instead of being silently absorbed into cohort 0 (a single NaN label moved `estimate` by ~1.4% with no warning). Within-unit invariance now uses nunique(dropna=False) at all three invariance sites; non-finite period labels are rejected up front. - Balance roll-up propagates NaN as R does: `_frac_treated_extreme` returns NA for a covariate with <3 distinct values, and the summary no longer turns that into 0.0 via pandas' NaN-skipping sum. Masks on the `post` column, not on a zero roll-up weight (a zero-weight post cell still contributes). - `aggregation="twfe"` now enforces R's third restriction (xformla == ~1): fits record their covariate names on the aggregation kit, at both build sites (staggered.py and dml_did.py), and a covariate-adjusted fit raises. A kit predating the bookkeeping warns; a non-CS result is a TypeError. - ATT(g,t)-frame input: duplicate cells, non-finite group/time labels and non-finite effects are rejected, and an incomplete grid fails closed for all three aggregations. Two structural gaps mirror R instead of raising: a cohort with no estimable post cell is dropped (did's first-period drop), and under control_group="not_yet_treated" the CS estimands average over each cohort's available post periods (aggte). Both warn. - Unit weights must be finite, non-negative, with positive total and treated mass; never-treated mass is required only where the comparison group enters the formula, so overall/simple still work without one. Parity and house conventions (items 6, 7, 9): - Parity now asserts pretrend_bias, post_only, effective_sample_size and cell ess/remainder. At the documented 0/0 cells the expectation is rebuilt from R's OWN cells with our limit substituted only where R's number is noise, so the assertion stays anchored to R. The scalar-split relaxation is restricted to fwl_gmin1, and fwl_gmin1 cells are asserted. - The FWL linear algebra goes through the house helpers: within_transform for the two-way demeaning and solve_ols for the weighted solve. The bespoke pivoted QR is gone - it was the same norm-pivoted QR solve_ols uses, so its "drops later columns first" docstring was inaccurate and nothing R-specific was lost. A frozen-numbers pin captured on the pre-refactor code guards the weighted branches, which no parity fixture covers. - negative_weight_share counted pre cells, so it read ~0.5 in every staggered design. n_negative_post / negative_post_weight_share report the actual pathology; summary() leads with them. Tests (items 8, 13, 18, 19): decomposition edge cases, both plot backends, collinear covariates, and regression tests for items 1-5. plot_twfe_weights gains the backend="plotly" split its file-mates have. The constructed fixtures are derived from their cells instead of carrying stale literals, and the two weak tests now assert hand-computed values and all five exported names. Goldens (items 10-12): regeneration is numerically inert - every one of the 192 changed numbers is a group/time label, none elsewhere. Cells now carry original period labels throughout (implicit_* run in positional time), and the tests assert labels rather than array position. The duplicated mpdta panel CSV is dropped in favour of the shared mpdta_stata_panel.csv plus a derived-column expression, with the generator asserting the two agree. The AIPW blocks are labelled reserved for the follow-up rather than left looking unused. Docs (items 14-17, 20, 21): README back to the sibling shape, changelog trimmed, REGISTRY rewritten (the new hard errors, both R-mirroring carve-outs, the 0/0 magnitudes enumerated, the annihilation note recast as numerical hygiene with its threshold limitation stated), llms-full contract prose and a plot subsection, doc-deps banner, requirements.R (twfeweights is not on CRAN), and twfe_weights.rst registered in the snippet harness after fixing its first_treat column name and making each block self-contained. The weighted ECDF is O(n log n) instead of quadratic, holding parity at 1e-9. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- benchmarks/R/generate_twfeweights_golden.R | 159 +- benchmarks/R/requirements.R | 6 +- benchmarks/data/twfeweights_golden.json | 40 +- benchmarks/data/twfeweights_mpdta_panel.csv | 2501 ----------------- .../20260831-twfe-weight-diagnostics.md | 41 +- diff_diff/dml_did.py | 1 + diff_diff/guides/llms-full.txt | 42 +- diff_diff/staggered.py | 9 +- diff_diff/twfe_weights.py | 626 +++-- diff_diff/twfe_weights_results.py | 93 +- diff_diff/visualization/_diagnostic.py | 247 +- docs/api/twfe_weights.rst | 18 +- docs/api/visualization.rst | 6 + docs/doc-deps.yaml | 5 +- docs/methodology/REGISTRY.md | 25 +- tests/helpers/results_foundation.py | 39 +- tests/test_doc_snippets.py | 1 + tests/test_naming_guard.py | 42 + tests/test_twfe_weights.py | 727 ++++- tests/test_twfe_weights_parity.py | 84 +- tests/test_visualization_plotly.py | 60 + 22 files changed, 1929 insertions(+), 2845 deletions(-) delete mode 100644 benchmarks/data/twfeweights_mpdta_panel.csv diff --git a/README.md b/README.md index e1879cc80..17ba66226 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html) - Cattaneo, Jansson & Ma (2020) density-discontinuity test (`RDDensityTest`): rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out -- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna (2025) implicit weights on ATT(g,t): `attgt_weights(cs_result, aggregation='twfe'|'overall'|'simple')` shows what a TWFE regression (vs ATT^O / ATT^simple) implicitly puts on each group-time effect, including negative weights; `decompose_twfe_weights(panel, method='fwl')` re-derives the estimate from its building blocks with the pre-trend-violation contribution and implicit covariate balance. Ported from Brantly Callaway's `twfeweights` R package (MIT) +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-trend contribution. Ports Callaway's `twfeweights` (MIT) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R index 8e4541f24..5e3e619a5 100644 --- a/benchmarks/R/generate_twfeweights_golden.R +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -4,10 +4,13 @@ # Requires: twfeweights (>= 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, # DRDID, jsonlite # Output: benchmarks/data/twfeweights_golden.json -# benchmarks/data/twfeweights_mpdta_panel.csv # benchmarks/data/twfeweights_sim_panel.csv # benchmarks/data/twfeweights_unbalanced_panel.csv # +# The mpdta fixture reads the EXISTING benchmarks/data/mpdta_stata_panel.csv +# rather than writing a renamed copy of it; this script asserts the two agree +# bit-for-bit on every shared column before using it. +# # Run from the repository root: # Rscript benchmarks/R/generate_twfeweights_golden.R # @@ -23,9 +26,15 @@ # implicit_aipw_weights # # plus covariate balance as a result-object method (<- twfe_cov_bal / -# aipw_cov_bal / mp_covariate_bal_summary_helper) and two private two-period -# kernels (<- two_period_reg_weights / two_period_aipw_weights) that are -# pinned here because they have no public Python surface of their own. +# aipw_cov_bal / mp_covariate_bal_summary_helper). +# +# RESERVED BLOCKS (pinned, but not yet read by any test): `decompose.aipw`, +# `balance.aipw` and `two_period.*` pin implicit_aipw_weights, aipw_cov_bal +# and the two_period_reg_weights / two_period_aipw_weights kernels. None of +# these has a Python surface yet - `method="aipw"` is a documented follow-up - +# so they are captured here so that follow-up needs no R re-run. Note the AIPW +# golden is COVARIATE-ADJUSTED: a time-invariant covariate is annihilated by +# double-demeaning but is NOT a no-op in a propensity score. # # --------------------------------------------------------------------------- # NOTE (upstream bug — do NOT "simplify" the no-covariate calls below) @@ -101,7 +110,17 @@ extract_mp_weights <- function(obj) { ) } -extract_fwl <- function(obj) { +# implicit_* run in POSITIONAL time (BMisc::orig2t), so their $g / $tp are +# 1..T. attgt_weights' goldens carry RAW labels. Map back here so every block +# in the JSON shares one convention and the Python tests can assert labels. +to_orig <- function(pos, periods) { + out <- as.numeric(pos) + keep <- !is.na(out) & out >= 1 & out <= length(periods) + out[keep] <- as.numeric(periods[out[keep]]) + out +} + +extract_fwl <- function(obj, periods) { cells <- obj$twfe_gt g <- unlist(BMisc::getListElement(cells, "g")) tp <- unlist(BMisc::getListElement(cells, "tp")) @@ -110,6 +129,8 @@ extract_fwl <- function(obj) { ess <- unlist(BMisc::getListElement(cells, "ess")) rem <- unlist(BMisc::getListElement(cells, "remainder")) post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) list( cells = list( group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), @@ -126,7 +147,7 @@ extract_fwl <- function(obj) { ) } -extract_aipw <- function(obj) { +extract_aipw <- function(obj, periods) { cells <- obj$aipw_gt g <- unlist(BMisc::getListElement(cells, "g")) tp <- unlist(BMisc::getListElement(cells, "tp")) @@ -134,6 +155,8 @@ extract_aipw <- function(obj) { wt <- unlist(BMisc::getListElement(cells, "att_weight")) ess <- unlist(BMisc::getListElement(cells, "ess")) post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) list( cells = list( group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), @@ -151,14 +174,17 @@ extract_aipw <- function(obj) { } # Per-cell balance tables, one row per (g, t) x covariate. -extract_balance_cells <- function(cells) { +extract_balance_cells <- function(cells, periods) { g <- unlist(BMisc::getListElement(cells, "g")) tp <- unlist(BMisc::getListElement(cells, "tp")) dfs <- BMisc::getListElement(cells, "cov_bal_df") + post_i <- 1 * (tp >= g) + g_o <- to_orig(g, periods) + tp_o <- to_orig(tp, periods) rows <- do.call(rbind.data.frame, lapply(seq_along(dfs), function(i) { d <- dfs[[i]] cbind.data.frame( - group = g[i], time = tp[i], post = 1 * (tp[i] >= g[i]), + group = g_o[i], time = tp_o[i], post = post_i[i], covariate = rownames(d), d, row.names = NULL ) })) @@ -192,9 +218,12 @@ extract_two_period <- function(obj) { # --------------------------------------------------------------------------- build_fixture <- function(df, data_file, outcome, unit, time, first_treat, - invariant_cov, varying_cov, two_period_g) { + invariant_cov, varying_cov, two_period_g, + data_file_out = NULL, columns_out = NULL, + derived_columns = NULL) { stopifnot(all(tapply(df[[invariant_cov]], df[[unit]], function(z) length(unique(z))) == 1)) + periods <- sort(unique(df[[time]])) # Slice the two-period sub-panel FIRST. Several upstream entry points # (did::att_gt, and BMisc helpers reached from implicit_*) call @@ -249,11 +278,15 @@ build_fixture <- function(df, data_file, outcome, unit, time, first_treat, tp_aipw <- quiet(do.call(two_period_aipw_weights, c(sub_common, list(xformula = var_f)))) - list( - data_file = data_file, - columns = list(outcome = outcome, unit = unit, time = time, - first_treat = first_treat, - invariant_cov = invariant_cov, varying_cov = varying_cov), + out <- list( + data_file = if (is.null(data_file_out)) data_file else data_file_out, + columns = if (is.null(columns_out)) { + list(outcome = outcome, unit = unit, time = time, + first_treat = first_treat, + invariant_cov = invariant_cov, varying_cov = varying_cov) + } else { + columns_out + }, two_period_group = two_period_g, attgt_weights = list( twfe = extract_mp_weights(quiet(twfe_weights(ag))), @@ -261,20 +294,24 @@ build_fixture <- function(df, data_file, outcome, unit, time, first_treat, simple = extract_mp_weights(quiet(att_simple_weights(ag))) ), decompose = list( - fwl_nocov = extract_fwl(fwl_nocov), - fwl_cov = extract_fwl(fwl_cov), - fwl_gmin1 = extract_fwl(fwl_gmin1), - aipw = extract_aipw(aipw) + fwl_nocov = extract_fwl(fwl_nocov, periods), + fwl_cov = extract_fwl(fwl_cov, periods), + fwl_gmin1 = extract_fwl(fwl_gmin1, periods), + aipw = extract_aipw(aipw, periods) ), balance = list( - fwl = list(cells = extract_balance_cells(bal_fwl$twfe_gt), + fwl = list(cells = extract_balance_cells(bal_fwl$twfe_gt, periods), summary = extract_balance_summary(bal_fwl$twfe_gt)), - aipw = list(cells = extract_balance_cells(bal_aipw$aipw_gt), + aipw = list(cells = extract_balance_cells(bal_aipw$aipw_gt, periods), summary = extract_balance_summary(bal_aipw$aipw_gt)) ), two_period = list(reg = extract_two_period(tp_reg), aipw = extract_two_period(tp_aipw)) ) + if (!is.null(derived_columns)) { + out$derived_columns <- derived_columns + } + out } # --------------------------------------------------------------------------- @@ -298,13 +335,49 @@ mpdta_df <- mpdta_df[order(mpdta_df$unit, mpdta_df$period), ] # the covariate-adjusted branch is non-degenerate on this fixture too. mpdta_df$lpop_t <- mpdta_df$lpop * (mpdta_df$period - 2002) / 5 +# The fixture READS benchmarks/data/mpdta_stata_panel.csv (already in the repo +# for the Stata parity suites) instead of writing a renamed copy. Assert the +# two sources agree bit-for-bit on every shared column, so they cannot drift. +stata_path <- file.path(out_dir, "mpdta_stata_panel.csv") +if (!file.exists(stata_path)) { + stop("expected ", stata_path, " (the mpdta fixture now reads it)") +} +stata_df <- read.csv(stata_path) +stata_df <- stata_df[order(stata_df$countyreal, stata_df$year), ] +# The identifiers must match exactly; the float columns are compared at CSV +# round-trip precision, NOT bit-for-bit. write.csv emits 15 significant digits, +# so a CSV column always sits within ~1e-15 relative of the in-memory double it +# came from. That gap is pre-existing and unchanged by this switch: the fixture +# previously read twfeweights_mpdta_panel.csv, itself a 15-digit round-trip of +# these same values, and the parity tolerances already absorb it. +rt_tol <- 1e-14 +stopifnot( + nrow(stata_df) == nrow(mpdta_df), + identical(as.numeric(stata_df$countyreal), mpdta_df$unit), + identical(as.numeric(stata_df$year), mpdta_df$period), + identical(as.numeric(stata_df$first.treat), mpdta_df$first_treat), + max(abs(as.numeric(stata_df$lemp) - mpdta_df$outcome)) <= + rt_tol * max(1, max(abs(mpdta_df$outcome))), + max(abs(as.numeric(stata_df$lpop) - mpdta_df$lpop)) <= + rt_tol * max(1, max(abs(mpdta_df$lpop))) +) + # --------------------------------------------------------------------------- # Fixture 2 — sim_staggered (simulated) # -# Well-conditioned by construction: 3 equal cohorts of 100 so no (g,t) cell is -# degenerate and 100 controls per cell keep the AIPW propensity score bounded -# away from 0/1; `0.3 * x1 * period` induces a REAL pre-trend so -# pretrend_bias != 0 and the diagnostic is not testing a trivial zero. +# 3 equal cohorts of 100, which keeps the AIPW propensity score bounded away +# from 0/1 (100 controls per cell). The equal cohorts are also exactly what +# makes the comparison-group normalizer VANISH at t = 3 (-1/3 + 1/3), so this +# fixture deliberately exercises the documented 0/0 cells - it is not a +# "no cell is degenerate" design. +# +# `0.3 * x1 * period` gives each unit a trend, so pretrend_bias is non-zero, +# but x1 is iid and cohorts are assigned by unit INDEX, so E[x1 | g] does not +# vary by cohort: the differential pre-trend is zero in expectation and the +# observed value (~0.093) is sampling noise, not a designed pre-trend. +# `xtv`'s two structured terms (0.2 * period and 0.5 * x1) are absorbed by the +# two-way fixed effects, so the covariate branch regresses on the residual +# noise - adequate for parity, but not a "well-conditioned" design. # --------------------------------------------------------------------------- make_sim <- function(seed, cohort_sizes, cohort_times, n_periods) { @@ -344,17 +417,24 @@ unb_df <- make_sim(20260901, c(120, 70, 60), c(0, 3, 5), 6) # Build + write # --------------------------------------------------------------------------- -write.csv(mpdta_df, file.path(out_dir, "twfeweights_mpdta_panel.csv"), - row.names = FALSE) write.csv(sim_df, file.path(out_dir, "twfeweights_sim_panel.csv"), row.names = FALSE) write.csv(unb_df, file.path(out_dir, "twfeweights_unbalanced_panel.csv"), row.names = FALSE) cat("building mpdta ...\n") -fx_mpdta <- build_fixture(mpdta_df, "twfeweights_mpdta_panel.csv", - "outcome", "unit", "period", "first_treat", - "lpop", "lpop_t", two_period_g = 2004) +fx_mpdta <- build_fixture( + mpdta_df, "twfeweights_mpdta_panel.csv", + "outcome", "unit", "period", "first_treat", + "lpop", "lpop_t", two_period_g = 2004, + # Emitted names point at the SHARED stata panel; the R calls above keep + # using mpdta_df's own names, so nothing inside build_fixture changes. + data_file_out = "mpdta_stata_panel.csv", + columns_out = list(outcome = "lemp", unit = "countyreal", time = "year", + first_treat = "first.treat", + invariant_cov = "lpop", varying_cov = "lpop_t"), + derived_columns = list(lpop_t = "lpop * (year - 2002) / 5") +) cat("building sim_staggered ...\n") fx_sim <- build_fixture(sim_df, "twfeweights_sim_panel.csv", "outcome", "unit", "period", "first_treat", @@ -382,6 +462,27 @@ payload <- list( BMisc_version = as.character(packageVersion("BMisc")), DRDID_version = as.character(packageVersion("DRDID")), seeds = list(sim_staggered = 20260831L, unbalanced_cohorts = 20260901L), + mpdta_provenance = paste( + "fixtures.mpdta is data(mpdta, package = \"did\") version", + as.character(packageVersion("did")), + "- read from the shared benchmarks/data/mpdta_stata_panel.csv, whose", + "columns this generator asserts are bit-identical to data(mpdta).", + "`lpop_t` is derived (see fixtures.mpdta.derived_columns)." + ), + reserved_blocks = paste( + "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED:", + "they capture implicit_aipw_weights, aipw_cov_bal and the", + "two_period_reg_weights / two_period_aipw_weights kernels, none of which", + "has a Python surface yet (method=\"aipw\" is a documented follow-up).", + "They are kept so that follow-up needs no R re-run. NOTE the AIPW golden", + "is covariate-adjusted: a time-invariant covariate is annihilated by", + "double-demeaning but is NOT a no-op in a propensity score." + ), + label_convention = paste( + "Every cells block (attgt_weights.*, decompose.*, balance.*) carries", + "ORIGINAL period labels. implicit_* run in positional time internally;", + "the generator maps them back before emitting." + ), no_covariate_note = paste( "decompose.fwl_nocov is generated with xformula = ~,", "which is numerically the ~1 branch (double-demeaning annihilates a", diff --git a/benchmarks/R/requirements.R b/benchmarks/R/requirements.R index c85a33f4e..ba65b8cd7 100644 --- a/benchmarks/R/requirements.R +++ b/benchmarks/R/requirements.R @@ -20,6 +20,8 @@ required_packages <- c( "Synth", # Abadie-Diamond-Hainmueller (2010) synthetic control (SyntheticControl R-parity; ships data(basque)) "qte", # Callaway qte package (Athey-Imbens CiC + QDiD R-parity; ships data(lalonde)) "ptetools", # Callaway ptetools (badcontrols dependency; bad-control DMLDiD lane black-box parity) + "BMisc", # Callaway utility package (twfeweights dependency: weighted_ecdf, orig2t) + "DRDID", # Sant'Anna & Zhao (2020) doubly-robust DiD (twfeweights AIPW dependency) # Utilities "jsonlite", # JSON output for Python interop @@ -28,7 +30,9 @@ required_packages <- c( # synthdid must be installed from GitHub github_packages <- list( - synthdid = "synth-inference/synthdid" + synthdid = "synth-inference/synthdid", + # TWFE weight diagnostics parity goldens (not on CRAN) + twfeweights = "bcallaway11/twfeweights" ) install_if_missing <- function(pkg) { diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json index 771a0d15d..30cfbe030 100644 --- a/benchmarks/data/twfeweights_golden.json +++ b/benchmarks/data/twfeweights_golden.json @@ -12,16 +12,19 @@ "sim_staggered": 20260831, "unbalanced_cohorts": 20260901 }, + "mpdta_provenance": "fixtures.mpdta is data(mpdta, package = \"did\") version 2.5.1 - read from the shared benchmarks/data/mpdta_stata_panel.csv, whose columns this generator asserts are bit-identical to data(mpdta). `lpop_t` is derived (see fixtures.mpdta.derived_columns).", + "reserved_blocks": "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED: they capture implicit_aipw_weights, aipw_cov_bal and the two_period_reg_weights / two_period_aipw_weights kernels, none of which has a Python surface yet (method=\"aipw\" is a documented follow-up). They are kept so that follow-up needs no R re-run. NOTE the AIPW golden is covariate-adjusted: a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score.", + "label_convention": "Every cells block (attgt_weights.*, decompose.*, balance.*) carries ORIGINAL period labels. implicit_* run in positional time internally; the generator maps them back before emitting.", "no_covariate_note": "decompose.fwl_nocov is generated with xformula = ~, which is numerically the ~1 branch (double-demeaning annihilates a time-invariant regressor exactly). The ~1 branch itself cannot be called: fixest::demean segfaults on the zero-column model matrix it builds. See the comment block at the top of the generator." }, "fixtures": { "mpdta": { - "data_file": "twfeweights_mpdta_panel.csv", + "data_file": "mpdta_stata_panel.csv", "columns": { - "outcome": "outcome", - "unit": "unit", - "time": "period", - "first_treat": "first_treat", + "outcome": "lemp", + "unit": "countyreal", + "time": "year", + "first_treat": "first.treat", "invariant_cov": "lpop", "varying_cov": "lpop_t" }, @@ -55,8 +58,8 @@ "decompose": { "fwl_nocov": { "cells": { - "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], - "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, 0, 0.00652011242423291, 0.00376929367371423, -0.000825313279148546, -0.0374551778725037, 0, 0.0305066555832921, 0.0277807626971762, -0.00330635669251204, -0.0293607674117092], "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], @@ -72,8 +75,8 @@ }, "fwl_cov": { "cells": { - "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], - "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "att": [0, -0.0133665432176392, -0.0704231581031491, -0.267917985756793, -0.0863164461874393, 0, 0.00207806356666779, 0.00376929367371422, -0.126820852395217, -0.0470847199292006, 0, 0.0267475453560011, 0.0277807626971762, -0.12743173909835, -0.0323931143733938], "weight": [-0.1135230713667, 0.0464885411048002, 0.0461635736399947, 0.0324772245138304, -0.0116062678919253, -0.0912244608682691, -0.10633971008501, -0.108093577640393, 0.197464333015477, 0.108193415578195, -0.0876606039551204, -0.133316485481803, -0.135213842067535, -0.224628248535168, 0.580819180039627], @@ -89,8 +92,8 @@ }, "fwl_gmin1": { "cells": { - "group": [2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5], - "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286273, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], @@ -106,8 +109,8 @@ }, "aipw": { "cells": { - "group": [2, 2, 2, 2, 4, 4, 5], - "time": [2, 3, 4, 5, 4, 5, 5], + "group": [2004, 2004, 2004, 2004, 2006, 2006, 2007], + "time": [2004, 2005, 2006, 2007, 2006, 2007, 2007], "post": [1, 1, 1, 1, 1, 1, 1], "att": [-0.0145296683111164, -0.0764218817440481, -0.140448336820238, -0.106903898121729, 0.000960573746698802, -0.0412938655881805, -0.0287813610394872], "weight": [0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.104712041884817, 0.104712041884817, 0.68586387434555], @@ -124,8 +127,8 @@ "balance": { "fwl": { "cells": { - "group": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], - "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007], "post": [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1], "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877], @@ -157,8 +160,8 @@ }, "aipw": { "cells": { - "group": [2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5], - "time": [2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 5, 5, 5, 5], + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2007, 2007], + "time": [2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2006, 2006, 2007, 2007, 2007, 2007], "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], @@ -200,6 +203,9 @@ "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } + }, + "derived_columns": { + "lpop_t": "lpop * (year - 2002) / 5" } }, "sim_staggered": { diff --git a/benchmarks/data/twfeweights_mpdta_panel.csv b/benchmarks/data/twfeweights_mpdta_panel.csv deleted file mode 100644 index b88831b79..000000000 --- a/benchmarks/data/twfeweights_mpdta_panel.csv +++ /dev/null @@ -1,2501 +0,0 @@ -"unit","period","first_treat","outcome","lpop","lpop_t" -8001,2003,2007,8.46146904264388,5.8967609333053,1.17935218666106 -8001,2004,2007,8.33686963728496,5.8967609333053,2.35870437332212 -8001,2005,2007,8.34021732094704,5.8967609333053,3.53805655998318 -8001,2006,2007,8.37816098272068,5.8967609333053,4.71740874664424 -8001,2007,2007,8.48735234940522,5.8967609333053,5.8967609333053 -8019,2003,2007,4.99721227376411,2.23237719795055,0.44647543959011 -8019,2004,2007,5.08140436498446,2.23237719795055,0.89295087918022 -8019,2005,2007,4.78749174278205,2.23237719795055,1.33942631877033 -8019,2006,2007,4.99043258677874,2.23237719795055,1.78590175836044 -8019,2007,2007,5.03695260241363,2.23237719795055,2.23237719795055 -8023,2003,2007,2.19722457733622,1.29828248379668,0.259656496759335 -8023,2004,2007,2.30258509299405,1.29828248379668,0.519312993518671 -8023,2005,2007,2.77258872223978,1.29828248379668,0.778969490278006 -8023,2006,2007,1.94591014905531,1.29828248379668,1.03862598703734 -8023,2007,2007,2.99573227355399,1.29828248379668,1.29828248379668 -8029,2003,2007,5.88332238848828,3.32625829499766,0.665251658999532 -8029,2004,2007,5.78996017089725,3.32625829499766,1.33050331799906 -8029,2005,2007,5.85793315448346,3.32625829499766,1.9957549769986 -8029,2006,2007,5.99893656194668,3.32625829499766,2.66100663599813 -8029,2007,2007,6.10479323241498,3.32625829499766,3.32625829499766 -8041,2003,2007,8.96865090337487,6.24790553432335,1.24958110686467 -8041,2004,2007,8.9157008189569,6.24790553432335,2.49916221372934 -8041,2005,2007,8.90095787254506,6.24790553432335,3.74874332059401 -8041,2006,2007,8.90272766403552,6.24790553432335,4.99832442745868 -8041,2007,2007,8.88100262425557,6.24790553432335,6.24790553432335 -8063,2003,2007,4.9416424226093,2.08081559723298,0.416163119446596 -8063,2004,2007,4.86753445045558,2.08081559723298,0.832326238893192 -8063,2005,2007,4.66343909411207,2.08081559723298,1.24848935833979 -8063,2006,2007,4.81218435537242,2.08081559723298,1.66465247778638 -8063,2007,2007,4.91998092582813,2.08081559723298,2.08081559723298 -8075,2003,2007,5.6970934865054,3.02061998906177,0.604123997812354 -8075,2004,2007,5.63121178182137,3.02061998906177,1.20824799562471 -8075,2005,2007,5.72031177660741,3.02061998906177,1.81237199343706 -8075,2006,2007,5.79605775076537,3.02061998906177,2.41649599124942 -8075,2007,2007,5.56068163101553,3.02061998906177,3.02061998906177 -8103,2003,2007,4.45434729625351,1.78942340943051,0.357884681886101 -8103,2004,2007,4.58496747867057,1.78942340943051,0.715769363772203 -8103,2005,2007,4.75359019110636,1.78942340943051,1.0736540456583 -8103,2006,2007,4.90527477843843,1.78942340943051,1.43153872754441 -8103,2007,2007,4.71849887129509,1.78942340943051,1.78942340943051 -8117,2003,2007,6.37842618365159,3.15904089116601,0.631808178233203 -8117,2004,2007,6.27476202124194,3.15904089116601,1.26361635646641 -8117,2005,2007,6.2841341610708,3.15904089116601,1.89542453469961 -8117,2006,2007,6.3818160174061,3.15904089116601,2.52723271293281 -8117,2007,2007,6.43133108193348,3.15904089116601,3.15904089116601 -8123,2003,2007,7.7553388128465,5.19814337757751,1.0396286755155 -8123,2004,2007,7.68799716639302,5.19814337757751,2.079257351031 -8123,2005,2007,7.75405263903576,5.19814337757751,3.11888602654651 -8123,2006,2007,7.78945456608667,5.19814337757751,4.15851470206201 -8123,2007,2007,7.80302664363222,5.19814337757751,5.19814337757751 -12007,2003,2006,5.50125821054473,3.26147543848702,0.652295087697404 -12007,2004,2006,5.59471137960184,3.26147543848702,1.30459017539481 -12007,2005,2006,5.24702407216049,3.26147543848702,1.95688526309221 -12007,2006,2006,5.43372200355424,3.26147543848702,2.60918035078961 -12007,2007,2006,5.44241771052179,3.26147543848702,3.26147543848702 -12019,2003,2006,7.77275271646874,4.94743987059911,0.989487974119822 -12019,2004,2006,7.81116338502528,4.94743987059911,1.97897594823964 -12019,2005,2006,7.80139132029149,4.94743987059911,2.96846392235947 -12019,2006,2006,7.86326672400957,4.94743987059911,3.95795189647929 -12019,2007,2006,7.87016594646984,4.94743987059911,4.94743987059911 -12023,2003,2006,6.99117688712121,4.03447070018167,0.806894140036335 -12023,2004,2006,6.53813982376767,4.03447070018167,1.61378828007267 -12023,2005,2006,6.59030104819669,4.03447070018167,2.420682420109 -12023,2006,2006,6.5510803350434,4.03447070018167,3.22757656014534 -12023,2007,2006,6.71780469502369,4.03447070018167,4.03447070018167 -12029,2003,2006,3.85014760171006,2.62662320240648,0.525324640481296 -12029,2004,2006,4.07753744390572,2.62662320240648,1.05064928096259 -12029,2005,2006,4.35670882668959,2.62662320240648,1.57597392144389 -12029,2006,2006,4.31748811353631,2.62662320240648,2.10129856192518 -12029,2007,2006,4.33073334028633,2.62662320240648,2.62662320240648 -12051,2003,2006,6.42648845745769,3.58933532377755,0.71786706475551 -12051,2004,2006,6.42324696353352,3.58933532377755,1.43573412951102 -12051,2005,2006,6.37331978957701,3.58933532377755,2.15360119426653 -12051,2006,2006,6.4118182677099,3.58933532377755,2.87146825902204 -12051,2007,2006,6.37331978957701,3.58933532377755,3.58933532377755 -12071,2003,2006,8.75872660784204,6.08879087493748,1.2177581749875 -12071,2004,2006,8.79845469606509,6.08879087493748,2.43551634997499 -12071,2005,2006,8.99118884193151,6.08879087493748,3.65327452496249 -12071,2006,2006,9.1516513756275,6.08879087493748,4.87103269994998 -12071,2007,2006,9.06843112579349,6.08879087493748,6.08879087493748 -12087,2003,2006,6.72383244082121,4.37687589234631,0.875375178469263 -12087,2004,2006,6.79234442747081,4.37687589234631,1.75075035693853 -12087,2005,2006,6.75809450442773,4.37687589234631,2.62612553540779 -12087,2006,2006,6.74641212857337,4.37687589234631,3.50150071387705 -12087,2007,2006,6.74758652682932,4.37687589234631,4.37687589234631 -12093,2003,2006,5.72358510195238,3.58101580823799,0.716203161647598 -12093,2004,2006,6.01126717440416,3.58101580823799,1.4324063232952 -12093,2005,2006,6.10255859461357,3.58101580823799,2.14860948494279 -12093,2006,2006,6.0913098820777,3.58101580823799,2.86481264659039 -12093,2007,2006,6.14418563412565,3.58101580823799,3.58101580823799 -12097,2003,2006,7.59085212368858,5.15035665593789,1.03007133118758 -12097,2004,2006,7.59387784460512,5.15035665593789,2.06014266237516 -12097,2005,2006,7.76556908109732,5.15035665593789,3.09021399356273 -12097,2006,2006,7.87321705486274,5.15035665593789,4.12028532475031 -12097,2007,2006,7.84737183615979,5.15035665593789,5.15035665593789 -12099,2003,2006,9.76864100976636,7.03102015077071,1.40620403015414 -12099,2004,2006,9.62918227712599,7.03102015077071,2.81240806030828 -12099,2005,2006,9.72202562615406,7.03102015077071,4.21861209046242 -12099,2006,2006,9.81623978397185,7.03102015077071,5.62481612061657 -12099,2007,2006,9.89363921648131,7.03102015077071,7.03102015077071 -12109,2003,2006,7.55066124310534,4.81328131446834,0.962656262893668 -12109,2004,2006,7.62705741701893,4.81328131446834,1.92531252578734 -12109,2005,2006,7.71199650704767,4.81328131446834,2.887968788681 -12109,2006,2006,7.86288203464149,4.81328131446834,3.85062505157467 -12109,2007,2006,7.8811822022271,4.81328131446834,4.81328131446834 -12113,2003,2006,7.09090982207998,4.76850428314641,0.953700856629282 -12113,2004,2006,7.16394668434255,4.76850428314641,1.90740171325856 -12113,2005,2006,7.36073990305828,4.76850428314641,2.86110256988785 -12113,2006,2006,7.40306109109009,4.76850428314641,3.81480342651713 -12113,2007,2006,7.33367639565768,4.76850428314641,4.76850428314641 -12121,2003,2006,5.67332326717149,3.55088095595665,0.710176191191329 -12121,2004,2006,5.6970934865054,3.55088095595665,1.42035238238266 -12121,2005,2006,5.70044357339069,3.55088095595665,2.13052857357399 -12121,2006,2006,5.77765232322266,3.55088095595665,2.84070476476532 -12121,2007,2006,5.78382518232974,3.55088095595665,3.55088095595665 -13011,2003,0,5.27299955856375,2.66875481849457,0.533750963698914 -13011,2004,0,5.31320597904179,2.66875481849457,1.06750192739783 -13011,2005,0,5.49306144334055,2.66875481849457,1.60125289109674 -13011,2006,0,5.59842195899838,2.66875481849457,2.13500385479565 -13011,2007,0,5.57215403217776,2.66875481849457,2.66875481849457 -13013,2003,0,6.18826412308259,3.83176694166248,0.766353388332497 -13013,2004,0,6.18208490671663,3.83176694166248,1.53270677666499 -13013,2005,0,6.25575004175337,3.83176694166248,2.29906016499749 -13013,2006,0,6.4377516497364,3.83176694166248,3.06541355332999 -13013,2007,0,6.40191719672719,3.83176694166248,3.83176694166248 -13019,2003,0,5.56452040732269,2.78716940555481,0.557433881110962 -13019,2004,0,5.55682806169954,2.78716940555481,1.11486776222192 -13019,2005,0,5.65599181081985,2.78716940555481,1.67230164333289 -13019,2006,0,5.62762111369064,2.78716940555481,2.22973552444385 -13019,2007,0,4.99721227376411,2.78716940555481,2.78716940555481 -13021,2003,0,7.73892375743946,5.03621856684166,1.00724371336833 -13021,2004,0,7.72797554210556,5.03621856684166,2.01448742673666 -13021,2005,0,7.73149202924568,5.03621856684166,3.02173114010499 -13021,2006,0,7.70975686445416,5.03621856684166,4.02897485347333 -13021,2007,0,7.60738142563979,5.03621856684166,5.03621856684166 -13037,2003,0,2.77258872223978,1.84371920815877,0.368743841631753 -13037,2004,0,3.04452243772342,1.84371920815877,0.737487683263506 -13037,2005,0,2.70805020110221,1.84371920815877,1.10623152489526 -13037,2006,0,3.17805383034795,1.84371920815877,1.47497536652701 -13037,2007,0,3.46573590279973,1.84371920815877,1.84371920815877 -13039,2003,0,6.22455842927536,3.77652396396598,0.755304792793197 -13039,2004,0,6.29156913955832,3.77652396396598,1.51060958558639 -13039,2005,0,6.35610766069589,3.77652396396598,2.26591437837959 -13039,2006,0,6.34212141872115,3.77652396396598,3.02121917117279 -13039,2007,0,6.29341927884648,3.77652396396598,3.77652396396598 -13049,2003,0,4.62497281328427,2.33039479363331,0.466078958726663 -13049,2004,0,4.46590811865458,2.33039479363331,0.932157917453325 -13049,2005,0,4.66343909411207,2.33039479363331,1.39823687617999 -13049,2006,0,4.70048036579242,2.33039479363331,1.86431583490665 -13049,2007,0,4.75359019110636,2.33039479363331,2.33039479363331 -13057,2003,0,7.42237370098682,4.95514372559156,0.991028745118313 -13057,2004,0,7.49997654095212,4.95514372559156,1.98205749023663 -13057,2005,0,7.62608275807238,4.95514372559156,2.97308623535494 -13057,2006,0,7.71378461659875,4.95514372559156,3.96411498047325 -13057,2007,0,7.80343505695217,4.95514372559156,4.95514372559156 -13077,2003,0,7.11314210870709,4.49104918688339,0.898209837376678 -13077,2004,0,7.07834157955767,4.49104918688339,1.79641967475336 -13077,2005,0,7.10414409298753,4.49104918688339,2.69462951213004 -13077,2006,0,7.16162200293919,4.49104918688339,3.59283934950671 -13077,2007,0,7.26332961747684,4.49104918688339,4.49104918688339 -13079,2003,0,4.02535169073515,2.52532856428692,0.505065712857383 -13079,2004,0,3.66356164612965,2.52532856428692,1.01013142571477 -13079,2005,0,3.43398720448515,2.52532856428692,1.51519713857215 -13079,2006,0,3.40119738166216,2.52532856428692,2.02026285142953 -13079,2007,0,3.13549421592915,2.52532856428692,2.52532856428692 -13093,2003,0,3.76120011569356,2.44451858888271,0.488903717776542 -13093,2004,0,3.73766961828337,2.44451858888271,0.977807435553085 -13093,2005,0,3.63758615972639,2.44451858888271,1.46671115332963 -13093,2006,0,4.0943445622221,2.44451858888271,1.95561487110617 -13093,2007,0,4.34380542185368,2.44451858888271,2.44451858888271 -13097,2003,0,7.42714413340862,4.52367809513269,0.904735619026538 -13097,2004,0,7.46794233228585,4.52367809513269,1.80947123805308 -13097,2005,0,7.57507169950756,4.52367809513269,2.71420685707961 -13097,2006,0,7.71512360363211,4.52367809513269,3.61894247610615 -13097,2007,0,7.82763954636642,4.52367809513269,4.52367809513269 -13109,2003,0,5.19849703126583,2.3508989532726,0.470179790654519 -13109,2004,0,5.15905529921453,2.3508989532726,0.940359581309039 -13109,2005,0,5.21493575760899,2.3508989532726,1.41053937196356 -13109,2006,0,5.13579843705026,2.3508989532726,1.88071916261808 -13109,2007,0,5.20948615284142,2.3508989532726,2.3508989532726 -13113,2003,0,7.42535788702715,4.51374544806874,0.902749089613749 -13113,2004,0,7.44366368311559,4.51374544806874,1.8054981792275 -13113,2005,0,7.45182223652793,4.51374544806874,2.70824726884125 -13113,2006,0,7.51752085060303,4.51374544806874,3.610996358455 -13113,2007,0,7.63723438878947,4.51374544806874,4.51374544806874 -13117,2003,0,7.33106030521863,4.58911193973939,0.917822387947878 -13117,2004,0,7.38894609761844,4.58911193973939,1.83564477589576 -13117,2005,0,7.52779398772144,4.58911193973939,2.75346716384363 -13117,2006,0,7.64730883235624,4.58911193973939,3.67128955179151 -13117,2007,0,7.63819824428578,4.58911193973939,4.58911193973939 -13129,2003,0,6.44571981938558,3.78655048128738,0.757310096257476 -13129,2004,0,6.37672694789863,3.78655048128738,1.51462019251495 -13129,2005,0,6.361302477573,3.78655048128738,2.27193028877243 -13129,2006,0,6.35437004079735,3.78655048128738,3.02924038502991 -13129,2007,0,6.39859493453521,3.78655048128738,3.78655048128738 -13135,2003,0,9.19482056031815,6.37748856255844,1.27549771251169 -13135,2004,0,9.14590851181679,6.37748856255844,2.55099542502338 -13135,2005,0,9.19482056031815,6.37748856255844,3.82649313753507 -13135,2006,0,9.29504909990314,6.37748856255844,5.10199085004675 -13135,2007,0,9.29569203907569,6.37748856255844,6.37748856255844 -13147,2003,0,5.5834963087817,3.13536377263918,0.627072754527837 -13147,2004,0,5.44241771052179,3.13536377263918,1.25414550905567 -13147,2005,0,5.5834963087817,3.13536377263918,1.88121826358351 -13147,2006,0,5.51342874616498,3.13536377263918,2.50829101811135 -13147,2007,0,5.57215403217776,3.13536377263918,3.13536377263918 -13155,2003,0,4.06044301054642,2.29566117792122,0.459132235584244 -13155,2004,0,3.8286413964891,2.29566117792122,0.918264471168489 -13155,2005,0,4.12713438504509,2.29566117792122,1.37739670675273 -13155,2006,0,3.8286413964891,2.29566117792122,1.83652894233698 -13155,2007,0,3.87120101090789,2.29566117792122,2.29566117792122 -13161,2003,0,5.01063529409626,2.5403413566761,0.50806827133522 -13161,2004,0,4.91265488573605,2.5403413566761,1.01613654267044 -13161,2005,0,5.07517381523383,2.5403413566761,1.52420481400566 -13161,2006,0,5.19849703126583,2.5403413566761,2.03227308534088 -13161,2007,0,5.23644196282995,2.5403413566761,2.5403413566761 -13179,2003,0,6.02344759296103,4.12082419502648,0.824164839005296 -13179,2004,0,5.91620206260743,4.12082419502648,1.64832967801059 -13179,2005,0,6.17586727010576,4.12082419502648,2.47249451701589 -13179,2006,0,6.13556489108174,4.12082419502648,3.29665935602118 -13179,2007,0,6.15060276844628,4.12082419502648,4.12082419502648 -13195,2003,0,4.75359019110636,3.24765762629365,0.649531525258731 -13195,2004,0,4.77912349311153,3.24765762629365,1.29906305051746 -13195,2005,0,4.66343909411207,3.24765762629365,1.94859457577619 -13195,2006,0,4.87519732320115,3.24765762629365,2.59812610103492 -13195,2007,0,5.00394630594546,3.24765762629365,3.24765762629365 -13197,2003,0,3.73766961828337,1.9662728435742,0.39325456871484 -13197,2004,0,3.63758615972639,1.9662728435742,0.786509137429679 -13197,2005,0,3.52636052461616,1.9662728435742,1.17976370614452 -13197,2006,0,3.89182029811063,1.9662728435742,1.57301827485936 -13197,2007,0,3.95124371858143,1.9662728435742,1.9662728435742 -13209,2003,0,3.8286413964891,2.1126345090356,0.42252690180712 -13209,2004,0,3.40119738166216,2.1126345090356,0.84505380361424 -13209,2005,0,3.55534806148941,2.1126345090356,1.26758070542136 -13209,2006,0,3.66356164612965,2.1126345090356,1.69010760722848 -13209,2007,0,3.58351893845611,2.1126345090356,2.1126345090356 -13211,2003,0,5.19295685089021,2.73806197517018,0.547612395034036 -13211,2004,0,5.18738580584075,2.73806197517018,1.09522479006807 -13211,2005,0,5.28320372873799,2.73806197517018,1.64283718510211 -13211,2006,0,5.605802066296,2.73806197517018,2.19044958013615 -13211,2007,0,5.59471137960184,2.73806197517018,2.73806197517018 -13213,2003,0,5.71042701737487,3.59747663064059,0.719495326128119 -13213,2004,0,5.68357976733868,3.59747663064059,1.43899065225624 -13213,2005,0,5.38449506278909,3.59747663064059,2.15848597838436 -13213,2006,0,5.72031177660741,3.59747663064059,2.87798130451247 -13213,2007,0,5.65248918026865,3.59747663064059,3.59747663064059 -13217,2003,0,6.45204895443723,4.12715051394728,0.825430102789456 -13217,2004,0,6.49677499018586,4.12715051394728,1.65086020557891 -13217,2005,0,6.51619307604296,4.12715051394728,2.47629030836837 -13217,2006,0,6.62804137617953,4.12715051394728,3.30172041115782 -13217,2007,0,6.62539236800796,4.12715051394728,4.12715051394728 -13223,2003,0,6.38519439899773,4.40278468775839,0.880556937551678 -13223,2004,0,6.48463523563525,4.40278468775839,1.76111387510336 -13223,2005,0,6.71295620067707,4.40278468775839,2.64167081265503 -13223,2006,0,6.80682936039218,4.40278468775839,3.52222775020671 -13223,2007,0,6.98471632011827,4.40278468775839,4.40278468775839 -13227,2003,0,5.32300997913841,3.13475481220279,0.626950962440559 -13227,2004,0,5.4380793089232,3.13475481220279,1.25390192488112 -13227,2005,0,5.37527840768417,3.13475481220279,1.88085288732168 -13227,2006,0,5.40717177146012,3.13475481220279,2.50780384976223 -13227,2007,0,5.4510384535657,3.13475481220279,3.13475481220279 -13233,2003,0,5.96614673912369,3.64092269256804,0.728184538513609 -13233,2004,0,5.92692602597041,3.64092269256804,1.45636907702722 -13233,2005,0,5.94803498918065,3.64092269256804,2.18455361554083 -13233,2006,0,5.90808293816893,3.64092269256804,2.91273815405443 -13233,2007,0,5.91079664404053,3.64092269256804,3.64092269256804 -13237,2003,0,5.00394630594546,2.9344949640828,0.58689899281656 -13237,2004,0,5.03695260241363,2.9344949640828,1.17379798563312 -13237,2005,0,5.08759633523238,2.9344949640828,1.76069697844968 -13237,2006,0,5.52545293913178,2.9344949640828,2.34759597126624 -13237,2007,0,5.4249500174814,2.9344949640828,2.9344949640828 -13241,2003,0,5.4510384535657,2.71137799119488,0.542275598238977 -13241,2004,0,5.26269018890489,2.71137799119488,1.08455119647795 -13241,2005,0,5.4249500174814,2.71137799119488,1.62682679471693 -13241,2006,0,5.37063802812766,2.71137799119488,2.16910239295591 -13241,2007,0,5.40717177146012,2.71137799119488,2.71137799119488 -13263,2003,0,2.39789527279837,1.87149443724691,0.374298887449382 -13263,2004,0,2.77258872223978,1.87149443724691,0.748597774898763 -13263,2005,0,1.94591014905531,1.87149443724691,1.12289666234815 -13263,2006,0,1.79175946922805,1.87149443724691,1.49719554979753 -13263,2007,0,2.30258509299405,1.87149443724691,1.87149443724691 -13271,2003,0,4.68213122712422,2.46759092757831,0.493518185515662 -13271,2004,0,4.86753445045558,2.46759092757831,0.987036371031325 -13271,2005,0,5.16478597392351,2.46759092757831,1.48055455654699 -13271,2006,0,5.28320372873799,2.46759092757831,1.97407274206265 -13271,2007,0,5.19295685089021,2.46759092757831,2.46759092757831 -13273,2003,0,3.98898404656427,2.39516427428714,0.479032854857428 -13273,2004,0,4.0943445622221,2.39516427428714,0.958065709714856 -13273,2005,0,4.17438726989564,2.39516427428714,1.43709856457228 -13273,2006,0,4.18965474202643,2.39516427428714,1.91613141942971 -13273,2007,0,4.0943445622221,2.39516427428714,2.39516427428714 -13297,2003,0,6.53233429222235,4.10572950708866,0.821145901417733 -13297,2004,0,6.4085287910595,4.10572950708866,1.64229180283547 -13297,2005,0,6.35437004079735,4.10572950708866,2.4634377042532 -13297,2006,0,6.52649485957079,4.10572950708866,3.28458360567093 -13297,2007,0,6.62936325343745,4.10572950708866,4.10572950708866 -13303,2003,0,5.11799381241676,3.0528684649248,0.61057369298496 -13303,2004,0,4.99043258677874,3.0528684649248,1.22114738596992 -13303,2005,0,4.969813299576,3.0528684649248,1.83172107895488 -13303,2006,0,4.93447393313069,3.0528684649248,2.44229477193984 -13303,2007,0,5.04342511691925,3.0528684649248,3.0528684649248 -13313,2003,0,7.15148546390474,4.42514598824282,0.885029197648564 -13313,2004,0,7.15539630189673,4.42514598824282,1.77005839529713 -13313,2005,0,7.20042489294496,4.42514598824282,2.65508759294569 -13313,2006,0,7.27586460054653,4.42514598824282,3.54011679059426 -13313,2007,0,7.18841273649695,4.42514598824282,4.42514598824282 -13317,2003,0,4.92725368515721,2.36902804954276,0.473805609908552 -13317,2004,0,4.74493212836325,2.36902804954276,0.947611219817104 -13317,2005,0,4.64439089914137,2.36902804954276,1.42141682972566 -13317,2006,0,4.59511985013459,2.36902804954276,1.89522243963421 -13317,2007,0,4.54329478227,2.36902804954276,2.36902804954276 -13321,2003,0,4.96284463025991,3.08954132723205,0.61790826544641 -13321,2004,0,4.97673374242057,3.08954132723205,1.23581653089282 -13321,2005,0,5.08759633523238,3.08954132723205,1.85372479633923 -13321,2006,0,5.15329159449778,3.08954132723205,2.47163306178564 -13321,2007,0,4.89034912822175,3.08954132723205,3.08954132723205 -16003,2003,0,3.25809653802148,1.24588220740315,0.249176441480629 -16003,2004,0,3.61091791264422,1.24588220740315,0.498352882961258 -16003,2005,0,3.98898404656427,1.24588220740315,0.747529324441887 -16003,2006,0,3.98898404656427,1.24588220740315,0.996705765922517 -16003,2007,0,3.89182029811063,1.24588220740315,1.24588220740315 -16015,2003,0,3.89182029811063,1.89761985992753,0.379523971985506 -16015,2004,0,3.91202300542815,1.89761985992753,0.759047943971013 -16015,2005,0,4.17438726989564,1.89761985992753,1.13857191595652 -16015,2006,0,4.02535169073515,1.89761985992753,1.51809588794203 -16015,2007,0,4.14313472639153,1.89761985992753,1.89761985992753 -16017,2003,0,6.29710931993394,3.60644848013478,0.721289696026955 -16017,2004,0,6.29894924685594,3.60644848013478,1.44257939205391 -16017,2005,0,6.4281052726846,3.60644848013478,2.16386908808087 -16017,2006,0,6.42971947803914,3.60644848013478,2.88515878410782 -16017,2007,0,6.40522845803084,3.60644848013478,3.60644848013478 -16021,2003,0,4.89783979995091,2.28960116543575,0.45792023308715 -16021,2004,0,4.83628190695148,2.28960116543575,0.9158404661743 -16021,2005,0,4.96284463025991,2.28960116543575,1.37376069926145 -16021,2006,0,4.99721227376411,2.28960116543575,1.8316809323486 -16021,2007,0,4.82028156560504,2.28960116543575,2.28960116543575 -16029,2003,0,4.9416424226093,1.98842214329267,0.397684428658533 -16029,2004,0,4.79579054559674,1.98842214329267,0.795368857317067 -16029,2005,0,4.98360662170834,1.98842214329267,1.1930532859756 -16029,2006,0,4.78749174278205,1.98842214329267,1.59073771463413 -16029,2007,0,5.03695260241363,1.98842214329267,1.98842214329267 -16041,2003,0,5.63835466933375,2.42736580989135,0.48547316197827 -16041,2004,0,5.53733426701854,2.42736580989135,0.970946323956541 -16041,2005,0,5.62762111369064,2.42736580989135,1.45641948593481 -16041,2006,0,5.6937321388027,2.42736580989135,1.94189264791308 -16041,2007,0,5.74939298590825,2.42736580989135,2.42736580989135 -16045,2003,0,5.01727983681492,2.72004464595058,0.544008929190116 -16045,2004,0,4.92725368515721,2.72004464595058,1.08801785838023 -16045,2005,0,5.12989871492307,2.72004464595058,1.63202678757035 -16045,2006,0,5.08759633523238,2.72004464595058,2.17603571676046 -16045,2007,0,5.19849703126583,2.72004464595058,2.72004464595058 -16067,2003,0,5.73334127689775,3.00439464663264,0.600878929326529 -16067,2004,0,5.73979291217923,3.00439464663264,1.20175785865306 -16067,2005,0,5.73334127689775,3.00439464663264,1.80263678797959 -16067,2006,0,5.77455154554441,3.00439464663264,2.40351571730611 -16067,2007,0,5.80814248998044,3.00439464663264,3.00439464663264 -16079,2003,0,5.09375020080676,2.62256493174499,0.524512986348998 -16079,2004,0,5.17614973257383,2.62256493174499,1.049025972698 -16079,2005,0,5.14749447681345,2.62256493174499,1.57353895904699 -16079,2006,0,5.24702407216049,2.62256493174499,2.09805194539599 -16079,2007,0,5.38449506278909,2.62256493174499,2.62256493174499 -16083,2003,0,7.20489251020467,4.16331076668682,0.832662153337365 -16083,2004,0,7.18311170174328,4.16331076668682,1.66532430667473 -16083,2005,0,7.14045304310116,4.16331076668682,2.49798646001209 -16083,2006,0,7.23489842031483,4.16331076668682,3.33064861334946 -16083,2007,0,7.28961052145117,4.16331076668682,4.16331076668682 -17005,2003,2004,5.18738580584075,2.86977214642579,0.573954429285157 -17005,2004,2004,5.06890420222023,2.86977214642579,1.14790885857031 -17005,2005,2004,4.93447393313069,2.86977214642579,1.72186328785547 -17005,2006,2004,4.89034912822175,2.86977214642579,2.29581771714063 -17005,2007,2004,4.99043258677874,2.86977214642579,2.86977214642579 -17015,2003,2004,5.42934562895444,2.81385061998842,0.562770123997684 -17015,2004,2004,5.42053499927229,2.81385061998842,1.12554024799537 -17015,2005,2004,5.50125821054473,2.81385061998842,1.68831037199305 -17015,2006,2004,5.46806014113513,2.81385061998842,2.25108049599074 -17015,2007,2004,5.4553211153577,2.81385061998842,2.81385061998842 -17025,2003,2004,5.08759633523238,2.67827804276854,0.535655608553708 -17025,2004,2004,5.00394630594546,2.67827804276854,1.07131121710742 -17025,2005,2004,4.85203026391962,2.67827804276854,1.60696682566112 -17025,2006,2004,4.98360662170834,2.67827804276854,2.14262243421483 -17025,2007,2004,4.97673374242057,2.67827804276854,2.67827804276854 -17035,2003,2004,4.92725368515721,2.42063475976786,0.484126951953572 -17035,2004,2004,4.59511985013459,2.42063475976786,0.968253903907144 -17035,2005,2004,4.44265125649032,2.42063475976786,1.45238085586072 -17035,2006,2004,4.30406509320417,2.42063475976786,1.93650780781429 -17035,2007,2004,4.46590811865458,2.42063475976786,2.42063475976786 -17047,2003,2004,4.35670882668959,1.94175868650426,0.388351737300852 -17047,2004,2004,4.39444915467244,1.94175868650426,0.776703474601705 -17047,2005,2004,4.20469261939097,1.94175868650426,1.16505521190256 -17047,2006,2004,4.00733318523247,1.94175868650426,1.55340694920341 -17047,2007,2004,4.12713438504509,1.94175868650426,1.94175868650426 -17049,2003,2004,6.99025650049388,3.53409524031307,0.706819048062614 -17049,2004,2004,6.90975328164481,3.53409524031307,1.41363809612523 -17049,2005,2004,6.87523208727658,3.53409524031307,2.12045714418784 -17049,2006,2004,6.8330317327862,3.53409524031307,2.82727619225046 -17049,2007,2004,6.87626461189077,3.53409524031307,3.53409524031307 -17053,2003,2004,5.61677109766657,2.65612512823729,0.531225025647458 -17053,2004,2004,5.48893772615669,2.65612512823729,1.06245005129492 -17053,2005,2004,5.48893772615669,2.65612512823729,1.59367507694237 -17053,2006,2004,5.51342874616498,2.65612512823729,2.12490010258983 -17053,2007,2004,5.31320597904179,2.65612512823729,2.65612512823729 -17063,2003,2004,6.58063913728495,3.62527383102497,0.725054766204993 -17063,2004,2004,6.47697236288968,3.62527383102497,1.45010953240999 -17063,2005,2004,6.49375383985169,3.62527383102497,2.17516429861498 -17063,2006,2004,6.4831073514572,3.62527383102497,2.90021906481997 -17063,2007,2004,6.5694814204143,3.62527383102497,3.62527383102497 -17089,2003,2004,9.0067543198775,6.00170938904395,1.20034187780879 -17089,2004,2004,8.96763166516713,6.00170938904395,2.40068375561758 -17089,2005,2004,8.99118884193151,6.00170938904395,3.60102563342637 -17089,2006,2004,8.97802993283716,6.00170938904395,4.80136751123516 -17089,2007,2004,9.02509454366498,6.00170938904395,6.00170938904395 -17097,2003,2004,9.27369068747694,6.46825136840052,1.2936502736801 -17097,2004,2004,9.22147811638664,6.46825136840052,2.58730054736021 -17097,2005,2004,9.2072355570227,6.46825136840052,3.88095082104031 -17097,2006,2004,9.24599704918422,6.46825136840052,5.17460109472042 -17097,2007,2004,9.23824732522919,6.46825136840052,6.46825136840052 -17113,2003,2004,7.89095671613892,5.01351780234143,1.00270356046829 -17113,2004,2004,7.77653502818524,5.01351780234143,2.00540712093657 -17113,2005,2004,7.755767170103,5.01351780234143,3.00811068140486 -17113,2006,2004,7.74022952476318,5.01351780234143,4.01081424187314 -17113,2007,2004,7.8188324438034,5.01351780234143,5.01351780234143 -17125,2003,2004,5.24702407216049,2.77496090638484,0.554992181276967 -17125,2004,2004,5.25227342804663,2.77496090638484,1.10998436255393 -17125,2005,2004,4.97673374242057,2.77496090638484,1.6649765438309 -17125,2006,2004,4.99043258677874,2.77496090638484,2.21996872510787 -17125,2007,2004,5.05624580534831,2.77496090638484,2.77496090638484 -17135,2003,2004,6.07764224334903,3.42269791310816,0.684539582621632 -17135,2004,2004,6.10255859461357,3.42269791310816,1.36907916524326 -17135,2005,2004,6.15485809401642,3.42269791310816,2.05361874786489 -17135,2006,2004,6.0137151560428,3.42269791310816,2.73815833048653 -17135,2007,2004,6.01615715969835,3.42269791310816,3.42269791310816 -17155,2003,2004,4.02535169073515,1.80599105147478,0.361198210294956 -17155,2004,2004,3.73766961828337,1.80599105147478,0.722396420589912 -17155,2005,2004,3.91202300542815,1.80599105147478,1.08359463088487 -17155,2006,2004,3.78418963391826,1.80599105147478,1.44479284117982 -17155,2007,2004,4.24849524204936,1.80599105147478,1.80599105147478 -17161,2003,2004,7.87663846097546,5.0064532281027,1.00129064562054 -17161,2004,2004,7.82404601085629,5.0064532281027,2.00258129124108 -17161,2005,2004,7.86557175768479,5.0064532281027,3.00387193686162 -17161,2006,2004,7.85166117788927,5.0064532281027,4.00516258248216 -17161,2007,2004,7.82923253754359,5.0064532281027,5.0064532281027 -17169,2003,2004,4.52178857704904,1.97255208000173,0.394510416000347 -17169,2004,2004,4.53259949315326,1.97255208000173,0.789020832000694 -17169,2005,2004,4.31748811353631,1.97255208000173,1.18353124800104 -17169,2006,2004,4.38202663467388,1.97255208000173,1.57804166400139 -17169,2007,2004,4.43081679884331,1.97255208000173,1.97255208000173 -17193,2003,2004,5.4510384535657,2.7324826172428,0.546496523448559 -17193,2004,2004,5.3890717298165,2.7324826172428,1.09299304689712 -17193,2005,2004,5.25227342804663,2.7324826172428,1.63948957034568 -17193,2006,2004,5.17614973257383,2.7324826172428,2.18598609379424 -17193,2007,2004,5.27299955856375,2.7324826172428,2.7324826172428 -17195,2003,2004,6.71780469502369,4.10516909830525,0.82103381966105 -17195,2004,2004,6.6682282484174,4.10516909830525,1.6420676393221 -17195,2005,2004,6.76849321164863,4.10516909830525,2.46310145898315 -17195,2006,2004,6.68959926917897,4.10516909830525,3.2841352786442 -17195,2007,2004,6.73221070646721,4.10516909830525,4.10516909830525 -17199,2003,2004,6.81014245011514,4.11571458795829,0.823142917591659 -17199,2004,2004,6.8596149036542,4.11571458795829,1.64628583518332 -17199,2005,2004,6.77764659363512,4.11571458795829,2.46942875277498 -17199,2006,2004,6.72982407048948,4.11571458795829,3.29257167036663 -17199,2007,2004,6.77878489768518,4.11571458795829,4.11571458795829 -17203,2003,2004,6.5191472879404,3.56865907554909,0.713731815109817 -17203,2004,2004,6.44094654063292,3.56865907554909,1.42746363021963 -17203,2005,2004,6.41673228251233,3.56865907554909,2.14119544532945 -17203,2006,2004,6.46925031679577,3.56865907554909,2.85492726043927 -17203,2007,2004,6.48616078894409,3.56865907554909,3.56865907554909 -18005,2003,0,7.15305163493748,4.26878794531712,0.853757589063424 -18005,2004,0,7.11720550316434,4.26878794531712,1.70751517812685 -18005,2005,0,7.16626597413364,4.26878794531712,2.56127276719027 -18005,2006,0,7.10002716662926,4.26878794531712,3.4150303562537 -18005,2007,0,7.08170858610557,4.26878794531712,4.26878794531712 -18017,2003,0,6.29156913955832,3.71186329050431,0.742372658100862 -18017,2004,0,6.20455776256869,3.71186329050431,1.48474531620172 -18017,2005,0,6.10255859461357,3.71186329050431,2.22711797430259 -18017,2006,0,6.0913098820777,3.71186329050431,2.96949063240345 -18017,2007,0,6.06145691892802,3.71186329050431,3.71186329050431 -18019,2003,0,7.40610338123702,4.56925281080129,0.913850562160258 -18019,2004,0,7.31388683163346,4.56925281080129,1.82770112432052 -18019,2005,0,7.27931883541462,4.56925281080129,2.74155168648077 -18019,2006,0,7.40974195408092,4.56925281080129,3.65540224864103 -18019,2007,0,7.36897040219479,4.56925281080129,4.56925281080129 -18021,2003,0,5.47227067367148,3.27925571085691,0.655851142171383 -18021,2004,0,5.73657229747919,3.27925571085691,1.31170228434277 -18021,2005,0,5.75574221358691,3.27925571085691,1.96755342651415 -18021,2006,0,5.60947179518496,3.27925571085691,2.62340456868553 -18021,2007,0,5.56068163101553,3.27925571085691,3.27925571085691 -18023,2003,0,6.12029741895095,3.52241156124315,0.70448231224863 -18023,2004,0,6.07073772800249,3.52241156124315,1.40896462449726 -18023,2005,0,6.04263283368238,3.52241156124315,2.11344693674589 -18023,2006,0,6.00141487796115,3.52241156124315,2.81792924899452 -18023,2007,0,6.02344759296103,3.52241156124315,3.52241156124315 -18029,2003,0,6.80239476332431,3.83100815871387,0.766201631742774 -18029,2004,0,6.74405918631135,3.83100815871387,1.53240326348555 -18029,2005,0,6.75693238924755,3.83100815871387,2.29860489522832 -18029,2006,0,6.64898455002478,3.83100815871387,3.0648065269711 -18029,2007,0,6.81014245011514,3.83100815871387,3.83100815871387 -18033,2003,0,6.50727771238501,3.69597919122916,0.739195838245832 -18033,2004,0,6.42162226780652,3.69597919122916,1.47839167649166 -18033,2005,0,6.43615036836943,3.69597919122916,2.2175875147375 -18033,2006,0,6.51174532964473,3.69597919122916,2.95678335298333 -18033,2007,0,6.37842618365159,3.69597919122916,3.69597919122916 -18035,2003,0,7.34729970074316,4.77718043011612,0.955436086023223 -18035,2004,0,7.22766249872865,4.77718043011612,1.91087217204645 -18035,2005,0,7.1929342212158,4.77718043011612,2.86630825806967 -18035,2006,0,7.18992217074581,4.77718043011612,3.82174434409289 -18035,2007,0,7.17472430983638,4.77718043011612,4.77718043011612 -18053,2003,0,6.87109129461055,4.29596480671984,0.859192961343968 -18053,2004,0,6.78105762593618,4.29596480671984,1.71838592268794 -18053,2005,0,6.75809450442773,4.29596480671984,2.5775788840319 -18053,2006,0,6.67076632084587,4.29596480671984,3.43677184537587 -18053,2007,0,6.66568371778241,4.29596480671984,4.29596480671984 -18063,2003,0,7.56060116276856,4.64528473032447,0.929056946064893 -18063,2004,0,7.59789795052178,4.64528473032447,1.85811389212979 -18063,2005,0,7.64444076155657,4.64528473032447,2.78717083819468 -18063,2006,0,7.70616297019958,4.64528473032447,3.71622778425957 -18063,2007,0,7.74066440191724,4.64528473032447,4.64528473032447 -18081,2003,0,7.74975340627444,4.74674787021019,0.949349574042038 -18081,2004,0,7.73805229768932,4.74674787021019,1.89869914808408 -18081,2005,0,7.71868549519847,4.74674787021019,2.84804872212611 -18081,2006,0,7.7106533235012,4.74674787021019,3.79739829616815 -18081,2007,0,7.71646080017636,4.74674787021019,4.74674787021019 -18083,2003,0,6.33150184989369,3.6701042987876,0.734020859757521 -18083,2004,0,6.24997524225948,3.6701042987876,1.46804171951504 -18083,2005,0,6.24804287450843,3.6701042987876,2.20206257927256 -18083,2006,0,6.29341927884648,3.6701042987876,2.93608343903008 -18083,2007,0,6.18208490671663,3.6701042987876,3.6701042987876 -18091,2003,0,7.43543801981455,4.70144353815659,0.940288707631318 -18091,2004,0,7.37337430991005,4.70144353815659,1.88057741526264 -18091,2005,0,7.39510754656249,4.70144353815659,2.82086612289396 -18091,2006,0,7.3790081276283,4.70144353815659,3.76115483052527 -18091,2007,0,7.37148929521428,4.70144353815659,4.70144353815659 -18093,2003,0,6.21060007702465,3.82694430506983,0.765388861013966 -18093,2004,0,6.15485809401642,3.82694430506983,1.53077772202793 -18093,2005,0,6.15697898558556,3.82694430506983,2.2961665830419 -18093,2006,0,6.15909538849193,3.82694430506983,3.06155544405587 -18093,2007,0,6.20253551718792,3.82694430506983,3.82694430506983 -18099,2003,0,6.57368016696065,3.80950289643767,0.761900579287535 -18099,2004,0,6.51471269087253,3.80950289643767,1.52380115857507 -18099,2005,0,6.48004456192665,3.80950289643767,2.28570173786261 -18099,2006,0,6.52356230614951,3.80950289643767,3.04760231715014 -18099,2007,0,6.45833828334479,3.80950289643767,3.80950289643768 -18109,2003,0,6.68085467879022,4.20004002177996,0.840008004355991 -18109,2004,0,6.63068338564237,4.20004002177996,1.68001600871198 -18109,2005,0,6.65929391968364,4.20004002177996,2.52002401306797 -18109,2006,0,6.61606518513282,4.20004002177996,3.36003201742396 -18109,2007,0,6.59714570188665,4.20004002177996,4.20004002177996 -18113,2003,0,6.49072353450251,3.83460185845956,0.766920371691912 -18113,2004,0,6.45676965557216,3.83460185845956,1.53384074338382 -18113,2005,0,6.3919171133926,3.83460185845956,2.30076111507574 -18113,2006,0,6.41999492814714,3.83460185845956,3.06768148676765 -18113,2007,0,6.39692965521615,3.83460185845956,3.83460185845956 -18115,2003,0,4.29045944114839,1.72686532931006,0.345373065862013 -18115,2004,0,4.26267987704132,1.72686532931006,0.690746131724026 -18115,2005,0,4.17438726989564,1.72686532931006,1.03611919758604 -18115,2006,0,4.04305126783455,1.72686532931006,1.38149226344805 -18115,2007,0,4.07753744390572,1.72686532931006,1.72686532931006 -18119,2003,0,5.28320372873799,3.08126756166025,0.616253512332051 -18119,2004,0,5.30330490805908,3.08126756166025,1.2325070246641 -18119,2005,0,5.35185813347607,3.08126756166025,1.84876053699615 -18119,2006,0,5.30826769740121,3.08126756166025,2.4650140493282 -18119,2007,0,5.27299955856375,3.08126756166025,3.08126756166025 -18133,2003,0,6.4425401664682,3.58404657700818,0.716809315401636 -18133,2004,0,6.47697236288968,3.58404657700818,1.43361863080327 -18133,2005,0,6.43294009273918,3.58404657700818,2.15042794620491 -18133,2006,0,6.28785856016178,3.58404657700818,2.86723726160654 -18133,2007,0,6.24610676548156,3.58404657700818,3.58404657700818 -18145,2003,0,6.48768401848461,3.77149577029117,0.754299154058234 -18145,2004,0,6.4723462945009,3.77149577029117,1.50859830811647 -18145,2005,0,6.31173480915291,3.77149577029117,2.2628974621747 -18145,2006,0,6.18826412308259,3.77149577029117,3.01719661623294 -18145,2007,0,6.13556489108174,3.77149577029117,3.77149577029117 -18157,2003,0,7.70616297019958,5.0036442469074,1.00072884938148 -18157,2004,0,7.6425241342329,5.0036442469074,2.00145769876296 -18157,2005,0,7.70751219460034,5.0036442469074,3.00218654814444 -18157,2006,0,7.73587031995257,5.0036442469074,4.00291539752592 -18157,2007,0,7.74196789982069,5.0036442469074,5.0036442469074 -18175,2003,0,5.35185813347607,3.30406220422892,0.660812440845785 -18175,2004,0,5.31811999384422,3.30406220422892,1.32162488169157 -18175,2005,0,5.22035582507832,3.30406220422892,1.98243732253735 -18175,2006,0,5.24702407216049,3.30406220422892,2.64324976338314 -18175,2007,0,5.27299955856375,3.30406220422892,3.30406220422892 -18179,2003,0,6.08449941307517,3.3178157727231,0.663563154544621 -18179,2004,0,5.94803498918065,3.3178157727231,1.32712630908924 -18179,2005,0,5.92958914338989,3.3178157727231,1.99068946363386 -18179,2006,0,6.05208916892442,3.3178157727231,2.65425261817848 -18179,2007,0,5.83188247728352,3.3178157727231,3.3178157727231 -18181,2003,0,5.53338948872752,3.22949919650533,0.645899839301066 -18181,2004,0,5.32300997913841,3.22949919650533,1.29179967860213 -18181,2005,0,5.28826703069454,3.22949919650533,1.9376995179032 -18181,2006,0,5.25749537202778,3.22949919650533,2.58359935720427 -18181,2007,0,5.23110861685459,3.22949919650533,3.22949919650533 -19013,2003,0,7.9043348420851,4.85212400952536,0.970424801905072 -19013,2004,0,7.83794891602528,4.85212400952536,1.94084960381014 -19013,2005,0,7.8407064517494,4.85212400952536,2.91127440571522 -19013,2006,0,7.85476918349913,4.85212400952536,3.88169920762029 -19013,2007,0,7.87929148508227,4.85212400952536,4.85212400952536 -19025,2003,0,5.46806014113513,2.40829554741616,0.481659109483232 -19025,2004,0,5.2257466737132,2.40829554741616,0.963318218966464 -19025,2005,0,5.18738580584075,2.40829554741616,1.4449773284497 -19025,2006,0,5.22035582507832,2.40829554741616,1.92663643793293 -19025,2007,0,5.15329159449778,2.40829554741616,2.40829554741616 -19069,2003,0,5.28320372873799,2.37061750338587,0.474123500677175 -19069,2004,0,5.21493575760899,2.37061750338587,0.948247001354349 -19069,2005,0,5.31320597904179,2.37061750338587,1.42237050203152 -19069,2006,0,5.19849703126583,2.37061750338587,1.8964940027087 -19069,2007,0,5.2040066870768,2.37061750338587,2.37061750338587 -19077,2003,0,5.05624580534831,2.4294820261824,0.485896405236479 -19077,2004,0,4.97673374242057,2.4294820261824,0.971792810472959 -19077,2005,0,4.9416424226093,2.4294820261824,1.45768921570944 -19077,2006,0,4.84418708645859,2.4294820261824,1.94358562094592 -19077,2007,0,4.87519732320115,2.4294820261824,2.4294820261824 -19083,2003,0,5.87493073085203,2.9344949640828,0.58689899281656 -19083,2004,0,5.78689738136671,2.9344949640828,1.17379798563312 -19083,2005,0,5.75257263882563,2.9344949640828,1.76069697844968 -19083,2006,0,5.77144112313002,2.9344949640828,2.34759597126624 -19083,2007,0,5.75257263882563,2.9344949640828,2.9344949640828 -19087,2003,0,6.20253551718792,3.0123927144471,0.60247854288942 -19087,2004,0,6.01126717440416,3.0123927144471,1.20495708577884 -19087,2005,0,6.0137151560428,3.0123927144471,1.80743562866826 -19087,2006,0,5.98896141688986,3.0123927144471,2.40991417155768 -19087,2007,0,5.92692602597041,3.0123927144471,3.0123927144471 -19111,2003,0,6.40687998606931,3.63895364534421,0.727790729068842 -19111,2004,0,6.32076829425058,3.63895364534421,1.45558145813768 -19111,2005,0,6.39359075395063,3.63895364534421,2.18337218720653 -19111,2006,0,6.43615036836943,3.63895364534421,2.91116291627537 -19111,2007,0,6.43294009273918,3.63895364534421,3.63895364534421 -19113,2003,0,8.30992298925832,5.25593686652208,1.05118737330442 -19113,2004,0,8.28702502516506,5.25593686652208,2.10237474660883 -19113,2005,0,8.25400859056484,5.25593686652208,3.15356211991325 -19113,2006,0,8.28526113406895,5.25593686652208,4.20474949321767 -19113,2007,0,8.36985260351753,5.25593686652208,5.25593686652208 -19123,2003,0,5.89989735358249,3.10615495480408,0.621230990960816 -19123,2004,0,5.8406416573734,3.10615495480408,1.24246198192163 -19123,2005,0,5.73657229747919,3.10615495480408,1.86369297288245 -19123,2006,0,5.80211837537706,3.10615495480408,2.48492396384327 -19123,2007,0,5.77455154554441,3.10615495480408,3.10615495480408 -19131,2003,0,5.26785815906333,2.38637461872373,0.477274923744746 -19131,2004,0,5.18738580584075,2.38637461872373,0.954549847489492 -19131,2005,0,5.27811465923052,2.38637461872373,1.43182477123424 -19131,2006,0,5.25749537202778,2.38637461872373,1.90909969497898 -19131,2007,0,5.31811999384422,2.38637461872373,2.38637461872373 -19137,2003,0,5.37527840768417,2.46563887943034,0.493127775886069 -19137,2004,0,5.44673737166631,2.46563887943034,0.986255551772137 -19137,2005,0,5.37989735354046,2.46563887943034,1.47938332765821 -19137,2006,0,5.4510384535657,2.46563887943034,1.97251110354427 -19137,2007,0,5.4553211153577,2.46563887943034,2.46563887943034 -19141,2003,0,6.08904487544685,2.71482718538123,0.542965437076247 -19141,2004,0,6.00141487796115,2.71482718538123,1.08593087415249 -19141,2005,0,6.0330862217988,2.71482718538123,1.62889631122874 -19141,2006,0,6.05208916892442,2.71482718538123,2.17186174830499 -19141,2007,0,5.88610403145016,2.71482718538123,2.71482718538123 -19153,2003,0,9.19136140537838,5.92586145952057,1.18517229190411 -19153,2004,0,9.12565356380899,5.92586145952057,2.37034458380823 -19153,2005,0,9.12956406194889,5.92586145952057,3.55551687571234 -19153,2006,0,9.14729407025855,5.92586145952057,4.74068916761646 -19153,2007,0,9.1557785839835,5.92586145952057,5.92586145952057 -19157,2003,0,6.3456363608286,2.9346544240454,0.58693088480908 -19157,2004,0,6.09582456243222,2.9346544240454,1.17386176961816 -19157,2005,0,6.06378520868761,2.9346544240454,1.76079265442724 -19157,2006,0,6.1463292576689,2.9346544240454,2.34772353923632 -19157,2007,0,6.15485809401642,2.9346544240454,2.9346544240454 -19159,2003,0,4.64439089914137,1.69909578436464,0.339819156872927 -19159,2004,0,4.53259949315326,1.69909578436464,0.679638313745854 -19159,2005,0,4.33073334028633,1.69909578436464,1.01945747061878 -19159,2006,0,4.27666611901606,1.69909578436464,1.35927662749171 -19159,2007,0,4.36944785246702,1.69909578436464,1.69909578436464 -19183,2003,0,6.09582456243222,3.02868337369368,0.605736674738735 -19183,2004,0,5.92692602597041,3.02868337369368,1.21147334947747 -19183,2005,0,5.88887795833288,3.02868337369368,1.81721002421621 -19183,2006,0,5.97888576490112,3.02868337369368,2.42294669895494 -19183,2007,0,6.12468339089421,3.02868337369368,3.02868337369368 -19185,2003,0,4.24849524204936,1.90657514365664,0.381315028731327 -19185,2004,0,3.93182563272433,1.90657514365664,0.762630057462655 -19185,2005,0,4.02535169073515,1.90657514365664,1.14394508619398 -19185,2006,0,4.06044301054642,1.90657514365664,1.52526011492531 -19185,2007,0,4.4188406077966,1.90657514365664,1.90657514365664 -19187,2003,0,6.73340189183736,3.6947372635981,0.738947452719619 -19187,2004,0,6.68336094576627,3.6947372635981,1.47789490543924 -19187,2005,0,6.64898455002478,3.6947372635981,2.21684235815886 -19187,2006,0,6.68336094576627,3.6947372635981,2.95578981087848 -19187,2007,0,6.67329796776765,3.6947372635981,3.6947372635981 -20013,2003,0,5.42934562895444,2.37248422037371,0.474496844074743 -20013,2004,0,5.45958551414416,2.37248422037371,0.948993688149485 -20013,2005,0,5.50125821054473,2.37248422037371,1.42349053222423 -20013,2006,0,5.31811999384422,2.37248422037371,1.89798737629897 -20013,2007,0,5.28320372873799,2.37248422037371,2.37248422037371 -20021,2003,0,5.59098698051086,3.11817112074657,0.623634224149314 -20021,2004,0,5.35658627467201,3.11817112074657,1.24726844829863 -20021,2005,0,5.54517744447956,3.11817112074657,1.87090267244794 -20021,2006,0,5.40267738187228,3.11817112074657,2.49453689659725 -20021,2007,0,5.31811999384422,3.11817112074657,3.11817112074657 -20027,2003,0,5.14166355650266,2.17724860168275,0.43544972033655 -20027,2004,0,5.23644196282995,2.17724860168275,0.870899440673099 -20027,2005,0,5.37063802812766,2.17724860168275,1.30634916100965 -20027,2006,0,5.39362754635236,2.17724860168275,1.7417988813462 -20027,2007,0,5.40267738187228,2.17724860168275,2.17724860168275 -20031,2003,0,5.28826703069454,2.18211093952617,0.436422187905234 -20031,2004,0,5.19849703126583,2.18211093952617,0.872844375810468 -20031,2005,0,4.94875989037817,2.18211093952617,1.3092665637157 -20031,2006,0,5.37527840768417,2.18211093952617,1.74568875162094 -20031,2007,0,5.39362754635236,2.18211093952617,2.18211093952617 -20045,2003,0,7.52240023138712,4.6047901137698,0.920958022753959 -20045,2004,0,7.3664451483276,4.6047901137698,1.84191604550792 -20045,2005,0,7.37525577800975,4.6047901137698,2.76287406826188 -20045,2006,0,7.40306109109009,4.6047901137698,3.68383209101584 -20045,2007,0,7.4265490723973,4.6047901137698,4.6047901137698 -20047,2003,0,3.66356164612965,1.2380843339547,0.247616866790941 -20047,2004,0,3.80666248977032,1.2380843339547,0.495233733581882 -20047,2005,0,3.63758615972639,1.2380843339547,0.742850600372822 -20047,2006,0,2.94443897916644,1.2380843339547,0.990467467163763 -20047,2007,0,3.8286413964891,1.2380843339547,1.2380843339547 -20063,2003,0,4.26267987704132,1.12102588350501,0.224205176701002 -20063,2004,0,4.18965474202643,1.12102588350501,0.448410353402004 -20063,2005,0,4.00733318523247,1.12102588350501,0.672615530103006 -20063,2006,0,4.04305126783455,1.12102588350501,0.896820706804008 -20063,2007,0,3.97029191355212,1.12102588350501,1.12102588350501 -20085,2003,0,5.4380793089232,2.53821042181031,0.507642084362062 -20085,2004,0,5.37527840768417,2.53821042181031,1.01528416872412 -20085,2005,0,5.3890717298165,2.53821042181031,1.52292625308619 -20085,2006,0,5.35185813347607,2.53821042181031,2.03056833744825 -20085,2007,0,5.40717177146012,2.53821042181031,2.53821042181031 -20095,2003,0,4.92725368515721,2.16021475170232,0.432042950340464 -20095,2004,0,4.91265488573605,2.16021475170232,0.864085900680928 -20095,2005,0,5.03695260241363,2.16021475170232,1.29612885102139 -20095,2006,0,4.969813299576,2.16021475170232,1.72817180136186 -20095,2007,0,5.05624580534831,2.16021475170232,2.16021475170232 -20103,2003,0,6.59578051396131,4.22961818627967,0.845923637255934 -20103,2004,0,6.58892647753352,4.22961818627967,1.69184727451187 -20103,2005,0,6.46458830368996,4.22961818627967,2.5377709117678 -20103,2006,0,6.49828214947643,4.22961818627967,3.38369454902374 -20103,2007,0,6.45676965557216,4.22961818627967,4.22961818627967 -20133,2003,0,6.11146733950268,2.83303685789521,0.566607371579043 -20133,2004,0,5.92157841964382,2.83303685789521,1.13321474315809 -20133,2005,0,5.82008293035236,2.83303685789521,1.69982211473713 -20133,2006,0,5.84354441703136,2.83303685789521,2.26642948631617 -20133,2007,0,5.72358510195238,2.83303685789521,2.83303685789521 -20137,2003,0,4.7361984483945,1.78389529417125,0.356779058834251 -20137,2004,0,4.70953020131233,1.78389529417125,0.713558117668501 -20137,2005,0,4.85203026391962,1.78389529417125,1.07033717650275 -20137,2006,0,4.79579054559674,1.78389529417125,1.427116235337 -20137,2007,0,4.64439089914137,1.78389529417125,1.78389529417125 -20143,2003,0,3.95124371858143,1.81856367198011,0.363712734396021 -20143,2004,0,3.98898404656427,1.81856367198011,0.727425468792043 -20143,2005,0,3.98898404656427,1.81856367198011,1.09113820318806 -20143,2006,0,4.18965474202643,1.81856367198011,1.45485093758409 -20143,2007,0,4.06044301054642,1.81856367198011,1.81856367198011 -20145,2003,0,4.49980967033027,1.97865388786714,0.395730777573429 -20145,2004,0,4.48863636973214,1.97865388786714,0.791461555146857 -20145,2005,0,4.63472898822964,1.97865388786714,1.18719233272029 -20145,2006,0,4.80402104473326,1.97865388786714,1.58292311029371 -20145,2007,0,4.70048036579242,1.97865388786714,1.97865388786714 -20171,2003,0,5.03043792139244,1.63315443905142,0.326630887810283 -20171,2004,0,4.91265488573605,1.63315443905142,0.653261775620567 -20171,2005,0,4.81218435537242,1.63315443905142,0.97989266343085 -20171,2006,0,4.8283137373023,1.63315443905142,1.30652355124113 -20171,2007,0,4.79579054559674,1.63315443905142,1.63315443905142 -20183,2003,0,3.97029191355212,1.51204556642545,0.30240911328509 -20183,2004,0,3.93182563272433,1.51204556642545,0.60481822657018 -20183,2005,0,3.63758615972639,1.51204556642545,0.90722733985527 -20183,2006,0,4.17438726989564,1.51204556642545,1.20963645314036 -20183,2007,0,4.38202663467388,1.51204556642545,1.51204556642545 -20205,2003,0,4.79579054559674,2.33524587523359,0.467049175046719 -20205,2004,0,4.79579054559674,2.33524587523359,0.934098350093438 -20205,2005,0,4.81218435537242,2.33524587523359,1.40114752514016 -20205,2006,0,4.94875989037817,2.33524587523359,1.86819670018688 -20205,2007,0,5.16478597392351,2.33524587523359,2.33524587523359 -22007,2003,0,4.77912349311153,3.15222307031342,0.630444614062685 -22007,2004,0,4.83628190695148,3.15222307031342,1.26088922812537 -22007,2005,0,4.57471097850338,3.15222307031342,1.89133384218805 -22007,2006,0,4.91998092582813,3.15222307031342,2.52177845625074 -22007,2007,0,5.07517381523383,3.15222307031342,3.15222307031342 -22033,2003,0,8.74957394808293,6.02308917522617,1.20461783504523 -22033,2004,0,8.68016201969438,6.02308917522617,2.40923567009047 -22033,2005,0,8.68440111040014,6.02308917522617,3.6138535051357 -22033,2006,0,8.78094111357239,6.02308917522617,4.81847134018094 -22033,2007,0,8.65782432115598,6.02308917522617,6.02308917522617 -22045,2003,0,6.65415252018322,4.29409665407894,0.858819330815788 -22045,2004,0,6.58063913728495,4.29409665407894,1.71763866163158 -22045,2005,0,6.58202513889283,4.29409665407894,2.57645799244736 -22045,2006,0,6.76272950693188,4.29409665407894,3.43527732326315 -22045,2007,0,6.77992190747225,4.29409665407894,4.29409665407894 -22069,2003,0,5.64190707093811,3.66561082717458,0.733122165434916 -22069,2004,0,5.53733426701854,3.66561082717458,1.46624433086983 -22069,2005,0,5.40717177146012,3.66561082717458,2.19936649630475 -22069,2006,0,5.53733426701854,3.66561082717458,2.93248866173967 -22069,2007,0,5.78689738136671,3.66561082717458,3.66561082717458 -22071,2003,0,8.31164394850298,6.18347649998489,1.23669529999698 -22071,2004,0,8.1371033896393,6.18347649998489,2.47339059999396 -22071,2005,0,8.14206328310415,6.18347649998489,3.71008589999093 -22071,2006,0,7.1785454837637,6.18347649998489,4.94678119998791 -22071,2007,0,7.75147531802146,6.18347649998489,6.18347649998489 -22073,2003,0,7.51043055637801,4.9921318225317,0.998426364506339 -22073,2004,0,7.35115822643069,4.9921318225317,1.99685272901268 -22073,2005,0,7.40367029001237,4.9921318225317,2.99527909351902 -22073,2006,0,7.45240245122364,4.9921318225317,3.99370545802536 -22073,2007,0,7.50988306115491,4.9921318225317,4.9921318225317 -22081,2003,0,4.02535169073515,2.26405214327726,0.452810428655452 -22081,2004,0,3.78418963391826,2.26405214327726,0.905620857310905 -22081,2005,0,3.36729582998647,2.26405214327726,1.35843128596636 -22081,2006,0,3.3322045101752,2.26405214327726,1.81124171462181 -22081,2007,0,3.3322045101752,2.26405214327726,2.26405214327726 -22087,2003,0,6.74051935960622,4.20810470205826,0.841620940411652 -22087,2004,0,6.58892647753352,4.20810470205826,1.6832418808233 -22087,2005,0,6.5792512120101,4.20810470205826,2.52486282123496 -22087,2006,0,4.45434729625351,4.20810470205826,3.36648376164661 -22087,2007,0,5.21493575760899,4.20810470205826,4.20810470205826 -22091,2003,0,3.2188758248682,2.35375337956845,0.470750675913689 -22091,2004,0,3.55534806148941,2.35375337956845,0.941501351827378 -22091,2005,0,3.17805383034795,2.35375337956845,1.41225202774107 -22091,2006,0,3.3322045101752,2.35375337956845,1.88300270365476 -22091,2007,0,3.36729582998647,2.35375337956845,2.35375337956845 -22111,2003,0,5.05624580534831,3.12689210625201,0.625378421250403 -22111,2004,0,4.69134788222914,3.12689210625201,1.25075684250081 -22111,2005,0,5.05624580534831,3.12689210625201,1.87613526375121 -22111,2006,0,4.82028156560504,3.12689210625201,2.50151368500161 -22111,2007,0,4.88280192258637,3.12689210625201,3.12689210625201 -22115,2003,0,5.605802066296,3.96140347152558,0.792280694305117 -22115,2004,0,5.64544689764324,3.96140347152558,1.58456138861023 -22115,2005,0,5.65599181081985,3.96140347152558,2.37684208291535 -22115,2006,0,5.74620319054015,3.96140347152558,3.16912277722047 -22115,2007,0,5.78382518232974,3.96140347152558,3.96140347152558 -22119,2003,0,5.6970934865054,3.73363769145571,0.746727538291142 -22119,2004,0,5.46806014113513,3.73363769145571,1.49345507658228 -22119,2005,0,5.61677109766657,3.73363769145571,2.24018261487343 -22119,2006,0,5.57594910314632,3.73363769145571,2.98691015316457 -22119,2007,0,5.81413053182507,3.73363769145571,3.73363769145571 -24009,2003,2007,7.02553831463852,4.31164440561939,0.862328881123878 -24009,2004,2007,7.14677217945264,4.31164440561939,1.72465776224776 -24009,2005,2007,7.16162200293919,4.31164440561939,2.58698664337163 -24009,2006,2007,7.1066061377273,4.31164440561939,3.44931552449551 -24009,2007,2007,7.1929342212158,4.31164440561939,4.31164440561939 -24015,2003,2007,7.04053639021596,4.4537773664325,0.890755473286499 -24015,2004,2007,7.15305163493748,4.4537773664325,1.781510946573 -24015,2005,2007,7.14440718032114,4.4537773664325,2.6722664198595 -24015,2006,2007,7.18841273649695,4.4537773664325,3.563021893146 -24015,2007,2007,7.11151211649616,4.4537773664325,4.4537773664325 -24017,2003,2007,7.68017564043659,4.79203142282408,0.958406284564816 -24017,2004,2007,7.69256964806791,4.79203142282408,1.91681256912963 -24017,2005,2007,7.69074316354187,4.79203142282408,2.87521885369445 -24017,2006,2007,7.75491027202143,4.79203142282408,3.83362513825926 -24017,2007,2007,7.75619534394812,4.79203142282408,4.79203142282408 -24021,2003,2007,8.31016902198191,5.27441906341037,1.05488381268207 -24021,2004,2007,8.36264243156764,5.27441906341037,2.10976762536415 -24021,2005,2007,8.30819906320645,5.27441906341037,3.16465143804622 -24021,2006,2007,8.32748441618826,5.27441906341037,4.2195352507283 -24021,2007,2007,8.28803156777646,5.27441906341037,5.27441906341037 -24027,2003,2007,8.76826314537129,5.5127914463584,1.10255828927168 -24027,2004,2007,8.74273386733017,5.5127914463584,2.20511657854336 -24027,2005,2007,8.67077227934454,5.5127914463584,3.30767486781504 -24027,2006,2007,8.71719093322313,5.5127914463584,4.41023315708672 -24027,2007,2007,8.75131624677346,5.5127914463584,5.5127914463584 -24033,2003,2007,9.10830764641103,6.68650368678403,1.33730073735681 -24033,2004,2007,9.14034700015565,6.68650368678403,2.67460147471361 -24033,2005,2007,9.17232692977797,6.68650368678403,4.01190221207042 -24033,2006,2007,9.17926241640532,6.68650368678403,5.34920294942722 -24033,2007,2007,9.15567297063387,6.68650368678403,6.68650368678403 -26005,2003,2007,7.1693500166706,4.66027371221412,0.932054742442823 -26005,2004,2007,7.15773548424991,4.66027371221412,1.86410948488565 -26005,2005,2007,7.10578612948127,4.66027371221412,2.79616422732847 -26005,2006,2007,7.05185562295589,4.66027371221412,3.72821896977129 -26005,2007,2007,6.98749024700099,4.66027371221412,4.66027371221412 -26015,2003,2007,6.34388043412633,4.0387437582073,0.807748751641461 -26015,2004,2007,6.25190388316589,4.0387437582073,1.61549750328292 -26015,2005,2007,6.28599809450886,4.0387437582073,2.42324625492438 -26015,2006,2007,6.18001665365257,4.0387437582073,3.23099500656584 -26015,2007,2007,6.19440539110467,4.0387437582073,4.0387437582073 -26027,2003,2007,5.42053499927229,3.9338627720326,0.78677255440652 -26027,2004,2007,5.62401750618734,3.9338627720326,1.57354510881304 -26027,2005,2007,5.72358510195238,3.9338627720326,2.36031766321956 -26027,2006,2007,5.58724865840025,3.9338627720326,3.14709021762608 -26027,2007,2007,5.27299955856375,3.9338627720326,3.9338627720326 -26031,2003,2007,5.27811465923052,3.27518054107867,0.655036108215734 -26031,2004,2007,5.12989871492307,3.27518054107867,1.31007221643147 -26031,2005,2007,5.12989871492307,3.27518054107867,1.9651083246472 -26031,2006,2007,5.11198778835654,3.27518054107867,2.62014443286293 -26031,2007,2007,4.99721227376411,3.27518054107867,3.27518054107867 -26041,2003,2007,6.31173480915291,3.65117758692992,0.730235517385985 -26041,2004,2007,6.32076829425058,3.65117758692992,1.46047103477197 -26041,2005,2007,6.25766758788264,3.65117758692992,2.19070655215795 -26041,2006,2007,6.25382881157547,3.65117758692992,2.92094206954394 -26041,2007,2007,6.13988455222626,3.65117758692992,3.65117758692992 -26057,2003,2007,6.13988455222626,3.74443241323419,0.748886482646837 -26057,2004,2007,6.09582456243222,3.74443241323419,1.49777296529367 -26057,2005,2007,6.25190388316589,3.74443241323419,2.24665944794051 -26057,2006,2007,6.12249280951439,3.74443241323419,2.99554593058735 -26057,2007,2007,5.92692602597041,3.74443241323419,3.74443241323419 -26061,2003,2007,6.42648845745769,3.58396328416438,0.716792656832875 -26061,2004,2007,6.40025744530882,3.58396328416438,1.43358531366575 -26061,2005,2007,6.33859407820318,3.58396328416438,2.15037797049863 -26061,2006,2007,6.22851100359118,3.58396328416438,2.8671706273315 -26061,2007,2007,6.13556489108174,3.58396328416438,3.58396328416438 -26065,2003,2007,8.4142741374084,5.63235807797783,1.12647161559557 -26065,2004,2007,8.33254893925264,5.63235807797783,2.25294323119113 -26065,2005,2007,8.24643378616036,5.63235807797783,3.3794148467867 -26065,2006,2007,8.21283958467648,5.63235807797783,4.50588646238227 -26065,2007,2007,8.16365617616843,5.63235807797783,5.63235807797783 -26073,2003,2007,6.73933662735717,4.14869069196221,0.829738138392443 -26073,2004,2007,6.59850902861452,4.14869069196221,1.65947627678489 -26073,2005,2007,6.68586094706836,4.14869069196221,2.48921441517733 -26073,2006,2007,6.53087762772588,4.14869069196221,3.31895255356977 -26073,2007,2007,6.51025834052315,4.14869069196221,4.14869069196221 -26085,2003,2007,3.66356164612965,2.42771882375081,0.485543764750162 -26085,2004,2007,3.40119738166216,2.42771882375081,0.971087529500325 -26085,2005,2007,3.63758615972639,2.42771882375081,1.45663129425049 -26085,2006,2007,3.2188758248682,2.42771882375081,1.94217505900065 -26085,2007,2007,3.36729582998647,2.42771882375081,2.42771882375081 -26089,2003,2007,5.64190707093811,3.05017310923224,0.610034621846448 -26089,2004,2007,5.52942908751142,3.05017310923224,1.2200692436929 -26089,2005,2007,5.47227067367148,3.05017310923224,1.83010386553934 -26089,2006,2007,5.28320372873799,3.05017310923224,2.44013848738579 -26089,2007,2007,5.34233425196481,3.05017310923224,3.05017310923224 -26105,2003,2007,5.75890177387728,3.34194265444484,0.668388530888968 -26105,2004,2007,5.59842195899838,3.34194265444484,1.33677706177794 -26105,2005,2007,5.75574221358691,3.34194265444484,2.0051655926669 -26105,2006,2007,5.61312810638807,3.34194265444484,2.67355412355587 -26105,2007,2007,5.52942908751142,3.34194265444484,3.34194265444484 -26127,2003,2007,5.52942908751142,3.29112206507397,0.658224413014795 -26127,2004,2007,5.51745289646471,3.29112206507397,1.31644882602959 -26127,2005,2007,5.54907608489522,3.29112206507397,1.97467323904438 -26127,2006,2007,5.39362754635236,3.29112206507397,2.63289765205918 -26127,2007,2007,5.37527840768417,3.29112206507397,3.29112206507397 -26129,2003,2007,5.74300318780948,3.07477448089394,0.614954896178789 -26129,2004,2007,5.75574221358691,3.07477448089394,1.22990979235758 -26129,2005,2007,5.75890177387728,3.07477448089394,1.84486468853637 -26129,2006,2007,5.5683445037611,3.07477448089394,2.45981958471515 -26129,2007,2007,5.52942908751142,3.07477448089394,3.07477448089394 -26137,2003,2007,6.2363695902037,3.14849627810562,0.629699255621124 -26137,2004,2007,6.22851100359118,3.14849627810562,1.25939851124225 -26137,2005,2007,6.18620862390049,3.14849627810562,1.88909776686337 -26137,2006,2007,6.10479323241498,3.14849627810562,2.5187970224845 -26137,2007,2007,5.94542060860658,3.14849627810562,3.14849627810562 -26149,2003,2007,6.62273632394984,4.13391777734183,0.826783555468365 -26149,2004,2007,6.45204895443723,4.13391777734183,1.65356711093673 -26149,2005,2007,6.50727771238501,4.13391777734183,2.4803506664051 -26149,2006,2007,6.41836493593621,4.13391777734183,3.30713422187346 -26149,2007,2007,6.36990098282823,4.13391777734183,4.13391777734183 -27005,2003,2006,6.84374994900622,3.40119738166216,0.680239476332431 -27005,2004,2006,6.76157276880406,3.40119738166216,1.36047895266486 -27005,2005,2006,6.78671695060508,3.40119738166216,2.04071842899729 -27005,2006,2006,6.96790920180188,3.40119738166216,2.72095790532972 -27005,2007,2006,6.91075078796194,3.40119738166216,3.40119738166216 -27009,2003,2006,6.52356230614951,3.53298558915526,0.706597117831052 -27009,2004,2006,6.45362499889269,3.53298558915526,1.4131942356621 -27009,2005,2006,6.47697236288968,3.53298558915526,2.11979135349316 -27009,2006,2006,6.51767127291227,3.53298558915526,2.82638847132421 -27009,2007,2006,6.41673228251233,3.53298558915526,3.53298558915526 -27023,2003,2006,5.79301360838414,2.57169577986039,0.514339155972078 -27023,2004,2006,5.65599181081985,2.57169577986039,1.02867831194416 -27023,2005,2006,5.79909265446053,2.57169577986039,1.54301746791623 -27023,2006,2006,5.6970934865054,2.57169577986039,2.05735662388831 -27023,2007,2006,5.72358510195238,2.57169577986039,2.57169577986039 -27035,2003,2006,7.4899708988348,4.00913156717385,0.80182631343477 -27035,2004,2006,7.22983877815125,4.00913156717385,1.60365262686954 -27035,2005,2006,7.30114780585603,4.00913156717385,2.40547894030431 -27035,2006,2006,7.39510754656249,4.00913156717385,3.20730525373908 -27035,2007,2006,7.22329567956231,4.00913156717385,4.00913156717385 -27043,2003,2006,5.57594910314632,2.78383771441767,0.556767542883534 -27043,2004,2006,5.39362754635236,2.78383771441767,1.11353508576707 -27043,2005,2006,5.54907608489522,2.78383771441767,1.6703026286506 -27043,2006,2006,5.46383180502561,2.78383771441767,2.22707017153414 -27043,2007,2006,5.41164605185504,2.78383771441767,2.78383771441767 -27049,2003,2006,7.2991214627108,3.78707184000532,0.757414368001063 -27049,2004,2006,7.03174125876313,3.78707184000532,1.51482873600213 -27049,2005,2006,6.93731408122368,3.78707184000532,2.27224310400319 -27049,2006,2006,6.96979066990159,3.78707184000532,3.02965747200425 -27049,2007,2006,6.95844839329766,3.78707184000532,3.78707184000532 -27051,2003,2006,4.54329478227,1.83880207556137,0.367760415112273 -27051,2004,2006,4.49980967033027,1.83880207556137,0.735520830224547 -27051,2005,2006,4.33073334028633,1.83880207556137,1.10328124533682 -27051,2006,2006,4.44265125649032,1.83880207556137,1.47104166044909 -27051,2007,2006,4.06044301054642,1.83880207556137,1.83880207556137 -27055,2003,2006,5.77765232322266,2.98153192415285,0.59630638483057 -27055,2004,2006,5.74300318780948,2.98153192415285,1.19261276966114 -27055,2005,2006,5.73979291217923,2.98153192415285,1.78891915449171 -27055,2006,2006,5.71373280550937,2.98153192415285,2.38522553932228 -27055,2007,2006,5.66296048013595,2.98153192415285,2.98153192415285 -27091,2003,2006,6.30991827822652,3.08200170870617,0.616400341741233 -27091,2004,2006,6.28785856016178,3.08200170870617,1.23280068348247 -27091,2005,2006,6.3578422665081,3.08200170870617,1.8492010252237 -27091,2006,2006,6.40522845803084,3.08200170870617,2.46560136696493 -27091,2007,2006,6.3456363608286,3.08200170870617,3.08200170870617 -27115,2003,2006,6.12686918411419,3.27827616814965,0.65565523362993 -27115,2004,2006,6.12468339089421,3.27827616814965,1.31131046725986 -27115,2005,2006,6.09582456243222,3.27827616814965,1.96696570088979 -27115,2006,2006,5.96614673912369,3.27827616814965,2.62262093451972 -27115,2007,2006,6.00388706710654,3.27827616814965,3.27827616814965 -27123,2003,2006,9.2617936535651,6.23643808100884,1.24728761620177 -27123,2004,2006,9.18450961194166,6.23643808100884,2.49457523240354 -27123,2005,2006,9.13841463240459,6.23643808100884,3.7418628486053 -27123,2006,2006,9.19146331620649,6.23643808100884,4.98915046480707 -27123,2007,2006,9.11294802596753,6.23643808100884,6.23643808100884 -27129,2003,2006,5.77455154554441,2.84223138256823,0.568446276513646 -27129,2004,2006,5.72358510195238,2.84223138256823,1.13689255302729 -27129,2005,2006,5.75257263882563,2.84223138256823,1.70533882954094 -27129,2006,2006,5.64190707093811,2.84223138256823,2.27378510605458 -27129,2007,2006,5.87773578177964,2.84223138256823,2.84223138256823 -27149,2003,2006,5.35185813347607,2.30787109742328,0.461574219484657 -27149,2004,2006,5.14166355650266,2.30787109742328,0.923148438969313 -27149,2005,2006,5.2040066870768,2.30787109742328,1.38472265845397 -27149,2006,2006,5.17614973257383,2.30787109742328,1.84629687793863 -27149,2007,2006,5.35185813347607,2.30787109742328,2.30787109742328 -27157,2003,2006,6.04737217904628,3.07315617051879,0.614631234103759 -27157,2004,2006,5.95324333428778,3.07315617051879,1.22926246820752 -27157,2005,2006,5.89164421182577,3.07315617051879,1.84389370231128 -27157,2006,2006,5.92157841964382,3.07315617051879,2.45852493641504 -27157,2007,2006,5.91620206260743,3.07315617051879,3.07315617051879 -27163,2003,2006,8.27231514795602,5.30395146516513,1.06079029303303 -27163,2004,2006,8.21419441485256,5.30395146516513,2.12158058606605 -27163,2005,2006,8.17075142375753,5.30395146516513,3.18237087909908 -27163,2006,2006,8.25035895147729,5.30395146516513,4.2431611721321 -27163,2007,2006,8.20303024171486,5.30395146516513,5.30395146516513 -27165,2003,2006,5.48479693349065,2.47451955690151,0.494903911380303 -27165,2004,2006,5.40267738187228,2.47451955690151,0.989807822760605 -27165,2005,2006,5.30330490805908,2.47451955690151,1.48471173414091 -27165,2006,2006,5.41610040220442,2.47451955690151,1.97961564552121 -27165,2007,2006,5.38449506278909,2.47451955690151,2.47451955690151 -29003,2003,2007,4.83628190695148,2.80287541484465,0.560575082968931 -29003,2004,2007,4.85981240436167,2.80287541484465,1.12115016593786 -29003,2005,2007,4.87519732320115,2.80287541484465,1.68172524890679 -29003,2006,2007,4.88280192258637,2.80287541484465,2.24230033187572 -29003,2007,2007,5.01063529409626,2.80287541484465,2.80287541484465 -29011,2003,2007,5.51342874616498,2.52900327684191,0.505800655368383 -29011,2004,2007,5.51745289646471,2.52900327684191,1.01160131073677 -29011,2005,2007,5.40267738187228,2.52900327684191,1.51740196610515 -29011,2006,2007,5.30330490805908,2.52900327684191,2.02320262147353 -29011,2007,2007,5.25749537202778,2.52900327684191,2.52900327684191 -29013,2003,2007,5.36129216570943,2.81259038037613,0.562518076075227 -29013,2004,2007,5.25227342804663,2.81259038037613,1.12503615215045 -29013,2005,2007,5.32787616878958,2.81259038037613,1.68755422822568 -29013,2006,2007,5.18738580584075,2.81259038037613,2.25007230430091 -29013,2007,2007,5.12396397940326,2.81259038037613,2.81259038037613 -29023,2003,2007,6.40025744530882,3.71032289140759,0.742064578281518 -29023,2004,2007,6.51174532964473,3.71032289140759,1.48412915656304 -29023,2005,2007,6.53813982376767,3.71032289140759,2.22619373484455 -29023,2006,2007,6.41345895716736,3.71032289140759,2.96825831312607 -29023,2007,2007,6.29341927884648,3.71032289140759,3.71032289140759 -29033,2003,2007,4.82028156560504,2.33068652310492,0.466137304620984 -29033,2004,2007,4.80402104473326,2.33068652310492,0.932274609241968 -29033,2005,2007,4.65396035015752,2.33068652310492,1.39841191386295 -29033,2006,2007,4.69134788222914,2.33068652310492,1.86454921848394 -29033,2007,2007,4.8283137373023,2.33068652310492,2.33068652310492 -29041,2003,2007,4.71849887129509,2.13274531370215,0.426549062740429 -29041,2004,2007,4.65396035015752,2.13274531370215,0.853098125480859 -29041,2005,2007,4.64439089914137,2.13274531370215,1.27964718822129 -29041,2006,2007,4.36944785246702,2.13274531370215,1.70619625096172 -29041,2007,2007,4.4188406077966,2.13274531370215,2.13274531370215 -29065,2003,2007,5.07517381523383,2.7031716536511,0.540634330730221 -29065,2004,2007,5.13579843705026,2.7031716536511,1.08126866146044 -29065,2005,2007,5.11799381241676,2.7031716536511,1.62190299219066 -29065,2006,2007,5.09375020080676,2.7031716536511,2.16253732292088 -29065,2007,2007,5.02388052084628,2.7031716536511,2.7031716536511 -29073,2003,2007,5.8111409929767,2.7305941655371,0.546118833107421 -29073,2004,2007,5.76205138278018,2.7305941655371,1.09223766621484 -29073,2005,2007,5.87211778947542,2.7305941655371,1.63835649932226 -29073,2006,2007,5.94542060860658,2.7305941655371,2.18447533242968 -29073,2007,2007,5.8805329864007,2.7305941655371,2.7305941655371 -29077,2003,2007,8.44827174594982,5.48226676435625,1.09645335287125 -29077,2004,2007,8.39117635083275,5.48226676435625,2.1929067057425 -29077,2005,2007,8.39976009452414,5.48226676435625,3.28936005861375 -29077,2006,2007,8.46779284112111,5.48226676435625,4.385813411485 -29077,2007,2007,8.44891435066294,5.48226676435625,5.48226676435625 -29079,2003,2007,4.94875989037817,2.3448780051843,0.468975601036859 -29079,2004,2007,4.84418708645859,2.3448780051843,0.937951202073719 -29079,2005,2007,4.969813299576,2.3448780051843,1.40692680311058 -29079,2006,2007,4.8283137373023,2.3448780051843,1.87590240414744 -29079,2007,2007,4.8283137373023,2.3448780051843,2.3448780051843 -29081,2003,2007,5.11198778835654,2.18041745901984,0.436083491803968 -29081,2004,2007,5.06259503302697,2.18041745901984,0.872166983607935 -29081,2005,2007,5.17048399503815,2.18041745901984,1.3082504754119 -29081,2006,2007,4.94875989037817,2.18041745901984,1.74433396721587 -29081,2007,2007,4.90527477843843,2.18041745901984,2.18041745901984 -29085,2003,2007,3.46573590279973,2.19053558918542,0.438107117837085 -29085,2004,2007,3.25809653802148,2.19053558918542,0.876214235674169 -29085,2005,2007,3.3322045101752,2.19053558918542,1.31432135351125 -29085,2006,2007,3.40119738166216,2.19053558918542,1.75242847134834 -29085,2007,2007,3.49650756146648,2.19053558918542,2.19053558918542 -29089,2003,2007,4.70048036579242,2.32356349937924,0.464712699875848 -29089,2004,2007,4.7361984483945,2.32356349937924,0.929425399751695 -29089,2005,2007,4.70953020131233,2.32356349937924,1.39413809962754 -29089,2006,2007,4.69134788222914,2.32356349937924,1.85885079950339 -29089,2007,2007,4.62497281328427,2.32356349937924,2.32356349937924 -29099,2003,2007,7.80628928926703,5.28876690573619,1.05775338114724 -29099,2004,2007,7.78572089653462,5.28876690573619,2.11550676229447 -29099,2005,2007,7.74283595543075,5.28876690573619,3.17326014344171 -29099,2006,2007,7.73848812249465,5.28876690573619,4.23101352458895 -29099,2007,2007,7.7367436824535,5.28876690573619,5.28876690573619 -29101,2003,2007,6.07993319509559,3.87656161714999,0.775312323429997 -29101,2004,2007,6.14203740558736,3.87656161714999,1.55062464685999 -29101,2005,2007,6.20050917404269,3.87656161714999,2.32593697028999 -29101,2006,2007,6.16120732169508,3.87656161714999,3.10124929371999 -29101,2007,2007,6.18414889093748,3.87656161714999,3.87656161714999 -29105,2003,2007,6.13556489108174,3.48164000935702,0.696328001871404 -29105,2004,2007,6.0591231955818,3.48164000935702,1.39265600374281 -29105,2005,2007,6.1463292576689,3.48164000935702,2.08898400561421 -29105,2006,2007,6.28226674689601,3.48164000935702,2.78531200748562 -29105,2007,2007,6.27476202124194,3.48164000935702,3.48164000935702 -29107,2003,2007,6.1527326947041,3.49529470504127,0.699058941008254 -29107,2004,2007,6.19236248947487,3.49529470504127,1.39811788201651 -29107,2005,2007,6.1463292576689,3.49529470504127,2.09717682302476 -29107,2006,2007,6.09582456243222,3.49529470504127,2.79623576403302 -29107,2007,2007,6.12029741895095,3.49529470504127,3.49529470504127 -29117,2003,2007,5.7268477475872,2.67814067069607,0.535628134139213 -29117,2004,2007,5.75574221358691,2.67814067069607,1.07125626827843 -29117,2005,2007,5.64544689764324,2.67814067069607,1.60688440241764 -29117,2006,2007,5.67675380226828,2.67814067069607,2.14251253655685 -29117,2007,2007,5.54517744447956,2.67814067069607,2.67814067069607 -29119,2003,2007,5.35658627467201,3.07643630096895,0.615287260193791 -29119,2004,2007,5.23110861685459,3.07643630096895,1.23057452038758 -29119,2005,2007,5.16478597392351,3.07643630096895,1.84586178058137 -29119,2006,2007,5.28320372873799,3.07643630096895,2.46114904077516 -29119,2007,2007,5.33753807970132,3.07643630096895,3.07643630096895 -29127,2003,2007,6.39359075395063,3.34247303650915,0.668494607301829 -29127,2004,2007,6.33150184989369,3.34247303650915,1.33698921460366 -29127,2005,2007,6.42324696353352,3.34247303650915,2.00548382190549 -29127,2006,2007,6.39692965521615,3.34247303650915,2.67397842920732 -29127,2007,2007,6.36818718635049,3.34247303650915,3.34247303650915 -29131,2003,2007,5.52942908751142,3.15972012365944,0.631944024731888 -29131,2004,2007,5.34233425196481,3.15972012365944,1.26388804946378 -29131,2005,2007,5.34710753071747,3.15972012365944,1.89583207419566 -29131,2006,2007,5.41610040220442,3.15972012365944,2.52777609892755 -29131,2007,2007,5.38449506278909,3.15972012365944,3.15972012365944 -29149,2003,2007,4.21950770517611,2.33640664146956,0.467281328293911 -29149,2004,2007,4.18965474202643,2.33640664146956,0.934562656587823 -29149,2005,2007,4.35670882668959,2.33640664146956,1.40184398488173 -29149,2006,2007,4.35670882668959,2.33640664146956,1.86912531317565 -29149,2007,2007,4.51085950651685,2.33640664146956,2.33640664146956 -29155,2003,2007,5.4971682252932,2.99807951662234,0.599615903324468 -29155,2004,2007,5.34233425196481,2.99807951662234,1.19923180664894 -29155,2005,2007,5.26269018890489,2.99807951662234,1.7988477099734 -29155,2006,2007,5.45958551414416,2.99807951662234,2.39846361329787 -29155,2007,2007,5.55295958492162,2.99807951662234,2.99807951662234 -29163,2003,2007,5.605802066296,2.90968406892854,0.581936813785707 -29163,2004,2007,5.69035945432406,2.90968406892854,1.16387362757141 -29163,2005,2007,5.72031177660741,2.90968406892854,1.74581044135712 -29163,2006,2007,5.7037824746562,2.90968406892854,2.32774725514283 -29163,2007,2007,5.63835466933375,2.90968406892854,2.90968406892854 -29177,2003,2007,5.48063892334199,3.15076827565352,0.630153655130705 -29177,2004,2007,5.26785815906333,3.15076827565352,1.26030731026141 -29177,2005,2007,5.28320372873799,3.15076827565352,1.89046096539211 -29177,2006,2007,5.18738580584075,3.15076827565352,2.52061462052282 -29177,2007,2007,5.22035582507832,3.15076827565352,3.15076827565352 -29201,2003,2007,6.08904487544685,3.69937419120658,0.739874838241316 -29201,2004,2007,6.19031540585315,3.69937419120658,1.47974967648263 -29201,2005,2007,6.15060276844628,3.69937419120658,2.21962451472395 -29201,2006,2007,6.09807428216624,3.69937419120658,2.95949935296526 -29201,2007,2007,6.00635315960173,3.69937419120658,3.69937419120658 -29209,2003,2007,5.00394630594546,3.35543263629441,0.671086527258882 -29209,2004,2007,5.19849703126583,3.35543263629441,1.34217305451776 -29209,2005,2007,5.19849703126583,3.35543263629441,2.01325958177665 -29209,2006,2007,5.23644196282995,3.35543263629441,2.68434610903553 -29209,2007,2007,5.32300997913841,3.35543263629441,3.35543263629441 -29211,2003,2007,4.26267987704132,1.97671643915702,0.395343287831405 -29211,2004,2007,4.24849524204936,1.97671643915702,0.79068657566281 -29211,2005,2007,3.93182563272433,1.97671643915702,1.18602986349421 -29211,2006,2007,4.02535169073515,1.97671643915702,1.58137315132562 -29211,2007,2007,4.12713438504509,1.97671643915702,1.97671643915702 -29215,2003,2007,5.37989735354046,3.13562464220588,0.627124928441176 -29215,2004,2007,5.29831736654804,3.13562464220588,1.25424985688235 -29215,2005,2007,5.43372200355424,3.13562464220588,1.88137478532353 -29215,2006,2007,5.41610040220442,3.13562464220588,2.50849971376471 -29215,2007,2007,5.40267738187228,3.13562464220588,3.13562464220588 -29229,2003,2007,5.52942908751142,2.88786862767805,0.577573725535609 -29229,2004,2007,5.54517744447956,2.88786862767805,1.15514745107122 -29229,2005,2007,5.51745289646471,2.88786862767805,1.73272117660683 -29229,2006,2007,5.42053499927229,2.88786862767805,2.31029490214244 -29229,2007,2007,5.26269018890489,2.88786862767805,2.88786862767805 -29510,2003,2007,8.40200678160712,5.85274543579545,1.17054908715909 -29510,2004,2007,8.34616759436413,5.85274543579545,2.34109817431818 -29510,2005,2007,8.29704514908183,5.85274543579545,3.51164726147727 -29510,2006,2007,8.29679586577005,5.85274543579545,4.68219634863636 -29510,2007,2007,8.25816336153762,5.85274543579545,5.85274543579545 -30009,2003,2007,5.08140436498446,2.25675055665025,0.451350111330049 -30009,2004,2007,4.92725368515721,2.25675055665025,0.902700222660098 -30009,2005,2007,4.77912349311153,2.25675055665025,1.35405033399015 -30009,2006,2007,4.87519732320115,2.25675055665025,1.8054004453202 -30009,2007,2007,4.97673374242057,2.25675055665025,2.25675055665025 -30011,2003,2007,2.484906649788,0.307484699747961,0.0614969399495921 -30011,2004,2007,1.94591014905531,0.307484699747961,0.122993879899184 -30011,2005,2007,1.6094379124341,0.307484699747961,0.184490819848776 -30011,2006,2007,1.94591014905531,0.307484699747961,0.245987759798369 -30011,2007,2007,2.07944154167984,0.307484699747961,0.307484699747961 -30013,2003,2007,7.27793857294566,4.3864792072439,0.877295841448781 -30013,2004,2007,7.16626597413364,4.3864792072439,1.75459168289756 -30013,2005,2007,7.19142933003638,4.3864792072439,2.63188752434634 -30013,2006,2007,7.25981961036319,4.3864792072439,3.50918336579512 -30013,2007,2007,7.28138566357028,4.3864792072439,4.3864792072439 -30021,2003,2007,5.37063802812766,2.20375873868715,0.44075174773743 -30021,2004,2007,5.30826769740121,2.20375873868715,0.88150349547486 -30021,2005,2007,5.16478597392351,2.20375873868715,1.32225524321229 -30021,2006,2007,5.44241771052179,2.20375873868715,1.76300699094972 -30021,2007,2007,5.25227342804663,2.20375873868715,2.20375873868715 -30033,2003,2007,2.39789527279837,0.246078522596706,0.0492157045193411 -30033,2004,2007,2.89037175789616,0.246078522596706,0.0984314090386822 -30033,2005,2007,2.484906649788,0.246078522596706,0.147647113558023 -30033,2006,2007,2.83321334405622,0.246078522596706,0.196862818077364 -30033,2007,2007,2.89037175789616,0.246078522596706,0.246078522596706 -30049,2003,2007,7.12929754892937,4.02026735882578,0.804053471765157 -30049,2004,2007,7.0604763659998,4.02026735882578,1.60810694353031 -30049,2005,2007,7.00397413672268,4.02026735882578,2.41216041529547 -30049,2006,2007,7.09506437728713,4.02026735882578,3.21621388706063 -30049,2007,2007,7.09589322109753,4.02026735882578,4.02026735882578 -30081,2003,2007,6.21260609575152,3.58546149491545,0.71709229898309 -30081,2004,2007,6.21860011969173,3.58546149491545,1.43418459796618 -30081,2005,2007,6.18208490671663,3.58546149491545,2.15127689694927 -30081,2006,2007,6.26909628370626,3.58546149491545,2.86836919593236 -30081,2007,2007,6.27852142416584,3.58546149491545,3.58546149491545 -30087,2003,2007,5.35185813347607,2.23889954130791,0.447779908261583 -30087,2004,2007,5.32300997913841,2.23889954130791,0.895559816523165 -30087,2005,2007,5.11198778835654,2.23889954130791,1.34333972478475 -30087,2006,2007,5.0998664278242,2.23889954130791,1.79111963304633 -30087,2007,2007,4.969813299576,2.23889954130791,2.23889954130791 -30095,2003,2007,4.56434819146784,2.10352421219579,0.420704842439159 -30095,2004,2007,4.38202663467388,2.10352421219579,0.841409684878317 -30095,2005,2007,4.65396035015752,2.10352421219579,1.26211452731748 -30095,2006,2007,4.65396035015752,2.10352421219579,1.68281936975663 -30095,2007,2007,4.63472898822964,2.10352421219579,2.10352421219579 -30101,2003,2007,4.24849524204936,1.66146094050125,0.33229218810025 -30101,2004,2007,4.00733318523247,1.66146094050125,0.664584376200501 -30101,2005,2007,4.26267987704132,1.66146094050125,0.996876564300752 -30101,2006,2007,4.12713438504509,1.66146094050125,1.329168752401 -30101,2007,2007,4.29045944114839,1.66146094050125,1.66146094050125 -30109,2003,2007,2.70805020110221,0.0657877405380032,0.0131575481076006 -30109,2004,2007,2.63905732961526,0.0657877405380032,0.0263150962152013 -30109,2005,2007,2.19722457733622,0.0657877405380032,0.0394726443228019 -30109,2006,2007,2.19722457733622,0.0657877405380032,0.0526301924304025 -30109,2007,2007,2.30258509299405,0.0657877405380032,0.0657877405380032 -31003,2003,0,4.62497281328427,2.00848245273934,0.401696490547868 -31003,2004,0,4.44265125649032,2.00848245273934,0.803392981095737 -31003,2005,0,4.4188406077966,2.00848245273934,1.20508947164361 -31003,2006,0,4.55387689160054,2.00848245273934,1.60678596219147 -31003,2007,0,4.51085950651685,2.00848245273934,2.00848245273934 -31023,2003,0,4.76217393479776,2.17099467260645,0.43419893452129 -31023,2004,0,4.51085950651685,2.17099467260645,0.868397869042579 -31023,2005,0,4.70953020131233,2.17099467260645,1.30259680356387 -31023,2006,0,4.62497281328427,2.17099467260645,1.73679573808516 -31023,2007,0,4.77068462446567,2.17099467260645,2.17099467260645 -31031,2003,0,4.88280192258637,1.81612682567635,0.36322536513527 -31031,2004,0,5.01063529409626,1.81612682567635,0.72645073027054 -31031,2005,0,5.07517381523383,1.81612682567635,1.08967609540581 -31031,2006,0,4.96284463025991,1.81612682567635,1.45290146054108 -31031,2007,0,4.87519732320115,1.81612682567635,1.81612682567635 -31033,2003,0,5.49306144334055,2.28543893415908,0.457087786831815 -31033,2004,0,5.5683445037611,2.28543893415908,0.91417557366363 -31033,2005,0,5.5834963087817,2.28543893415908,1.37126336049545 -31033,2006,0,5.54126354515843,2.28543893415908,1.82835114732726 -31033,2007,0,5.59842195899838,2.28543893415908,2.28543893415908 -31041,2003,0,5.11198778835654,2.46750613510783,0.493501227021567 -31041,2004,0,5.04342511691925,2.46750613510783,0.987002454043133 -31041,2005,0,5.06890420222023,2.46750613510783,1.4805036810647 -31041,2006,0,5.21493575760899,2.46750613510783,1.97400490808627 -31041,2007,0,5.28826703069454,2.46750613510783,2.46750613510783 -31055,2003,0,9.36546200884348,6.13898975546329,1.22779795109266 -31055,2004,0,9.3321154289021,6.13898975546329,2.45559590218531 -31055,2005,0,9.31118068690225,6.13898975546329,3.68339385327797 -31055,2006,0,9.32197118814737,6.13898975546329,4.91119180437063 -31055,2007,0,9.32027043134838,6.13898975546329,6.13898975546329 -31059,2003,0,4.97673374242057,1.89220794052486,0.378441588104972 -31059,2004,0,4.77912349311153,1.89220794052486,0.756883176209944 -31059,2005,0,4.74493212836325,1.89220794052486,1.13532476431492 -31059,2006,0,4.70048036579242,1.89220794052486,1.51376635241989 -31059,2007,0,4.64439089914137,1.89220794052486,1.89220794052486 -31061,2003,0,3.71357206670431,1.27368541673724,0.254737083347447 -31061,2004,0,3.61091791264422,1.27368541673724,0.509474166694895 -31061,2005,0,3.40119738166216,1.27368541673724,0.764211250042342 -31061,2006,0,3.3322045101752,1.27368541673724,1.01894833338979 -31061,2007,0,3.61091791264422,1.27368541673724,1.27368541673724 -31073,2003,0,2.70805020110221,0.76220671649144,0.152441343298288 -31073,2004,0,2.77258872223978,0.76220671649144,0.304882686596576 -31073,2005,0,2.83321334405622,0.76220671649144,0.457324029894864 -31073,2006,0,2.70805020110221,0.76220671649144,0.609765373193152 -31073,2007,0,2.63905732961526,0.76220671649144,0.76220671649144 -31079,2003,0,7.1708884785125,3.98031696606713,0.796063393213427 -31079,2004,0,7.09920174355309,3.98031696606713,1.59212678642685 -31079,2005,0,7.06646697013696,3.98031696606713,2.38819017964028 -31079,2006,0,7.16549347506085,3.98031696606713,3.18425357285371 -31079,2007,0,7.17548971362422,3.98031696606713,3.98031696606713 -31081,2003,0,5.21493575760899,2.24102878729494,0.448205757458988 -31081,2004,0,5.24174701505964,2.24102878729494,0.896411514917976 -31081,2005,0,5.20948615284142,2.24102878729494,1.34461727237696 -31081,2006,0,5.15329159449778,2.24102878729494,1.79282302983595 -31081,2007,0,5.14749447681345,2.24102878729494,2.24102878729494 -31083,2003,0,3.85014760171006,1.33131005278713,0.266262010557426 -31083,2004,0,3.73766961828337,1.33131005278713,0.532524021114852 -31083,2005,0,3.89182029811063,1.33131005278713,0.798786031672279 -31083,2006,0,3.98898404656427,1.33131005278713,1.0650480422297 -31083,2007,0,3.85014760171006,1.33131005278713,1.33131005278713 -31101,2003,0,5.4249500174814,2.18323833536148,0.436647667072296 -31101,2004,0,5.30826769740121,2.18323833536148,0.873295334144592 -31101,2005,0,5.30330490805908,2.18323833536148,1.30994300121689 -31101,2006,0,5.23110861685459,2.18323833536148,1.74659066828918 -31101,2007,0,5.26269018890489,2.18323833536148,2.18323833536148 -31107,2003,0,5.04985600724954,2.23793989950051,0.447587979900103 -31107,2004,0,5.19295685089021,2.23793989950051,0.895175959800206 -31107,2005,0,5.08140436498446,2.23793989950051,1.34276393970031 -31107,2006,0,5.03695260241363,2.23793989950051,1.79035191960041 -31107,2007,0,5.14166355650266,2.23793989950051,2.23793989950051 -31109,2003,0,8.57828829077605,5.52262424093949,1.1045248481879 -31109,2004,0,8.52694548285892,5.52262424093949,2.2090496963758 -31109,2005,0,8.53168763756669,5.52262424093949,3.31357454456369 -31109,2006,0,8.52038808231276,5.52262424093949,4.41809939275159 -31109,2007,0,8.5571828396324,5.52262424093949,5.52262424093949 -31123,2003,0,4.46590811865458,1.69377906086785,0.33875581217357 -31123,2004,0,4.40671924726425,1.69377906086785,0.677511624347141 -31123,2005,0,4.20469261939097,1.69377906086785,1.01626743652071 -31123,2006,0,4.24849524204936,1.69377906086785,1.35502324869428 -31123,2007,0,4.04305126783455,1.69377906086785,1.69377906086785 -31127,2003,0,4.92725368515721,2.02498535588378,0.404997071176755 -31127,2004,0,4.95582705760126,2.02498535588378,0.809994142353511 -31127,2005,0,4.71849887129509,2.02498535588378,1.21499121353027 -31127,2006,0,4.64439089914137,2.02498535588378,1.61998828470702 -31127,2007,0,4.67282883446191,2.02498535588378,2.02498535588378 -31131,2003,0,5.94542060860658,2.7341077354215,0.5468215470843 -31131,2004,0,5.70711026474888,2.7341077354215,1.0936430941686 -31131,2005,0,5.69035945432406,2.7341077354215,1.6404646412529 -31131,2006,0,5.74939298590825,2.7341077354215,2.1872861883372 -31131,2007,0,5.80814248998044,2.7341077354215,2.7341077354215 -31133,2003,0,3.13549421592915,1.12719974552002,0.225439949104004 -31133,2004,0,3.09104245335832,1.12719974552002,0.450879898208009 -31133,2005,0,3.13549421592915,1.12719974552002,0.676319847312013 -31133,2006,0,3.49650756146648,1.12719974552002,0.901759796416018 -31133,2007,0,3.55534806148941,1.12719974552002,1.12719974552002 -31147,2003,0,4.83628190695148,2.25454964395549,0.450909928791098 -31147,2004,0,4.92725368515721,2.25454964395549,0.901819857582196 -31147,2005,0,4.93447393313069,2.25454964395549,1.35272978637329 -31147,2006,0,4.72738781871234,2.25454964395549,1.80363971516439 -31147,2007,0,4.74493212836325,2.25454964395549,2.25454964395549 -31151,2003,0,5.66296048013595,2.6277796897056,0.52555593794112 -31151,2004,0,5.44241771052179,2.6277796897056,1.05111187588224 -31151,2005,0,5.50125821054473,2.6277796897056,1.57666781382336 -31151,2006,0,5.4971682252932,2.6277796897056,2.10222375176448 -31151,2007,0,5.40267738187228,2.6277796897056,2.6277796897056 -31153,2003,0,7.53315880745556,4.8088862396362,0.961777247927241 -31153,2004,0,7.49665243816828,4.8088862396362,1.92355449585448 -31153,2005,0,7.53529670244409,4.8088862396362,2.88533174378172 -31153,2006,0,7.53743003658651,4.8088862396362,3.84710899170896 -31153,2007,0,7.61283103040736,4.8088862396362,4.8088862396362 -31155,2003,0,5.60947179518496,2.9871959425317,0.597439188506341 -31155,2004,0,5.6021188208797,2.9871959425317,1.19487837701268 -31155,2005,0,5.46806014113513,2.9871959425317,1.79231756551902 -31155,2006,0,5.50533153593236,2.9871959425317,2.38975675402536 -31155,2007,0,5.63835466933375,2.9871959425317,2.9871959425317 -31161,2003,0,4.45434729625351,1.82422665936556,0.364845331873111 -31161,2004,0,4.31748811353631,1.82422665936556,0.729690663746223 -31161,2005,0,4.26267987704132,1.82422665936556,1.09453599561933 -31161,2006,0,4.27666611901606,1.82422665936556,1.45938132749245 -31161,2007,0,4.11087386417331,1.82422665936556,1.82422665936556 -31177,2003,0,5.84932477994686,2.93279247378012,0.586558494756023 -31177,2004,0,5.83773044716594,2.93279247378012,1.17311698951205 -31177,2005,0,5.73009978297357,2.93279247378012,1.75967548426807 -31177,2006,0,5.67332326717149,2.93279247378012,2.34623397902409 -31177,2007,0,5.59842195899838,2.93279247378012,2.93279247378012 -32007,2003,2007,6.51323011091231,3.81310833725371,0.762621667450742 -32007,2004,2007,6.67203294546107,3.81310833725371,1.52524333490148 -32007,2005,2007,6.75110146893676,3.81310833725371,2.28786500235223 -32007,2006,2007,6.81563999007433,3.81310833725371,3.05048666980297 -32007,2007,2007,6.88653164253051,3.81310833725371,3.81310833725371 -32011,2003,2007,2.56494935746154,0.501381164937991,0.100276232987598 -32011,2004,2007,3.13549421592915,0.501381164937991,0.200552465975196 -32011,2005,2007,2.83321334405622,0.501381164937991,0.300828698962795 -32011,2006,2007,3.25809653802148,0.501381164937991,0.401104931950393 -32011,2007,2007,3.3322045101752,0.501381164937991,0.501381164937991 -32033,2003,2007,4.98360662170834,2.21713613116156,0.443427226232313 -32033,2004,2007,4.91265488573605,2.21713613116156,0.886854452464625 -32033,2005,2007,5.07517381523383,2.21713613116156,1.33028167869694 -32033,2006,2007,5.18738580584075,2.21713613116156,1.77370890492925 -32033,2007,2007,5.18178355029209,2.21713613116156,2.21713613116156 -35006,2003,0,5.44241771052179,3.24239701990955,0.648479403981909 -35006,2004,0,5.13579843705026,3.24239701990955,1.29695880796382 -35006,2005,0,5.26269018890489,3.24239701990955,1.94543821194573 -35006,2006,0,5.47227067367148,3.24239701990955,2.59391761592764 -35006,2007,0,5.44241771052179,3.24239701990955,3.24239701990955 -35013,2003,0,7.57865685059476,5.16296717805949,1.0325934356119 -35013,2004,0,7.56060116276856,5.16296717805949,2.06518687122379 -35013,2005,0,7.62851762657506,5.16296717805949,3.09778030683569 -35013,2006,0,7.59689443814454,5.16296717805949,4.13037374244759 -35013,2007,0,7.65681009148038,5.16296717805949,5.16296717805949 -35025,2003,0,6.71901315438526,4.01658119931192,0.803316239862384 -35025,2004,0,6.70563909486,4.01658119931192,1.60663247972477 -35025,2005,0,6.79122146272619,4.01658119931192,2.40994871958715 -35025,2006,0,6.89162589705225,4.01658119931192,3.21326495944954 -35025,2007,0,7.03262426102801,4.01658119931192,4.01658119931192 -35033,2003,0,3.04452243772342,1.64480505627139,0.328961011254278 -35033,2004,0,2.19722457733622,1.64480505627139,0.657922022508557 -35033,2005,0,2.484906649788,1.64480505627139,0.986883033762835 -35033,2006,0,3.04452243772342,1.64480505627139,1.31584404501711 -35033,2007,0,2.83321334405622,1.64480505627139,1.64480505627139 -35045,2003,0,7.44658509915773,4.73445130899984,0.946890261799969 -35045,2004,0,7.37713371283395,4.73445130899984,1.89378052359994 -35045,2005,0,7.35308192051543,4.73445130899984,2.84067078539991 -35045,2006,0,7.36770857237437,4.73445130899984,3.78756104719987 -35045,2007,0,7.44775128004791,4.73445130899984,4.73445130899984 -37005,2003,2007,4.62497281328427,2.36809189519617,0.473618379039233 -37005,2004,2007,4.58496747867057,2.36809189519617,0.947236758078466 -37005,2005,2007,4.67282883446191,2.36809189519617,1.4208551371177 -37005,2006,2007,4.63472898822964,2.36809189519617,1.89447351615693 -37005,2007,2007,4.62497281328427,2.36809189519617,2.36809189519617 -37021,2003,2007,8.0487882835342,5.32947682879714,1.06589536575943 -37021,2004,2007,8.06589354696427,5.32947682879714,2.13179073151886 -37021,2005,2007,8.14206328310415,5.32947682879714,3.19768609727828 -37021,2006,2007,8.18395571730495,5.32947682879714,4.26358146303771 -37021,2007,2007,8.24800570160062,5.32947682879714,5.32947682879714 -37025,2003,2007,7.6231530684769,4.87567812362863,0.975135624725727 -37025,2004,2007,7.68570306123455,4.87567812362863,1.95027124945145 -37025,2005,2007,7.70571282389443,4.87567812362863,2.92540687417718 -37025,2006,2007,7.8087293067444,4.87567812362863,3.90054249890291 -37025,2007,2007,7.900636613018,4.87567812362863,4.87567812362863 -37029,2003,2007,3.29583686600433,1.92934513218062,0.385869026436124 -37029,2004,2007,2.94443897916644,1.92934513218062,0.771738052872247 -37029,2005,2007,3.46573590279973,1.92934513218062,1.15760707930837 -37029,2006,2007,3.36729582998647,1.92934513218062,1.54347610574449 -37029,2007,2007,3.25809653802148,1.92934513218062,1.92934513218062 -37041,2003,2007,5.06890420222023,2.67594014718398,0.535188029436795 -37041,2004,2007,5.18738580584075,2.67594014718398,1.07037605887359 -37041,2005,2007,4.93447393313069,2.67594014718398,1.60556408831039 -37041,2006,2007,5.08140436498446,2.67594014718398,2.14075211774718 -37041,2007,2007,5.08140436498446,2.67594014718398,2.67594014718398 -37051,2003,2007,8.03624994213212,5.71361068584184,1.14272213716837 -37051,2004,2007,8.04269949689764,5.71361068584184,2.28544427433674 -37051,2005,2007,8.08948247436075,5.71361068584184,3.42816641150511 -37051,2006,2007,8.12622252945853,5.71361068584184,4.57088854867348 -37051,2007,2007,8.13973227971767,5.71361068584184,5.71361068584184 -37057,2003,2007,7.2841348061952,4.99210465747682,0.998420931495364 -37057,2004,2007,7.2283884515736,4.99210465747682,1.99684186299073 -37057,2005,2007,7.22983877815125,4.99210465747682,2.99526279448609 -37057,2006,2007,7.24208235925696,4.99210465747682,3.99368372598146 -37057,2007,2007,7.24064969425547,4.99210465747682,4.99210465747682 -37059,2003,2007,5.95842469302978,3.55062262848208,0.710124525696416 -37059,2004,2007,5.9427993751267,3.55062262848208,1.42024905139283 -37059,2005,2007,6.13988455222626,3.55062262848208,2.13037357708925 -37059,2006,2007,6.13556489108174,3.55062262848208,2.84049810278567 -37059,2007,2007,6.10924758276437,3.55062262848208,3.55062262848208 -37063,2003,2007,7.96797317966293,5.40857885280557,1.08171577056111 -37063,2004,2007,7.96485088744731,5.40857885280557,2.16343154112223 -37063,2005,2007,7.95296679092313,5.40857885280557,3.24514731168334 -37063,2006,2007,8.02059914989697,5.40857885280557,4.32686308224446 -37063,2007,2007,8.02845516411425,5.40857885280557,5.40857885280557 -37075,2003,2007,4.02535169073515,2.07856615864388,0.415713231728776 -37075,2004,2007,4.24849524204936,2.07856615864388,0.831426463457553 -37075,2005,2007,4.17438726989564,2.07856615864388,1.24713969518633 -37075,2006,2007,4.47733681447821,2.07856615864388,1.66285292691511 -37075,2007,2007,4.75359019110636,2.07856615864388,2.07856615864388 -37079,2003,2007,4.61512051684126,2.94306962097069,0.588613924194137 -37079,2004,2007,4.70953020131233,2.94306962097069,1.17722784838827 -37079,2005,2007,4.51085950651685,2.94306962097069,1.76584177258241 -37079,2006,2007,4.51085950651685,2.94306962097069,2.35445569677655 -37079,2007,2007,4.51085950651685,2.94306962097069,2.94306962097069 -37085,2003,2007,6.84374994900622,4.5111341940615,0.902226838812299 -37085,2004,2007,6.85751406254539,4.5111341940615,1.8044536776246 -37085,2005,2007,6.93731408122368,4.5111341940615,2.7066805164369 -37085,2006,2007,7.0352685992811,4.5111341940615,3.6089073552492 -37085,2007,2007,6.99301512293296,4.5111341940615,4.5111341940615 -37093,2003,2007,5.23110861685459,3.51589417829697,0.703178835659394 -37093,2004,2007,5.14166355650266,3.51589417829697,1.40635767131879 -37093,2005,2007,5.28826703069454,3.51589417829697,2.10953650697818 -37093,2006,2007,5.25227342804663,3.51589417829697,2.81271534263758 -37093,2007,2007,5.30826769740121,3.51589417829697,3.51589417829697 -37099,2003,2007,5.79301360838414,3.50016752229797,0.700033504459593 -37099,2004,2007,5.75574221358691,3.50016752229797,1.40006700891919 -37099,2005,2007,5.82600010738045,3.50016752229797,2.10010051337878 -37099,2006,2007,5.84932477994686,3.50016752229797,2.80013401783837 -37099,2007,2007,5.97380961186926,3.50016752229797,3.50016752229797 -37103,2003,2007,3.8286413964891,2.33997721221111,0.467995442442222 -37103,2004,2007,3.66356164612965,2.33997721221111,0.935990884884443 -37103,2005,2007,3.76120011569356,2.33997721221111,1.40398632732667 -37103,2006,2007,4.11087386417331,2.33997721221111,1.87198176976889 -37103,2007,2007,3.13549421592915,2.33997721221111,2.33997721221111 -37105,2003,2007,6.59850902861452,3.89263629162796,0.778527258325591 -37105,2004,2007,6.58617165485467,3.89263629162796,1.55705451665118 -37105,2005,2007,6.67582322163485,3.89263629162796,2.33558177497677 -37105,2006,2007,6.59304453414244,3.89263629162796,3.11410903330236 -37105,2007,2007,6.65027904858742,3.89263629162796,3.89263629162796 -37117,2003,2007,5.35658627467201,3.24231887659467,0.648463775318934 -37117,2004,2007,5.40267738187228,3.24231887659467,1.29692755063787 -37117,2005,2007,5.48893772615669,3.24231887659467,1.9453913259568 -37117,2006,2007,5.44241771052179,3.24231887659467,2.59385510127573 -37117,2007,2007,5.41164605185504,3.24231887659467,3.24231887659467 -37123,2003,2007,5.53338948872752,3.28922244628875,0.65784448925775 -37123,2004,2007,5.4380793089232,3.28922244628875,1.3156889785155 -37123,2005,2007,5.55682806169954,3.28922244628875,1.97353346777325 -37123,2006,2007,5.605802066296,3.28922244628875,2.631377957031 -37123,2007,2007,5.57972982598622,3.28922244628875,3.28922244628875 -37127,2003,2007,7.1708884785125,4.47072408943517,0.894144817887034 -37127,2004,2007,7.11558212618445,4.47072408943517,1.78828963577407 -37127,2005,2007,7.11639414409346,4.47072408943517,2.6824344536611 -37127,2006,2007,7.22548147278229,4.47072408943517,3.57657927154813 -37127,2007,2007,7.35627987655075,4.47072408943517,4.47072408943517 -37155,2003,2007,6.85751406254539,4.81493666186062,0.962987332372125 -37155,2004,2007,6.85329909318608,4.81493666186062,1.92597466474425 -37155,2005,2007,7.05444965813294,4.81493666186062,2.88896199711637 -37155,2006,2007,6.99393297522319,4.81493666186062,3.8519493294885 -37155,2007,2007,7.03790596344718,4.81493666186062,4.81493666186062 -37161,2003,2007,6.30078579466324,4.14153026532882,0.828306053065765 -37161,2004,2007,6.24610676548156,4.14153026532882,1.65661210613153 -37161,2005,2007,6.25575004175337,4.14153026532882,2.48491815919729 -37161,2006,2007,6.29710931993394,4.14153026532882,3.31322421226306 -37161,2007,2007,6.4281052726846,4.14153026532882,4.14153026532882 -37163,2003,2007,6.31716468674728,4.09702430184386,0.819404860368772 -37163,2004,2007,6.38519439899773,4.09702430184386,1.63880972073754 -37163,2005,2007,6.32435896238131,4.09702430184386,2.45821458110632 -37163,2006,2007,6.34212141872115,4.09702430184386,3.27761944147509 -37163,2007,2007,6.44730586254121,4.09702430184386,4.09702430184386 -37165,2003,2007,6.15485809401642,3.58346338135729,0.716692676271457 -37165,2004,2007,6.20657592672493,3.58346338135729,1.43338535254291 -37165,2005,2007,6.14846829591765,3.58346338135729,2.15007802881437 -37165,2006,2007,6.18208490671663,3.58346338135729,2.86677070508583 -37165,2007,2007,6.16331480403464,3.58346338135729,3.58346338135729 -37167,2003,2007,6.62539236800796,4.06216566385787,0.812433132771573 -37167,2004,2007,6.54391184556479,4.06216566385787,1.62486626554315 -37167,2005,2007,6.61338421837956,4.06216566385787,2.43729939831472 -37167,2006,2007,6.62936325343745,4.06216566385787,3.24973253108629 -37167,2007,2007,6.61740297797448,4.06216566385787,4.06216566385787 -37175,2003,2007,5.67675380226828,3.37874725282456,0.675749450564913 -37175,2004,2007,5.70711026474888,3.37874725282456,1.35149890112983 -37175,2005,2007,5.75890177387728,3.37874725282456,2.02724835169474 -37175,2006,2007,5.78074351579233,3.37874725282456,2.70299780225965 -37175,2007,2007,5.71373280550937,3.37874725282456,3.37874725282456 -37189,2003,2007,6.3261494731551,3.75408181735957,0.750816363471914 -37189,2004,2007,6.35610766069589,3.75408181735957,1.50163272694383 -37189,2005,2007,6.35088571671474,3.75408181735957,2.25244909041574 -37189,2006,2007,6.44413125670044,3.75408181735957,3.00326545388766 -37189,2007,2007,6.43294009273918,3.75408181735957,3.75408181735957 -37191,2003,2007,7.24708058458576,4.73029509291699,0.946059018583398 -37191,2004,2007,7.2283884515736,4.73029509291699,1.8921180371668 -37191,2005,2007,7.21744343169653,4.73029509291699,2.8381770557502 -37191,2006,2007,7.30988148582479,4.73029509291699,3.78423607433359 -37191,2007,2007,7.30451594646016,4.73029509291699,4.73029509291699 -38001,2003,0,4.11087386417331,0.952815506556527,0.190563101311305 -38001,2004,0,4.0943445622221,0.952815506556527,0.381126202622611 -38001,2005,0,4.06044301054642,0.952815506556527,0.571689303933916 -38001,2006,0,3.89182029811063,0.952815506556527,0.762252405245222 -38001,2007,0,4.17438726989564,0.952815506556527,0.952815506556527 -38021,2003,0,4.68213122712422,1.75041650569367,0.350083301138734 -38021,2004,0,4.29045944114839,1.75041650569367,0.700166602277469 -38021,2005,0,4.51085950651685,1.75041650569367,1.0502499034162 -38021,2006,0,4.74493212836325,1.75041650569367,1.40033320455494 -38021,2007,0,4.62497281328427,1.75041650569367,1.75041650569367 -38023,2003,0,4.11087386417331,0.825490367547658,0.165098073509532 -38023,2004,0,4.11087386417331,0.825490367547658,0.330196147019063 -38023,2005,0,3.87120101090789,0.825490367547658,0.495294220528595 -38023,2006,0,3.89182029811063,0.825490367547658,0.660392294038127 -38023,2007,0,3.85014760171006,0.825490367547658,0.825490367547659 -38035,2003,0,7.28138566357028,4.19130489492644,0.838260978985288 -38035,2004,0,7.20637729147225,4.19130489492644,1.67652195797058 -38035,2005,0,7.29369772060144,4.19130489492644,2.51478293695586 -38035,2006,0,7.28207365809346,4.19130489492644,3.35304391594115 -38035,2007,0,7.28207365809346,4.19130489492644,4.19130489492644 -38037,2003,0,3.46573590279973,1.04415610287205,0.20883122057441 -38037,2004,0,3.2188758248682,1.04415610287205,0.41766244114882 -38037,2005,0,3.13549421592915,1.04415610287205,0.626493661723231 -38037,2006,0,3.29583686600433,1.04415610287205,0.835324882297641 -38037,2007,0,3.46573590279973,1.04415610287205,1.04415610287205 -38049,2003,0,3.98898404656427,1.78959045194322,0.357918090388643 -38049,2004,0,3.63758615972639,1.78959045194322,0.715836180777286 -38049,2005,0,3.89182029811063,1.78959045194322,1.07375427116593 -38049,2006,0,3.8286413964891,1.78959045194322,1.43167236155457 -38049,2007,0,4.02535169073515,1.78959045194322,1.78959045194322 -38061,2003,0,4.9416424226093,1.89175562238673,0.378351124477346 -38061,2004,0,4.70953020131233,1.89175562238673,0.756702248954692 -38061,2005,0,4.64439089914137,1.89175562238673,1.13505337343204 -38061,2006,0,4.59511985013459,1.89175562238673,1.51340449790938 -38061,2007,0,4.7361984483945,1.89175562238673,1.89175562238673 -38063,2003,0,3.91202300542815,1.31237867816972,0.262475735633944 -38063,2004,0,4.04305126783455,1.31237867816972,0.524951471267889 -38063,2005,0,4.04305126783455,1.31237867816972,0.787427206901833 -38063,2006,0,3.80666248977032,1.31237867816972,1.04990294253578 -38063,2007,0,3.71357206670431,1.31237867816972,1.31237867816972 -38073,2003,0,4.83628190695148,1.7732559976635,0.354651199532699 -38073,2004,0,4.87519732320115,1.7732559976635,0.709302399065398 -38073,2005,0,4.75359019110636,1.7732559976635,1.0639535985981 -38073,2006,0,4.76217393479776,1.7732559976635,1.4186047981308 -38073,2007,0,4.79579054559674,1.7732559976635,1.7732559976635 -38097,2003,0,5.16478597392351,2.13735661362627,0.427471322725254 -38097,2004,0,5.14166355650266,2.13735661362627,0.854942645450507 -38097,2005,0,5.01727983681492,2.13735661362627,1.28241396817576 -38097,2006,0,5.08759633523238,2.13735661362627,1.70988529090101 -38097,2007,0,5.05624580534831,2.13735661362627,2.13735661362627 -38099,2003,0,5.84643877505772,2.5168089821336,0.50336179642672 -38099,2004,0,5.77144112313002,2.5168089821336,1.00672359285344 -38099,2005,0,5.6970934865054,2.5168089821336,1.51008538928016 -38099,2006,0,5.69035945432406,2.5168089821336,2.01344718570688 -38099,2007,0,5.67332326717149,2.5168089821336,2.5168089821336 -38105,2003,0,6.16751649088834,2.98371029832667,0.596742059665333 -38105,2004,0,6.07073772800249,2.98371029832667,1.19348411933067 -38105,2005,0,6.12468339089421,2.98371029832667,1.790226178996 -38105,2006,0,6.16961073249146,2.98371029832667,2.38696823866133 -38105,2007,0,6.18826412308259,2.98371029832667,2.98371029832667 -39019,2003,2007,5.46383180502561,3.36162460654233,0.672324921308466 -39019,2004,2007,5.44241771052179,3.36162460654233,1.34464984261693 -39019,2005,2007,5.40267738187228,3.36162460654233,2.0169747639254 -39019,2006,2007,5.27811465923052,3.36162460654233,2.68929968523387 -39019,2007,2007,5.35185813347607,3.36162460654233,3.36162460654233 -39021,2003,2007,5.89989735358249,3.66073714816766,0.732147429633531 -39021,2004,2007,5.77455154554441,3.66073714816766,1.46429485926706 -39021,2005,2007,5.80211837537706,3.66073714816766,2.19644228890059 -39021,2006,2007,5.8636311755981,3.66073714816766,2.92858971853412 -39021,2007,2007,5.78382518232974,3.66073714816766,3.66073714816766 -39035,2003,2007,10.0020198086004,7.23991680927341,1.44798336185468 -39035,2004,2007,9.94755228369823,7.23991680927341,2.89596672370936 -39035,2005,2007,9.91467474192669,7.23991680927341,4.34395008556404 -39035,2006,2007,9.91595945403145,7.23991680927341,5.79193344741873 -39035,2007,2007,9.89374020108125,7.23991680927341,7.23991680927341 -39047,2003,2007,6.34212141872115,3.34755044243138,0.669510088486276 -39047,2004,2007,6.3297209055227,3.34755044243138,1.33902017697255 -39047,2005,2007,6.33859407820318,3.34755044243138,2.00853026545883 -39047,2006,2007,6.23244801655052,3.34755044243138,2.6780403539451 -39047,2007,2007,6.24416690066374,3.34755044243138,3.34755044243138 -39055,2003,2007,7.29437729928882,4.50970499417002,0.901940998834005 -39055,2004,2007,7.24208235925696,4.50970499417002,1.80388199766801 -39055,2005,2007,7.22037383672395,4.50970499417002,2.70582299650201 -39055,2006,2007,7.30921236569276,4.50970499417002,3.60776399533602 -39055,2007,2007,7.24279792279376,4.50970499417002,4.50970499417002 -39063,2003,2007,7.15461535691366,4.26682619873408,0.853365239746817 -39063,2004,2007,7.06219163228656,4.26682619873408,1.70673047949363 -39063,2005,2007,7.12929754892937,4.26682619873408,2.56009571924045 -39063,2006,2007,7.11395610956603,4.26682619873408,3.41346095898727 -39063,2007,2007,7.14677217945264,4.26682619873408,4.26682619873408 -39071,2003,2007,6.11368217983223,3.71051862921742,0.742103725843483 -39071,2004,2007,6.14418563412565,3.71051862921742,1.48420745168697 -39071,2005,2007,6.21060007702465,3.71051862921742,2.22631117753045 -39071,2006,2007,6.19236248947487,3.71051862921742,2.96841490337393 -39071,2007,2007,6.10702288774225,3.71051862921742,3.71051862921742 -39079,2003,2007,5.8348107370626,3.4855691668815,0.697113833376299 -39079,2004,2007,5.75574221358691,3.4855691668815,1.3942276667526 -39079,2005,2007,5.86929691313377,3.4855691668815,2.0913415001289 -39079,2006,2007,5.82600010738045,3.4855691668815,2.7884553335052 -39079,2007,2007,5.78996017089725,3.4855691668815,3.4855691668815 -39095,2003,2007,9.01054706927019,6.12041609322756,1.22408321864551 -39095,2004,2007,8.92718138825199,6.12041609322756,2.44816643729102 -39095,2005,2007,8.84202652949881,6.12041609322756,3.67224965593654 -39095,2006,2007,8.84029066908897,6.12041609322756,4.89633287458205 -39095,2007,2007,8.82173238093444,6.12041609322756,6.12041609322756 -39097,2003,2007,5.97635090929793,3.69419032643251,0.738838065286503 -39097,2004,2007,5.94017125272043,3.69419032643251,1.47767613057301 -39097,2005,2007,6.13339804299665,3.69419032643251,2.21651419585951 -39097,2006,2007,5.95064255258773,3.69419032643251,2.95535226114601 -39097,2007,2007,5.85507192220243,3.69419032643251,3.69419032643251 -39107,2003,2007,6.77650699237218,3.71171668801669,0.742343337603339 -39107,2004,2007,6.7696419768525,3.71171668801669,1.48468667520668 -39107,2005,2007,6.73459165997295,3.71171668801669,2.22703001281002 -39107,2006,2007,6.80572255341699,3.71171668801669,2.96937335041335 -39107,2007,2007,6.71901315438526,3.71171668801669,3.71171668801669 -39111,2003,2007,3.3322045101752,2.71997877196748,0.543995754393497 -39111,2004,2007,3.25809653802148,2.71997877196748,1.08799150878699 -39111,2005,2007,3.52636052461616,2.71997877196748,1.63198726318049 -39111,2006,2007,3.49650756146648,2.71997877196748,2.17598301757399 -39111,2007,2007,3.40119738166216,2.71997877196748,2.71997877196748 -39119,2003,2007,7.12044437239249,4.43775694592726,0.887551389185451 -39119,2004,2007,7.02553831463852,4.43775694592726,1.7751027783709 -39119,2005,2007,6.94601399109923,4.43775694592726,2.66265416755635 -39119,2006,2007,6.8596149036542,4.43775694592726,3.5502055567418 -39119,2007,2007,6.89568269774787,4.43775694592726,4.43775694592726 -39143,2003,2007,6.83195356556585,4.1237739062461,0.824754781249221 -39143,2004,2007,6.82871207164168,4.1237739062461,1.64950956249844 -39143,2005,2007,6.74523634948436,4.1237739062461,2.47426434374766 -39143,2006,2007,6.73340189183736,4.1237739062461,3.29901912499688 -39143,2007,2007,6.76503897678054,4.1237739062461,4.1237739062461 -39163,2003,2007,4.23410650459726,2.54991381109661,0.509982762219322 -39163,2004,2007,4.24849524204936,2.54991381109661,1.01996552443864 -39163,2005,2007,4.0943445622221,2.54991381109661,1.52994828665797 -39163,2006,2007,3.87120101090789,2.54991381109661,2.03993104887729 -39163,2007,2007,4.00733318523247,2.54991381109661,2.54991381109661 -39173,2003,2007,7.55799495853081,4.79632759144445,0.959265518288889 -39173,2004,2007,7.52131798019924,4.79632759144445,1.91853103657778 -39173,2005,2007,7.46508273639955,4.79632759144445,2.87779655486667 -39173,2006,2007,7.47363710849621,4.79632759144445,3.83706207315556 -39173,2007,2007,7.51261754467451,4.79632759144445,4.79632759144445 -40003,2003,0,4.02535169073515,1.80910810756267,0.361821621512534 -40003,2004,0,3.76120011569356,1.80910810756267,0.723643243025067 -40003,2005,0,3.91202300542815,1.80910810756267,1.0854648645376 -40003,2006,0,3.85014760171006,1.80910810756267,1.44728648605013 -40003,2007,0,3.98898404656427,1.80910810756267,1.80910810756267 -40013,2003,0,6.1779441140506,3.59824333385449,0.719648666770898 -40013,2004,0,6.24222326545517,3.59824333385449,1.4392973335418 -40013,2005,0,6.2803958389602,3.59824333385449,2.1589460003127 -40013,2006,0,6.13556489108174,3.59824333385449,2.87859466708359 -40013,2007,0,6.19236248947487,3.59824333385449,3.59824333385449 -40015,2003,0,5.45958551414416,3.40618492317319,0.681236984634639 -40015,2004,0,5.27811465923052,3.40618492317319,1.36247396926928 -40015,2005,0,5.32787616878958,3.40618492317319,2.04371095390392 -40015,2006,0,5.41164605185504,3.40618492317319,2.72494793853856 -40015,2007,0,5.3890717298165,3.40618492317319,3.40618492317319 -40017,2003,0,7.12125245324454,4.47388769126739,0.894777538253478 -40017,2004,0,7.08590146436561,4.47388769126739,1.78955507650696 -40017,2005,0,7.00033446027523,4.47388769126739,2.68433261476043 -40017,2006,0,7.07834157955767,4.47388769126739,3.57911015301391 -40017,2007,0,7.09090982207998,4.47388769126739,4.47388769126739 -40025,2003,0,3.80666248977032,1.14676733055516,0.229353466111031 -40025,2004,0,3.58351893845611,1.14676733055516,0.458706932222063 -40025,2005,0,3.55534806148941,1.14676733055516,0.688060398333094 -40025,2006,0,3.93182563272433,1.14676733055516,0.917413864444126 -40025,2007,0,3.55534806148941,1.14676733055516,1.14676733055516 -40039,2003,0,6.04500531403601,3.26354321636336,0.652708643272672 -40039,2004,0,5.98645200528444,3.26354321636336,1.30541728654534 -40039,2005,0,5.89989735358249,3.26354321636336,1.95812592981802 -40039,2006,0,5.98393628068719,3.26354321636336,2.61083457309069 -40039,2007,0,6.02344759296103,3.26354321636336,3.26354321636336 -40063,2003,0,4.62497281328427,2.64999726965359,0.529999453930719 -40063,2004,0,4.45434729625351,2.64999726965359,1.05999890786144 -40063,2005,0,4.27666611901606,2.64999726965359,1.58999836179216 -40063,2006,0,4.11087386417331,2.64999726965359,2.11999781572287 -40063,2007,0,4.52178857704904,2.64999726965359,2.64999726965359 -40069,2003,0,4.49980967033027,2.35261258659369,0.470522517318739 -40069,2004,0,4.38202663467388,2.35261258659369,0.941045034637477 -40069,2005,0,4.18965474202643,2.35261258659369,1.41156755195622 -40069,2006,0,4.18965474202643,2.35261258659369,1.88209006927495 -40069,2007,0,4.66343909411207,2.35261258659369,2.35261258659369 -40075,2003,0,4.60517018598809,2.32503128182388,0.465006256364775 -40075,2004,0,4.38202663467388,2.32503128182388,0.93001251272955 -40075,2005,0,4.26267987704132,2.32503128182388,1.39501876909433 -40075,2006,0,4.26267987704132,2.32503128182388,1.8600250254591 -40075,2007,0,4.11087386417331,2.32503128182388,2.32503128182388 -40083,2003,0,5.65599181081985,3.52412272849945,0.704824545699891 -40083,2004,0,5.38449506278909,3.52412272849945,1.40964909139978 -40083,2005,0,5.42934562895444,3.52412272849945,2.11447363709967 -40083,2006,0,5.4380793089232,3.52412272849945,2.81929818279956 -40083,2007,0,5.47227067367148,3.52412272849945,3.52412272849945 -40101,2003,0,7.02642680869964,4.24062146794478,0.848124293588956 -40101,2004,0,6.7945865808765,4.24062146794478,1.69624858717791 -40101,2005,0,6.91075078796194,4.24062146794478,2.54437288076687 -40101,2006,0,6.95177216439891,4.24062146794478,3.39249717435583 -40101,2007,0,6.78332520060396,4.24062146794478,4.24062146794478 -40119,2003,0,6.96034772910131,4.22229792653256,0.844459585306512 -40119,2004,0,6.87729607149743,4.22229792653256,1.68891917061302 -40119,2005,0,6.7202201551353,4.22229792653256,2.53337875591954 -40119,2006,0,6.81014245011514,4.22229792653256,3.37783834122605 -40119,2007,0,6.83625927727707,4.22229792653256,4.22229792653256 -40137,2003,0,6.20657592672493,3.76542374175372,0.753084748350744 -40137,2004,0,6.17170059741091,3.76542374175372,1.50616949670149 -40137,2005,0,6.19440539110467,3.76542374175372,2.25925424505223 -40137,2006,0,6.30261897574491,3.76542374175372,3.01233899340298 -40137,2007,0,6.30809844150953,3.76542374175372,3.76542374175372 -40143,2003,0,9.27556615689389,6.33381057064612,1.26676211412922 -40143,2004,0,9.18502256399189,6.33381057064612,2.53352422825845 -40143,2005,0,9.21303673352392,6.33381057064612,3.80028634238767 -40143,2006,0,9.31775876874345,6.33381057064612,5.0670484565169 -40143,2007,0,9.36108525866861,6.33381057064612,6.33381057064612 -40145,2003,0,5.89164421182577,4.05162841381337,0.810325682762674 -40145,2004,0,5.85220247977447,4.05162841381337,1.62065136552535 -40145,2005,0,5.92425579741453,4.05162841381337,2.43097704828802 -40145,2006,0,6.00635315960173,4.05162841381337,3.24130273105069 -40145,2007,0,6.02102334934953,4.05162841381337,4.05162841381337 -40153,2003,0,5.73334127689775,2.91701368884259,0.583402737768517 -40153,2004,0,5.64190707093811,2.91701368884259,1.16680547553703 -40153,2005,0,5.7268477475872,2.91701368884259,1.75020821330555 -40153,2006,0,5.86929691313377,2.91701368884259,2.33361095107407 -40153,2007,0,5.95064255258773,2.91701368884259,2.91701368884259 -45007,2003,0,7.6586995582683,5.1104202954209,1.02208405908418 -45007,2004,0,7.61184239958042,5.1104202954209,2.04416811816836 -45007,2005,0,7.62217459481762,5.1104202954209,3.06625217725254 -45007,2006,0,7.65822752616135,5.1104202954209,4.08833623633672 -45007,2007,0,7.63964228785801,5.1104202954209,5.1104202954209 -45009,2003,0,5.03043792139244,2.81289058151315,0.56257811630263 -45009,2004,0,4.93447393313069,2.81289058151315,1.12515623260526 -45009,2005,0,5.12989871492307,2.81289058151315,1.68773434890789 -45009,2006,0,5.05624580534831,2.81289058151315,2.25031246521052 -45009,2007,0,5.14166355650266,2.81289058151315,2.81289058151315 -45013,2003,0,7.58222919427646,4.79526974884863,0.959053949769726 -45013,2004,0,7.57044325205737,4.79526974884863,1.91810789953945 -45013,2005,0,7.6511201757027,4.79526974884863,2.87716184930918 -45013,2006,0,7.74586822979227,4.79526974884863,3.8362157990789 -45013,2007,0,7.71779621101358,4.79526974884863,4.79526974884863 -45017,2003,0,4.21950770517611,2.72030809849771,0.544061619699542 -45017,2004,0,4.12713438504509,2.72030809849771,1.08812323939908 -45017,2005,0,4.14313472639153,2.72030809849771,1.63218485909863 -45017,2006,0,4.17438726989564,2.72030809849771,2.17624647879817 -45017,2007,0,4.23410650459726,2.72030809849771,2.72030809849771 -45019,2003,0,8.52237971810354,5.73647229247886,1.14729445849577 -45019,2004,0,8.50936261230105,5.73647229247886,2.29458891699154 -45019,2005,0,8.60575336839572,5.73647229247886,3.44188337548732 -45019,2006,0,8.70018119275335,5.73647229247886,4.58917783398309 -45019,2007,0,8.71292443512011,5.73647229247886,5.73647229247886 -45023,2003,0,5.34710753071747,3.52835852727883,0.705671705455767 -45023,2004,0,5.35185813347607,3.52835852727883,1.41134341091153 -45023,2005,0,5.35658627467201,3.52835852727883,2.1170151163673 -45023,2006,0,5.47646355193151,3.52835852727883,2.82268682182307 -45023,2007,0,5.52545293913178,3.52835852727883,3.52835852727883 -45025,2003,0,5.80513496891649,3.75579015939656,0.751158031879313 -45025,2004,0,5.73334127689775,3.75579015939656,1.50231606375863 -45025,2005,0,5.85220247977447,3.75579015939656,2.25347409563794 -45025,2006,0,5.78382518232974,3.75579015939656,3.00463212751725 -45025,2007,0,5.89989735358249,3.75579015939656,3.75579015939656 -45037,2003,0,5.0998664278242,3.2025431702473,0.640508634049461 -45037,2004,0,4.969813299576,3.2025431702473,1.28101726809892 -45037,2005,0,5.03043792139244,3.2025431702473,1.92152590214838 -45037,2006,0,4.99721227376411,3.2025431702473,2.56203453619784 -45037,2007,0,5.27299955856375,3.2025431702473,3.2025431702473 -45051,2003,0,7.98786409608569,5.28131870446459,1.05626374089292 -45051,2004,0,7.97453284413023,5.28131870446459,2.11252748178584 -45051,2005,0,8.11282747875137,5.28131870446459,3.16879122267875 -45051,2006,0,8.19973896063079,5.28131870446459,4.22505496357167 -45051,2007,0,8.28652137368124,5.28131870446459,5.28131870446459 -45055,2003,0,6.1779441140506,3.96360925689958,0.792721851379916 -45055,2004,0,6.30261897574491,3.96360925689958,1.58544370275983 -45055,2005,0,6.4085287910595,3.96360925689958,2.37816555413975 -45055,2006,0,6.28599809450886,3.96360925689958,3.17088740551966 -45055,2007,0,6.3818160174061,3.96360925689958,3.96360925689958 -45059,2003,0,6.42162226780652,4.24229031697041,0.848458063394082 -45059,2004,0,6.25766758788264,4.24229031697041,1.69691612678816 -45059,2005,0,6.32435896238131,4.24229031697041,2.54537419018224 -45059,2006,0,6.35610766069589,4.24229031697041,3.39383225357633 -45059,2007,0,6.34212141872115,4.24229031697041,4.24229031697041 -45067,2003,0,5.69035945432406,3.56857449107029,0.713714898214058 -45067,2004,0,5.605802066296,3.56857449107029,1.42742979642812 -45067,2005,0,5.61312810638807,3.56857449107029,2.14114469464217 -45067,2006,0,5.64544689764324,3.56857449107029,2.85485959285623 -45067,2007,0,5.63835466933375,3.56857449107029,3.56857449107029 -45073,2003,0,6.56526497003536,4.19290702337893,0.838581404675786 -45073,2004,0,6.43294009273918,4.19290702337893,1.67716280935157 -45073,2005,0,6.39692965521615,4.19290702337893,2.51574421402736 -45073,2006,0,6.41836493593621,4.19290702337893,3.35432561870314 -45073,2007,0,6.44413125670044,4.19290702337893,4.19290702337893 -45081,2003,0,5.04985600724954,2.95392020573955,0.59078404114791 -45081,2004,0,5.10594547390058,2.95392020573955,1.18156808229582 -45081,2005,0,5.15329159449778,2.95392020573955,1.77235212344373 -45081,2006,0,5.16478597392351,2.95392020573955,2.36313616459164 -45081,2007,0,5.11198778835654,2.95392020573955,2.95392020573955 -45087,2003,0,5.2257466737132,3.39722282690674,0.679444565381349 -45087,2004,0,5.01727983681492,3.39722282690674,1.3588891307627 -45087,2005,0,5.09375020080676,3.39722282690674,2.03833369614405 -45087,2006,0,5.32787616878958,3.39722282690674,2.7177782615254 -45087,2007,0,5.40717177146012,3.39722282690674,3.39722282690674 -46013,2003,0,6.74993119378857,3.56840530064606,0.713681060129212 -46013,2004,0,6.68586094706836,3.56840530064606,1.42736212025842 -46013,2005,0,6.70196036600254,3.56840530064606,2.14104318038764 -46013,2006,0,6.78219205600679,3.56840530064606,2.85472424051685 -46013,2007,0,6.78671695060508,3.56840530064606,3.56840530064606 -46021,2003,0,3.04452243772342,0.577736329048618,0.115547265809724 -46021,2004,0,2.99573227355399,0.577736329048618,0.231094531619447 -46021,2005,0,2.70805020110221,0.577736329048618,0.346641797429171 -46021,2006,0,2.70805020110221,0.577736329048618,0.462189063238894 -46021,2007,0,2.56494935746154,0.577736329048618,0.577736329048618 -46029,2003,0,6.61606518513282,3.25412713188084,0.650825426376167 -46029,2004,0,6.56244409369372,3.25412713188084,1.30165085275233 -46029,2005,0,6.62406522779989,3.25412713188084,1.9524762791285 -46029,2006,0,6.59304453414244,3.25412713188084,2.60330170550467 -46029,2007,0,6.57228254269401,3.25412713188084,3.25412713188084 -46033,2003,0,4.23410650459726,1.98444381305756,0.396888762611511 -46033,2004,0,4.18965474202643,1.98444381305756,0.793777525223023 -46033,2005,0,4.4188406077966,1.98444381305756,1.19066628783453 -46033,2006,0,4.47733681447821,1.98444381305756,1.58755505044605 -46033,2007,0,4.39444915467244,1.98444381305756,1.98444381305756 -46047,2003,0,4.60517018598809,2.00861663589956,0.401723327179911 -46047,2004,0,4.69134788222914,2.00861663589956,0.803446654359823 -46047,2005,0,4.53259949315326,2.00861663589956,1.20516998153973 -46047,2006,0,4.8283137373023,2.00861663589956,1.60689330871965 -46047,2007,0,4.83628190695148,2.00861663589956,2.00861663589956 -46065,2003,0,6.13339804299665,2.80220820225204,0.560441640450409 -46065,2004,0,6.12686918411419,2.80220820225204,1.12088328090082 -46065,2005,0,6.16751649088834,2.80220820225204,1.68132492135123 -46065,2006,0,6.13556489108174,2.80220820225204,2.24176656180163 -46065,2007,0,6.10479323241498,2.80220820225204,2.80220820225204 -46075,2003,0,3.49650756146648,0.176471143115779,0.0352942286231558 -46075,2004,0,3.49650756146648,0.176471143115779,0.0705884572463116 -46075,2005,0,3.25809653802148,0.176471143115779,0.105882685869467 -46075,2006,0,2.94443897916644,0.176471143115779,0.141176914492623 -46075,2007,0,3.17805383034795,0.176471143115779,0.176471143115779 -46079,2003,0,5.45958551414416,2.42267657325187,0.484535314650373 -46079,2004,0,5.31320597904179,2.42267657325187,0.969070629300746 -46079,2005,0,5.4249500174814,2.42267657325187,1.45360594395112 -46079,2006,0,5.30330490805908,2.42267657325187,1.93814125860149 -46079,2007,0,5.32300997913841,2.42267657325187,2.42267657325187 -46095,2003,0,2.39789527279837,0.733809162278835,0.146761832455767 -46095,2004,0,2.70805020110221,0.733809162278835,0.293523664911534 -46095,2005,0,2.83321334405622,0.733809162278835,0.440285497367301 -46095,2006,0,2.94443897916644,0.733809162278835,0.587047329823068 -46095,2007,0,3.36729582998647,0.733809162278835,0.733809162278835 -46105,2003,0,4.04305126783455,1.21283343275813,0.242566686551626 -46105,2004,0,4.02535169073515,1.21283343275813,0.485133373103253 -46105,2005,0,4.20469261939097,1.21283343275813,0.727700059654879 -46105,2006,0,4.17438726989564,1.21283343275813,0.970266746206506 -46105,2007,0,4.07753744390572,1.21283343275813,1.21283343275813 -46109,2003,0,5.14749447681345,2.30418381435774,0.460836762871549 -46109,2004,0,5.15905529921453,2.30418381435774,0.921673525743097 -46109,2005,0,5.24702407216049,2.30418381435774,1.38251028861465 -46109,2006,0,5.21493575760899,2.30418381435774,1.84334705148619 -46109,2007,0,5.12989871492307,2.30418381435774,2.30418381435774 -46115,2003,0,4.81218435537242,2.00875080105707,0.401750160211413 -46115,2004,0,4.91265488573605,2.00875080105707,0.803500320422827 -46115,2005,0,5.02388052084628,2.00875080105707,1.20525048063424 -46115,2006,0,5.08759633523238,2.00875080105707,1.60700064084565 -46115,2007,0,5.05624580534831,2.00875080105707,2.00875080105707 -46119,2003,0,2.83321334405622,0.4421184257562,0.08842368515124 -46119,2004,0,2.99573227355399,0.4421184257562,0.17684737030248 -46119,2005,0,2.70805020110221,0.4421184257562,0.26527105545372 -46119,2006,0,2.99573227355399,0.4421184257562,0.35369474060496 -46119,2007,0,2.89037175789616,0.4421184257562,0.4421184257562 -46121,2003,0,3.49650756146648,2.20276475771183,0.440552951542367 -46121,2004,0,3.76120011569356,2.20276475771183,0.881105903084734 -46121,2005,0,3.46573590279973,2.20276475771183,1.3216588546271 -46121,2006,0,3.58351893845611,2.20276475771183,1.76221180616947 -46121,2007,0,3.17805383034795,2.20276475771183,2.20276475771183 -47005,2003,0,5.06890420222023,2.80560029466806,0.561120058933612 -47005,2004,0,4.95582705760126,2.80560029466806,1.12224011786722 -47005,2005,0,4.99043258677874,2.80560029466806,1.68336017680084 -47005,2006,0,5.00394630594546,2.80560029466806,2.24448023573445 -47005,2007,0,4.9416424226093,2.80560029466806,2.80560029466806 -47007,2003,0,3.97029191355212,2.51503163475992,0.503006326951984 -47007,2004,0,3.85014760171006,2.51503163475992,1.00601265390397 -47007,2005,0,4.07753744390572,2.51503163475992,1.50901898085595 -47007,2006,0,3.93182563272433,2.51503163475992,2.01202530780794 -47007,2007,0,4.15888308335967,2.51503163475992,2.51503163475992 -47009,2003,0,6.94312242281943,4.66176788710248,0.932353577420496 -47009,2004,0,7.00215595440362,4.66176788710248,1.86470715484099 -47009,2005,0,7.06817200038804,4.66176788710248,2.79706073226149 -47009,2006,0,7.12044437239249,4.66176788710248,3.72941430968199 -47009,2007,0,7.09672137849476,4.66176788710248,4.66176788710248 -47013,2003,0,5.53733426701854,3.68522277661039,0.737044555322078 -47013,2004,0,5.54126354515843,3.68522277661039,1.47408911064416 -47013,2005,0,5.68357976733868,3.68522277661039,2.21113366596624 -47013,2006,0,5.61312810638807,3.68522277661039,2.94817822128831 -47013,2007,0,5.65599181081985,3.68522277661039,3.68522277661039 -47029,2003,0,5.36129216570943,3.51348385739071,0.702696771478143 -47029,2004,0,5.23644196282995,3.51348385739071,1.40539354295629 -47029,2005,0,5.34710753071747,3.51348385739071,2.10809031443443 -47029,2006,0,5.32300997913841,3.51348385739071,2.81078708591257 -47029,2007,0,5.4380793089232,3.51348385739071,3.51348385739071 -47047,2003,0,5.06890420222023,3.36058369877686,0.672116739755372 -47047,2004,0,5.07517381523383,3.36058369877686,1.34423347951074 -47047,2005,0,5.14749447681345,3.36058369877686,2.01635021926611 -47047,2006,0,5.40717177146012,3.36058369877686,2.68846695902149 -47047,2007,0,5.48063892334199,3.36058369877686,3.36058369877686 -47065,2003,0,8.57319538153152,5.72976206361515,1.14595241272303 -47065,2004,0,8.60300384782935,5.72976206361515,2.29190482544606 -47065,2005,0,8.63194942871443,5.72976206361515,3.43785723816909 -47065,2006,0,8.5577591531629,5.72976206361515,4.58380965089212 -47065,2007,0,8.56369502506766,5.72976206361515,5.72976206361515 -47073,2003,0,6.12686918411419,3.98085853120358,0.796171706240715 -47073,2004,0,6.00881318544259,3.98085853120358,1.59234341248143 -47073,2005,0,5.93489419561959,3.98085853120358,2.38851511872215 -47073,2006,0,6.04263283368238,3.98085853120358,3.18468682496286 -47073,2007,0,5.93753620508243,3.98085853120358,3.98085853120358 -47081,2003,0,4.98360662170834,3.10436243807944,0.620872487615888 -47081,2004,0,4.85203026391962,3.10436243807944,1.24174497523178 -47081,2005,0,4.8283137373023,3.10436243807944,1.86261746284766 -47081,2006,0,4.89034912822175,3.10436243807944,2.48348995046355 -47081,2007,0,5.01063529409626,3.10436243807944,3.10436243807944 -47097,2003,0,4.7361984483945,3.29957062757386,0.659914125514771 -47097,2004,0,4.72738781871234,3.29957062757386,1.31982825102954 -47097,2005,0,4.52178857704904,3.29957062757386,1.97974237654431 -47097,2006,0,4.78749174278205,3.29957062757386,2.63965650205909 -47097,2007,0,4.88280192258637,3.29957062757386,3.29957062757386 -47099,2003,0,5.89715386763674,3.68702774075046,0.737405548150092 -47099,2004,0,5.91889385427315,3.68702774075046,1.47481109630018 -47099,2005,0,5.96614673912369,3.68702774075046,2.21221664445028 -47099,2006,0,5.96614673912369,3.68702774075046,2.94962219260037 -47099,2007,0,6.05208916892442,3.68702774075046,3.68702774075046 -47125,2003,0,7.31654817718298,4.90355478157301,0.980710956314602 -47125,2004,0,7.33498187887181,4.90355478157301,1.9614219126292 -47125,2005,0,7.38894609761844,4.90355478157301,2.9421328689438 -47125,2006,0,7.42595365707754,4.90355478157301,3.92284382525841 -47125,2007,0,7.51479976048867,4.90355478157301,4.90355478157301 -47145,2003,0,6.02344759296103,3.94951144983914,0.789902289967828 -47145,2004,0,6.10255859461357,3.94951144983914,1.57980457993566 -47145,2005,0,6.19236248947487,3.94951144983914,2.36970686990348 -47145,2006,0,6.07993319509559,3.94951144983914,3.15960915987131 -47145,2007,0,6.04973345523196,3.94951144983914,3.94951144983914 -47157,2003,0,9.48607637340956,6.79958192210419,1.35991638442084 -47157,2004,0,9.44770236052795,6.79958192210419,2.71983276884167 -47157,2005,0,9.40640045488065,6.79958192210419,4.07974915326251 -47157,2006,0,9.47031717333574,6.79958192210419,5.43966553768335 -47157,2007,0,9.52493210973482,6.79958192210419,6.79958192210419 -47171,2003,0,4.99721227376411,2.87169849263054,0.574339698526109 -47171,2004,0,4.69134788222914,2.87169849263054,1.14867939705222 -47171,2005,0,4.85981240436167,2.87169849263054,1.72301909557833 -47171,2006,0,4.89034912822175,2.87169849263054,2.29735879410443 -47171,2007,0,4.85981240436167,2.87169849263054,2.87169849263054 -47173,2003,0,4.51085950651685,2.87964779453319,0.575929558906638 -47173,2004,0,4.34380542185368,2.87964779453319,1.15185911781328 -47173,2005,0,4.33073334028633,2.87964779453319,1.72778867671991 -47173,2006,0,4.63472898822964,2.87964779453319,2.30371823562655 -47173,2007,0,4.45434729625351,2.87964779453319,2.87964779453319 -48015,2003,0,5.73979291217923,3.16082289341958,0.632164578683917 -48015,2004,0,5.8664680569333,3.16082289341958,1.26432915736783 -48015,2005,0,5.8664680569333,3.16082289341958,1.89649373605175 -48015,2006,0,5.89989735358249,3.16082289341958,2.52865831473567 -48015,2007,0,5.99893656194668,3.16082289341958,3.16082289341958 -48019,2003,0,4.75359019110636,2.87045245712723,0.574090491425447 -48019,2004,0,4.67282883446191,2.87045245712723,1.14818098285089 -48019,2005,0,4.83628190695148,2.87045245712723,1.72227147427634 -48019,2006,0,4.79579054559674,2.87045245712723,2.29636196570179 -48019,2007,0,4.79579054559674,2.87045245712723,2.87045245712723 -48025,2003,0,5.22035582507832,3.47689218936423,0.695378437872845 -48025,2004,0,5.24174701505964,3.47689218936423,1.39075687574569 -48025,2005,0,5.19295685089021,3.47689218936423,2.08613531361854 -48025,2006,0,5.36597601502185,3.47689218936423,2.78151375149138 -48025,2007,0,5.33271879326537,3.47689218936423,3.47689218936423 -48031,2003,0,4.57471097850338,2.13037227034838,0.426074454069676 -48031,2004,0,4.60517018598809,2.13037227034838,0.852148908139352 -48031,2005,0,4.4188406077966,2.13037227034838,1.27822336220903 -48031,2006,0,4.45434729625351,2.13037227034838,1.7042978162787 -48031,2007,0,4.59511985013459,2.13037227034838,2.13037227034838 -48043,2003,0,4.83628190695148,2.18222373632286,0.436444747264572 -48043,2004,0,4.89034912822175,2.18222373632286,0.872889494529145 -48043,2005,0,5.16478597392351,2.18222373632286,1.30933424179372 -48043,2006,0,5.32787616878958,2.18222373632286,1.74577898905829 -48043,2007,0,5.4971682252932,2.18222373632286,2.18222373632286 -48055,2003,0,5.35185813347607,3.47178009978404,0.694356019956808 -48055,2004,0,5.4971682252932,3.47178009978404,1.38871203991362 -48055,2005,0,5.37989735354046,3.47178009978404,2.08306805987042 -48055,2006,0,5.44673737166631,3.47178009978404,2.77742407982723 -48055,2007,0,5.43372200355424,3.47178009978404,3.47178009978404 -48061,2003,0,7.84149292446001,5.81480791429005,1.16296158285801 -48061,2004,0,7.8164169836918,5.81480791429005,2.32592316571602 -48061,2005,0,7.74240202181578,5.81480791429005,3.48888474857403 -48061,2006,0,7.81843027207066,5.81480791429005,4.65184633143204 -48061,2007,0,7.92768504561578,5.81480791429005,5.81480791429005 -48073,2003,0,6.35262939631957,3.84286583473053,0.768573166946107 -48073,2004,0,6.07764224334903,3.84286583473053,1.53714633389221 -48073,2005,0,6.19440539110467,3.84286583473053,2.30571950083832 -48073,2006,0,6.21060007702465,3.84286583473053,3.07429266778443 -48073,2007,0,6.22653666928747,3.84286583473053,3.84286583473053 -48077,2003,0,4.34380542185368,2.39844057863757,0.479688115727513 -48077,2004,0,4.11087386417331,2.39844057863757,0.959376231455027 -48077,2005,0,4.15888308335967,2.39844057863757,1.43906434718254 -48077,2006,0,3.91202300542815,2.39844057863757,1.91875246291005 -48077,2007,0,3.80666248977032,2.39844057863757,2.39844057863757 -48113,2003,0,10.4426965306939,7.70476640589851,1.5409532811797 -48113,2004,0,10.3391261606518,7.70476640589851,3.0819065623594 -48113,2005,0,10.3371189079706,7.70476640589851,4.62285984353911 -48113,2006,0,10.3784787232929,7.70476640589851,6.16381312471881 -48113,2007,0,10.4352622050525,7.70476640589851,7.70476640589851 -48117,2003,0,5.4249500174814,2.92106260521696,0.584212521043392 -48117,2004,0,5.28320372873799,2.92106260521696,1.16842504208678 -48117,2005,0,5.26785815906333,2.92106260521696,1.75263756313018 -48117,2006,0,5.46383180502561,2.92106260521696,2.33685008417357 -48117,2007,0,5.4971682252932,2.92106260521696,2.92106260521696 -48121,2003,0,8.62353322718756,6.07068229921461,1.21413645984292 -48121,2004,0,8.63034328934889,6.07068229921461,2.42827291968584 -48121,2005,0,8.68253812400308,6.07068229921461,3.64240937952877 -48121,2006,0,8.75731184693641,6.07068229921461,4.85654583937169 -48121,2007,0,8.80956425335415,6.07068229921461,6.07068229921461 -48151,2003,0,2.94443897916644,1.46879558263163,0.293759116526327 -48151,2004,0,2.99573227355399,1.46879558263163,0.587518233052654 -48151,2005,0,2.56494935746154,1.46879558263163,0.881277349578981 -48151,2006,0,2.99573227355399,1.46879558263163,1.17503646610531 -48151,2007,0,2.484906649788,1.46879558263163,1.46879558263163 -48173,2003,0,2.39789527279837,0.340748793388473,0.0681497586776946 -48173,2004,0,1.79175946922805,0.340748793388473,0.136299517355389 -48173,2005,0,1.94591014905531,0.340748793388473,0.204449276033084 -48173,2006,0,2.07944154167984,0.340748793388473,0.272599034710778 -48173,2007,0,1.38629436111989,0.340748793388473,0.340748793388473 -48213,2003,0,6.57088296233958,4.29424678066323,0.858849356132646 -48213,2004,0,6.47389069635227,4.29424678066323,1.71769871226529 -48213,2005,0,6.39692965521615,4.29424678066323,2.57654806839794 -48213,2006,0,6.49526555593701,4.29424678066323,3.43539742453058 -48213,2007,0,6.46458830368996,4.29424678066323,4.29424678066323 -48221,2003,0,6.2363695902037,3.71600812150219,0.743201624300438 -48221,2004,0,6.13772705408623,3.71600812150219,1.48640324860088 -48221,2005,0,6.05678401322862,3.71600812150219,2.22960487290131 -48221,2006,0,6.25382881157547,3.71600812150219,2.97280649720175 -48221,2007,0,6.36302810354046,3.71600812150219,3.71600812150219 -48223,2003,0,5.95583736946483,3.46448512089807,0.692897024179615 -48223,2004,0,5.91889385427315,3.46448512089807,1.38579404835923 -48223,2005,0,5.85793315448346,3.46448512089807,2.07869107253884 -48223,2006,0,5.95064255258773,3.46448512089807,2.77158809671846 -48223,2007,0,5.90536184805457,3.46448512089807,3.46448512089807 -48245,2003,0,8.01763715990848,5.52963144798754,1.10592628959751 -48245,2004,0,7.92334821193015,5.52963144798754,2.21185257919502 -48245,2005,0,8.09162741160107,5.52963144798754,3.31777886879253 -48245,2006,0,8.18618599422608,5.52963144798754,4.42370515839003 -48245,2007,0,8.17301131172497,5.52963144798754,5.52963144798754 -48249,2003,0,5.62040086571715,3.67188587773737,0.734377175547473 -48249,2004,0,5.76205138278018,3.67188587773737,1.46875435109495 -48249,2005,0,5.73979291217923,3.67188587773737,2.20313152664242 -48249,2006,0,5.78689738136671,3.67188587773737,2.93750870218989 -48249,2007,0,5.86929691313377,3.67188587773737,3.67188587773737 -48271,2003,0,2.63905732961526,1.21757980773215,0.243515961546431 -48271,2004,0,1.6094379124341,1.21757980773215,0.487031923092861 -48271,2005,0,1.6094379124341,1.21757980773215,0.730547884639292 -48271,2006,0,2.19722457733622,1.21757980773215,0.974063846185722 -48271,2007,0,2.30258509299405,1.21757980773215,1.21757980773215 -48277,2003,0,6.32256523992728,3.88154317917417,0.776308635834834 -48277,2004,0,6.24027584517077,3.88154317917417,1.55261727166967 -48277,2005,0,6.15060276844628,3.88154317917417,2.3289259075045 -48277,2006,0,6.37672694789863,3.88154317917417,3.10523454333934 -48277,2007,0,6.45676965557216,3.88154317917417,3.88154317917417 -48291,2003,0,6.26530121273771,4.25069282559285,0.850138565118569 -48291,2004,0,6.24610676548156,4.25069282559285,1.70027713023714 -48291,2005,0,6.19644412779452,4.25069282559285,2.55041569535571 -48291,2006,0,6.27664348934164,4.25069282559285,3.40055426047428 -48291,2007,0,6.289715570909,4.25069282559285,4.25069282559285 -48295,2003,0,3.58351893845611,1.1174340429087,0.223486808581739 -48295,2004,0,3.91202300542815,1.1174340429087,0.446973617163479 -48295,2005,0,4.04305126783455,1.1174340429087,0.670460425745219 -48295,2006,0,3.98898404656427,1.1174340429087,0.893947234326958 -48295,2007,0,4.20469261939097,1.1174340429087,1.1174340429087 -48305,2003,0,3.78418963391826,1.87946504964716,0.375893009929432 -48305,2004,0,3.71357206670431,1.87946504964716,0.751786019858864 -48305,2005,0,3.80666248977032,1.87946504964716,1.1276790297883 -48305,2006,0,3.87120101090789,1.87946504964716,1.50357203971773 -48305,2007,0,2.99573227355399,1.87946504964716,1.87946504964716 -48317,2003,0,2.56494935746154,1.55730215801357,0.311460431602714 -48317,2004,0,1.09861228866811,1.55730215801357,0.622920863205429 -48317,2005,0,2.07944154167984,1.55730215801357,0.934381294808143 -48317,2006,0,1.38629436111989,1.55730215801357,1.24584172641086 -48317,2007,0,1.94591014905531,1.55730215801357,1.55730215801357 -48323,2003,0,5.78996017089725,3.85644686853929,0.771289373707859 -48323,2004,0,5.4553211153577,3.85644686853929,1.54257874741572 -48323,2005,0,5.50125821054473,3.85644686853929,2.31386812112358 -48323,2006,0,5.71702770140622,3.85644686853929,3.08515749483144 -48323,2007,0,5.91620206260743,3.85644686853929,3.85644686853929 -48341,2003,0,5.74939298590825,3.00176404578571,0.600352809157142 -48341,2004,0,5.59471137960184,3.00176404578571,1.20070561831428 -48341,2005,0,5.58724865840025,3.00176404578571,1.80105842747143 -48341,2006,0,5.6937321388027,3.00176404578571,2.40141123662857 -48341,2007,0,5.6970934865054,3.00176404578571,3.00176404578571 -48351,2003,0,3.3322045101752,2.71283871783401,0.542567743566801 -48351,2004,0,3.46573590279973,2.71283871783401,1.0851354871336 -48351,2005,0,3.80666248977032,2.71283871783401,1.6277032307004 -48351,2006,0,3.61091791264422,2.71283871783401,2.17027097426721 -48351,2007,0,3.8286413964891,2.71283871783401,2.71283871783401 -48357,2003,0,5.30330490805908,2.19789102187938,0.439578204375876 -48357,2004,0,5.23644196282995,2.19789102187938,0.879156408751752 -48357,2005,0,5.18178355029209,2.19789102187938,1.31873461312763 -48357,2006,0,5.25227342804663,2.19789102187938,1.7583128175035 -48357,2007,0,5.25749537202778,2.19789102187938,2.19789102187938 -48363,2003,0,5.61312810638807,3.29679936561589,0.659359873123179 -48363,2004,0,5.62762111369064,3.29679936561589,1.31871974624636 -48363,2005,0,5.67332326717149,3.29679936561589,1.97807961936954 -48363,2006,0,5.78689738136671,3.29679936561589,2.63743949249272 -48363,2007,0,5.78996017089725,3.29679936561589,3.29679936561589 -48369,2003,0,4.76217393479776,2.30418381435774,0.460836762871549 -48369,2004,0,4.88280192258637,2.30418381435774,0.921673525743097 -48369,2005,0,4.87519732320115,2.30418381435774,1.38251028861465 -48369,2006,0,4.77912349311153,2.30418381435774,1.84334705148619 -48369,2007,0,4.79579054559674,2.30418381435774,2.30418381435774 -48383,2003,0,3.43398720448515,1.20177038077074,0.240354076154147 -48383,2004,0,3.61091791264422,1.20177038077074,0.480708152308294 -48383,2005,0,3.29583686600433,1.20177038077074,0.721062228462442 -48383,2006,0,3.61091791264422,1.20177038077074,0.961416304616589 -48383,2007,0,3.76120011569356,1.20177038077074,1.20177038077074 -48385,2003,0,3.09104245335832,1.11415750000357,0.222831500000714 -48385,2004,0,2.70805020110221,1.11415750000357,0.445663000001429 -48385,2005,0,2.39789527279837,1.11415750000357,0.668494500002143 -48385,2006,0,2.70805020110221,1.11415750000357,0.891326000002858 -48385,2007,0,2.94443897916644,1.11415750000357,1.11415750000357 -48397,2003,0,6.55535689181067,3.76305885228819,0.752611770457637 -48397,2004,0,6.6052979209482,3.76305885228819,1.50522354091527 -48397,2005,0,6.68835471394676,3.76305885228819,2.25783531137291 -48397,2006,0,6.78332520060396,3.76305885228819,3.01044708183055 -48397,2007,0,6.93828448401696,3.76305885228819,3.76305885228819 -48399,2003,0,4.86753445045558,2.44191215821514,0.488382431643029 -48399,2004,0,4.75359019110636,2.44191215821514,0.976764863286058 -48399,2005,0,4.65396035015752,2.44191215821514,1.46514729492909 -48399,2006,0,4.62497281328427,2.44191215821514,1.95352972657212 -48399,2007,0,4.62497281328427,2.44191215821514,2.44191215821514 -48409,2003,0,6.12249280951439,4.20675020260656,0.841350040521313 -48409,2004,0,6.13988455222626,4.20675020260656,1.68270008104263 -48409,2005,0,6.19644412779452,4.20675020260656,2.52405012156394 -48409,2006,0,6.30627528694802,4.20675020260656,3.36540016208525 -48409,2007,0,6.38856140554563,4.20675020260656,4.20675020260656 -48415,2003,0,5.19849703126583,2.79490045400094,0.558980090800188 -48415,2004,0,5.03043792139244,2.79490045400094,1.11796018160038 -48415,2005,0,4.969813299576,2.79490045400094,1.67694027240056 -48415,2006,0,5.07517381523383,2.79490045400094,2.23592036320075 -48415,2007,0,5.19849703126583,2.79490045400094,2.79490045400094 -48417,2003,0,3.58351893845611,1.19452834549794,0.238905669099587 -48417,2004,0,3.55534806148941,1.19452834549794,0.477811338199175 -48417,2005,0,3.66356164612965,1.19452834549794,0.716717007298762 -48417,2006,0,4.15888308335967,1.19452834549794,0.955622676398349 -48417,2007,0,3.93182563272433,1.19452834549794,1.19452834549794 -48447,2003,0,2.56494935746154,0.615185639090233,0.123037127818047 -48447,2004,0,2.19722457733622,0.615185639090233,0.246074255636093 -48447,2005,0,2.484906649788,0.615185639090233,0.36911138345414 -48447,2006,0,2.484906649788,0.615185639090233,0.492148511272187 -48447,2007,0,2.484906649788,0.615185639090233,0.615185639090233 -48461,2003,0,2.77258872223978,1.22495121071113,0.244990242142226 -48461,2004,0,2.30258509299405,1.22495121071113,0.489980484284451 -48461,2005,0,2.77258872223978,1.22495121071113,0.734970726426677 -48461,2006,0,2.83321334405622,1.22495121071113,0.979960968568902 -48461,2007,0,2.56494935746154,1.22495121071113,1.22495121071113 -48465,2003,0,6.01126717440416,3.80345735882137,0.760691471764274 -48465,2004,0,5.82304589548302,3.80345735882137,1.52138294352855 -48465,2005,0,5.78382518232974,3.80345735882137,2.28207441529282 -48465,2006,0,5.80513496891649,3.80345735882137,3.0427658870571 -48465,2007,0,5.92692602597041,3.80345735882137,3.80345735882137 -48477,2003,0,6.32435896238131,3.41355405587153,0.682710811174305 -48477,2004,0,6.32435896238131,3.41355405587153,1.36542162234861 -48477,2005,0,6.23244801655052,3.41355405587153,2.04813243352292 -48477,2006,0,6.22059017009974,3.41355405587153,2.73084324469722 -48477,2007,0,6.39692965521615,3.41355405587153,3.41355405587153 -48493,2003,0,5.55295958492162,3.47840530590039,0.695681061180078 -48493,2004,0,5.50938833662798,3.47840530590039,1.39136212236016 -48493,2005,0,5.46806014113513,3.47840530590039,2.08704318354023 -48493,2006,0,5.73334127689775,3.47840530590039,2.78272424472031 -48493,2007,0,5.69035945432406,3.47840530590039,3.47840530590039 -48503,2003,0,5.48063892334199,2.88720006673053,0.577440013346106 -48503,2004,0,5.4971682252932,2.88720006673053,1.15488002669221 -48503,2005,0,5.51342874616498,2.88720006673053,1.73232004003832 -48503,2006,0,5.54517744447956,2.88720006673053,2.30976005338442 -48503,2007,0,5.51342874616498,2.88720006673053,2.88720006673053 -48505,2003,0,3.87120101090789,2.49995945241429,0.499991890482859 -48505,2004,0,4.12713438504509,2.49995945241429,0.999983780965717 -48505,2005,0,4.26267987704132,2.49995945241429,1.49997567144858 -48505,2006,0,4.39444915467244,2.49995945241429,1.99996756193143 -48505,2007,0,4.33073334028633,2.49995945241429,2.49995945241429 -48507,2003,0,3.40119738166216,2.45100509811232,0.490201019622464 -48507,2004,0,3.43398720448515,2.45100509811232,0.980402039244928 -48507,2005,0,3.3322045101752,2.45100509811232,1.47060305886739 -48507,2006,0,3.40119738166216,2.45100509811232,1.96080407848986 -48507,2007,0,3.55534806148941,2.45100509811232,2.45100509811232 -49003,2003,0,6.84481547920826,3.7552522294782,0.751050445895639 -49003,2004,0,6.79234442747081,3.7552522294782,1.50210089179128 -49003,2005,0,6.79682371827486,3.7552522294782,2.25315133768692 -49003,2006,0,6.82654522355659,3.7552522294782,3.00420178358256 -49003,2007,0,6.93049476595163,3.7552522294782,3.7552522294782 -49011,2003,0,8.52257866369258,5.47643844701387,1.09528768940277 -49011,2004,0,8.49412925181769,5.47643844701387,2.19057537880555 -49011,2005,0,8.51959031601596,5.47643844701387,3.28586306820832 -49011,2006,0,8.59192953753026,5.47643844701387,4.3811507576111 -49011,2007,0,8.6909784171879,5.47643844701387,5.47643844701387 -49023,2003,0,5.29330482472449,2.10875759601389,0.421751519202778 -49023,2004,0,5.22035582507832,2.10875759601389,0.843503038405556 -49023,2005,0,5.27811465923052,2.10875759601389,1.26525455760833 -49023,2006,0,5.46383180502561,2.10875759601389,1.68700607681111 -49023,2007,0,5.48893772615669,2.10875759601389,2.10875759601389 -49037,2003,0,5.01727983681492,2.66813057710097,0.533626115420193 -49037,2004,0,5.12989871492307,2.66813057710097,1.06725223084039 -49037,2005,0,5.07517381523383,2.66813057710097,1.60087834626058 -49037,2006,0,5.23644196282995,2.66813057710097,2.13450446168077 -49037,2007,0,5.01063529409626,2.66813057710097,2.66813057710097 -49053,2003,0,7.84109976542212,4.50373528833274,0.900747057666547 -49053,2004,0,7.82763954636642,4.50373528833274,1.80149411533309 -49053,2005,0,7.92840602618053,4.50373528833274,2.70224117299964 -49053,2006,0,8.06180227453835,4.50373528833274,3.60298823066619 -49053,2007,0,8.08978917578932,4.50373528833274,4.50373528833274 -51003,2003,0,7.08086789669078,4.37243074100043,0.874486148200087 -51003,2004,0,7.12044437239249,4.37243074100043,1.74897229640017 -51003,2005,0,7.06987412845857,4.37243074100043,2.62345844460026 -51003,2006,0,7.18916773842032,4.37243074100043,3.49794459280035 -51003,2007,0,7.27931883541462,4.37243074100043,4.37243074100043 -51015,2003,0,6.46925031679577,4.18380432834825,0.83676086566965 -51015,2004,0,6.39359075395063,4.18380432834825,1.6735217313393 -51015,2005,0,6.46458830368996,4.18380432834825,2.51028259700895 -51015,2006,0,6.49677499018586,4.18380432834825,3.3470434626786 -51015,2007,0,6.48463523563525,4.18380432834825,4.18380432834825 -51017,2003,0,2.89037175789616,1.61899212523891,0.323798425047782 -51017,2004,0,2.39789527279837,1.61899212523891,0.647596850095565 -51017,2005,0,2.77258872223978,1.61899212523891,0.971395275143347 -51017,2006,0,2.56494935746154,1.61899212523891,1.29519370019113 -51017,2007,0,2.30258509299405,1.61899212523891,1.61899212523891 -51019,2003,0,5.80513496891649,4.10050885718995,0.820101771437989 -51019,2004,0,5.82304589548302,4.10050885718995,1.64020354287598 -51019,2005,0,5.89715386763674,4.10050885718995,2.46030531431397 -51019,2006,0,5.94542060860658,4.10050885718995,3.28040708575196 -51019,2007,0,5.88332238848828,4.10050885718995,4.10050885718995 -51023,2003,0,5.56452040732269,3.41759552747179,0.683519105494358 -51023,2004,0,5.50938833662798,3.41759552747179,1.36703821098872 -51023,2005,0,5.65599181081985,3.41759552747179,2.05055731648307 -51023,2006,0,5.89440283426485,3.41759552747179,2.73407642197743 -51023,2007,0,5.98393628068719,3.41759552747179,3.41759552747179 -51025,2003,0,4.64439089914137,2.91338274053697,0.582676548107393 -51025,2004,0,4.54329478227,2.91338274053697,1.16535309621479 -51025,2005,0,4.56434819146784,2.91338274053697,1.74802964432218 -51025,2006,0,4.54329478227,2.91338274053697,2.33070619242957 -51025,2007,0,4.61512051684126,2.91338274053697,2.91338274053697 -51041,2003,0,8.49801037199946,5.56030848448194,1.11206169689639 -51041,2004,0,8.50875771259514,5.56030848448194,2.22412339379278 -51041,2005,0,8.53129331579502,5.56030848448194,3.33618509068916 -51041,2006,0,8.57395152523485,5.56030848448194,4.44824678758555 -51041,2007,0,8.56254889313703,5.56030848448194,5.56030848448194 -51047,2003,0,6.04500531403601,3.53403686830827,0.706807373661653 -51047,2004,0,6.13772705408623,3.53403686830827,1.41361474732331 -51047,2005,0,6.1779441140506,3.53403686830827,2.12042212098496 -51047,2006,0,6.24027584517077,3.53403686830827,2.82722949464661 -51047,2007,0,6.28226674689601,3.53403686830827,3.53403686830827 -51053,2003,0,5.45958551414416,3.2000191500179,0.640003830003579 -51053,2004,0,5.4249500174814,3.2000191500179,1.28000766000716 -51053,2005,0,5.26269018890489,3.2000191500179,1.92001149001074 -51053,2006,0,5.39816270151775,3.2000191500179,2.56001532001432 -51053,2007,0,5.46806014113513,3.2000191500179,3.2000191500179 -51059,2003,0,9.39499254410842,6.87703727512594,1.37540745502519 -51059,2004,0,9.38681151509196,6.87703727512594,2.75081491005038 -51059,2005,0,9.45297277587424,6.87703727512594,4.12622236507556 -51059,2006,0,9.4922055591064,6.87703727512594,5.50162982010075 -51059,2007,0,9.4753933263676,6.87703727512594,6.87703727512594 -51061,2003,0,6.71174039505618,4.0098572697765,0.8019714539553 -51061,2004,0,6.77078942390898,4.0098572697765,1.6039429079106 -51061,2005,0,6.74758652682932,4.0098572697765,2.4059143618659 -51061,2006,0,6.76041469108343,4.0098572697765,3.2078858158212 -51061,2007,0,6.86693328446188,4.0098572697765,4.0098572697765 -51077,2003,0,4.06044301054642,2.88574998281776,0.577149996563552 -51077,2004,0,4.26267987704132,2.88574998281776,1.1542999931271 -51077,2005,0,4.07753744390572,2.88574998281776,1.73144998969066 -51077,2006,0,4.26267987704132,2.88574998281776,2.30859998625421 -51077,2007,0,4.14313472639153,2.88574998281776,2.88574998281776 -51079,2003,0,5.30826769740121,2.72418598301161,0.544837196602323 -51079,2004,0,4.90527477843843,2.72418598301161,1.08967439320465 -51079,2005,0,4.8283137373023,2.72418598301161,1.63451158980697 -51079,2006,0,4.78749174278205,2.72418598301161,2.17934878640929 -51079,2007,0,4.70953020131233,2.72418598301161,2.72418598301161 -51089,2003,0,6.15909538849193,4.05923538510853,0.811847077021707 -51089,2004,0,5.96357934361845,4.05923538510853,1.62369415404341 -51089,2005,0,6.06610809010375,4.05923538510853,2.43554123106512 -51089,2006,0,5.98141421125448,4.05923538510853,3.24738830808683 -51089,2007,0,6.0591231955818,4.05923538510853,4.05923538510853 -51109,2003,0,5.45958551414416,3.24364648319341,0.648729296638683 -51109,2004,0,5.40267738187228,3.24364648319341,1.29745859327737 -51109,2005,0,5.55682806169954,3.24364648319341,1.94618788991605 -51109,2006,0,5.52942908751142,3.24364648319341,2.59491718655473 -51109,2007,0,5.71042701737487,3.24364648319341,3.24364648319341 -51111,2003,0,4.39444915467244,2.57611752984138,0.515223505968277 -51111,2004,0,4.35670882668959,2.57611752984138,1.03044701193655 -51111,2005,0,4.38202663467388,2.57611752984138,1.54567051790483 -51111,2006,0,4.38202663467388,2.57611752984138,2.06089402387311 -51111,2007,0,4.20469261939097,2.57611752984138,2.57611752984138 -51121,2003,0,6.96885037834195,4.4263903498904,0.88527806997808 -51121,2004,0,6.89972310728487,4.4263903498904,1.77055613995616 -51121,2005,0,6.95749737087695,4.4263903498904,2.65583420993424 -51121,2006,0,7.0057890192535,4.4263903498904,3.54111227991232 -51121,2007,0,7.05703698169789,4.4263903498904,4.4263903498904 -51149,2003,0,5.97888576490112,3.49793079061946,0.699586158123893 -51149,2004,0,5.8664680569333,3.49793079061946,1.39917231624779 -51149,2005,0,5.78074351579233,3.49793079061946,2.09875847437168 -51149,2006,0,5.71373280550937,3.49793079061946,2.79834463249557 -51149,2007,0,5.73009978297357,3.49793079061946,3.49793079061946 -51185,2003,0,6.21460809842219,3.79768901497121,0.759537802994242 -51185,2004,0,6.02586597382531,3.79768901497121,1.51907560598848 -51185,2005,0,6.19440539110467,3.79768901497121,2.27861340898273 -51185,2006,0,6.26530121273771,3.79768901497121,3.03815121197697 -51185,2007,0,6.14846829591765,3.79768901497121,3.79768901497121 -51515,2003,0,5.07517381523383,1.84039089063979,0.368078178127958 -51515,2004,0,5.11799381241676,1.84039089063979,0.736156356255917 -51515,2005,0,5.03695260241363,1.84039089063979,1.10423453438388 -51515,2006,0,5.20948615284142,1.84039089063979,1.47231271251183 -51515,2007,0,5.21493575760899,1.84039089063979,1.84039089063979 -51520,2003,0,6.10255859461357,2.85457185376498,0.570914370752995 -51520,2004,0,6.16331480403464,2.85457185376498,1.14182874150599 -51520,2005,0,6.20253551718792,2.85457185376498,1.71274311225899 -51520,2006,0,6.22455842927536,2.85457185376498,2.28365748301198 -51520,2007,0,6.289715570909,2.85457185376498,2.85457185376498 -51630,2003,0,6.81892406527552,2.95901642061733,0.591803284123466 -51630,2004,0,6.92067150424868,2.95901642061733,1.18360656824693 -51630,2005,0,7.02197642307216,2.95901642061733,1.7754098523704 -51630,2006,0,6.99301512293296,2.95901642061733,2.36721313649387 -51630,2007,0,7.00306545878646,2.95901642061733,2.95901642061733 -51650,2003,0,7.62657020629066,4.98659530183624,0.997319060367247 -51650,2004,0,7.6275443904885,4.98659530183624,1.99463812073449 -51650,2005,0,7.70616297019958,4.98659530183624,2.99195718110174 -51650,2006,0,7.65207074611648,4.98659530183624,3.98927624146899 -51650,2007,0,7.57814547241947,4.98659530183624,4.98659530183624 -51660,2003,0,6.86380339145295,3.70051153834364,0.740102307668729 -51660,2004,0,6.85118492749374,3.70051153834364,1.48020461533746 -51660,2005,0,6.90975328164481,3.70051153834364,2.22030692300619 -51660,2006,0,6.9555926083963,3.70051153834364,2.96040923067492 -51660,2007,0,7.01571242048723,3.70051153834364,3.70051153834364 -51680,2003,0,7.21744343169653,4.17851719155535,0.835703438311069 -51680,2004,0,7.1785454837637,4.17851719155535,1.67140687662214 -51680,2005,0,7.2211050981825,4.17851719155535,2.50711031493321 -51680,2006,0,7.27031288607902,4.17851719155535,3.34281375324428 -51680,2007,0,7.37713371283395,4.17851719155535,4.17851719155535 -51700,2003,0,7.6685611080159,5.1937898371941,1.03875796743882 -51700,2004,0,7.73236922228439,5.1937898371941,2.07751593487764 -51700,2005,0,7.67229245562876,5.1937898371941,3.11627390231646 -51700,2006,0,7.73061406606374,5.1937898371941,4.15503186975528 -51700,2007,0,7.78113850984502,5.1937898371941,5.1937898371941 -51710,2003,0,7.98309894071089,5.45704185625577,1.09140837125115 -51710,2004,0,7.95962530509811,5.45704185625577,2.18281674250231 -51710,2005,0,7.95331834656043,5.45704185625577,3.27422511375346 -51710,2006,0,8.00869818298853,5.45704185625577,4.36563348500461 -51710,2007,0,8.0149968943483,5.45704185625577,5.45704185625577 -51735,2003,0,4.85981240436167,2.44806975973603,0.489613951947205 -51735,2004,0,4.91998092582813,2.44806975973603,0.979227903894411 -51735,2005,0,4.96284463025991,2.44806975973603,1.46884185584162 -51735,2006,0,4.99721227376411,2.44806975973603,1.95845580778882 -51735,2007,0,4.98360662170834,2.44806975973603,2.44806975973603 -51770,2003,0,7.6685611080159,4.55293961038444,0.910587922076888 -51770,2004,0,7.59438124255182,4.55293961038444,1.82117584415378 -51770,2005,0,7.60837447438078,4.55293961038444,2.73176376623066 -51770,2006,0,7.66152708135852,4.55293961038444,3.64235168830755 -51770,2007,0,7.73543335249969,4.55293961038444,4.55293961038444 -51790,2003,0,6.21260609575152,3.17190999558746,0.634381999117491 -51790,2004,0,6.19236248947487,3.17190999558746,1.26876399823498 -51790,2005,0,6.1779441140506,3.17190999558746,1.90314599735247 -51790,2006,0,6.19847871649231,3.17190999558746,2.53752799646997 -51790,2007,0,6.31173480915291,3.17190999558746,3.17190999558746 -51800,2003,0,6.60800062529609,4.15382342987363,0.830764685974725 -51800,2004,0,6.70441435496411,4.15382342987363,1.66152937194945 -51800,2005,0,6.71659477352098,4.15382342987363,2.49229405792418 -51800,2006,0,6.77992190747225,4.15382342987363,3.3230587438989 -51800,2007,0,6.83195356556585,4.15382342987363,4.15382342987363 -51830,2003,0,6.45833828334479,2.4847399692309,0.49694799384618 -51830,2004,0,6.4281052726846,2.4847399692309,0.99389598769236 -51830,2005,0,6.35437004079735,2.4847399692309,1.49084398153854 -51830,2006,0,6.42648845745769,2.4847399692309,1.98779197538472 -51830,2007,0,6.35610766069589,2.4847399692309,2.4847399692309 -51840,2003,0,6.96508034560141,3.16061091673622,0.632122183347245 -51840,2004,0,7.02908756414966,3.16061091673622,1.26424436669449 -51840,2005,0,7.00669522683704,3.16061091673622,1.89636655004173 -51840,2006,0,6.98933526597456,3.16061091673622,2.52848873338898 -51840,2007,0,7.02286808608264,3.16061091673622,3.16061091673622 -54005,2003,2007,5.04342511691925,3.24005006009961,0.648010012019921 -54005,2004,2007,4.79579054559674,3.24005006009961,1.29602002403984 -54005,2005,2007,4.72738781871234,3.24005006009961,1.94403003605976 -54005,2006,2007,4.36944785246702,3.24005006009961,2.59204004807969 -54005,2007,2007,4.4188406077966,3.24005006009961,3.24005006009961 -54017,2003,2007,3.36729582998647,2.00188532346096,0.400377064692192 -54017,2004,2007,3.17805383034795,2.00188532346096,0.800754129384385 -54017,2005,2007,2.484906649788,2.00188532346096,1.20113119407658 -54017,2006,2007,2.19722457733622,2.00188532346096,1.60150825876877 -54017,2007,2007,2.07944154167984,2.00188532346096,2.00188532346096 -54021,2003,2007,2.77258872223978,1.96850998097255,0.393701996194511 -54021,2004,2007,2.94443897916644,1.96850998097255,0.787403992389022 -54021,2005,2007,3.52636052461616,1.96850998097255,1.18110598858353 -54021,2006,2007,3.46573590279973,1.96850998097255,1.57480798477804 -54021,2007,2007,3.09104245335832,1.96850998097255,1.96850998097255 -54027,2003,2007,4.89783979995091,3.00583110823141,0.601166221646281 -54027,2004,2007,4.72738781871234,3.00583110823141,1.20233244329256 -54027,2005,2007,4.80402104473326,3.00583110823141,1.80349866493884 -54027,2006,2007,4.96284463025991,3.00583110823141,2.40466488658512 -54027,2007,2007,4.85981240436167,3.00583110823141,3.00583110823141 -54029,2003,2007,5.78382518232974,3.48636539403203,0.697273078806407 -54029,2004,2007,5.73334127689775,3.48636539403203,1.39454615761281 -54029,2005,2007,5.67332326717149,3.48636539403203,2.09181923641922 -54029,2006,2007,5.65948221575962,3.48636539403203,2.78909231522563 -54029,2007,2007,5.60947179518496,3.48636539403203,3.48636539403203 -54037,2003,2007,6.17170059741091,3.74218322613625,0.74843664522725 -54037,2004,2007,6.32435896238131,3.74218322613625,1.4968732904545 -54037,2005,2007,6.36647044773144,3.74218322613625,2.24530993568175 -54037,2006,2007,6.30444880242198,3.74218322613625,2.993746580909 -54037,2007,2007,6.27852142416584,3.74218322613625,3.74218322613625 -54047,2003,2007,4.07753744390572,3.30794840944252,0.661589681888505 -54047,2004,2007,3.73766961828337,3.30794840944252,1.32317936377701 -54047,2005,2007,3.80666248977032,3.30794840944252,1.98476904566552 -54047,2006,2007,4.18965474202643,3.30794840944252,2.64635872755402 -54047,2007,2007,4.06044301054642,3.30794840944252,3.30794840944252 -54051,2003,2007,5.21493575760899,3.57006776457451,0.714013552914902 -54051,2004,2007,5.18738580584075,3.57006776457451,1.4280271058298 -54051,2005,2007,5.37527840768417,3.57006776457451,2.14204065874471 -54051,2006,2007,5.33753807970132,3.57006776457451,2.85605421165961 -54051,2007,2007,5.4249500174814,3.57006776457451,3.57006776457451 -54085,2003,2007,3.71357206670431,2.33630996239567,0.467261992479133 -54085,2004,2007,3.52636052461616,2.33630996239567,0.934523984958267 -54085,2005,2007,3.66356164612965,2.33630996239567,1.4017859774374 -54085,2006,2007,4.20469261939097,2.33630996239567,1.86904796991653 -54085,2007,2007,4.26267987704132,2.33630996239567,2.33630996239567 -54087,2003,2007,3.93182563272433,2.73735007014707,0.547470014029414 -54087,2004,2007,3.8286413964891,2.73735007014707,1.09494002805883 -54087,2005,2007,4.02535169073515,2.73735007014707,1.64241004208824 -54087,2006,2007,4.14313472639153,2.73735007014707,2.18988005611766 -54087,2007,2007,4.38202663467388,2.73735007014707,2.73735007014707 -54099,2003,2007,5.23110861685459,3.75894175355891,0.751788350711782 -54099,2004,2007,5.27811465923052,3.75894175355891,1.50357670142356 -54099,2005,2007,5.15329159449778,3.75894175355891,2.25536505213534 -54099,2006,2007,5.05624580534831,3.75894175355891,3.00715340284713 -54099,2007,2007,5.14166355650266,3.75894175355891,3.75894175355891 -55001,2003,2006,5.08140436498446,2.92547074051332,0.585094148102665 -55001,2004,2006,4.7361984483945,2.92547074051332,1.17018829620533 -55001,2005,2006,4.59511985013459,2.92547074051332,1.75528244430799 -55001,2006,2006,4.8283137373023,2.92547074051332,2.34037659241066 -55001,2007,2006,4.79579054559674,2.92547074051332,2.92547074051332 -55051,2003,2006,4.76217393479776,1.92585320370875,0.385170640741751 -55051,2004,2006,4.70048036579242,1.92585320370875,0.770341281483501 -55051,2005,2006,4.64439089914137,1.92585320370875,1.15551192222525 -55051,2006,2006,4.63472898822964,1.92585320370875,1.540682562967 -55051,2007,2006,4.4188406077966,1.92585320370875,1.92585320370875 -55059,2003,2006,8.00235954625271,5.00781131040515,1.00156226208103 -55059,2004,2006,7.91644286012226,5.00781131040515,2.00312452416206 -55059,2005,2006,7.95050243480885,5.00781131040515,3.00468678624309 -55059,2006,2006,7.9208096792886,5.00781131040515,4.00624904832412 -55059,2007,2006,7.91205688817901,5.00781131040515,5.00781131040515 -55069,2003,2006,6.59578051396131,3.3891585380504,0.67783170761008 -55069,2004,2006,6.55535689181067,3.3891585380504,1.35566341522016 -55069,2005,2006,6.5366915975913,3.3891585380504,2.03349512283024 -55069,2006,2006,6.4150969591716,3.3891585380504,2.71132683044032 -55069,2007,2006,6.45204895443723,3.3891585380504,3.3891585380504 -55081,2003,2006,6.65801104587075,3.7111056128721,0.74222112257442 -55081,2004,2006,6.68710860786651,3.7111056128721,1.48444224514884 -55081,2005,2006,6.62406522779989,3.7111056128721,2.22666336772326 -55081,2006,2006,6.68210859744981,3.7111056128721,2.96888449029768 -55081,2007,2006,6.71901315438526,3.7111056128721,3.7111056128721 -55085,2003,2006,6.72503364216684,3.60484545850393,0.720969091700785 -55085,2004,2006,6.59850902861452,3.60484545850393,1.44193818340157 -55085,2005,2006,6.63463335786169,3.60484545850393,2.16290727510236 -55085,2006,2006,6.58479139238572,3.60484545850393,2.88387636680314 -55085,2007,2006,6.56526497003536,3.60484545850393,3.60484545850393 -55103,2003,2006,5.6937321388027,2.8861405969239,0.577228119384779 -55103,2004,2006,5.54907608489522,2.8861405969239,1.15445623876956 -55103,2005,2006,5.48063892334199,2.8861405969239,1.73168435815434 -55103,2006,2006,5.53338948872752,2.8861405969239,2.30891247753912 -55103,2007,2006,5.55682806169954,2.8861405969239,2.8861405969239 -55109,2003,2006,7.45760928971561,4.14559202222594,0.829118404445188 -55109,2004,2006,7.37713371283395,4.14559202222594,1.65823680889038 -55109,2005,2006,7.35755620091035,4.14559202222594,2.48735521333556 -55109,2006,2006,7.35627987655075,4.14559202222594,3.31647361778075 -55109,2007,2006,7.31121838441963,4.14559202222594,4.14559202222594 -55123,2003,2006,6.04263283368238,3.33420251283788,0.666840502567575 -55123,2004,2006,6.01859321449623,3.33420251283788,1.33368100513515 -55123,2005,2006,5.91350300563827,3.33420251283788,2.00052150770273 -55123,2006,2006,5.89440283426485,3.33420251283788,2.6673620102703 -55123,2007,2006,5.91079664404053,3.33420251283788,3.33420251283788 -55131,2003,2006,8.0870254706677,4.7663787573415,0.9532757514683 -55131,2004,2006,8.0925452638913,4.7663787573415,1.9065515029366 -55131,2005,2006,8.12118324207883,4.7663787573415,2.8598272544049 -55131,2006,2006,8.07340296898641,4.7663787573415,3.8131030058732 -55131,2007,2006,8.11820704940578,4.7663787573415,4.7663787573415 -55137,2003,2006,5.95583736946483,3.14216755178354,0.628433510356707 -55137,2004,2006,5.92425579741453,3.14216755178354,1.25686702071341 -55137,2005,2006,5.9427993751267,3.14216755178354,1.88530053107012 -55137,2006,2006,5.93753620508243,3.14216755178354,2.51373404142683 -55137,2007,2006,5.96357934361845,3.14216755178354,3.14216755178354 diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index 07d7420b3..25b3c8cd7 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -4,30 +4,17 @@ on staggered-adoption data. - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), - plus the negative-weight share. Takes a fitted `CallawaySantAnnaResults` - (reading cohort masses off its aggregation bookkeeping, so no raw panel is - needed); a `(gt_frame, data=, unit=, time=, first_treat=)` fallback - consumes `result.to_dataframe("group_time")` verbatim. Returns - `ATTGTWeightsResult`. `aggregation="twfe"` requires - `base_period="universal"` and `control_group="never_treated"`, matching - the restrictions R enforces. - - `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, - method="fwl", covariates=, base_period="first_period"|"gmin1")` re-derives - the estimate from its ATT(g,t) building blocks and returns - `TWFEDecompositionResult` with `pretrend_bias` — the contribution of - pre-treatment cells, i.e. of parallel-trends violations rather than of - treatment. With `balance_covariates=`, `result.covariate_balance()` - reports whether the implicit weights actually balance those covariates. - - `plot_twfe_weights()` renders either view. - - Names are deliberately separate from the existing `twowayfeweights` / - `TWFEWeightsResult` (de Chaisemartin & D'Haultfoeuille) surface, which - weights (unit, time) cells rather than ATT(g,t) parameters. - - Validated against R `twfeweights` 0.9.0 output on three fixtures (`mpdta` - plus two simulated panels); goldens at - `benchmarks/data/twfeweights_golden.json`, regenerated by - `benchmarks/R/generate_twfeweights_golden.R`. R is never needed to run the - test suite. Methodology: Baker, Callaway, Cunningham, Goodman-Bacon & - Sant'Anna (2025); Callaway & Sant'Anna (2021) for the ATT^O / ATT^simple - weights. + plus post-period negative-weight counts. Returns `ATTGTWeightsResult`. + - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate + from its ATT(g,t) building blocks and returns `TWFEDecompositionResult` + with `pretrend_bias` - the contribution of pre-treatment cells, i.e. of + parallel-trends violations rather than of treatment - and, with + `balance_covariates=`, implicit-weight covariate balance. + `plot_twfe_weights()` renders either view (matplotlib or plotly). + - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits + under `aggregation="twfe"`, duplicated or non-finite ATT(g,t) cells, an + incomplete group-time grid, and invalid sampling weights. Two structural + gaps are handled as R does instead of raising: a cohort with no estimable + post cell is dropped (`did`'s first-period drop), and under + `control_group="not_yet_treated"` the CS estimands average over each + cohort's available post periods (`aggte`). diff --git a/diff_diff/dml_did.py b/diff_diff/dml_did.py index 16e6f4bdf..bb2d17a15 100644 --- a/diff_diff/dml_did.py +++ b/diff_diff/dml_did.py @@ -3362,6 +3362,7 @@ def fit( # stays admitted). is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.results_ = results self.is_fitted_ = True diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 8a51fedb6..d9ccc3515 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1563,10 +1563,44 @@ decompose_twfe_weights( plot_twfe_weights(result, kind="auto") # "weights" | "balance" ``` -`aggregation="twfe"` requires a fit with `base_period="universal"` and -`control_group="never_treated"`; it raises otherwise. ATT^O and ATT^simple -weights are non-negative and sum to one, so comparing `implied_att` across -the three aggregations shows what the TWFE specification costs. +`aggregation="twfe"` requires a fit with `base_period="universal"`, +`control_group="never_treated"` AND no covariates (R twfe_weights' three +restrictions); it raises otherwise. ATT^O and ATT^simple weights are +non-negative and sum to one, so comparing `implied_att` across the three +aggregations shows what the TWFE specification costs. + +Both entry points fail closed on input R never faced: NaN / `-inf` cohort +labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite +ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x +period cell, the CS estimands every post cell), and sampling weights that are +not finite, non-negative and positive-mass. Two structural gaps mirror R +rather than raising, each with a `UserWarning`: a cohort with no estimable +post cell is dropped from the table and the cohort shares (`did`'s +first-period drop), and under `control_group="not_yet_treated"` the cells CS +marks `zero_treated_control` are treated as structurally absent, so +`"overall"`/`"simple"` average over each cohort's AVAILABLE post periods +(`aggte`). `n_negative_post` / `negative_post_weight_share` report the +pathology (negative weight on POST cells); `n_negative` counts pre cells too, +and is near-half in every staggered design because the TWFE weights sum to +zero over the full grid. + +### plot_twfe_weights + +```python +plot_twfe_weights( + results, # ATTGTWeightsResult | TWFEDecompositionResult + kind="auto", # "weights" | "balance" ("auto" picks balance + # when the result carries a balance table) + standardize=True, absolute_value=True, # balance view + annotate=False, ax=None, show=True, + backend="matplotlib", # or "plotly" +) +``` + +`kind="weights"` scatters weight against ATT(g,t), coloured by pre/post - points +left of the vertical zero line carry negative weight. `kind="balance"` scatters +unweighted against implicitly-weighted covariate differences; points near the +horizontal axis are covariates the implicit weights balance. `decompose_twfe_weights` takes the raw panel rather than a fitted result because it re-estimates. It is tied to `attgt_weights` by an identity: diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 37a544907..8278d950c 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -7,7 +7,7 @@ import bisect import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple import numpy as np import pandas as pd @@ -3056,6 +3056,7 @@ def fit( group_time_effects, is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.is_fitted_ = True @@ -5151,6 +5152,7 @@ def _build_aggregation_kit( *, is_survey_fit: bool = False, bootstrap_results: Optional["CSBootstrapResults"] = None, + covariates: Optional[Sequence[str]] = None, ) -> Optional["AggregationKit"]: """Distil the fit-time state post-fit re-aggregation needs. @@ -5185,6 +5187,11 @@ def _build_aggregation_kit( # (or DDD) survey fit does not warn as "CallawaySantAnna" on post-fit # aggregate(). Legacy kits without the key default at the read site. bookkeeping["bootstrap_label"] = getattr(estimator, "_BOOTSTRAP_LABEL", "CallawaySantAnna") + # Covariate usage, recorded so downstream diagnostics can refuse designs + # their formulas do not cover (``attgt_weights(aggregation="twfe")`` + # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES + # only - never values - so the data-minimization contract holds. + bookkeeping["covariates"] = tuple(covariates or ()) # Data minimization: the results object is picklable and users share # result artifacts, so the kit must not turn it into a carrier for raw diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index 7464c6012..eda573824 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -55,12 +55,13 @@ import numpy as np import pandas as pd -from scipy.linalg import qr as scipy_qr +from diff_diff.linalg import solve_ols from diff_diff.twfe_weights_results import ( ATTGTWeightsResult, TWFEDecompositionResult, ) +from diff_diff.utils import within_transform if TYPE_CHECKING: # pragma: no cover - typing only from diff_diff.staggered_results import CallawaySantAnnaResults @@ -74,10 +75,40 @@ def _is_never(values: np.ndarray) -> np.ndarray: """Boolean mask for never-treated cohort labels. diff-diff and R ``did`` have both used ``0`` and ``+inf`` as the - never-treated sentinel over time; accept either and normalize to ``0``. + never-treated sentinel over time; accept exactly those two and normalize + to ``0``. Every OTHER non-finite label (NaN, ``-inf``) is an input error - + see :func:`_validate_cohort_labels` - not a never-treated unit. """ arr = np.asarray(values, dtype=float) - return ~np.isfinite(arr) | (arr == 0) + return (arr == 0) | (arr == np.inf) + + +def _validate_cohort_labels( + values: np.ndarray, *, unit_ids: Optional[np.ndarray] = None, what: str = "first_treat" +) -> None: + """Reject NaN / ``-inf`` cohort labels instead of silently treating them as never-treated.""" + arr = np.asarray(values, dtype=float) + bad = np.isnan(arr) | (arr == -np.inf) + if bad.any(): + idx = np.flatnonzero(bad)[:5] + who = [unit_ids[i] for i in idx] if unit_ids is not None else idx.tolist() + raise ValueError( + f"{what!r} contains NaN or -inf cohort label(s) for unit(s) {who!r}; " + "never-treated units must be coded exactly 0 or +inf, and every " + "other unit needs a finite first-treatment period" + ) + + +def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> None: + """Reject NaN / non-finite period labels before any grid is formed.""" + arr = pd.to_numeric(pd.Series(np.asarray(values)), errors="coerce").to_numpy(dtype=float) + bad = ~np.isfinite(arr) + if bad.any(): + raise ValueError( + f"{what!r} contains {int(bad.sum())} non-finite or non-numeric period " + f"label(s) (first at row {int(np.flatnonzero(bad)[0])}); every observation " + "must carry a finite period" + ) def _positional_grid( @@ -117,10 +148,42 @@ def _to_positional_cohort(cohorts: np.ndarray, grid: Dict[float, int]) -> np.nda return out +def _validate_unit_weights( + w: np.ndarray, is_never: np.ndarray, *, require_control_mass: bool +) -> None: + """Shared contract for unit-level sampling weights. + + Finite, non-negative, positive total, positive TREATED mass; positive + never-treated mass only where the never-treated group enters the formula + (``aggregation="twfe"`` and the decomposition) - ATT^O / ATT^simple are + defined without one. + """ + if not np.all(np.isfinite(w)): + raise ValueError("unit weights must be finite; got NaN or infinite weight(s)") + if (w < 0).any(): + idx = np.flatnonzero(w < 0)[:5].tolist() + raise ValueError( + f"unit weights must be non-negative; negative weight(s) at unit index {idx!r}" + ) + if w.sum() <= 0: + raise ValueError("unit weights sum to zero; cannot form cohort shares") + if w[~is_never].sum() <= 0: + raise ValueError( + "the ever-treated units carry zero total weight; cannot form cohort shares" + ) + if require_control_mass and w[is_never].sum() <= 0: + raise ValueError( + "the never-treated comparison group carries zero total weight, so " + "every group-time contrast is undefined" + ) + + def _cohort_masses( unit_cohorts: np.ndarray, grid: Dict[float, int], weights: Optional[np.ndarray], + *, + require_control_mass: bool = False, ) -> Tuple[Dict[int, float], Dict[int, float], Dict[int, float], float]: """Cohort shares and treated-share-by-period, all in positional time. @@ -140,19 +203,15 @@ def _cohort_masses( w = np.ones(len(g_pos)) if weights is None else np.asarray(weights, dtype=float) if len(w) != len(g_pos): raise ValueError(f"weights has length {len(w)} but there are {len(g_pos)} units") + _validate_unit_weights(w, g_pos == 0, require_control_mass=require_control_mass) total = w.sum() - if total <= 0: - raise ValueError("unit weights sum to zero; cannot form cohort shares") treated = g_pos != 0 treated_mass = w[treated].sum() cohorts = sorted({int(g) for g in g_pos if g != 0}) p_all = {g: float(w[g_pos == g].sum() / total) for g in cohorts} - if treated_mass > 0: - p_treated = {g: float(w[g_pos == g].sum() / treated_mass) for g in cohorts} - else: # pragma: no cover - guarded upstream by the never-treated check - p_treated = {g: 0.0 for g in cohorts} + p_treated = {g: float(w[g_pos == g].sum() / treated_mass) for g in cohorts} periods = sorted(grid.values()) e_dt = {t: float(w[treated & (g_pos <= t)].sum() / total) for t in periods} @@ -197,17 +256,22 @@ def _overall_weight_vector( times: np.ndarray, n_periods: int, p_treated: Dict[int, float], + n_post_available: Optional[Dict[int, int]] = None, ) -> np.ndarray: """ATT^O weights: ``1[t >= g] * pbar_g / (maxT - g + 1)``. Not renormalized - the ``(maxT - g + 1)`` divisor already makes them sum - to one over a complete post-treatment grid. + to one over a complete post-treatment grid. ``n_post_available`` replaces + that divisor with each cohort's number of AVAILABLE post periods when + some post cells are structurally absent (``control_group="not_yet_treated"`` + runs out of comparison units) - what R ``aggte(type="group")`` averages + over on such a fit. """ - return ( - (times >= groups).astype(float) - * np.array([p_treated[int(g)] for g in groups]) - / (n_periods - groups + 1.0) - ) + if n_post_available is None: + divisor = n_periods - groups + 1.0 + else: + divisor = np.array([float(n_post_available[int(g)]) for g in groups]) + return (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor def _simple_weight_vector( @@ -228,35 +292,42 @@ def _simple_weight_vector( def _attgt_from_cs( results: "CallawaySantAnnaResults", -) -> Tuple[pd.DataFrame, int]: +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: """Extract the ``(g, t, att)`` table from a fitted CS result. - Non-estimable cells (``skip_reason`` set, NaN effect) are dropped and - counted, so a partially-estimable fit still produces weights over the - cells that exist rather than propagating NaN through every aggregate. + Non-estimable cells (``skip_reason`` set, NaN effect) are left out of the + table and reported in the returned ``{(g, t): skip_reason}`` map, so the + caller can decide - per aggregation - whether the gap is structural, a + harmless pre-period drop, or a hard error. """ rows: List[Dict[str, Any]] = [] - dropped = 0 + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} for (g, t), cell in results.group_time_effects.items(): effect = cell.get("effect", np.nan) if cell.get("skip_reason") is not None or not np.isfinite(effect): - dropped += 1 + skipped[(g, t)] = cell.get("skip_reason") continue rows.append({"group": g, "time": t, "att": float(effect)}) if not rows: raise ValueError( - "the fitted result has no estimable group-time cells; there is " "nothing to weight" + "the fitted result has no estimable group-time cells; there is nothing to weight" ) table = pd.DataFrame(rows).sort_values(["group", "time"]).reset_index(drop=True) - return table, dropped + return table, skipped -def _attgt_from_frame(frame: pd.DataFrame) -> Tuple[pd.DataFrame, int]: +def _attgt_from_frame( + frame: pd.DataFrame, +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: """Extract ``(g, t, att)`` from a user-supplied ATT(g, t) frame. ``effect`` is preferred over ``att`` because that is the column ``CallawaySantAnnaResults.to_dataframe("group_time")`` emits - so the - fallback consumes our own frame verbatim. + fallback consumes our own frame verbatim, including its ``skip_reason`` + column when present. Duplicate cells and non-finite ``group`` / ``time`` + labels are rejected; a non-finite effect is reported in the skip map, not + silently kept (an ``inf`` ATT would otherwise propagate into + ``implied_att``). """ missing = {"group", "time"} - set(frame.columns) if missing: @@ -272,18 +343,38 @@ def _attgt_from_frame(frame: pd.DataFrame) -> Tuple[pd.DataFrame, int]: raise ValueError( "ATT(g,t) frame must carry an 'effect' or 'att' column; got " f"{list(frame.columns)!r}" ) + groups = pd.to_numeric(frame["group"], errors="coerce").to_numpy(dtype=float) + times = pd.to_numeric(frame["time"], errors="coerce").to_numpy(dtype=float) + _validate_cohort_labels(groups, what="group") + _validate_time_labels(frame["time"].to_numpy(), what="time") + key = pd.MultiIndex.from_arrays([frame["group"].to_numpy(), frame["time"].to_numpy()]) + if key.duplicated().any(): + dupes = sorted({tuple(k) for k in key[key.duplicated()].tolist()})[:5] + raise ValueError( + f"ATT(g,t) frame has duplicated (group, time) cell(s) {dupes!r}; each " + "cell must appear exactly once" + ) + att = pd.to_numeric(frame[value_col], errors="coerce").to_numpy(dtype=float) + reasons = ( + frame["skip_reason"].tolist() if "skip_reason" in frame.columns else [None] * len(frame) + ) table = pd.DataFrame( - { - "group": frame["group"].to_numpy(), - "time": frame["time"].to_numpy(), - "att": pd.to_numeric(frame[value_col], errors="coerce").to_numpy(), - } + {"group": frame["group"].to_numpy(), "time": frame["time"].to_numpy(), "att": att} ) - dropped = int(table["att"].isna().sum()) - table = table.dropna(subset=["att"]) + finite = np.isfinite(att) + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} + for i in np.flatnonzero(~finite): + reason = reasons[i] + skipped[(table["group"].iat[i], table["time"].iat[i])] = ( + None + if reason is None or (isinstance(reason, float) and np.isnan(reason)) + else str(reason) + ) + table = table[finite] if table.empty: raise ValueError("ATT(g,t) frame has no finite effects to weight") - return table.sort_values(["group", "time"]).reset_index(drop=True), dropped + _ = groups, times # validated above; positional mapping happens in the caller + return table.sort_values(["group", "time"]).reset_index(drop=True), skipped def _unit_cohorts_from_frame( @@ -293,14 +384,19 @@ def _unit_cohorts_from_frame( for col in (unit, time, first_treat): if col not in data.columns: raise ValueError(f"column {col!r} not found in data") - per_unit = data.groupby(unit, sort=True)[first_treat].nunique() + _validate_time_labels(data[time].to_numpy(), what=time) + # dropna=False: a unit whose label is NaN in one period must fail the + # invariance check, not slip through because nunique() skipped the NaN. + per_unit = data.groupby(unit, sort=True)[first_treat].nunique(dropna=False) if (per_unit > 1).any(): offenders = per_unit[per_unit > 1].index.tolist()[:5] raise ValueError( f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " "membership must be time-invariant" ) - cohorts = data.groupby(unit, sort=True)[first_treat].first().to_numpy() + firsts = data.groupby(unit, sort=True)[first_treat].first() + cohorts = firsts.to_numpy() + _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) periods = np.asarray(sorted(data[time].unique())) return cohorts, periods, None @@ -341,6 +437,13 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No These are hard errors rather than warnings: a silently wrong weight table is worse than no weight table, and every one of these has a concrete fix. """ + from diff_diff.staggered_results import CallawaySantAnnaResults + + if not isinstance(results, CallawaySantAnnaResults): + raise TypeError( + "attgt_weights takes a CallawaySantAnna (or DMLDiD) fitted result, or an " + f"ATT(g,t) DataFrame; got {type(results).__name__}" + ) if not getattr(results, "panel", True): raise ValueError( "attgt_weights requires a panel fit: E_t[D] and the cohort shares " @@ -371,6 +474,31 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No "grid, including the pre-treatment cells that a varying base does " "not report. Refit with base_period='universal'." ) + # R's third restriction: xformla == ~1. The fit records its covariate + # column names on the aggregation kit; a kit without the key predates that + # bookkeeping (an old pickle) and can only be warned about. A missing kit + # is left to _resolve_cs_inputs, whose error is the useful one. + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + return + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + if "covariates" not in bookkeeping: + warnings.warn( + "this fit predates covariate bookkeeping, so attgt_weights cannot " + "verify it used no covariates; the TWFE weight formula assumes an " + "unadjusted regression (R twfe_weights requires xformla == ~1)", + UserWarning, + stacklevel=3, + ) + elif bookkeeping["covariates"]: + raise ValueError( + f"aggregation='twfe' requires a fit without covariates, but this one " + f"adjusted for {list(bookkeeping['covariates'])!r}. The TWFE weight " + "formula describes the unadjusted regression (R's twfe_weights stops " + "unless xformla == ~1); refit with covariates=None, or use " + "decompose_twfe_weights(covariates=...) for the covariate-adjusted " + "decomposition." + ) def attgt_weights( @@ -396,9 +524,13 @@ def attgt_weights( ---------- results : CallawaySantAnnaResults or pd.DataFrame A fitted Callaway & Sant'Anna result (preferred), or a frame with - ``group`` / ``time`` / ``effect`` (or ``att``) columns. On the frame - path, ``data``, ``unit``, ``time`` and ``first_treat`` are required - so cohort shares can be formed. + ``group`` / ``time`` / ``effect`` (or ``att``) columns - the output of + ``result.to_dataframe("group_time")`` is consumed verbatim, including + its ``skip_reason`` column. On the frame path, ``data``, ``unit``, + ``time`` and ``first_treat`` are required so cohort shares can be + formed, and the caller is responsible for the fit having used no + covariates under ``aggregation="twfe"`` (a frame carries no record of + that; the fitted path checks it). aggregation : {"twfe", "overall", "simple"}, default "twfe" Which estimand's weights to report. data : pd.DataFrame, optional @@ -409,23 +541,46 @@ def attgt_weights( weights : str or array-like, optional Unit-level sampling weights (R's ``w=``): a column name in ``data``, or one value per unit. Rejected when the fit already carries survey - weights, which take precedence. + weights, which take precedence. Must be finite and non-negative with + positive treated mass (and positive never-treated mass for ``"twfe"``). Returns ------- ATTGTWeightsResult - Per-cell weights plus the negative-weight roll-up. + Per-cell weights plus the negative-weight roll-ups. Raises ------ ValueError On an unknown ``aggregation``; on a design the formula does not support (repeated cross-sections, unbalanced fallback, and - for - ``aggregation="twfe"`` - a non-never-treated control group or a - non-universal base period); or on an incomplete fallback spec. + ``aggregation="twfe"`` - a non-never-treated control group, a + non-universal base period, or a covariate-adjusted fit); on NaN / + ``-inf`` cohort labels, invalid weights, duplicated or non-finite + cells; or on an INCOMPLETE grid: ``"twfe"`` needs every cohort x period + cell, ``"overall"`` / ``"simple"`` every post-treatment cell. + TypeError + When ``results`` is neither a CallawaySantAnna-family result nor a + DataFrame. Notes ----- + Two structural gaps are handled rather than raised, mirroring R: + + * A cohort with NO estimable post-treatment cell (typically one treated in + the first observed period, which has no base period) is dropped from the + table AND from the cohort masses with a warning - what + ``did::pre_process_did`` does when it drops units already treated in the + first period. + * Under ``control_group="not_yet_treated"`` the last cohorts run out of + comparison units, and CS marks those post cells ``zero_treated_control``. + For ``"overall"`` / ``"simple"`` they are treated as structurally absent: + ``"overall"`` divides each cohort by its number of AVAILABLE post periods + and ``"simple"`` renormalizes over the available post cells - what + R ``aggte()`` computes on such a fit. A warning names the cells. + (``"twfe"`` requires a never-treated control group and never reaches + this branch.) + R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows with ``attgt = 0`` to mirror an internal vector layout; those rows are excluded from every normalization and contribute exactly zero, so the @@ -446,6 +601,7 @@ def attgt_weights( ) frame_path = isinstance(results, pd.DataFrame) + frame = results if isinstance(results, pd.DataFrame) else None fallback_args = {"data": data, "unit": unit, "time": time, "first_treat": first_treat} supplied = {k: v for k, v in fallback_args.items() if v is not None} @@ -460,12 +616,14 @@ def attgt_weights( ) assert data is not None and unit is not None assert time is not None and first_treat is not None - table, dropped = _attgt_from_frame(results) + assert frame is not None + table, skipped = _attgt_from_frame(frame) cohorts, periods, _ = _unit_cohorts_from_frame(data, unit, time, first_treat) unit_weights = _resolve_frame_weights(weights, data, unit) source = "DataFrame" control_group = None base_period = None + has_skip_reasons = "skip_reason" in frame.columns else: if supplied: raise ValueError( @@ -475,7 +633,7 @@ def attgt_weights( "result.to_dataframe('group_time') as the first argument." ) _guard_cs_design(results, aggregation) - table, dropped = _attgt_from_cs(results) + table, skipped = _attgt_from_cs(results) cohorts, survey_weights = _resolve_cs_inputs(results) if survey_weights is not None and weights is not None: raise ValueError( @@ -495,32 +653,126 @@ def attgt_weights( source = "CallawaySantAnnaResults" control_group = getattr(results, "control_group", None) base_period = getattr(results, "base_period", None) + has_skip_reasons = True - if dropped: - warnings.warn( - f"{dropped} group-time cell(s) had no estimable ATT(g,t) and were " - "excluded from the weight table; the reported weights renormalize " - "over the remaining cells", - UserWarning, - stacklevel=2, - ) - + _validate_cohort_labels(cohorts, what="first_treat") grid = _positional_grid(periods) n_periods = len(grid) - p_all, p_treated, e_dt, mean_e_dt = _cohort_masses(cohorts, grid, unit_weights) - if not p_treated: + first_period_pos = 1 + + # Positional mapping FIRST: the cohort universe the masses are formed over + # must be known before the masses are formed. + unit_g_pos = _to_positional_cohort(cohorts, grid) + if not (unit_g_pos != 0).any(): raise ValueError( "no ever-treated units found; cohort labels are all never-treated " "sentinels (0 or inf)" ) - g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + post_mask = t_pos >= g_pos + + # --- whole-cohort exclusion (R did drops units treated in the first period) + panel_cohorts = sorted({int(g) for g in unit_g_pos if g != 0}) + cohorts_with_post = {int(g) for g in g_pos[post_mask]} + excluded = [g for g in panel_cohorts if g not in cohorts_with_post] + if excluded: + if not has_skip_reasons: + not_structural = [g for g in excluded if g != first_period_pos] + if not_structural: + labels = [_label_for(grid, g) for g in not_structural] + raise ValueError( + f"cohort(s) {labels!r} are present in data= but have no " + "post-treatment cell in the ATT(g,t) frame. A bare frame " + "cannot say why; pass result.to_dataframe('group_time') " + "verbatim (it carries skip_reason) or the fitted result itself." + ) + n_units_excl = int(np.isin(unit_g_pos, excluded).sum()) + warnings.warn( + f"cohort(s) {[_label_for(grid, g) for g in excluded]!r} ({n_units_excl} " + "unit(s)) have no estimable post-treatment cell and were dropped from " + "the weight table and the cohort shares, matching R did's drop of units " + "already treated in the first observed period", + UserWarning, + stacklevel=2, + ) + keep_units = ~np.isin(unit_g_pos, excluded) + cohorts = cohorts[keep_units] + unit_g_pos = unit_g_pos[keep_units] + if unit_weights is not None: + unit_weights = np.asarray(unit_weights, dtype=float)[keep_units] + keep_rows = ~np.isin(g_pos, excluded) + table = table[keep_rows].reset_index(drop=True) + g_pos, t_pos, post_mask = g_pos[keep_rows], t_pos[keep_rows], post_mask[keep_rows] + skipped = {k: v for k, v in skipped.items() if _pos_of(grid, k[0]) not in excluded} + + p_all, p_treated, e_dt, mean_e_dt = _cohort_masses( + cohorts, grid, unit_weights, require_control_mass=(aggregation == "twfe") + ) + + # --- grid completeness + present = set(zip(g_pos.tolist(), t_pos.tolist())) + surviving = sorted(cohorts_with_post) + if aggregation == "twfe": + required = {(g, t) for g in surviving for t in range(1, n_periods + 1)} + else: + required = {(g, t) for g in surviving for t in range(g, n_periods + 1)} + missing_cells = sorted(required - present) + structurally_absent: List[Tuple[Any, Any]] = [] + if missing_cells: + carve_out_ok = aggregation != "twfe" and control_group == "not_yet_treated" + hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] + for g, t in missing_cells: + label = (_label_for(grid, g), _label_for(grid, t)) + reason = skipped.get(label) + if carve_out_ok and reason == "zero_treated_control": + structurally_absent.append(label) + else: + hard.append((label, reason)) + if hard: + what = "cohort x period" if aggregation == "twfe" else "post-treatment" + detail = ", ".join( + f"{lab} [{reason or 'not in source table'}]" for lab, reason in hard[:6] + ) + raise ValueError( + f"aggregation={aggregation!r} needs the complete {what} grid, but " + f"{len(hard)} required cell(s) are missing: {detail}. A weight table " + "over a partial grid is not the named estimand. Fix the source fit " + "(or pass the complete to_dataframe('group_time') output)." + ) + warnings.warn( + f"{len(structurally_absent)} post-treatment cell(s) {structurally_absent[:6]!r} " + "have no not-yet-treated comparison units (skip_reason " + "'zero_treated_control') and are treated as structurally absent: " + f"aggregation={aggregation!r} averages over each cohort's AVAILABLE " + "post periods, as R aggte() does on a not-yet-treated fit", + UserWarning, + stacklevel=2, + ) + + # Non-estimable PRE cells of surviving cohorts are the only drops left; + # the CS estimands ignore pre cells, so they change nothing. + dropped = sum( + 1 + for (g_lab, t_lab) in skipped + if _pos_of(grid, g_lab) in cohorts_with_post and _pos_of(grid, t_lab) < _pos_of(grid, g_lab) + ) + if dropped and aggregation != "twfe": + warnings.warn( + f"{dropped} pre-treatment group-time cell(s) had no estimable ATT(g,t) " + f"and were excluded; aggregation={aggregation!r} places no weight on " + "pre-treatment cells, so the weights are unaffected", + UserWarning, + stacklevel=2, + ) if aggregation == "twfe": weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) elif aggregation == "overall": - weight_vec = _overall_weight_vector(g_pos, t_pos, n_periods, p_treated) + n_post_available = None + if structurally_absent: + n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} + weight_vec = _overall_weight_vector(g_pos, t_pos, n_periods, p_treated, n_post_available) else: weight_vec = _simple_weight_vector(g_pos, t_pos, p_treated) @@ -528,7 +780,7 @@ def attgt_weights( { "group": table["group"].to_numpy(), "time": table["time"].to_numpy(), - "post": (t_pos >= g_pos).astype(int), + "post": post_mask.astype(int), "weight": weight_vec, "att": table["att"].to_numpy(), } @@ -536,6 +788,8 @@ def attgt_weights( negative = weight_vec < 0 abs_total = float(np.abs(weight_vec).sum()) + negative_post = negative & post_mask + abs_post_total = float(np.abs(weight_vec[post_mask]).sum()) return ATTGTWeightsResult( weights=out, aggregation=aggregation, @@ -544,6 +798,12 @@ def attgt_weights( negative_weight_share=( float(np.abs(weight_vec[negative]).sum() / abs_total) if abs_total > 0 else 0.0 ), + n_negative_post=int(negative_post.sum()), + negative_post_weight_share=( + float(np.abs(weight_vec[negative_post]).sum() / abs_post_total) + if abs_post_total > 0 + else 0.0 + ), n_cells=len(out), source=source, control_group=control_group, @@ -552,6 +812,25 @@ def attgt_weights( ) +def _label_for(grid: Dict[float, int], pos: int) -> Any: + """Positional period -> original label (inverse of ``_positional_grid``).""" + for label, p in grid.items(): + if p == pos: + return int(label) if float(label).is_integer() else label + return pos + + +def _pos_of(grid: Dict[float, int], label: Any) -> int: + """Original label -> positional period; never-treated sentinel stays 0.""" + try: + value = float(label) + except (TypeError, ValueError): + return -1 + if value == 0 or value == np.inf: + return 0 + return grid.get(value, -1) + + def _resolve_frame_weights( weights: Optional[Union[str, np.ndarray]], data: pd.DataFrame, @@ -563,7 +842,7 @@ def _resolve_frame_weights( if isinstance(weights, str): if weights not in data.columns: raise ValueError(f"weights column {weights!r} not found in data") - per_unit = data.groupby(unit, sort=True)[weights].nunique() + per_unit = data.groupby(unit, sort=True)[weights].nunique(dropna=False) if (per_unit > 1).any(): offenders = per_unit[per_unit > 1].index.tolist()[:5] raise ValueError( @@ -587,78 +866,6 @@ def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: return float((values * weights).sum() / total) -def _demean_two_way( - values: np.ndarray, - weights: np.ndarray, - *, - tol: float = 1e-12, - max_iter: int = 100, -) -> np.ndarray: - """Two-way (unit and period) demeaning of a ``(n_units, n_periods, k)`` block. - - Alternating projections, matching what ``fixest::demean`` does. On a - balanced panel with uniform weights this converges after a single sweep - to the closed form ``x - xbar_i - xbar_t + xbar``; the loop exists so - sampling weights (which break that identity) are still handled exactly - rather than approximately. - - ``weights`` is ``(n_units, n_periods)`` and broadcasts over the trailing - covariate axis. - """ - out = np.array(values, dtype=float, copy=True) - if out.size == 0: - return out - w = weights[:, :, None] - for _ in range(max_iter): - unit_mass = w.sum(axis=1, keepdims=True) - out -= np.divide( - (out * w).sum(axis=1, keepdims=True), - unit_mass, - out=np.zeros_like(unit_mass), - where=unit_mass > 0, - ) - time_mass = w.sum(axis=0, keepdims=True) - shift = np.divide( - (out * w).sum(axis=0, keepdims=True), - time_mass, - out=np.zeros_like(time_mass), - where=time_mass > 0, - ) - out -= shift - if np.max(np.abs(shift)) < tol: - break - return out - - -def _drop_collinear(matrix: np.ndarray) -> Tuple[np.ndarray, List[int]]: - """Drop linearly dependent columns via a pivoted QR. - - Mirrors ``BMisc::drop_collinear`` (which delegates to - ``caret::findLinearCombos``) in effect: keep a maximal independent set, - dropping later columns first. - """ - if matrix.shape[1] == 0: - return matrix, [] - _, r_mat, piv = scipy_qr(matrix, mode="economic", pivoting=True) - diag = np.abs(np.diag(r_mat)) - if diag.size == 0: - return matrix[:, :0], list(range(matrix.shape[1])) - tol = diag.max() * max(matrix.shape) * np.finfo(float).eps - rank = int((diag > tol).sum()) - keep = sorted(piv[:rank].tolist()) - dropped = [j for j in range(matrix.shape[1]) if j not in keep] - return matrix[:, keep], dropped - - -def _wls_coefficients(design: np.ndarray, target: np.ndarray, weights: np.ndarray) -> np.ndarray: - """Weighted least squares through the origin (R's ``lm(y ~ -1 + X, w)``).""" - if design.shape[1] == 0: - return np.zeros(0) - root_w = np.sqrt(weights) - coef, *_ = np.linalg.lstsq(design * root_w[:, None], target * root_w, rcond=None) - return coef - - def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray) -> float: """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors.""" sw = sampling_weights / sampling_weights.mean() @@ -694,6 +901,7 @@ def __init__( if weights is not None and weights not in data.columns: raise ValueError(f"weights column {weights!r} not found in data") + _validate_time_labels(data[time].to_numpy(), what=time) frame = data.sort_values([unit, time]).reset_index(drop=True) units = frame[unit].to_numpy() periods = frame[time].to_numpy() @@ -720,26 +928,42 @@ def __init__( self.n_periods = n_periods cohort_long = frame[first_treat].to_numpy() - per_unit = frame.groupby(unit, sort=True)[first_treat].nunique() + # dropna=False: a NaN label in one period must fail invariance, not + # be skipped by nunique(). + per_unit = frame.groupby(unit, sort=True)[first_treat].nunique(dropna=False) if (per_unit > 1).any(): offenders = per_unit[per_unit > 1].index.tolist()[:5] raise ValueError( f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " "membership must be time-invariant" ) - self.cohorts = _to_positional_cohort( - cohort_long.reshape(n_units, n_periods)[:, 0], self.grid - ) + raw_cohorts = cohort_long.reshape(n_units, n_periods)[:, 0] + _validate_cohort_labels(raw_cohorts, unit_ids=self.unit_ids, what=first_treat) + self.cohorts = _to_positional_cohort(raw_cohorts, self.grid) + if not (self.cohorts == 0).any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) self.outcome = frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods) if weights is None: self.weights = np.ones((n_units, n_periods)) else: - self.weights = frame[weights].to_numpy(dtype=float).reshape(n_units, n_periods) - if not np.allclose(self.weights, self.weights[:, :1]): + block = frame[weights].to_numpy(dtype=float).reshape(n_units, n_periods) + # Finite check FIRST: np.allclose is False on any NaN, which would + # otherwise be misreported as "varies within unit". + if not np.all(np.isfinite(block)): + raise ValueError( + f"weights column {weights!r} must be finite; got NaN or infinite weight(s)" + ) + if not np.allclose(block, block[:, :1]): raise ValueError( f"weights column {weights!r} varies within unit; sampling " "weights must be time-invariant" ) + _validate_unit_weights(block[:, 0], self.cohorts == 0, require_control_mass=True) + self.weights = block self.covariates = tuple(covariates) if covariates: self.design = ( @@ -755,6 +979,43 @@ def __init__( (periods_positional[None, :] >= self.cohorts[:, None]) & (self.cohorts[:, None] != 0) ).astype(float) + # Two-way demeaning through the house helper (the same alternating + # projections fixest::demean runs), on the sorted long frame so the + # (unit, period) reshape afterwards is a plain view. The treatment + # indicator is DERIVED from cohorts x positional periods, not an input + # column, so it is synthesized here before the call. Both the RAW and + # the demeaned covariate blocks are kept: the annihilation filter in + # _fwl_residuals compares one against the other. + demean_frame = pd.DataFrame( + {"_unit": frame[unit].to_numpy(), "_time": frame[time].to_numpy()} + ) + demean_frame["_treated"] = self.treated.reshape(-1) + for j, name in enumerate(self.covariates): + demean_frame[f"_x{j}"] = self.design[:, :, j].reshape(-1) + row_weights = None if weights is None else self.weights.reshape(-1) + demeaned = within_transform( + demean_frame, + ["_treated", *(f"_x{j}" for j in range(len(self.covariates)))], + "_unit", + "_time", + weights=row_weights, + suffix="_dm", + tol=1e-12, + ) + self.treated_demeaned = ( + demeaned["_treated_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + ) + if self.covariates: + self.design_demeaned = np.stack( + [ + demeaned[f"_x{j}_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + for j in range(len(self.covariates)) + ], + axis=2, + ) + else: + self.design_demeaned = np.zeros((n_units, n_periods, 0)) + def covariate_block( self, names: Sequence[str], data: pd.DataFrame, unit: str, time: str ) -> np.ndarray: @@ -788,8 +1049,8 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: builds for ``xformula = ~1``. """ weights = panel.weights - d_dot = _demean_two_way(panel.treated[:, :, None], weights)[:, :, 0] - x_dot = _demean_two_way(panel.design, weights) + d_dot = panel.treated_demeaned + x_dot = panel.design_demeaned flat_d = d_dot.reshape(-1) flat_w = weights.reshape(-1) @@ -798,14 +1059,19 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: # on which fixest::demean segfaults; here it simply has to be spelled out. flat_x = x_dot.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]) - # Drop covariates that double-demeaning ANNIHILATED before anything is - # projected on them. A time-invariant regressor leaves a column of pure - # rounding noise (~1e-16 against a raw scale of ~1), and regressing on - # that amplifies the noise by ~1e16 - which silently corrupts the per-cell - # weights. The test is scale-relative: a column counts as having no - # within-variation when its demeaned norm is negligible NEXT TO ITS OWN - # raw norm, which a rank test on the demeaned matrix alone cannot see - # (there, 1e-16 is simply the largest pivot). + # Numerical hygiene: drop covariates that double-demeaning ANNIHILATED + # before anything is projected on them. A time-invariant regressor leaves a + # column of pure rounding noise (~1e-16 against a raw scale of ~1). Keeping + # it is not catastrophic - the column lies in the FE span and is orthogonal + # to the treatment residual, so on mpdta's `lpop` it moves the FWL residual + # by ~2e-18 - but regressing on an exactly-zero column is meaningless, and + # dropping it is what makes covariates=None and covariates=[] + # agree exactly. The test is scale-relative: a column counts as having no + # within-variation when its demeaned norm is negligible NEXT TO ITS OWN raw + # norm, which a rank test on the demeaned matrix alone cannot see (there, + # 1e-16 is simply the largest pivot). The 1e-10 relative threshold is a + # blunt instrument: a covariate with a large level and genuinely small + # within-variation can trip it, which is why the warning says so. raw_scale = np.linalg.norm( panel.design.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]), axis=0, @@ -816,26 +1082,44 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: names = [panel.covariates[j] for j in np.flatnonzero(annihilated)] warnings.warn( f"covariate(s) {names!r} have no within-unit-and-period variation " - "and were dropped: two-way demeaning annihilates them, so they " - "cannot affect a two-way fixed effects regression", + "(or within-variation below 1e-10 of their own level) and were " + "dropped: two-way demeaning annihilates them, so they cannot affect " + "a two-way fixed effects regression. If that is not intended, " + "centre or rescale the covariate so its within-variation is not " + "negligible next to its level", UserWarning, stacklevel=3, ) flat_x = flat_x[:, ~annihilated] surviving = [name for name, drop in zip(panel.covariates, annihilated) if not drop] - kept, dropped = _drop_collinear(flat_x) - if dropped: - names = [surviving[j] for j in dropped] - warnings.warn( - f"dropped collinear covariate column(s) {names!r} after " - "double-demeaning; they carry no within-variation independent of " - "the others", - UserWarning, - stacklevel=3, + if flat_x.shape[1]: + # House solver: WLS through the origin (R's lm(y ~ -1 + X, w)). On a + # rank-deficient design it fits the maximal independent set, sets the + # dropped coefficients to NaN (R-style) and computes the residual from + # the identified ones - so the residual is the FWL residual we need + # and the NaN positions name the collinear columns. + gamma, resid, _ = solve_ols( + flat_x, + flat_d, + weights=flat_w, + return_vcov=False, + rank_deficient_action="silent", + column_names=list(surviving), ) - gamma = _wls_coefficients(kept, flat_d, flat_w) - resid = flat_d - (kept @ gamma if kept.shape[1] else 0.0) + dropped = np.flatnonzero(np.isnan(gamma)) + if dropped.size: + names = [surviving[j] for j in dropped] + warnings.warn( + f"dropped collinear covariate column(s) {names!r} after " + "double-demeaning; they carry no within-variation independent of " + "the others", + UserWarning, + stacklevel=3, + ) + resid = np.asarray(resid, dtype=float) + else: + resid = flat_d alpha_den = _weighted_mean(resid * flat_d, flat_w) if not np.isfinite(alpha_den) or alpha_den == 0: raise ValueError( @@ -992,22 +1276,22 @@ def _decompose_fwl( ) frame = pd.DataFrame(cells) - weight_col = frame["weight"].to_numpy() + weight_vec = frame["weight"].to_numpy() att_col = frame["att"].to_numpy() post_col = frame["post"].to_numpy().astype(bool) - decomposition = float((weight_col * att_col).sum()) - remainder_total = float((frame["remainder"].to_numpy() * weight_col).sum()) + decomposition = float((weight_vec * att_col).sum()) + remainder_total = float((frame["remainder"].to_numpy() * weight_vec).sum()) ess_col = frame["ess"].to_numpy() return { "cells": frame, "estimate": decomposition + remainder_total, "decomposition": decomposition, "remainder": remainder_total, - "pretrend_bias": float((weight_col[~post_col] * att_col[~post_col]).sum()), - "post_only": float((weight_col[post_col] * att_col[post_col]).sum()), + "pretrend_bias": float((weight_vec[~post_col] * att_col[~post_col]).sum()), + "post_only": float((weight_vec[post_col] * att_col[post_col]).sum()), # summary.decomposed_twfe: post cells only, on both factors "effective_sample_size": float( - post_col.sum() * (weight_col[post_col] * ess_col[post_col]).sum() + post_col.sum() * (weight_vec[post_col] * ess_col[post_col]).sum() ), "balance": pd.DataFrame(balance_rows) if balance_block is not None else None, } @@ -1025,8 +1309,16 @@ def _weighted_ecdf(values: np.ndarray, weights: np.ndarray) -> Tuple[np.ndarray, values, and ``F(knot_j) = mean(w * (y <= knot_j))``. """ w = weights / weights.mean() + # Sort once and read cumulative mass at each unique-value boundary, rather + # than rescanning the full vector per knot (the naive form is O(n * k), and + # this runs per covariate x cohort x period). `np.unique` returns the + # sorted knots, so a single searchsorted locates each boundary. + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(w[order]) knots = np.unique(values) - heights = np.array([float((w * (values <= knot)).mean()) for knot in knots]) + last = np.searchsorted(sorted_values, knots, side="right") - 1 + heights = cumulative[last] / len(values) return knots, heights diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py index bdf4831b3..3f7299873 100644 --- a/diff_diff/twfe_weights_results.py +++ b/diff_diff/twfe_weights_results.py @@ -76,12 +76,23 @@ class ATTGTWeightsResult(Diagnostic): ``sum(weight * att)`` - what the estimand delivers given these ATT(g, t). For ``aggregation="twfe"`` this is the TWFE coefficient. n_negative : int - Number of cells receiving a negative weight. Non-zero is the - classic staggered-adoption pathology: the regression is subtracting - treatment effects it should be adding. + Number of cells - PRE and post - receiving a negative weight. Under + ``aggregation="twfe"`` the weights over the full ``g != 0`` grid sum + to zero (post to +1, pre to -1), so this is non-zero in every + staggered design; read ``n_negative_post`` for the pathology. negative_weight_share : float - ``sum(|w| : w < 0) / sum(|w|)`` - how much of the total weight mass - points the wrong way. ``0.0`` when no weight is negative. + ``sum(|w| : w < 0) / sum(|w|)`` over ALL cells - how much of the total + weight mass points the wrong way. Near 0.5 is normal under + ``"twfe"`` for the same reason. ``0.0`` when no weight is negative. + n_negative_post : int + Number of POST-treatment cells receiving a negative weight. This is + the classic staggered-adoption pathology: the regression subtracts + treatment effects it should be adding. Zero for ``"overall"`` and + ``"simple"`` by construction. + negative_post_weight_share : float + ``sum(|w| : w < 0, post) / sum(|w| : post)`` - the share of + post-treatment weight mass that is negative. No R counterpart; see + the methodology registry. n_cells : int Number of ``(g, t)`` cells contributing. source : str or None @@ -98,6 +109,8 @@ class ATTGTWeightsResult(Diagnostic): implied_att: float n_negative: int negative_weight_share: float + n_negative_post: int + negative_post_weight_share: float n_cells: int source: Optional[str] = None control_group: Optional[str] = None @@ -146,15 +159,24 @@ def summary(self) -> str: "-" * width, "", f"{'Implied estimate:':<28} {_fmt(self.implied_att)}", - f"{'Negative-weight cells:':<28} {self.n_negative:>12}", - f"{'Negative-weight share:':<28} {_fmt(self.negative_weight_share)}", + f"{'Negative POST-period cells:':<28} {self.n_negative_post:>12}", + f"{'Negative POST-weight share:':<28} {_fmt(self.negative_post_weight_share)}", + f"{'Negative cells (all):':<28} {self.n_negative:>12}", + f"{'Negative share (all):':<28} {_fmt(self.negative_weight_share)}", "", ] - if self.n_negative: + if self.n_negative_post: lines += [ - "Note: negative weights mean this estimand subtracts some ATT(g, t).", - " Under heterogeneous effects the estimate need not lie in the", - " convex hull of the underlying group-time effects.", + "Note: negative POST-period weights mean this estimand subtracts some", + " treatment-period ATT(g, t). Under heterogeneous effects the", + " estimate need not lie in the convex hull of those effects.", + "", + ] + elif self.n_negative: + lines += [ + "Note: the negative weights fall on PRE-treatment cells only, which is", + " how the TWFE weights sum to zero over the full grid; no", + " treatment-period effect is being subtracted.", "", ] lines.append("=" * width) @@ -176,6 +198,8 @@ def to_dict(self) -> Dict[str, Any]: "n_cells": self.n_cells, "n_negative": self.n_negative, "negative_weight_share": self.negative_weight_share, + "n_negative_post": self.n_negative_post, + "negative_post_weight_share": self.negative_post_weight_share, "n_dropped_cells": self.n_dropped_cells, "source": self.source, "control_group": self.control_group, @@ -186,29 +210,27 @@ def to_dict(self) -> Dict[str, Any]: @dataclass class TWFEDecompositionResult(Diagnostic): - """Decomposition of a TWFE (or AIPW) estimate into weighted ATT(g, t). + """Decomposition of a TWFE estimate into weighted ATT(g, t). Returned by :func:`diff_diff.decompose_twfe_weights`. Attributes ---------- cells : pd.DataFrame - Columns ``group``, ``time``, ``post``, ``att``, ``weight``, ``ess``, - and (``method="fwl"`` only) ``remainder``. ``weight`` is the implicit - weight the regression places on that cell's ATT(g, t) - R's - ``alpha_weight`` under ``method="fwl"`` and ``att_weight`` under - ``method="aipw"``. + Columns ``group``, ``time``, ``post``, ``att``, ``weight``, ``ess`` + and ``remainder``. ``weight`` is the implicit weight the regression + places on that cell's ATT(g, t) - R's ``alpha_weight``. method : str - ``"fwl"`` (Frisch-Waugh-Lovell residual weights from the TWFE - regression) or ``"aipw"`` (per-cell doubly-robust weights). + ``"fwl"`` - Frisch-Waugh-Lovell residual weights from the TWFE + regression. (The only method currently implemented; upstream's AIPW + decomposition is a documented follow-up.) estimate : float The estimate being decomposed - ``decomposition + remainder``. decomposition : float ``sum(weight * att)`` over all cells, pre and post. remainder : float Part of ``estimate`` not attributable to any ATT(g, t) cell. - Identically ``0.0`` except under ``method="fwl"`` with - ``base_period="gmin1"``. + Identically ``0.0`` except under ``base_period="gmin1"``. pretrend_bias : float ``sum(weight * att)`` over PRE-treatment cells only. Under parallel trends every pre-treatment ATT(g, t) is zero and this vanishes; a @@ -217,7 +239,7 @@ class TWFEDecompositionResult(Diagnostic): post_only : float ``sum(weight * att)`` over post-treatment cells only. base_period : str or None - ``"first_period"`` or ``"gmin1"`` (``method="fwl"`` only). + ``"first_period"`` or ``"gmin1"``. covariates : tuple of str Covariates the regression adjusted for. Empty tuple when none. effective_sample_size : float @@ -258,7 +280,6 @@ def summary(self) -> str: width = 78 method_label = { "fwl": "TWFE regression (Frisch-Waugh-Lovell implicit weights)", - "aipw": "AIPW (doubly-robust per-cell weights)", }.get(self.method, self.method) covs = ", ".join(self.covariates) if self.covariates else "(none)" lines = [ @@ -399,17 +420,23 @@ def covariate_balance( weights = self.cells.set_index(["group", "time"])["weight"] keys = pd.MultiIndex.from_arrays([table["group"], table["time"]]) - cell_weight = weights.reindex(keys).to_numpy() + table["_w"] = weights.reindex(keys).to_numpy() if post_only: - cell_weight = cell_weight * table["post"].to_numpy() - - table["_w"] = cell_weight - rolled = ( - table[list(_BALANCE_STATS)] - .mul(table["_w"], axis=0) - .groupby(table["covariate"].to_numpy(), sort=False) - .sum() - ) + # Mask on the `post` COLUMN, not on a zero roll-up weight: R's + # post-only helper never touches the pre cells, but a post cell + # whose implicit weight happens to be exactly zero still + # contributes - and still propagates its NA. + table = table[table["post"].to_numpy().astype(bool)] + + # R propagates NA: if ANY contributing cell of a (covariate, statistic) + # is NA, the summary is NA. pandas' sum() skips NaN, which would turn + # the documented NA return of frac_treated_extreme (fewer than three + # distinct values) into a spurious 0.0. + stats = table[list(_BALANCE_STATS)].mul(table["_w"], axis=0) + groups = table["covariate"].to_numpy() + rolled = stats.groupby(groups, sort=False).sum(min_count=1) + any_nan = table[list(_BALANCE_STATS)].isna().groupby(groups, sort=False).any() + rolled = rolled.mask(any_nan) rolled.index.name = "covariate" out = rolled.reset_index() if standardize: diff --git a/diff_diff/visualization/_diagnostic.py b/diff_diff/visualization/_diagnostic.py index 599a6216a..ddb99e224 100644 --- a/diff_diff/visualization/_diagnostic.py +++ b/diff_diff/visualization/_diagnostic.py @@ -1,12 +1,20 @@ -"""Diagnostic visualization functions (sensitivity, Bacon decomposition).""" +"""Diagnostic visualization functions (sensitivity, Bacon decomposition). -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +``plot_twfe_weights`` renders the implicit ATT(g, t) weights (or their covariate +balance) from :func:`diff_diff.attgt_weights` / :func:`diff_diff.decompose_twfe_weights`. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np if TYPE_CHECKING: from diff_diff.bacon import BaconDecompositionResults from diff_diff.honest_did import SensitivityResults + from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, + ) def plot_sensitivity( @@ -820,7 +828,7 @@ def _render_bacon_plotly( def plot_twfe_weights( - results: Any, + results: "Union[ATTGTWeightsResult, TWFEDecompositionResult]", *, kind: str = "auto", standardize: bool = True, @@ -836,6 +844,7 @@ def plot_twfe_weights( annotate: bool = False, ax: Optional[Any] = None, show: bool = True, + backend: str = "matplotlib", ) -> Any: """Visualize implicit TWFE weights on ATT(g, t), or their covariate balance. @@ -875,28 +884,34 @@ def plot_twfe_weights( annotate : bool, default False Label each point with its ``(group, time)`` or covariate name. ax : matplotlib Axes, optional - Axes to draw on. A new figure is created when omitted. + Axes to draw on. A new figure is created when omitted. Matplotlib only. show : bool, default True - Call ``plt.show()`` before returning. + Call ``plt.show()`` / ``fig.show()`` before returning. + backend : str, default "matplotlib" + Plotting backend: ``"matplotlib"`` or ``"plotly"``. Returns ------- - matplotlib.axes.Axes + matplotlib.axes.Axes or plotly.graph_objects.Figure Raises ------ ValueError - On an unknown ``kind``, or when ``kind="balance"`` is requested for a - result that carries no balance table. + On an unknown ``kind`` or ``backend``; when ``kind="balance"`` is + requested for a result that carries no balance table; or when the + balance table has no finite differences to plot. Examples -------- >>> import diff_diff # doctest: +SKIP >>> w = diff_diff.attgt_weights(cs_result) # doctest: +SKIP >>> diff_diff.plot_twfe_weights(w) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w, backend="plotly") # doctest: +SKIP """ if kind not in ("auto", "weights", "balance"): raise ValueError(f"kind must be one of ['auto', 'weights', 'balance'], got {kind!r}") + if backend not in ("matplotlib", "plotly"): + raise ValueError(f"backend must be 'matplotlib' or 'plotly', got {backend!r}") has_balance = getattr(results, "balance", None) is not None if kind == "auto": kind = "balance" if has_balance else "weights" @@ -907,6 +922,82 @@ def plot_twfe_weights( "decompose_twfe_weights(..., balance_covariates=[...])." ) + payload = _twfe_weights_payload(results, kind, standardize, absolute_value) + render = _render_twfe_weights_plotly if backend == "plotly" else _render_twfe_weights_mpl + return render( + payload, + kind=kind, + standardize=standardize, + figsize=figsize, + title=title, + xlabel=xlabel, + ylabel=ylabel, + post_color=post_color, + pre_color=pre_color, + markersize=markersize, + alpha=alpha, + annotate=annotate, + ax=ax, + show=show, + ) + + +def _twfe_weights_payload( + results: Any, kind: str, standardize: bool, absolute_value: bool +) -> Dict[str, Any]: + """Backend-agnostic data for either view (shared by the two renderers).""" + if kind == "weights": + table = getattr(results, "weights", None) + if table is None: + table = results.cells + weight = table["weight"].to_numpy(dtype=float) + return { + "post": table["post"].to_numpy().astype(bool), + "weight": weight, + "att": table["att"].to_numpy(dtype=float), + "labels": [f"({g}, {t})" for g, t in zip(table["group"], table["time"])], + "n_negative": int((weight < 0).sum()), + } + balance = results.covariate_balance(level="summary", standardize=standardize) + suffix = "_std_diff" if standardize else "_diff" + unweighted = balance["unweighted" + suffix].to_numpy(dtype=float) + weighted = balance["weighted" + suffix].to_numpy(dtype=float) + if absolute_value: + unweighted, weighted = np.abs(unweighted), np.abs(weighted) + finite = np.isfinite(unweighted) & np.isfinite(weighted) + if not finite.any(): + raise ValueError( + "the covariate balance table has no finite differences to plot " + "(every covariate is degenerate - e.g. constant or binary with a zero " + "pooled SD under standardize=True); try standardize=False or check the " + "balance_covariates" + ) + limit = float(np.max(np.abs(np.concatenate([unweighted[finite], weighted[finite]]))) or 1.0) + return { + "unweighted": unweighted, + "weighted": weighted, + "labels": [str(c) for c in balance["covariate"]], + "limit": limit, + } + + +def _render_twfe_weights_mpl( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: from diff_diff.visualization._common import _require_matplotlib plt = _require_matplotlib() @@ -914,12 +1005,7 @@ def plot_twfe_weights( _, ax = plt.subplots(figsize=figsize) if kind == "weights": - table = getattr(results, "weights", None) - if table is None: - table = results.cells - post = table["post"].to_numpy().astype(bool) - weight = table["weight"].to_numpy() - att = table["att"].to_numpy() + post, weight, att = payload["post"], payload["weight"], payload["att"] ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) ax.axvline(0, color="0.4", linewidth=1.2, zorder=1) for mask, color, label in ( @@ -937,36 +1023,19 @@ def plot_twfe_weights( zorder=3, ) if annotate: - for w, a, g, t in zip(weight, att, table["group"], table["time"]): - ax.annotate( - f"({g}, {t})", (w, a), fontsize=8, xytext=(4, 4), textcoords="offset points" - ) + for w, a, lab in zip(weight, att, payload["labels"]): + ax.annotate(lab, (w, a), fontsize=8, xytext=(4, 4), textcoords="offset points") ax.set_xlabel(xlabel or "Implicit weight") ax.set_ylabel(ylabel or "ATT(g, t)") default_title = "Implicit weights on group-time effects" - n_negative = int((weight < 0).sum()) - if n_negative: - default_title += f" ({n_negative} negative)" + if payload["n_negative"]: + default_title += f" ({payload['n_negative']} negative)" ax.set_title(title or default_title) ax.legend(frameon=False) else: - balance = results.covariate_balance(level="summary", standardize=standardize) - suffix = "_std_diff" if standardize else "_diff" - unweighted = balance["unweighted" + suffix].to_numpy(dtype=float) - weighted = balance["weighted" + suffix].to_numpy(dtype=float) - if absolute_value: - unweighted = np.abs(unweighted) - weighted = np.abs(weighted) + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) - ax.scatter( - unweighted, - weighted, - s=markersize, - alpha=alpha, - color=post_color, - zorder=3, - ) - limit = float(np.nanmax(np.abs(np.concatenate([unweighted, weighted]))) or 1.0) + ax.scatter(unweighted, weighted, s=markersize, alpha=alpha, color=post_color, zorder=3) ax.plot( [0, limit], [0, limit], @@ -977,10 +1046,8 @@ def plot_twfe_weights( label="no improvement", ) if annotate: - for x, y, name in zip(unweighted, weighted, balance["covariate"]): - ax.annotate( - str(name), (x, y), fontsize=8, xytext=(4, 4), textcoords="offset points" - ) + for x, y, name in zip(unweighted, weighted, payload["labels"]): + ax.annotate(name, (x, y), fontsize=8, xytext=(4, 4), textcoords="offset points") kindword = "standardized " if standardize else "" ax.set_xlabel(xlabel or f"Unweighted {kindword}difference") ax.set_ylabel(ylabel or f"Implicitly-weighted {kindword}difference") @@ -990,3 +1057,101 @@ def plot_twfe_weights( if show: plt.show() return ax + + +def _render_twfe_weights_plotly( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: + from diff_diff.visualization._common import _require_plotly + + go = _require_plotly() + fig = go.Figure() + marker_px = max(4.0, float(np.sqrt(markersize))) # matplotlib area -> plotly diameter + mode = "markers+text" if annotate else "markers" + + if kind == "weights": + post, weight, att = payload["post"], payload["weight"], payload["att"] + labels = np.asarray(payload["labels"], dtype=object) + for mask, color, name in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + fig.add_trace( + go.Scatter( + x=weight[mask], + y=att[mask], + mode=mode, + name=name, + text=labels[mask] if annotate else None, + textposition="top right", + marker={"size": marker_px, "color": color, "opacity": alpha}, + hovertemplate="%{text}
weight=%{x:.4f}
ATT(g,t)=%{y:.4f}", + customdata=None, + ) + ) + fig.data[-1].text = labels[mask] + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + fig.add_vline(x=0, line={"color": "gray", "width": 1.2}) + default_title = "Implicit weights on group-time effects" + if payload["n_negative"]: + default_title += f" ({payload['n_negative']} negative)" + fig.update_layout( + title=title or default_title, + xaxis_title=xlabel or "Implicit weight", + yaxis_title=ylabel or "ATT(g, t)", + ) + else: + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] + fig.add_trace( + go.Scatter( + x=unweighted, + y=weighted, + mode=mode, + name="covariates", + text=payload["labels"], + textposition="top right", + marker={"size": marker_px, "color": post_color, "opacity": alpha}, + hovertemplate="%{text}
unweighted=%{x:.4f}
weighted=%{y:.4f}", + ) + ) + fig.add_trace( + go.Scatter( + x=[0, limit], + y=[0, limit], + mode="lines", + name="no improvement", + line={"color": "gray", "dash": "dash", "width": 1.0}, + ) + ) + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + kindword = "standardized " if standardize else "" + fig.update_layout( + title=title or "Covariate balance under the implicit weights", + xaxis_title=xlabel or f"Unweighted {kindword}difference", + yaxis_title=ylabel or f"Implicitly-weighted {kindword}difference", + ) + + fig.update_layout( + width=int(figsize[0] * 100), + height=int(figsize[1] * 100), + template="plotly_white", + legend={"orientation": "h", "y": -0.15}, + ) + if show: + fig.show() + return fig diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst index cfd3cae6e..ebf809ef2 100644 --- a/docs/api/twfe_weights.rst +++ b/docs/api/twfe_weights.rst @@ -92,7 +92,7 @@ Inspecting what a TWFE regression weights base_period="universal", # required for aggregation="twfe" ).fit( panel, outcome="lemp", unit="countyreal", time="year", - first_treat="first.treat", + first_treat="first_treat", ) weights = diff_diff.attgt_weights(cs, aggregation="twfe") @@ -108,6 +108,16 @@ between ``implied_att`` values is the cost of the TWFE specification: .. code-block:: python + import diff_diff + + panel = diff_diff.load_mpdta() + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal", + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + ) + for aggregation in ("twfe", "overall", "simple"): w = diff_diff.attgt_weights(cs, aggregation=aggregation) print(f"{aggregation:8s} {w.implied_att: .4f} " @@ -118,10 +128,14 @@ Separating treatment effects from pre-trend violations .. code-block:: python + import diff_diff + + panel = diff_diff.load_mpdta() + decomposition = diff_diff.decompose_twfe_weights( panel, outcome="lemp", unit="countyreal", time="year", - first_treat="first.treat", + first_treat="first_treat", covariates=["lpop"], balance_covariates=["lpop"], ) diff --git a/docs/api/visualization.rst b/docs/api/visualization.rst index b143d2d20..2b03605a0 100644 --- a/docs/api/visualization.rst +++ b/docs/api/visualization.rst @@ -194,6 +194,12 @@ plot_bacon Visualize Goodman-Bacon decomposition results. +.. seealso:: + + :func:`diff_diff.plot_twfe_weights` renders the implicit ATT(g, t) weights + and their covariate balance. It is documented on its own page, + :doc:`twfe_weights`, and supports the same ``backend=`` options. + .. autofunction:: diff_diff.plot_bacon Example diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 61120b816..a5046c8f3 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -1204,7 +1204,7 @@ sources: - path: docs/r_comparison.rst type: user_guide - # ── BaconDecomposition ───���───────────────────────────────────────── + # ── TWFE Weight Diagnostics ───���───────────────────────────────────────── diff_diff/twfe_weights.py: drift_risk: low @@ -1233,6 +1233,9 @@ sources: - path: docs/methodology/REGISTRY.md section: "TWFE Weight Diagnostics" type: methodology + + # ── BaconDecomposition ───���───────────────────────────────────────── + diff_diff/bacon.py: drift_risk: low docs: diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 495888e74..18109b6b8 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6291,7 +6291,12 @@ Verified on `mpdta` at `-0.03654894` from both directions. ### Edge cases -- Non-estimable `(g,t)` cells (NaN ATT) are dropped from `attgt_weights` with a `UserWarning` and counted in `n_dropped_cells`; the remaining weights renormalize. +- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `aggregation="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. +- **Note (matches R `did`'s first-period drop):** a cohort with NO estimable post-treatment cell, canonically one treated in the first observed period (no base period), is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. +- **Note (matches R `aggte`'s available-period averaging under not-yet-treated controls):** on a `control_group="not_yet_treated"` fit the last cohorts run out of comparison units and CS marks those post cells `skip_reason="zero_treated_control"`. For `"overall"` / `"simple"` they are treated as structurally absent: `"overall"` divides each cohort by its number of AVAILABLE post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells, which is what R `aggte(type="group")` / `aggte(type="simple")` compute on such a fit. A `UserWarning` names the cells. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. +- **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. +- **Note (`aggregation="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. +- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `aggregation="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. - `attgt_weights` rejects repeated-cross-section fits and unbalanced-panel fallbacks: `E_t[D]` and the cohort shares average over a fixed unit set. @@ -6299,23 +6304,26 @@ Verified on `mpdta` at `-0.03654894` from both directions. ### Notes and deviations - **Note (upstream `fixest::demean` segfault on the no-covariate branch):** `twfeweights::implicit_twfe_weights(xformula = ~1)` builds `model.matrix(~-1, data)`, an `nT x 0` matrix, and `fixest::demean()` SEGFAULTS on a zero-column matrix (reproduced in isolation on R 4.6.1 / fixest 0.14.2: `fixest::demean(matrix(numeric(0), 10, 0), ids)` → `*** caught segfault *** memory not mapped`). This is a zero-column bug, not a property of any fixture. The no-covariate golden is therefore generated with a TIME-INVARIANT covariate, which double-demeaning annihilates exactly, making the call numerically the `~1` branch; the parity test asserts BOTH `covariates=None` and `covariates=[]` against that single golden, so the equivalence is proven rather than assumed. Verified on `mpdta`: `twfe_weights(att_gt(...))` aggregate and `implicit_twfe_weights(xformula = ~lpop)$est` both equal `-0.03654894`. -- **Note (annihilated covariates are dropped before the projection):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Regressing on it amplifies that noise by ~1e16 and corrupts the per-cell weights. diff-diff drops such columns, judged against each column's own PRE-demeaning norm — a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. A `UserWarning` names the dropped covariates. This is what makes `covariates=None` and `covariates=[]` agree to 1e-15. -- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **The aggregate is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), which is why `estimate` matches R to 1e-15 while the individual `ATT(g,t)` do not. +- **Note (annihilated covariates are dropped before the projection, as numerical hygiene):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Keeping it is not catastrophic: the column lies in the fixed-effect span and is orthogonal to the treatment residual, so on mpdta's `lpop` it moves the FWL residual by ~2e-18. But regressing on an exactly-zero column is meaningless, and dropping it is what makes `covariates=None` and `covariates=[]` agree exactly rather than approximately. diff-diff judges each column against its own PRE-demeaning norm; a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. **Limitation:** the 1e-10 relative threshold is blunt, so a covariate with a large level and genuinely small within-variation can trip it. The `UserWarning` says so and suggests centring or rescaling. +- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pretrend_bias` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. - **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. - **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. -- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The two-period kernels, per-cell helpers and balance statistics are private; they are pinned directly by the parity suite since they have no public surface. +- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. - **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. - **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. - **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. -- **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. +- **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values, and that `NA` survives the summary roll-up. R propagates NA if any contributing cell is NA, whereas a plain pandas `.sum()` skips it and would report a spurious `0.0` for a binary or constant covariate; the roll-up therefore masks on the `post` column (not on a zero roll-up weight, since a post cell whose implicit weight is exactly zero still contributes) and returns NaN whenever any contributing post cell is NaN. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. +- **Note (negative-weight statistics; no R counterpart):** over the `g != 0` grid the TWFE weights sum to zero, post cells to +1 and pre cells to -1, so `n_negative` is non-zero and `negative_weight_share` sits near 0.5 in EVERY staggered design, including one with no negative post-period weight. The pathology the literature describes is negative weight on POST cells, so `ATTGTWeightsResult` reports both, labelled: `n_negative` / `negative_weight_share` over all cells, and `n_negative_post` / `negative_post_weight_share` restricted to post cells (the share of post-period weight MASS that is negative). `summary()` leads with the post-only figures. R reports neither statistic. +- **Note (weighted `aggregation="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, aggregation="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). +- **Note (linear algebra runs through the house helpers):** the two-way demeaning is `diff_diff.utils.within_transform` (the same alternating projections `fixest::demean` runs) applied to the sorted long frame before the `(unit, period)` reshape, with the treatment indicator synthesized as a column since it is derived from cohorts x positional periods rather than supplied. The Frisch-Waugh-Lovell solve is `diff_diff.linalg.solve_ols(..., weights=, rank_deficient_action="silent")`: on a rank-deficient design it fits the maximal independent set, sets the aliased coefficients to `NaN` (R-style) and returns the residual computed from the identified ones, so the module reads the dropped column names off the `NaN` positions and uses the returned residual directly. This replaced a bespoke pivoted QR whose docstring claimed to drop "later columns first"; it did not, being the same norm-pivoted QR `solve_ols` uses, and dropping the same column. - **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. - **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. ### R output parity -Goldens: `benchmarks/data/twfeweights_golden.json` (+ three sibling panel CSVs), regenerated by `benchmarks/R/generate_twfeweights_golden.R`. R is needed only to regenerate them, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. +Goldens: `benchmarks/data/twfeweights_golden.json`, plus two simulated sibling panel CSVs. The `mpdta` fixture reads the shared `benchmarks/data/mpdta_stata_panel.csv` and derives `lpop_t` from a `derived_columns` expression in the golden, rather than committing a renamed copy; the generator asserts the two sources agree to CSV round-trip precision. Every cells block carries ORIGINAL period labels: `implicit_*` run in positional time internally, and the generator maps them back so one convention holds throughout and the tests assert labels rather than array position. Regenerated by `benchmarks/R/generate_twfeweights_golden.R`; R is needed only to regenerate, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. -Three fixtures: `mpdta` (real; non-`1..T` period labels), `sim_staggered` (equal cohorts, a real pre-trend so `pretrend_bias != 0`, and the degenerate `t=3` cells above), and `unbalanced_cohorts` (120/70/60 — breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). +Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, package="did")`), `sim_staggered` (three equal cohorts of 100, which is exactly what makes the comparison-group normalizer vanish at `t=3`, so this fixture deliberately exercises the degenerate cells above; its `pretrend_bias` is non-zero but the ~0.093 is sampling noise, since `x1` is iid and cohorts are assigned by unit index, so no differential pre-trend is designed in), and `unbalanced_cohorts` (120/70/60, which breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). | Surface | Gate | Rationale | |---------|------|-----------| @@ -6325,7 +6333,8 @@ Three fixtures: `mpdta` (real; non-`1..T` period labels), `sim_staggered` (equal | FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | | FWL with covariates | `atol=1e-8` | The demeaning slack propagates through the OLS projection of `Ddot` on `Xdot`. | | Covariate balance (11 statistics) | `atol=1e-9` | Smooth functions of the weights above. Observed max deviation 7.3e-11. | -| Per-cell ATT and the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. | +| Per-cell ATT, `pretrend_bias` / `post_only`, and (under `gmin1`) the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. Under `base_period="first_period"` the remainder is identically zero, so the split is gated tight even where the mask fires. | +| `effective_sample_size` and per-cell `ess` at DEGENERATE cells | expected value rebuilt from R's own cells | R's `ess` at a 0/0 cell is a ratio of rounding errors (scalar gap ~0.99, per-cell up to ~0.53). The expectation uses R's weights and R's `ess` wherever the degeneracy is not detected, substituting our limit value only at the detected cells, so the assertion is anchored to R rather than to our own implementation. | --- diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index ce81e9e78..326df5098 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -332,12 +332,27 @@ def make_constructed_diagnostics() -> Dict[str, Any]: interpretation="All applicable checks passed.", applicable_checks=("parallel_trends",), ), + # Every count / share is DERIVED from the cells so the fixture cannot + # drift from the object it imitates. "ATTGTWeightsResult": diff_diff.ATTGTWeightsResult( weights=attgt_weight_cells, aggregation="twfe", implied_att=float((attgt_weight_cells["weight"] * attgt_weight_cells["att"]).sum()), - n_negative=1, - negative_weight_share=0.25, + n_negative=int((attgt_weight_cells["weight"] < 0).sum()), + negative_weight_share=float( + attgt_weight_cells["weight"].clip(upper=0).abs().sum() + / attgt_weight_cells["weight"].abs().sum() + ), + n_negative_post=int( + ((attgt_weight_cells["weight"] < 0) & (attgt_weight_cells["post"] == 1)).sum() + ), + negative_post_weight_share=float( + attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"] + .clip(upper=0) + .abs() + .sum() + / attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"].abs().sum() + ), n_cells=len(attgt_weight_cells), source="CallawaySantAnnaResults", control_group="never_treated", @@ -349,11 +364,25 @@ def make_constructed_diagnostics() -> Dict[str, Any]: estimate=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), decomposition=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), remainder=0.0, - pretrend_bias=0.0, - post_only=0.5, + pretrend_bias=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 0 + ].sum() + ), + post_only=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 1 + ].sum() + ), base_period="first_period", covariates=("x1",), - effective_sample_size=42.0, + # Module identity: post_count * sum_post(weight * ess). + effective_sample_size=float( + (decomposition_cells["post"] == 1).sum() + * (decomposition_cells["weight"] * decomposition_cells["ess"])[ + decomposition_cells["post"] == 1 + ].sum() + ), n_units=12, n_periods=4, balance=decomposition_balance, diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index 79586aff3..45a32cf4e 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -47,6 +47,7 @@ "api/dml_did.rst", "api/mmm.rst", "api/triple_diff.rst", + "api/twfe_weights.rst", "practitioner_decision_tree.rst", "practitioner_getting_started.rst", "python_comparison.rst", diff --git a/tests/test_naming_guard.py b/tests/test_naming_guard.py index c018814bf..953c371c6 100644 --- a/tests/test_naming_guard.py +++ b/tests/test_naming_guard.py @@ -565,6 +565,31 @@ def _build_rowed_index(): "TripleDifference.fit[group]": ( "rule-3 reserved treated-group 0/1 indicator (v4-design section 8 rule 3)" ), + # The TWFE weight diagnostics share vocabulary with three rename families + # without reading any of them. `time=` here is the panel PERIOD COLUMN + # NAME (the same role as CallawaySantAnna.fit[time], which no row touches), + # not the two-period 0/1 post dummy M-030/M-031/M-082/M-137/M-138 rename to + # `post`: both functions are staggered-only. `aggregation=` selects an + # ESTIMAND ("twfe" / "overall" / "simple"), not the Wooldridge output + # granularity M-044 renames to `level` and M-087 removes. + **{ + f"{fn}[time]": ( + "panel PERIOD COLUMN NAME (as in CallawaySantAnna.fit[time]), not " + "the two-period 0/1 post dummy renamed to `post` by " + "M-030/M-031/M-082/M-137/M-138; survives 4.0" + ) + for fn in ("attgt_weights", "decompose_twfe_weights") + }, + "attgt_weights[aggregation]": ( + "ESTIMAND selector ('twfe' / 'overall' / 'simple'), not the " + "WooldridgeDiDResults output granularity M-044 renames to `level` and " + "M-087 removes; survives 4.0" + ), + "ATTGTWeightsResult.aggregation": ( + "records which ESTIMAND's weights the result holds - the " + "attgt_weights[aggregation] value, not a Wooldridge output granularity " + "(M-044 / M-087); survives 4.0" + ), "run_placebo_test[time]": ( "OVERLOADED pass-through, redesign pending (TODO.md): forwarded as " "the calendar column to placebo_timing_test/placebo_group_test AND " @@ -989,6 +1014,23 @@ def _token_family_code_refs(tok): # field. The one file that DID name it - diff_diff/guides/llms-full.txt - # was migrated in this same diff (migrate-first rule) and remains a lane # hit only through its unrelated backticked schema key. + # The TWFE weight diagnostics document their own `time=` (panel period + # COLUMN) and `aggregation=` (estimand selector) on these two surfaces; + # neither reads a renamed name. See the SURFACE_ALLOWLIST entries for + # attgt_weights / decompose_twfe_weights. + ("time", "diff_diff/guides/llms.txt"): ( + "attgt_weights / decompose_twfe_weights document a panel PERIOD COLUMN " + "named `time`, not the two-period 0/1 post dummy renamed by " + "M-030/M-031/M-082/M-137/M-138" + ), + ("aggregation", "diff_diff/guides/llms.txt"): ( + "attgt_weights' ESTIMAND selector, not WooldridgeDiDResults' output " + "granularity (M-044 / M-087)" + ), + ("aggregation", "docs/methodology/REGISTRY.md"): ( + "the TWFE Weight Diagnostics section documents attgt_weights' ESTIMAND " + "selector, not WooldridgeDiDResults' output granularity (M-044 / M-087)" + ), ("estimator", "diff_diff/aggregation.py"): ( "AggregationResult.estimator - independent field holding a CLASS NAME" ), diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index d55d567f3..3c93df678 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -108,7 +108,8 @@ def test_twfe_weights_can_be_negative(self, fitted): result = attgt_weights(fitted, aggregation="twfe") assert result.n_negative > 0 assert 0.0 < result.negative_weight_share < 1.0 - assert "Negative-weight cells:" in result.summary() + assert "Negative POST-period cells:" in result.summary() + assert result.n_negative_post <= result.n_negative def test_pre_treatment_cells_carry_weight_under_twfe(self, fitted): """TWFE loads on pre-treatment cells; the CS estimands do not.""" @@ -357,3 +358,727 @@ def test_rejects_a_cohort_label_off_the_period_grid(self, panel): time="period", first_treat="first_treat", ) + + +class TestWeightedRegressionPin: + """Frozen-numbers pin for the WEIGHTED branches. + + No parity fixture passes ``weights=`` and R ``twfe_weights`` has no ``w=``, + so the weighted code paths (weighted two-way demeaning, weighted FWL solve, + weighted cohort masses) have no external oracle. These literals were + captured from the implementation BEFORE the linear algebra was routed + through ``diff_diff.linalg.solve_ols`` / ``diff_diff.utils.within_transform`` + and pin that behaviour: any refactor must leave them green at 1e-12. + """ + + @staticmethod + def _weighted_panel(): + rng = np.random.default_rng(20260907) + n_per, n_periods = 12, 5 + cohorts = [0] * n_per + [3] * n_per + [4] * n_per + rows = [] + for i, g in enumerate(cohorts): + w = float(rng.choice([0.5, 1.0, 1.5, 2.5])) + alpha = rng.normal() + for t in range(1, n_periods + 1): + x = rng.normal() + 0.3 * t + effect = 1.0 * (t - g + 1) if (g and t >= g) else 0.0 + y = alpha + 0.2 * t + 0.5 * x + effect + rng.normal(scale=0.3) + rows.append({"id": i, "t": t, "g": g, "y": y, "x": x, "w": w}) + return pd.DataFrame(rows) + + _DEC = { + "nocov": dict( + kwargs={}, + estimate=1.4221735897240102, + pretrend_bias=0.5127724996498023, + post_only=0.9094010900742079, + ess=59.999999999999986, + weight=[ + -0.26383763837638374, + -0.26383763837638374, + 0.3726937269372693, + 0.07749077490774903, + 0.07749077490774903, + -0.059040590405904064, + -0.059040590405904064, + -0.3542435424354244, + 0.23616236162361626, + 0.23616236162361626, + ], + att=[ + 0.0, + -1.4436560985156102, + 0.2605332041282682, + 1.5423748763069138, + 2.020457826078349, + 0.0, + -0.8817595960499504, + -0.22533107108782402, + 1.0544057969164409, + 1.2161310006314463, + ], + ), + "cov": dict( + kwargs={"covariates": ["x"]}, + estimate=1.3930847792561663, + pretrend_bias=0.5397671816632773, + post_only=0.8533175975928889, + ess=58.816020983744, + weight=[ + -0.26642873140338846, + -0.26217358219812914, + 0.3731356281678529, + 0.07703988099301252, + 0.07842680444065203, + -0.05851961726129342, + -0.05862391113360511, + -0.35425415800358373, + 0.23433448557874187, + 0.23706320081974044, + ], + att=[ + 0.0, + -1.4333600946359863, + 0.1470326297519402, + 1.502553090163623, + 1.9646825183908976, + 0.0, + -0.8405505765908995, + -0.32378354968013, + 1.0394622134176261, + 1.2023475510925157, + ], + ), + "gmin1": dict( + kwargs={"base_period": "gmin1"}, + estimate=1.4221735897240106, + pretrend_bias=-0.3554385674608017, + post_only=1.777612157184812, + ess=59.999999999999986, + weight=None, # identical to nocov (weights do not depend on the base period) + att=[ + 1.4436560985156102, + 0.0, + 1.7041893026438784, + 2.986030974822525, + 3.4641139245939594, + 0.2253310710878238, + -0.6564285249621264, + 0.0, + 1.279736868004265, + 1.4414620717192705, + ], + ), + } + + @pytest.mark.parametrize("key", ["nocov", "cov", "gmin1"]) + def test_decomposition_weighted_branches(self, key): + spec = self._DEC[key] + df = self._weighted_panel() + result = diff_diff.decompose_twfe_weights( + df, outcome="y", unit="id", time="t", first_treat="g", weights="w", **spec["kwargs"] + ) + assert result.estimate == pytest.approx(spec["estimate"], abs=1e-12) + assert result.pretrend_bias == pytest.approx(spec["pretrend_bias"], abs=1e-12) + assert result.post_only == pytest.approx(spec["post_only"], abs=1e-12) + assert result.effective_sample_size == pytest.approx(spec["ess"], abs=1e-9) + expected_w = spec["weight"] if spec["weight"] is not None else self._DEC["nocov"]["weight"] + np.testing.assert_allclose(result.cells["weight"].to_numpy(), expected_w, atol=1e-12) + np.testing.assert_allclose(result.cells["att"].to_numpy(), spec["att"], atol=1e-12) + + _AGG = { + "twfe": ( + 1.4348838554104435, + [ + -0.2638376383763837, + -0.2638376383763837, + 0.3726937269372694, + 0.0774907749077491, + 0.0774907749077491, + -0.059040590405904085, + -0.059040590405904085, + -0.3542435424354242, + 0.2361623616236162, + 0.2361623616236162, + ], + ), + "overall": ( + 2.101822892918353, + [ + 0, + 0, + 0.17874396135265702, + 0.17874396135265702, + 0.17874396135265702, + 0, + 0, + 0, + 0.2318840579710145, + 0.2318840579710145, + ], + ), + "simple": ( + 2.239531384413449, + [ + 0, + 0, + 0.21142857142857147, + 0.21142857142857147, + 0.21142857142857147, + 0, + 0, + 0, + 0.18285714285714288, + 0.18285714285714288, + ], + ), + } + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_attgt_weighted_branches(self, aggregation): + df = self._weighted_panel() + cs = diff_diff.CallawaySantAnna(base_period="universal", control_group="never_treated").fit( + df, outcome="y", unit="id", time="t", first_treat="g" + ) + unit_w = df.groupby("id", sort=True)["w"].first().to_numpy() + result = attgt_weights(cs, aggregation=aggregation, weights=unit_w) + implied, weight = self._AGG[aggregation] + assert result.implied_att == pytest.approx(implied, abs=1e-12) + np.testing.assert_allclose(result.weights["weight"].to_numpy(), weight, atol=1e-12) + assert list(zip(result.weights["group"], result.weights["time"])) == [ + (3, 1), + (3, 2), + (3, 3), + (3, 4), + (3, 5), + (4, 1), + (4, 2), + (4, 3), + (4, 4), + (4, 5), + ] + + +# --------------------------------------------------------------------------- +# Review-response regression tests (PR #812 items 1-5, 9, 17, 19) +# --------------------------------------------------------------------------- + + +def _gt_frame(fitted): + return fitted.to_dataframe("group_time") + + +def _frame_call(frame, panel, **kw): + return attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat", **kw + ) + + +class TestCohortLabelValidation: + """Item 1: NaN / -inf labels are errors, never a silent never-treated unit.""" + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_decompose_rejects_non_finite_labels(self, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_frame_path_rejects_non_finite_labels(self, fitted, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(_gt_frame(fitted), df) + + def test_plus_inf_is_never_treated(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["first_treat"] == 0, "first_treat"] = np.inf + with_inf = _frame_call(_gt_frame(fitted), df) + with_zero = _frame_call(_gt_frame(fitted), panel) + np.testing.assert_allclose( + with_inf.weights["weight"].to_numpy(), + with_zero.weights["weight"].to_numpy(), + atol=1e-15, + ) + + def test_nan_in_one_period_fails_invariance(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "first_treat"] = np.nan + with pytest.raises(ValueError, match="varies within unit"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + def test_nan_period_label_is_rejected_up_front(self, fitted, panel): + df = panel.copy() + df["period"] = df["period"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "period"] = np.nan + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +class TestBalanceNaNPropagation: + """Item 2: frac_extreme's NA for <3 distinct values survives the summary roll-up.""" + + @pytest.fixture(scope="class") + def decomposed(self): + df = _panel() + rng = np.random.RandomState(3) + df["binary"] = rng.binomial(1, 0.4, size=len(df)).astype(float) + df["const"] = 1.0 + df["cont"] = rng.normal(size=len(df)) + # Make the binary / constant columns unit-invariant so the unit mean + # keeps them at <3 distinct values. + df["binary"] = df.groupby("unit")["binary"].transform("first") + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["cont"], + balance_covariates=["binary", "const", "cont"], + ) + + def test_cell_level_is_nan_for_degenerate_covariates(self, decomposed): + cells = decomposed.covariate_balance(level="cell") + for cov in ("binary", "const"): + sub = cells[cells["covariate"] == cov] + assert sub["unweighted_frac_extreme"].isna().all() + assert sub["weighted_frac_extreme"].isna().all() + cont = cells[cells["covariate"] == "cont"] + assert np.isfinite(cont["unweighted_frac_extreme"]).all() + + def test_summary_level_propagates_nan_not_zero(self, decomposed): + summary = decomposed.covariate_balance(level="summary").set_index("covariate") + for cov in ("binary", "const"): + assert np.isnan(summary.loc[cov, "unweighted_frac_extreme"]) + assert np.isnan(summary.loc[cov, "weighted_frac_extreme"]) + # The other statistics are ordinary sums and stay finite. + assert np.isfinite(summary.loc[cov, "unweighted_diff"]) + assert np.isfinite(summary.loc["cont", "weighted_frac_extreme"]) + + +class TestCovariateGuard: + """Item 3: a covariate-adjusted CS fit is not a TWFE regression.""" + + def test_covariate_adjusted_fit_is_rejected_under_twfe(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(5).normal(size=len(df)) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + ) + assert fit._aggregation_kit.bookkeeping["covariates"] == ("x",) + with pytest.raises(ValueError, match="requires a fit without covariates"): + attgt_weights(fit, aggregation="twfe") + # The CS estimands do not depend on the regression specification. + assert attgt_weights(fit, aggregation="overall").n_negative_post == 0 + + def test_unadjusted_fit_records_empty_covariates(self, fitted): + assert fitted._aggregation_kit.bookkeeping["covariates"] == () + + def test_legacy_kit_without_the_key_warns(self, fitted): + kit = fitted._aggregation_kit + saved = kit.bookkeeping.pop("covariates") + try: + with pytest.warns(UserWarning, match="predates covariate bookkeeping"): + attgt_weights(fitted, aggregation="twfe") + finally: + kit.bookkeeping["covariates"] = saved + + def test_wrong_result_type_is_a_type_error(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(TypeError, match="CallawaySantAnna"): + attgt_weights(dec) # type: ignore[arg-type] + + +class TestFrameValidationAndGrid: + """Item 4: duplicates, non-finite cells, and incomplete grids fail closed.""" + + def test_duplicate_cells_are_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + dup = pd.concat([frame, frame.iloc[[2]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicated \\(group, time\\)"): + _frame_call(dup, panel) + + def test_non_finite_group_label_is_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + frame.loc[0, "group"] = np.nan + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(frame, panel) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_inf_att_on_a_post_cell_is_an_incomplete_grid(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + idx = frame.index[(frame["group"] == 3) & (frame["time"] == 3)][0] + frame.loc[idx, "effect"] = np.inf + with pytest.raises(ValueError, match="complete .* grid"): + _frame_call(frame, panel, aggregation=aggregation) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_missing_post_cell_raises_for_every_aggregation(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 3) & (frame["time"] == 3))] + with pytest.raises(ValueError, match="required cell\\(s\\) are missing"): + _frame_call(frame, panel, aggregation=aggregation) + + def test_missing_pre_cell_raises_for_twfe_but_warns_for_cs_estimands(self, fitted, panel): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 4) & (frame["time"] == 1))] + with pytest.raises(ValueError, match="complete cohort x period grid"): + _frame_call(frame, panel, aggregation="twfe") + complete = _frame_call(_gt_frame(fitted), panel, aggregation="overall") + for aggregation in ("overall", "simple"): + # An ABSENT pre row is not a drop: nothing to warn about, weights unchanged. + partial = _frame_call(frame, panel, aggregation=aggregation) + assert partial.n_dropped_cells == 0 + assert len(partial.weights) == len(frame) + ref = _frame_call(_gt_frame(fitted), panel, aggregation=aggregation) + assert partial.implied_att == pytest.approx(ref.implied_att, abs=1e-15) + # A NaN pre cell (present but non-estimable) is what n_dropped_cells counts. + frame = _gt_frame(fitted) + frame.loc[frame.index[(frame["group"] == 4) & (frame["time"] == 1)][0], "effect"] = np.nan + with pytest.warns(UserWarning, match="pre-treatment group-time cell"): + dropped = _frame_call(frame, panel, aggregation="overall") + assert dropped.n_dropped_cells == 1 + assert dropped.implied_att == pytest.approx(complete.implied_att, abs=1e-15) + + def test_first_period_cohort_is_dropped_like_r_did(self): + df = _panel(cohorts=(0, 1, 3, 4), n_periods=5) + fit = _fit(df) + with pytest.warns(UserWarning, match="no estimable post-treatment cell"): + result = attgt_weights(fit, aggregation="overall") + assert 1 not in set(result.weights["group"]) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + # Same numbers as fitting on the panel with those units removed up front. + pre_filtered = _fit(df[df["first_treat"] != 1]) + reference = attgt_weights(pre_filtered, aggregation="overall") + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-12 + ) + + def test_bare_frame_missing_a_whole_non_first_cohort_raises(self, fitted, panel): + frame = _gt_frame(fitted).drop(columns=["skip_reason"], errors="ignore") + frame = frame[frame["group"] != 4] + with pytest.raises(ValueError, match="no post-treatment cell in the ATT"): + _frame_call(frame, panel, aggregation="overall") + + def test_not_yet_treated_carve_out_mirrors_aggte(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + with pytest.warns(UserWarning) as record: + result = attgt_weights(fit, aggregation="overall") + messages = " | ".join(str(w.message) for w in record) + assert "structurally absent" in messages # (3,5),(3,6),(4,5),(4,6) + assert "no estimable post-treatment cell" in messages # cohort 5 + assert set(result.weights["group"]) == {3, 4} + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + post = result.weights[result.weights["post"] == 1] + # cohort 3 keeps (3,3),(3,4): divisor 2; cohort 4 keeps (4,4): divisor 1 + assert post[post["group"] == 3]["time"].tolist() == [3, 4] + assert post[post["group"] == 4]["time"].tolist() == [4] + w3 = post[post["group"] == 3]["weight"].to_numpy() + w4 = post[post["group"] == 4]["weight"].to_numpy() + assert w3[0] == pytest.approx(w3[1]) + assert w4[0] == pytest.approx(2 * w3[0]) # equal cohorts: pbar_3 == pbar_4 + expected = float((post["weight"] * post["att"]).sum()) + assert result.implied_att == pytest.approx(expected, abs=1e-12) + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + +class TestWeightValidation: + """Item 5: finite, non-negative, positive treated (and control) mass.""" + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda w: np.where(np.arange(len(w)) < 50, -1.0, w), "non-negative"), + (lambda w: np.where(np.arange(len(w)) == 0, np.nan, w), "must be finite"), + (lambda w: np.where(np.arange(len(w)) == 0, np.inf, w), "must be finite"), + (lambda w: np.zeros_like(w), "sum to zero"), + ], + ) + def test_bad_unit_weights_are_rejected(self, fitted, mutate, match): + w = mutate(np.ones(len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]))) + with pytest.raises(ValueError, match=match): + attgt_weights(fitted, aggregation="overall", weights=w) + + def test_zero_control_mass_only_matters_where_controls_enter(self, fitted, panel): + cohorts = np.asarray(fitted._aggregation_kit.bookkeeping["unit_cohorts"], dtype=float) + w = np.where(cohorts == 0, 0.0, 1.0) + with pytest.raises(ValueError, match="never-treated comparison group carries zero"): + attgt_weights(fitted, aggregation="twfe", weights=w) + for aggregation in ("overall", "simple"): + assert attgt_weights(fitted, aggregation=aggregation, weights=w).n_cells > 0 + + def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): + df = panel.copy() + df["w"] = 1.0 + df.loc[(df["unit"] == 3) & (df["period"] == 2), "w"] = np.nan + with pytest.raises(ValueError, match="must be finite"): + diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + +class TestNegativePostWeights: + """Item 9: the pathology is negative weight on POST cells.""" + + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_cs_estimands_have_no_negative_post_weight(self, fitted, aggregation): + result = attgt_weights(fitted, aggregation=aggregation) + assert result.n_negative_post == 0 + assert result.negative_post_weight_share == 0.0 + + def test_twfe_reports_both_labelled(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative_post <= result.n_negative + assert 0.0 <= result.negative_post_weight_share <= 1.0 + d = result.to_dict() + assert {"n_negative_post", "negative_post_weight_share"} <= set(d) + assert "Negative POST-period cells:" in result.summary() + + +class TestWeightedTwfeExtension: + """Item 17: weighted aggregation="twfe" has no R counterpart; tie it to the decomposition.""" + + def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): + df = panel.copy() + rng = np.random.RandomState(9) + unit_w = pd.Series( + rng.choice([0.5, 1.0, 2.0], size=df["unit"].nunique()), + index=sorted(df["unit"].unique()), + ) + df["w"] = df["unit"].map(unit_w) + fit = _fit(df) + weighted = attgt_weights(fit, aggregation="twfe", weights=unit_w.to_numpy()) + decomposed = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + weighted.weights["weight"].to_numpy(), decomposed.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestHandComputedWeights: + """Item 19: assert against numbers computed OUTSIDE the module.""" + + def test_implied_att_equals_hand_computed_overall(self): + # 2 cohorts (g=2,3) x 3 periods, 10 + 30 treated units + 20 never-treated. + frame = pd.DataFrame( + { + "group": [2, 2, 2, 3, 3, 3], + "time": [1, 2, 3, 1, 2, 3], + "effect": [0.0, 1.0, 2.0, 0.0, 0.0, 4.0], + } + ) + rows = [] + for u, g in enumerate([0] * 20 + [2] * 10 + [3] * 30): + for t in (1, 2, 3): + rows.append({"unit": u, "period": t, "first_treat": g, "outcome": 0.0}) + panel = pd.DataFrame(rows) + result = _frame_call(frame, panel, aggregation="overall") + # pbar_2 = 10/40, pbar_3 = 30/40; cohort 2 has 2 post periods, cohort 3 has 1. + expected = (10 / 40) / 2 * (1.0 + 2.0) + (30 / 40) / 1 * 4.0 + assert result.implied_att == pytest.approx(expected, abs=1e-15) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-15) + + def test_all_five_public_names_are_exported(self): + for name in ( + "attgt_weights", + "decompose_twfe_weights", + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "plot_twfe_weights", + ): + assert name in diff_diff.__all__, name + assert hasattr(diff_diff, name) + + +class TestCollinearCovariates: + """Item 7: rank-deficient designs go through solve_ols' R-style NaN handling.""" + + def test_exactly_collinear_pair_warns_once_and_leaves_the_estimate_unchanged(self, panel): + df = panel.copy() + rng = np.random.RandomState(21) + df["x1"] = rng.normal(size=len(df)) + df["x2"] = 2.0 * df["x1"] # exactly collinear twin + common = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + with pytest.warns(UserWarning, match="dropped collinear covariate") as record: + both = diff_diff.decompose_twfe_weights(df, covariates=["x1", "x2"], **common) + collinear = [w for w in record if "dropped collinear" in str(w.message)] + assert len(collinear) == 1 + message = str(collinear[0].message) + assert ("'x1'" in message) != ("'x2'" in message) # exactly one of the pair + alone = diff_diff.decompose_twfe_weights(df, covariates=["x1"], **common) + assert both.estimate == pytest.approx(alone.estimate, abs=1e-12) + np.testing.assert_allclose( + both.cells["weight"].to_numpy(), alone.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestDecompositionEdgeCases: + """Item 8: the REGISTRY edge cases for decompose_twfe_weights, asserted.""" + + COMMON = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + + def test_unbalanced_panel_is_rejected(self, panel): + df = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) + with pytest.raises(ValueError, match="balanced panel"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_no_never_treated_group_is_rejected(self): + df = _panel(cohorts=(3, 4)) + with pytest.raises(ValueError, match="never-treated units"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_time_varying_cohort_is_rejected(self, panel): + df = panel.copy() + df.loc[(df["unit"] == 50) & (df["period"] == 5), "first_treat"] = 4 + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_gmin1_with_a_first_period_cohort_is_rejected(self): + df = _panel(cohorts=(0, 1, 3)) + with pytest.raises(ValueError, match="gmin1"): + diff_diff.decompose_twfe_weights(df, base_period="gmin1", **self.COMMON) + + def test_balance_without_request_and_bad_level(self, panel): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + with pytest.raises(ValueError, match="balance_covariates="): + result.covariate_balance() + df = panel.copy() + df["x"] = np.random.RandomState(4).normal(size=len(df)) + with_balance = diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **self.COMMON) + with pytest.raises(ValueError, match="level must be"): + with_balance.covariate_balance(level="cohort") + + def test_bad_method_and_base_period(self, panel): + with pytest.raises(ValueError, match="method must be"): + diff_diff.decompose_twfe_weights(panel, method="aipw", **self.COMMON) + with_bad = dict(self.COMMON) + with pytest.raises(ValueError, match="base_period must be"): + diff_diff.decompose_twfe_weights(panel, base_period="universal", **with_bad) + + def test_identities_hold(self, panel, fitted): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + assert result.pretrend_bias + result.post_only == pytest.approx( + result.decomposition, abs=1e-12 + ) + assert result.remainder == 0.0 + assert attgt_weights(fitted, aggregation="twfe").implied_att == pytest.approx( + result.estimate, abs=1e-6 + ) + gmin1 = diff_diff.decompose_twfe_weights(panel, base_period="gmin1", **self.COMMON) + assert gmin1.estimate == pytest.approx(result.estimate, abs=1e-10) + assert gmin1.estimate == pytest.approx(gmin1.decomposition + gmin1.remainder, abs=1e-12) + + +class TestPlotTWFEWeights: + """Item 8 / 13: matplotlib behaviour of plot_twfe_weights.""" + + @pytest.fixture(autouse=True) + def _agg_backend(self): + matplotlib = pytest.importorskip("matplotlib") + matplotlib.use("Agg") + yield + import matplotlib.pyplot as plt + + plt.close("all") + + def test_weights_view(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + ax = diff_diff.plot_twfe_weights(result, show=False) + assert ax.get_xlabel() == "Implicit weight" + assert "negative" in ax.get_title() + assert len(ax.collections) == 2 # post + pre scatters + + def test_ax_reuse_and_annotate(self, fitted): + import matplotlib.pyplot as plt + + _, ax = plt.subplots() + out = diff_diff.plot_twfe_weights( + attgt_weights(fitted, aggregation="overall"), ax=ax, annotate=True, show=False + ) + assert out is ax + assert len(ax.texts) == 10 + + def test_balance_view_and_auto(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(8).normal(size=len(df)) + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + ax = diff_diff.plot_twfe_weights(dec, show=False) # auto -> balance + assert "balance" in ax.get_title().lower() + ax2 = diff_diff.plot_twfe_weights(dec, kind="weights", show=False) + assert ax2.get_ylabel() == "ATT(g, t)" + + def test_balance_requested_without_table_raises(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(ValueError, match="no covariate balance table"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_all_nan_balance_table_is_a_clear_error(self, panel): + df = panel.copy() + df["const"] = 1.0 # zero pooled SD -> standardized diffs are all NaN + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + balance_covariates=["const"], + ) + with pytest.raises(ValueError, match="no finite differences"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_bad_kind_and_backend(self, fitted): + result = attgt_weights(fitted) + with pytest.raises(ValueError, match="kind must be"): + diff_diff.plot_twfe_weights(result, kind="heat", show=False) + with pytest.raises(ValueError, match="backend must be"): + diff_diff.plot_twfe_weights(result, backend="bokeh", show=False) diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py index a668573a6..c0c35f31c 100644 --- a/tests/test_twfe_weights_parity.py +++ b/tests/test_twfe_weights_parity.py @@ -6,7 +6,8 @@ implementation matches R ``twfeweights`` 0.9.0. **R is only needed to regenerate the JSON**, never to run these tests. The -committed JSON plus its sibling panel CSVs are the source of truth and the +committed JSON plus the panel CSVs it names (two simulated siblings, plus +the shared ``mpdta_stata_panel.csv``) are the source of truth and the assertions run on any Python-only environment. Tests skip ONLY if a fixture file is absent. @@ -54,11 +55,21 @@ def golden(): def _fixture(golden, name): + """Panel for a fixture, with any golden-declared derived columns applied. + + ``fixtures.mpdta`` points at the SHARED ``mpdta_stata_panel.csv`` rather + than a renamed copy of it, and declares its one derived column + (``lpop_t``) as an expression evaluated against that file's own names. + Fixtures without a ``derived_columns`` block are unaffected. + """ payload = golden["fixtures"][name] path = DATA_DIR / payload["data_file"] if not path.exists(): pytest.skip(f"panel {path} not found; run: {REGENERATE}") - return payload, pd.read_csv(path) + df = pd.read_csv(path) + for column, expression in (payload.get("derived_columns") or {}).items(): + df[column] = df.eval(expression) + return payload, df def _sorted_golden_weights(block): @@ -285,6 +296,16 @@ def _decompose(df, cols, **kwargs): ) +def _assert_cell_labels(cells, golden_cells): + """Row-for-row (group, time, post) agreement before any value comparison.""" + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + cells[column].to_numpy(dtype=float), + np.asarray(golden_cells[column], dtype=float), + err_msg=f"cell {column!r} labels differ from the golden", + ) + + def _degenerate_mask(cells, golden_cells): """Rows where R's ATT(g,t) is a 0/0 artifact rather than a disagreement. @@ -330,6 +351,7 @@ def test_no_covariate_branch_two_ways(self, golden, fixture): assert without.estimate == pytest.approx(with_invariant.estimate, abs=1e-15) for result in (without, with_invariant): + _assert_cell_labels(result.cells, expected["cells"]) np.testing.assert_allclose(result.estimate, expected["estimate"], atol=DEMEAN_ATOL) np.testing.assert_allclose( result.cells["weight"].to_numpy(), @@ -360,17 +382,47 @@ def test_scalars(self, golden, fixture, key): # leaving their sum exact. np.testing.assert_allclose(result.estimate, expected["estimate"], atol=atol) - split_atol = atol - if _degenerate_mask(result.cells, expected["cells"]).any(): - split_atol = DEGENERATE_CELL_ATOL + _assert_cell_labels(result.cells, expected["cells"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + # The decomposition/remainder SPLIT only moves under gmin1 (the remainder + # is built from the degenerate comparison-group weights). Under + # first_period the remainder is identically zero, so the split is gated + # tight even where the mask fires (observed gap on sim_staggered/fwl_nocov + # is 3e-15). + split_atol = DEGENERATE_CELL_ATOL if (degenerate.any() and key == "fwl_gmin1") else atol for field in ("decomposition", "remainder"): np.testing.assert_allclose(getattr(result, field), expected[field], atol=split_atol) + # pretrend_bias / post_only straddle the pre/post split, so R's 0/0 noise + # at the degenerate cells (which sit on opposite sides of it on + # sim_staggered) moves each by ~1.2e-4 while their sum stays exact. + pp_atol = DEGENERATE_CELL_ATOL if degenerate.any() else atol + for field in ("pretrend_bias", "post_only"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=pp_atol) + + # effective_sample_size: post_count * sum_post(weight * ess). At the + # degenerate cells R's ess is a ratio of rounding errors (gap ~0.99 on + # sim_staggered), so the expected value is rebuilt from R's OWN cells, + # substituting our limit ess only where R's is noise - independent of + # our implementation, and not a tautology. + g_cells = expected["cells"] + r_w = np.asarray(g_cells["weight"], dtype=float) + r_ess = np.asarray(g_cells["ess"], dtype=float) + r_post = np.asarray(g_cells["post"], dtype=bool) + our_ess = result.cells["ess"].to_numpy(dtype=float) + ess_ref = np.where(degenerate, our_ess, r_ess) + expected_ess = r_post.sum() * float((r_w[r_post] * ess_ref[r_post]).sum()) + np.testing.assert_allclose(result.effective_sample_size, expected_ess, atol=1e-6) + if not degenerate.any(): + np.testing.assert_allclose( + result.effective_sample_size, expected["effective_sample_size"], atol=1e-6 + ) + # estimate == decomposition + remainder is an identity, not a fit assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) @pytest.mark.parametrize("fixture", FIXTURES) - @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov"]) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) def test_cells(self, golden, fixture, key): payload, df = _fixture(golden, fixture) cols = payload["columns"] @@ -378,10 +430,15 @@ def test_cells(self, golden, fixture, key): kwargs = { "fwl_nocov": {"covariates": None}, "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, }[key] atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL result = _decompose(df, cols, **kwargs) + # Label alignment: the golden now carries ORIGINAL period labels in + # every block, so the row-for-row comparisons below are anchored rather + # than merely positional. + _assert_cell_labels(result.cells, expected["cells"]) np.testing.assert_allclose( result.cells["weight"].to_numpy(), np.asarray(expected["cells"]["weight"]), @@ -399,6 +456,15 @@ def test_cells(self, golden, fixture, key): atol=DEGENERATE_CELL_ATOL, ) + # Cell ess / remainder: tight where R is a valid reference; at the + # degenerate cells R divides rounding errors (cell-ess gap up to ~0.53 + # on sim_staggered), so only finiteness is asserted there. + for field in ("ess", "remainder"): + ours = result.cells[field].to_numpy(dtype=float) + theirs = np.asarray(expected["cells"][field], dtype=float) + np.testing.assert_allclose(ours[~degenerate], theirs[~degenerate], atol=atol) + assert np.isfinite(ours[degenerate]).all() + class TestDecompositionIsExactAtDegenerateCells: """Where R reports 0/0 noise, we report the analytic limit.""" @@ -453,6 +519,12 @@ def test_cell_level(self, golden, fixture): expected = expected.sort_values(key).reset_index(drop=True) actual = actual.sort_values(key).reset_index(drop=True) assert actual["covariate"].tolist() == expected["covariate"].tolist() + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + actual[column].to_numpy(dtype=float), + expected[column].to_numpy(dtype=float), + err_msg=f"balance cell {column!r} labels differ from the golden", + ) for r_name, our_name in R_BALANCE_COLUMNS.items(): np.testing.assert_allclose( diff --git a/tests/test_visualization_plotly.py b/tests/test_visualization_plotly.py index d3c4d774b..f8fb915f4 100644 --- a/tests/test_visualization_plotly.py +++ b/tests/test_visualization_plotly.py @@ -662,3 +662,63 @@ def test_band_labels(self): results.alpha = 0.025 fig_r_frac = plot_dose_response(results, backend="plotly", show=False) assert self._band_traces(fig_r_frac)[0].name == "97.5% CI" + + +class TestPlotlyTWFEWeights: + """Plotly backend for plot_twfe_weights (both views).""" + + @staticmethod + def _panel_and_fit(): + import pandas as pd + + import diff_diff + + rng = np.random.RandomState(11) + first_treat = np.repeat(np.array([0, 3, 4]), 30) + rows = [] + for t in range(1, 6): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(len(first_treat)), + "period": t, + "first_treat": first_treat, + "outcome": rng.normal(size=len(first_treat)) + treated * 1.0, + "x": rng.normal(size=len(first_treat)), + } + ) + ) + df = pd.concat(rows, ignore_index=True) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, outcome="outcome", unit="unit", time="period", first_treat="first_treat") + return df, fit + + def test_weights_view(self): + import diff_diff + + _, fit = self._panel_and_fit() + fig = diff_diff.plot_twfe_weights( + diff_diff.attgt_weights(fit), backend="plotly", show=False + ) + assert isinstance(fig, go.Figure) + assert len(fig.data) == 2 # post + pre traces + assert len(fig.layout.shapes) >= 2 # zero lines + + def test_balance_view(self): + import diff_diff + + df, _ = self._panel_and_fit() + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + fig = diff_diff.plot_twfe_weights(dec, backend="plotly", show=False) + assert isinstance(fig, go.Figure) + assert any(trace.name == "no improvement" for trace in fig.data) From ec975c846c1eae352ba793a1cbe4e1498aa75e0c Mon Sep 17 00:00:00 2001 From: yiyi Date: Mon, 14 Sep 2026 15:34:36 +0800 Subject: [PATCH 07/13] fix(twfeweights): address second review round (#812) Addresses igerber's second review of #812 (1 rename decision + items 1-18), rebased onto current main. As before, most of it is about the paths R never had to handle: the ATT(g,t) frame, anticipation, unbalanced panels, and non-standard period labels. Decision - rename `pretrend_bias` to `pre_period_contribution`. The field is the SAMPLE contribution of the pre-treatment cells, which can reflect differential pre-trends OR sampling variation (the diagnostic carries no inference), so it is diagnostic evidence about the earlier-period restrictions, not proof the identifying assumption fails post-treatment. The three doc surfaces say so. Correctness (items 1-6), all reproduced by the reviewer: - One canonical numeric time key. Period labels are coerced once at validation and that key drives sorting, reshaping, cohort mapping and the grid; using the raw column lets "10" sort before "2" and silently rebuilds a different panel. Numeric and numeric-string labels now decompose identically ([1,2,10,11,12] pinned). - `anticipation` is honoured on `"overall"`/`"simple"`: post cells become `t >= g - anticipation` and the window enters the required-cell grid, the available-period divisors and the `post` column. The fit is read off the aggregation kit; the frame path takes an explicit `anticipation=` and the fitted path rejects the kwarg instead of ignoring it. `"twfe"` deliberately keeps `t >= g` (the regression's own indicator does not anticipate, and R's twfe_weights has no anticipation argument). Each CS estimand is pinned against `results.aggregate(...)` with anticipation=1 under both control groups. - Unbalanced panels are rejected: the kit records `is_balanced`, and the frame path requires exactly one observation per unit-period. The cohort shares and E_t[D] assume a fixed unit set. - Non-finite outcomes, regression covariates and balance covariates fail closed instead of returning an all-NaN decomposition. - The zero_treated_control carve-out keys on the `skip_reason` VALUE, not on `control_group`, so the fitted and frame paths agree (the frame path has no control_group to read). A not-yet-treated fit is added to the fit-vs-frame agreement test. - A cohort is dropped only when the drop is structural: treated in the first observed period, or every missing post cell carries zero_treated_control. A mid cohort blanked out any other way (NaN effects, skip_reason None) raises instead of silently leaving the estimand. Numerics and plotting (items 8-11): - The annihilation threshold is the accumulated rounding-noise scale (`sqrt(n_obs) * 64 * eps * max(raw, 1)`) instead of a fixed 1e-10, which was five orders too loose (it discarded a 1e6-level covariate with genuine within-sd 1e-4) while still annihilating mpdta's lpop. A dropped covariate no longer appears in `result.covariates`. - The weights-view title counts POST-only negatives (a healthy panel was titled "(5 negative)"); the signed balance view draws the no-improvement diagonal over [-limit, limit] in both backends; the plotly renderer's dead `text`/`customdata` lines are gone while `text` still feeds the hover. - A wrong-length `weights=` now fails with both lengths before the excluded-cohort slice, instead of a raw IndexError. Docs (items 7, 16, 17): REGISTRY corrects R's mechanism for the second carve-out (`pre_process_did` truncating to t < max(g) and recoding the last cohort, not aggte averaging) and records the new guarantees; the R generator header, the mpdta check comment and `meta.mpdta_provenance` now say identical-on-integers / 1e-14-on-floats rather than "bit-for-bit"; the `plot_twfe_weights` subsection moves out of the estimator catalog to sit beside plot_bacon under Visualization. Tests (items 12-15): degenerate-cell ESS is asserted at the analytic `n_control`; a two-covariate frozen-numbers pin covers the multi-column solve_ols branch for the first time; the re-implementation test that could not fail is deleted and the export test covers all five names; parity asserts `n_negative_post` / `negative_post_weight_share` against R's own golden weights. New regression tests cover items 1-6, 10 and 11. Verified: the two twfe suites at 181 passed; docs IA, doc-deps, guides, changelog fragments, diagnostic roster, serialization, aggregate-contract, spillover, staggered, dml_did and all visualization suites green; full run 14793 passed. ruff/black clean, mypy at zero errors. --- benchmarks/R/generate_twfeweights_golden.R | 15 +- benchmarks/data/twfeweights_golden.json | 26 +- .../20260831-twfe-weight-diagnostics.md | 24 +- diff_diff/guides/llms-full.txt | 45 ++- diff_diff/guides/llms.txt | 2 +- diff_diff/staggered.py | 5 + diff_diff/twfe_weights.py | 325 +++++++++++++---- diff_diff/twfe_weights_results.py | 31 +- diff_diff/visualization/_diagnostic.py | 36 +- docs/api/twfe_weights.rst | 12 +- docs/methodology/REGISTRY.md | 22 +- tests/helpers/results_foundation.py | 2 +- tests/test_twfe_weights.py | 345 +++++++++++++++++- tests/test_twfe_weights_parity.py | 42 ++- tests/test_visualization_plotly.py | 30 ++ 15 files changed, 766 insertions(+), 196 deletions(-) diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R index 5e3e619a5..2666e8b1c 100644 --- a/benchmarks/R/generate_twfeweights_golden.R +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -8,8 +8,9 @@ # benchmarks/data/twfeweights_unbalanced_panel.csv # # The mpdta fixture reads the EXISTING benchmarks/data/mpdta_stata_panel.csv -# rather than writing a renamed copy of it; this script asserts the two agree -# bit-for-bit on every shared column before using it. +# rather than writing a renamed copy of it; this script asserts the integer +# identifier columns match exactly and the float columns agree to CSV +# round-trip precision (1e-14 relative) before using it. # # Run from the repository root: # Rscript benchmarks/R/generate_twfeweights_golden.R @@ -140,7 +141,7 @@ extract_fwl <- function(obj, periods) { estimate = obj$est, decomposition = obj$decomposition_est, remainder = obj$decomposition_remainder, - pretrend_bias = obj$pt_violations_bias, + pre_period_contribution = obj$pt_violations_bias, post_only = sum(wt[post == 1] * att[post == 1]), # summary.decomposed_twfe:351 effective_sample_size = sum(post) * sum(wt[post == 1] * ess[post == 1]) @@ -165,7 +166,7 @@ extract_aipw <- function(obj, periods) { estimate = obj$est, decomposition = obj$decomposition_est, remainder = obj$decomposition_remainder, - pretrend_bias = obj$pt_violations_bias, + pre_period_contribution = obj$pt_violations_bias, post_only = sum(wt[post == 1] * att[post == 1]), # summary.decomposed_aipw:1006 — note the inner sum is NOT post-filtered, # unlike the twfe roll-up. Preserved verbatim. @@ -371,7 +372,7 @@ stopifnot( # fixture deliberately exercises the documented 0/0 cells - it is not a # "no cell is degenerate" design. # -# `0.3 * x1 * period` gives each unit a trend, so pretrend_bias is non-zero, +# `0.3 * x1 * period` gives each unit a trend, so pre_period_contribution is non-zero, # but x1 is iid and cohorts are assigned by unit INDEX, so E[x1 | g] does not # vary by cohort: the differential pre-trend is zero in expectation and the # observed value (~0.093) is sampling noise, not a designed pre-trend. @@ -466,7 +467,9 @@ payload <- list( "fixtures.mpdta is data(mpdta, package = \"did\") version", as.character(packageVersion("did")), "- read from the shared benchmarks/data/mpdta_stata_panel.csv, whose", - "columns this generator asserts are bit-identical to data(mpdta).", + "integer identifier columns this generator asserts are identical to", + "data(mpdta) and whose float columns it asserts agree to CSV round-trip", + "precision (1e-14 relative).", "`lpop_t` is derived (see fixtures.mpdta.derived_columns)." ), reserved_blocks = paste( diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json index 30cfbe030..69502f40c 100644 --- a/benchmarks/data/twfeweights_golden.json +++ b/benchmarks/data/twfeweights_golden.json @@ -12,7 +12,7 @@ "sim_staggered": 20260831, "unbalanced_cohorts": 20260901 }, - "mpdta_provenance": "fixtures.mpdta is data(mpdta, package = \"did\") version 2.5.1 - read from the shared benchmarks/data/mpdta_stata_panel.csv, whose columns this generator asserts are bit-identical to data(mpdta). `lpop_t` is derived (see fixtures.mpdta.derived_columns).", + "mpdta_provenance": "fixtures.mpdta is data(mpdta, package = \"did\") version 2.5.1 - read from the shared benchmarks/data/mpdta_stata_panel.csv, whose integer identifier columns this generator asserts are identical to data(mpdta) and whose float columns it asserts agree to CSV round-trip precision (1e-14 relative). `lpop_t` is derived (see fixtures.mpdta.derived_columns).", "reserved_blocks": "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED: they capture implicit_aipw_weights, aipw_cov_bal and the two_period_reg_weights / two_period_aipw_weights kernels, none of which has a Python surface yet (method=\"aipw\" is a documented follow-up). They are kept so that follow-up needs no R re-run. NOTE the AIPW golden is covariate-adjusted: a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score.", "label_convention": "Every cells block (attgt_weights.*, decompose.*, balance.*) carries ORIGINAL period labels. implicit_* run in positional time internally; the generator maps them back before emitting.", "no_covariate_note": "decompose.fwl_nocov is generated with xformula = ~, which is numerically the ~1 branch (double-demeaning annihilates a time-invariant regressor exactly). The ~1 branch itself cannot be called: fixest::demean segfaults on the zero-column model matrix it builds. See the comment block at the top of the generator." @@ -69,7 +69,7 @@ "estimate": -0.0365489366740672, "decomposition": -0.0365489366740672, "remainder": 0, - "pretrend_bias": -0.00817768207864544, + "pre_period_contribution": -0.00817768207864544, "post_only": -0.0283712545954217, "effective_sample_size": 2163 }, @@ -86,7 +86,7 @@ "estimate": -0.0398490715024001, "decomposition": -0.0398490715024001, "remainder": 0, - "pretrend_bias": 0.0206741188438872, + "pre_period_contribution": 0.0206741188438872, "post_only": -0.0605231903462873, "effective_sample_size": 1650.8229562172 }, @@ -103,7 +103,7 @@ "estimate": -0.0365489366740662, "decomposition": -0.0365489366740672, "remainder": 9.57567358739198e-16, - "pretrend_bias": -0.00893136238322589, + "pre_period_contribution": -0.00893136238322589, "post_only": -0.0276175742908413, "effective_sample_size": 2163 }, @@ -119,7 +119,7 @@ "estimate": -0.0328195971622414, "decomposition": -0.0328195971622414, "remainder": 0, - "pretrend_bias": 0, + "pre_period_contribution": 0, "post_only": -0.0328195971622414, "effective_sample_size": 2002.7927075831 } @@ -259,7 +259,7 @@ "estimate": 1.38176729464315, "decomposition": 1.38176729464315, "remainder": 0, - "pretrend_bias": 0.0929885879714431, + "pre_period_contribution": 0.0929885879714431, "post_only": 1.28877870667171, "effective_sample_size": 499.00962938647 }, @@ -276,7 +276,7 @@ "estimate": 1.38188157642338, "decomposition": 1.38188157642338, "remainder": 0, - "pretrend_bias": -12.4573008473221, + "pre_period_contribution": -12.4573008473221, "post_only": 13.8391824237455, "effective_sample_size": 312.37838960871 }, @@ -293,7 +293,7 @@ "estimate": 1.38176729464315, "decomposition": 1.38628305525382, "remainder": -0.00451576061067098, - "pretrend_bias": -0.0508955308063108, + "pre_period_contribution": -0.0508955308063108, "post_only": 1.43717858606014, "effective_sample_size": 499.00962938647 }, @@ -309,7 +309,7 @@ "estimate": 1.80832529747437, "decomposition": 1.80832529747437, "remainder": 0, - "pretrend_bias": 0, + "pre_period_contribution": 0, "post_only": 1.80832529747437, "effective_sample_size": 491.268276272983 } @@ -446,7 +446,7 @@ "estimate": 1.88846922090823, "decomposition": 1.88846922090823, "remainder": 0, - "pretrend_bias": -0.0541461639412249, + "pre_period_contribution": -0.0541461639412249, "post_only": 1.94261538484946, "effective_sample_size": 719.999999999998 }, @@ -463,7 +463,7 @@ "estimate": 1.88897945007637, "decomposition": 1.88897945007637, "remainder": 0, - "pretrend_bias": -0.0567591745747959, + "pre_period_contribution": -0.0567591745747959, "post_only": 1.94573862465116, "effective_sample_size": 630.228235498893 }, @@ -480,7 +480,7 @@ "estimate": 1.88846922090823, "decomposition": 1.88846922090823, "remainder": -1.52655665885959e-16, - "pretrend_bias": -0.0613032635598606, + "pre_period_contribution": -0.0613032635598606, "post_only": 1.94977248446809, "effective_sample_size": 719.999999999998 }, @@ -496,7 +496,7 @@ "estimate": 2.20641349591344, "decomposition": 2.20641349591344, "remainder": 0, - "pretrend_bias": 0, + "pre_period_contribution": 0, "post_only": 2.20641349591344, "effective_sample_size": 706.0420149054 } diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index 25b3c8cd7..ab016915f 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -5,16 +5,20 @@ - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), plus post-period negative-weight counts. Returns `ATTGTWeightsResult`. + The CS estimands honour the fit's `anticipation` window (an explicit + `anticipation=` on the frame path); `"twfe"` keeps `t >= g`. - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate from its ATT(g,t) building blocks and returns `TWFEDecompositionResult` - with `pretrend_bias` - the contribution of pre-treatment cells, i.e. of - parallel-trends violations rather than of treatment - and, with - `balance_covariates=`, implicit-weight covariate balance. - `plot_twfe_weights()` renders either view (matplotlib or plotly). + with `pre_period_contribution` - the sample contribution of the + pre-treatment cells, which can reflect parallel-trends violations or + sampling variation - and, with `balance_covariates=`, implicit-weight + covariate balance. `plot_twfe_weights()` renders either view (matplotlib + or plotly). - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits - under `aggregation="twfe"`, duplicated or non-finite ATT(g,t) cells, an - incomplete group-time grid, and invalid sampling weights. Two structural - gaps are handled as R does instead of raising: a cohort with no estimable - post cell is dropped (`did`'s first-period drop), and under - `control_group="not_yet_treated"` the CS estimands average over each - cohort's available post periods (`aggte`). + under `aggregation="twfe"`, unbalanced panels, non-finite outcomes / + covariates, duplicated or non-finite ATT(g,t) cells, an incomplete + group-time grid, and invalid sampling weights. Two structural gaps are + handled as R does instead of raising: a cohort with no estimable post cell + is dropped (`did`'s first-period drop, or running out of comparison + units), and under `control_group="not_yet_treated"` the CS estimands + average over each cohort's available post periods (`aggte`). diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index d9ccc3515..36c5289b5 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1584,24 +1584,6 @@ pathology (negative weight on POST cells); `n_negative` counts pre cells too, and is near-half in every staggered design because the TWFE weights sum to zero over the full grid. -### plot_twfe_weights - -```python -plot_twfe_weights( - results, # ATTGTWeightsResult | TWFEDecompositionResult - kind="auto", # "weights" | "balance" ("auto" picks balance - # when the result carries a balance table) - standardize=True, absolute_value=True, # balance view - annotate=False, ax=None, show=True, - backend="matplotlib", # or "plotly" -) -``` - -`kind="weights"` scatters weight against ATT(g,t), coloured by pre/post - points -left of the vertical zero line carry negative weight. `kind="balance"` scatters -unweighted against implicitly-weighted covariate differences; points near the -horizontal axis are covariates the implicit weights balance. - `decompose_twfe_weights` takes the raw panel rather than a fitted result because it re-estimates. It is tied to `attgt_weights` by an identity: `attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. @@ -2108,8 +2090,9 @@ Diagnostic result from `decompose_twfe_weights`. - `cells`: DataFrame with `group`, `time`, `post`, `att`, `weight`, `ess`, `remainder` - `estimate` == `decomposition` + `remainder` -- `pretrend_bias`: contribution of PRE-treatment cells, i.e. of - parallel-trends violations rather than of treatment +- `pre_period_contribution`: sample contribution of the PRE-treatment cells + (can reflect differential pre-trends OR sampling variation; diagnostic + evidence, not proof the post-treatment assumption fails) - `post_only`, `effective_sample_size`, `covariates`, `base_period` - `covariate_balance(level="summary"|"cell", standardize=True, post_only=True)`: implicit-weight covariate balance; raises when @@ -2631,6 +2614,28 @@ plot_bacon( ) ``` +### plot_twfe_weights + +```python +from diff_diff import plot_twfe_weights + +plot_twfe_weights( + results, # ATTGTWeightsResult | TWFEDecompositionResult + kind="auto", # "weights" | "balance" ("auto" picks balance + # when the result carries a balance table) + standardize=True, absolute_value=True, # balance view + annotate=False, ax=None, show=True, + backend="matplotlib", # or "plotly" +) +``` + +`kind="weights"` scatters weight against ATT(g,t), coloured by pre/post - points +left of the vertical zero line carry negative weight; the title counts POST-only +negatives. `kind="balance"` scatters unweighted against implicitly-weighted +covariate differences; points near the horizontal axis are covariates the +implicit weights balance. With `absolute_value=False` the reference diagonal +spans `[-limit, limit]` so signed differences read correctly. + ### plot_power_curve ```python diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 784fe8536..096f2b6a3 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -92,7 +92,7 @@ The site is organized into 5 sections, each with a landing page: - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html): Cattaneo, Jansson & Ma (2020) density-discontinuity manipulation test (`RDDensityTest`), parity with R rddensity 3.0 - boundary-adaptive local polynomial density estimation at the cutoff, robust bias-corrected inference, unrestricted/restricted models, jackknife/plugin variances, data-driven bandwidths, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics -- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pretrend_bias`, and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pre_period_contribution` (the pre-treatment cells' sample contribution, which can be pre-trends or sampling noise), and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 8278d950c..134acdafb 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -5192,6 +5192,11 @@ def _build_aggregation_kit( # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES # only - never values - so the data-minimization contract holds. bookkeeping["covariates"] = tuple(covariates or ()) + # Panel balance, recorded so ``attgt_weights`` can reject an unbalanced + # fitted result: its cohort shares and E_t[D] assume the same units in + # every period. Defaults True when the key is absent (a legacy kit, or a + # producer that never computed it). + bookkeeping["is_balanced"] = bool((precomputed or {}).get("is_balanced", True)) # Data minimization: the results object is picklable and users share # result artifacts, so the kit must not turn it into a carrier for raw diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index eda573824..7c1cda31f 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -99,8 +99,14 @@ def _validate_cohort_labels( ) -def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> None: - """Reject NaN / non-finite period labels before any grid is formed.""" +def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> np.ndarray: + """Validate period labels and return them as the canonical numeric key. + + Every downstream step - sorting, reshaping, the positional grid, cohort + mapping - MUST use this one key. Using the raw column instead lets a + numeric-string label ("10" sorts before "2" lexicographically) desynchronize + the sort order from the grid, silently rebuilding a different panel. + """ arr = pd.to_numeric(pd.Series(np.asarray(values)), errors="coerce").to_numpy(dtype=float) bad = ~np.isfinite(arr) if bad.any(): @@ -109,6 +115,7 @@ def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> None: f"label(s) (first at row {int(np.flatnonzero(bad)[0])}); every observation " "must carry a finite period" ) + return arr def _positional_grid( @@ -253,34 +260,30 @@ def _twfe_weight_vector( def _overall_weight_vector( groups: np.ndarray, - times: np.ndarray, - n_periods: int, p_treated: Dict[int, float], - n_post_available: Optional[Dict[int, int]] = None, + post_mask: np.ndarray, + n_post_available: Dict[int, int], ) -> np.ndarray: - """ATT^O weights: ``1[t >= g] * pbar_g / (maxT - g + 1)``. - - Not renormalized - the ``(maxT - g + 1)`` divisor already makes them sum - to one over a complete post-treatment grid. ``n_post_available`` replaces - that divisor with each cohort's number of AVAILABLE post periods when - some post cells are structurally absent (``control_group="not_yet_treated"`` - runs out of comparison units) - what R ``aggte(type="group")`` averages - over on such a fit. + """ATT^O weights: ``1[t >= g - anticipation] * pbar_g / n_post_g``. + + Not renormalized - the per-cohort divisor is each cohort's number of + AVAILABLE post periods, so the weights already sum to one (the + ``pbar_g`` sum to one over cohorts). Counting the available cells rather + than writing ``(maxT - g + 1)`` analytically is what makes the + anticipation window and the structurally-absent ``zero_treated_control`` + cells come out right, and the two agree exactly on a complete grid. """ - if n_post_available is None: - divisor = n_periods - groups + 1.0 - else: - divisor = np.array([float(n_post_available[int(g)]) for g in groups]) - return (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor + divisor = np.array([float(n_post_available[int(g)]) for g in groups]) + return post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor def _simple_weight_vector( groups: np.ndarray, - times: np.ndarray, p_treated: Dict[int, float], + post_mask: np.ndarray, ) -> np.ndarray: - """ATT^simple weights: ``1[t >= g] * pbar_g``, normalized to sum to one.""" - raw = (times >= groups).astype(float) * np.array([p_treated[int(g)] for g in groups]) + """ATT^simple weights: ``1[t >= g - anticipation] * pbar_g``, normalized.""" + raw = post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) total = raw.sum() if total == 0: raise ValueError( @@ -384,7 +387,24 @@ def _unit_cohorts_from_frame( for col in (unit, time, first_treat): if col not in data.columns: raise ValueError(f"column {col!r} not found in data") - _validate_time_labels(data[time].to_numpy(), what=time) + key = _validate_time_labels(data[time].to_numpy(), what=time) + # Exactly one observation per unit-period, on a rectangular grid: the cohort + # shares and E_t[D] assume the same units in every period, exactly as the + # fitted path does. + cells = pd.MultiIndex.from_arrays([data[unit].to_numpy(), key]) + if cells.duplicated().any(): + raise ValueError( + "data has duplicate (unit, period) row(s); the frame path needs " + "exactly one observation per unit-period" + ) + n_periods = len(pd.unique(key)) + observed = pd.Series(key).groupby(data[unit].to_numpy(), sort=True).nunique() + if not (observed == n_periods).all(): + raise ValueError( + "data is not a balanced panel: some units are missing periods, so " + "the cohort shares are not comparable across periods. The frame path " + "needs exactly one observation per unit-period (a balanced panel)." + ) # dropna=False: a unit whose label is NaN in one period must fail the # invariance check, not slip through because nunique() skipped the NaN. per_unit = data.groupby(unit, sort=True)[first_treat].nunique(dropna=False) @@ -397,13 +417,13 @@ def _unit_cohorts_from_frame( firsts = data.groupby(unit, sort=True)[first_treat].first() cohorts = firsts.to_numpy() _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) - periods = np.asarray(sorted(data[time].unique())) + periods = np.asarray(sorted(pd.unique(key))) return cohorts, periods, None def _resolve_cs_inputs( results: "CallawaySantAnnaResults", -) -> Tuple[np.ndarray, Optional[np.ndarray]]: +) -> Tuple[np.ndarray, Optional[np.ndarray], int, bool]: """Read cohort labels (and survey weights) off a fitted CS result. The aggregation kit is package-internal, but it is the same channel @@ -428,7 +448,17 @@ def _resolve_cs_inputs( "panel explicitly via data=/unit=/time=/first_treat=" ) weights = bookkeeping.get("survey_weights") - return np.asarray(cohorts), (None if weights is None else np.asarray(weights, dtype=float)) + anticipation = int(getattr(kit, "anticipation", 0) or 0) + # Default True: DMLDiD builds its kit through the same builder but its + # precomputed mapping predates the balance flag, and a false rejection + # would be worse than the (already-CS-only) check. + is_balanced = bool(bookkeeping.get("is_balanced", True)) + return ( + np.asarray(cohorts), + (None if weights is None else np.asarray(weights, dtype=float)), + anticipation, + is_balanced, + ) def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> None: @@ -510,6 +540,7 @@ def attgt_weights( time: Optional[str] = None, first_treat: Optional[str] = None, weights: Optional[Union[str, np.ndarray]] = None, + anticipation: Optional[int] = None, ) -> ATTGTWeightsResult: """Weights an estimand places on each group-time effect ATT(g, t). @@ -543,6 +574,15 @@ def attgt_weights( or one value per unit. Rejected when the fit already carries survey weights, which take precedence. Must be finite and non-negative with positive treated mass (and positive never-treated mass for ``"twfe"``). + anticipation : int, optional + Anticipation window for the CS estimands, i.e. the number of periods + before ``g`` whose cells count as post-treatment (``t >= g - + anticipation``). Only meaningful on the DataFrame path, where a bare + frame carries no record of the source fit's setting; the fitted path + reads it off the fit and rejects an explicit ``anticipation=``. It does + NOT affect ``aggregation="twfe"``: the TWFE regression's own treatment + indicator is ``1[t >= g]`` regardless of how the CS estimands treat the + run-up, and R's ``twfe_weights`` has no anticipation argument either. Returns ------- @@ -553,7 +593,7 @@ def attgt_weights( ------ ValueError On an unknown ``aggregation``; on a design the formula does not - support (repeated cross-sections, unbalanced fallback, and - for + support (repeated cross-sections, an unbalanced panel, and - for ``aggregation="twfe"`` - a non-never-treated control group, a non-universal base period, or a covariate-adjusted fit); on NaN / ``-inf`` cohort labels, invalid weights, duplicated or non-finite @@ -571,15 +611,24 @@ def attgt_weights( the first observed period, which has no base period) is dropped from the table AND from the cohort masses with a warning - what ``did::pre_process_did`` does when it drops units already treated in the - first period. + first period. This drop is allowed ONLY when the cohort is treated in the + first observed period, or when every one of its missing post cells + carries ``skip_reason="zero_treated_control"``; a mid cohort blanked out + by some other mechanism raises rather than disappearing. * Under ``control_group="not_yet_treated"`` the last cohorts run out of comparison units, and CS marks those post cells ``zero_treated_control``. For ``"overall"`` / ``"simple"`` they are treated as structurally absent: ``"overall"`` divides each cohort by its number of AVAILABLE post periods and ``"simple"`` renormalizes over the available post cells - what - R ``aggte()`` computes on such a fit. A warning names the cells. - (``"twfe"`` requires a never-treated control group and never reaches - this branch.) + R ``aggte()`` computes on such a fit. A warning names the cells. The + carve-out keys on the ``skip_reason`` values themselves (that reason is + only ever emitted on a not-yet-treated fit), so the fitted and frame + paths behave identically. (``"twfe"`` requires a never-treated control + group and never reaches this branch.) + + Cohort shares assume a BALANCED panel - the same units observed in every + period - so an unbalanced fitted result is rejected (as + :func:`decompose_twfe_weights` already rejects an unbalanced panel). R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows with ``attgt = 0`` to mirror an internal vector layout; those rows are @@ -599,6 +648,11 @@ def attgt_weights( raise ValueError( f"aggregation must be one of {list(_AGGREGATIONS)!r}, got " f"{aggregation!r}" ) + if anticipation is not None: + if isinstance(anticipation, bool) or not isinstance(anticipation, (int, np.integer)): + raise ValueError(f"anticipation must be a non-negative integer, got {anticipation!r}") + if int(anticipation) < 0: + raise ValueError(f"anticipation must be non-negative, got {anticipation!r}") frame_path = isinstance(results, pd.DataFrame) frame = results if isinstance(results, pd.DataFrame) else None @@ -624,6 +678,7 @@ def attgt_weights( control_group = None base_period = None has_skip_reasons = "skip_reason" in frame.columns + window = 0 if anticipation is None else int(anticipation) else: if supplied: raise ValueError( @@ -632,9 +687,22 @@ def attgt_weights( "bookkeeping - drop them, or pass " "result.to_dataframe('group_time') as the first argument." ) + if anticipation is not None: + raise ValueError( + "anticipation= is only for the DataFrame fallback; a fitted " + "CallawaySantAnnaResults already carries its own anticipation. " + "Drop anticipation=, or pass result.to_dataframe('group_time') " + "as the first argument." + ) _guard_cs_design(results, aggregation) table, skipped = _attgt_from_cs(results) - cohorts, survey_weights = _resolve_cs_inputs(results) + cohorts, survey_weights, window, is_balanced = _resolve_cs_inputs(results) + if not is_balanced: + raise ValueError( + "attgt_weights requires a balanced panel: the cohort shares and " + "E_t[D] assume the same units are observed in every period. " + "Balance the panel (diff_diff.balance_panel) and refit." + ) if survey_weights is not None and weights is not None: raise ValueError( "this fit already carries survey weights; passing weights= as " @@ -668,31 +736,81 @@ def attgt_weights( "no ever-treated units found; cohort labels are all never-treated " "sentinels (0 or inf)" ) + + # A wrong-length weights= must fail HERE, before the excluded-cohort + # boolean slice below, which would otherwise raise a raw IndexError. + if unit_weights is not None and len(np.asarray(unit_weights)) != len(unit_g_pos): + raise ValueError( + f"weights has length {len(np.asarray(unit_weights))} but the panel " + f"has {len(unit_g_pos)} units" + ) + g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) - post_mask = t_pos >= g_pos + + # Post-treatment mask. The TWFE regression's own indicator is 1[t >= g] + # regardless of the CS anticipation window; the CS target estimands shift + # it to 1[t >= g - anticipation]. + if aggregation == "twfe": + post_mask = t_pos >= g_pos + else: + post_mask = t_pos >= (g_pos - window) + + def _post_start(g: int) -> int: + raw = g if aggregation == "twfe" else g - window + return max(1, raw) # --- whole-cohort exclusion (R did drops units treated in the first period) + present = set(zip(g_pos.tolist(), t_pos.tolist())) panel_cohorts = sorted({int(g) for g in unit_g_pos if g != 0}) cohorts_with_post = {int(g) for g in g_pos[post_mask]} excluded = [g for g in panel_cohorts if g not in cohorts_with_post] if excluded: - if not has_skip_reasons: - not_structural = [g for g in excluded if g != first_period_pos] - if not_structural: - labels = [_label_for(grid, g) for g in not_structural] + # A cohort may be dropped ONLY when the drop is structural: it is + # treated in the first observed period (R did's first-period drop), or + # every one of its missing post cells carries zero_treated_control + # (it ran out of comparison units). Any other blanked-out cohort must + # fail closed rather than disappear. + structural: List[int] = [] + for g in excluded: + if g == first_period_pos: + structural.append(g) + continue + if has_skip_reasons: + missing_post = [ + (_label_for(grid, g), _label_for(grid, t)) + for t in range(_post_start(g), n_periods + 1) + if (g, t) not in present + ] + if missing_post and all( + skipped.get(lab) == "zero_treated_control" for lab in missing_post + ): + structural.append(g) + not_structural = [g for g in excluded if g not in structural] + if not_structural: + labels = [_label_for(grid, g) for g in not_structural] + if not has_skip_reasons: raise ValueError( f"cohort(s) {labels!r} are present in data= but have no " "post-treatment cell in the ATT(g,t) frame. A bare frame " "cannot say why; pass result.to_dataframe('group_time') " "verbatim (it carries skip_reason) or the fitted result itself." ) + raise ValueError( + f"cohort(s) {labels!r} are present in data= but have no estimable " + "post-treatment cell, and their missing post cell(s) do not all " + "carry skip_reason 'zero_treated_control'. A cohort is only dropped " + "like R did's first-period cohort, or when it runs out of comparison " + "units; blanking a mid cohort's effects is not one of those, so it " + "fails closed instead of silently leaving the estimand." + ) n_units_excl = int(np.isin(unit_g_pos, excluded).sum()) warnings.warn( f"cohort(s) {[_label_for(grid, g) for g in excluded]!r} ({n_units_excl} " "unit(s)) have no estimable post-treatment cell and were dropped from " - "the weight table and the cohort shares, matching R did's drop of units " - "already treated in the first observed period", + "the weight table and the cohort shares: either treated in the first " + "observed period (R did's first-period drop) or run out of comparison " + "units under a not-yet-treated control group (R did's panel truncation)", UserWarning, stacklevel=2, ) @@ -716,11 +834,15 @@ def attgt_weights( if aggregation == "twfe": required = {(g, t) for g in surviving for t in range(1, n_periods + 1)} else: - required = {(g, t) for g in surviving for t in range(g, n_periods + 1)} + required = {(g, t) for g in surviving for t in range(_post_start(g), n_periods + 1)} missing_cells = sorted(required - present) structurally_absent: List[Tuple[Any, Any]] = [] if missing_cells: - carve_out_ok = aggregation != "twfe" and control_group == "not_yet_treated" + # The zero_treated_control carve-out is keyed on the skip_reason VALUE, + # not on control_group: that reason is only ever emitted on a + # not-yet-treated fit, and the frame path has no control_group to read, + # so keying on the reason is what makes the two paths agree. + carve_out_ok = aggregation != "twfe" hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] for g, t in missing_cells: label = (_label_for(grid, g), _label_for(grid, t)) @@ -752,11 +874,12 @@ def attgt_weights( # Non-estimable PRE cells of surviving cohorts are the only drops left; # the CS estimands ignore pre cells, so they change nothing. - dropped = sum( - 1 - for (g_lab, t_lab) in skipped - if _pos_of(grid, g_lab) in cohorts_with_post and _pos_of(grid, t_lab) < _pos_of(grid, g_lab) - ) + dropped = 0 + surviving_positional = {int(g) for g in cohorts_with_post} + for g_lab, t_lab in skipped: + gp, tp = _pos_of(grid, g_lab), _pos_of(grid, t_lab) + if gp in surviving_positional and tp < _post_start(gp): + dropped += 1 if dropped and aggregation != "twfe": warnings.warn( f"{dropped} pre-treatment group-time cell(s) had no estimable ATT(g,t) " @@ -769,12 +892,10 @@ def attgt_weights( if aggregation == "twfe": weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) elif aggregation == "overall": - n_post_available = None - if structurally_absent: - n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} - weight_vec = _overall_weight_vector(g_pos, t_pos, n_periods, p_treated, n_post_available) + n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} + weight_vec = _overall_weight_vector(g_pos, p_treated, post_mask, n_post_available) else: - weight_vec = _simple_weight_vector(g_pos, t_pos, p_treated) + weight_vec = _simple_weight_vector(g_pos, p_treated, post_mask) out = pd.DataFrame( { @@ -876,6 +997,24 @@ def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray return float(ew.sum() ** 2 / denom) +def _require_finite(block: np.ndarray, name: str, *, what: str) -> np.ndarray: + """Fail closed on NaN / inf in an estimation input block. + + A NaN outcome otherwise propagates silently: the demeaned residual becomes + NaN and every ``(g, t)`` ATT(g, t) is NaN, so the decomposition returns an + all-NaN result with no error. Complete-case handling is a policy choice we + do not make here, so the caller must clean the input. + """ + if not np.all(np.isfinite(block)): + n_bad = int((~np.isfinite(block)).sum()) + raise ValueError( + f"{what} {name!r} contains {n_bad} non-finite value(s) (NaN or inf); " + "decompose_twfe_weights does not drop incomplete cases, so clean the " + f"panel first (e.g. drop or impute rows with a missing {what})" + ) + return block + + class _Panel: """Balanced panel reshaped to ``(n_units, n_periods)`` with positional time. @@ -901,10 +1040,11 @@ def __init__( if weights is not None and weights not in data.columns: raise ValueError(f"weights column {weights!r} not found in data") - _validate_time_labels(data[time].to_numpy(), what=time) - frame = data.sort_values([unit, time]).reset_index(drop=True) + time_key = _validate_time_labels(data[time].to_numpy(), what=time) + frame = data.assign(_twfe_time_key=time_key) + frame = frame.sort_values([unit, "_twfe_time_key"]).reset_index(drop=True) units = frame[unit].to_numpy() - periods = frame[time].to_numpy() + periods = frame["_twfe_time_key"].to_numpy(dtype=float) self.unit_ids = np.asarray(sorted(pd.unique(units))) self.period_labels = np.asarray(sorted(pd.unique(periods))) n_units = len(self.unit_ids) @@ -916,7 +1056,7 @@ def __init__( "Balance it first, e.g. diff_diff.balance_panel(data, unit=..., " "time=...)." ) - counts = frame.groupby(unit, sort=True)[time].nunique().to_numpy() + counts = frame.groupby(unit, sort=True)["_twfe_time_key"].nunique().to_numpy() if not np.all(counts == n_periods): raise ValueError( "decompose_twfe_weights requires a balanced panel: some units " @@ -946,7 +1086,11 @@ def __init__( "comparison group; none were found (matching R's twfeweights, " "which supports only a never-treated comparison)" ) - self.outcome = frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods) + self.outcome = _require_finite( + frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods), + outcome, + what="outcome", + ) if weights is None: self.weights = np.ones((n_units, n_periods)) else: @@ -966,10 +1110,12 @@ def __init__( self.weights = block self.covariates = tuple(covariates) if covariates: - self.design = ( + self.design = _require_finite( frame[list(covariates)] .to_numpy(dtype=float) - .reshape(n_units, n_periods, len(covariates)) + .reshape(n_units, n_periods, len(covariates)), + ", ".join(covariates), + what="covariate", ) else: self.design = np.zeros((n_units, n_periods, 0)) @@ -987,7 +1133,7 @@ def __init__( # the demeaned covariate blocks are kept: the annihilation filter in # _fwl_residuals compares one against the other. demean_frame = pd.DataFrame( - {"_unit": frame[unit].to_numpy(), "_time": frame[time].to_numpy()} + {"_unit": frame[unit].to_numpy(), "_time": frame["_twfe_time_key"].to_numpy()} ) demean_frame["_treated"] = self.treated.reshape(-1) for j, name in enumerate(self.covariates): @@ -1025,23 +1171,28 @@ def covariate_block( within a unit before comparing groups, so a time-varying covariate is summarized by its unit mean. """ - frame = data.sort_values([unit, time]).reset_index(drop=True) + frame = data.assign(_twfe_time_key=_validate_time_labels(data[time].to_numpy(), what=time)) + frame = frame.sort_values([unit, "_twfe_time_key"]).reset_index(drop=True) block = ( frame[list(names)] .to_numpy(dtype=float) .reshape(self.n_units, self.n_periods, len(names)) ) + for j, name in enumerate(names): + _require_finite(block[:, :, j], name, what="balance covariate") return block.mean(axis=1) -def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: +def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float, List[str]]: """Frisch-Waugh-Lovell residual of treatment on covariates, plus its scale. Double-demeans ``D`` and ``X``, projects the demeaned treatment on the demeaned covariates, and returns the residual. That residual IS the implicit weight the regression applies to each observation; ``alpha_den`` is the normalization ``E[resid * Ddot]`` from R's - ``combine_twfe_weights_gt``. + ``combine_twfe_weights_gt``. The third element is the names of the + covariates that SURVIVED the annihilation filter, so the result reports the + columns the regression actually used rather than the user's input list. With no covariates the projection is empty and the residual is just the double-demeaned treatment - which is exactly the branch R cannot run, @@ -1057,7 +1208,8 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: # Explicit row count: with zero covariates the trailing axis is 0 and # numpy cannot infer a -1 against it. This is the same no-covariate branch # on which fixest::demean segfaults; here it simply has to be spelled out. - flat_x = x_dot.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]) + n_obs = panel.n_units * panel.n_periods + flat_x = x_dot.reshape(n_obs, x_dot.shape[2]) # Numerical hygiene: drop covariates that double-demeaning ANNIHILATED # before anything is projected on them. A time-invariant regressor leaves a @@ -1069,24 +1221,31 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: # agree exactly. The test is scale-relative: a column counts as having no # within-variation when its demeaned norm is negligible NEXT TO ITS OWN raw # norm, which a rank test on the demeaned matrix alone cannot see (there, - # 1e-16 is simply the largest pivot). The 1e-10 relative threshold is a - # blunt instrument: a covariate with a large level and genuinely small - # within-variation can trip it, which is why the warning says so. + # 1e-16 is simply the largest pivot). + # + # The threshold is the accumulated ROUNDING-NOISE scale, not a fixed + # relative constant: demeaning an ``n_obs``-row column accumulates + # ``O(sqrt(n) * eps)`` of relative error, times a safety factor of 64. + # A fixed 1e-10 was five orders too loose - it discarded a covariate with + # level 1e6 and genuine within-sd 1e-4 (ratio 1e-10) - while still + # annihilating mpdta's `lpop`, whose true within-variation is many orders + # above the noise floor. raw_scale = np.linalg.norm( - panel.design.reshape(panel.n_units * panel.n_periods, x_dot.shape[2]), + panel.design.reshape(n_obs, x_dot.shape[2]), axis=0, ) demeaned_scale = np.linalg.norm(flat_x, axis=0) - annihilated = demeaned_scale <= 1e-10 * np.maximum(raw_scale, 1.0) + noise_floor = np.sqrt(n_obs) * 64.0 * np.finfo(float).eps + annihilated = demeaned_scale <= noise_floor * np.maximum(raw_scale, 1.0) if annihilated.any(): names = [panel.covariates[j] for j in np.flatnonzero(annihilated)] warnings.warn( f"covariate(s) {names!r} have no within-unit-and-period variation " - "(or within-variation below 1e-10 of their own level) and were " - "dropped: two-way demeaning annihilates them, so they cannot affect " - "a two-way fixed effects regression. If that is not intended, " - "centre or rescale the covariate so its within-variation is not " - "negligible next to its level", + "(or within-variation at the floating-point noise floor of their own " + "level) and were dropped: two-way demeaning annihilates them, so they " + "cannot affect a two-way fixed effects regression. If that is not " + "intended, centre or rescale the covariate so its within-variation is " + "not negligible next to its level", UserWarning, stacklevel=3, ) @@ -1127,7 +1286,7 @@ def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float]: "double-demeaning and covariate adjustment, so the TWFE " "coefficient is not identified" ) - return resid.reshape(panel.n_units, panel.n_periods), alpha_den + return resid.reshape(panel.n_units, panel.n_periods), alpha_den, surviving def _normalize_cell_weights( @@ -1173,7 +1332,7 @@ def _decompose_fwl( balance_block: Optional[np.ndarray], ) -> Dict[str, Any]: """R ``implicit_twfe_weights``: TWFE as weighted ATT(g, t) + a remainder.""" - resid, alpha_den = _fwl_residuals(panel) + resid, alpha_den, surviving_covariates = _fwl_residuals(panel) weights = panel.weights flat_w = weights.reshape(-1) cohorts = panel.cohorts @@ -1287,13 +1446,14 @@ def _decompose_fwl( "estimate": decomposition + remainder_total, "decomposition": decomposition, "remainder": remainder_total, - "pretrend_bias": float((weight_vec[~post_col] * att_col[~post_col]).sum()), + "pre_period_contribution": float((weight_vec[~post_col] * att_col[~post_col]).sum()), "post_only": float((weight_vec[post_col] * att_col[post_col]).sum()), # summary.decomposed_twfe: post cells only, on both factors "effective_sample_size": float( post_col.sum() * (weight_vec[post_col] * ess_col[post_col]).sum() ), "balance": pd.DataFrame(balance_rows) if balance_block is not None else None, + "covariates": tuple(surviving_covariates), } @@ -1526,9 +1686,12 @@ def decompose_twfe_weights( """Decompose a TWFE estimate into weighted group-time effects. Runs the regression, recovers the implicit weight it places on each - ATT(g, t), and separates the part of the estimate that comes from - PRE-treatment cells - i.e. from parallel-trends violations rather than - from treatment. + ATT(g, t), and separates out the part of the estimate that comes from + PRE-treatment cells (``pre_period_contribution``). That component can + reflect differential pre-trends OR plain sampling variation - the + diagnostic carries no inference - so it is diagnostic evidence about the + earlier-period restrictions, not proof that the identifying assumption + fails in the post-treatment counterfactual. Takes the raw panel rather than a fitted result, because it re-estimates: it double-demeans treatment and covariates and forms its own group-time @@ -1582,7 +1745,7 @@ def decompose_twfe_weights( ... panel, outcome="y", unit="id", time="t", first_treat="g", ... covariates=["x"], balance_covariates=["x"], ... ) - >>> dec.pretrend_bias # doctest: +SKIP + >>> dec.pre_period_contribution # doctest: +SKIP >>> dec.covariate_balance() # doctest: +SKIP """ if method not in _METHODS: @@ -1614,10 +1777,10 @@ def decompose_twfe_weights( estimate=payload["estimate"], decomposition=payload["decomposition"], remainder=payload["remainder"], - pretrend_bias=payload["pretrend_bias"], + pre_period_contribution=payload["pre_period_contribution"], post_only=payload["post_only"], base_period=base_period, - covariates=covariate_names, + covariates=payload["covariates"], effective_sample_size=payload["effective_sample_size"], n_units=panel.n_units, n_periods=panel.n_periods, diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py index 3f7299873..58a086a65 100644 --- a/diff_diff/twfe_weights_results.py +++ b/diff_diff/twfe_weights_results.py @@ -231,11 +231,13 @@ class TWFEDecompositionResult(Diagnostic): remainder : float Part of ``estimate`` not attributable to any ATT(g, t) cell. Identically ``0.0`` except under ``base_period="gmin1"``. - pretrend_bias : float - ``sum(weight * att)`` over PRE-treatment cells only. Under parallel - trends every pre-treatment ATT(g, t) is zero and this vanishes; a - non-zero value is the contribution of parallel-trends violations to - ``estimate``. + pre_period_contribution : float + ``sum(weight * att)`` over PRE-treatment cells only. This is the sample + contribution of the pre-treatment cells to ``estimate``. It can reflect + differential pre-trends, but it can also be sampling variation, since + the diagnostic carries no inference; read it as diagnostic evidence, + not as proof that the identifying assumption fails in the + post-treatment counterfactual. post_only : float ``sum(weight * att)`` over post-treatment cells only. base_period : str or None @@ -258,7 +260,7 @@ class TWFEDecompositionResult(Diagnostic): estimate: float decomposition: float remainder: float - pretrend_bias: float + pre_period_contribution: float post_only: float base_period: Optional[str] covariates: Tuple[str, ...] @@ -271,7 +273,7 @@ def __repr__(self) -> str: return ( f"TWFEDecompositionResult(method={self.method!r}, " f"estimate={self.estimate:.4f}, " - f"pretrend_bias={self.pretrend_bias:.4f}, " + f"pre_period_contribution={self.pre_period_contribution:.4f}, " f"n_cells={len(self.cells)})" ) @@ -313,17 +315,20 @@ def summary(self) -> str: f"{'Estimate:':<30} {_fmt(self.estimate)}", f"{' from ATT(g,t) cells:':<30} {_fmt(self.decomposition)}", f"{' post-treatment only:':<30} {_fmt(self.post_only)}", - f"{' pre-trend violations:':<30} {_fmt(self.pretrend_bias)}", + f"{' from pre-period cells:':<30} {_fmt(self.pre_period_contribution)}", f"{' remainder:':<30} {_fmt(self.remainder)}", "", f"{'Effective sample size:':<30} {_fmt(self.effective_sample_size)}", "", ] - if abs(self.pretrend_bias) > 1e-10: + if abs(self.pre_period_contribution) > 1e-10: lines += [ - "Note: a non-zero pre-trend contribution means pre-treatment", - " ATT(g, t) are not zero, so part of the estimate reflects", - " parallel-trends violations rather than treatment effects.", + "Note: the pre-period contribution is the sample contribution of", + " the pre-treatment cells. It can reflect differential", + " pre-trends, but it can also be sampling variation - this", + " diagnostic carries no inference - so treat it as evidence", + " about the earlier-period restrictions, not as proof that the", + " identifying assumption fails post-treatment.", "", ] if self.balance is not None: @@ -349,7 +354,7 @@ def to_dict(self) -> Dict[str, Any]: "estimate": self.estimate, "decomposition": self.decomposition, "remainder": self.remainder, - "pretrend_bias": self.pretrend_bias, + "pre_period_contribution": self.pre_period_contribution, "post_only": self.post_only, "base_period": self.base_period, "covariates": list(self.covariates), diff --git a/diff_diff/visualization/_diagnostic.py b/diff_diff/visualization/_diagnostic.py index ddb99e224..da496f518 100644 --- a/diff_diff/visualization/_diagnostic.py +++ b/diff_diff/visualization/_diagnostic.py @@ -951,12 +951,16 @@ def _twfe_weights_payload( if table is None: table = results.cells weight = table["weight"].to_numpy(dtype=float) + post = table["post"].to_numpy().astype(bool) return { - "post": table["post"].to_numpy().astype(bool), + "post": post, "weight": weight, "att": table["att"].to_numpy(dtype=float), "labels": [f"({g}, {t})" for g, t in zip(table["group"], table["time"])], - "n_negative": int((weight < 0).sum()), + # POST-only: over the full grid the TWFE weights sum to zero, so a + # healthy staggered panel would still count ~half its cells as + # negative. The pathology is negative weight on post cells. + "n_negative_post": int(((weight < 0) & post).sum()), } balance = results.covariate_balance(level="summary", standardize=standardize) suffix = "_std_diff" if standardize else "_diff" @@ -978,6 +982,7 @@ def _twfe_weights_payload( "weighted": weighted, "labels": [str(c) for c in balance["covariate"]], "limit": limit, + "absolute_value": absolute_value, } @@ -1028,17 +1033,20 @@ def _render_twfe_weights_mpl( ax.set_xlabel(xlabel or "Implicit weight") ax.set_ylabel(ylabel or "ATT(g, t)") default_title = "Implicit weights on group-time effects" - if payload["n_negative"]: - default_title += f" ({payload['n_negative']} negative)" + if payload["n_negative_post"]: + default_title += f" ({payload['n_negative_post']} negative post)" ax.set_title(title or default_title) ax.legend(frameon=False) else: unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) ax.scatter(unweighted, weighted, s=markersize, alpha=alpha, color=post_color, zorder=3) + # The "no improvement" diagonal: on a SIGNED plot differences can be + # negative, so the line must span [-limit, limit], not [0, limit]. + diagonal_lo = 0.0 if payload["absolute_value"] else -limit ax.plot( - [0, limit], - [0, limit], + [diagonal_lo, limit], + [diagonal_lo, limit], color="0.6", linestyle="--", linewidth=1.0, @@ -1097,19 +1105,20 @@ def _render_twfe_weights_plotly( y=att[mask], mode=mode, name=name, - text=labels[mask] if annotate else None, + # `text` always carries the labels: with mode="markers" + # they feed the hover template; with "markers+text" + # (annotate=True) they also render. + text=labels[mask], textposition="top right", marker={"size": marker_px, "color": color, "opacity": alpha}, hovertemplate="%{text}
weight=%{x:.4f}
ATT(g,t)=%{y:.4f}", - customdata=None, ) ) - fig.data[-1].text = labels[mask] fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) fig.add_vline(x=0, line={"color": "gray", "width": 1.2}) default_title = "Implicit weights on group-time effects" - if payload["n_negative"]: - default_title += f" ({payload['n_negative']} negative)" + if payload["n_negative_post"]: + default_title += f" ({payload['n_negative_post']} negative post)" fig.update_layout( title=title or default_title, xaxis_title=xlabel or "Implicit weight", @@ -1129,10 +1138,11 @@ def _render_twfe_weights_plotly( hovertemplate="%{text}
unweighted=%{x:.4f}
weighted=%{y:.4f}", ) ) + diagonal_lo = 0.0 if payload["absolute_value"] else -limit fig.add_trace( go.Scatter( - x=[0, limit], - y=[0, limit], + x=[diagonal_lo, limit], + y=[diagonal_lo, limit], mode="lines", name="no improvement", line={"color": "gray", "dash": "dash", "width": 1.0}, diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst index ebf809ef2..9b39b31f0 100644 --- a/docs/api/twfe_weights.rst +++ b/docs/api/twfe_weights.rst @@ -15,8 +15,10 @@ back into its building blocks. - You have a staggered design and want to see, cell by cell, what your TWFE specification is actually averaging -- You want to quantify how much of a TWFE estimate comes from *pre-treatment* - cells -- i.e. from parallel-trends violations rather than from treatment +- You want to quantify what the *pre-treatment* cells contribute to a TWFE + estimate -- a sample contribution that can reflect parallel-trends + violations, but can also be sampling variation, so read it as diagnostic + evidence rather than proof - Your TWFE and :class:`~diff_diff.CallawaySantAnna` estimates disagree and you want to see which cells drive the gap - You adjusted for covariates and want to check whether the regression's @@ -123,8 +125,8 @@ between ``implied_att`` values is the cost of the TWFE specification: print(f"{aggregation:8s} {w.implied_att: .4f} " f"({w.n_negative} negative weights)") -Separating treatment effects from pre-trend violations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Separating treatment effects from the pre-period contribution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: python @@ -141,7 +143,7 @@ Separating treatment effects from pre-trend violations ) print(decomposition.summary()) - print("from pre-treatment cells:", decomposition.pretrend_bias) + print("from pre-treatment cells:", decomposition.pre_period_contribution) # Do the implicit weights balance the covariates? print(decomposition.covariate_balance()) diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 18109b6b8..60478b08f 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6276,10 +6276,10 @@ where `Ytilde` is the outcome measured against the base period (`Y_t - Y_1` unde decomposition = sum over all cells of alpha_weight * ATT remainder = sum of alpha_weight * cell remainder (0 unless base_period="gmin1") estimate = decomposition + remainder -pretrend_bias = sum over PRE cells (t < g) of alpha_weight * ATT +pre_period_contribution = sum over PRE cells (t < g) of alpha_weight * ATT ``` -Under parallel trends every pre-treatment ATT(g,t) is zero and `pretrend_bias` vanishes; a non-zero value is the contribution of parallel-trends violations to the TWFE coefficient. +`pre_period_contribution` is the sample contribution of the pre-treatment cells to `estimate`. It can reflect differential pre-trends, but it can also be sampling variation - the diagnostic carries no inference - so it is evidence about the earlier-period restrictions, not proof that the identifying assumption fails in the post-treatment counterfactual. *Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, @@ -6292,20 +6292,24 @@ Verified on `mpdta` at `-0.03654894` from both directions. ### Edge cases - **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `aggregation="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. -- **Note (matches R `did`'s first-period drop):** a cohort with NO estimable post-treatment cell, canonically one treated in the first observed period (no base period), is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. -- **Note (matches R `aggte`'s available-period averaging under not-yet-treated controls):** on a `control_group="not_yet_treated"` fit the last cohorts run out of comparison units and CS marks those post cells `skip_reason="zero_treated_control"`. For `"overall"` / `"simple"` they are treated as structurally absent: `"overall"` divides each cohort by its number of AVAILABLE post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells, which is what R `aggte(type="group")` / `aggte(type="simple")` compute on such a fit. A `UserWarning` names the cells. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. +- **Note (matches R `did`'s first-period drop, but only when the drop is structural):** a cohort with NO estimable post-treatment cell is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. The drop is allowed ONLY when the cohort is treated in the first observed period, OR when every one of its missing post cells carries `skip_reason="zero_treated_control"` (it ran out of comparison units). A mid cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. +- **Note (matches R `did`'s mechanism behind the second structural gap):** with no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g)` and recodes the last-treated cohort to never-treated. That truncation - not `aggte` averaging over available periods - is what produces R's number. diff-diff reaches the same numerics by treating the cells CS marks `skip_reason="zero_treated_control"` as structurally ABSENT (they name exactly the cohort x period cells that truncation removes): `"overall"` divides each cohort by its number of available post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group`, because that reason is only ever emitted on a not-yet-treated fit and the frame path carries no `control_group`; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. - **Note (`aggregation="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. - **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `aggregation="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. -- `attgt_weights` rejects repeated-cross-section fits and unbalanced-panel fallbacks: `E_t[D]` and the cohort shares average over a fixed unit set. +- `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The frame path requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). +- **Note (the anticipation window is honoured on the CS estimands):** the fitted path reads `anticipation` off the aggregation kit; the frame path takes it as an explicit `anticipation=` argument (a bare frame carries no record of the source fit's setting, and the fitted path rejects the kwarg rather than silently ignoring it). Post cells become `t >= g - anticipation` for `"overall"` / `"simple"`, and the same window enters the required-cell grid, the available-period divisors, and the `post` column. `"twfe"` deliberately keeps `t >= g`: the regression's own treatment indicator does not anticipate, and R's `twfe_weights` has no anticipation argument. Each CS estimand is pinned against `results.aggregate(...)` with positive anticipation under both control groups (`"simple"` against `aggregate("simple")`, `"overall"` against the cohort-mass-weighted combination of `aggregate("group")`). +- **Note (one canonical numeric time key):** period labels are coerced to a single numeric key at validation, and that key is used for sorting, reshaping, cohort mapping, and the positional grid. Using the raw column instead lets a numeric-string label (`"10"` sorts before `"2"` lexicographically) desynchronize the sort order from the grid and silently rebuild a different panel; numeric and numeric-string labels now decompose identically. +- **Note (inputs are validated finite):** `decompose_twfe_weights` fails closed on non-finite outcomes, regression covariates, or balance covariates. A single NaN outcome would otherwise propagate to every cell and return an all-NaN decomposition with no error; complete-case handling is a policy choice the caller owns. +- **Note (a wrong-length `weights=` fails closed):** the length check runs before the excluded-cohort boolean slice, so a wrong-length vector is a clear `ValueError` naming both lengths rather than a raw `IndexError`. ### Notes and deviations - **Note (upstream `fixest::demean` segfault on the no-covariate branch):** `twfeweights::implicit_twfe_weights(xformula = ~1)` builds `model.matrix(~-1, data)`, an `nT x 0` matrix, and `fixest::demean()` SEGFAULTS on a zero-column matrix (reproduced in isolation on R 4.6.1 / fixest 0.14.2: `fixest::demean(matrix(numeric(0), 10, 0), ids)` → `*** caught segfault *** memory not mapped`). This is a zero-column bug, not a property of any fixture. The no-covariate golden is therefore generated with a TIME-INVARIANT covariate, which double-demeaning annihilates exactly, making the call numerically the `~1` branch; the parity test asserts BOTH `covariates=None` and `covariates=[]` against that single golden, so the equivalence is proven rather than assumed. Verified on `mpdta`: `twfe_weights(att_gt(...))` aggregate and `implicit_twfe_weights(xformula = ~lpop)$est` both equal `-0.03654894`. -- **Note (annihilated covariates are dropped before the projection, as numerical hygiene):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Keeping it is not catastrophic: the column lies in the fixed-effect span and is orthogonal to the treatment residual, so on mpdta's `lpop` it moves the FWL residual by ~2e-18. But regressing on an exactly-zero column is meaningless, and dropping it is what makes `covariates=None` and `covariates=[]` agree exactly rather than approximately. diff-diff judges each column against its own PRE-demeaning norm; a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. **Limitation:** the 1e-10 relative threshold is blunt, so a covariate with a large level and genuinely small within-variation can trip it. The `UserWarning` says so and suggests centring or rescaling. -- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pretrend_bias` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. +- **Note (annihilated covariates are dropped before the projection, as numerical hygiene):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Keeping it is not catastrophic: the column lies in the fixed-effect span and is orthogonal to the treatment residual, so on mpdta's `lpop` it moves the FWL residual by ~2e-18. But regressing on an exactly-zero column is meaningless, and dropping it is what makes `covariates=None` and `covariates=[]` agree exactly rather than approximately. diff-diff judges each column against its own PRE-demeaning norm; a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. The threshold is the accumulated ROUNDING-NOISE scale - `sqrt(n_obs) * 64 * eps * max(raw_scale, 1)` - not a fixed relative constant: a fixed `1e-10` is five orders too loose (a covariate with level `1e6` and genuine within-sd `1e-4` has ratio `1e-10` and would be wrongly discarded) while still annihilating mpdta's `lpop`, whose true within-variation is far above the noise floor. When a column IS dropped the result reports the SURVIVING covariate names, not the user's input list, so `result.covariates` never claims a column that was annihilated. The `UserWarning` names the dropped columns. +- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pre_period_contribution` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. - **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. - **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. - **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. @@ -6323,7 +6327,7 @@ Verified on `mpdta` at `-0.03654894` from both directions. Goldens: `benchmarks/data/twfeweights_golden.json`, plus two simulated sibling panel CSVs. The `mpdta` fixture reads the shared `benchmarks/data/mpdta_stata_panel.csv` and derives `lpop_t` from a `derived_columns` expression in the golden, rather than committing a renamed copy; the generator asserts the two sources agree to CSV round-trip precision. Every cells block carries ORIGINAL period labels: `implicit_*` run in positional time internally, and the generator maps them back so one convention holds throughout and the tests assert labels rather than array position. Regenerated by `benchmarks/R/generate_twfeweights_golden.R`; R is needed only to regenerate, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. -Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, package="did")`), `sim_staggered` (three equal cohorts of 100, which is exactly what makes the comparison-group normalizer vanish at `t=3`, so this fixture deliberately exercises the degenerate cells above; its `pretrend_bias` is non-zero but the ~0.093 is sampling noise, since `x1` is iid and cohorts are assigned by unit index, so no differential pre-trend is designed in), and `unbalanced_cohorts` (120/70/60, which breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). +Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, package="did")`), `sim_staggered` (three equal cohorts of 100, which is exactly what makes the comparison-group normalizer vanish at `t=3`, so this fixture deliberately exercises the degenerate cells above; its `pre_period_contribution` is non-zero but the ~0.093 is sampling noise, since `x1` is iid and cohorts are assigned by unit index, so no differential pre-trend is designed in), and `unbalanced_cohorts` (120/70/60, which breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). | Surface | Gate | Rationale | |---------|------|-----------| @@ -6333,7 +6337,7 @@ Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, | FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | | FWL with covariates | `atol=1e-8` | The demeaning slack propagates through the OLS projection of `Ddot` on `Xdot`. | | Covariate balance (11 statistics) | `atol=1e-9` | Smooth functions of the weights above. Observed max deviation 7.3e-11. | -| Per-cell ATT, `pretrend_bias` / `post_only`, and (under `gmin1`) the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. Under `base_period="first_period"` the remainder is identically zero, so the split is gated tight even where the mask fires. | +| Per-cell ATT, `pre_period_contribution` / `post_only`, and (under `gmin1`) the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. Under `base_period="first_period"` the remainder is identically zero, so the split is gated tight even where the mask fires. | | `effective_sample_size` and per-cell `ess` at DEGENERATE cells | expected value rebuilt from R's own cells | R's `ess` at a 0/0 cell is a ratio of rounding errors (scalar gap ~0.99, per-cell up to ~0.53). The expectation uses R's weights and R's `ess` wherever the degeneracy is not detected, substituting our limit value only at the detected cells, so the assertion is anchored to R rather than to our own implementation. | --- diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index 326df5098..3b7251ff3 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -364,7 +364,7 @@ def make_constructed_diagnostics() -> Dict[str, Any]: estimate=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), decomposition=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), remainder=0.0, - pretrend_bias=float( + pre_period_contribution=float( (decomposition_cells["weight"] * decomposition_cells["att"])[ decomposition_cells["post"] == 0 ].sum() diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index 3c93df678..da87fd4b8 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -61,8 +61,15 @@ def fitted(panel): class TestPublicSurface: def test_exported_from_package_root(self): assert diff_diff.attgt_weights is attgt_weights - for name in ("attgt_weights", "ATTGTWeightsResult", "TWFEDecompositionResult"): - assert name in diff_diff.__all__ + for name in ( + "attgt_weights", + "decompose_twfe_weights", + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "plot_twfe_weights", + ): + assert name in diff_diff.__all__, name + assert hasattr(diff_diff, name), name def test_name_is_distinct_from_the_dcdh_surface(self): """The two weight surfaces must stay separately addressable.""" @@ -90,12 +97,6 @@ def test_result_renders(self, fitted): class TestAggregationBehaviour: - @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) - def test_implied_att_is_the_weighted_sum(self, fitted, aggregation): - result = attgt_weights(fitted, aggregation=aggregation) - expected = (result.weights["weight"] * result.weights["att"]).sum() - assert result.implied_att == pytest.approx(expected, abs=1e-14) - @pytest.mark.parametrize("aggregation", ["overall", "simple"]) def test_target_estimands_are_convex(self, fitted, aggregation): """ATT^O / ATT^simple weights are non-negative and sum to one.""" @@ -391,7 +392,7 @@ def _weighted_panel(): "nocov": dict( kwargs={}, estimate=1.4221735897240102, - pretrend_bias=0.5127724996498023, + pre_period_contribution=0.5127724996498023, post_only=0.9094010900742079, ess=59.999999999999986, weight=[ @@ -422,7 +423,7 @@ def _weighted_panel(): "cov": dict( kwargs={"covariates": ["x"]}, estimate=1.3930847792561663, - pretrend_bias=0.5397671816632773, + pre_period_contribution=0.5397671816632773, post_only=0.8533175975928889, ess=58.816020983744, weight=[ @@ -453,7 +454,7 @@ def _weighted_panel(): "gmin1": dict( kwargs={"base_period": "gmin1"}, estimate=1.4221735897240106, - pretrend_bias=-0.3554385674608017, + pre_period_contribution=-0.3554385674608017, post_only=1.777612157184812, ess=59.999999999999986, weight=None, # identical to nocov (weights do not depend on the base period) @@ -480,7 +481,9 @@ def test_decomposition_weighted_branches(self, key): df, outcome="y", unit="id", time="t", first_treat="g", weights="w", **spec["kwargs"] ) assert result.estimate == pytest.approx(spec["estimate"], abs=1e-12) - assert result.pretrend_bias == pytest.approx(spec["pretrend_bias"], abs=1e-12) + assert result.pre_period_contribution == pytest.approx( + spec["pre_period_contribution"], abs=1e-12 + ) assert result.post_only == pytest.approx(spec["post_only"], abs=1e-12) assert result.effective_sample_size == pytest.approx(spec["ess"], abs=1e-9) expected_w = spec["weight"] if spec["weight"] is not None else self._DEC["nocov"]["weight"] @@ -997,7 +1000,7 @@ def test_bad_method_and_base_period(self, panel): def test_identities_hold(self, panel, fitted): result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) - assert result.pretrend_bias + result.post_only == pytest.approx( + assert result.pre_period_contribution + result.post_only == pytest.approx( result.decomposition, abs=1e-12 ) assert result.remainder == 0.0 @@ -1025,9 +1028,22 @@ def test_weights_view(self, fitted): result = attgt_weights(fitted, aggregation="twfe") ax = diff_diff.plot_twfe_weights(result, show=False) assert ax.get_xlabel() == "Implicit weight" - assert "negative" in ax.get_title() + # Item 9: the title counts POST-only negatives. This panel has negative + # weights, but all of them are on pre cells, so the title must not claim + # any negative treatment-period weight. + assert result.n_negative > 0 + assert result.n_negative_post == 0 + assert "negative" not in ax.get_title() assert len(ax.collections) == 2 # post + pre scatters + def test_weights_title_reports_post_only_count(self): + """A panel WITH negative post weight labels the post-only count.""" + fit = _fit(_panel(cohorts=(0, 2, 4))) + result = attgt_weights(fit, aggregation="twfe") + assert result.n_negative_post > 0 + ax = diff_diff.plot_twfe_weights(result, show=False) + assert f"({result.n_negative_post} negative post)" in ax.get_title() + def test_ax_reuse_and_annotate(self, fitted): import matplotlib.pyplot as plt @@ -1082,3 +1098,304 @@ def test_bad_kind_and_backend(self, fitted): diff_diff.plot_twfe_weights(result, kind="heat", show=False) with pytest.raises(ValueError, match="backend must be"): diff_diff.plot_twfe_weights(result, backend="bokeh", show=False) + + +# --------------------------------------------------------------------------- +# Second review round (PR #812): canonical time keys, anticipation, balance/ +# provenance, and an independent multi-covariate pin. +# --------------------------------------------------------------------------- + +_DECO = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + + +class TestCanonicalTimeKey: + """Item 1: numeric and numeric-string period labels must agree. + + ``"10" < "2"`` lexicographically, so a raw-label sort desynchronizes the + reshape from the positional grid and silently rebuilds a different panel. + """ + + @staticmethod + def _gapped(panel, labels): + df = panel.copy() + remap = {1: labels[0], 2: labels[1], 3: labels[2], 4: labels[3], 5: labels[4]} + df["period"] = df["period"].map(remap) + df["first_treat"] = df["first_treat"].map(lambda g: remap.get(g, 0)) + return df + + def test_numeric_string_labels_agree(self, panel): + as_int = diff_diff.decompose_twfe_weights(self._gapped(panel, [1, 2, 10, 11, 12]), **_DECO) + as_str = diff_diff.decompose_twfe_weights( + self._gapped(panel, ["1", "2", "10", "11", "12"]), **_DECO + ) + assert as_str.estimate == pytest.approx(as_int.estimate, abs=1e-12) + np.testing.assert_allclose( + as_str.cells["weight"].to_numpy(), as_int.cells["weight"].to_numpy(), atol=1e-12 + ) + np.testing.assert_allclose( + as_str.cells["att"].to_numpy(), as_int.cells["att"].to_numpy(), atol=1e-12 + ) + + +class TestAnticipation: + """Item 2: the anticipation window shifts the CS estimands, never TWFE.""" + + @staticmethod + def _fit(df, anticipation, control_group): + return diff_diff.CallawaySantAnna( + control_group=control_group, base_period="universal", anticipation=anticipation + ).fit(df, **_DECO) + + @staticmethod + def _group_overall(fit, panel): + group = fit.aggregate("group") + ever = panel.drop_duplicates("unit")["first_treat"].to_numpy() + ever = ever[ever != 0] + vals, counts = np.unique(ever, return_counts=True) + pg = dict(zip(vals.tolist(), (counts / counts.sum()).tolist())) + return sum(pg[lab] * att for lab, att in zip(group.label, group.att)) + + @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) + def test_simple_matches_aggregate(self, panel, control_group): + fit = self._fit(panel, 1, control_group) + got = attgt_weights(fit, aggregation="simple").implied_att + assert got == pytest.approx(fit.aggregate("simple").att[0], abs=1e-12) + + @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) + def test_overall_matches_group_combination(self, panel, control_group): + fit = self._fit(panel, 1, control_group) + got = attgt_weights(fit, aggregation="overall").implied_att + assert got == pytest.approx(self._group_overall(fit, panel), abs=1e-12) + + def test_twfe_ignores_anticipation(self, panel): + ant0 = attgt_weights(self._fit(panel, 0, "never_treated"), aggregation="twfe") + ant1 = attgt_weights(self._fit(panel, 1, "never_treated"), aggregation="twfe") + np.testing.assert_allclose( + ant1.weights["weight"].to_numpy(), ant0.weights["weight"].to_numpy(), atol=1e-12 + ) + + def test_frame_path_takes_explicit_anticipation(self, panel): + fit = self._fit(panel, 1, "never_treated") + from_fit = attgt_weights(fit, aggregation="simple") + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation="simple", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + anticipation=1, + ) + assert from_frame.implied_att == pytest.approx(from_fit.implied_att, abs=1e-15) + + def test_rejects_negative_anticipation(self, fitted): + with pytest.raises(ValueError, match="non-negative"): + attgt_weights(fitted, anticipation=-1) + + def test_fitted_path_rejects_the_kwarg(self, panel): + fit = self._fit(panel, 1, "never_treated") + with pytest.raises(ValueError, match="only for the DataFrame fallback"): + attgt_weights(fit, anticipation=1) + + +class TestUnbalancedFittedResult: + """Item 3: cohort shares assume a balanced panel; reject otherwise.""" + + def test_unbalanced_fitted_result_is_rejected(self): + df = _panel() + broken = df.copy() + broken.loc[broken.index[0], "outcome"] = np.nan # row present, value missing + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(broken, **_DECO) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="balanced panel"): + attgt_weights(fit, aggregation="twfe") + + def test_balanced_fit_records_the_flag(self, fitted): + assert fitted._aggregation_kit.bookkeeping["is_balanced"] is True + + def test_frame_path_requires_one_obs_per_unit_period(self, fitted, panel): + broken = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) + with pytest.raises(ValueError, match="exactly one observation per unit-period"): + _frame_call(_gt_frame(fitted), broken) + + def test_frame_path_rejects_duplicate_cells(self, fitted, panel): + dup = pd.concat([panel, panel.iloc[[0]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicate \\(unit, period\\)"): + _frame_call(_gt_frame(fitted), dup) + + +class TestFiniteInputs: + """Item 4: NaN inputs fail closed instead of returning an all-NaN result.""" + + def test_nan_outcome_is_rejected(self, panel): + df = panel.copy() + df.loc[df.index[0], "outcome"] = np.nan + with pytest.raises(ValueError, match="outcome.*non-finite"): + diff_diff.decompose_twfe_weights(df, **_DECO) + + def test_nan_covariate_is_rejected(self, panel): + df = panel.copy() + df["x"] = 1.0 + df.loc[df.index[0], "x"] = np.inf + with pytest.raises(ValueError, match="covariate.*non-finite"): + diff_diff.decompose_twfe_weights(df, covariates=["x"], **_DECO) + + def test_nan_balance_covariate_is_rejected(self, panel): + df = panel.copy() + df["x"] = 1.0 + df.loc[df.index[0], "x"] = np.nan + with pytest.raises(ValueError, match="balance covariate.*non-finite"): + diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **_DECO) + + +class TestCarveOutConsistency: + """Item 5: the frame path honours zero_treated_control, not control_group.""" + + def test_paths_agree_on_a_not_yet_treated_fit(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + with pytest.warns(UserWarning, match="structurally absent"): + from_fit = attgt_weights(fit, aggregation="overall") + with pytest.warns(UserWarning, match="structurally absent"): + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation="overall", + data=df, + unit="unit", + time="period", + first_treat="first_treat", + ) + np.testing.assert_allclose( + from_fit.weights["weight"].to_numpy(), + from_frame.weights["weight"].to_numpy(), + atol=1e-15, + ) + assert from_fit.implied_att == pytest.approx(from_frame.implied_att, abs=1e-15) + + +class TestCohortDropIsStructural: + """Item 6: a mid cohort blanked out for a non-structural reason fails closed.""" + + def test_fitted_mid_cohort_without_a_reason_raises(self): + fit = _fit(_panel()) + for t in (4, 5): + cell = fit.group_time_effects[(4, t)] + cell["effect"] = np.nan + cell["skip_reason"] = None + with pytest.raises(ValueError, match="do not all carry"): + attgt_weights(fit, aggregation="overall") + + def test_frame_mid_cohort_without_a_reason_raises(self, fitted, panel): + frame = _gt_frame(fitted).copy() + mask = (frame["group"] == 4) & (frame["time"] >= 4) + frame.loc[mask, "effect"] = np.nan + frame.loc[mask, "skip_reason"] = None + with pytest.raises(ValueError, match="do not all carry"): + _frame_call(frame, panel, aggregation="overall") + + +class TestWeightsLength: + """Item 11: a wrong-length weights= is a clear error, never an IndexError.""" + + def test_wrong_length_with_an_excluded_cohort(self): + fit = _fit(_panel(cohorts=(0, 1, 3, 4))) + n_units = len(fit._aggregation_kit.bookkeeping["unit_cohorts"]) + with pytest.raises(ValueError, match="weights has length"): + attgt_weights(fit, aggregation="overall", weights=np.ones(n_units - 1)) + + +class TestMultiCovariatePin: + """Item 13: a frozen-numbers pin for the multi-column solve_ols branch. + + Two NON-collinear covariates are the first coverage of ``k > 1`` (the only + prior test used an exactly collinear pair, which exercises the rank-drop + path, not the multi-column solve). Captured from the implementation; any + refactor must leave these green at 1e-12. + """ + + @staticmethod + def _panel_two_covariates(): + rng = np.random.default_rng(20260914) + n_per, n_periods = 15, 5 + cohorts = [0] * n_per + [3] * n_per + [4] * n_per + rows = [] + for i, g in enumerate(cohorts): + alpha = rng.normal() + x1u, x2u = rng.normal(), rng.normal() + for t in range(1, n_periods + 1): + x1 = x1u + 0.4 * t + rng.normal(scale=0.1) + x2 = x2u - 0.2 * t + rng.normal(scale=0.1) + effect = 1.0 * (t - g + 1) if (g and t >= g) else 0.0 + y = alpha + 0.2 * t + 0.5 * x1 - 0.3 * x2 + effect + rng.normal(scale=0.3) + rows.append({"id": i, "t": t, "g": g, "y": y, "x1": x1, "x2": x2}) + return pd.DataFrame(rows) + + def test_two_covariate_decomposition(self): + df = self._panel_two_covariates() + result = diff_diff.decompose_twfe_weights( + df, outcome="y", unit="id", time="t", first_treat="g", covariates=["x1", "x2"] + ) + assert result.estimate == pytest.approx(1.5221483563387275, abs=1e-12) + assert result.pre_period_contribution == pytest.approx(-0.0878362103974791, abs=1e-12) + assert result.post_only == pytest.approx(1.6099845667362065, abs=1e-12) + assert result.remainder == 0.0 + assert result.covariates == ("x1", "x2") + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + [ + -0.245653467623804, + -0.253634060114308, + 0.381887799688092, + 0.055465013979673, + 0.061934714070347, + -0.068055542075608, + -0.055903047966494, + -0.376753882219787, + 0.256467817300994, + 0.244244654960894, + ], + atol=1e-12, + ) + + +class TestSignedBalancePlot: + """Item 10: the signed balance view draws the full reference diagonal.""" + + @pytest.fixture(autouse=True) + def _agg_backend(self): + matplotlib = pytest.importorskip("matplotlib") + matplotlib.use("Agg") + yield + import matplotlib.pyplot as plt + + plt.close("all") + + @staticmethod + def _decomposed(panel): + df = panel.copy() + df["x"] = np.random.RandomState(8).normal(size=len(df)) + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + + def test_signed_reference_line_spans_negative(self, panel): + ax = diff_diff.plot_twfe_weights( + self._decomposed(panel), kind="balance", absolute_value=False, show=False + ) + line = next(ln for ln in ax.lines if ln.get_label() == "no improvement") + assert line.get_xdata()[0] < 0 + + def test_absolute_reference_line_starts_at_zero(self, panel): + ax = diff_diff.plot_twfe_weights( + self._decomposed(panel), kind="balance", absolute_value=True, show=False + ) + line = next(ln for ln in ax.lines if ln.get_label() == "no improvement") + assert line.get_xdata()[0] == 0 diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py index c0c35f31c..777257912 100644 --- a/tests/test_twfe_weights_parity.py +++ b/tests/test_twfe_weights_parity.py @@ -209,10 +209,24 @@ def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): assert twfe.n_negative > 0 assert twfe.negative_weight_share > 0 + # Post-only negative fields, anchored to R's OWN golden weights rather + # than to our implementation (which would be a tautology). + block = payload["attgt_weights"]["twfe"] + g_w = np.asarray(block["weight"], dtype=float) + g_post = np.asarray(block["post"], dtype=bool) + neg_post = (g_w < 0) & g_post + abs_post = float(np.abs(g_w[g_post]).sum()) + assert twfe.n_negative_post == int(neg_post.sum()) + assert twfe.negative_post_weight_share == pytest.approx( + float(np.abs(g_w[neg_post]).sum() / abs_post), abs=1e-12 + ) + for aggregation in ("overall", "simple"): benign = attgt_weights(fit, aggregation=aggregation) assert benign.n_negative == 0 assert benign.negative_weight_share == 0.0 + assert benign.n_negative_post == 0 + assert benign.negative_post_weight_share == 0.0 @pytest.mark.parametrize("fixture", FIXTURES) def test_target_estimand_weights_sum_to_one(self, golden, fixture): @@ -393,11 +407,11 @@ def test_scalars(self, golden, fixture, key): for field in ("decomposition", "remainder"): np.testing.assert_allclose(getattr(result, field), expected[field], atol=split_atol) - # pretrend_bias / post_only straddle the pre/post split, so R's 0/0 noise + # pre_period_contribution / post_only straddle the pre/post split, so R's 0/0 noise # at the degenerate cells (which sit on opposite sides of it on # sim_staggered) moves each by ~1.2e-4 while their sum stays exact. pp_atol = DEGENERATE_CELL_ATOL if degenerate.any() else atol - for field in ("pretrend_bias", "post_only"): + for field in ("pre_period_contribution", "post_only"): np.testing.assert_allclose(getattr(result, field), expected[field], atol=pp_atol) # effective_sample_size: post_count * sum_post(weight * ess). At the @@ -456,14 +470,22 @@ def test_cells(self, golden, fixture, key): atol=DEGENERATE_CELL_ATOL, ) - # Cell ess / remainder: tight where R is a valid reference; at the - # degenerate cells R divides rounding errors (cell-ess gap up to ~0.53 - # on sim_staggered), so only finiteness is asserted there. - for field in ("ess", "remainder"): - ours = result.cells[field].to_numpy(dtype=float) - theirs = np.asarray(expected["cells"][field], dtype=float) - np.testing.assert_allclose(ours[~degenerate], theirs[~degenerate], atol=atol) - assert np.isfinite(ours[degenerate]).all() + # Cell ESS: tight against R where R is a valid reference; at a 0/0 cell + # the comparison weights are all one, so the cell ESS is exactly + # n_control - an anchor independent of BOTH implementations. + ours_ess = result.cells["ess"].to_numpy(dtype=float) + theirs_ess = np.asarray(expected["cells"]["ess"], dtype=float) + np.testing.assert_allclose(ours_ess[~degenerate], theirs_ess[~degenerate], atol=atol) + n_control = int((df.groupby(cols["unit"])[cols["first_treat"]].first() == 0).sum()) + np.testing.assert_allclose(ours_ess[degenerate], float(n_control), rtol=1e-12) + + # remainder: at the degenerate cells R divides rounding errors + # (cell-ess gap up to ~0.53 on sim_staggered), so only finiteness is + # asserted there. + ours_rem = result.cells["remainder"].to_numpy(dtype=float) + theirs_rem = np.asarray(expected["cells"]["remainder"], dtype=float) + np.testing.assert_allclose(ours_rem[~degenerate], theirs_rem[~degenerate], atol=atol) + assert np.isfinite(ours_rem[degenerate]).all() class TestDecompositionIsExactAtDegenerateCells: diff --git a/tests/test_visualization_plotly.py b/tests/test_visualization_plotly.py index f8fb915f4..309ee053e 100644 --- a/tests/test_visualization_plotly.py +++ b/tests/test_visualization_plotly.py @@ -722,3 +722,33 @@ def test_balance_view(self): fig = diff_diff.plot_twfe_weights(dec, backend="plotly", show=False) assert isinstance(fig, go.Figure) assert any(trace.name == "no improvement" for trace in fig.data) + + def test_signed_balance_reference_line_spans_negative(self): + import diff_diff + + df, _ = self._panel_and_fit() + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + fig = diff_diff.plot_twfe_weights( + dec, kind="balance", absolute_value=False, backend="plotly", show=False + ) + line = next(trace for trace in fig.data if trace.name == "no improvement") + assert min(line.x) < 0 + + def test_hover_labels_do_not_need_annotation(self): + """Item 18: `text` feeds the hover template even when it is not drawn.""" + import diff_diff + + _, fit = self._panel_and_fit() + fig = diff_diff.plot_twfe_weights( + diff_diff.attgt_weights(fit), backend="plotly", annotate=False, show=False + ) + assert fig.data[0].mode == "markers" + assert len(fig.data[0].text) > 0 From d1ba5be971f2c98757a24020110dfbfdd97fa129 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 23 Sep 2026 17:11:37 -0400 Subject: [PATCH 08/13] refactor(twfeweights): attgt_weights(type=) and ATTGTWeightsResult.level Rename the estimand selector to the house aggregate(type=) vocabulary and the result field to .level (values unchanged: "twfe" | "overall" | "simple"). Declare the accepted values as ATTGTWeightsResult.LEVELS (ClassVar) and source the module's _TYPES from it. The private helper's parameter becomes `estimand` because its body needs the builtin type(). Naming-guard entries for the old keyword are removed (no hit left to explain); the surviving [time] entries keep a note on the level/M-044 coexistence. The feature is unreleased, so no deprecation alias or ledger row. --- benchmarks/R/generate_twfeweights_golden.R | 2 +- .../20260831-twfe-weight-diagnostics.md | 7 +- diff_diff/guides/llms-full.txt | 19 ++- diff_diff/guides/llms.txt | 2 +- diff_diff/staggered.py | 2 +- diff_diff/twfe_weights.py | 73 ++++----- diff_diff/twfe_weights_results.py | 32 ++-- docs/api/twfe_weights.rst | 16 +- docs/methodology/REGISTRY.md | 22 +-- tests/helpers/results_foundation.py | 2 +- tests/test_naming_guard.py | 29 +--- tests/test_twfe_weights.py | 148 +++++++++--------- tests/test_twfe_weights_parity.py | 46 +++--- 13 files changed, 202 insertions(+), 198 deletions(-) diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R index 2666e8b1c..957cd1300 100644 --- a/benchmarks/R/generate_twfeweights_golden.R +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -21,7 +21,7 @@ # `diff_diff/twfe_weights.py` exposes two entry points, each folding several # upstream R functions: # -# attgt_weights(aggregation=) <- twfe_weights / attO_weights / +# attgt_weights(type=) <- twfe_weights / attO_weights / # att_simple_weights # decompose_twfe_weights(method=) <- implicit_twfe_weights / # implicit_aipw_weights diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index ab016915f..c72520b26 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -2,9 +2,10 @@ - **TWFE weight diagnostics** (port of Brantly Callaway's `twfeweights` R package, MIT): what a two-way fixed effects regression *implicitly* weights on staggered-adoption data. - - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the + - `attgt_weights(results, type="twfe"|"overall"|"simple")` reports the weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), - plus post-period negative-weight counts. Returns `ATTGTWeightsResult`. + plus post-period negative-weight counts. Returns `ATTGTWeightsResult` + (`.level` records the type). The CS estimands honour the fit's `anticipation` window (an explicit `anticipation=` on the frame path); `"twfe"` keeps `t >= g`. - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate @@ -15,7 +16,7 @@ covariate balance. `plot_twfe_weights()` renders either view (matplotlib or plotly). - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits - under `aggregation="twfe"`, unbalanced panels, non-finite outcomes / + under `type="twfe"`, unbalanced panels, non-finite outcomes / covariates, duplicated or non-finite ATT(g,t) cells, an incomplete group-time grid, and invalid sampling weights. Two structural gaps are handled as R does instead of raising: a cohort with no estimable post cell diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 36c5289b5..3a84d4134 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1545,9 +1545,10 @@ ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package ```python attgt_weights( results, # CallawaySantAnnaResults, or a (g,t) frame - aggregation="twfe", # "twfe" | "overall" (ATT^O) | "simple" + type="twfe", # "twfe" | "overall" (ATT^O) | "simple" data=None, unit=None, time=None, first_treat=None, # frame path only weights=None, # unit-level sampling weights + anticipation=None, # frame path only; the fit's own otherwise ) -> ATTGTWeightsResult decompose_twfe_weights( @@ -1563,11 +1564,12 @@ decompose_twfe_weights( plot_twfe_weights(result, kind="auto") # "weights" | "balance" ``` -`aggregation="twfe"` requires a fit with `base_period="universal"`, +`type="twfe"` requires a fit with `base_period="universal"`, `control_group="never_treated"` AND no covariates (R twfe_weights' three restrictions); it raises otherwise. ATT^O and ATT^simple weights are non-negative and sum to one, so comparing `implied_att` across the three -aggregations shows what the TWFE specification costs. +types shows what the TWFE specification costs. The result records its +`type=` as `.level` (`ATTGTWeightsResult.LEVELS` lists the values). Both entry points fail closed on input R never faced: NaN / `-inf` cohort labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite @@ -1586,7 +1588,7 @@ zero over the full grid. `decompose_twfe_weights` takes the raw panel rather than a fitted result because it re-estimates. It is tied to `attgt_weights` by an identity: -`attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. +`attgt_weights(cs, type="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. ### StaggeredTripleDifference @@ -2078,9 +2080,12 @@ decomposition is an algebraic identity. - `weights`: DataFrame with `group`, `time`, `post`, `weight`, `att` - `implied_att`: `sum(weight * att)` - the TWFE coefficient when - `aggregation="twfe"` -- `n_negative`, `negative_weight_share`: the staggered-TWFE pathology -- `aggregation`, `source`, `control_group`, `base_period`, `n_cells` + `type="twfe"` +- `n_negative_post`, `negative_post_weight_share`: the staggered-TWFE + pathology (negative weight on POST cells); `n_negative`, + `negative_weight_share` count every cell and are structural under `"twfe"` +- `level` (the `type=` used), `source`, `control_group`, `base_period`, + `n_cells` - `summary()`, `to_dataframe()`, `to_dict()` ### TWFEDecompositionResult diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 096f2b6a3..f7125471e 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -92,7 +92,7 @@ The site is organized into 5 sections, each with a landing page: - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html): Cattaneo, Jansson & Ma (2020) density-discontinuity manipulation test (`RDDensityTest`), parity with R rddensity 3.0 - boundary-adaptive local polynomial density estimation at the cutoff, robust bias-corrected inference, unrestricted/restricted models, jackknife/plugin variances, data-driven bandwidths, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics -- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pre_period_contribution` (the pre-treatment cells' sample contribution, which can be pre-trends or sampling noise), and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, type='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pre_period_contribution` (the pre-treatment cells' sample contribution, which can be pre-trends or sampling noise), and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 134acdafb..48fa874e6 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -5188,7 +5188,7 @@ def _build_aggregation_kit( # aggregate(). Legacy kits without the key default at the read site. bookkeeping["bootstrap_label"] = getattr(estimator, "_BOOTSTRAP_LABEL", "CallawaySantAnna") # Covariate usage, recorded so downstream diagnostics can refuse designs - # their formulas do not cover (``attgt_weights(aggregation="twfe")`` + # their formulas do not cover (``attgt_weights(type="twfe")`` # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES # only - never values - so the data-minimization contract holds. bookkeeping["covariates"] = tuple(covariates or ()) diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index 7c1cda31f..082ffc7ce 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -68,7 +68,7 @@ __all__ = ["attgt_weights", "decompose_twfe_weights"] -_AGGREGATIONS = ("twfe", "overall", "simple") +_TYPES = ATTGTWeightsResult.LEVELS def _is_never(values: np.ndarray) -> np.ndarray: @@ -162,7 +162,7 @@ def _validate_unit_weights( Finite, non-negative, positive total, positive TREATED mass; positive never-treated mass only where the never-treated group enters the formula - (``aggregation="twfe"`` and the decomposition) - ATT^O / ATT^simple are + (``type="twfe"`` and the decomposition) - ATT^O / ATT^simple are defined without one. """ if not np.all(np.isfinite(w)): @@ -300,7 +300,7 @@ def _attgt_from_cs( Non-estimable cells (``skip_reason`` set, NaN effect) are left out of the table and reported in the returned ``{(g, t): skip_reason}`` map, so the - caller can decide - per aggregation - whether the gap is structural, a + caller can decide - per estimand type - whether the gap is structural, a harmless pre-period drop, or a hard error. """ rows: List[Dict[str, Any]] = [] @@ -461,7 +461,7 @@ def _resolve_cs_inputs( ) -def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> None: +def _guard_cs_design(results: "CallawaySantAnnaResults", estimand: str) -> None: """Reject fits whose design breaks the weight formulas. These are hard errors rather than warnings: a silently wrong weight table @@ -486,12 +486,12 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No "unbalanced panel, so the cohort shares are not comparable across " "periods. Balance the panel (diff_diff.balance_panel) and refit." ) - if aggregation != "twfe": + if estimand != "twfe": return control_group = getattr(results, "control_group", None) if control_group not in (None, "never_treated"): raise ValueError( - f"aggregation='twfe' requires control_group='never_treated', got " + f"type='twfe' requires control_group='never_treated', got " f"{control_group!r}. The TWFE weight formula is derived against a " "never-treated comparison group (matching R's twfe_weights, which " "raises the same restriction)." @@ -499,13 +499,13 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No base_period = getattr(results, "base_period", None) if base_period not in (None, "universal"): raise ValueError( - f"aggregation='twfe' requires base_period='universal', got " + f"type='twfe' requires base_period='universal', got " f"{base_period!r}. The formula needs the complete cohort x period " "grid, including the pre-treatment cells that a varying base does " "not report. Refit with base_period='universal'." ) # R's third restriction: xformla == ~1. The fit records its covariate - # column names on the aggregation kit; a kit without the key predates that + # column names on the estimand kit; a kit without the key predates that # bookkeeping (an old pickle) and can only be warned about. A missing kit # is left to _resolve_cs_inputs, whose error is the useful one. kit = getattr(results, "_aggregation_kit", None) @@ -522,7 +522,7 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No ) elif bookkeeping["covariates"]: raise ValueError( - f"aggregation='twfe' requires a fit without covariates, but this one " + f"type='twfe' requires a fit without covariates, but this one " f"adjusted for {list(bookkeeping['covariates'])!r}. The TWFE weight " "formula describes the unadjusted regression (R's twfe_weights stops " "unless xformla == ~1); refit with covariates=None, or use " @@ -534,7 +534,7 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> No def attgt_weights( results: Union["CallawaySantAnnaResults", pd.DataFrame], *, - aggregation: str = "twfe", + type: str = "twfe", # noqa: A002 - matches results.aggregate(type=) vocabulary data: Optional[pd.DataFrame] = None, unit: Optional[str] = None, time: Optional[str] = None, @@ -560,10 +560,13 @@ def attgt_weights( its ``skip_reason`` column. On the frame path, ``data``, ``unit``, ``time`` and ``first_treat`` are required so cohort shares can be formed, and the caller is responsible for the fit having used no - covariates under ``aggregation="twfe"`` (a frame carries no record of + covariates under ``type="twfe"`` (a frame carries no record of that; the fitted path checks it). - aggregation : {"twfe", "overall", "simple"}, default "twfe" - Which estimand's weights to report. + type : {"twfe", "overall", "simple"}, default "twfe" + Which estimand's weights to report (the same keyword as + ``results.aggregate(type=...)``; the accepted values are + ``ATTGTWeightsResult.LEVELS``). ``"overall"`` is ATT^O, R ``did``'s + ``attO``. The result records the choice as ``.level``. data : pd.DataFrame, optional Balanced panel backing the ATT(g, t) frame. Only for the fallback path; passing it alongside a fitted result raises. @@ -580,7 +583,7 @@ def attgt_weights( anticipation``). Only meaningful on the DataFrame path, where a bare frame carries no record of the source fit's setting; the fitted path reads it off the fit and rejects an explicit ``anticipation=``. It does - NOT affect ``aggregation="twfe"``: the TWFE regression's own treatment + NOT affect ``type="twfe"``: the TWFE regression's own treatment indicator is ``1[t >= g]`` regardless of how the CS estimands treat the run-up, and R's ``twfe_weights`` has no anticipation argument either. @@ -592,9 +595,9 @@ def attgt_weights( Raises ------ ValueError - On an unknown ``aggregation``; on a design the formula does not + On an unknown ``type``; on a design the formula does not support (repeated cross-sections, an unbalanced panel, and - for - ``aggregation="twfe"`` - a non-never-treated control group, a + ``type="twfe"`` - a non-never-treated control group, a non-universal base period, or a covariate-adjusted fit); on NaN / ``-inf`` cohort labels, invalid weights, duplicated or non-finite cells; or on an INCOMPLETE grid: ``"twfe"`` needs every cohort x period @@ -641,13 +644,11 @@ def attgt_weights( >>> cs = diff_diff.CallawaySantAnna(base_period="universal") # doctest: +SKIP >>> res = cs.fit(df, outcome="y", unit="id", time="t", ... first_treat="g") # doctest: +SKIP - >>> w = diff_diff.attgt_weights(res, aggregation="twfe") # doctest: +SKIP + >>> w = diff_diff.attgt_weights(res, type="twfe") # doctest: +SKIP >>> print(w.summary()) # doctest: +SKIP """ - if aggregation not in _AGGREGATIONS: - raise ValueError( - f"aggregation must be one of {list(_AGGREGATIONS)!r}, got " f"{aggregation!r}" - ) + if type not in _TYPES: + raise ValueError(f"type must be one of {list(_TYPES)!r}, got " f"{type!r}") if anticipation is not None: if isinstance(anticipation, bool) or not isinstance(anticipation, (int, np.integer)): raise ValueError(f"anticipation must be a non-negative integer, got {anticipation!r}") @@ -694,7 +695,7 @@ def attgt_weights( "Drop anticipation=, or pass result.to_dataframe('group_time') " "as the first argument." ) - _guard_cs_design(results, aggregation) + _guard_cs_design(results, type) table, skipped = _attgt_from_cs(results) cohorts, survey_weights, window, is_balanced = _resolve_cs_inputs(results) if not is_balanced: @@ -751,13 +752,13 @@ def attgt_weights( # Post-treatment mask. The TWFE regression's own indicator is 1[t >= g] # regardless of the CS anticipation window; the CS target estimands shift # it to 1[t >= g - anticipation]. - if aggregation == "twfe": + if type == "twfe": post_mask = t_pos >= g_pos else: post_mask = t_pos >= (g_pos - window) def _post_start(g: int) -> int: - raw = g if aggregation == "twfe" else g - window + raw = g if type == "twfe" else g - window return max(1, raw) # --- whole-cohort exclusion (R did drops units treated in the first period) @@ -825,13 +826,13 @@ def _post_start(g: int) -> int: skipped = {k: v for k, v in skipped.items() if _pos_of(grid, k[0]) not in excluded} p_all, p_treated, e_dt, mean_e_dt = _cohort_masses( - cohorts, grid, unit_weights, require_control_mass=(aggregation == "twfe") + cohorts, grid, unit_weights, require_control_mass=(type == "twfe") ) # --- grid completeness present = set(zip(g_pos.tolist(), t_pos.tolist())) surviving = sorted(cohorts_with_post) - if aggregation == "twfe": + if type == "twfe": required = {(g, t) for g in surviving for t in range(1, n_periods + 1)} else: required = {(g, t) for g in surviving for t in range(_post_start(g), n_periods + 1)} @@ -842,7 +843,7 @@ def _post_start(g: int) -> int: # not on control_group: that reason is only ever emitted on a # not-yet-treated fit, and the frame path has no control_group to read, # so keying on the reason is what makes the two paths agree. - carve_out_ok = aggregation != "twfe" + carve_out_ok = type != "twfe" hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] for g, t in missing_cells: label = (_label_for(grid, g), _label_for(grid, t)) @@ -852,12 +853,12 @@ def _post_start(g: int) -> int: else: hard.append((label, reason)) if hard: - what = "cohort x period" if aggregation == "twfe" else "post-treatment" + what = "cohort x period" if type == "twfe" else "post-treatment" detail = ", ".join( f"{lab} [{reason or 'not in source table'}]" for lab, reason in hard[:6] ) raise ValueError( - f"aggregation={aggregation!r} needs the complete {what} grid, but " + f"type={type!r} needs the complete {what} grid, but " f"{len(hard)} required cell(s) are missing: {detail}. A weight table " "over a partial grid is not the named estimand. Fix the source fit " "(or pass the complete to_dataframe('group_time') output)." @@ -866,7 +867,7 @@ def _post_start(g: int) -> int: f"{len(structurally_absent)} post-treatment cell(s) {structurally_absent[:6]!r} " "have no not-yet-treated comparison units (skip_reason " "'zero_treated_control') and are treated as structurally absent: " - f"aggregation={aggregation!r} averages over each cohort's AVAILABLE " + f"type={type!r} averages over each cohort's AVAILABLE " "post periods, as R aggte() does on a not-yet-treated fit", UserWarning, stacklevel=2, @@ -880,18 +881,18 @@ def _post_start(g: int) -> int: gp, tp = _pos_of(grid, g_lab), _pos_of(grid, t_lab) if gp in surviving_positional and tp < _post_start(gp): dropped += 1 - if dropped and aggregation != "twfe": + if dropped and type != "twfe": warnings.warn( f"{dropped} pre-treatment group-time cell(s) had no estimable ATT(g,t) " - f"and were excluded; aggregation={aggregation!r} places no weight on " + f"and were excluded; type={type!r} places no weight on " "pre-treatment cells, so the weights are unaffected", UserWarning, stacklevel=2, ) - if aggregation == "twfe": + if type == "twfe": weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) - elif aggregation == "overall": + elif type == "overall": n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} weight_vec = _overall_weight_vector(g_pos, p_treated, post_mask, n_post_available) else: @@ -913,7 +914,7 @@ def _post_start(g: int) -> int: abs_post_total = float(np.abs(weight_vec[post_mask]).sum()) return ATTGTWeightsResult( weights=out, - aggregation=aggregation, + level=type, implied_att=float((weight_vec * table["att"].to_numpy()).sum()), n_negative=int(negative.sum()), negative_weight_share=( @@ -1700,7 +1701,7 @@ def decompose_twfe_weights( by an identity that holds when the fit used ``base_period="universal"``, ``control_group="never_treated"`` and no covariates:: - sum(attgt_weights(cs, aggregation="twfe").weights.eval("weight * att")) + sum(attgt_weights(cs, type="twfe").weights.eval("weight * att")) == decompose_twfe_weights(panel, ...).estimate Parameters diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py index 58a086a65..7635c37a9 100644 --- a/diff_diff/twfe_weights_results.py +++ b/diff_diff/twfe_weights_results.py @@ -15,7 +15,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Optional, Tuple +from typing import Any, ClassVar, Dict, Optional, Tuple import numpy as np import pandas as pd @@ -69,15 +69,17 @@ class ATTGTWeightsResult(Diagnostic): ``0`` for pre-treatment cells. ``att`` is the ATT(g, t) the weight multiplies, carried through from the source so that ``(weight * att).sum()`` reproduces ``implied_att``. - aggregation : str - Which estimand's weights these are: ``"twfe"``, ``"overall"`` - (ATT^O), or ``"simple"`` (ATT^simple). + level : str + The ``type=`` this result was built with, one of :attr:`LEVELS`: + ``"twfe"``, ``"overall"`` (ATT^O) or ``"simple"`` (ATT^simple). + Unrelated to :meth:`TWFEDecompositionResult.covariate_balance`'s + ``level=`` (an output granularity). implied_att : float ``sum(weight * att)`` - what the estimand delivers given these - ATT(g, t). For ``aggregation="twfe"`` this is the TWFE coefficient. + ATT(g, t). For ``type="twfe"`` this is the TWFE coefficient. n_negative : int Number of cells - PRE and post - receiving a negative weight. Under - ``aggregation="twfe"`` the weights over the full ``g != 0`` grid sum + ``type="twfe"`` the weights over the full ``g != 0`` grid sum to zero (post to +1, pre to -1), so this is non-zero in every staggered design; read ``n_negative_post`` for the pathology. negative_weight_share : float @@ -102,10 +104,20 @@ class ATTGTWeightsResult(Diagnostic): Design metadata carried from the source fit, when available. n_dropped_cells : int Cells excluded because their ATT(g, t) was non-estimable (NaN). + LEVELS : tuple of str + The values ``level`` can take: ``("twfe", "overall", "simple")``. + ``"overall"`` here is ATT^O (R ``did``'s ``attO``) and is unrelated + to the ``"overall"`` row label an :class:`~diff_diff.AggregationResult` + carries under ``level="simple"``. """ + # Accepted ``type=`` values, declared on the class the way the results + # classes declare ``_AGGREGATE_SUPPORTED``. ``ClassVar`` keeps it out of + # the dataclass fields (see stacked_did_results.py for the trap). + LEVELS: ClassVar[Tuple[str, ...]] = ("twfe", "overall", "simple") + weights: pd.DataFrame - aggregation: str + level: str implied_att: float n_negative: int negative_weight_share: float @@ -119,7 +131,7 @@ class ATTGTWeightsResult(Diagnostic): def __repr__(self) -> str: return ( - f"ATTGTWeightsResult(aggregation={self.aggregation!r}, " + f"ATTGTWeightsResult(level={self.level!r}, " f"implied_att={self.implied_att:.4f}, " f"n_cells={self.n_cells}, n_negative={self.n_negative})" ) @@ -127,7 +139,7 @@ def __repr__(self) -> str: def summary(self) -> str: """Formatted per-cell weight table with the negative-weight roll-up.""" width = 72 - label = _AGGREGATION_LABELS.get(self.aggregation, self.aggregation) + label = _AGGREGATION_LABELS.get(self.level, self.level) lines = [ "=" * width, "Implicit Weights on ATT(g, t)".center(width), @@ -193,7 +205,7 @@ def to_dataframe(self) -> pd.DataFrame: def to_dict(self) -> Dict[str, Any]: """Serializable view of the result.""" return { - "aggregation": self.aggregation, + "level": self.level, "implied_att": self.implied_att, "n_cells": self.n_cells, "n_negative": self.n_negative, diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst index 9b39b31f0..62a8d9f3b 100644 --- a/docs/api/twfe_weights.rst +++ b/docs/api/twfe_weights.rst @@ -71,6 +71,10 @@ Result Objects :show-inheritance: :no-index: +``ATTGTWeightsResult.LEVELS`` lists the accepted ``type=`` values +(``"twfe"``, ``"overall"``, ``"simple"``); the result records the one it was +built with as ``.level``. + .. autoclass:: diff_diff.TWFEDecompositionResult :members: :undoc-members: @@ -91,20 +95,20 @@ Inspecting what a TWFE regression weights cs = diff_diff.CallawaySantAnna( control_group="never_treated", - base_period="universal", # required for aggregation="twfe" + base_period="universal", # required for type="twfe" ).fit( panel, outcome="lemp", unit="countyreal", time="year", first_treat="first_treat", ) - weights = diff_diff.attgt_weights(cs, aggregation="twfe") + weights = diff_diff.attgt_weights(cs, type="twfe") print(weights.summary()) print(weights.n_negative, "cells carry negative weight") Comparing against the estimand you meant to report ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``aggregation="overall"`` and ``"simple"`` give the Callaway & Sant'Anna +``type="overall"`` and ``"simple"`` give the Callaway & Sant'Anna target-parameter weights, which are non-negative and sum to one. The gap between ``implied_att`` values is the cost of the TWFE specification: @@ -120,9 +124,9 @@ between ``implied_att`` values is the cost of the TWFE specification: first_treat="first_treat", ) - for aggregation in ("twfe", "overall", "simple"): - w = diff_diff.attgt_weights(cs, aggregation=aggregation) - print(f"{aggregation:8s} {w.implied_att: .4f} " + for level in ("twfe", "overall", "simple"): + w = diff_diff.attgt_weights(cs, type=level) + print(f"{level:8s} {w.implied_att: .4f} " f"({w.n_negative} negative weights)") Separating treatment effects from the pre-period contribution diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 60478b08f..0a69fcdfb 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6231,9 +6231,9 @@ Where `n_k` is the sample share of timing group `k`, `n_{kℓ} = n_k / (n_k + n_ All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T`. -*ATT(g,t) weights — `attgt_weights(aggregation=...)`:* +*ATT(g,t) weights — `attgt_weights(type=...)`:* -`aggregation="twfe"` (R `twfe_weights`), with `p_g` the share of ALL units in cohort `g` and `E_t[D]` the share of units treated by `t`: +`type="twfe"` (R `twfe_weights`), with `p_g` the share of ALL units in cohort `g` and `E_t[D]` the share of units treated by `t`: ``` h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D] @@ -6241,13 +6241,13 @@ num(g,t) = h(g,t) * p_g w(g,t) = num(g,t) / sum over {t >= g, g != 0} of num(g,t) ``` -`aggregation="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: +`type="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: ``` w(g,t) = 1[t >= g] * pbar_g / (maxT - g + 1) ``` -`aggregation="simple"` (ATT^simple, R `att_simple_weights`): +`type="simple"` (ATT^simple, R `att_simple_weights`): ``` w(g,t) = 1[t >= g] * pbar_g, then normalized to sum to one @@ -6284,19 +6284,19 @@ pre_period_contribution = sum over PRE cells (t < g) of alpha_weight * ATT *Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, ``` -attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate +attgt_weights(cs, type="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate ``` Verified on `mpdta` at `-0.03654894` from both directions. ### Edge cases -- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `aggregation="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. +- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `type="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. - **Note (matches R `did`'s first-period drop, but only when the drop is structural):** a cohort with NO estimable post-treatment cell is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. The drop is allowed ONLY when the cohort is treated in the first observed period, OR when every one of its missing post cells carries `skip_reason="zero_treated_control"` (it ran out of comparison units). A mid cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. - **Note (matches R `did`'s mechanism behind the second structural gap):** with no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g)` and recodes the last-treated cohort to never-treated. That truncation - not `aggte` averaging over available periods - is what produces R's number. diff-diff reaches the same numerics by treating the cells CS marks `skip_reason="zero_treated_control"` as structurally ABSENT (they name exactly the cohort x period cells that truncation removes): `"overall"` divides each cohort by its number of available post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group`, because that reason is only ever emitted on a not-yet-treated fit and the frame path carries no `control_group`; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. -- **Note (`aggregation="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. -- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `aggregation="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. +- **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. +- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. - `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The frame path requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). @@ -6312,13 +6312,13 @@ Verified on `mpdta` at `-0.03654894` from both directions. - **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pre_period_contribution` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. - **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. - **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. -- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. +- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `type=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. - **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. - **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. - **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. - **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values, and that `NA` survives the summary roll-up. R propagates NA if any contributing cell is NA, whereas a plain pandas `.sum()` skips it and would report a spurious `0.0` for a binary or constant covariate; the roll-up therefore masks on the `post` column (not on a zero roll-up weight, since a post cell whose implicit weight is exactly zero still contributes) and returns NaN whenever any contributing post cell is NaN. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. - **Note (negative-weight statistics; no R counterpart):** over the `g != 0` grid the TWFE weights sum to zero, post cells to +1 and pre cells to -1, so `n_negative` is non-zero and `negative_weight_share` sits near 0.5 in EVERY staggered design, including one with no negative post-period weight. The pathology the literature describes is negative weight on POST cells, so `ATTGTWeightsResult` reports both, labelled: `n_negative` / `negative_weight_share` over all cells, and `n_negative_post` / `negative_post_weight_share` restricted to post cells (the share of post-period weight MASS that is negative). `summary()` leads with the post-only figures. R reports neither statistic. -- **Note (weighted `aggregation="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, aggregation="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). +- **Note (weighted `type="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, type="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). - **Note (linear algebra runs through the house helpers):** the two-way demeaning is `diff_diff.utils.within_transform` (the same alternating projections `fixest::demean` runs) applied to the sorted long frame before the `(unit, period)` reshape, with the treatment indicator synthesized as a column since it is derived from cohorts x positional periods rather than supplied. The Frisch-Waugh-Lovell solve is `diff_diff.linalg.solve_ols(..., weights=, rank_deficient_action="silent")`: on a rank-deficient design it fits the maximal independent set, sets the aliased coefficients to `NaN` (R-style) and returns the residual computed from the identified ones, so the module reads the dropped column names off the `NaN` positions and uses the returned residual directly. This replaced a bespoke pivoted QR whose docstring claimed to drop "later columns first"; it did not, being the same norm-pivoted QR `solve_ols` uses, and dropping the same column. - **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. - **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. @@ -6331,7 +6331,7 @@ Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, | Surface | Gate | Rationale | |---------|------|-----------| -| ATT(g,t) weights, all three aggregations | `atol=1e-12` | Closed-form rational expression in cohort masses; only double-precision representation error separates the two sides. Observed max deviation 4.7e-16. | +| ATT(g,t) weights, all three types | `atol=1e-12` | Closed-form rational expression in cohort masses; only double-precision representation error separates the two sides. Observed max deviation 4.7e-16. | | `implied_att` (R's own ATT(g,t) fed back in) | `atol=1e-12` | Isolates the weight arithmetic from CallawaySantAnna-vs-`did` parity. | | End-to-end from a CS fit | `rtol=1e-6` | COMPOSED check — carries the pre-existing CS parity band, not this module's. | | FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index 3b7251ff3..4506c53bd 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -336,7 +336,7 @@ def make_constructed_diagnostics() -> Dict[str, Any]: # drift from the object it imitates. "ATTGTWeightsResult": diff_diff.ATTGTWeightsResult( weights=attgt_weight_cells, - aggregation="twfe", + level="twfe", implied_att=float((attgt_weight_cells["weight"] * attgt_weight_cells["att"]).sum()), n_negative=int((attgt_weight_cells["weight"] < 0).sum()), negative_weight_share=float( diff --git a/tests/test_naming_guard.py b/tests/test_naming_guard.py index 953c371c6..e30ecc4d0 100644 --- a/tests/test_naming_guard.py +++ b/tests/test_naming_guard.py @@ -569,9 +569,9 @@ def _build_rowed_index(): # without reading any of them. `time=` here is the panel PERIOD COLUMN # NAME (the same role as CallawaySantAnna.fit[time], which no row touches), # not the two-period 0/1 post dummy M-030/M-031/M-082/M-137/M-138 rename to - # `post`: both functions are staggered-only. `aggregation=` selects an - # ESTIMAND ("twfe" / "overall" / "simple"), not the Wooldridge output - # granularity M-044 renames to `level` and M-087 removes. + # `post`: both functions are staggered-only. (`attgt_weights(type=)` / + # `ATTGTWeightsResult.level` reuse the `aggregate(type=)` / `.level` pair + # as an ESTIMAND selector, distinct from the M-044 granularity sense.) **{ f"{fn}[time]": ( "panel PERIOD COLUMN NAME (as in CallawaySantAnna.fit[time]), not " @@ -580,16 +580,6 @@ def _build_rowed_index(): ) for fn in ("attgt_weights", "decompose_twfe_weights") }, - "attgt_weights[aggregation]": ( - "ESTIMAND selector ('twfe' / 'overall' / 'simple'), not the " - "WooldridgeDiDResults output granularity M-044 renames to `level` and " - "M-087 removes; survives 4.0" - ), - "ATTGTWeightsResult.aggregation": ( - "records which ESTIMAND's weights the result holds - the " - "attgt_weights[aggregation] value, not a Wooldridge output granularity " - "(M-044 / M-087); survives 4.0" - ), "run_placebo_test[time]": ( "OVERLOADED pass-through, redesign pending (TODO.md): forwarded as " "the calendar column to placebo_timing_test/placebo_group_test AND " @@ -1015,22 +1005,13 @@ def _token_family_code_refs(tok): # was migrated in this same diff (migrate-first rule) and remains a lane # hit only through its unrelated backticked schema key. # The TWFE weight diagnostics document their own `time=` (panel period - # COLUMN) and `aggregation=` (estimand selector) on these two surfaces; - # neither reads a renamed name. See the SURFACE_ALLOWLIST entries for - # attgt_weights / decompose_twfe_weights. + # COLUMN) on this surface; it does not read a renamed name. See the + # SURFACE_ALLOWLIST entries for attgt_weights / decompose_twfe_weights. ("time", "diff_diff/guides/llms.txt"): ( "attgt_weights / decompose_twfe_weights document a panel PERIOD COLUMN " "named `time`, not the two-period 0/1 post dummy renamed by " "M-030/M-031/M-082/M-137/M-138" ), - ("aggregation", "diff_diff/guides/llms.txt"): ( - "attgt_weights' ESTIMAND selector, not WooldridgeDiDResults' output " - "granularity (M-044 / M-087)" - ), - ("aggregation", "docs/methodology/REGISTRY.md"): ( - "the TWFE Weight Diagnostics section documents attgt_weights' ESTIMAND " - "selector, not WooldridgeDiDResults' output granularity (M-044 / M-087)" - ), ("estimator", "diff_diff/aggregation.py"): ( "AggregationResult.estimator - independent field holding a CLASS NAME" ), diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index da87fd4b8..cd1570116 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -92,21 +92,21 @@ def test_result_renders(self, fitted): # to_dataframe hands back a copy, not the live table frame.loc[0, "weight"] = 999.0 assert result.weights.loc[0, "weight"] != 999.0 - assert set(result.to_dict()) >= {"aggregation", "implied_att", "weights"} - assert "aggregation='twfe'" in repr(result) + assert set(result.to_dict()) >= {"level", "implied_att", "weights"} + assert "level='twfe'" in repr(result) class TestAggregationBehaviour: - @pytest.mark.parametrize("aggregation", ["overall", "simple"]) - def test_target_estimands_are_convex(self, fitted, aggregation): + @pytest.mark.parametrize("level", ["overall", "simple"]) + def test_target_estimands_are_convex(self, fitted, level): """ATT^O / ATT^simple weights are non-negative and sum to one.""" - weights = attgt_weights(fitted, aggregation=aggregation).weights["weight"] + weights = attgt_weights(fitted, type=level).weights["weight"] assert (weights >= 0).all() assert weights.sum() == pytest.approx(1.0, abs=1e-12) def test_twfe_weights_can_be_negative(self, fitted): """The whole point of the diagnostic: staggered TWFE is not convex.""" - result = attgt_weights(fitted, aggregation="twfe") + result = attgt_weights(fitted, type="twfe") assert result.n_negative > 0 assert 0.0 < result.negative_weight_share < 1.0 assert "Negative POST-period cells:" in result.summary() @@ -114,34 +114,34 @@ def test_twfe_weights_can_be_negative(self, fitted): def test_pre_treatment_cells_carry_weight_under_twfe(self, fitted): """TWFE loads on pre-treatment cells; the CS estimands do not.""" - twfe = attgt_weights(fitted, aggregation="twfe").weights + twfe = attgt_weights(fitted, type="twfe").weights assert (twfe.loc[twfe["post"] == 0, "weight"].abs() > 0).any() - for aggregation in ("overall", "simple"): - benign = attgt_weights(fitted, aggregation=aggregation).weights + for level in ("overall", "simple"): + benign = attgt_weights(fitted, type=level).weights assert (benign.loc[benign["post"] == 0, "weight"] == 0).all() - def test_rejects_unknown_aggregation(self, fitted): - with pytest.raises(ValueError, match="aggregation must be one of"): - attgt_weights(fitted, aggregation="everything") + def test_rejects_unknown_type(self, fitted): + with pytest.raises(ValueError, match="type must be one of"): + attgt_weights(fitted, type="everything") class TestDesignGuards: def test_rejects_non_universal_base_period_for_twfe(self, panel): fit = _fit(panel, base_period="varying") with pytest.raises(ValueError, match="base_period='universal'"): - attgt_weights(fit, aggregation="twfe") + attgt_weights(fit, type="twfe") def test_varying_base_is_fine_for_the_cs_estimands(self, panel): """Only the TWFE formula needs the complete grid.""" fit = _fit(panel, base_period="varying") - for aggregation in ("overall", "simple"): - result = attgt_weights(fit, aggregation=aggregation) + for level in ("overall", "simple"): + result = attgt_weights(fit, type=level) assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) def test_rejects_not_yet_treated_control_for_twfe(self, panel): fit = _fit(panel, control_group="not_yet_treated") with pytest.raises(ValueError, match="control_group='never_treated'"): - attgt_weights(fit, aggregation="twfe") + attgt_weights(fit, type="twfe") def test_rejects_repeated_cross_sections(self, panel): # A true RCS needs one observation per unit id, so re-key the rows @@ -224,13 +224,13 @@ class TestNonConsecutiveTimeLabels: """Positional rescaling: gapped period labels must not change the weights.""" def test_gapped_periods_match_consecutive_ones(self, panel): - consecutive = attgt_weights(_fit(panel), aggregation="twfe") + consecutive = attgt_weights(_fit(panel), type="twfe") gapped = panel.copy() remap = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50} gapped["period"] = gapped["period"].map(remap) gapped["first_treat"] = gapped["first_treat"].map(lambda g: remap.get(g, 0)) - result = attgt_weights(_fit(gapped), aggregation="twfe") + result = attgt_weights(_fit(gapped), type="twfe") np.testing.assert_allclose( result.weights["weight"].to_numpy(), @@ -244,7 +244,7 @@ class TestSamplingWeights: def test_uniform_weights_are_a_no_op(self, fitted, panel): baseline = attgt_weights( fitted.to_dataframe("group_time"), - aggregation="overall", + type="overall", data=panel, unit="unit", time="period", @@ -253,7 +253,7 @@ def test_uniform_weights_are_a_no_op(self, fitted, panel): weighted_panel = panel.assign(w=1.0) weighted = attgt_weights( fitted.to_dataframe("group_time"), - aggregation="overall", + type="overall", data=weighted_panel, unit="unit", time="period", @@ -268,7 +268,7 @@ def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): """Doubling a cohort's sampling weight raises its share of ATT^O.""" baseline = attgt_weights( fitted.to_dataframe("group_time"), - aggregation="overall", + type="overall", data=panel, unit="unit", time="period", @@ -277,7 +277,7 @@ def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): tilted_panel = panel.assign(w=np.where(panel["first_treat"] == 3, 2.0, 1.0)) tilted = attgt_weights( fitted.to_dataframe("group_time"), - aggregation="overall", + type="overall", data=tilted_panel, unit="unit", time="period", @@ -337,7 +337,7 @@ def test_warns_and_renormalizes_when_cells_are_dropped(self, fitted, panel): with pytest.warns(UserWarning, match="had no estimable ATT"): result = attgt_weights( frame, - aggregation="overall", + type="overall", data=panel, unit="unit", time="period", @@ -538,15 +538,15 @@ def test_decomposition_weighted_branches(self, key): ), } - @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) - def test_attgt_weighted_branches(self, aggregation): + @pytest.mark.parametrize("level", ["twfe", "overall", "simple"]) + def test_attgt_weighted_branches(self, level): df = self._weighted_panel() cs = diff_diff.CallawaySantAnna(base_period="universal", control_group="never_treated").fit( df, outcome="y", unit="id", time="t", first_treat="g" ) unit_w = df.groupby("id", sort=True)["w"].first().to_numpy() - result = attgt_weights(cs, aggregation=aggregation, weights=unit_w) - implied, weight = self._AGG[aggregation] + result = attgt_weights(cs, type=level, weights=unit_w) + implied, weight = self._AGG[level] assert result.implied_att == pytest.approx(implied, abs=1e-12) np.testing.assert_allclose(result.weights["weight"].to_numpy(), weight, atol=1e-12) assert list(zip(result.weights["group"], result.weights["time"])) == [ @@ -694,9 +694,9 @@ def test_covariate_adjusted_fit_is_rejected_under_twfe(self, panel): ) assert fit._aggregation_kit.bookkeeping["covariates"] == ("x",) with pytest.raises(ValueError, match="requires a fit without covariates"): - attgt_weights(fit, aggregation="twfe") + attgt_weights(fit, type="twfe") # The CS estimands do not depend on the regression specification. - assert attgt_weights(fit, aggregation="overall").n_negative_post == 0 + assert attgt_weights(fit, type="overall").n_negative_post == 0 def test_unadjusted_fit_records_empty_covariates(self, fitted): assert fitted._aggregation_kit.bookkeeping["covariates"] == () @@ -706,7 +706,7 @@ def test_legacy_kit_without_the_key_warns(self, fitted): saved = kit.bookkeeping.pop("covariates") try: with pytest.warns(UserWarning, match="predates covariate bookkeeping"): - attgt_weights(fitted, aggregation="twfe") + attgt_weights(fitted, type="twfe") finally: kit.bookkeeping["covariates"] = saved @@ -733,39 +733,39 @@ def test_non_finite_group_label_is_rejected(self, fitted, panel): with pytest.raises(ValueError, match="NaN or -inf cohort label"): _frame_call(frame, panel) - @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) - def test_inf_att_on_a_post_cell_is_an_incomplete_grid(self, fitted, panel, aggregation): + @pytest.mark.parametrize("level", ["twfe", "overall", "simple"]) + def test_inf_att_on_a_post_cell_is_an_incomplete_grid(self, fitted, panel, level): frame = _gt_frame(fitted) idx = frame.index[(frame["group"] == 3) & (frame["time"] == 3)][0] frame.loc[idx, "effect"] = np.inf with pytest.raises(ValueError, match="complete .* grid"): - _frame_call(frame, panel, aggregation=aggregation) + _frame_call(frame, panel, type=level) - @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) - def test_missing_post_cell_raises_for_every_aggregation(self, fitted, panel, aggregation): + @pytest.mark.parametrize("level", ["twfe", "overall", "simple"]) + def test_missing_post_cell_raises_for_every_type(self, fitted, panel, level): frame = _gt_frame(fitted) frame = frame[~((frame["group"] == 3) & (frame["time"] == 3))] with pytest.raises(ValueError, match="required cell\\(s\\) are missing"): - _frame_call(frame, panel, aggregation=aggregation) + _frame_call(frame, panel, type=level) def test_missing_pre_cell_raises_for_twfe_but_warns_for_cs_estimands(self, fitted, panel): frame = _gt_frame(fitted) frame = frame[~((frame["group"] == 4) & (frame["time"] == 1))] with pytest.raises(ValueError, match="complete cohort x period grid"): - _frame_call(frame, panel, aggregation="twfe") - complete = _frame_call(_gt_frame(fitted), panel, aggregation="overall") - for aggregation in ("overall", "simple"): + _frame_call(frame, panel, type="twfe") + complete = _frame_call(_gt_frame(fitted), panel, type="overall") + for level in ("overall", "simple"): # An ABSENT pre row is not a drop: nothing to warn about, weights unchanged. - partial = _frame_call(frame, panel, aggregation=aggregation) + partial = _frame_call(frame, panel, type=level) assert partial.n_dropped_cells == 0 assert len(partial.weights) == len(frame) - ref = _frame_call(_gt_frame(fitted), panel, aggregation=aggregation) + ref = _frame_call(_gt_frame(fitted), panel, type=level) assert partial.implied_att == pytest.approx(ref.implied_att, abs=1e-15) # A NaN pre cell (present but non-estimable) is what n_dropped_cells counts. frame = _gt_frame(fitted) frame.loc[frame.index[(frame["group"] == 4) & (frame["time"] == 1)][0], "effect"] = np.nan with pytest.warns(UserWarning, match="pre-treatment group-time cell"): - dropped = _frame_call(frame, panel, aggregation="overall") + dropped = _frame_call(frame, panel, type="overall") assert dropped.n_dropped_cells == 1 assert dropped.implied_att == pytest.approx(complete.implied_att, abs=1e-15) @@ -773,12 +773,12 @@ def test_first_period_cohort_is_dropped_like_r_did(self): df = _panel(cohorts=(0, 1, 3, 4), n_periods=5) fit = _fit(df) with pytest.warns(UserWarning, match="no estimable post-treatment cell"): - result = attgt_weights(fit, aggregation="overall") + result = attgt_weights(fit, type="overall") assert 1 not in set(result.weights["group"]) assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) # Same numbers as fitting on the panel with those units removed up front. pre_filtered = _fit(df[df["first_treat"] != 1]) - reference = attgt_weights(pre_filtered, aggregation="overall") + reference = attgt_weights(pre_filtered, type="overall") np.testing.assert_allclose( result.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-12 ) @@ -787,13 +787,13 @@ def test_bare_frame_missing_a_whole_non_first_cohort_raises(self, fitted, panel) frame = _gt_frame(fitted).drop(columns=["skip_reason"], errors="ignore") frame = frame[frame["group"] != 4] with pytest.raises(ValueError, match="no post-treatment cell in the ATT"): - _frame_call(frame, panel, aggregation="overall") + _frame_call(frame, panel, type="overall") def test_not_yet_treated_carve_out_mirrors_aggte(self): df = _panel(cohorts=(3, 4, 5), n_periods=6) fit = _fit(df, control_group="not_yet_treated") with pytest.warns(UserWarning) as record: - result = attgt_weights(fit, aggregation="overall") + result = attgt_weights(fit, type="overall") messages = " | ".join(str(w.message) for w in record) assert "structurally absent" in messages # (3,5),(3,6),(4,5),(4,6) assert "no estimable post-treatment cell" in messages # cohort 5 @@ -810,7 +810,7 @@ def test_not_yet_treated_carve_out_mirrors_aggte(self): expected = float((post["weight"] * post["att"]).sum()) assert result.implied_att == pytest.approx(expected, abs=1e-12) with pytest.raises(ValueError, match="control_group='never_treated'"): - attgt_weights(fit, aggregation="twfe") + attgt_weights(fit, type="twfe") class TestWeightValidation: @@ -828,15 +828,15 @@ class TestWeightValidation: def test_bad_unit_weights_are_rejected(self, fitted, mutate, match): w = mutate(np.ones(len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]))) with pytest.raises(ValueError, match=match): - attgt_weights(fitted, aggregation="overall", weights=w) + attgt_weights(fitted, type="overall", weights=w) def test_zero_control_mass_only_matters_where_controls_enter(self, fitted, panel): cohorts = np.asarray(fitted._aggregation_kit.bookkeeping["unit_cohorts"], dtype=float) w = np.where(cohorts == 0, 0.0, 1.0) with pytest.raises(ValueError, match="never-treated comparison group carries zero"): - attgt_weights(fitted, aggregation="twfe", weights=w) - for aggregation in ("overall", "simple"): - assert attgt_weights(fitted, aggregation=aggregation, weights=w).n_cells > 0 + attgt_weights(fitted, type="twfe", weights=w) + for level in ("overall", "simple"): + assert attgt_weights(fitted, type=level, weights=w).n_cells > 0 def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): df = panel.copy() @@ -856,14 +856,14 @@ def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): class TestNegativePostWeights: """Item 9: the pathology is negative weight on POST cells.""" - @pytest.mark.parametrize("aggregation", ["overall", "simple"]) - def test_cs_estimands_have_no_negative_post_weight(self, fitted, aggregation): - result = attgt_weights(fitted, aggregation=aggregation) + @pytest.mark.parametrize("level", ["overall", "simple"]) + def test_cs_estimands_have_no_negative_post_weight(self, fitted, level): + result = attgt_weights(fitted, type=level) assert result.n_negative_post == 0 assert result.negative_post_weight_share == 0.0 def test_twfe_reports_both_labelled(self, fitted): - result = attgt_weights(fitted, aggregation="twfe") + result = attgt_weights(fitted, type="twfe") assert result.n_negative_post <= result.n_negative assert 0.0 <= result.negative_post_weight_share <= 1.0 d = result.to_dict() @@ -872,7 +872,7 @@ def test_twfe_reports_both_labelled(self, fitted): class TestWeightedTwfeExtension: - """Item 17: weighted aggregation="twfe" has no R counterpart; tie it to the decomposition.""" + """Item 17: weighted type="twfe" has no R counterpart; tie it to the decomposition.""" def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): df = panel.copy() @@ -883,7 +883,7 @@ def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): ) df["w"] = df["unit"].map(unit_w) fit = _fit(df) - weighted = attgt_weights(fit, aggregation="twfe", weights=unit_w.to_numpy()) + weighted = attgt_weights(fit, type="twfe", weights=unit_w.to_numpy()) decomposed = diff_diff.decompose_twfe_weights( df, outcome="outcome", @@ -914,7 +914,7 @@ def test_implied_att_equals_hand_computed_overall(self): for t in (1, 2, 3): rows.append({"unit": u, "period": t, "first_treat": g, "outcome": 0.0}) panel = pd.DataFrame(rows) - result = _frame_call(frame, panel, aggregation="overall") + result = _frame_call(frame, panel, type="overall") # pbar_2 = 10/40, pbar_3 = 30/40; cohort 2 has 2 post periods, cohort 3 has 1. expected = (10 / 40) / 2 * (1.0 + 2.0) + (30 / 40) / 1 * 4.0 assert result.implied_att == pytest.approx(expected, abs=1e-15) @@ -1004,7 +1004,7 @@ def test_identities_hold(self, panel, fitted): result.decomposition, abs=1e-12 ) assert result.remainder == 0.0 - assert attgt_weights(fitted, aggregation="twfe").implied_att == pytest.approx( + assert attgt_weights(fitted, type="twfe").implied_att == pytest.approx( result.estimate, abs=1e-6 ) gmin1 = diff_diff.decompose_twfe_weights(panel, base_period="gmin1", **self.COMMON) @@ -1025,7 +1025,7 @@ def _agg_backend(self): plt.close("all") def test_weights_view(self, fitted): - result = attgt_weights(fitted, aggregation="twfe") + result = attgt_weights(fitted, type="twfe") ax = diff_diff.plot_twfe_weights(result, show=False) assert ax.get_xlabel() == "Implicit weight" # Item 9: the title counts POST-only negatives. This panel has negative @@ -1039,7 +1039,7 @@ def test_weights_view(self, fitted): def test_weights_title_reports_post_only_count(self): """A panel WITH negative post weight labels the post-only count.""" fit = _fit(_panel(cohorts=(0, 2, 4))) - result = attgt_weights(fit, aggregation="twfe") + result = attgt_weights(fit, type="twfe") assert result.n_negative_post > 0 ax = diff_diff.plot_twfe_weights(result, show=False) assert f"({result.n_negative_post} negative post)" in ax.get_title() @@ -1049,7 +1049,7 @@ def test_ax_reuse_and_annotate(self, fitted): _, ax = plt.subplots() out = diff_diff.plot_twfe_weights( - attgt_weights(fitted, aggregation="overall"), ax=ax, annotate=True, show=False + attgt_weights(fitted, type="overall"), ax=ax, annotate=True, show=False ) assert out is ax assert len(ax.texts) == 10 @@ -1158,28 +1158,28 @@ def _group_overall(fit, panel): @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) def test_simple_matches_aggregate(self, panel, control_group): fit = self._fit(panel, 1, control_group) - got = attgt_weights(fit, aggregation="simple").implied_att + got = attgt_weights(fit, type="simple").implied_att assert got == pytest.approx(fit.aggregate("simple").att[0], abs=1e-12) @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) def test_overall_matches_group_combination(self, panel, control_group): fit = self._fit(panel, 1, control_group) - got = attgt_weights(fit, aggregation="overall").implied_att + got = attgt_weights(fit, type="overall").implied_att assert got == pytest.approx(self._group_overall(fit, panel), abs=1e-12) def test_twfe_ignores_anticipation(self, panel): - ant0 = attgt_weights(self._fit(panel, 0, "never_treated"), aggregation="twfe") - ant1 = attgt_weights(self._fit(panel, 1, "never_treated"), aggregation="twfe") + ant0 = attgt_weights(self._fit(panel, 0, "never_treated"), type="twfe") + ant1 = attgt_weights(self._fit(panel, 1, "never_treated"), type="twfe") np.testing.assert_allclose( ant1.weights["weight"].to_numpy(), ant0.weights["weight"].to_numpy(), atol=1e-12 ) def test_frame_path_takes_explicit_anticipation(self, panel): fit = self._fit(panel, 1, "never_treated") - from_fit = attgt_weights(fit, aggregation="simple") + from_fit = attgt_weights(fit, type="simple") from_frame = attgt_weights( fit.to_dataframe("group_time"), - aggregation="simple", + type="simple", data=panel, unit="unit", time="period", @@ -1210,7 +1210,7 @@ def test_unbalanced_fitted_result_is_rejected(self): ).fit(broken, **_DECO) assert fit._aggregation_kit.bookkeeping["is_balanced"] is False with pytest.raises(ValueError, match="balanced panel"): - attgt_weights(fit, aggregation="twfe") + attgt_weights(fit, type="twfe") def test_balanced_fit_records_the_flag(self, fitted): assert fitted._aggregation_kit.bookkeeping["is_balanced"] is True @@ -1257,11 +1257,11 @@ def test_paths_agree_on_a_not_yet_treated_fit(self): df = _panel(cohorts=(3, 4, 5), n_periods=6) fit = _fit(df, control_group="not_yet_treated") with pytest.warns(UserWarning, match="structurally absent"): - from_fit = attgt_weights(fit, aggregation="overall") + from_fit = attgt_weights(fit, type="overall") with pytest.warns(UserWarning, match="structurally absent"): from_frame = attgt_weights( fit.to_dataframe("group_time"), - aggregation="overall", + type="overall", data=df, unit="unit", time="period", @@ -1285,7 +1285,7 @@ def test_fitted_mid_cohort_without_a_reason_raises(self): cell["effect"] = np.nan cell["skip_reason"] = None with pytest.raises(ValueError, match="do not all carry"): - attgt_weights(fit, aggregation="overall") + attgt_weights(fit, type="overall") def test_frame_mid_cohort_without_a_reason_raises(self, fitted, panel): frame = _gt_frame(fitted).copy() @@ -1293,7 +1293,7 @@ def test_frame_mid_cohort_without_a_reason_raises(self, fitted, panel): frame.loc[mask, "effect"] = np.nan frame.loc[mask, "skip_reason"] = None with pytest.raises(ValueError, match="do not all carry"): - _frame_call(frame, panel, aggregation="overall") + _frame_call(frame, panel, type="overall") class TestWeightsLength: @@ -1303,7 +1303,7 @@ def test_wrong_length_with_an_excluded_cohort(self): fit = _fit(_panel(cohorts=(0, 1, 3, 4))) n_units = len(fit._aggregation_kit.bookkeeping["unit_cohorts"]) with pytest.raises(ValueError, match="weights has length"): - attgt_weights(fit, aggregation="overall", weights=np.ones(n_units - 1)) + attgt_weights(fit, type="overall", weights=np.ones(n_units - 1)) class TestMultiCovariatePin: diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py index 777257912..f29eda096 100644 --- a/tests/test_twfe_weights_parity.py +++ b/tests/test_twfe_weights_parity.py @@ -34,7 +34,7 @@ # Closed-form weights: both sides evaluate the same rational expression in # cohort masses in double precision, so only representation error separates -# them. Observed max deviation across all 3 fixtures x 3 aggregations is +# them. Observed max deviation across all 3 fixtures x 3 estimand types is # 4.7e-16 - two orders of margin below this gate. WEIGHT_ATOL = 1e-12 WEIGHT_RTOL = 0.0 @@ -109,16 +109,16 @@ class TestATTGTWeightsParity: """ @pytest.mark.parametrize("fixture", FIXTURES) - @pytest.mark.parametrize("aggregation", AGGREGATIONS) - def test_weight_column(self, golden, fixture, aggregation): + @pytest.mark.parametrize("level", AGGREGATIONS) + def test_weight_column(self, golden, fixture, level): payload, df = _fixture(golden, fixture) - expected = _sorted_golden_weights(payload["attgt_weights"][aggregation]) + expected = _sorted_golden_weights(payload["attgt_weights"][level]) gt_frame = expected[["group", "time", "att"]].rename(columns={"att": "effect"}) cols = payload["columns"] result = attgt_weights( gt_frame, - aggregation=aggregation, + type=level, data=df, unit=cols["unit"], time=cols["time"], @@ -138,16 +138,16 @@ def test_weight_column(self, golden, fixture, aggregation): ) @pytest.mark.parametrize("fixture", FIXTURES) - @pytest.mark.parametrize("aggregation", AGGREGATIONS) - def test_implied_att(self, golden, fixture, aggregation): + @pytest.mark.parametrize("level", AGGREGATIONS) + def test_implied_att(self, golden, fixture, level): payload, df = _fixture(golden, fixture) - block = payload["attgt_weights"][aggregation] + block = payload["attgt_weights"][level] expected = _sorted_golden_weights(block) cols = payload["columns"] result = attgt_weights( expected[["group", "time", "att"]].rename(columns={"att": "effect"}), - aggregation=aggregation, + type=level, data=df, unit=cols["unit"], time=cols["time"], @@ -171,11 +171,11 @@ class TestATTGTWeightsFromCSFit: """ @pytest.mark.parametrize("fixture", FIXTURES) - @pytest.mark.parametrize("aggregation", AGGREGATIONS) - def test_end_to_end(self, golden, fixture, aggregation): + @pytest.mark.parametrize("level", AGGREGATIONS) + def test_end_to_end(self, golden, fixture, level): payload, df = _fixture(golden, fixture) - block = payload["attgt_weights"][aggregation] - result = attgt_weights(_fit_cs(df, payload["columns"]), aggregation=aggregation) + block = payload["attgt_weights"][level] + result = attgt_weights(_fit_cs(df, payload["columns"]), type=level) actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) expected = _sorted_golden_weights(block) @@ -205,7 +205,7 @@ def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): payload, df = _fixture(golden, fixture) fit = _fit_cs(df, payload["columns"]) - twfe = attgt_weights(fit, aggregation="twfe") + twfe = attgt_weights(fit, type="twfe") assert twfe.n_negative > 0 assert twfe.negative_weight_share > 0 @@ -221,8 +221,8 @@ def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): float(np.abs(g_w[neg_post]).sum() / abs_post), abs=1e-12 ) - for aggregation in ("overall", "simple"): - benign = attgt_weights(fit, aggregation=aggregation) + for level in ("overall", "simple"): + benign = attgt_weights(fit, type=level) assert benign.n_negative == 0 assert benign.negative_weight_share == 0.0 assert benign.n_negative_post == 0 @@ -233,8 +233,8 @@ def test_target_estimand_weights_sum_to_one(self, golden, fixture): """ATT^O and ATT^simple are proper averages of the post cells.""" payload, df = _fixture(golden, fixture) fit = _fit_cs(df, payload["columns"]) - for aggregation in ("overall", "simple"): - weights = attgt_weights(fit, aggregation=aggregation).weights + for level in ("overall", "simple"): + weights = attgt_weights(fit, type=level).weights np.testing.assert_allclose(weights["weight"].sum(), 1.0, atol=1e-12) @@ -242,16 +242,16 @@ class TestCSFitAndFrameAgree: """The DataFrame fallback reproduces the fitted-result path exactly.""" @pytest.mark.parametrize("fixture", FIXTURES) - @pytest.mark.parametrize("aggregation", AGGREGATIONS) - def test_paths_agree(self, golden, fixture, aggregation): + @pytest.mark.parametrize("level", AGGREGATIONS) + def test_paths_agree(self, golden, fixture, level): payload, df = _fixture(golden, fixture) cols = payload["columns"] fit = _fit_cs(df, cols) - from_fit = attgt_weights(fit, aggregation=aggregation) + from_fit = attgt_weights(fit, type=level) from_frame = attgt_weights( fit.to_dataframe("group_time"), - aggregation=aggregation, + type=level, data=df, unit=cols["unit"], time=cols["time"], @@ -605,7 +605,7 @@ def test_twfe_weights_reproduce_the_decomposition(self, golden, fixture): payload, df = _fixture(golden, fixture) cols = payload["columns"] - weighted = attgt_weights(_fit_cs(df, cols), aggregation="twfe") + weighted = attgt_weights(_fit_cs(df, cols), type="twfe") decomposed = _decompose(df, cols, covariates=None) assert weighted.implied_att == pytest.approx(decomposed.estimate, abs=1e-6) From 0d817e15a11dc29b59304b7be7c00ac140424b50 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 23 Sep 2026 17:27:21 -0400 Subject: [PATCH 09/13] fix(twfeweights): raw-time anticipation, skip_reason-keyed structural rule, producer-recorded completeness attgt_weights applied the anticipation window to POSITIONAL indices while CallawaySantAnna applies it to raw period labels, so on a gapped calendar (10, 20, 30, 40; anticipation=1) it flagged (20,10) as post and returned 1.5230 against aggregate("simple") 2.2844. The window now counts raw time units on the post mask, the required grid, the available-cell divisors and the structural rule; "twfe" keeps t >= g for its own post window. The whole-cohort drop and the incomplete-grid carve-out are now keyed on the estimator's own skip_reason set {missing_period, zero_treated_control, zero_weight_mass} (the cells aggregate() finite-masks) instead of zero_treated_control alone plus a positional first-period shortcut, so a cohort CS could not estimate at all under the window is dropped for every type (as did::pre_process_did drops it) and a cohort kept through a single estimable cell is averaged over what it has (as aggregate() keeps it; a documented deviation from R's drop under base_period="varying"). A cohort blanked without a reason still fails closed under any window. Bare frames keep only the first-period shortcut, in raw time. Panel completeness on the aggregation kit is now the producer's own record: every outcome cell finite AND no unit dropped by the estimator's per-cell complete-case mask (DMLDiD sets complete_case_drops from the units its cell loops excluded; CS's NaN-covariate fallback keeps masses intact). Kits without the record or the covariates record (pickled by <= 3.12.0) fail closed with a refit message instead of warn-then-continue. decompose_twfe_weights reports the ORIGINAL time labels on every surface (cells, summary, balance rows, plots); the numeric key stays internal, a column mixing two spellings of one period is rejected, and the frame path detects duplicate cells on the numeric keys so "3" and 3.0 cannot be double-counted. Docs: REGISTRY equations in label form, the structural-rule / completeness / anticipation / canonical-key Notes rewritten (R did 2.5.1 pre_process_did provenance stamped), identity conditions tightened, guide and changelog updated. Tests cover the structural drop for all three types, the varying-base keep, reason-less blanks under wide windows, gapped calendars against aggregate(), zero_weight_mass, the bare-frame shortcut, DML complete-case drops (incl. the bad-control lane), CS inf outcomes, the legacy-kit refusal for both producers, label round-trips, and the weighted identity with the same survey weights on both sides. --- .../20260831-twfe-weight-diagnostics.md | 22 +- diff_diff/dml_did.py | 5 + diff_diff/guides/llms-full.txt | 21 +- diff_diff/staggered.py | 20 +- diff_diff/twfe_weights.py | 309 ++++++++----- diff_diff/twfe_weights_results.py | 16 +- docs/methodology/REGISTRY.md | 24 +- tests/test_twfe_weights.py | 419 +++++++++++++++++- 8 files changed, 691 insertions(+), 145 deletions(-) diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index c72520b26..8d9f8847e 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -6,8 +6,10 @@ weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), plus post-period negative-weight counts. Returns `ATTGTWeightsResult` (`.level` records the type). - The CS estimands honour the fit's `anticipation` window (an explicit - `anticipation=` on the frame path); `"twfe"` keeps `t >= g`. + The CS estimands honour the fit's `anticipation` window in raw time units + (an explicit `anticipation=` on the frame path); `"twfe"` keeps `t >= g`, + but a cohort with no estimable post cell under the window is dropped for + every type. - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate from its ATT(g,t) building blocks and returns `TWFEDecompositionResult` with `pre_period_contribution` - the sample contribution of the @@ -16,10 +18,12 @@ covariate balance. `plot_twfe_weights()` renders either view (matplotlib or plotly). - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits - under `type="twfe"`, unbalanced panels, non-finite outcomes / - covariates, duplicated or non-finite ATT(g,t) cells, an incomplete - group-time grid, and invalid sampling weights. Two structural gaps are - handled as R does instead of raising: a cohort with no estimable post cell - is dropped (`did`'s first-period drop, or running out of comparison - units), and under `control_group="not_yet_treated"` the CS estimands - average over each cohort's available post periods (`aggte`). + under `type="twfe"`, unbalanced panels, fitted results that dropped any + unit by their per-cell complete-case rules or that carry no completeness + record (fitted by <= 3.12.0), non-finite outcomes / covariates, + duplicated or non-finite ATT(g,t) cells, an incomplete group-time grid, + and invalid sampling weights. Gaps the estimator itself could not fill + (`skip_reason` missing_period / zero_treated_control / zero_weight_mass) + are handled as `aggregate()` does instead of raising: a cohort with no + estimable post cell under the window is dropped, and the CS estimands + average over each surviving cohort's available post cells (`aggte`). diff --git a/diff_diff/dml_did.py b/diff_diff/dml_did.py index bb2d17a15..f6020ff89 100644 --- a/diff_diff/dml_did.py +++ b/diff_diff/dml_did.py @@ -3352,6 +3352,11 @@ def fit( bad_control_covariates=(tuple(w_names) if w_names is not None else None), bad_control_diagnostics=(bad_control_diagnostics if bad_control_diagnostics else None), ) + # Every unit a cell loop excluded by its complete-case mask (outcome, + # dY, base-period covariates, bad-control columns, W) lands in + # ``dropped_units``; the kit records whether that ever happened so + # ``attgt_weights`` can refuse a fit whose cohort masses are reduced. + precomputed["complete_case_drops"] = bool(dropped_units) results._aggregation_kit = _build_aggregation_kit( cast(Any, self), # duck-typed host contract (alpha/anticipation/cband) precomputed, diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 3a84d4134..d5c716f5f 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1575,13 +1575,20 @@ Both entry points fail closed on input R never faced: NaN / `-inf` cohort labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x period cell, the CS estimands every post cell), and sampling weights that are -not finite, non-negative and positive-mass. Two structural gaps mirror R -rather than raising, each with a `UserWarning`: a cohort with no estimable -post cell is dropped from the table and the cohort shares (`did`'s -first-period drop), and under `control_group="not_yet_treated"` the cells CS -marks `zero_treated_control` are treated as structurally absent, so -`"overall"`/`"simple"` average over each cohort's AVAILABLE post periods -(`aggte`). `n_negative_post` / `negative_post_weight_share` report the +not finite, non-negative and positive-mass. A fitted result is rejected +unless every unit-period outcome cell is present and finite and the estimator +dropped no unit by its own complete-case rules (DMLDiD records this; results +pickled by <= 3.12.0 carry no record and are refused). Gaps the estimator +itself could not fill are handled rather than raised, each with a +`UserWarning`, keyed on the cells' `skip_reason` (`missing_period`, +`zero_treated_control`, `zero_weight_mass`): a cohort with no estimable post +cell under the anticipation window is dropped from the table and the cohort +shares for every `type` (`did`'s first-period drop), and a cohort that keeps +one estimable post cell has its absent cells carved out, so `"overall"` / +`"simple"` average over each cohort's AVAILABLE post cells (`aggte`, +`aggregate()`). The window counts raw time units, as CallawaySantAnna does; +`"twfe"`'s own post window stays `t >= g`. `n_negative_post` / +`negative_post_weight_share` report the pathology (negative weight on POST cells); `n_negative` counts pre cells too, and is near-half in every staggered design because the TWFE weights sum to zero over the full grid. diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 48fa874e6..3ff685562 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -5192,11 +5192,21 @@ def _build_aggregation_kit( # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES # only - never values - so the data-minimization contract holds. bookkeeping["covariates"] = tuple(covariates or ()) - # Panel balance, recorded so ``attgt_weights`` can reject an unbalanced - # fitted result: its cohort shares and E_t[D] assume the same units in - # every period. Defaults True when the key is absent (a legacy kit, or a - # producer that never computed it). - bookkeeping["is_balanced"] = bool((precomputed or {}).get("is_balanced", True)) + # Panel completeness, recorded so ``attgt_weights`` can reject a fitted + # result whose cohort shares and E_t[D] are not those of a fixed unit set: + # every unit-period outcome cell present and finite, AND the producer + # dropped no unit from any cell by its own complete-case mask (DMLDiD + # sets ``complete_case_drops``; CS's NaN-covariate fallback keeps the + # masses intact and sets nothing). False for RCS precomputes (no outcome + # matrix). Stricter than CallawaySantAnna's own ``is_balanced`` (an + # ``isnan``-only check for its fast paths), which stays as it is. + _pre = precomputed or {} + _om = _pre.get("outcome_matrix") + bookkeeping["is_balanced"] = bool( + _om is not None + and np.isfinite(np.asarray(_om, dtype=float)).all() + and not _pre.get("complete_case_drops", False) + ) # Data minimization: the results object is picklable and users share # result artifacts, so the kit must not turn it into a carrier for raw diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index 082ffc7ce..08d382f91 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -6,8 +6,9 @@ coefficient need not lie in the convex hull of the effects it summarizes. :func:`attgt_weights` reports those weights, next to the weights the target estimands ATT^O and ATT^simple would use. :func:`decompose_twfe_weights` -re-derives the regression from its building blocks and separates the part -driven by pre-treatment parallel-trends violations. +re-derives the regression from its building blocks and separates the sample +contribution of the pre-treatment cells, which can reflect differential +pre-trends or sampling variation. Distinct from :func:`diff_diff.twowayfeweights`, which implements the de Chaisemartin & D'Haultfoeuille (2020) Theorem 1 decomposition: that one @@ -70,6 +71,14 @@ _TYPES = ATTGTWeightsResult.LEVELS +# ``skip_reason`` values CS / DMLDiD emit for cells they structurally could not +# form: no base period under the anticipation window (``missing_period``), no +# comparison units (``zero_treated_control``), or zero survey mass +# (``zero_weight_mass``). A cohort whose only gaps carry these reasons is +# handled the way ``aggregate()`` handles it - dropped when it has no estimable +# post cell, otherwise averaged over the cells it has; any other gap fails closed. +_STRUCTURAL_REASONS = frozenset({"missing_period", "zero_treated_control", "zero_weight_mass"}) + def _is_never(values: np.ndarray) -> np.ndarray: """Boolean mask for never-treated cohort labels. @@ -264,14 +273,16 @@ def _overall_weight_vector( post_mask: np.ndarray, n_post_available: Dict[int, int], ) -> np.ndarray: - """ATT^O weights: ``1[t >= g - anticipation] * pbar_g / n_post_g``. - - Not renormalized - the per-cohort divisor is each cohort's number of - AVAILABLE post periods, so the weights already sum to one (the - ``pbar_g`` sum to one over cohorts). Counting the available cells rather - than writing ``(maxT - g + 1)`` analytically is what makes the - anticipation window and the structurally-absent ``zero_treated_control`` - cells come out right, and the two agree exactly on a complete grid. + """ATT^O weights: ``1[label(t) >= label(g) - a] * pbar_g / n_post(g)``. + + Not renormalized - the per-cohort divisor ``n_post(g)`` is each cohort's + number of AVAILABLE post cells under the anticipation window ``a`` (raw + time units), so the weights already sum to one (the ``pbar_g`` sum to one + over cohorts). On a complete grid that is ``maxT - post_start(g) + 1``, + which reduces to R's ``(maxT - g + 1)`` only when ``a == 0``; counting the + available cells rather than writing it analytically is what makes the + window and the structurally-absent cells (no comparison units, no base + period, zero survey mass) come out right. """ divisor = np.array([float(n_post_available[int(g)]) for g in groups]) return post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor @@ -282,7 +293,11 @@ def _simple_weight_vector( p_treated: Dict[int, float], post_mask: np.ndarray, ) -> np.ndarray: - """ATT^simple weights: ``1[t >= g - anticipation] * pbar_g``, normalized.""" + """ATT^simple weights: ``1[label(t) >= label(g) - a] * pbar_g``, normalized. + + ``a`` is the anticipation window in raw time units; the vector + self-normalizes, so no per-cohort divisor is needed. + """ raw = post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) total = raw.sum() if total == 0: @@ -328,9 +343,11 @@ def _attgt_from_frame( ``CallawaySantAnnaResults.to_dataframe("group_time")`` emits - so the fallback consumes our own frame verbatim, including its ``skip_reason`` column when present. Duplicate cells and non-finite ``group`` / ``time`` - labels are rejected; a non-finite effect is reported in the skip map, not - silently kept (an ``inf`` ATT would otherwise propagate into - ``implied_att``). + labels are rejected - duplicates are detected on the NUMERIC keys, so two + spellings of one period (``3`` and ``"3"``) are the same cell, exactly as + the positional grid treats them; a non-finite effect is reported in the + skip map, not silently kept (an ``inf`` ATT would otherwise propagate + into ``implied_att``). """ missing = {"group", "time"} - set(frame.columns) if missing: @@ -350,12 +367,13 @@ def _attgt_from_frame( times = pd.to_numeric(frame["time"], errors="coerce").to_numpy(dtype=float) _validate_cohort_labels(groups, what="group") _validate_time_labels(frame["time"].to_numpy(), what="time") - key = pd.MultiIndex.from_arrays([frame["group"].to_numpy(), frame["time"].to_numpy()]) + key = pd.MultiIndex.from_arrays([groups, times]) if key.duplicated().any(): dupes = sorted({tuple(k) for k in key[key.duplicated()].tolist()})[:5] raise ValueError( f"ATT(g,t) frame has duplicated (group, time) cell(s) {dupes!r}; each " - "cell must appear exactly once" + "cell must appear exactly once (labels compare as numbers, so '3' and " + "3.0 name the same cell)" ) att = pd.to_numeric(frame[value_col], errors="coerce").to_numpy(dtype=float) reasons = ( @@ -449,10 +467,13 @@ def _resolve_cs_inputs( ) weights = bookkeeping.get("survey_weights") anticipation = int(getattr(kit, "anticipation", 0) or 0) - # Default True: DMLDiD builds its kit through the same builder but its - # precomputed mapping predates the balance flag, and a false rejection - # would be worse than the (already-CS-only) check. - is_balanced = bool(bookkeeping.get("is_balanced", True)) + # Fail closed on a kit without the completeness record: a result pickled + # by diff-diff <= 3.12.0 cannot say whether its cohort masses are those of + # a complete panel, and a silently wrong weight table is worse than a + # refit. (Same policy as the missing-`covariates` key in _guard_cs_design.) + if "is_balanced" not in bookkeeping: + raise ValueError(_LEGACY_KIT_MESSAGE) + is_balanced = bool(bookkeeping["is_balanced"]) return ( np.asarray(cohorts), (None if weights is None else np.asarray(weights, dtype=float)), @@ -461,6 +482,13 @@ def _resolve_cs_inputs( ) +_LEGACY_KIT_MESSAGE = ( + "this result was fitted by diff-diff <= 3.12.0 and does not record the " + "bookkeeping attgt_weights needs (panel completeness and covariate usage); " + "refit it to use attgt_weights" +) + + def _guard_cs_design(results: "CallawaySantAnnaResults", estimand: str) -> None: """Reject fits whose design breaks the weight formulas. @@ -505,22 +533,17 @@ def _guard_cs_design(results: "CallawaySantAnnaResults", estimand: str) -> None: "not report. Refit with base_period='universal'." ) # R's third restriction: xformla == ~1. The fit records its covariate - # column names on the estimand kit; a kit without the key predates that - # bookkeeping (an old pickle) and can only be warned about. A missing kit - # is left to _resolve_cs_inputs, whose error is the useful one. + # column names on the aggregation kit; a kit without the key predates that + # bookkeeping (a result pickled by <= 3.12.0) and fails closed, the same + # policy as the missing completeness record in _resolve_cs_inputs. A + # missing kit is left to _resolve_cs_inputs, whose error is the useful one. kit = getattr(results, "_aggregation_kit", None) if kit is None: return bookkeeping = getattr(kit, "bookkeeping", {}) or {} if "covariates" not in bookkeeping: - warnings.warn( - "this fit predates covariate bookkeeping, so attgt_weights cannot " - "verify it used no covariates; the TWFE weight formula assumes an " - "unadjusted regression (R twfe_weights requires xformla == ~1)", - UserWarning, - stacklevel=3, - ) - elif bookkeeping["covariates"]: + raise ValueError(_LEGACY_KIT_MESSAGE) + if bookkeeping["covariates"]: raise ValueError( f"type='twfe' requires a fit without covariates, but this one " f"adjusted for {list(bookkeeping['covariates'])!r}. The TWFE weight " @@ -559,9 +582,13 @@ def attgt_weights( ``result.to_dataframe("group_time")`` is consumed verbatim, including its ``skip_reason`` column. On the frame path, ``data``, ``unit``, ``time`` and ``first_treat`` are required so cohort shares can be - formed, and the caller is responsible for the fit having used no - covariates under ``type="twfe"`` (a frame carries no record of - that; the fitted path checks it). + formed. A frame carries no record of the producing fit, so on this + path the caller is responsible for what the fitted path checks: the + fit used no covariates under ``type="twfe"``; it dropped no unit from + any cell by its own complete-case rules; ``anticipation=`` is the + window the fit used; and no cell was hand-built for a cohort the + estimator would not have estimated. None of these can be detected + from the frame. type : {"twfe", "overall", "simple"}, default "twfe" Which estimand's weights to report (the same keyword as ``results.aggregate(type=...)``; the accepted values are @@ -578,14 +605,19 @@ def attgt_weights( weights, which take precedence. Must be finite and non-negative with positive treated mass (and positive never-treated mass for ``"twfe"``). anticipation : int, optional - Anticipation window for the CS estimands, i.e. the number of periods - before ``g`` whose cells count as post-treatment (``t >= g - - anticipation``). Only meaningful on the DataFrame path, where a bare - frame carries no record of the source fit's setting; the fitted path - reads it off the fit and rejects an explicit ``anticipation=``. It does - NOT affect ``type="twfe"``: the TWFE regression's own treatment - indicator is ``1[t >= g]`` regardless of how the CS estimands treat the - run-up, and R's ``twfe_weights`` has no anticipation argument either. + Anticipation window for the CS estimands, in the calendar's own time + units exactly as ``CallawaySantAnna(anticipation=)`` counts them: a + cell counts as post-treatment when ``time >= group - anticipation`` + on the raw labels (on a calendar 10, 20, 30 an ``anticipation=10`` + shifts the window by one period; ``anticipation=1`` shifts it by + none). Only accepted on the DataFrame path, where it must be the value + the producing fit used; the fitted path reads it off the fit and + rejects an explicit ``anticipation=``. It does not change + ``type="twfe"``'s post window, which stays ``1[t >= g]`` (the + regression's own indicator; R's ``twfe_weights`` has no anticipation + argument) - but a cohort the estimator could not estimate at all under + the window is dropped for every ``type``, and the remaining cohorts' + masses and weights are those of the panel without it. Returns ------- @@ -600,38 +632,58 @@ def attgt_weights( ``type="twfe"`` - a non-never-treated control group, a non-universal base period, or a covariate-adjusted fit); on NaN / ``-inf`` cohort labels, invalid weights, duplicated or non-finite - cells; or on an INCOMPLETE grid: ``"twfe"`` needs every cohort x period - cell, ``"overall"`` / ``"simple"`` every post-treatment cell. + cells; on a fitted result that carries no completeness / covariate + bookkeeping (pickled by diff-diff <= 3.12.0); or on an INCOMPLETE + grid: ``"twfe"`` needs every cohort x period cell, ``"overall"`` / + ``"simple"`` every cell in the anticipation window that the estimator + did not itself mark structurally absent (see Notes). TypeError When ``results`` is neither a CallawaySantAnna-family result nor a DataFrame. Notes ----- - Two structural gaps are handled rather than raised, mirroring R: - - * A cohort with NO estimable post-treatment cell (typically one treated in - the first observed period, which has no base period) is dropped from the - table AND from the cohort masses with a warning - what + Gaps the estimator itself could not fill are handled rather than raised, + the way ``results.aggregate()`` handles them. A cell is *structurally + absent* when its ``skip_reason`` is one of ``missing_period`` (no base + period under the anticipation window - R ``did``'s first-period drop), + ``zero_treated_control`` (no comparison units, as under + ``control_group="not_yet_treated"`` for the last cohorts) or + ``zero_weight_mass`` (zero survey mass in the cell). Then: + + * A cohort with NO estimable post-treatment cell, every missing post cell + of which is structurally absent, is dropped from the table AND from the + cohort masses with a warning, for every ``type`` - what ``did::pre_process_did`` does when it drops units already treated in the - first period. This drop is allowed ONLY when the cohort is treated in the - first observed period, or when every one of its missing post cells - carries ``skip_reason="zero_treated_control"``; a mid cohort blanked out - by some other mechanism raises rather than disappearing. - * Under ``control_group="not_yet_treated"`` the last cohorts run out of - comparison units, and CS marks those post cells ``zero_treated_control``. - For ``"overall"`` / ``"simple"`` they are treated as structurally absent: - ``"overall"`` divides each cohort by its number of AVAILABLE post periods - and ``"simple"`` renormalizes over the available post cells - what - R ``aggte()`` computes on such a fit. A warning names the cells. The - carve-out keys on the ``skip_reason`` values themselves (that reason is - only ever emitted on a not-yet-treated fit), so the fitted and frame - paths behave identically. (``"twfe"`` requires a never-treated control - group and never reaches this branch.) - - Cohort shares assume a BALANCED panel - the same units observed in every - period - so an unbalanced fitted result is rejected (as - :func:`decompose_twfe_weights` already rejects an unbalanced panel). + first period (or within the anticipation window). The remaining cohorts' + masses and weights are those of the panel without it. A cohort blanked + out any other way (NaN effects with no reason) raises rather than + disappearing. + * A cohort that keeps at least one estimable post cell is kept, and for + ``"overall"`` / ``"simple"`` its structurally absent post cells are left + out of the grid: ``"overall"`` divides the cohort by its number of + AVAILABLE post cells and ``"simple"`` renormalizes over the available + cells - what R ``aggte()`` computes on a not-yet-treated fit, and what + ``aggregate()`` computes on a varying-base fit that keeps a cohort + through a single estimable cell. A warning names the cells and the + reasons. (``"twfe"`` requires a never-treated, universal-base fit and + never reaches this branch.) + + The classification keys on the ``skip_reason`` values, so a + ``to_dataframe("group_time")`` frame behaves exactly like the fitted + result. A bare frame WITHOUT ``skip_reason`` has only one structural + route: a cohort whose window starts at or before the first observed + period (``group - anticipation <= first period``, R's + ``g <= first.period + anticipation``) may be dropped; every other gap + fails closed, being indistinguishable from user truncation. + + Cohort shares assume a COMPLETE panel - the same units in every period - + so a fitted result is rejected unless every unit-period outcome cell is + present and finite and the estimator dropped no unit from any cell by its + own complete-case rules (DMLDiD records its drops; CallawaySantAnna's + NaN-covariate fallback keeps the cohort masses intact and is not a + rejection cause). :func:`decompose_twfe_weights` rejects an unbalanced + panel the same way. R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows with ``attgt = 0`` to mirror an internal vector layout; those rows are @@ -700,9 +752,12 @@ def attgt_weights( cohorts, survey_weights, window, is_balanced = _resolve_cs_inputs(results) if not is_balanced: raise ValueError( - "attgt_weights requires a balanced panel: the cohort shares and " - "E_t[D] assume the same units are observed in every period. " - "Balance the panel (diff_diff.balance_panel) and refit." + "attgt_weights requires a balanced panel: every unit-period " + "outcome cell present and finite, and no unit dropped by the " + "producing estimator's per-cell complete-case rules, because the " + "cohort shares and E_t[D] assume the same units in every period. " + "Balance the panel (diff_diff.balance_panel), clean the " + "non-finite cells, and refit." ) if survey_weights is not None and weights is not None: raise ValueError( @@ -748,35 +803,43 @@ def attgt_weights( g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + # Raw label of each position, 1-based (index 0 unused): the anticipation + # window is defined in the calendar's own units, as CallawaySantAnna + # applies it (``t < g - anticipation`` on labels), NOT in positions - on a + # gapped calendar the two differ. + label_of = np.array([np.nan] + sorted(grid)) + g_int = g_pos.astype(int) # Post-treatment mask. The TWFE regression's own indicator is 1[t >= g] # regardless of the CS anticipation window; the CS target estimands shift - # it to 1[t >= g - anticipation]. + # it to 1[label(t) >= label(g) - anticipation] in raw time units. if type == "twfe": post_mask = t_pos >= g_pos else: - post_mask = t_pos >= (g_pos - window) + post_mask = label_of[t_pos] >= label_of[g_int] - window + + def _cs_post_start(g: int) -> int: + """First position whose raw label is ``>= label(g) - window``; never below 1.""" + return int(np.searchsorted(label_of[1:], label_of[g] - window, side="left")) + 1 def _post_start(g: int) -> int: - raw = g if type == "twfe" else g - window - return max(1, raw) + return g if type == "twfe" else _cs_post_start(g) - # --- whole-cohort exclusion (R did drops units treated in the first period) + # --- whole-cohort exclusion (R did drops cohorts it cannot estimate at all) present = set(zip(g_pos.tolist(), t_pos.tolist())) panel_cohorts = sorted({int(g) for g in unit_g_pos if g != 0}) cohorts_with_post = {int(g) for g in g_pos[post_mask]} excluded = [g for g in panel_cohorts if g not in cohorts_with_post] if excluded: - # A cohort may be dropped ONLY when the drop is structural: it is - # treated in the first observed period (R did's first-period drop), or - # every one of its missing post cells carries zero_treated_control - # (it ran out of comparison units). Any other blanked-out cohort must - # fail closed rather than disappear. + # A cohort may be dropped ONLY when the drop is structural. With + # skip_reason available (fitted result, or its to_dataframe frame): + # every one of its missing post cells carries a reason the estimator + # itself emits for a cell it could not form (no base period under the + # window, no comparison units, zero survey mass). On a bare frame the + # only route is R did's first-period rule in raw time, g - anticipation + # <= first period. Any other blanked-out cohort fails closed. structural: List[int] = [] for g in excluded: - if g == first_period_pos: - structural.append(g) - continue if has_skip_reasons: missing_post = [ (_label_for(grid, g), _label_for(grid, t)) @@ -784,9 +847,11 @@ def _post_start(g: int) -> int: if (g, t) not in present ] if missing_post and all( - skipped.get(lab) == "zero_treated_control" for lab in missing_post + skipped.get(lab) in _STRUCTURAL_REASONS for lab in missing_post ): structural.append(g) + elif _cs_post_start(g) <= first_period_pos: + structural.append(g) not_structural = [g for g in excluded if g not in structural] if not_structural: labels = [_label_for(grid, g) for g in not_structural] @@ -800,18 +865,33 @@ def _post_start(g: int) -> int: raise ValueError( f"cohort(s) {labels!r} are present in data= but have no estimable " "post-treatment cell, and their missing post cell(s) do not all " - "carry skip_reason 'zero_treated_control'. A cohort is only dropped " - "like R did's first-period cohort, or when it runs out of comparison " - "units; blanking a mid cohort's effects is not one of those, so it " - "fails closed instead of silently leaving the estimand." + "carry a structural skip_reason (one of " + f"{sorted(_STRUCTURAL_REASONS)!r}: no base period under the " + "anticipation window, no comparison units, or zero survey mass). " + "A cohort is only dropped when the estimator itself could not " + "form any of its post cells; blanking a mid cohort's effects is " + "not that, so it fails closed instead of silently leaving the " + "estimand." ) n_units_excl = int(np.isin(unit_g_pos, excluded).sum()) + reasons_seen = sorted( + { + str(skipped[lab]) + for lab in skipped + if _pos_of(grid, lab[0]) in excluded and skipped[lab] in _STRUCTURAL_REASONS + } + ) + why = ( + f"skip_reason {reasons_seen!r}" + if reasons_seen + else "treated at or before the first observed period plus the anticipation window" + ) warnings.warn( f"cohort(s) {[_label_for(grid, g) for g in excluded]!r} ({n_units_excl} " - "unit(s)) have no estimable post-treatment cell and were dropped from " - "the weight table and the cohort shares: either treated in the first " - "observed period (R did's first-period drop) or run out of comparison " - "units under a not-yet-treated control group (R did's panel truncation)", + "unit(s)) have no estimable post-treatment cell under the anticipation " + f"window ({why}) and were dropped from the weight table and the cohort " + "shares, as R did drops cohorts it cannot estimate; the remaining " + "cohorts' masses and weights are those of the panel without them", UserWarning, stacklevel=2, ) @@ -839,17 +919,20 @@ def _post_start(g: int) -> int: missing_cells = sorted(required - present) structurally_absent: List[Tuple[Any, Any]] = [] if missing_cells: - # The zero_treated_control carve-out is keyed on the skip_reason VALUE, - # not on control_group: that reason is only ever emitted on a - # not-yet-treated fit, and the frame path has no control_group to read, - # so keying on the reason is what makes the two paths agree. + # The carve-out is keyed on the skip_reason VALUE, not on control_group + # or base_period: the estimator emits these reasons only for cells it + # structurally could not form, and the frame path has no design + # metadata to read, so keying on the reason is what makes the two + # paths agree. `aggregate()` finite-masks the same cells. carve_out_ok = type != "twfe" hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] + absent_reasons: set = set() for g, t in missing_cells: label = (_label_for(grid, g), _label_for(grid, t)) reason = skipped.get(label) - if carve_out_ok and reason == "zero_treated_control": + if carve_out_ok and reason in _STRUCTURAL_REASONS: structurally_absent.append(label) + absent_reasons.add(str(reason)) else: hard.append((label, reason)) if hard: @@ -865,10 +948,11 @@ def _post_start(g: int) -> int: ) warnings.warn( f"{len(structurally_absent)} post-treatment cell(s) {structurally_absent[:6]!r} " - "have no not-yet-treated comparison units (skip_reason " - "'zero_treated_control') and are treated as structurally absent: " - f"type={type!r} averages over each cohort's AVAILABLE " - "post periods, as R aggte() does on a not-yet-treated fit", + f"could not be estimated (skip_reason {sorted(absent_reasons)!r}: no " + "comparison units, no base period under the anticipation window, or " + "zero survey mass) and are treated as structurally absent: " + f"type={type!r} averages over each cohort's AVAILABLE post cells, as " + "R aggte() and results.aggregate() do", UserWarning, stacklevel=2, ) @@ -1047,9 +1131,12 @@ def __init__( units = frame[unit].to_numpy() periods = frame["_twfe_time_key"].to_numpy(dtype=float) self.unit_ids = np.asarray(sorted(pd.unique(units))) - self.period_labels = np.asarray(sorted(pd.unique(periods))) + # The numeric key orders, reshapes and maps cohorts; the ORIGINAL + # labels are what every reporting surface (cells, summary(), balance + # rows, plots) shows, so a string-labelled panel reports strings. + self.period_keys = np.asarray(sorted(pd.unique(periods))) n_units = len(self.unit_ids) - n_periods = len(self.period_labels) + n_periods = len(self.period_keys) if len(frame) != n_units * n_periods: raise ValueError( f"decompose_twfe_weights requires a balanced panel: got " @@ -1064,9 +1151,16 @@ def __init__( "are missing periods" ) - self.grid = _positional_grid(self.period_labels) + self.grid = _positional_grid(self.period_keys) self.n_units = n_units self.n_periods = n_periods + labels = frame.groupby("_twfe_time_key")[time] + if (labels.nunique() > 1).any(): + raise ValueError( + f"{time!r} mixes representations of the same period (e.g. '2' and " + "2.0); use one label per period" + ) + self.period_labels = labels.first().loc[self.period_keys].to_numpy() cohort_long = frame[first_treat].to_numpy() # dropna=False: a NaN label in one period must fail invariance, not @@ -1699,7 +1793,12 @@ def decompose_twfe_weights( contrasts, so there is no ATT(g, t) table it could consume. Its companion :func:`attgt_weights` is the fitted-result surface, and the two are tied by an identity that holds when the fit used ``base_period="universal"``, - ``control_group="never_treated"`` and no covariates:: + ``control_group="never_treated"`` and no covariates, no cohort was + dropped by :func:`attgt_weights`' structural rule (every cohort has at + least one estimable post cell under the fit's anticipation window - a + cohort treated in the first observed period is dropped there but kept + here under ``base_period="first_period"``), and both sides use the same + unit weights:: sum(attgt_weights(cs, type="twfe").weights.eval("weight * att")) == decompose_twfe_weights(panel, ...).estimate diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py index 7635c37a9..d7c7586ff 100644 --- a/diff_diff/twfe_weights_results.py +++ b/diff_diff/twfe_weights_results.py @@ -65,10 +65,11 @@ class ATTGTWeightsResult(Diagnostic): ---------- weights : pd.DataFrame Columns ``group``, ``time``, ``post``, ``weight``, ``att``. ``post`` - is ``1`` when ``time >= group`` (the cells the estimand targets), - ``0`` for pre-treatment cells. ``att`` is the ATT(g, t) the weight - multiplies, carried through from the source so that - ``(weight * att).sum()`` reproduces ``implied_att``. + is ``1`` for the cells the estimand targets - ``time >= group`` for + ``level="twfe"``, ``time >= group - anticipation`` (raw time units) + for the CS estimands - and ``0`` for pre-treatment cells. ``att`` is + the ATT(g, t) the weight multiplies, carried through from the source + so that ``(weight * att).sum()`` reproduces ``implied_att``. level : str The ``type=`` this result was built with, one of :attr:`LEVELS`: ``"twfe"``, ``"overall"`` (ATT^O) or ``"simple"`` (ATT^simple). @@ -103,7 +104,10 @@ class ATTGTWeightsResult(Diagnostic): control_group, base_period : str or None Design metadata carried from the source fit, when available. n_dropped_cells : int - Cells excluded because their ATT(g, t) was non-estimable (NaN). + PRE-window cells of surviving cohorts excluded because their ATT(g, t) + was non-estimable (NaN); the CS estimands place no weight there, so + the count is informational. Post-window cells the estimator could not + form are carved out of the grid (with a warning), not counted here. LEVELS : tuple of str The values ``level`` can take: ``("twfe", "overall", "simple")``. ``"overall"`` here is ATT^O (R ``did``'s ``attO``) and is unrelated @@ -290,7 +294,7 @@ def __repr__(self) -> str: ) def summary(self) -> str: - """Formatted decomposition table with the pre-trend contribution.""" + """Formatted decomposition table with the pre-period contribution.""" width = 78 method_label = { "fwl": "TWFE regression (Frisch-Waugh-Lovell implicit weights)", diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 0a69fcdfb..fdc167fff 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6225,11 +6225,11 @@ Where `n_k` is the sample share of timing group `k`, `n_{kℓ} = n_k / (n_k + n_ ### Relationship to neighbouring surfaces - **vs `twowayfeweights` (de Chaisemartin & D'Haultfoeuille 2020, Theorem 1):** that surface weights **(unit, time) cells**; these functions weight **ATT(g,t) parameters** — the cohort-by-period building blocks. Both detect negative weighting in staggered TWFE, but they decompose along different axes and their weight tables are not comparable row-for-row. The names are deliberately disjoint (`attgt_weights` / `ATTGTWeightsResult` vs `twowayfeweights` / `TWFEWeightsResult`). -- **vs `BaconDecomposition` (Goodman-Bacon 2021):** Bacon decomposes TWFE into **2x2 DiD comparisons** and asks which comparisons drive the estimate. `decompose_twfe_weights` decomposes it into **group-time effects** and additionally isolates a pre-trend-violation term. Use Bacon to see the forbidden comparisons; use this to see the per-`(g,t)` weights and how much of the estimate is not a treatment effect at all. +- **vs `BaconDecomposition` (Goodman-Bacon 2021):** Bacon decomposes TWFE into **2x2 DiD comparisons** and asks which comparisons drive the estimate. `decompose_twfe_weights` decomposes it into **group-time effects** and additionally isolates the pre-period contribution - the sample contribution of the pre-treatment cells, which can reflect differential pre-trends or sampling variation (see the Note under the equations). Use Bacon to see the forbidden comparisons; use this to see the per-`(g,t)` weights and how much of the estimate comes from pre-treatment cells. ### Estimator equations (as implemented) -All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T`. +All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T` - except the anticipation window `a`, which is applied in RAW time units exactly as `CallawaySantAnna(anticipation=)` applies it (`label(t) >= label(g) - a`, with `label(.)` the raw period label of a position); on a gapped calendar the two differ. *ATT(g,t) weights — `attgt_weights(type=...)`:* @@ -6244,13 +6244,15 @@ w(g,t) = num(g,t) / sum over {t >= g, g != 0} of num(g,t) `type="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: ``` -w(g,t) = 1[t >= g] * pbar_g / (maxT - g + 1) +w(g,t) = 1[label(t) >= label(g) - a] * pbar_g / n_post(g) ``` +with `a` the anticipation window in raw time units and `n_post(g)` the cohort's number of AVAILABLE post cells - `maxT - post_start(g) + 1` on a complete grid, which is R's `(maxT - g + 1)` only when `a = 0`; `post_start(g)` is the first position whose label is `>= label(g) - a`. Counting the available cells is what `n_post_available` feeds `_overall_weight_vector`, and is what makes the window and the structurally absent cells below come out right. + `type="simple"` (ATT^simple, R `att_simple_weights`): ``` -w(g,t) = 1[t >= g] * pbar_g, then normalized to sum to one +w(g,t) = 1[label(t) >= label(g) - a] * pbar_g, then normalized to sum to one ``` *FWL decomposition — `decompose_twfe_weights(method="fwl")` (R `implicit_twfe_weights`):* @@ -6281,7 +6283,7 @@ pre_period_contribution = sum over PRE cells (t < g) of alpha_weight * ATT `pre_period_contribution` is the sample contribution of the pre-treatment cells to `estimate`. It can reflect differential pre-trends, but it can also be sampling variation - the diagnostic carries no inference - so it is evidence about the earlier-period restrictions, not proof that the identifying assumption fails in the post-treatment counterfactual. -*Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, +*Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, no cohort was dropped by `attgt_weights`' structural rule (every cohort has at least one estimable post cell under the fit's anticipation window - `anticipation=0` alone is not sufficient: a cohort treated in the first observed period is dropped by `attgt_weights` but kept by `decompose_twfe_weights` under `base_period="first_period"`, and only `"gmin1"` raises on it), and both sides use the same unit weights (a weighted fit against an unweighted decomposition breaks it; `tests/test_twfe_weights.py::TestWeightedTwfeExtension` pins the weighted identity with the same survey weights on both sides), ``` attgt_weights(cs, type="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate @@ -6291,17 +6293,17 @@ Verified on `mpdta` at `-0.03654894` from both directions. ### Edge cases -- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `type="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. -- **Note (matches R `did`'s first-period drop, but only when the drop is structural):** a cohort with NO estimable post-treatment cell is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. The drop is allowed ONLY when the cohort is treated in the first observed period, OR when every one of its missing post cells carries `skip_reason="zero_treated_control"` (it ran out of comparison units). A mid cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. -- **Note (matches R `did`'s mechanism behind the second structural gap):** with no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g)` and recodes the last-treated cohort to never-treated. That truncation - not `aggte` averaging over available periods - is what produces R's number. diff-diff reaches the same numerics by treating the cells CS marks `skip_reason="zero_treated_control"` as structurally ABSENT (they name exactly the cohort x period cells that truncation removes): `"overall"` divides each cohort by its number of available post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group`, because that reason is only ever emitted on a not-yet-treated fit and the frame path carries no `control_group`; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. +- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `type="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. +- **Note (structural gaps are keyed on the estimator's own `skip_reason`, for every `type`):** a cell is *structurally absent* when its `skip_reason` is one of `missing_period` (no base period under the anticipation window), `zero_treated_control` (no comparison units) or `zero_weight_mass` (zero survey mass) - the reasons CS / DMLDiD emit only for cells they could not form, and the cells `_aggregate_overall` finite-masks. A cohort with NO estimable post-treatment cell, every missing post cell of which is structurally absent, is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, for every `type` including `"twfe"` - exactly as `did::pre_process_did` drops units treated at or before `first.period + anticipation` (verified against `did` 2.5.1 `pre_process_did` source); the remaining cohorts' masses and weights are those of the panel without it. The criterion is *post* cells under the window, not all cells. A cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing, under any window - the fitted path has no first-period shortcut. On a bare ATT(g,t) frame with no `skip_reason` column the only structural route IS that shortcut, in raw time: a cohort with `label(g) - anticipation <= label(first period)` may be excluded; any other absent cohort raises, being indistinguishable from user truncation. The frame path cannot detect a mismatched window or hand-built cells for cohorts the estimator would not have estimated, so on that path the caller must pass the window the producing fit used and must not hand-build such cells (stated in the docstring; no detection is promised). +- **Note (a cohort that keeps one estimable post cell is kept, as `aggregate()` keeps it; deviation from R's `pre_process_did` drop):** for `"overall"` / `"simple"`, a surviving cohort's structurally absent post cells are carved out of the required grid: `"overall"` divides each cohort by its number of AVAILABLE post cells instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells and the reasons present. Two mechanisms reach this branch. (i) With no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g) - anticipation` (and recodes the last-treated cohort to never-treated only under `control_group="never_treated"`) (verified against `did` 2.5.1 `pre_process_did` source); that truncation - not `aggte` averaging over available periods - is what produces R's number, and the cells CS marks `zero_treated_control` name exactly the cells truncation removes, so the numerics agree. (ii) Under `base_period="varying"` with `anticipation > 0`, CS keeps a cohort through a single estimable pre-window cell while marking every later cell `missing_period` (verified by execution: cohorts (0,3,5), periods 1..6, `anticipation=2` keeps cohort 3 through `(3,2)`), and `aggregate("group")` / `aggregate("simple")` INCLUDE that cohort; R's `pre_process_did` would drop it outright. `attgt_weights` follows diff-diff's own aggregates, so its `implied_att` matches `aggregate()` and not R there. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group` / `base_period`, because the frame path carries no design metadata; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated, universal-base comparison and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. - **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. - **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. -- `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The frame path requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). -- **Note (the anticipation window is honoured on the CS estimands):** the fitted path reads `anticipation` off the aggregation kit; the frame path takes it as an explicit `anticipation=` argument (a bare frame carries no record of the source fit's setting, and the fitted path rejects the kwarg rather than silently ignoring it). Post cells become `t >= g - anticipation` for `"overall"` / `"simple"`, and the same window enters the required-cell grid, the available-period divisors, and the `post` column. `"twfe"` deliberately keeps `t >= g`: the regression's own treatment indicator does not anticipate, and R's `twfe_weights` has no anticipation argument. Each CS estimand is pinned against `results.aggregate(...)` with positive anticipation under both control groups (`"simple"` against `aggregate("simple")`, `"overall"` against the cohort-mass-weighted combination of `aggregate("group")`). -- **Note (one canonical numeric time key):** period labels are coerced to a single numeric key at validation, and that key is used for sorting, reshaping, cohort mapping, and the positional grid. Using the raw column instead lets a numeric-string label (`"10"` sorts before `"2"` lexicographically) desynchronize the sort order from the grid and silently rebuild a different panel; numeric and numeric-string labels now decompose identically. +- **Note (panel completeness is the producer's own record):** `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The kit's predicate is "every unit-period outcome cell present and finite, and no unit dropped by the producing estimator's own per-cell complete-case mask" - DMLDiD records its drops (`complete_case_drops`, set whenever a cell loop excluded a unit for a non-finite outcome, `dY`, base-period covariate, bad-control column or `W`; a non-finite covariate in a period no cell reads is not a drop); CallawaySantAnna's NaN-covariate fallback keeps the cohort masses intact and is not a rejection cause. This is stricter than CallawaySantAnna's internal `is_balanced` (`isnan`-only, for its own fast paths), which is unchanged. Results without the record (fitted by diff-diff <= 3.12.0) are refused with a refit message, as is a kit without the `covariates` record - one fail-closed policy for every legacy kit, since CS masses CAN diverge on an incomplete-panel pickle (`_aggregate_overall` weights by the reduced per-cell `n_treated`). The frame path carries no record and is the caller's responsibility; it requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). +- **Note (the anticipation window is honoured on the CS estimands, in raw time units):** the fitted path reads `anticipation` off the aggregation kit; the frame path takes it as an explicit `anticipation=` argument (a bare frame carries no record of the source fit's setting, and the fitted path rejects the kwarg rather than silently ignoring it) - on the frame path the caller MUST pass the value the producing fit used, since a mismatch cannot be detected. Post cells become `label(t) >= label(g) - anticipation` for `"overall"` / `"simple"`, counted in the calendar's own units as CallawaySantAnna counts them (a positional window was a defect on gapped calendars: periods 10..40 with `anticipation=1` flagged `(20,10)` as post and returned 1.5230 against `aggregate("simple")` 2.2844), and the same window enters the required-cell grid, the available-cell divisors, the structural rule, and the `post` column. `"twfe"` deliberately keeps `t >= g` for its post window: the regression's own treatment indicator does not anticipate, and R's `twfe_weights` has no anticipation argument - but a cohort the estimator could not estimate at all under the window is dropped for every `type`, so `"twfe"`'s weights on a fit with such a cohort are those of the panel without it (its `excluded` set is computed on the unwindowed `t >= g` post set, a superset of the windowed one, so a cohort whose only estimable cell sits at `g - 1` is kept by the CS estimands but dropped by `"twfe"`). Each CS estimand is pinned against `results.aggregate(...)` with positive anticipation under both control groups (`"simple"` against `aggregate("simple")`, `"overall"` against the cohort-mass-weighted combination of `aggregate("group")`), on consecutive and on gapped calendars. +- **Note (one canonical numeric time key; reporting keeps the original labels):** period labels are coerced to a single numeric key at validation, and that key is used for sorting, reshaping, cohort mapping, and the positional grid. Using the raw column instead lets a numeric-string label (`"10"` sorts before `"2"` lexicographically) desynchronize the sort order from the grid and silently rebuild a different panel; numeric and numeric-string labels now decompose identically. The key is internal: every reporting surface (`cells`, `summary()`, `covariate_balance(level="cell")` rows, plots) carries the ORIGINAL `time` values in key order, so an integer-labelled panel reports integers and a string-labelled one reports strings. A `time` column that mixes two spellings of one period (`"2"` and `2.0`) is rejected by `decompose_twfe_weights`; on `attgt_weights`' frame path the two spellings are one cell (duplicates are detected on the numeric keys) and are rejected as a duplicate. - **Note (inputs are validated finite):** `decompose_twfe_weights` fails closed on non-finite outcomes, regression covariates, or balance covariates. A single NaN outcome would otherwise propagate to every cell and return an all-NaN decomposition with no error; complete-case handling is a policy choice the caller owns. - **Note (a wrong-length `weights=` fails closed):** the length check runs before the excluded-cohort boolean slice, so a wrong-length vector is a clear `ValueError` naming both lengths rather than a raw `IndexError`. diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index cd1570116..b3f04a0b6 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -5,6 +5,8 @@ result-object surface, and the design restrictions we enforce as errors. """ +import warnings + import numpy as np import pandas as pd import pytest @@ -701,11 +703,12 @@ def test_covariate_adjusted_fit_is_rejected_under_twfe(self, panel): def test_unadjusted_fit_records_empty_covariates(self, fitted): assert fitted._aggregation_kit.bookkeeping["covariates"] == () - def test_legacy_kit_without_the_key_warns(self, fitted): + def test_legacy_kit_without_the_key_raises(self, fitted): + """A kit pickled by <= 3.12.0 fails closed rather than warn-then-continue.""" kit = fitted._aggregation_kit saved = kit.bookkeeping.pop("covariates") try: - with pytest.warns(UserWarning, match="predates covariate bookkeeping"): + with pytest.raises(ValueError, match="does not record the bookkeeping"): attgt_weights(fitted, type="twfe") finally: kit.bookkeeping["covariates"] = saved @@ -727,6 +730,14 @@ def test_duplicate_cells_are_rejected(self, fitted, panel): with pytest.raises(ValueError, match="duplicated \\(group, time\\)"): _frame_call(dup, panel) + def test_duplicate_cells_are_detected_on_numeric_keys(self, fitted, panel): + """``time=3`` and ``time="3"`` are one cell on the positional grid.""" + frame = _gt_frame(fitted).astype({"time": object}) + twin = frame[(frame["group"] == 3) & (frame["time"] == 3)].assign(time="3") + dup = pd.concat([frame, twin], ignore_index=True) + with pytest.raises(ValueError, match="duplicated \\(group, time\\)"): + _frame_call(dup, panel) + def test_non_finite_group_label_is_rejected(self, fitted, panel): frame = _gt_frame(fitted) frame.loc[0, "group"] = np.nan @@ -896,6 +907,26 @@ def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): weighted.weights["weight"].to_numpy(), decomposed.cells["weight"].to_numpy(), atol=1e-12 ) + def test_weighted_identity_with_the_same_survey_weights_on_both_sides(self, panel): + """``implied_att == decompose(...).estimate`` for a fit built with the weights.""" + df = panel.copy() + rng = np.random.RandomState(9) + unit_w = pd.Series( + rng.choice([0.5, 1.0, 2.0], size=df["unit"].nunique()), + index=sorted(df["unit"].unique()), + ) + df["w"] = df["unit"].map(unit_w) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, survey_design=diff_diff.SurveyDesign(weights="w"), **_DECO) + weighted = attgt_weights(fit, type="twfe") + decomposed = diff_diff.decompose_twfe_weights(df, weights="w", **_DECO) + assert weighted.implied_att == pytest.approx(decomposed.estimate, abs=1e-10) + # ...and an UNWEIGHTED decomposition breaks it, which is why the identity + # is stated with "the same unit weights on both sides". + unweighted = diff_diff.decompose_twfe_weights(df, **_DECO) + assert abs(weighted.implied_att - unweighted.estimate) > 1e-6 + class TestHandComputedWeights: """Item 19: assert against numbers computed OUTSIDE the module.""" @@ -1136,6 +1167,36 @@ def test_numeric_string_labels_agree(self, panel): as_str.cells["att"].to_numpy(), as_int.cells["att"].to_numpy(), atol=1e-12 ) + def test_reporting_labels_are_the_original_time_values(self, panel): + """The numeric key is internal; cells and balance rows carry the input labels.""" + df = panel.copy() + df["x"] = np.random.RandomState(6).normal(size=len(df)) + assert df["period"].dtype == np.int64 and df["first_treat"].dtype == np.int64 + dec = diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **_DECO) + assert dec.cells["time"].dtype == df["period"].dtype + assert sorted(dec.cells["time"].unique().tolist()) == sorted(df["period"].unique().tolist()) + assert set(dec.cells["group"].tolist()) == set( + df.loc[df["first_treat"] != 0, "first_treat"] + ) + balance = dec.covariate_balance(level="cell") + assert ( + balance[["group", "time"]].values.tolist() + == dec.cells[["group", "time"]].values.tolist() + ) + + def test_string_labels_round_trip(self, panel): + labels = ["1", "2", "10", "11", "12"] + dec = diff_diff.decompose_twfe_weights(self._gapped(panel, labels), **_DECO) + assert set(dec.cells["time"]) == set(labels) + assert set(dec.cells["group"]) == {"10", "11"} + assert "10" in dec.summary() + + def test_mixed_representations_of_one_period_are_rejected(self, panel): + df = panel.copy().astype({"period": object}) + df.loc[(df["period"] == 2) & (df["unit"] < 60), "period"] = "2" + with pytest.raises(ValueError, match="mixes representations"): + diff_diff.decompose_twfe_weights(df, **_DECO) + class TestAnticipation: """Item 2: the anticipation window shifts the CS estimands, never TWFE.""" @@ -1168,6 +1229,9 @@ def test_overall_matches_group_combination(self, panel, control_group): assert got == pytest.approx(self._group_overall(fit, panel), abs=1e-12) def test_twfe_ignores_anticipation(self, panel): + """Holds when no cohort is dropped: cohorts 3 and 4 keep estimable cells + under ``anticipation=1``. A cohort the window makes non-estimable is + dropped for every ``type`` (``TestAnticipationStructuralDrop``).""" ant0 = attgt_weights(self._fit(panel, 0, "never_treated"), type="twfe") ant1 = attgt_weights(self._fit(panel, 1, "never_treated"), type="twfe") np.testing.assert_allclose( @@ -1215,6 +1279,114 @@ def test_unbalanced_fitted_result_is_rejected(self): def test_balanced_fit_records_the_flag(self, fitted): assert fitted._aggregation_kit.bookkeeping["is_balanced"] is True + def test_inf_outcome_is_incomplete(self, panel): + """``isnan`` alone would miss it; the kit predicate is ``isfinite``.""" + broken = panel.copy() + broken.loc[broken.index[0], "outcome"] = np.inf + fit = _fit(broken) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="present and finite"): + attgt_weights(fit, type="overall") + + def test_cs_nan_covariate_fallback_is_not_a_rejection(self, panel): + """CS falls back to unconditional estimation and keeps the cohort masses.""" + df = panel.copy() + df["x"] = np.random.RandomState(2).normal(size=len(df)) + df.loc[(df["unit"] == 3) & (df["period"] == 1), "x"] = np.nan + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # CS's own fallback / skip warnings + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, covariates=["x"], **_DECO) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is True + assert attgt_weights(fit, type="overall").n_cells > 0 + + @pytest.mark.parametrize("producer", ["cs", "dml"]) + def test_legacy_kit_without_the_record_raises(self, producer): + from tests.test_dml_did import COV, FIT_KW, make_staggered_dml_data + + if producer == "cs": + fit = _fit(_panel()) + else: + fit = diff_diff.DMLDiD(seed=0).fit(make_staggered_dml_data(), **FIT_KW, **COV) + kit = fit._aggregation_kit + saved = kit.bookkeeping.pop("is_balanced") + try: + with pytest.raises(ValueError, match="does not record the bookkeeping"): + attgt_weights(fit, type="overall") + finally: + kit.bookkeeping["is_balanced"] = saved + + +class TestDMLCompleteCaseRecord: + """DMLDiD records its own per-cell complete-case drops on the kit. + + Its cell loops fold outcomes, ``dY``, base-period covariates, bad-control + columns and ``W`` into one validity mask and collect every excluded unit; + the kit turns that into ``is_balanced`` so ``attgt_weights`` refuses a fit + whose cohort masses are reduced. A NaN in a period no cell reads is NOT a + drop, and must not false-reject. + """ + + @staticmethod + def _dml_fit(df, **extra): + from tests.test_dml_did import COV, FIT_KW + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return diff_diff.DMLDiD(seed=0).fit(df, **FIT_KW, **COV, **extra) + + @pytest.fixture(scope="class") + def data(self): + from tests.test_dml_did import make_staggered_dml_data + + return make_staggered_dml_data() + + def test_balanced_dml_fit_is_accepted(self, data): + fit = self._dml_fit(data) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is True + assert attgt_weights(fit, type="overall").n_cells > 0 + + def test_nan_outcome_is_rejected(self, data): + df = data.copy() + df.loc[df.index[0], "y"] = np.nan + fit = self._dml_fit(df) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="complete-case"): + attgt_weights(fit, type="overall") + + def test_nan_base_period_covariate_is_rejected(self, data): + """Every cell reads the universal base period's covariates.""" + df = data.copy() + df.loc[(df["unit"] == 3) & (df["time"] == 2000), "x1"] = np.nan + fit = self._dml_fit(df) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="complete-case"): + attgt_weights(fit, type="overall") + + def test_nan_covariate_in_an_unread_period_is_accepted(self, data): + """A post-period covariate cell is never read under a universal base.""" + df = data.copy() + df.loc[(df["unit"] == 3) & (df["time"] == 2003), "x1"] = np.nan + fit = self._dml_fit(df) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is True + assert attgt_weights(fit, type="overall").n_cells > 0 + + def test_bad_control_lane_records_its_drops(self, data): + """Pins the ``_compute_ccps_gt`` lane: a non-finite bad-control cell drops the unit.""" + from tests.test_dml_did import BC_KW, add_bad_control + + bc = add_bad_control(data) + fit = self._dml_fit(bc, **BC_KW) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is True + broken = bc.copy() + treated_unit = broken.loc[broken["first_treat"] == 2001, "unit"].iloc[0] + broken.loc[(broken["unit"] == treated_unit) & (broken["time"] == 2001), "xbad"] = np.nan + fit = self._dml_fit(broken, **BC_KW) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="complete-case"): + attgt_weights(fit, type="overall") + def test_frame_path_requires_one_obs_per_unit_period(self, fitted, panel): broken = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) with pytest.raises(ValueError, match="exactly one observation per unit-period"): @@ -1342,6 +1514,15 @@ def test_two_covariate_decomposition(self): assert result.post_only == pytest.approx(1.6099845667362065, abs=1e-12) assert result.remainder == 0.0 assert result.covariates == ("x1", "x2") + # Independent anchor: the pin is the covariate-adjusted TWFE coefficient, + # so a brute-force OLS of y on the treatment indicator, x1, x2 and unit + # and time dummies must land on the same number. + df = df.sort_values(["id", "t"]).reset_index(drop=True) + treated = ((df["g"] != 0) & (df["t"] >= df["g"])).astype(float).to_numpy() + dummies = pd.get_dummies(df[["id", "t"]].astype(str), drop_first=True).to_numpy(dtype=float) + design = np.column_stack([treated, df[["x1", "x2"]].to_numpy(), np.ones(len(df)), dummies]) + beta, *_ = np.linalg.lstsq(design, df["y"].to_numpy(), rcond=None) + assert result.estimate == pytest.approx(beta[0], abs=1e-10) np.testing.assert_allclose( result.cells["weight"].to_numpy(), [ @@ -1399,3 +1580,237 @@ def test_absolute_reference_line_starts_at_zero(self, panel): ) line = next(ln for ln in ax.lines if ln.get_label() == "no improvement") assert line.get_xdata()[0] == 0 + + +# --------------------------------------------------------------------------- +# Maintainer follow-up (PR #812 takeover): anticipation in raw time, the +# structural-drop rule keyed on skip_reason, and the bare-frame shortcut. +# --------------------------------------------------------------------------- + + +def _cs(df, **kwargs): + params = {"control_group": "never_treated", "base_period": "universal"} + params.update(kwargs) + return diff_diff.CallawaySantAnna(**params).fit(df, **_DECO) + + +def _group_overall_over(fit, panel, surviving): + """Cohort-share-weighted combination of ``aggregate("group")`` rows, + with the shares RENORMALIZED over ``surviving`` (the cohorts + ``attgt_weights`` kept).""" + group = fit.aggregate("group") + ever = panel.drop_duplicates("unit")["first_treat"].to_numpy() + ever = ever[np.isin(ever, list(surviving))] + vals, counts = np.unique(ever, return_counts=True) + share = dict(zip(vals.tolist(), (counts / counts.sum()).tolist())) + return sum(share[lab] * att for lab, att in zip(group.label, group.att) if lab in share) + + +class TestAnticipationStructuralDrop: + """A cohort CS cannot estimate at all under the window is dropped for every + type, exactly as ``aggregate()`` leaves it out; a cohort with one estimable + post cell is kept and averaged over what it has; a cohort blanked without + a reason still fails closed.""" + + def test_window_drops_the_cohort_for_every_type(self): + # cohorts 2, 4, 5 on periods 1..6 under anticipation=1: cohort 2's + # window starts at period 1, the universal base, so it has no + # estimable cell at all (every cell is missing_period); 4 and 5 survive, + # so the renormalization over surviving cohorts is exercised. + df = _panel(n_periods=6, cohorts=(0, 2, 4, 5)) + fit = _cs(df, anticipation=1) + with pytest.warns(UserWarning, match="skip_reason \\['missing_period'\\]"): + simple = attgt_weights(fit, type="simple") + assert set(simple.weights["group"]) == {4, 5} + assert simple.implied_att == pytest.approx(fit.aggregate("simple").att[0], abs=1e-12) + with pytest.warns(UserWarning, match="dropped from the weight table"): + overall = attgt_weights(fit, type="overall") + assert overall.implied_att == pytest.approx(_group_overall_over(fit, df, {4, 5}), abs=1e-12) + # "twfe" drops the cohort too, and the surviving cohorts' masses and + # weights are those of the panel WITHOUT it (prefiltered refit). + with pytest.warns(UserWarning, match="dropped from the weight table"): + twfe = attgt_weights(fit, type="twfe") + assert 2 not in set(twfe.weights["group"]) + reference = attgt_weights(_cs(df[df["first_treat"] != 2], anticipation=1), type="twfe") + np.testing.assert_allclose( + twfe.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-12 + ) + assert twfe.implied_att == pytest.approx(reference.implied_att, abs=1e-12) + # The frame path (skip_reason carried, matching window) agrees for all three. + for level, from_fit in (("simple", simple), ("overall", overall), ("twfe", twfe)): + with pytest.warns(UserWarning, match="dropped from the weight table"): + from_frame = _frame_call(_gt_frame(fit), df, type=level, anticipation=1) + np.testing.assert_allclose( + from_frame.weights["weight"].to_numpy(), + from_fit.weights["weight"].to_numpy(), + atol=1e-15, + ) + + def test_varying_base_keeps_a_cohort_through_one_estimable_cell(self): + # base_period="varying", anticipation=2, cohorts (0,3,5): CS keeps cohort 3 + # through its single estimable cell (3,2) and aggregate() includes it, + # while every other windowed cell is missing_period. R's pre_process_did + # would drop the cohort outright; attgt_weights follows diff-diff's own + # aggregates (documented deviation). + df = _panel(n_periods=6, cohorts=(0, 3, 5)) + fit = _cs(df, anticipation=2, base_period="varying") + reasons = {t: fit.group_time_effects[(3, t)].get("skip_reason") for t in range(1, 7)} + assert reasons[2] is None and all(reasons[t] == "missing_period" for t in (1, 3, 4, 5, 6)) + with pytest.warns(UserWarning, match="structurally absent") as record: + simple = attgt_weights(fit, type="simple") + overall = attgt_weights(fit, type="overall") + assert "missing_period" in " | ".join(str(w.message) for w in record) + assert set(simple.weights["group"]) == {3, 5} + assert simple.implied_att == pytest.approx(fit.aggregate("simple").att[0], abs=1e-12) + assert overall.implied_att == pytest.approx(_group_overall_over(fit, df, {3, 5}), abs=1e-12) + assert overall.weights.query("group == 3 and post == 1")["time"].tolist() == [2] + with pytest.raises(ValueError, match="base_period='universal'"): + attgt_weights(fit, type="twfe") + + @pytest.mark.parametrize("anticipation", [1, 3]) + def test_blanked_mid_cohort_still_fails_closed(self, anticipation): + """The fitted path has no raw-time shortcut: a reason-less blank raises + under any window, including one wide enough to reach the first period.""" + df = _panel(n_periods=6, cohorts=(0, 2, 4, 5)) + fit = _cs(df, anticipation=anticipation) + for (g, t), cell in fit.group_time_effects.items(): + if g == 4 and t >= 4 - anticipation: + cell["effect"] = np.nan + cell["skip_reason"] = None + with pytest.raises(ValueError, match="do not all carry"): + attgt_weights(fit, type="overall") + with pytest.raises(ValueError, match="do not all carry"): + _frame_call(_gt_frame(fit), df, type="overall", anticipation=anticipation) + + +class TestAnticipationGappedCalendar: + """The window is applied in the calendar's own units, as CS applies it.""" + + @staticmethod + def _gapped(): + rng = np.random.RandomState(11) + first_treat = np.repeat(np.array([0, 30, 40]), 40) + unit_fe = rng.normal(size=len(first_treat)) + rows = [] + for t in (10, 20, 30, 40, 50, 60): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(len(first_treat)), + "period": t, + "first_treat": first_treat, + "outcome": unit_fe + + 0.05 * t + + 0.1 * treated * (t - first_treat + 10) / 10 + + rng.normal(scale=0.3, size=len(first_treat)), + } + ) + ) + return pd.concat(rows, ignore_index=True) + + @pytest.mark.parametrize("level", ["simple", "overall"]) + def test_window_counts_raw_time_units(self, level): + df = self._gapped() + fits = {a: _cs(df, anticipation=a) for a in (0, 1, 10)} + results = {a: attgt_weights(fit, type=level) for a, fit in fits.items()} + + def oracle(a): + if level == "simple": + return fits[a].aggregate("simple").att[0] + return _group_overall_over(fits[a], df, {30, 40}) + + def post_flag(a, g, t): + row = results[a].weights.query("group == @g and time == @t") + return int(row["post"].iloc[0]) + + # anticipation=1 on a calendar stepping by 10 shifts nothing. + np.testing.assert_allclose( + results[1].weights["weight"].to_numpy(), + results[0].weights["weight"].to_numpy(), + atol=1e-14, + ) + assert results[1].weights["post"].tolist() == results[0].weights["post"].tolist() + assert results[1].implied_att == pytest.approx(results[0].implied_att, abs=1e-14) + assert post_flag(1, 30, 20) == 0 + # anticipation=10 shifts exactly one period. + assert post_flag(10, 30, 20) == 1 and post_flag(10, 30, 10) == 0 + assert results[10].implied_att != pytest.approx(results[0].implied_att, abs=1e-6) + for a in (0, 1, 10): + assert results[a].implied_att == pytest.approx(oracle(a), abs=1e-12) + + def test_twfe_is_identical_across_windows(self): + df = self._gapped() + results = [attgt_weights(_cs(df, anticipation=a), type="twfe") for a in (0, 1, 10)] + for other in results[1:]: + np.testing.assert_allclose( + other.weights["weight"].to_numpy(), + results[0].weights["weight"].to_numpy(), + atol=1e-14, + ) + assert other.weights["post"].tolist() == results[0].weights["post"].tolist() + + +class TestZeroWeightMassDrop: + """``zero_weight_mass`` is the third structural reason. On a panel fit the + survey weight is per unit, so a zero-mass cohort has NO estimable cell and + takes the drop branch (the carve-out branch needs per-period mass, which + only repeated cross-sections could produce, and those are rejected).""" + + def test_zero_mass_cohort_is_dropped_like_aggregate(self, panel): + df = panel.copy() + rng = np.random.RandomState(9) + unit_w = pd.Series( + rng.choice([0.5, 1.0, 2.0], size=df["unit"].nunique()), + index=sorted(df["unit"].unique()), + ) + df["w"] = df["unit"].map(unit_w) + df.loc[df["first_treat"] == 3, "w"] = 0.0 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # CS's consolidated skip warning + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, survey_design=diff_diff.SurveyDesign(weights="w"), **_DECO) + reasons = {c.get("skip_reason") for (g, _), c in fit.group_time_effects.items() if g == 3} + assert reasons == {"zero_weight_mass"} + with pytest.warns(UserWarning, match="skip_reason \\['zero_weight_mass'\\]"): + result = attgt_weights(fit, type="overall") + assert set(result.weights["group"]) == {4} + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # pweight normalization notice + reference = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit( + df[df["first_treat"] != 3], + survey_design=diff_diff.SurveyDesign(weights="w"), + **_DECO, + ) + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), + attgt_weights(reference, type="overall").weights["weight"].to_numpy(), + atol=1e-12, + ) + + +class TestBareFrameFirstPeriodShortcut: + """Without ``skip_reason`` the only structural route is R did's first-period + rule in raw time: ``group - anticipation <= first period``.""" + + def test_in_window_cohort_is_dropped_and_matches_the_fitted_path(self): + df = _panel(n_periods=6, cohorts=(0, 2, 4, 5)) + fit = _cs(df, anticipation=1) + bare = _gt_frame(fit).drop(columns=["skip_reason"]) + assert "skip_reason" not in bare.columns + with pytest.warns(UserWarning, match="first observed period plus the anticipation"): + from_bare = _frame_call(bare, df, type="simple", anticipation=1) + assert set(from_bare.weights["group"]) == {4, 5} + np.testing.assert_allclose( + from_bare.weights["weight"].to_numpy(), + attgt_weights(fit, type="simple").weights["weight"].to_numpy(), + atol=1e-15, + ) + # The same bare frame under the WRONG window (0) has cohort 2 out of + # reach of the shortcut, so it fails closed - the caller owns the window. + with pytest.raises(ValueError, match="A bare frame cannot say why"): + _frame_call(bare, df, type="simple", anticipation=0) From a0237cfa5ad7e068de03b650e1147eee91b88449 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 23 Sep 2026 17:27:21 -0400 Subject: [PATCH 10/13] docs(twfeweights): wording nits, post-only pathology in examples, tracking rows - "pre-trend contribution" -> "pre-period contribution" on README, the API page, the module docstring and the REGISTRY intro (the term is a sample contribution that can be pre-trends or sampling variation) - API-page examples print n_negative_post: the post-only counts are the pathology, the all-cell counts are structural - R generator header pins twfeweights == 0.9.0 (matches the stopifnot) and states the fixture cross-check precision honestly - TODO rows for decompose_twfe_weights(method="aipw") and the tutorial section (both follow-on PRs from #812) - multi-covariate pin gains a brute-force OLS anchor --- README.md | 2 +- TODO.md | 2 ++ benchmarks/R/generate_twfeweights_golden.R | 5 +++-- docs/api/twfe_weights.rst | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 17ba66226..3c5acfc07 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html) - Cattaneo, Jansson & Ma (2020) density-discontinuity test (`RDDensityTest`): rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out -- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-trend contribution. Ports Callaway's `twfeweights` (MIT) +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-period contribution. Ports Callaway's `twfeweights` (MIT) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/TODO.md b/TODO.md index 334c0b855..f875a1914 100644 --- a/TODO.md +++ b/TODO.md @@ -21,6 +21,7 @@ Related tracking surfaces: | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| +| `decompose_twfe_weights(method="aipw")`: port upstream `implicit_aipw_weights` (covariate-adjusted only; `twfe_weights.py` rejects the value today and lists `"fwl"`). The R goldens already carry the `decompose.aipw` / `balance.aipw` / `two_period.*` blocks under `meta.reserved_blocks`, so no R re-run is needed | `diff_diff/twfe_weights.py` | #812 | Mid | Medium | | Consolidate the remaining estimator-entangled DR/logit score variants (`staggered.py::_doubly_robust` + RC twins, `triple_diff.py`, `lwdid.py`, `wooldridge.py`) onto the shared `_dr_scores.py` module, each migration with its own committed oracle capture (the ContinuousDiD lift's two-tier pattern in `tests/test_dr_scores.py`); and add a ridge vcov path to `solve_ridge` if an estimator ever needs analytical ridge inference | `diff_diff/_dr_scores.py` | dml-b0 | Mid | Low | | Numeric between-period cohorts (e.g. `first_treat=4.5` with integer times) are rejected by LWDiD while CallawaySantAnna estimates them and LWDiD's own datetime/Period cohorts map to the next observed period — close the dtype asymmetry by adopting the next-observed-period mapping for numeric cohorts too (contract documented in REGISTRY cohort-encodings Note + `docs/api/lwdid.rst` Input Contract). Lands only after PR #588 merges | `diff_diff/lwdid.py` | #588 | Quick | Low | | Implement the LW 2026 eq. 7.9/7.10 unit-average cohort estimand (regress per-unit post-average transformed outcomes on `[1, D_g]` vs never-treated) as an alternative to the documented cell-mass `cohort_effects` convention (REGISTRY within-cohort aggregation Note; the two differ on unbalanced panels, where cell-mass weights units by observed post periods). Needs the 7.10 regression + its covariance on the NT path. Lands only after PR #588 merges | `diff_diff/lwdid_staggered.py` | #588 | Quick | Low | @@ -83,6 +84,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| +| TWFE-weights tutorial section: `attgt_weights` (all three `type=` values side by side) + `decompose_twfe_weights` + `plot_twfe_weights` on `mpdta`, either as a section of `02_staggered_did.ipynb` or a new notebook registered in `docs/tutorials/index.rst`; follow-on PR by decision, the feature ships without it | `docs/tutorials/02_staggered_did.ipynb` | #812 | Mid | Medium | | Geo-experiments tutorial: the SDiD fits on the `generate_factor_data(...)` panel (conversions ~1,500, sigma-hat ~310) hit the Frank-Wolfe iteration cap - the placebo fit warns once and the bootstrap cross-check warns on 100/100 draws - yet the narrative presents the bootstrap SE as a paper-faithful robustness check. Pre-existing on `main` under BOTH backends (surfaced when the notebook outputs were refreshed for the shape-only fit diagnostic); the committed April outputs predate the aggregated non-convergence warning. Either rescale / re-tune the DGP (or raise `max_iter` / loosen `min_decrease`) so the refits converge and re-execute, or label the bootstrap result unreliable and drop it from the "Validity evidence" claim. | `docs/tutorials/18_geo_experiments.ipynb` | local codex review of the SDiD shape-only fit-RMSE PR | Mid | Low | | Replicate Chang (2020) §4.2.1 ML-design RCS DGP — requires a penalized (Lasso-class) propensity learner or a maintainer fixture via the duck-typed `propensity_learner=` object route; native logit+linear verified pure noise at (N=500, p=100) across ~20 review seeds (att ~ −23..+16, SE ~4-10, EPV ~1.4-1.6 vs threshold 10, fitted out-of-fold clipping ~5-29%, mean ~14%); at (N=200, p=100) `outcome_learner="linear"` fails closed on control-fold rank deficiency even under an oracle propensity (use ridge/sieve there); §4.2.2 is replicated | `tests/test_methodology_dml_did.py`, `diff_diff/_learners.py` | DML PR-B2 | Mid | Low | | Optional scheduled end-to-end execution gate for the MMM tutorials (29/30): a cron-only workflow (or extension of `mmm-interop.yml`) that executes both notebooks in isolated exact-pin environments, so a stale/invalid committed posterior cannot stay green indefinitely - today the hybrid posture (deliberate: notebooks execute locally with committed outputs; CI smoke-tests the exporters without sampling; drift tests pin source + committed-output needles) leaves the MCMC claims un-re-executed in CI | `.github/workflows/mmm-interop.yml`, `docs/tutorials/29_mmm_calibration_pymc.ipynb`, `docs/tutorials/30_mmm_calibration_meridian.ipynb` | mmm-interop | Mid | Low | diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R index 957cd1300..1481846fa 100644 --- a/benchmarks/R/generate_twfeweights_golden.R +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -1,7 +1,7 @@ #!/usr/bin/env Rscript # Generate R `twfeweights` parity goldens for the diff-diff TWFE weight diagnostics. # -# Requires: twfeweights (>= 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, +# Requires: twfeweights (== 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, # DRDID, jsonlite # Output: benchmarks/data/twfeweights_golden.json # benchmarks/data/twfeweights_sim_panel.csv @@ -338,7 +338,8 @@ mpdta_df$lpop_t <- mpdta_df$lpop * (mpdta_df$period - 2002) / 5 # The fixture READS benchmarks/data/mpdta_stata_panel.csv (already in the repo # for the Stata parity suites) instead of writing a renamed copy. Assert the -# two sources agree bit-for-bit on every shared column, so they cannot drift. +# two sources agree on every shared column - integer identifiers exactly, float +# columns at CSV round-trip precision (1e-14) - so they cannot drift. stata_path <- file.path(out_dir, "mpdta_stata_panel.csv") if (!file.exists(stata_path)) { stop("expected ", stata_path, " (the mpdta fixture now reads it)") diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst index 62a8d9f3b..529f7ff5b 100644 --- a/docs/api/twfe_weights.rst +++ b/docs/api/twfe_weights.rst @@ -31,7 +31,7 @@ back into its building blocks. functions here weight **ATT(g, t) parameters**. - :class:`diff_diff.BaconDecomposition` decomposes TWFE into **2x2 DiD comparisons**. :func:`diff_diff.decompose_twfe_weights` decomposes it into - **group-time effects**, plus a pre-trend-violation term. + **group-time effects**, plus the pre-period contribution. **Reference:** Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025). Difference-in-Differences Designs: A @@ -103,7 +103,7 @@ Inspecting what a TWFE regression weights weights = diff_diff.attgt_weights(cs, type="twfe") print(weights.summary()) - print(weights.n_negative, "cells carry negative weight") + print(weights.n_negative_post, "post-treatment cells carry negative weight") Comparing against the estimand you meant to report ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -127,7 +127,7 @@ between ``implied_att`` values is the cost of the TWFE specification: for level in ("twfe", "overall", "simple"): w = diff_diff.attgt_weights(cs, type=level) print(f"{level:8s} {w.implied_att: .4f} " - f"({w.n_negative} negative weights)") + f"({w.n_negative_post} negative post-treatment weights)") Separating treatment effects from the pre-period contribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 3389b0da9a1abb5da27292e673e705334954925a Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 23 Sep 2026 17:50:21 -0400 Subject: [PATCH 11/13] fix(twfeweights): fitted path takes the fit's own weights; fail closed on zero-mass cohorts and phantom cohorts attgt_weights(fitted_result, weights=w) recomputed only the weight vector under w and multiplied it into ATT(g,t) estimated under the fit's own weights, so implied_att was neither the weighted nor the unweighted estimand (1.4349 vs the weighted fit's 1.4222 on the frozen weighted fixture). The fitted path now rejects an explicit weights= and uses the fit's SurveyDesign weights; the DataFrame path keeps weights= under the documented contract that the frame's ATT(g,t) were estimated under them. The weighted pins are re-captured from a natively weighted fit and anchored to decompose_twfe_weights(weights=).estimate and aggregate("simple"). decompose_twfe_weights rejects a treated cohort whose units all carry zero sampling weight by name (its cells were 0/0 and poisoned the decomposition with NaN silently). attgt_weights' frame path rejects a cohort in the ATT(g,t) frame that no unit in data= belongs to (was a raw KeyError), and keys its skip_reason lookups on canonical labels so a numeric-string frame keeps the structural carve-out (a string-keyed skip was searched as an int and wrongly rejected). effective_sample_size is R's verbatim (sampling weights cancel); documented as a REGISTRY Note rather than changed, since the per-cell ess is R-gated. The per-cohort share in the FWL loop is computed once per cohort instead of once per cell. Class-scoped instance-method fixtures moved to module level (pytest 10 removal). REGISTRY legacy-kit sentence aligned with the code. --- .../20260831-twfe-weight-diagnostics.md | 3 +- diff_diff/guides/llms-full.txt | 2 +- diff_diff/twfe_weights.py | 100 ++++++--- docs/methodology/REGISTRY.md | 7 +- tests/test_twfe_weights.py | 197 +++++++++++++----- 5 files changed, 226 insertions(+), 83 deletions(-) diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index 8d9f8847e..090552e9e 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -22,7 +22,8 @@ unit by their per-cell complete-case rules or that carry no completeness record (fitted by <= 3.12.0), non-finite outcomes / covariates, duplicated or non-finite ATT(g,t) cells, an incomplete group-time grid, - and invalid sampling weights. Gaps the estimator itself could not fill + invalid sampling weights, an explicit `weights=` on a fitted result (the + fit's own survey weights are used), and a zero-mass treated cohort. Gaps the estimator itself could not fill (`skip_reason` missing_period / zero_treated_control / zero_weight_mass) are handled as `aggregate()` does instead of raising: a cohort with no estimable post cell under the window is dropped, and the CS estimands diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index d5c716f5f..03ed4e2ab 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1547,7 +1547,7 @@ attgt_weights( results, # CallawaySantAnnaResults, or a (g,t) frame type="twfe", # "twfe" | "overall" (ATT^O) | "simple" data=None, unit=None, time=None, first_treat=None, # frame path only - weights=None, # unit-level sampling weights + weights=None, # frame path only; a fitted result uses its own survey weights anticipation=None, # frame path only; the fit's own otherwise ) -> ATTGTWeightsResult diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index 08d382f91..a1dab11c3 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -143,6 +143,18 @@ def _positional_grid( return {t: i + 1 for i, t in enumerate(ordered)} +def _canonical_label(value: Any) -> Any: + """One representation per period label: ``int`` for integral values, + ``float`` otherwise, so ``"3"``, ``3.0`` and ``np.int64(3)`` key the same + cell. This is the form ``_label_for`` emits, so ``skipped`` lookups keyed + here always find their cell whatever the caller's label dtype.""" + try: + as_float = float(value) + except (TypeError, ValueError): + return value + return int(as_float) if as_float.is_integer() else as_float + + def _to_positional_cohort(cohorts: np.ndarray, grid: Dict[float, int]) -> np.ndarray: """Cohort labels -> positional time; never-treated stays 0. @@ -323,7 +335,7 @@ def _attgt_from_cs( for (g, t), cell in results.group_time_effects.items(): effect = cell.get("effect", np.nan) if cell.get("skip_reason") is not None or not np.isfinite(effect): - skipped[(g, t)] = cell.get("skip_reason") + skipped[(_canonical_label(g), _canonical_label(t))] = cell.get("skip_reason") continue rows.append({"group": g, "time": t, "att": float(effect)}) if not rows: @@ -386,7 +398,7 @@ def _attgt_from_frame( skipped: Dict[Tuple[Any, Any], Optional[str]] = {} for i in np.flatnonzero(~finite): reason = reasons[i] - skipped[(table["group"].iat[i], table["time"].iat[i])] = ( + skipped[(_canonical_label(groups[i]), _canonical_label(times[i]))] = ( None if reason is None or (isinstance(reason, float) and np.isnan(reason)) else str(reason) @@ -586,9 +598,9 @@ def attgt_weights( path the caller is responsible for what the fitted path checks: the fit used no covariates under ``type="twfe"``; it dropped no unit from any cell by its own complete-case rules; ``anticipation=`` is the - window the fit used; and no cell was hand-built for a cohort the - estimator would not have estimated. None of these can be detected - from the frame. + window the fit used; ``weights=`` are the weights the fit used; and + no cell was hand-built for a cohort the estimator would not have + estimated. None of these can be detected from the frame. type : {"twfe", "overall", "simple"}, default "twfe" Which estimand's weights to report (the same keyword as ``results.aggregate(type=...)``; the accepted values are @@ -600,10 +612,16 @@ def attgt_weights( unit, time, first_treat : str, optional Column names in ``data``. Required together with ``data``. weights : str or array-like, optional - Unit-level sampling weights (R's ``w=``): a column name in ``data``, - or one value per unit. Rejected when the fit already carries survey - weights, which take precedence. Must be finite and non-negative with - positive treated mass (and positive never-treated mass for ``"twfe"``). + Unit-level sampling weights (R's ``w=``), DataFrame path only: a + column name in ``data``, or one value per unit. The fitted path uses + the fit's own survey weights (``SurveyDesign(weights=)``) and rejects + an explicit ``weights=``: the ATT(g, t) were estimated under the + fit's weights, and a different weight vector would make + ``implied_att`` neither the weighted nor the unweighted estimand. The + same contract binds the frame path - the frame's ATT(g, t) must have + been estimated under these weights. Must be finite and non-negative + with positive treated mass (and positive never-treated mass for + ``"twfe"``). anticipation : int, optional Anticipation window for the CS estimands, in the calendar's own time units exactly as ``CallawaySantAnna(anticipation=)`` counts them: a @@ -759,20 +777,17 @@ def attgt_weights( "Balance the panel (diff_diff.balance_panel), clean the " "non-finite cells, and refit." ) - if survey_weights is not None and weights is not None: + if weights is not None: raise ValueError( - "this fit already carries survey weights; passing weights= as " - "well is ambiguous. Drop weights= to use the fit's own." + "weights= is only for the DataFrame path. A fitted result's " + "ATT(g,t) were estimated under the fit's own survey weights (if " + "any); combining them with a different weight vector would report " + "an implied_att that is neither the weighted nor the unweighted " + "estimand. Refit with survey_design=SurveyDesign(weights=...) - " + "the fitted path then uses those weights automatically - or pass " + "result.to_dataframe('group_time') with data= and weights=." ) - if weights is not None and not isinstance(weights, str): - unit_weights = np.asarray(weights, dtype=float) - elif isinstance(weights, str): - raise ValueError( - "weights= may only name a column on the DataFrame path; pass " - "an array of per-unit weights instead" - ) - else: - unit_weights = survey_weights + unit_weights = survey_weights periods = np.asarray(results.time_periods) source = "CallawaySantAnnaResults" control_group = getattr(results, "control_group", None) @@ -803,6 +818,13 @@ def attgt_weights( g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + phantom = sorted({int(g) for g in g_pos} - {int(g) for g in unit_g_pos if g != 0}) + if phantom: + raise ValueError( + f"ATT(g,t) frame carries cohort(s) {[_label_for(grid, g) for g in phantom]!r} " + "that no unit in data= belongs to; every group in the frame must be a " + "first_treat value of the panel" + ) # Raw label of each position, 1-based (index 0 unused): the anticipation # window is defined in the calendar's own units, as CallawaySantAnna # applies it (``t < g - anticipation`` on labels), NOT in positions - on a @@ -1073,7 +1095,15 @@ def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray) -> float: - """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors.""" + """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors. + + Verbatim ``twfeweights::effective_sample_size``: the sampling weights + enter ONLY through the mean-one normalization of the estimation weights, + and since the ratio is scale-invariant they cancel - the statistic + measures the concentration of the ESTIMATION weights alone, not of a + composite ``sampling * estimation`` weight. Preserved for parity (the + per-cell ``ess`` is R-gated at 1e-9); documented in the registry. + """ sw = sampling_weights / sampling_weights.mean() ew = est_weights / _weighted_mean(est_weights, sw) denom = float((ew**2).sum()) @@ -1202,6 +1232,22 @@ def __init__( "weights must be time-invariant" ) _validate_unit_weights(block[:, 0], self.cohorts == 0, require_control_mass=True) + # Per-cohort mass: a treated cohort whose units all carry zero + # weight has no cohort share and no cell contrast, and every one + # of its cells would come out NaN and poison the decomposition + # silently. Fail closed and name the cohort. + zero_mass = [ + g + for g in sorted({int(c) for c in self.cohorts if c != 0}) + if block[self.cohorts == g, 0].sum() <= 0 + ] + if zero_mass: + labels = [_label_for(self.grid, g) for g in zero_mass] + raise ValueError( + f"cohort(s) {labels!r} carry zero total sampling weight in " + f"{weights!r}, so their cohort share and cells are undefined; " + "drop those units or give them positive weight" + ) self.weights = block self.covariates = tuple(covariates) if covariates: @@ -1454,6 +1500,12 @@ def _decompose_fwl( degenerate_cells: List[Tuple[Any, Any]] = [] for g in treated_cohorts: treated_mask = cohorts == g + # Cohort share of the (weighted) sample, R's ``pg``; constant over + # periods on a balanced panel, so computed once per cohort. + p_g = _weighted_mean( + treated_mask.astype(float)[:, None].repeat(panel.n_periods, axis=1).reshape(-1), + flat_w, + ) for t_pos in range(1, panel.n_periods + 1): col = t_pos - 1 w_treated = weights[treated_mask, col] @@ -1476,10 +1528,6 @@ def _decompose_fwl( gpart = _weighted_mean(gpart_w * adjusted[treated_mask], w_treated) upart = _weighted_mean(upart_w * adjusted[control_mask], w_control) - p_g = _weighted_mean( - (cohorts == g).astype(float)[:, None].repeat(panel.n_periods, axis=1).reshape(-1), - flat_w, - ) alpha_weight = ( _weighted_mean(r_treated, w_treated) * p_g / (alpha_den * panel.n_periods) ) diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index fdc167fff..5e54fbd05 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6297,9 +6297,9 @@ Verified on `mpdta` at `-0.03654894` from both directions. - **Note (structural gaps are keyed on the estimator's own `skip_reason`, for every `type`):** a cell is *structurally absent* when its `skip_reason` is one of `missing_period` (no base period under the anticipation window), `zero_treated_control` (no comparison units) or `zero_weight_mass` (zero survey mass) - the reasons CS / DMLDiD emit only for cells they could not form, and the cells `_aggregate_overall` finite-masks. A cohort with NO estimable post-treatment cell, every missing post cell of which is structurally absent, is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, for every `type` including `"twfe"` - exactly as `did::pre_process_did` drops units treated at or before `first.period + anticipation` (verified against `did` 2.5.1 `pre_process_did` source); the remaining cohorts' masses and weights are those of the panel without it. The criterion is *post* cells under the window, not all cells. A cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing, under any window - the fitted path has no first-period shortcut. On a bare ATT(g,t) frame with no `skip_reason` column the only structural route IS that shortcut, in raw time: a cohort with `label(g) - anticipation <= label(first period)` may be excluded; any other absent cohort raises, being indistinguishable from user truncation. The frame path cannot detect a mismatched window or hand-built cells for cohorts the estimator would not have estimated, so on that path the caller must pass the window the producing fit used and must not hand-build such cells (stated in the docstring; no detection is promised). - **Note (a cohort that keeps one estimable post cell is kept, as `aggregate()` keeps it; deviation from R's `pre_process_did` drop):** for `"overall"` / `"simple"`, a surviving cohort's structurally absent post cells are carved out of the required grid: `"overall"` divides each cohort by its number of AVAILABLE post cells instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells and the reasons present. Two mechanisms reach this branch. (i) With no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g) - anticipation` (and recodes the last-treated cohort to never-treated only under `control_group="never_treated"`) (verified against `did` 2.5.1 `pre_process_did` source); that truncation - not `aggte` averaging over available periods - is what produces R's number, and the cells CS marks `zero_treated_control` name exactly the cells truncation removes, so the numerics agree. (ii) Under `base_period="varying"` with `anticipation > 0`, CS keeps a cohort through a single estimable pre-window cell while marking every later cell `missing_period` (verified by execution: cohorts (0,3,5), periods 1..6, `anticipation=2` keeps cohort 3 through `(3,2)`), and `aggregate("group")` / `aggregate("simple")` INCLUDE that cohort; R's `pre_process_did` would drop it outright. `attgt_weights` follows diff-diff's own aggregates, so its `implied_att` matches `aggregate()` and not R there. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group` / `base_period`, because the frame path carries no design metadata; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated, universal-base comparison and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. -- **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. +- **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping (pickled by diff-diff <= 3.12.0) fails closed with a refit `ValueError`, the same policy as the missing completeness record; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. - **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. -- `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. +- `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. A treated cohort whose units all carry zero sampling weight is rejected by name: its cohort share and every one of its cells would be `0/0`, and the NaN would otherwise propagate silently into an all-NaN decomposition. On `attgt_weights`' frame path a cohort in the ATT(g,t) frame that no unit in `data=` belongs to is rejected by name rather than surfacing as a raw `KeyError`. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. - **Note (panel completeness is the producer's own record):** `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The kit's predicate is "every unit-period outcome cell present and finite, and no unit dropped by the producing estimator's own per-cell complete-case mask" - DMLDiD records its drops (`complete_case_drops`, set whenever a cell loop excluded a unit for a non-finite outcome, `dY`, base-period covariate, bad-control column or `W`; a non-finite covariate in a period no cell reads is not a drop); CallawaySantAnna's NaN-covariate fallback keeps the cohort masses intact and is not a rejection cause. This is stricter than CallawaySantAnna's internal `is_balanced` (`isnan`-only, for its own fast paths), which is unchanged. Results without the record (fitted by diff-diff <= 3.12.0) are refused with a refit message, as is a kit without the `covariates` record - one fail-closed policy for every legacy kit, since CS masses CAN diverge on an incomplete-panel pickle (`_aggregate_overall` weights by the reduced per-cell `n_treated`). The frame path carries no record and is the caller's responsibility; it requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). - **Note (the anticipation window is honoured on the CS estimands, in raw time units):** the fitted path reads `anticipation` off the aggregation kit; the frame path takes it as an explicit `anticipation=` argument (a bare frame carries no record of the source fit's setting, and the fitted path rejects the kwarg rather than silently ignoring it) - on the frame path the caller MUST pass the value the producing fit used, since a mismatch cannot be detected. Post cells become `label(t) >= label(g) - anticipation` for `"overall"` / `"simple"`, counted in the calendar's own units as CallawaySantAnna counts them (a positional window was a defect on gapped calendars: periods 10..40 with `anticipation=1` flagged `(20,10)` as post and returned 1.5230 against `aggregate("simple")` 2.2844), and the same window enters the required-cell grid, the available-cell divisors, the structural rule, and the `post` column. `"twfe"` deliberately keeps `t >= g` for its post window: the regression's own treatment indicator does not anticipate, and R's `twfe_weights` has no anticipation argument - but a cohort the estimator could not estimate at all under the window is dropped for every `type`, so `"twfe"`'s weights on a fit with such a cohort are those of the panel without it (its `excluded` set is computed on the unwindowed `t >= g` post set, a superset of the windowed one, so a cohort whose only estimable cell sits at `g - 1` is kept by the CS estimands but dropped by `"twfe"`). Each CS estimand is pinned against `results.aggregate(...)` with positive anticipation under both control groups (`"simple"` against `aggregate("simple")`, `"overall"` against the cohort-mass-weighted combination of `aggregate("group")`), on consecutive and on gapped calendars. @@ -6317,10 +6317,11 @@ Verified on `mpdta` at `-0.03654894` from both directions. - **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `type=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. - **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. - **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. +- **Note (`effective_sample_size` is preserved verbatim; sampling weights cancel):** upstream's `effective_sample_size(est_weights, sampling_weights)` normalizes the sampling weights to mean one, divides the estimation weights by their sampling-weighted mean, and returns `sum(w)^2 / sum(w^2)` of the ESTIMATION weights - a scale-invariant ratio, so the sampling weights cancel and the statistic measures the concentration of the estimation weights alone, not of a composite `sampling * estimation` weight (three units with sampling weights `[1, 0, 0]` and equal estimation weights report 3). Preserved as-is: the per-cell `ess` and `effective_sample_size` are R-gated (table below), and a composite-weight variant would be a different statistic. Read it as R's. - **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. - **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values, and that `NA` survives the summary roll-up. R propagates NA if any contributing cell is NA, whereas a plain pandas `.sum()` skips it and would report a spurious `0.0` for a binary or constant covariate; the roll-up therefore masks on the `post` column (not on a zero roll-up weight, since a post cell whose implicit weight is exactly zero still contributes) and returns NaN whenever any contributing post cell is NaN. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. - **Note (negative-weight statistics; no R counterpart):** over the `g != 0` grid the TWFE weights sum to zero, post cells to +1 and pre cells to -1, so `n_negative` is non-zero and `negative_weight_share` sits near 0.5 in EVERY staggered design, including one with no negative post-period weight. The pathology the literature describes is negative weight on POST cells, so `ATTGTWeightsResult` reports both, labelled: `n_negative` / `negative_weight_share` over all cells, and `n_negative_post` / `negative_post_weight_share` restricted to post cells (the share of post-period weight MASS that is negative). `summary()` leads with the post-only figures. R reports neither statistic. -- **Note (weighted `type="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, type="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). +- **Note (weighted `type="twfe"` is a diff-diff extension; the fitted path takes the fit's own weights only):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with sampling weights they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. On the fitted path the weights are the fit's own `SurveyDesign(weights=)` and an explicit `weights=` is rejected: the ATT(g,t) were estimated under the fit's weights, and recomputing only the weight vector under a different `w` would report an `implied_att` that is neither the weighted nor the unweighted estimand (measured on the frozen weighted fixture: 1.4349 from an unweighted fit with post-fit weights against 1.4222 from the weighted fit, which is `decompose_twfe_weights(weights=)`'s estimate). The DataFrame path keeps `weights=` under the documented contract that the frame's ATT(g,t) were estimated under the same weights. Pinned by asserting that a survey-weighted fit's `attgt_weights(type="twfe")` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12 AND the same `implied_att` / `estimate`, plus a frozen-numbers regression test on a synthetic weighted panel fitted natively under its weights (every parity fixture is unweighted, so R cannot gate this path). - **Note (linear algebra runs through the house helpers):** the two-way demeaning is `diff_diff.utils.within_transform` (the same alternating projections `fixest::demean` runs) applied to the sorted long frame before the `(unit, period)` reshape, with the treatment indicator synthesized as a column since it is derived from cohorts x positional periods rather than supplied. The Frisch-Waugh-Lovell solve is `diff_diff.linalg.solve_ols(..., weights=, rank_deficient_action="silent")`: on a rank-deficient design it fits the maximal independent set, sets the aliased coefficients to `NaN` (R-style) and returns the residual computed from the identified ones, so the module reads the dropped column names off the `NaN` positions and uses the returned residual directly. This replaced a bespoke pivoted QR whose docstring claimed to drop "later columns first"; it did not, being the same norm-pivoted QR `solve_ols` uses, and dropping the same column. - **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. - **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index b3f04a0b6..bfb07a84d 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -304,9 +304,16 @@ def test_rejects_time_varying_sampling_weights(self, fitted, panel): weights="w", ) - def test_rejects_a_weights_column_name_on_the_fitted_path(self, fitted): - with pytest.raises(ValueError, match="only name a column"): + def test_rejects_weights_on_the_fitted_path(self, fitted): + """The fit's own survey weights are the only weights a fitted result + can carry: its ATT(g, t) were estimated under them, and a different + vector would make implied_att neither the weighted nor the unweighted + estimand (a hybrid measured at 1.4349 vs 1.4222 on the weighted pin).""" + n_units = len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]) + with pytest.raises(ValueError, match="only for the DataFrame path"): attgt_weights(fitted, weights="w") + with pytest.raises(ValueError, match="only for the DataFrame path"): + attgt_weights(fitted, weights=np.ones(n_units)) class TestDegenerateInputs: @@ -492,9 +499,12 @@ def test_decomposition_weighted_branches(self, key): np.testing.assert_allclose(result.cells["weight"].to_numpy(), expected_w, atol=1e-12) np.testing.assert_allclose(result.cells["att"].to_numpy(), spec["att"], atol=1e-12) + # Captured from a fit made NATIVELY under the panel's weights + # (``SurveyDesign(weights="w")``); the "twfe" implied_att is the weighted + # TWFE coefficient and equals ``_DEC["nocov"]["estimate"]``. _AGG = { "twfe": ( - 1.4348838554104435, + 1.4221735897240106, [ -0.2638376383763837, -0.2638376383763837, @@ -509,7 +519,7 @@ def test_decomposition_weighted_branches(self, key): ], ), "overall": ( - 2.101822892918353, + 2.0885406501592416, [ 0, 0, @@ -524,7 +534,7 @@ def test_decomposition_weighted_branches(self, key): ], ), "simple": ( - 2.239531384413449, + 2.221649894556495, [ 0, 0, @@ -543,14 +553,28 @@ def test_decomposition_weighted_branches(self, key): @pytest.mark.parametrize("level", ["twfe", "overall", "simple"]) def test_attgt_weighted_branches(self, level): df = self._weighted_panel() - cs = diff_diff.CallawaySantAnna(base_period="universal", control_group="never_treated").fit( - df, outcome="y", unit="id", time="t", first_treat="g" - ) - unit_w = df.groupby("id", sort=True)["w"].first().to_numpy() - result = attgt_weights(cs, type=level, weights=unit_w) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # pweight normalization notice + cs = diff_diff.CallawaySantAnna( + base_period="universal", control_group="never_treated" + ).fit( + df, + outcome="y", + unit="id", + time="t", + first_treat="g", + survey_design=diff_diff.SurveyDesign(weights="w"), + ) + result = attgt_weights(cs, type=level) implied, weight = self._AGG[level] assert result.implied_att == pytest.approx(implied, abs=1e-12) np.testing.assert_allclose(result.weights["weight"].to_numpy(), weight, atol=1e-12) + # Independent anchors: the weighted TWFE coefficient is the weighted + # decomposition's estimate; ATT^simple is CS's own weighted aggregate. + if level == "twfe": + assert result.implied_att == pytest.approx(self._DEC["nocov"]["estimate"], abs=1e-12) + if level == "simple": + assert result.implied_att == pytest.approx(cs.aggregate("simple").att[0], abs=1e-12) assert list(zip(result.weights["group"], result.weights["time"])) == [ (3, 1), (3, 2), @@ -636,29 +660,31 @@ def test_nan_period_label_is_rejected_up_front(self, fitted, panel): ) +@pytest.fixture(scope="module") +def decomposed(): + """Balance table with <3-distinct-value covariates (TestBalanceNaNPropagation).""" + df = _panel() + rng = np.random.RandomState(3) + df["binary"] = rng.binomial(1, 0.4, size=len(df)).astype(float) + df["const"] = 1.0 + df["cont"] = rng.normal(size=len(df)) + # Make the binary / constant columns unit-invariant so the unit mean + # keeps them at <3 distinct values. + df["binary"] = df.groupby("unit")["binary"].transform("first") + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["cont"], + balance_covariates=["binary", "const", "cont"], + ) + + class TestBalanceNaNPropagation: """Item 2: frac_extreme's NA for <3 distinct values survives the summary roll-up.""" - @pytest.fixture(scope="class") - def decomposed(self): - df = _panel() - rng = np.random.RandomState(3) - df["binary"] = rng.binomial(1, 0.4, size=len(df)).astype(float) - df["const"] = 1.0 - df["cont"] = rng.normal(size=len(df)) - # Make the binary / constant columns unit-invariant so the unit mean - # keeps them at <3 distinct values. - df["binary"] = df.groupby("unit")["binary"].transform("first") - return diff_diff.decompose_twfe_weights( - df, - outcome="outcome", - unit="unit", - time="period", - first_treat="first_treat", - covariates=["cont"], - balance_covariates=["binary", "const", "cont"], - ) - def test_cell_level_is_nan_for_degenerate_covariates(self, decomposed): cells = decomposed.covariate_balance(level="cell") for cov in ("binary", "const"): @@ -836,18 +862,20 @@ class TestWeightValidation: (lambda w: np.zeros_like(w), "sum to zero"), ], ) - def test_bad_unit_weights_are_rejected(self, fitted, mutate, match): - w = mutate(np.ones(len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]))) + def test_bad_unit_weights_are_rejected(self, fitted, panel, mutate, match): + w = mutate(np.ones(panel["unit"].nunique())) with pytest.raises(ValueError, match=match): - attgt_weights(fitted, type="overall", weights=w) + _frame_call(_gt_frame(fitted), panel, type="overall", weights=w) def test_zero_control_mass_only_matters_where_controls_enter(self, fitted, panel): - cohorts = np.asarray(fitted._aggregation_kit.bookkeeping["unit_cohorts"], dtype=float) + cohorts = ( + panel.sort_values("unit").drop_duplicates("unit")["first_treat"].to_numpy(dtype=float) + ) w = np.where(cohorts == 0, 0.0, 1.0) with pytest.raises(ValueError, match="never-treated comparison group carries zero"): - attgt_weights(fitted, type="twfe", weights=w) + _frame_call(_gt_frame(fitted), panel, type="twfe", weights=w) for level in ("overall", "simple"): - assert attgt_weights(fitted, type=level, weights=w).n_cells > 0 + assert _frame_call(_gt_frame(fitted), panel, type=level, weights=w).n_cells > 0 def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): df = panel.copy() @@ -893,8 +921,10 @@ def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): index=sorted(df["unit"].unique()), ) df["w"] = df["unit"].map(unit_w) - fit = _fit(df) - weighted = attgt_weights(fit, type="twfe", weights=unit_w.to_numpy()) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, survey_design=diff_diff.SurveyDesign(weights="w"), **_DECO) + weighted = attgt_weights(fit, type="twfe") decomposed = diff_diff.decompose_twfe_weights( df, outcome="outcome", @@ -1318,6 +1348,13 @@ def test_legacy_kit_without_the_record_raises(self, producer): kit.bookkeeping["is_balanced"] = saved +@pytest.fixture(scope="module") +def dml_data(): + from tests.test_dml_did import make_staggered_dml_data + + return make_staggered_dml_data() + + class TestDMLCompleteCaseRecord: """DMLDiD records its own per-cell complete-case drops on the kit. @@ -1336,18 +1373,14 @@ def _dml_fit(df, **extra): warnings.simplefilter("ignore") return diff_diff.DMLDiD(seed=0).fit(df, **FIT_KW, **COV, **extra) - @pytest.fixture(scope="class") - def data(self): - from tests.test_dml_did import make_staggered_dml_data - - return make_staggered_dml_data() - - def test_balanced_dml_fit_is_accepted(self, data): + def test_balanced_dml_fit_is_accepted(self, dml_data): + data = dml_data fit = self._dml_fit(data) assert fit._aggregation_kit.bookkeeping["is_balanced"] is True assert attgt_weights(fit, type="overall").n_cells > 0 - def test_nan_outcome_is_rejected(self, data): + def test_nan_outcome_is_rejected(self, dml_data): + data = dml_data df = data.copy() df.loc[df.index[0], "y"] = np.nan fit = self._dml_fit(df) @@ -1355,7 +1388,8 @@ def test_nan_outcome_is_rejected(self, data): with pytest.raises(ValueError, match="complete-case"): attgt_weights(fit, type="overall") - def test_nan_base_period_covariate_is_rejected(self, data): + def test_nan_base_period_covariate_is_rejected(self, dml_data): + data = dml_data """Every cell reads the universal base period's covariates.""" df = data.copy() df.loc[(df["unit"] == 3) & (df["time"] == 2000), "x1"] = np.nan @@ -1364,7 +1398,8 @@ def test_nan_base_period_covariate_is_rejected(self, data): with pytest.raises(ValueError, match="complete-case"): attgt_weights(fit, type="overall") - def test_nan_covariate_in_an_unread_period_is_accepted(self, data): + def test_nan_covariate_in_an_unread_period_is_accepted(self, dml_data): + data = dml_data """A post-period covariate cell is never read under a universal base.""" df = data.copy() df.loc[(df["unit"] == 3) & (df["time"] == 2003), "x1"] = np.nan @@ -1372,7 +1407,8 @@ def test_nan_covariate_in_an_unread_period_is_accepted(self, data): assert fit._aggregation_kit.bookkeeping["is_balanced"] is True assert attgt_weights(fit, type="overall").n_cells > 0 - def test_bad_control_lane_records_its_drops(self, data): + def test_bad_control_lane_records_its_drops(self, dml_data): + data = dml_data """Pins the ``_compute_ccps_gt`` lane: a non-finite bad-control cell drops the unit.""" from tests.test_dml_did import BC_KW, add_bad_control @@ -1472,10 +1508,11 @@ class TestWeightsLength: """Item 11: a wrong-length weights= is a clear error, never an IndexError.""" def test_wrong_length_with_an_excluded_cohort(self): - fit = _fit(_panel(cohorts=(0, 1, 3, 4))) - n_units = len(fit._aggregation_kit.bookkeeping["unit_cohorts"]) + df = _panel(cohorts=(0, 1, 3, 4)) + fit = _fit(df) + n_units = df["unit"].nunique() with pytest.raises(ValueError, match="weights has length"): - attgt_weights(fit, type="overall", weights=np.ones(n_units - 1)) + _frame_call(_gt_frame(fit), df, type="overall", weights=np.ones(n_units - 1)) class TestMultiCovariatePin: @@ -1814,3 +1851,59 @@ def test_in_window_cohort_is_dropped_and_matches_the_fitted_path(self): # reach of the shortcut, so it fails closed - the caller owns the window. with pytest.raises(ValueError, match="A bare frame cannot say why"): _frame_call(bare, df, type="simple", anticipation=0) + + +class TestZeroMassCohortInDecomposition: + """A treated cohort with zero sampling weight fails closed by name instead of + poisoning the decomposition with NaN.""" + + def test_zero_mass_treated_cohort_is_rejected(self, panel): + df = panel.copy() + df["w"] = 1.0 + df.loc[df["first_treat"] == 3, "w"] = 0.0 + with pytest.raises(ValueError, match=r"cohort\(s\) \[3\] carry zero total sampling weight"): + diff_diff.decompose_twfe_weights(df, weights="w", **_DECO) + + +class TestFramePathLabelDtypes: + """Structural-skip lookups key on canonical labels, so a numeric-string + frame and panel behave exactly like the integer-labelled ones.""" + + def test_string_labels_keep_the_structural_carve_out(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + gt = _gt_frame(fit) + gt_str = gt.assign(group=gt["group"].astype(str), time=gt["time"].astype(str)) + df_str = df.assign( + period=df["period"].astype(str), first_treat=df["first_treat"].astype(str) + ) + with pytest.warns(UserWarning, match="structurally absent"): + reference = _frame_call(gt, df, type="overall") + with pytest.warns(UserWarning, match="structurally absent"): + as_str = _frame_call(gt_str, df_str, type="overall") + np.testing.assert_allclose( + as_str.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-15 + ) + assert as_str.implied_att == pytest.approx(reference.implied_att, abs=1e-15) + + def test_phantom_cohort_in_the_frame_is_rejected(self, fitted, panel): + """A group on the period grid but absent from data='s cohorts is a clear error.""" + gt = _gt_frame(fitted) + phantom = pd.concat([gt, gt[gt["group"] == 3].assign(group=2)], ignore_index=True) + for level in ("twfe", "overall"): + with pytest.raises(ValueError, match=r"cohort\(s\) \[2\] that no unit in data="): + _frame_call(phantom, panel, type=level) + + +class TestEffectiveSampleSizeIsUpstreams: + """``effective_sample_size`` is R's: the sampling weights cancel.""" + + def test_sampling_weights_cancel(self): + from diff_diff.twfe_weights import _effective_sample_size + + est = np.ones(3) + assert _effective_sample_size(est, np.array([1.0, 0.0, 0.0])) == pytest.approx(3.0) + assert _effective_sample_size(est, np.ones(3)) == pytest.approx(3.0) + # ...and the estimation weights alone drive it: sum(w)^2 / sum(w^2). + est = np.array([1.0, 3.0]) + assert _effective_sample_size(est, np.ones(2)) == pytest.approx(16.0 / 10.0) From 6d6b0d4ee7e767fdd9c330a7e4073b548596a155 Mon Sep 17 00:00:00 2001 From: igerber Date: Thu, 24 Sep 2026 05:51:17 -0400 Subject: [PATCH 12/13] fix(twfeweights): align frame-path weights= by unit label On the DataFrame path a pd.Series of per-unit weights had its index discarded and a bare array was taken positionally against an undocumented order (sorted unit ids, the order the cohort vector is built in). On a panel with non-monotonic unit ids a Series in panel-row order moved ATT^O's implied_att from 1.7607 (column route) to 1.7882. A Series is now reindexed to the panel's units by label and must cover exactly those units; the array contract (sorted unit-id order) is stated in the docstring and the registry; the column route is unchanged. Regression test on a shuffled-id panel pins column == Series (any order) == array (sorted order) and the rejections. --- .../20260831-twfe-weight-diagnostics.md | 3 +- diff_diff/twfe_weights.py | 38 +++++++++++++++---- docs/methodology/REGISTRY.md | 2 +- tests/test_twfe_weights.py | 34 +++++++++++++++++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md index 090552e9e..d22f6e3ca 100644 --- a/changelog.d/20260831-twfe-weight-diagnostics.md +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -23,7 +23,8 @@ record (fitted by <= 3.12.0), non-finite outcomes / covariates, duplicated or non-finite ATT(g,t) cells, an incomplete group-time grid, invalid sampling weights, an explicit `weights=` on a fitted result (the - fit's own survey weights are used), and a zero-mass treated cohort. Gaps the estimator itself could not fill + fit's own survey weights are used), and a zero-mass treated cohort; a + `pd.Series` of frame-path weights is aligned by unit label. Gaps the estimator itself could not fill (`skip_reason` missing_period / zero_treated_control / zero_weight_mass) are handled as `aggregate()` does instead of raising: a cohort with no estimable post cell under the window is dropped, and the CS estimands diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index a1dab11c3..711e0563a 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -448,7 +448,7 @@ def _unit_cohorts_from_frame( cohorts = firsts.to_numpy() _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) periods = np.asarray(sorted(pd.unique(key))) - return cohorts, periods, None + return cohorts, periods, firsts.index.to_numpy() def _resolve_cs_inputs( @@ -611,9 +611,14 @@ def attgt_weights( path; passing it alongside a fitted result raises. unit, time, first_treat : str, optional Column names in ``data``. Required together with ``data``. - weights : str or array-like, optional + weights : str, pd.Series or array-like, optional Unit-level sampling weights (R's ``w=``), DataFrame path only: a - column name in ``data``, or one value per unit. The fitted path uses + column name in ``data`` (time-invariant within unit), a ``pd.Series`` + indexed by unit id (aligned by label, so its order does not matter; + it must cover exactly the units in ``data``), or a bare array with + one value per unit in SORTED unit-id order (``sorted(data[unit].unique())``, + the order the cohort vector is built in - a label-free vector has no + other meaning, so prefer the column or the Series). The fitted path uses the fit's own survey weights (``SurveyDesign(weights=)``) and rejects an explicit ``weights=``: the ATT(g, t) were estimated under the fit's weights, and a different weight vector would make @@ -743,8 +748,8 @@ def attgt_weights( assert time is not None and first_treat is not None assert frame is not None table, skipped = _attgt_from_frame(frame) - cohorts, periods, _ = _unit_cohorts_from_frame(data, unit, time, first_treat) - unit_weights = _resolve_frame_weights(weights, data, unit) + cohorts, periods, unit_ids = _unit_cohorts_from_frame(data, unit, time, first_treat) + unit_weights = _resolve_frame_weights(weights, data, unit, unit_ids) source = "DataFrame" control_group = None base_period = None @@ -1060,11 +1065,19 @@ def _pos_of(grid: Dict[float, int], label: Any) -> int: def _resolve_frame_weights( - weights: Optional[Union[str, np.ndarray]], + weights: Optional[Union[str, np.ndarray, pd.Series]], data: pd.DataFrame, unit: str, + unit_ids: np.ndarray, ) -> Optional[np.ndarray]: - """Turn ``weights=`` into one value per unit, or None.""" + """Turn ``weights=`` into one value per unit, aligned to ``unit_ids``. + + ``unit_ids`` is the sorted unit-id order the cohort vector was built in. + A column name is collapsed per unit in that order; a ``pd.Series`` is + aligned BY LABEL (its index must be exactly the unit ids); a bare array + is taken positionally and must therefore already be in sorted unit-id + order - the only order a label-free vector can mean. + """ if weights is None: return None if isinstance(weights, str): @@ -1078,6 +1091,17 @@ def _resolve_frame_weights( f"{offenders!r}; sampling weights must be time-invariant" ) return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) + if isinstance(weights, pd.Series): + if weights.index.has_duplicates: + raise ValueError("weights Series index has duplicate unit labels") + missing = [u for u in unit_ids if u not in weights.index] + extra = [u for u in weights.index if u not in set(unit_ids.tolist())] + if missing or extra: + raise ValueError( + "weights Series must be indexed by exactly the units in data=; " + f"missing {missing[:5]!r}, unexpected {extra[:5]!r}" + ) + return weights.reindex(unit_ids).to_numpy(dtype=float) return np.asarray(weights, dtype=float) diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 5e54fbd05..3e0b617e0 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6298,7 +6298,7 @@ Verified on `mpdta` at `-0.03654894` from both directions. - **Note (a cohort that keeps one estimable post cell is kept, as `aggregate()` keeps it; deviation from R's `pre_process_did` drop):** for `"overall"` / `"simple"`, a surviving cohort's structurally absent post cells are carved out of the required grid: `"overall"` divides each cohort by its number of AVAILABLE post cells instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells and the reasons present. Two mechanisms reach this branch. (i) With no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g) - anticipation` (and recodes the last-treated cohort to never-treated only under `control_group="never_treated"`) (verified against `did` 2.5.1 `pre_process_did` source); that truncation - not `aggte` averaging over available periods - is what produces R's number, and the cells CS marks `zero_treated_control` name exactly the cells truncation removes, so the numerics agree. (ii) Under `base_period="varying"` with `anticipation > 0`, CS keeps a cohort through a single estimable pre-window cell while marking every later cell `missing_period` (verified by execution: cohorts (0,3,5), periods 1..6, `anticipation=2` keeps cohort 3 through `(3,2)`), and `aggregate("group")` / `aggregate("simple")` INCLUDE that cohort; R's `pre_process_did` would drop it outright. `attgt_weights` follows diff-diff's own aggregates, so its `implied_att` matches `aggregate()` and not R there. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group` / `base_period`, because the frame path carries no design metadata; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated, universal-base comparison and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. - **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping (pickled by diff-diff <= 3.12.0) fails closed with a refit `ValueError`, the same policy as the missing completeness record; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. -- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. +- **Note (sampling weights are validated, and aligned by unit label on the frame path):** unit weights must be finite, non-negative, with positive total and positive treated mass. On `attgt_weights`' DataFrame path a column name is collapsed per unit, a `pd.Series` is aligned to the panel's units BY LABEL (it must cover exactly those units), and a bare array is taken positionally in sorted unit-id order - the order the cohort vector is built in, stated in the docstring; a Series in panel-row order on a panel with non-monotonic ids was measured to move ATT^O's `implied_att` from 1.7607 to 1.7882 before the label alignment. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. A treated cohort whose units all carry zero sampling weight is rejected by name: its cohort share and every one of its cells would be `0/0`, and the NaN would otherwise propagate silently into an all-NaN decomposition. On `attgt_weights`' frame path a cohort in the ATT(g,t) frame that no unit in `data=` belongs to is rejected by name rather than surfacing as a raw `KeyError`. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. - **Note (panel completeness is the producer's own record):** `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The kit's predicate is "every unit-period outcome cell present and finite, and no unit dropped by the producing estimator's own per-cell complete-case mask" - DMLDiD records its drops (`complete_case_drops`, set whenever a cell loop excluded a unit for a non-finite outcome, `dY`, base-period covariate, bad-control column or `W`; a non-finite covariate in a period no cell reads is not a drop); CallawaySantAnna's NaN-covariate fallback keeps the cohort masses intact and is not a rejection cause. This is stricter than CallawaySantAnna's internal `is_balanced` (`isnan`-only, for its own fast paths), which is unchanged. Results without the record (fitted by diff-diff <= 3.12.0) are refused with a refit message, as is a kit without the `covariates` record - one fail-closed policy for every legacy kit, since CS masses CAN diverge on an incomplete-panel pickle (`_aggregate_overall` weights by the reduced per-cell `n_treated`). The frame path carries no record and is the caller's responsibility; it requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index bfb07a84d..5ce9feeab 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -291,6 +291,40 @@ def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): assert mass_3_after > mass_3_before assert tilted.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + def test_weights_align_by_unit_label_on_a_shuffled_panel(self): + """Non-monotonic unit ids, shuffled rows, non-uniform weights: the + column route, a Series in ANY order, and an array in sorted-id order + agree; a Series that does not cover the units is rejected.""" + df = _panel() + rng = np.random.RandomState(3) + ids = rng.permutation(np.arange(1000, 1000 + df["unit"].nunique())) + df["unit"] = df["unit"].map(dict(enumerate(ids))) + w_by_unit = pd.Series(rng.choice([0.5, 1.0, 2.0], size=len(ids)), index=ids) + df["w"] = df["unit"].map(w_by_unit) + df = df.sample(frac=1, random_state=1) + fit = _fit(df) + gt = _gt_frame(fit) + kw = dict(unit="unit", time="period", first_treat="first_treat", type="overall") + via_column = attgt_weights(gt, data=df, weights="w", **kw) + panel_order = df.drop_duplicates("unit")["unit"].to_numpy() + via_series = attgt_weights(gt, data=df, weights=w_by_unit.loc[panel_order], **kw) + via_array = attgt_weights(gt, data=df, weights=w_by_unit.loc[sorted(ids)].to_numpy(), **kw) + for other in (via_series, via_array): + np.testing.assert_allclose( + other.weights["weight"].to_numpy(), + via_column.weights["weight"].to_numpy(), + atol=1e-15, + ) + assert other.implied_att == pytest.approx(via_column.implied_att, abs=1e-15) + # The weights are non-uniform, so misalignment would have moved the number. + uniform = attgt_weights(gt, data=df, **kw) + assert abs(via_column.implied_att - uniform.implied_att) > 1e-6 + with pytest.raises(ValueError, match="indexed by exactly the units"): + attgt_weights(gt, data=df, weights=w_by_unit.iloc[1:], **kw) + with pytest.raises(ValueError, match="indexed by exactly the units"): + extra = pd.concat([w_by_unit, pd.Series([1.0], index=[-1])]) + attgt_weights(gt, data=df, weights=extra, **kw) + def test_rejects_time_varying_sampling_weights(self, fitted, panel): broken = panel.copy() broken["w"] = np.arange(len(broken), dtype=float) From 56aabebb3f02ba8c6aee271c4fc1f5ad043f310d Mon Sep 17 00:00:00 2001 From: igerber Date: Thu, 24 Sep 2026 06:34:01 -0400 Subject: [PATCH 13/13] fix(twfeweights): one value-sorted unit order on the frame path A categorical unit column is ordered by CATEGORY order under groupby, so the documented bare-array contract (value-sorted unit ids) did not match the order the cohort vector was actually built in: on ids with reversed categories the sorted-array route gave 1.7607 against 1.7919 from the column and Series routes. The frame path now builds one explicit value-sorted unit_ids array and reindexes the cohort vector and the weight column to it (observed=True so unused categories add no ghost groups); the Series route's extra-label check builds its set once. _Panel.unit_ids is the sorted frame's row order, which is what its reshaped blocks use. Regression test: categorical ids with reversed category order agree with the integer panel on all three weight routes and in the decomposition. --- diff_diff/twfe_weights.py | 43 +++++++++++++++++++++++++++--------- docs/methodology/REGISTRY.md | 2 +- tests/test_twfe_weights.py | 32 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py index 711e0563a..965dcecde 100644 --- a/diff_diff/twfe_weights.py +++ b/diff_diff/twfe_weights.py @@ -410,6 +410,12 @@ def _attgt_from_frame( return table.sort_values(["group", "time"]).reset_index(drop=True), skipped +def _sorted_unit_ids(units: pd.Series) -> np.ndarray: + """Value-sorted unit ids, as plain values (a categorical column's category + order is NOT used - the documented array contract is value order).""" + return np.asarray(sorted(pd.unique(np.asarray(units)))) + + def _unit_cohorts_from_frame( data: pd.DataFrame, unit: str, time: str, first_treat: str ) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: @@ -437,18 +443,26 @@ def _unit_cohorts_from_frame( ) # dropna=False: a unit whose label is NaN in one period must fail the # invariance check, not slip through because nunique() skipped the NaN. - per_unit = data.groupby(unit, sort=True)[first_treat].nunique(dropna=False) + # observed=True: a categorical unit column must not contribute ghost + # groups for unused categories. + per_unit = data.groupby(unit, sort=True, observed=True)[first_treat].nunique(dropna=False) if (per_unit > 1).any(): offenders = per_unit[per_unit > 1].index.tolist()[:5] raise ValueError( f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " "membership must be time-invariant" ) - firsts = data.groupby(unit, sort=True)[first_treat].first() + # ONE explicit unit order - value-sorted - for the cohort vector and for + # everything aligned to it (a bare weights= array, a weights column). A + # groupby(sort=True) on a categorical column orders by CATEGORY order, + # which need not be value order, so the cohorts are reindexed to the + # value-sorted ids rather than taken in groupby order. + unit_ids = _sorted_unit_ids(data[unit]) + firsts = data.groupby(unit, sort=True, observed=True)[first_treat].first().reindex(unit_ids) cohorts = firsts.to_numpy() - _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) + _validate_cohort_labels(cohorts, unit_ids=unit_ids, what=first_treat) periods = np.asarray(sorted(pd.unique(key))) - return cohorts, periods, firsts.index.to_numpy() + return cohorts, periods, unit_ids def _resolve_cs_inputs( @@ -617,8 +631,9 @@ def attgt_weights( indexed by unit id (aligned by label, so its order does not matter; it must cover exactly the units in ``data``), or a bare array with one value per unit in SORTED unit-id order (``sorted(data[unit].unique())``, - the order the cohort vector is built in - a label-free vector has no - other meaning, so prefer the column or the Series). The fitted path uses + the order the cohort vector is built in, value order even for a + categorical unit column - a label-free vector has no other meaning, + so prefer the column or the Series). The fitted path uses the fit's own survey weights (``SurveyDesign(weights=)``) and rejects an explicit ``weights=``: the ATT(g, t) were estimated under the fit's weights, and a different weight vector would make @@ -1083,19 +1098,25 @@ def _resolve_frame_weights( if isinstance(weights, str): if weights not in data.columns: raise ValueError(f"weights column {weights!r} not found in data") - per_unit = data.groupby(unit, sort=True)[weights].nunique(dropna=False) + per_unit = data.groupby(unit, sort=True, observed=True)[weights].nunique(dropna=False) if (per_unit > 1).any(): offenders = per_unit[per_unit > 1].index.tolist()[:5] raise ValueError( f"weights column {weights!r} varies within unit(s) " f"{offenders!r}; sampling weights must be time-invariant" ) - return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) + return ( + data.groupby(unit, sort=True, observed=True)[weights] + .first() + .reindex(unit_ids) + .to_numpy(dtype=float) + ) if isinstance(weights, pd.Series): if weights.index.has_duplicates: raise ValueError("weights Series index has duplicate unit labels") + known = set(unit_ids.tolist()) missing = [u for u in unit_ids if u not in weights.index] - extra = [u for u in weights.index if u not in set(unit_ids.tolist())] + extra = [u for u in weights.index if u not in known] if missing or extra: raise ValueError( "weights Series must be indexed by exactly the units in data=; " @@ -1184,7 +1205,9 @@ def __init__( frame = frame.sort_values([unit, "_twfe_time_key"]).reset_index(drop=True) units = frame[unit].to_numpy() periods = frame["_twfe_time_key"].to_numpy(dtype=float) - self.unit_ids = np.asarray(sorted(pd.unique(units))) + # Row order of the sorted frame (category order for a categorical unit + # column), i.e. the order every reshaped block below is in. + self.unit_ids = pd.unique(units) # The numeric key orders, reshapes and maps cohorts; the ORIGINAL # labels are what every reporting surface (cells, summary(), balance # rows, plots) shows, so a string-labelled panel reports strings. diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 3e0b617e0..1f2357c27 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -6298,7 +6298,7 @@ Verified on `mpdta` at `-0.03654894` from both directions. - **Note (a cohort that keeps one estimable post cell is kept, as `aggregate()` keeps it; deviation from R's `pre_process_did` drop):** for `"overall"` / `"simple"`, a surviving cohort's structurally absent post cells are carved out of the required grid: `"overall"` divides each cohort by its number of AVAILABLE post cells instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells and the reasons present. Two mechanisms reach this branch. (i) With no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g) - anticipation` (and recodes the last-treated cohort to never-treated only under `control_group="never_treated"`) (verified against `did` 2.5.1 `pre_process_did` source); that truncation - not `aggte` averaging over available periods - is what produces R's number, and the cells CS marks `zero_treated_control` name exactly the cells truncation removes, so the numerics agree. (ii) Under `base_period="varying"` with `anticipation > 0`, CS keeps a cohort through a single estimable pre-window cell while marking every later cell `missing_period` (verified by execution: cohorts (0,3,5), periods 1..6, `anticipation=2` keeps cohort 3 through `(3,2)`), and `aggregate("group")` / `aggregate("simple")` INCLUDE that cohort; R's `pre_process_did` would drop it outright. `attgt_weights` follows diff-diff's own aggregates, so its `implied_att` matches `aggregate()` and not R there. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group` / `base_period`, because the frame path carries no design metadata; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated, universal-base comparison and never reaches this branch; a bare frame without `skip_reason` stays strict. - **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. - **Note (`type="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping (pickled by diff-diff <= 3.12.0) fails closed with a refit `ValueError`, the same policy as the missing completeness record; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. -- **Note (sampling weights are validated, and aligned by unit label on the frame path):** unit weights must be finite, non-negative, with positive total and positive treated mass. On `attgt_weights`' DataFrame path a column name is collapsed per unit, a `pd.Series` is aligned to the panel's units BY LABEL (it must cover exactly those units), and a bare array is taken positionally in sorted unit-id order - the order the cohort vector is built in, stated in the docstring; a Series in panel-row order on a panel with non-monotonic ids was measured to move ATT^O's `implied_att` from 1.7607 to 1.7882 before the label alignment. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. +- **Note (sampling weights are validated, and aligned by unit label on the frame path):** unit weights must be finite, non-negative, with positive total and positive treated mass. On `attgt_weights`' DataFrame path a column name is collapsed per unit, a `pd.Series` is aligned to the panel's units BY LABEL (it must cover exactly those units), and a bare array is taken positionally in value-sorted unit-id order - the one explicit order the cohort vector and the weight column are both reindexed to (a categorical unit column's category order is not used; `groupby` would order by it), stated in the docstring; a Series in panel-row order on a panel with non-monotonic ids was measured to move ATT^O's `implied_att` from 1.7607 to 1.7882 before the label alignment. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `type="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. - `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. A treated cohort whose units all carry zero sampling weight is rejected by name: its cohort share and every one of its cells would be `0/0`, and the NaN would otherwise propagate silently into an all-NaN decomposition. On `attgt_weights`' frame path a cohort in the ATT(g,t) frame that no unit in `data=` belongs to is rejected by name rather than surfacing as a raw `KeyError`. - `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. - **Note (panel completeness is the producer's own record):** `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The kit's predicate is "every unit-period outcome cell present and finite, and no unit dropped by the producing estimator's own per-cell complete-case mask" - DMLDiD records its drops (`complete_case_drops`, set whenever a cell loop excluded a unit for a non-finite outcome, `dY`, base-period covariate, bad-control column or `W`; a non-finite covariate in a period no cell reads is not a drop); CallawaySantAnna's NaN-covariate fallback keeps the cohort masses intact and is not a rejection cause. This is stricter than CallawaySantAnna's internal `is_balanced` (`isnan`-only, for its own fast paths), which is unchanged. Results without the record (fitted by diff-diff <= 3.12.0) are refused with a refit message, as is a kit without the `covariates` record - one fail-closed policy for every legacy kit, since CS masses CAN diverge on an incomplete-panel pickle (`_aggregate_overall` weights by the reduced per-cell `n_treated`). The frame path carries no record and is the caller's responsibility; it requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py index 5ce9feeab..6ef71df30 100644 --- a/tests/test_twfe_weights.py +++ b/tests/test_twfe_weights.py @@ -325,6 +325,38 @@ def test_weights_align_by_unit_label_on_a_shuffled_panel(self): extra = pd.concat([w_by_unit, pd.Series([1.0], index=[-1])]) attgt_weights(gt, data=df, weights=extra, **kw) + def test_categorical_unit_ids_use_value_order_not_category_order(self): + """groupby orders a categorical column by CATEGORY order; the array + contract is value order, so cohorts and weights are reindexed to it.""" + df = _panel() + ids = np.arange(1000, 1000 + df["unit"].nunique()) + df["unit"] = df["unit"].map(dict(enumerate(ids))) + rng = np.random.RandomState(5) + w_by_unit = pd.Series(rng.choice([0.5, 1.0, 2.0], size=len(ids)), index=ids) + df["w"] = df["unit"].map(w_by_unit) + reversed_categories = pd.CategoricalDtype(categories=list(ids[::-1]), ordered=True) + categorical = df.assign(unit=df["unit"].astype(reversed_categories)) + gt = _gt_frame(_fit(df)) + kw = dict(unit="unit", time="period", first_treat="first_treat", type="overall") + reference = attgt_weights(gt, data=df, weights="w", **kw) + via_column = attgt_weights(gt, data=categorical, weights="w", **kw) + via_series = attgt_weights(gt, data=categorical, weights=w_by_unit, **kw) + via_array = attgt_weights( + gt, data=categorical, weights=w_by_unit.loc[sorted(ids)].to_numpy(), **kw + ) + for other in (via_column, via_series, via_array): + np.testing.assert_allclose( + other.weights["weight"].to_numpy(), + reference.weights["weight"].to_numpy(), + atol=1e-15, + ) + assert other.implied_att == pytest.approx(reference.implied_att, abs=1e-15) + # decompose_twfe_weights aligns everything through one sorted frame and + # is unaffected by the category order. + as_int = diff_diff.decompose_twfe_weights(df, weights="w", **_DECO) + as_cat = diff_diff.decompose_twfe_weights(categorical, weights="w", **_DECO) + assert as_cat.estimate == pytest.approx(as_int.estimate, abs=1e-12) + def test_rejects_time_varying_sampling_weights(self, fitted, panel): broken = panel.copy() broken["w"] = np.arange(len(broken), dtype=float)