File size: 7,656 Bytes
028081c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 | #!/usr/bin/env python3
"""Ensemble builder — reproduces go.csv from 19 per-cand prediction CSVs.
Reads `candidate_csvs/<descriptive_name>.csv` and computes V2 medoid_ngram per row.
No GPU needed. Pure CPU + ROUGE scoring (rouge-score==0.1.2).
Run from /mnt/msrh/Magic_submission/ folder:
python /mnt/msrh/Magic_submission/scripts/build_ensemble.py
Output:
- A regenerated submission CSV written under /mnt/msrh/Magic_submission/.
- The script also prints its md5 alongside the shipped go.csv (a2ecca4a8e1aa01acf9a8b9a1d56ebf2)
so you can compare. md5 byte-equality is NOT guaranteed across machines (see README §3
"Note on byte-identity vs functional reproducibility"); functional LB equivalence is.
"""
import csv, pathlib, json, sys
from rouge_score import rouge_scorer
# Resolve paths relative to this script
ROOT = pathlib.Path("/mnt/msrh/Magic_submission")
CAND_DIR = ROOT / "candidate_csvs"
OUT_DIR = ROOT
DATA_DIR = ROOT / "data"
if not (DATA_DIR / "Test.csv").exists():
print(f"ERROR: Test.csv not found at {DATA_DIR/'Test.csv'}")
print("Place Test.csv + SampleSubmission.csv in /mnt/msrh/Magic_submission/data/ then re-run.")
sys.exit(1)
TEST_CSV = DATA_DIR / "Test.csv"
SAMPLE_CSV = DATA_DIR / "SampleSubmission.csv"
# 19 candidates in EXACT ORDER — order matters because medoid ties resolve to
# the first-encountered cand. Reordering changes ties → different output bytes.
CAND_ORDER = [
# ─── 12 fewshot LoRA cands + 3 no-fewshot mediators ───
# 3 Q3.5-27B K=3 variants (3ep ck-1200/1100, 5ep ck-1200)
"Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv",
"Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv",
"Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv",
# Q3.5-27B K=4, K=7 peak ckpts (v1 prompt)
"Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv",
"Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv",
# Q3.6-27B family (K=3 RecA, K=4, K=5 peak, K=7)
"Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv",
"Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv",
"Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv",
"Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv",
# Q3-32B family (K=3, K=5, K=7)
"Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv",
"Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv",
"Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv",
# 3 no-fewshot baselines (mediators) — long-trained without demos for decorrelation
"Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv",
"Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv",
"Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv",
# ─── v8 anchored-extraction prompt (Q3.5 K=5) ───
"Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv", # ⭐ best standalone 0.72325 LB
# ─── 3 cross-arch EARLY ckpts (anti-overfit) ───
"Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv", # ⭐ early-tap (vs peak ck-1600)
"Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv", # ⭐ early-tap
"Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv", # ⭐ very early-tap (vs peak ck-1200)
]
# Standalone LB for each cand (verified on Zindi public LB, for documentation only)
CAND_LB = {
"Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1200.csv": 0.7148,
"Qwen3.5-27B-3fewshots-bs64-3eps-ckpt-1100.csv": 0.7124,
"Qwen3.5-27B-3fewshots-bs64-5eps-ckpt-1200.csv": 0.7102,
"Qwen3.5-27B-4fewshots-bs64-3eps-ckpt-1600.csv": 0.7162,
"Qwen3.5-27B-5fewshots-bs64-3eps-v8prompt-ckpt-1500.csv": 0.72325,
"Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.7150,
"Qwen3.5-27B-7fewshots-bs64-3eps-ckpt-1200.csv": 0.7084,
"Qwen3.6-27B-3fewshots-bs64-3eps-ckpt-1600.csv": 0.7060,
"Qwen3.6-27B-4fewshots-bs64-3eps-ckpt-1400.csv": 0.7091,
"Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1200.csv": 0.7136,
"Qwen3.6-27B-5fewshots-bs64-3eps-ckpt-1000.csv": 0.7086,
"Qwen3.6-27B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.7194,
"Qwen3-32B-3fewshots-bs64-3eps-ckpt-1400.csv": 0.7011,
"Qwen3-32B-5fewshots-bs64-3eps-ckpt-1700.csv": 0.7081,
"Qwen3-32B-7fewshots-bs64-3eps-ckpt-1600.csv": 0.71383,
"Qwen3-32B-7fewshots-bs64-3eps-ckpt-1200.csv": 0.7111,
"Qwen3.5-27B-NoFewshots-bs64-5eps-ckpt-2800.csv": 0.6948,
"Qwen3.6-27B-NoFewshots-bs64-5eps-ckpt-2600.csv": 0.6933,
"Qwen3-32B-NoFewshots-bs64-4eps-ckpt-6500.csv": 0.6884,
}
scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2"], use_stemmer=False)
def load(path):
with open(path, newline="") as f:
r = csv.DictReader(f)
col = "TargetRLF1" if "TargetRLF1" in r.fieldnames else r.fieldnames[1]
return {row["ID"]: str(row[col]) for row in r}
def medoid(texts):
"""V2 medoid_ngram: pick text with HIGHEST sum of pairwise (R1.F + R2.F) to all others."""
best, best_s = 0, -1.0
for i in range(len(texts)):
s = 0.0
for j in range(len(texts)):
if i == j:
continue
rr = scorer.score(texts[j], texts[i])
s += rr["rouge1"].fmeasure + rr["rouge2"].fmeasure
if s > best_s:
best_s, best = s, i
return best
def main():
print(f"Loading {len(CAND_ORDER)} cand CSVs from {CAND_DIR}...")
all_preds = {}
for name in CAND_ORDER:
path = CAND_DIR / name
if not path.exists():
print(f" MISSING: {name}")
continue
all_preds[name] = load(path)
print(f" OK: {name} ({len(all_preds[name])} rows)")
if len(all_preds) != len(CAND_ORDER):
print(f"ERROR: only {len(all_preds)}/{len(CAND_ORDER)} cand CSVs loaded.")
sys.exit(1)
# Load Test IDs in order
with open(TEST_CSV, newline="") as f:
ids = [r["ID"] for r in csv.DictReader(f)]
with open(SAMPLE_CSV, newline="") as f:
sample_cols = next(csv.reader(f))
target_cols = [c for c in sample_cols if c.startswith("Target")]
print(f"\nProcessing {len(ids)} Test rows × {len(CAND_ORDER)} cands "
f"({len(CAND_ORDER)*(len(CAND_ORDER)-1)//2} pairwise per row)...")
members = list(CAND_ORDER) # ORDER critical for medoid tiebreak
picks, chosen = [], []
for i, id_ in enumerate(ids):
if i % 500 == 0:
print(f" row {i}/{len(ids)}")
texts = [all_preds[m].get(id_, "") for m in members]
p = medoid(texts)
picks.append(p)
chosen.append(texts[p])
out_path = OUT_DIR / "go_reproduced.csv"
with open(out_path, "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=sample_cols, lineterminator="\n")
w.writeheader()
for id_, ans in zip(ids, chosen):
w.writerow({"ID": id_, **{tc: ans for tc in target_cols}})
print(f"\n=== DONE ===")
print(f"Output: {out_path}")
# Pick distribution
print("\nPick distribution (per cand %):")
for i, m in enumerate(members):
n = picks.count(i)
pct = n / len(picks) * 100
marker = " ⭐" if "v8prompt" in m or "ckpt-1200" in m or "ckpt-1000" in m else ""
print(f" {m:<60} {n:>4} ({pct:>5.1f}%){marker}")
# Verify match
import hashlib
h = hashlib.md5(open(out_path, "rb").read()).hexdigest()
EXPECTED = "a2ecca4a8e1aa01acf9a8b9a1d56ebf2"
if h == EXPECTED:
print(f"\n✅ MD5 MATCH: {h} (byte-identical to go.csv)")
else:
print(f"\n❌ MD5 MISMATCH:")
print(f" Reproduced: {h}")
print(f" Expected : {EXPECTED}")
if __name__ == "__main__":
main()
|