52 lines
1.6 KiB
Python
52 lines
1.6 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
def load_rows(path: Path):
|
||
|
|
rows = []
|
||
|
|
with path.open('r', encoding='utf-8') as f:
|
||
|
|
for line in f:
|
||
|
|
line = line.strip()
|
||
|
|
if line:
|
||
|
|
rows.append(json.loads(line))
|
||
|
|
return rows
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
ap.add_argument('--results', required=True)
|
||
|
|
ap.add_argument('--out', required=True)
|
||
|
|
ap.add_argument('--max-token-ratio', type=float, default=8.0)
|
||
|
|
args = ap.parse_args()
|
||
|
|
|
||
|
|
rows = load_rows(Path(args.results))
|
||
|
|
low_yield = []
|
||
|
|
for r in rows:
|
||
|
|
ab = r.get('ab') or {}
|
||
|
|
ta = int(ab.get('path_a_total_tokens', 0) or 0)
|
||
|
|
tb = int(ab.get('path_b_total_tokens', 0) or 0)
|
||
|
|
ready_b = bool(ab.get('path_b_ready', False))
|
||
|
|
ratio = (tb / ta) if ta > 0 else 0.0
|
||
|
|
if ratio > args.max_token_ratio and not ready_b:
|
||
|
|
low_yield.append({
|
||
|
|
'id': r.get('id'),
|
||
|
|
'language_exec': r.get('language_exec'),
|
||
|
|
'category': r.get('category'),
|
||
|
|
'ratio': round(ratio, 4),
|
||
|
|
'path_b_failure_reasons': ab.get('path_b_failure_reasons', []),
|
||
|
|
})
|
||
|
|
|
||
|
|
out = {
|
||
|
|
'max_token_ratio': args.max_token_ratio,
|
||
|
|
'total_runs': len(rows),
|
||
|
|
'low_yield_count': len(low_yield),
|
||
|
|
'pass': len(low_yield) == 0,
|
||
|
|
'low_yield_runs': low_yield,
|
||
|
|
}
|
||
|
|
Path(args.out).write_text(json.dumps(out, indent=2) + '\n', encoding='utf-8')
|
||
|
|
print(json.dumps(out))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
raise SystemExit(main())
|