#!/usr/bin/env python3 """ gen_implementer_synthetic.py Generates synthetic implementer training examples with explicit positive signals. Problem: gen_worker_type_data.py labels implementer by keyword ABSENCE (anything that isn't reviewer/architect/qa). This produces noisy boundaries — the model never sees strong positive implementer signal, so tasks like "Build review moderation" get misclassified as reviewer because "review" is in the text. Fix: generate examples where: 1. Explicit construction verbs are present (Implement, Build, Add, Write, etc.) 2. Ambiguous nouns appear as OBJECTS of construction, not as the action e.g., "Build audit log service" = implementer (building it), NOT reviewer (doing audit) Output: writes synthetic rows to data/generated/worker_type_implementer_synthetic.tsv and a new combined file worker_type_v2_{train,eval}.tsv that includes the existing combined data + synthetic implementer. Usage: python3 specialists/scripts/gen_implementer_synthetic.py (run from whetstone_DSL root) """ import random import sys from pathlib import Path ROOT = Path(__file__).parent.parent.parent OUT_DIR = ROOT / "specialists" / "data" / "generated" COMBINED_TRAIN = OUT_DIR / "worker_type_combined_train.tsv" COMBINED_EVAL = OUT_DIR / "worker_type_combined_eval.tsv" SYNTH_OUT = OUT_DIR / "worker_type_implementer_synthetic.tsv" V2_TRAIN = OUT_DIR / "worker_type_v2_train.tsv" V2_EVAL = OUT_DIR / "worker_type_v2_eval.tsv" IMPLEMENTER_LABEL = 0 # --------------------------------------------------------------------------- # Short title templates (sprint-plan style) # These are the critical ones — they teach the model that "Build X", "Add X", # "Implement X" = implementer even when X contains ambiguous words. # --------------------------------------------------------------------------- SHORT_TITLES = [ # Clear construction verbs — unambiguous "Implement authentication middleware", "Implement rate limiter with sliding window", "Implement JWT token refresh endpoint", "Implement password reset flow", "Implement webhook dispatcher with retry logic", "Implement idempotency keys for payment API", "Implement session store with expiry", "Implement background job worker", "Implement batch processing pipeline", "Implement file upload endpoint with validation", "Implement pagination for list endpoints", "Implement soft delete for user records", "Implement caching layer with TTL", "Implement event sourcing for order state", "Implement circuit breaker for external calls", "Add authentication to all protected routes", "Add request validation middleware", "Add retry logic to job processor", "Add structured logging to API handlers", "Add metrics collection to request pipeline", "Add database connection pooling", "Add rate limiting to public endpoints", "Add health check endpoint", "Add gzip compression to response middleware", "Add CORS policy to API gateway", "Build webhook dispatcher with backoff", "Build order state machine with transitions", "Build notification fanout service", "Build background task scheduler", "Build file processing pipeline", "Build token bucket rate limiter", "Build ledger reconciliation service", "Build event deduplication processor", "Build incremental snapshot service", "Build search index synchronization worker", "Write parser for incoming event payloads", "Write serialization layer for queue messages", "Write migration script for schema v3", "Write batch loader for warehouse ingestion", "Write retry wrapper for flaky external calls", "Write adapter for third-party payment provider", "Create database schema for subscription billing", "Create index for high-frequency query paths", "Create worker pool for parallel job execution", "Create shared serialization core for all services", "Extend billing API with proration support", "Extend user profile with privacy settings", "Extend search with fuzzy match and ranking", "Extend notification service with quiet hours", "Port request validator to use shared schema library", "Port legacy sync logic to new event bus", "Develop inventory reservation endpoint", "Code the ledger double-entry enforcement layer", "Wire up SMTP adapter to notification service", "Integrate carrier rate API into shipping worker", "Hook telemetry collector into request pipeline", "Expose GraphQL subscription for live updates", "Stand up gRPC service for internal messaging", "Scaffold CRUD endpoints for product catalog", "Generate typed client from OpenAPI spec", # Ambiguous-noun implementer — these are the hardest cases # "Build/Implement X" where X sounds like audit/review/check/scan work "Build audit log service", "Build audit trail persistence layer", "Implement audit event schema and writer", "Add audit log endpoint to admin API", "Write audit log retention policy enforcer", "Build review queue backend", "Build review moderation pipeline", "Implement review scoring engine", "Add review status transitions to moderation API", "Build code review comment threading API", "Implement peer review assignment logic", "Build inspection endpoint for object metadata", "Implement diagnostic data collection service", "Build diagnostic report export endpoint", "Write scan queue worker for document processing", "Implement scan result aggregation service", "Build check endpoint for idempotent replay safety", "Implement liveness check handler", "Implement readiness check with dependency probes", "Build validation pipeline for incoming records", "Implement validator chain for form submissions", "Add constraint validation to execution contract", "Write schema validation middleware", "Build compliance data export endpoint", "Implement compliance record writer", "Build analysis batch job for feature distributions", "Implement drift analysis worker", "Write data quality check runner", "Build quality scoring endpoint for artifacts", "Implement quality gate enforcer in build pipeline", "Build assessment result storage API", "Implement evaluation batch processor", "Build evaluation harness runner", "Implement triage queue persistence layer", "Build triage result export service", "Implement monitoring probe for SLO windows", "Build monitor agent for heartbeat tracking", # More construction verbs on mixed topics "Implement RBAC permission check middleware", "Build RBAC role assignment API", "Add permission enforcement to resource endpoints", "Write access policy evaluation engine", "Implement access control grant/revoke API", "Build fraud score persistence and query API", "Implement fraud detection rule evaluator", "Write rule engine for fraud signal aggregation", "Build PII redaction pipeline", "Implement PII scanner for log output", "Add PII masking to export endpoints", "Build secret rotation scheduler", "Implement credential refresh worker", "Write certificate expiry checker worker", "Build security event publisher", "Implement security policy enforcement layer", "Add encryption at rest for sensitive fields", "Implement key derivation service", "Build compliance report generator", "Write regulatory data export endpoint", "Implement GDPR deletion request handler", "Build consent management API", ] # --------------------------------------------------------------------------- # Title | intent templates (projects-pipeline style) # Format: " | <intent[:120]>" # --------------------------------------------------------------------------- TITLE_INTENT_TEMPLATES = [ # Clear implementer with intent ("Implement JWT auth service", "Build register, login, refresh, and logout endpoints with bcrypt hashing and token rotation"), ("Build rate limiter service", "Implement token bucket algorithm with Redis backend, per-key limits, and burst allowance"), ("Add webhook dispatcher", "Implement retry queue with exponential backoff, signature verification, and dead-letter drain"), ("Build order state machine", "Implement status transitions with guard conditions, event emission, and rollback support"), ("Implement billing proration engine", "Calculate mid-cycle charges and credits with timezone-aware billing periods"), ("Build notification fanout service", "Implement async dispatch to email, SMS, and push channels with per-user preferences"), ("Add search indexing worker", "Build incremental sync from Postgres to Elasticsearch with schema evolution support"), ("Implement session store", "Build Redis-backed session persistence with configurable TTL and concurrent access safety"), ("Build file metadata indexer", "Implement attribute extraction, deduplication, and full-text search endpoint"), ("Add idempotency key enforcement", "Implement request deduplication middleware with configurable TTL and replay detection"), ("Build CDC replication worker", "Implement change-data-capture from Postgres WAL with schema migration and checkpointing"), ("Implement feature flag API", "Build per-environment flag storage, targeting rules, rollout percentages, and kill switch"), ("Build task runner executor", "Implement YAML-defined task graph with topological scheduling and parallel execution"), ("Implement snapshot compaction job", "Build incremental event log compaction into query-optimized state tables"), ("Build a distributed rate limiter", "Implement fixed and sliding window modes, per-tenant policies, and lock-free counters"), # Ambiguous-noun implementer with intent ("Build audit log service", "Implement immutable event writer, retention policies, and export endpoints with filtering"), ("Implement audit trail persistence", "Build append-only audit records with tamper detection and compliance export format"), ("Build review queue backend", "Implement work assignment, priority ordering, and reviewer load balancing APIs"), ("Build code review comment API", "Implement threaded comments, reactions, resolution tracking, and diff anchoring"), ("Implement moderation pipeline", "Build content scoring with rule engine, human escalation queue, and status transitions"), ("Build review scoring engine", "Implement rubric evaluation, aggregate scoring, and confidence weighting logic"), ("Build diagnostic report service", "Implement data collection, aggregation, and structured export for operational diagnostics"), ("Implement scan result aggregator", "Build result storage, deduplication, and summary endpoint for document scan outputs"), ("Build validation pipeline", "Implement multi-stage validator chain with error collection and partial-pass semantics"), ("Implement schema validation middleware", "Build request body validator against versioned JSON schemas with detailed error paths"), ("Build compliance report generator", "Implement data aggregation, template rendering, and signed export for regulatory filings"), ("Implement data quality check runner", "Build null, range, uniqueness, and referential integrity checks over ingested datasets"), ("Build assessment result storage", "Implement structured result persistence, query API, and time-series aggregation"), ("Implement evaluation batch processor", "Build parallel rubric scoring with retry handling and result merging pipeline"), ("Build monitoring probe service", "Implement health check runners, SLO window tracking, and breach notification triggers"), ("Implement liveness and readiness handlers", "Build Kubernetes probe endpoints with dependency check logic and graceful degradation"), ("Build triage queue persistence", "Implement priority queue storage with lease management and completion acknowledgment"), ("Implement fraud rule evaluator", "Build rule execution engine with score aggregation, threshold gates, and audit trail"), ("Build RBAC permission API", "Implement role assignment, permission inheritance, and resource-scoped grant endpoints"), ("Implement access control enforcement", "Build middleware that evaluates hierarchical allow/deny policies with shadow-rule logging"), ("Build PII redaction pipeline", "Implement field-level masking for emails, phones, and IDs with configurable audit mode"), ("Implement security event publisher", "Build structured event emission for auth failures, permission denials, and anomalies"), ("Implement GDPR deletion handler", "Build cascading delete workflow across services with confirmation receipts and audit log"), ("Build credential rotation worker", "Implement scheduled secret rotation with staged rollout and rollback on verification fail"), ] def write_rows(path: Path, rows: list[tuple[int, str]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: for label, text in rows: f.write(f"{label}\t0\t{text}\n") print(f"Wrote {len(rows)} rows → {path}") def main() -> None: rng = random.Random(42) # Build synthetic implementer rows from all templates rows: list[tuple[int, str]] = [] for title in SHORT_TITLES: rows.append((IMPLEMENTER_LABEL, title)) for title, intent in TITLE_INTENT_TEMPLATES: rows.append((IMPLEMENTER_LABEL, f"{title} | {intent[:120]}")) rng.shuffle(rows) print(f"Generated {len(rows)} synthetic implementer rows") # Split 15% eval n_eval = max(1, int(len(rows) * 0.15)) synth_eval = rows[:n_eval] synth_train = rows[n_eval:] write_rows(SYNTH_OUT.with_suffix(".train.tsv"), synth_train) write_rows(SYNTH_OUT.with_suffix(".eval.tsv"), synth_eval) # Build v2 combined: existing combined + synthetic for split, existing, synth, out in [ ("train", COMBINED_TRAIN, synth_train, V2_TRAIN), ("eval", COMBINED_EVAL, synth_eval, V2_EVAL), ]: existing_rows = existing.read_text().splitlines(keepends=True) with open(out, "w") as f: f.writelines(existing_rows) for label, text in synth: f.write(f"{label}\t0\t{text}\n") total = len(existing_rows) + len(synth) print(f"Wrote {total} rows ({len(existing_rows)} existing + {len(synth)} synthetic) → {out}") # Print label distribution in v2 train from collections import Counter label_names = {0: "implementer", 1: "reviewer", 2: "architect", 3: "qa"} counts: Counter = Counter() for line in V2_TRAIN.read_text().splitlines(): if line.strip(): label = int(line.split("\t")[0]) counts[label] += 1 print("\nv2 train label distribution:") for label, name in sorted(label_names.items()): print(f" {name}: {counts.get(label, 0)}") print(f" total: {sum(counts.values())}") if __name__ == "__main__": main()