Step 221: basic evaluation tasks

2026-02-10 08:00:43 -07:00
parent 575e4b782d
commit 66c1b50ad7
34 changed files with 555 additions and 1 deletions
--- a/eval/basic/basic_22.json
+++ b/eval/basic/basic_22.json
@@ -0,0 +1,18 @@
+{
+    "id":  "basic_22",
+    "description":  "Generate code from spec and run pipeline",
+    "language":  "python",
+    "setupAst":  "def f(x):\\n    return x + 1\\n",
+    "expectedOutcome":  {
+                            "toolCalls":  [
+                                              "whetstone_generate_code",
+                                              "whetstone_run_pipeline"
+                                          ]
+                        },
+    "tools":  [
+                  "whetstone_generate_code",
+                  "whetstone_run_pipeline"
+              ],
+    "maxSteps":  8,
+    "allowPartial":  false
+}