Step 222: workflow evaluation tasks

2026-02-10 08:02:24 -07:00
parent 66c1b50ad7
commit 1ce68176d1
24 changed files with 406 additions and 1 deletions
--- a/eval/workflows/workflow_03.json
+++ b/eval/workflows/workflow_03.json
@@ -0,0 +1,20 @@
+{
+    "id":  "workflow_03",
+    "description":  "Annotate all functions in module",
+    "language":  "python",
+    "setupAst":  "def f(x):\\n    return x + 1\\n\\ndef g(y):\\n    return y * 2\\n",
+    "expectedOutcome":  {
+                            "toolCalls":  [
+                                              "whetstone_get_ast",
+                                              "whetstone_suggest_annotations",
+                                              "whetstone_apply_annotation"
+                                          ]
+                        },
+    "tools":  [
+                  "whetstone_get_ast",
+                  "whetstone_suggest_annotations",
+                  "whetstone_apply_annotation"
+              ],
+    "maxSteps":  12,
+    "allowPartial":  true
+}