bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 222: workflow evaluation tasks

Browse Source
This commit is contained in:
Bill
2026-02-10 08:02:24 -07:00
parent 66c1b50ad7
commit 1ce68176d1
24 changed files with 406 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
eval/workflows/workflow_01.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_01",
"description": "Annotate all functions in module",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_02.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_02",
"description": "Annotate all functions in module",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_03.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_03",
"description": "Annotate all functions in module",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

18
eval/workflows/workflow_04.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "workflow_04",
"description": "Cross-language projection with fixes",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_project",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_project",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

18
eval/workflows/workflow_05.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "workflow_05",
"description": "Cross-language projection with fixes",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_project",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_project",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

18
eval/workflows/workflow_06.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "workflow_06",
"description": "Cross-language projection with fixes",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_project",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_project",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

18
eval/workflows/workflow_07.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "workflow_07",
"description": "Cross-language projection with fixes",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_project",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_project",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

18
eval/workflows/workflow_08.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "workflow_08",
"description": "Cross-language projection with fixes",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_project",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_project",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_09.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_09",
"description": "Refactor pattern with batch mutate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_10.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_10",
"description": "Refactor pattern with batch mutate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_11.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_11",
"description": "Refactor pattern with batch mutate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_12.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_12",
"description": "Refactor pattern with batch mutate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_13.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_13",
"description": "Refactor pattern with batch mutate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate",
"whetstone_run_pipeline"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_14.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_14",
"description": "Debug annotation errors and revalidate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_15.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_15",
"description": "Debug annotation errors and revalidate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_16.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_16",
"description": "Debug annotation errors and revalidate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_17.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_17",
"description": "Debug annotation errors and revalidate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

20
eval/workflows/workflow_18.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "workflow_18",
"description": "Debug annotation errors and revalidate",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_run_pipeline",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 12,
"allowPartial": true
}

16
eval/workflows/workflow_19.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "workflow_19",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 12,
"allowPartial": true
}

16
eval/workflows/workflow_20.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "workflow_20",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n\\ndef g(y):\\n return y * 2\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 12,
"allowPartial": true
}