bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 221: basic evaluation tasks

Browse Source
This commit is contained in:
Bill
2026-02-10 08:00:43 -07:00
parent 575e4b782d
commit 66c1b50ad7
34 changed files with 555 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
eval/basic/basic_01.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_01",
"description": "Read AST and summarize structure",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast"
]
},
"tools": [
"whetstone_get_ast"
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_02.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_02",
"description": "Read AST and summarize structure",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast"
]
},
"tools": [
"whetstone_get_ast"
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_03.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_03",
"description": "Read AST and summarize structure",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast"
]
},
"tools": [
"whetstone_get_ast"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_04.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_04",
"description": "Apply a single mutation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_05.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_05",
"description": "Apply a single mutation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_06.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_06",
"description": "Apply a single mutation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_07.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_07",
"description": "Apply a single mutation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_08.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_08",
"description": "Apply a single mutation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_mutate"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_mutate"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_09.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_09",
"description": "Add a specific annotation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_10.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_10",
"description": "Add a specific annotation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_11.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_11",
"description": "Add a specific annotation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_12.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_12",
"description": "Add a specific annotation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_13.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_13",
"description": "Add a specific annotation",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

20
eval/basic/basic_14.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "basic_14",
"description": "Get annotation suggestions and apply",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

20
eval/basic/basic_15.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "basic_15",
"description": "Get annotation suggestions and apply",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

20
eval/basic/basic_16.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "basic_16",
"description": "Get annotation suggestions and apply",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

20
eval/basic/basic_17.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "basic_17",
"description": "Get annotation suggestions and apply",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

20
eval/basic/basic_18.json Normal file
View File

@@ -0,0 +1,20 @@
{
"id": "basic_18",
"description": "Get annotation suggestions and apply",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
]
},
"tools": [
"whetstone_get_ast",
"whetstone_suggest_annotations",
"whetstone_apply_annotation"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_19.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_19",
"description": "Generate code from spec and run pipeline",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_generate_code",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_generate_code",
"whetstone_run_pipeline"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_20.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_20",
"description": "Generate code from spec and run pipeline",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_generate_code",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_generate_code",
"whetstone_run_pipeline"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_21.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_21",
"description": "Generate code from spec and run pipeline",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_generate_code",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_generate_code",
"whetstone_run_pipeline"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_22.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_22",
"description": "Generate code from spec and run pipeline",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_generate_code",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_generate_code",
"whetstone_run_pipeline"
],
"maxSteps": 8,
"allowPartial": false
}

18
eval/basic/basic_23.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "basic_23",
"description": "Generate code from spec and run pipeline",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
"whetstone_generate_code",
"whetstone_run_pipeline"
]
},
"tools": [
"whetstone_generate_code",
"whetstone_run_pipeline"
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_24.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_24",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_25.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_25",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_26.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_26",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_27.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_27",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_28.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_28",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_29.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_29",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}

16
eval/basic/basic_30.json Normal file
View File

@@ -0,0 +1,16 @@
{
"id": "basic_30",
"description": "",
"language": "python",
"setupAst": "def f(x):\\n return x + 1\\n",
"expectedOutcome": {
"toolCalls": [
]
},
"tools": [
],
"maxSteps": 8,
"allowPartial": false
}