harry/patterm
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 4 Wiki Activity

Add idle-state classifier and Solo-parity timer tools

Browse Source 
Classifies every running child as idle/working/thinking/permission/error
using one of three pluggable strategies (output_activity,
osc_title_stability, osc_title_status) plus optional regex promoters
applied to the tail of recent output. State and last-match reason are
exposed via MCP on ProcessInfo and get_process_status. Per-preset
configuration lives on a new preset.IdleDetection block with bundled
defaults for the first-party claude/codex/opencode presets.

OSC title plumbing is exposed as Emulator.Title(), polled from the
session pump after each emulator write so title-change activity feeds
into the classifier without an extra cgo callback.

The MCP timer surface expands to match Solo: timer_set,
timer_fire_when_idle_any/all, timer_cancel, timer_pause, timer_resume,
timer_list. timer_wait is now a thin wrapper that shares the same
manager so it shows up in timer_list while pending. Timer bodies are
delivered to the owner process through the existing
InjectAsOrchestrator path. Top-level (non-agent) callers can attach
timers to a specific process via owner_process_id; omitting it grants
universal cancel/pause/resume/list privileges.

The sidebar gains a state glyph per process row and appends a
nearest-timer indicator when one is pending or paused.

Tests: idle_test.go covers the classify() pure function across the
three strategies and regex promotion; timers_test.go covers the
manager. Harness scenarios cover output_activity, osc_title_stability,
osc_title_status, and regex promotion, plus timer_set delivery,
cancel, pause/resume, idle_any-on-transition, idle_all-pending, and
idle_all-already-satisfied. A new wait_until_mcp harness step type
polls an MCP method until an assertion holds.
This commit is contained in:
Harry Bayliss
2026-05-15 09:49:59 +01:00
parent 1af032472b
commit 2b9e1ed77c
31 changed files with 2318 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
internal/harness/runner.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"regexp"
"strings"
"time"
)
type Event struct {
@@ -175,6 +176,41 @@ func runStep(s *Session, step Step, results map[string]json.RawMessage) error {
return fmt.Errorf("no saved result %q", step.From)
}
return assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring)
case "wait_until_mcp":
// Poll an MCP method until the assertion at Path holds (or
// Contains substring matches), or TimeoutMS elapses. Used by the
// idle-detection scenarios to wait for a child's idle_state to
// reach a target value without sprinkling sleeps.
params, perr := resolveParams(step.Params, results)
if perr != nil {
return perr
}
deadline := time.Now().Add(timeoutMS(step.TimeoutMS))
var lastRaw json.RawMessage
var lastErr error
for {
raw, err := s.MCPCall(step.Method, params)
if err == nil {
if aerr := assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring); aerr == nil {
if step.SaveAs != "" {
results[step.SaveAs] = raw
}
return nil
} else {
lastErr = aerr
lastRaw = raw
}
} else {
lastErr = err
}
if time.Now().After(deadline) {
if lastErr != nil {
return fmt.Errorf("wait_until_mcp timeout: %w (last response: %s)", lastErr, string(lastRaw))
}
return fmt.Errorf("wait_until_mcp timeout (no successful call)")
}
time.Sleep(100 * time.Millisecond)
}
}
return fmt.Errorf("unknown step type %q", step.Type)
}

22
internal/harness/scenario.go
View File

@@ -25,11 +25,23 @@ type ScenarioPresets struct {
}
type ScenarioPreset struct {
Name string `json:"name"`
Argv []string `json:"argv"`
Env map[string]string `json:"env,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Shell bool `json:"shell,omitempty"`
Name string `json:"name"`
Argv []string `json:"argv"`
Env map[string]string `json:"env,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Shell bool `json:"shell,omitempty"`
IdleDetection *ScenarioIdleDetection `json:"idle_detection,omitempty"`
}
// ScenarioIdleDetection mirrors preset.IdleDetection so scenarios can
// configure per-strategy idle detection for fake agent presets.
type ScenarioIdleDetection struct {
Strategy string `json:"strategy,omitempty"`
IdleThresholdMS int `json:"idle_threshold_ms,omitempty"`
TitleStatusMap map[string]string `json:"title_status_map,omitempty"`
PermissionPatterns []string `json:"permission_patterns,omitempty"`
ThinkingPatterns []string `json:"thinking_patterns,omitempty"`
ErrorPatterns []string `json:"error_patterns,omitempty"`
}
type ScenarioScript struct {

44
internal/harness/scenarios/idle_osc_title_stability.json Normal file
View File

@@ -0,0 +1,44 @@
{
"name": "idle_osc_title_stability",
"presets": {
"processes": [
{
"name": "titler",
"argv": [
"sh",
"-lc",
"i=0; while [ $i -lt 6 ]; do printf '\\033]2;step %d\\007' $i; i=$((i+1)); sleep 0.2; done; sleep 60"
],
"idle_detection": {
"strategy": "osc_title_stability",
"idle_threshold_ms": 1000
}
}
]
},
"trust": ["titler"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "titler", "name": "titler"},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "working",
"timeout_ms": 3000
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "idle",
"timeout_ms": 4000
}
]
}

48
internal/harness/scenarios/idle_osc_title_status.json Normal file
View File

@@ -0,0 +1,48 @@
{
"name": "idle_osc_title_status",
"presets": {
"processes": [
{
"name": "geminilike",
"argv": [
"sh",
"-lc",
"printf '\\033]2;Thinking\\007'; sleep 1; printf '\\033]2;Permission required\\007'; sleep 60"
],
"idle_detection": {
"strategy": "osc_title_status",
"idle_threshold_ms": 1000,
"title_status_map": {
"thinking": "thinking",
"permission": "permission"
}
}
}
]
},
"trust": ["geminilike"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "geminilike", "name": "geminilike"},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "thinking",
"timeout_ms": 3000
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "permission",
"timeout_ms": 4000
}
]
}

44
internal/harness/scenarios/idle_output_activity.json Normal file
View File

@@ -0,0 +1,44 @@
{
"name": "idle_output_activity",
"presets": {
"processes": [
{
"name": "blinker",
"argv": ["sh", "-lc", "echo step1; sleep 3; echo step2; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 1000
}
}
]
},
"trust": ["blinker"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {
"kind": "command",
"preset": "blinker",
"name": "blinker"
},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "working",
"timeout_ms": 4000
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "idle",
"timeout_ms": 4000
}
]
}

33
internal/harness/scenarios/idle_regex_promote.json Normal file
View File

@@ -0,0 +1,33 @@
{
"name": "idle_regex_promote",
"presets": {
"processes": [
{
"name": "approver",
"argv": ["sh", "-lc", "echo 'Do you want to proceed?'; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 500,
"permission_patterns": ["Do you want to proceed\\?"]
}
}
]
},
"trust": ["approver"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "approver", "name": "approver"},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "permission",
"timeout_ms": 4000
}
]
}

44
internal/harness/scenarios/timer_cancel.json Normal file
View File

@@ -0,0 +1,44 @@
{
"name": "timer_cancel",
"presets": {
"processes": [
{
"name": "echoer",
"argv": ["sh", "-lc", "while read line; do echo \"saw:$line\"; done"]
}
]
},
"trust": ["echoer"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "echoer", "name": "echoer"},
"save_as": "proc"
},
{ "type": "wait_stable", "timeout_ms": 1500 },
{
"type": "mcp_call",
"method": "timer_set",
"params": {"seconds": 1, "body": "should-not-arrive", "owner_process_id": "{{proc.process_id}}"},
"save_as": "tmr"
},
{
"type": "mcp_call",
"method": "timer_cancel",
"params": {"timer_id": "{{tmr.timer_id}}"}
},
{
"type": "mcp_call",
"method": "timer_list",
"params": {"owner_process_id": "{{proc.process_id}}"},
"save_as": "listed"
},
{
"type": "assert_saved",
"from": "listed",
"path": "",
"equals": []
}
]
}

48
internal/harness/scenarios/timer_idle_all_already_satisfied.json Normal file
View File

@@ -0,0 +1,48 @@
{
"name": "timer_idle_all_already_satisfied",
"presets": {
"processes": [
{
"name": "quiet",
"argv": ["sh", "-lc", "echo ready; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 500
}
}
]
},
"trust": ["quiet"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "quiet", "name": "quiet"},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{proc.process_id}}"},
"path": "idle_state",
"equals": "idle",
"timeout_ms": 4000
},
{
"type": "mcp_call",
"method": "timer_fire_when_idle_all",
"params": {
"watched": ["{{proc.process_id}}"],
"body": "all-idle",
"owner_process_id": "{{proc.process_id}}"
},
"save_as": "resp"
},
{
"type": "assert_saved",
"from": "resp",
"path": "status",
"equals": "already_satisfied"
}
]
}

89
internal/harness/scenarios/timer_idle_all_pending.json Normal file
View File

@@ -0,0 +1,89 @@
{
"name": "timer_idle_all_pending",
"presets": {
"processes": [
{
"name": "echoer",
"argv": ["sh", "-lc", "while read line; do echo \"saw:$line\"; done"]
},
{
"name": "quiet",
"argv": ["sh", "-lc", "echo ready; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 500
}
},
{
"name": "busy",
"argv": ["sh", "-lc", "for i in 1 2 3 4 5; do echo tick $i; sleep 0.2; done; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 500
}
}
]
},
"trust": ["echoer", "quiet", "busy"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "echoer", "name": "echoer"},
"save_as": "owner"
},
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "quiet", "name": "quiet"},
"save_as": "q"
},
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "busy", "name": "busy"},
"save_as": "b"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{q.process_id}}"},
"path": "idle_state",
"equals": "idle",
"timeout_ms": 3000
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{b.process_id}}"},
"path": "idle_state",
"equals": "working",
"timeout_ms": 3000
},
{
"type": "mcp_call",
"method": "timer_fire_when_idle_all",
"params": {
"watched": ["{{q.process_id}}", "{{b.process_id}}"],
"body": "all-idle",
"owner_process_id": "{{owner.process_id}}"
},
"save_as": "resp"
},
{
"type": "assert_saved",
"from": "resp",
"path": "status",
"equals": "pending"
},
{
"type": "wait_until_mcp",
"method": "get_process_output",
"params": {"process_id": "{{owner.process_id}}", "mode": "grid"},
"path": "content",
"contains": "saw:all-idle",
"allow_substring": true,
"timeout_ms": 6000
}
]
}

67
internal/harness/scenarios/timer_idle_any_fires_on_transition.json Normal file
View File

@@ -0,0 +1,67 @@
{
"name": "timer_idle_any_fires_on_transition",
"presets": {
"processes": [
{
"name": "echoer",
"argv": ["sh", "-lc", "while read line; do echo \"saw:$line\"; done"]
},
{
"name": "busy",
"argv": ["sh", "-lc", "for i in 1 2 3 4 5; do echo tick $i; sleep 0.2; done; sleep 60"],
"idle_detection": {
"strategy": "output_activity",
"idle_threshold_ms": 500
}
}
]
},
"trust": ["echoer", "busy"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "echoer", "name": "echoer"},
"save_as": "owner"
},
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "busy", "name": "busy"},
"save_as": "watch"
},
{
"type": "wait_until_mcp",
"method": "get_process_status",
"params": {"process_id": "{{watch.process_id}}"},
"path": "idle_state",
"equals": "working",
"timeout_ms": 3000
},
{
"type": "mcp_call",
"method": "timer_fire_when_idle_any",
"params": {
"watched": ["{{watch.process_id}}"],
"body": "any-idle",
"owner_process_id": "{{owner.process_id}}"
},
"save_as": "resp"
},
{
"type": "assert_saved",
"from": "resp",
"path": "status",
"equals": "pending"
},
{
"type": "wait_until_mcp",
"method": "get_process_output",
"params": {"process_id": "{{owner.process_id}}", "mode": "grid"},
"path": "content",
"contains": "saw:any-idle",
"allow_substring": true,
"timeout_ms": 6000
}
]
}

62
internal/harness/scenarios/timer_pause_resume.json Normal file
View File

@@ -0,0 +1,62 @@
{
"name": "timer_pause_resume",
"presets": {
"processes": [
{
"name": "echoer",
"argv": ["sh", "-lc", "while read line; do echo \"saw:$line\"; done"]
}
]
},
"trust": ["echoer"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "echoer", "name": "echoer"},
"save_as": "proc"
},
{ "type": "wait_stable", "timeout_ms": 1500 },
{
"type": "mcp_call",
"method": "timer_set",
"params": {
"seconds": 1,
"body": "after-resume",
"owner_process_id": "{{proc.process_id}}"
},
"save_as": "tmr"
},
{
"type": "mcp_call",
"method": "timer_pause",
"params": {"timer_id": "{{tmr.timer_id}}"}
},
{
"type": "mcp_call",
"method": "timer_list",
"params": {"owner_process_id": "{{proc.process_id}}"},
"save_as": "listed"
},
{
"type": "assert_saved",
"from": "listed",
"path": "0.status",
"equals": "paused"
},
{
"type": "mcp_call",
"method": "timer_resume",
"params": {"timer_id": "{{tmr.timer_id}}"}
},
{
"type": "wait_until_mcp",
"method": "get_process_output",
"params": {"process_id": "{{proc.process_id}}", "mode": "grid"},
"path": "content",
"contains": "saw:after-resume",
"allow_substring": true,
"timeout_ms": 5000
}
]
}

40
internal/harness/scenarios/timer_set_delivers.json Normal file
View File

@@ -0,0 +1,40 @@
{
"name": "timer_set_delivers",
"presets": {
"processes": [
{
"name": "echoer",
"argv": ["sh", "-lc", "while read line; do echo \"saw:$line\"; done"]
}
]
},
"trust": ["echoer"],
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {"kind": "command", "preset": "echoer", "name": "echoer"},
"save_as": "proc"
},
{ "type": "wait_stable", "timeout_ms": 1500 },
{
"type": "mcp_call",
"method": "timer_set",
"params": {
"seconds": 0.5,
"body": "hello-from-timer",
"owner_process_id": "{{proc.process_id}}"
},
"save_as": "tmr"
},
{
"type": "wait_until_mcp",
"method": "get_process_output",
"params": {"process_id": "{{proc.process_id}}", "mode": "grid"},
"path": "content",
"contains": "saw:hello-from-timer",
"allow_substring": true,
"timeout_ms": 5000
}
]
}