diff --git a/.gitignore b/.gitignore index ce72d8e..e41b561 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ spike-report-*.txt /.zig-cache/ /bin/ /spike +/.worktrees/ +internal/harness/.artifacts/ diff --git a/cmd/patterm/debug_harness.go b/cmd/patterm/debug_harness.go new file mode 100644 index 0000000..dee275b --- /dev/null +++ b/cmd/patterm/debug_harness.go @@ -0,0 +1,47 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/hjbdev/patterm/internal/harness" +) + +func runDebugHarness() { + var ( + scenarioPath = flag.String("scenario", "", "JSON scenario path") + pattermBin = flag.String("patterm-bin", "", "patterm binary under test (default: this executable)") + ) + flag.Parse() + if *scenarioPath == "" { + die("debug-harness: --scenario is required") + } + if *pattermBin == "" { + exe, err := os.Executable() + if err != nil { + die("debug-harness: os.Executable: %v", err) + } + *pattermBin = exe + } + sc, err := harness.LoadScenario(*scenarioPath) + if err != nil { + fmt.Fprintf(os.Stdout, "{\"ok\":false,\"error\":%q}\n", err.Error()) + os.Exit(2) + } + enc := json.NewEncoder(os.Stdout) + res, err := harness.RunScenario(harness.Options{ + Scenario: sc, + PattermBin: *pattermBin, + }, func(ev harness.Event) { + _ = enc.Encode(ev) + }) + _ = enc.Encode(res) + if err != nil { + if res.Steps == 0 { + os.Exit(2) + } + os.Exit(1) + } +} diff --git a/cmd/patterm/main.go b/cmd/patterm/main.go index 6f55d0d..e45d141 100644 --- a/cmd/patterm/main.go +++ b/cmd/patterm/main.go @@ -7,6 +7,8 @@ // patterm mcp-stdio --socket S --identity I // internal: stdio MCP proxy spawned for // children, forwards JSON-RPC over S +// patterm debug-harness --scenario S +// internal: run a black-box harness scenario package main import ( @@ -30,6 +32,11 @@ func main() { runMCPProxy() return } + if len(os.Args) >= 2 && os.Args[1] == "debug-harness" { + os.Args = append(os.Args[:1], os.Args[2:]...) + runDebugHarness() + return + } var projectDir = flag.String("project", "", "project directory (default $PWD)") flag.Parse() diff --git a/internal/harness/env.go b/internal/harness/env.go new file mode 100644 index 0000000..37a88f6 --- /dev/null +++ b/internal/harness/env.go @@ -0,0 +1,209 @@ +package harness + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/hjbdev/patterm/internal/projectkey" + "github.com/hjbdev/patterm/internal/trust" +) + +type Options struct { + PattermBin string + Scenario *Scenario +} + +type testEnv struct { + Root string `json:"root"` + ConfigHome string `json:"xdg_config_home"` + DataHome string `json:"xdg_data_home"` + RuntimeDir string `json:"xdg_runtime_dir"` + BinDir string `json:"bin_dir"` + ProjectDir string `json:"project_dir"` + PattermBin string `json:"patterm_bin"` + Cols uint16 `json:"cols"` + Rows uint16 `json:"rows"` +} + +func prepareEnv(opts Options) (*testEnv, []string, error) { + sc := opts.Scenario + root, err := os.MkdirTemp("", "patterm-harness-*") + if err != nil { + return nil, nil, err + } + env := &testEnv{ + Root: root, + ConfigHome: filepath.Join(root, "config"), + DataHome: filepath.Join(root, "data"), + RuntimeDir: filepath.Join(root, "runtime"), + BinDir: filepath.Join(root, "bin"), + Cols: sc.Cols, + Rows: sc.Rows, + PattermBin: opts.PattermBin, + } + if env.ProjectDir = sc.ProjectDir; env.ProjectDir == "" { + env.ProjectDir = filepath.Join(root, "project") + } + for _, dir := range []string{env.ConfigHome, env.DataHome, env.RuntimeDir, env.BinDir, env.ProjectDir} { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, nil, err + } + } + if env.PattermBin == "" { + env.PattermBin, err = defaultPattermBin() + if err != nil { + return nil, nil, err + } + } + if err := writeScripts(env.BinDir, sc.Scripts); err != nil { + return nil, nil, err + } + if err := writePresets(env.ConfigHome, sc.Presets); err != nil { + return nil, nil, err + } + if err := seedTrust(env, sc.Trust); err != nil { + return nil, nil, err + } + childEnv := append(os.Environ(), + "XDG_CONFIG_HOME="+env.ConfigHome, + "XDG_DATA_HOME="+env.DataHome, + "XDG_RUNTIME_DIR="+env.RuntimeDir, + "PATTERM_HARNESS=1", + "PATH="+env.BinDir+string(os.PathListSeparator)+os.Getenv("PATH"), + ) + for k, v := range sc.Env { + childEnv = append(childEnv, k+"="+v) + } + return env, childEnv, nil +} + +func writeScripts(bin string, scripts []ScenarioScript) error { + for _, script := range scripts { + if script.Name == "" { + return fmt.Errorf("script missing name") + } + if strings.Contains(script.Name, "/") { + return fmt.Errorf("script name %q must not contain /", script.Name) + } + path := filepath.Join(bin, script.Name) + if err := os.WriteFile(path, []byte(script.Body), 0o700); err != nil { + return err + } + } + return nil +} + +func writePresets(configHome string, presets ScenarioPresets) error { + base := filepath.Join(configHome, "patterm", "presets") + if err := os.MkdirAll(filepath.Join(base, "agents"), 0o700); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "processes"), 0o700); err != nil { + return err + } + for _, p := range presets.Agents { + if err := writePreset(filepath.Join(base, "agents", p.Name+".json"), p); err != nil { + return err + } + } + for _, p := range presets.Processes { + if err := writePreset(filepath.Join(base, "processes", p.Name+".json"), p); err != nil { + return err + } + } + return nil +} + +func writePreset(path string, p ScenarioPreset) error { + if p.Name == "" { + return fmt.Errorf("preset missing name") + } + b, err := json.MarshalIndent(p, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + return os.WriteFile(path, b, 0o600) +} + +func seedTrust(env *testEnv, presets []string) error { + if len(presets) == 0 { + return nil + } + key, err := projectkey.Key(env.ProjectDir) + if err != nil { + return err + } + restore := setenv(map[string]string{"XDG_DATA_HOME": env.DataHome}) + defer restore() + store, err := trust.Open(key) + if err != nil { + return err + } + for _, p := range presets { + if err := store.Grant(p); err != nil { + return err + } + } + return nil +} + +func setenv(vals map[string]string) func() { + type old struct { + v string + ok bool + } + prev := map[string]old{} + for k, v := range vals { + ov, ok := os.LookupEnv(k) + prev[k] = old{v: ov, ok: ok} + _ = os.Setenv(k, v) + } + return func() { + for k, ov := range prev { + if ov.ok { + _ = os.Setenv(k, ov.v) + } else { + _ = os.Unsetenv(k) + } + } + } +} + +func defaultPattermBin() (string, error) { + if p := os.Getenv("PATTERM_BIN"); p != "" { + return p, nil + } + return buildPattermBinary() +} + +func buildPattermBinary() (string, error) { + root, err := repoRoot() + if err != nil { + return "", err + } + out := filepath.Join(os.TempDir(), "patterm-harness-bin", "patterm") + if err := os.MkdirAll(filepath.Dir(out), 0o700); err != nil { + return "", err + } + cmd := exec.Command("go", "build", "-o", out, "./cmd/patterm") + cmd.Dir = root + cmd.Env = os.Environ() + if b, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("build patterm: %w\n%s", err, string(b)) + } + return out, nil +} + +func repoRoot() (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("runtime.Caller failed") + } + return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")), nil +} diff --git a/internal/harness/input.go b/internal/harness/input.go new file mode 100644 index 0000000..279740f --- /dev/null +++ b/internal/harness/input.go @@ -0,0 +1,35 @@ +package harness + +import "fmt" + +func EncodeChord(name string) ([]byte, error) { + switch name { + case "ctrl-k": + return []byte{0x0b}, nil + case "ctrl-k-kitty": + return []byte("\x1b[107;5u"), nil + case "ctrl-k-xterm": + return []byte("\x1b[27;5;107~"), nil + case "enter": + return []byte{'\r'}, nil + case "escape": + return []byte{0x1b}, nil + case "backspace": + return []byte{0x7f}, nil + case "up": + return []byte("\x1b[A"), nil + case "down": + return []byte("\x1b[B"), nil + case "left": + return []byte("\x1b[D"), nil + case "right": + return []byte("\x1b[C"), nil + case "ctrl-n": + return []byte{0x0e}, nil + case "ctrl-p": + return []byte{0x10}, nil + case "ctrl-u": + return []byte{0x15}, nil + } + return nil, fmt.Errorf("unknown chord %q", name) +} diff --git a/internal/harness/input_test.go b/internal/harness/input_test.go new file mode 100644 index 0000000..e8339c9 --- /dev/null +++ b/internal/harness/input_test.go @@ -0,0 +1,22 @@ +package harness + +import "testing" + +func TestEncodeChord(t *testing.T) { + tests := map[string]string{ + "ctrl-k": "\x0b", + "ctrl-k-kitty": "\x1b[107;5u", + "ctrl-k-xterm": "\x1b[27;5;107~", + "enter": "\r", + "down": "\x1b[B", + } + for name, want := range tests { + got, err := EncodeChord(name) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if string(got) != want { + t.Fatalf("%s = %q, want %q", name, string(got), want) + } + } +} diff --git a/internal/harness/mcp_client.go b/internal/harness/mcp_client.go new file mode 100644 index 0000000..dffcf02 --- /dev/null +++ b/internal/harness/mcp_client.go @@ -0,0 +1,82 @@ +package harness + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "sync" +) + +type MCPClient struct { + conn net.Conn + r *bufio.Reader + mu sync.Mutex + next int +} + +type RPCError struct { + Code int `json:"code"` + Message string `json:"message"` + Data any `json:"data,omitempty"` +} + +func (e *RPCError) Error() string { + return fmt.Sprintf("json-rpc error %d: %s", e.Code, e.Message) +} + +func DialMCP(socket string) (*MCPClient, error) { + conn, err := net.Dial("unix", socket) + if err != nil { + return nil, err + } + if _, err := conn.Write([]byte("{\"patterm_identity\":\"harness\"}\n")); err != nil { + _ = conn.Close() + return nil, err + } + return &MCPClient{conn: conn, r: bufio.NewReader(conn)}, nil +} + +func (c *MCPClient) Close() error { + if c == nil || c.conn == nil { + return nil + } + return c.conn.Close() +} + +func (c *MCPClient) Call(method string, params any) (json.RawMessage, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.next++ + req := map[string]any{ + "jsonrpc": "2.0", + "id": c.next, + "method": method, + } + if params != nil { + req["params"] = params + } + b, err := json.Marshal(req) + if err != nil { + return nil, err + } + b = append(b, '\n') + if _, err := c.conn.Write(b); err != nil { + return nil, err + } + line, err := c.r.ReadBytes('\n') + if err != nil { + return nil, err + } + var resp struct { + Result json.RawMessage `json:"result"` + Error *RPCError `json:"error"` + } + if err := json.Unmarshal(line, &resp); err != nil { + return nil, fmt.Errorf("parse response: %w: %s", err, string(line)) + } + if resp.Error != nil { + return nil, resp.Error + } + return resp.Result, nil +} diff --git a/internal/harness/observe.go b/internal/harness/observe.go new file mode 100644 index 0000000..55ca655 --- /dev/null +++ b/internal/harness/observe.go @@ -0,0 +1,92 @@ +package harness + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + "time" +) + +func timeoutMS(ms int) time.Duration { + if ms <= 0 { + ms = 5000 + } + return time.Duration(ms) * time.Millisecond +} + +func valueAt(v any, path string) (any, bool) { + if path == "" { + return v, true + } + cur := v + for _, part := range strings.Split(path, ".") { + switch x := cur.(type) { + case map[string]any: + next, ok := x[part] + if !ok { + return nil, false + } + cur = next + case []any: + i, err := strconv.Atoi(part) + if err != nil || i < 0 || i >= len(x) { + return nil, false + } + cur = x[i] + default: + return nil, false + } + } + return cur, true +} + +func assertJSONValue(raw json.RawMessage, path string, equals any, contains string, allowSubstring bool) error { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return err + } + got, ok := valueAt(v, path) + if !ok { + return fmt.Errorf("path %q not found in %s", path, string(raw)) + } + if equals != nil { + if !reflect.DeepEqual(normalizeJSON(equals), normalizeJSON(got)) { + return fmt.Errorf("path %q = %#v, want %#v", path, got, equals) + } + } + if contains != "" { + switch x := got.(type) { + case string: + if !strings.Contains(x, contains) { + return fmt.Errorf("path %q = %q, does not contain %q", path, x, contains) + } + default: + b, _ := json.Marshal(x) + if allowSubstring && strings.Contains(string(b), contains) { + return nil + } + return fmt.Errorf("path %q is not a string: %#v", path, got) + } + } + return nil +} + +func normalizeJSON(v any) any { + switch x := v.(type) { + case int: + return float64(x) + case int64: + return float64(x) + case []any: + for i := range x { + x[i] = normalizeJSON(x[i]) + } + case map[string]any: + for k := range x { + x[k] = normalizeJSON(x[k]) + } + } + return v +} diff --git a/internal/harness/recorder.go b/internal/harness/recorder.go new file mode 100644 index 0000000..095778f --- /dev/null +++ b/internal/harness/recorder.go @@ -0,0 +1,52 @@ +package harness + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +type Artifact struct { + Dir string `json:"artifact_dir"` +} + +func (s *Session) DumpArtifacts(sc *Scenario, failingStep int, cause error) (*Artifact, error) { + name := sc.Name + if name == "" { + name = strings.TrimSuffix(filepath.Base(sc.Path), filepath.Ext(sc.Path)) + } + if name == "" { + name = "scenario" + } + dir := filepath.Join("internal", "harness", ".artifacts", fmt.Sprintf("%s-%d", name, time.Now().Unix())) + abs, _ := filepath.Abs(dir) + if err := os.MkdirAll(abs, 0o700); err != nil { + return nil, err + } + screen, _ := s.em.ScreenText() + _ = os.WriteFile(filepath.Join(abs, "grid.log"), []byte(screen), 0o600) + _ = os.WriteFile(filepath.Join(abs, "raw.bytes"), s.rawBytes(), 0o600) + if b, err := s.em.SerializeVT(); err == nil { + _ = os.WriteFile(filepath.Join(abs, "serialize.vt"), b, 0o600) + } + envBytes, _ := json.MarshalIndent(s.env, "", " ") + _ = os.WriteFile(filepath.Join(abs, "env.json"), append(envBytes, '\n'), 0o600) + annotated := map[string]any{"scenario": sc, "failing_step": failingStep} + if cause != nil { + annotated["error"] = cause.Error() + } + ab, _ := json.MarshalIndent(annotated, "", " ") + _ = os.WriteFile(filepath.Join(abs, "scenario.annotated.json"), append(ab, '\n'), 0o600) + snapshot := map[string]json.RawMessage{} + for _, method := range []string{"whoami", "list_processes", "get_project_status"} { + if raw, err := s.mcp.Call(method, map[string]any{}); err == nil { + snapshot[method] = raw + } + } + sb, _ := json.MarshalIndent(snapshot, "", " ") + _ = os.WriteFile(filepath.Join(abs, "mcp-snapshot.json"), append(sb, '\n'), 0o600) + return &Artifact{Dir: abs}, nil +} diff --git a/internal/harness/runner.go b/internal/harness/runner.go new file mode 100644 index 0000000..2adc015 --- /dev/null +++ b/internal/harness/runner.go @@ -0,0 +1,205 @@ +package harness + +import ( + "encoding/json" + "fmt" + "strings" +) + +type Event struct { + Step int `json:"step"` + Type string `json:"type"` + OK bool `json:"ok"` + Result json.RawMessage `json:"result,omitempty"` + Error string `json:"error,omitempty"` + Artifact string `json:"artifact_dir,omitempty"` +} + +type RunResult struct { + OK bool `json:"ok"` + Steps int `json:"steps"` + Artifact string `json:"artifact_dir,omitempty"` + Error string `json:"error,omitempty"` + Events []Event `json:"-"` +} + +func RunScenario(opts Options, emit func(Event)) (RunResult, error) { + sc := opts.Scenario + s, err := NewCLI(opts) + if err != nil { + return RunResult{OK: false, Error: err.Error()}, err + } + defer s.Close() + results := map[string]json.RawMessage{} + for i, step := range sc.Steps { + ev := Event{Step: i + 1, Type: step.Type} + err := runStep(s, step, results) + if err != nil { + ev.OK = false + ev.Error = err.Error() + art, aerr := s.DumpArtifacts(sc, i, err) + if aerr == nil { + ev.Artifact = art.Dir + } + if emit != nil { + emit(ev) + } + return RunResult{OK: false, Steps: i + 1, Error: err.Error(), Artifact: ev.Artifact}, err + } + ev.OK = true + if step.SaveAs != "" { + ev.Result = results[step.SaveAs] + } + if emit != nil { + emit(ev) + } + } + return RunResult{OK: true, Steps: len(sc.Steps)}, nil +} + +func runStep(s *Session, step Step, results map[string]json.RawMessage) error { + switch step.Type { + case "wait_stable": + return s.WaitForStable(timeoutMS(step.TimeoutMS)) + case "send_chord": + return s.SendChord(step.Chord) + case "send_text": + return s.SendText(step.Text) + case "assert_contains": + screen, err := s.Screen() + if err != nil { + return err + } + if !strings.Contains(screen, step.Contains) { + return fmt.Errorf("screen does not contain %q:\n%s", step.Contains, screen) + } + return nil + case "assert_regex": + return s.WaitForRegex(step.Regex, timeoutMS(step.TimeoutMS)) + case "wait_text": + return s.WaitForText(step.Contains, timeoutMS(step.TimeoutMS)) + case "assert_cursor": + cur, err := s.Cursor() + if err != nil { + return err + } + if step.CursorRow != nil && int(cur.Row) != *step.CursorRow { + return fmt.Errorf("cursor row = %d, want %d", cur.Row, *step.CursorRow) + } + if step.CursorCol != nil && int(cur.Col) != *step.CursorCol { + return fmt.Errorf("cursor col = %d, want %d", cur.Col, *step.CursorCol) + } + return nil + case "mcp_call": + params, err := resolveParams(step.Params, results) + if err != nil { + return err + } + raw, err := s.MCPCall(step.Method, params) + if err != nil { + return err + } + if step.SaveAs != "" { + results[step.SaveAs] = raw + } + return nil + case "assert_mcp": + params, perr := resolveParams(step.Params, results) + if perr != nil { + return perr + } + raw, err := s.MCPCall(step.Method, params) + if step.ErrorKind != "" { + if err == nil { + return fmt.Errorf("expected MCP error kind %q, got success %s", step.ErrorKind, string(raw)) + } + rpcErr, ok := err.(*RPCError) + if !ok { + return err + } + if rpcErr.Data != nil { + if data, ok := rpcErr.Data.(map[string]any); ok && data["kind"] == step.ErrorKind { + return nil + } + } + if strings.Contains(rpcErr.Message, step.ErrorKind) || strings.Contains(rpcErr.Message, strings.ReplaceAll(step.ErrorKind, "_", " ")) { + return nil + } + return fmt.Errorf("MCP error = %#v, want kind %q", rpcErr, step.ErrorKind) + } + if err != nil { + return err + } + return assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring) + case "assert_saved": + raw, ok := results[step.From] + if !ok { + return fmt.Errorf("no saved result %q", step.From) + } + return assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring) + } + return fmt.Errorf("unknown step type %q", step.Type) +} + +func resolveParams(raw json.RawMessage, saved map[string]json.RawMessage) (json.RawMessage, error) { + if len(raw) == 0 { + return raw, nil + } + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return nil, err + } + resolved, err := resolveValue(v, saved) + if err != nil { + return nil, err + } + return json.Marshal(resolved) +} + +func resolveValue(v any, saved map[string]json.RawMessage) (any, error) { + switch x := v.(type) { + case string: + if strings.HasPrefix(x, "{{") && strings.HasSuffix(x, "}}") { + ref := strings.TrimSuffix(strings.TrimPrefix(x, "{{"), "}}") + parts := strings.SplitN(ref, ".", 2) + raw, ok := saved[parts[0]] + if !ok { + return nil, fmt.Errorf("unknown saved result %q", parts[0]) + } + var sv any + if err := json.Unmarshal(raw, &sv); err != nil { + return nil, err + } + path := "" + if len(parts) == 2 { + path = parts[1] + } + got, ok := valueAt(sv, path) + if !ok { + return nil, fmt.Errorf("path %q not found in saved result %q", path, parts[0]) + } + return got, nil + } + return x, nil + case []any: + for i := range x { + r, err := resolveValue(x[i], saved) + if err != nil { + return nil, err + } + x[i] = r + } + return x, nil + case map[string]any: + for k := range x { + r, err := resolveValue(x[k], saved) + if err != nil { + return nil, err + } + x[k] = r + } + return x, nil + default: + return x, nil + } +} diff --git a/internal/harness/scenario.go b/internal/harness/scenario.go new file mode 100644 index 0000000..591de50 --- /dev/null +++ b/internal/harness/scenario.go @@ -0,0 +1,76 @@ +package harness + +import ( + "encoding/json" + "fmt" + "os" +) + +type Scenario struct { + Path string `json:"-"` + Name string `json:"name,omitempty"` + ProjectDir string `json:"project_dir,omitempty"` + Cols uint16 `json:"cols,omitempty"` + Rows uint16 `json:"rows,omitempty"` + Env map[string]string `json:"env,omitempty"` + Trust []string `json:"trust,omitempty"` + Presets ScenarioPresets `json:"presets,omitempty"` + Scripts []ScenarioScript `json:"scripts,omitempty"` + Steps []Step `json:"steps"` +} + +type ScenarioPresets struct { + Agents []ScenarioPreset `json:"agents,omitempty"` + Processes []ScenarioPreset `json:"processes,omitempty"` +} + +type ScenarioPreset struct { + Name string `json:"name"` + Argv []string `json:"argv"` + Env map[string]string `json:"env,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + Shell bool `json:"shell,omitempty"` +} + +type ScenarioScript struct { + Name string `json:"name"` + Body string `json:"body"` +} + +type Step struct { + Type string `json:"type"` + Chord string `json:"chord,omitempty"` + Text string `json:"text,omitempty"` + Contains string `json:"contains,omitempty"` + Regex string `json:"regex,omitempty"` + TimeoutMS int `json:"timeout_ms,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + SaveAs string `json:"save_as,omitempty"` + From string `json:"from,omitempty"` + Path string `json:"path,omitempty"` + Equals any `json:"equals,omitempty"` + ErrorKind string `json:"error_kind,omitempty"` + CursorRow *int `json:"cursor_row,omitempty"` + CursorCol *int `json:"cursor_col,omitempty"` + AllowSubstring bool `json:"allow_substring,omitempty"` +} + +func LoadScenario(path string) (*Scenario, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var sc Scenario + if err := json.Unmarshal(b, &sc); err != nil { + return nil, fmt.Errorf("parse scenario %s: %w", path, err) + } + sc.Path = path + if sc.Cols == 0 { + sc.Cols = 120 + } + if sc.Rows == 0 { + sc.Rows = 40 + } + return &sc, nil +} diff --git a/internal/harness/scenario_test.go b/internal/harness/scenario_test.go new file mode 100644 index 0000000..dc47d01 --- /dev/null +++ b/internal/harness/scenario_test.go @@ -0,0 +1,49 @@ +package harness + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadScenario(t *testing.T) { + sc, err := LoadScenario(filepath.Join("scenarios", "spawn_process_via_palette.json")) + if err != nil { + t.Fatal(err) + } + if sc.Cols != 120 || sc.Rows != 40 { + t.Fatalf("default size = %dx%d, want 120x40", sc.Cols, sc.Rows) + } + if len(sc.Steps) == 0 { + t.Fatal("expected steps") + } +} + +func TestHarnessScenarios(t *testing.T) { + if testing.Short() { + t.Skip("skipping end-to-end harness scenarios in short mode") + } + paths, err := filepath.Glob(filepath.Join("scenarios", "*.json")) + if err != nil { + t.Fatal(err) + } + if len(paths) == 0 { + t.Fatal("no scenarios") + } + for _, path := range paths { + path := path + t.Run(filepath.Base(path), func(t *testing.T) { + sc, err := LoadScenario(path) + if err != nil { + t.Fatal(err) + } + res, err := RunScenario(Options{Scenario: sc, PattermBin: os.Getenv("PATTERM_BIN")}, nil) + if err != nil { + if res.Artifact != "" { + t.Fatalf("%v\nartifact: %s", err, res.Artifact) + } + t.Fatal(err) + } + }) + } +} diff --git a/internal/harness/scenarios/child_exit.json b/internal/harness/scenarios/child_exit.json new file mode 100644 index 0000000..10b49e5 --- /dev/null +++ b/internal/harness/scenarios/child_exit.json @@ -0,0 +1,23 @@ +{ + "name": "child_exit", + "trust": ["exit0"], + "presets": { + "processes": [ + { + "name": "exit0", + "argv": ["sh", "-lc", "exit 0"] + } + ] + }, + "steps": [ + { + "type": "mcp_call", + "method": "spawn_process", + "params": { "kind": "command", "preset": "exit0" }, + "save_as": "proc" + }, + { "type": "wait_stable", "timeout_ms": 3000 }, + { "type": "assert_mcp", "method": "list_processes", "path": "0.status", "equals": "exited" }, + { "type": "assert_mcp", "method": "list_processes", "path": "0.exit_code", "equals": 0 } + ] +} diff --git a/internal/harness/scenarios/mcp_send_input.json b/internal/harness/scenarios/mcp_send_input.json new file mode 100644 index 0000000..4996357 --- /dev/null +++ b/internal/harness/scenarios/mcp_send_input.json @@ -0,0 +1,30 @@ +{ + "name": "mcp_send_input", + "steps": [ + { + "type": "mcp_call", + "method": "spawn_process", + "params": { + "kind": "command", + "argv": ["sh", "-lc", "read line; echo got:$line; sleep 5"], + "name": "reader" + }, + "save_as": "proc" + }, + { + "type": "mcp_call", + "method": "send_input", + "params": { + "process_id": "{{proc.process_id}}", + "kind": "text", + "text": "hello", + "submit": true, + "wait_ms": 250, + "tail_mode": "grid" + }, + "save_as": "input" + }, + { "type": "assert_saved", "from": "input", "path": "tail.content", "contains": "got:hello" }, + { "type": "wait_text", "contains": "got:hello", "timeout_ms": 5000 } + ] +} diff --git a/internal/harness/scenarios/spawn_process_via_palette.json b/internal/harness/scenarios/spawn_process_via_palette.json new file mode 100644 index 0000000..26b445c --- /dev/null +++ b/internal/harness/scenarios/spawn_process_via_palette.json @@ -0,0 +1,26 @@ +{ + "name": "spawn_process_via_palette", + "presets": { + "processes": [ + { + "name": "ready", + "argv": ["ready-script"] + } + ] + }, + "scripts": [ + { + "name": "ready-script", + "body": "#!/bin/sh\necho READY\nsleep 5\n" + } + ], + "steps": [ + { "type": "wait_stable", "timeout_ms": 3000 }, + { "type": "send_chord", "chord": "ctrl-k" }, + { "type": "send_text", "text": "ready" }, + { "type": "send_chord", "chord": "enter" }, + { "type": "wait_text", "contains": "READY", "timeout_ms": 5000 }, + { "type": "assert_mcp", "method": "list_processes", "path": "0.name", "equals": "ready" }, + { "type": "assert_mcp", "method": "list_processes", "path": "0.status", "equals": "running" } + ] +} diff --git a/internal/harness/scenarios/switch_via_palette.json b/internal/harness/scenarios/switch_via_palette.json new file mode 100644 index 0000000..047b1f0 --- /dev/null +++ b/internal/harness/scenarios/switch_via_palette.json @@ -0,0 +1,34 @@ +{ + "name": "switch_via_palette", + "scripts": [ + { + "name": "named-loop", + "body": "#!/bin/sh\necho \"$1 READY\"\nsleep 5\n" + } + ], + "steps": [ + { + "type": "mcp_call", + "method": "spawn_process", + "params": { "kind": "command", "argv": ["named-loop", "first"], "name": "first" }, + "save_as": "first" + }, + { + "type": "mcp_call", + "method": "spawn_process", + "params": { "kind": "command", "argv": ["named-loop", "second"], "name": "second" }, + "save_as": "second" + }, + { "type": "wait_text", "contains": "second READY", "timeout_ms": 5000 }, + { "type": "send_chord", "chord": "ctrl-k" }, + { "type": "send_text", "text": "Switch to first" }, + { "type": "send_chord", "chord": "enter" }, + { "type": "wait_text", "contains": "first READY", "timeout_ms": 5000 }, + { + "type": "assert_mcp", + "method": "get_project_status", + "path": "processes.0.name", + "equals": "first" + } + ] +} diff --git a/internal/harness/scenarios/trust_required.json b/internal/harness/scenarios/trust_required.json new file mode 100644 index 0000000..be74074 --- /dev/null +++ b/internal/harness/scenarios/trust_required.json @@ -0,0 +1,19 @@ +{ + "name": "trust_required", + "presets": { + "processes": [ + { + "name": "untrusted", + "argv": ["sh", "-lc", "echo should-not-run"] + } + ] + }, + "steps": [ + { + "type": "assert_mcp", + "method": "spawn_process", + "params": { "kind": "command", "preset": "untrusted" }, + "error_kind": "needs_trust" + } + ] +} diff --git a/internal/harness/session.go b/internal/harness/session.go new file mode 100644 index 0000000..da56e60 --- /dev/null +++ b/internal/harness/session.go @@ -0,0 +1,264 @@ +package harness + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + pkgpty "github.com/hjbdev/patterm/internal/pty" + "github.com/hjbdev/patterm/internal/vt" +) + +type Session struct { + pty *pkgpty.PTY + em *vt.GhosttyEmulator + mcp *MCPClient + env *testEnv + + bytesMu sync.Mutex + bytes []byte + + lastWriteNS atomic.Int64 + readerDone chan struct{} + closeOnce sync.Once + closeErr error +} + +func New(t testing.TB, opts Options) *Session { + t.Helper() + s, err := NewCLI(opts) + if err != nil { + t.Fatalf("harness New: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func NewCLI(opts Options) (*Session, error) { + if opts.Scenario == nil { + return nil, fmt.Errorf("harness: Scenario required") + } + env, childEnv, err := prepareEnv(opts) + if err != nil { + return nil, err + } + em, err := vt.NewGhosttyEmulator(env.Cols, env.Rows) + if err != nil { + return nil, err + } + p, err := pkgpty.Start([]string{env.PattermBin, "--project", env.ProjectDir}, childEnv, env.Cols, env.Rows) + if err != nil { + _ = em.Close() + return nil, err + } + em.OnWritePTY(func(b []byte) { _, _ = p.Write(b) }) + s := &Session{pty: p, em: em, env: env, readerDone: make(chan struct{})} + go s.readLoop() + if err := s.bootstrapMCP(2 * time.Second); err != nil { + _ = s.Close() + return nil, err + } + return s, nil +} + +func (s *Session) readLoop() { + defer close(s.readerDone) + buf := make([]byte, 64*1024) + for { + n, err := s.pty.Read(buf) + if n > 0 { + chunk := make([]byte, n) + copy(chunk, buf[:n]) + _, _ = s.em.Write(chunk) + s.bytesMu.Lock() + s.bytes = append(s.bytes, chunk...) + s.bytesMu.Unlock() + s.lastWriteNS.Store(time.Now().UnixNano()) + } + if err != nil { + return + } + } +} + +func (s *Session) bootstrapMCP(timeout time.Duration) error { + socket := filepath.Join(s.env.RuntimeDir, "patterm", fmt.Sprintf("%d.sock", s.pty.Pid())) + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + if _, err := os.Stat(socket); err != nil { + last = err + time.Sleep(25 * time.Millisecond) + continue + } + c, err := DialMCP(socket) + if err != nil { + last = err + time.Sleep(25 * time.Millisecond) + continue + } + _, err = c.Call("whoami", map[string]any{}) + if err == nil { + s.mcp = c + return nil + } + last = err + _ = c.Close() + if strings.Contains(err.Error(), "tool host not initialized") { + time.Sleep(25 * time.Millisecond) + continue + } + time.Sleep(25 * time.Millisecond) + } + raw := strings.TrimSpace(string(s.rawBytes())) + if raw != "" { + return fmt.Errorf("mcp bootstrap timed out: %w; child output: %s", last, raw) + } + return fmt.Errorf("mcp bootstrap timed out: %w", last) +} + +func (s *Session) Close() error { + s.closeOnce.Do(func() { + if s.mcp != nil { + _ = s.mcp.Close() + } + pid := s.pty.Pid() + if pid > 0 { + if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil { + _ = syscall.Kill(pid, syscall.SIGTERM) + } + } + done := make(chan error, 1) + go func() { done <- s.pty.Wait() }() + select { + case <-done: + case <-time.After(2 * time.Second): + if pid > 0 { + if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + } + select { + case <-done: + case <-time.After(500 * time.Millisecond): + } + } + select { + case <-s.readerDone: + case <-time.After(time.Second): + if err := s.pty.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + s.closeErr = err + } + select { + case <-s.readerDone: + case <-time.After(500 * time.Millisecond): + } + } + _ = s.em.Close() + }) + return s.closeErr +} + +func (s *Session) SendChord(name string) error { + b, err := EncodeChord(name) + if err != nil { + return err + } + _, err = s.pty.Write(b) + return err +} + +func (s *Session) SendText(text string) error { + _, err := s.pty.Write([]byte(text)) + return err +} + +func (s *Session) Screen() (string, error) { return s.em.PlainText() } + +func (s *Session) Cursor() (vt.CursorState, error) { return s.em.Cursor() } + +func (s *Session) WaitForStable(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + tick := time.NewTicker(25 * time.Millisecond) + defer tick.Stop() + confirmed := false + for { + last := s.lastWriteNS.Load() + idle := last == 0 || time.Since(time.Unix(0, last)) >= time.Second + if idle { + if confirmed { + return nil + } + confirmed = true + } else { + confirmed = false + } + if time.Now().After(deadline) { + return fmt.Errorf("screen did not stabilize within %s", timeout) + } + <-tick.C + } +} + +func (s *Session) WaitForText(text string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + screen, err := s.Screen() + if err != nil { + return err + } + if strings.Contains(screen, text) { + return nil + } + time.Sleep(25 * time.Millisecond) + } + screen, _ := s.Screen() + return fmt.Errorf("text %q not found before timeout; screen:\n%s", text, screen) +} + +func (s *Session) WaitForRegex(pattern string, timeout time.Duration) error { + re, err := regexp.Compile(pattern) + if err != nil { + return err + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + screen, err := s.Screen() + if err != nil { + return err + } + if re.MatchString(screen) { + return nil + } + time.Sleep(25 * time.Millisecond) + } + screen, _ := s.Screen() + return fmt.Errorf("regex %q not found before timeout; screen:\n%s", pattern, screen) +} + +func (s *Session) MCPCall(method string, params json.RawMessage) (json.RawMessage, error) { + var v any = map[string]any{} + if len(params) > 0 { + if err := json.Unmarshal(params, &v); err != nil { + return nil, err + } + } + return s.mcp.Call(method, v) +} + +func (s *Session) rawBytes() []byte { + s.bytesMu.Lock() + defer s.bytesMu.Unlock() + out := make([]byte, len(s.bytes)) + copy(out, s.bytes) + return out +}