add black-box debug harness

.gitignore

@@ -6,3 +6,5 @@ spike-report-*.txt
 /.zig-cache/
 /bin/
 /spike
+/.worktrees/
+internal/harness/.artifacts/

cmd/patterm/debug_harness.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/hjbdev/patterm/internal/harness"
+)
+
+func runDebugHarness() {
+	var (
+		scenarioPath = flag.String("scenario", "", "JSON scenario path")
+		pattermBin   = flag.String("patterm-bin", "", "patterm binary under test (default: this executable)")
+	)
+	flag.Parse()
+	if *scenarioPath == "" {
+		die("debug-harness: --scenario is required")
+	}
+	if *pattermBin == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			die("debug-harness: os.Executable: %v", err)
+		}
+		*pattermBin = exe
+	}
+	sc, err := harness.LoadScenario(*scenarioPath)
+	if err != nil {
+		fmt.Fprintf(os.Stdout, "{\"ok\":false,\"error\":%q}\n", err.Error())
+		os.Exit(2)
+	}
+	enc := json.NewEncoder(os.Stdout)
+	res, err := harness.RunScenario(harness.Options{
+		Scenario:   sc,
+		PattermBin: *pattermBin,
+	}, func(ev harness.Event) {
+		_ = enc.Encode(ev)
+	})
+	_ = enc.Encode(res)
+	if err != nil {
+		if res.Steps == 0 {
+			os.Exit(2)
+		}
+		os.Exit(1)
+	}
+}

cmd/patterm/main.go

@@ -7,6 +7,8 @@
 //	patterm mcp-stdio --socket S --identity I
 //	                             internal: stdio MCP proxy spawned for
 //	                             children, forwards JSON-RPC over S
+//	patterm debug-harness --scenario S
+//	                             internal: run a black-box harness scenario
 package main
 
 import (
@@ -30,6 +32,11 @@ func main() {
 		runMCPProxy()
 		return
 	}
+	if len(os.Args) >= 2 && os.Args[1] == "debug-harness" {
+		os.Args = append(os.Args[:1], os.Args[2:]...)
+		runDebugHarness()
+		return
+	}
 
 	var projectDir = flag.String("project", "", "project directory (default $PWD)")
 	flag.Parse()

internal/harness/env.go

@@ -0,0 +1,209 @@
+package harness
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/hjbdev/patterm/internal/projectkey"
+	"github.com/hjbdev/patterm/internal/trust"
+)
+
+type Options struct {
+	PattermBin string
+	Scenario   *Scenario
+}
+
+type testEnv struct {
+	Root       string `json:"root"`
+	ConfigHome string `json:"xdg_config_home"`
+	DataHome   string `json:"xdg_data_home"`
+	RuntimeDir string `json:"xdg_runtime_dir"`
+	BinDir     string `json:"bin_dir"`
+	ProjectDir string `json:"project_dir"`
+	PattermBin string `json:"patterm_bin"`
+	Cols       uint16 `json:"cols"`
+	Rows       uint16 `json:"rows"`
+}
+
+func prepareEnv(opts Options) (*testEnv, []string, error) {
+	sc := opts.Scenario
+	root, err := os.MkdirTemp("", "patterm-harness-*")
+	if err != nil {
+		return nil, nil, err
+	}
+	env := &testEnv{
+		Root:       root,
+		ConfigHome: filepath.Join(root, "config"),
+		DataHome:   filepath.Join(root, "data"),
+		RuntimeDir: filepath.Join(root, "runtime"),
+		BinDir:     filepath.Join(root, "bin"),
+		Cols:       sc.Cols,
+		Rows:       sc.Rows,
+		PattermBin: opts.PattermBin,
+	}
+	if env.ProjectDir = sc.ProjectDir; env.ProjectDir == "" {
+		env.ProjectDir = filepath.Join(root, "project")
+	}
+	for _, dir := range []string{env.ConfigHome, env.DataHome, env.RuntimeDir, env.BinDir, env.ProjectDir} {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return nil, nil, err
+		}
+	}
+	if env.PattermBin == "" {
+		env.PattermBin, err = defaultPattermBin()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	if err := writeScripts(env.BinDir, sc.Scripts); err != nil {
+		return nil, nil, err
+	}
+	if err := writePresets(env.ConfigHome, sc.Presets); err != nil {
+		return nil, nil, err
+	}
+	if err := seedTrust(env, sc.Trust); err != nil {
+		return nil, nil, err
+	}
+	childEnv := append(os.Environ(),
+		"XDG_CONFIG_HOME="+env.ConfigHome,
+		"XDG_DATA_HOME="+env.DataHome,
+		"XDG_RUNTIME_DIR="+env.RuntimeDir,
+		"PATTERM_HARNESS=1",
+		"PATH="+env.BinDir+string(os.PathListSeparator)+os.Getenv("PATH"),
+	)
+	for k, v := range sc.Env {
+		childEnv = append(childEnv, k+"="+v)
+	}
+	return env, childEnv, nil
+}
+
+func writeScripts(bin string, scripts []ScenarioScript) error {
+	for _, script := range scripts {
+		if script.Name == "" {
+			return fmt.Errorf("script missing name")
+		}
+		if strings.Contains(script.Name, "/") {
+			return fmt.Errorf("script name %q must not contain /", script.Name)
+		}
+		path := filepath.Join(bin, script.Name)
+		if err := os.WriteFile(path, []byte(script.Body), 0o700); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writePresets(configHome string, presets ScenarioPresets) error {
+	base := filepath.Join(configHome, "patterm", "presets")
+	if err := os.MkdirAll(filepath.Join(base, "agents"), 0o700); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Join(base, "processes"), 0o700); err != nil {
+		return err
+	}
+	for _, p := range presets.Agents {
+		if err := writePreset(filepath.Join(base, "agents", p.Name+".json"), p); err != nil {
+			return err
+		}
+	}
+	for _, p := range presets.Processes {
+		if err := writePreset(filepath.Join(base, "processes", p.Name+".json"), p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writePreset(path string, p ScenarioPreset) error {
+	if p.Name == "" {
+		return fmt.Errorf("preset missing name")
+	}
+	b, err := json.MarshalIndent(p, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	return os.WriteFile(path, b, 0o600)
+}
+
+func seedTrust(env *testEnv, presets []string) error {
+	if len(presets) == 0 {
+		return nil
+	}
+	key, err := projectkey.Key(env.ProjectDir)
+	if err != nil {
+		return err
+	}
+	restore := setenv(map[string]string{"XDG_DATA_HOME": env.DataHome})
+	defer restore()
+	store, err := trust.Open(key)
+	if err != nil {
+		return err
+	}
+	for _, p := range presets {
+		if err := store.Grant(p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func setenv(vals map[string]string) func() {
+	type old struct {
+		v  string
+		ok bool
+	}
+	prev := map[string]old{}
+	for k, v := range vals {
+		ov, ok := os.LookupEnv(k)
+		prev[k] = old{v: ov, ok: ok}
+		_ = os.Setenv(k, v)
+	}
+	return func() {
+		for k, ov := range prev {
+			if ov.ok {
+				_ = os.Setenv(k, ov.v)
+			} else {
+				_ = os.Unsetenv(k)
+			}
+		}
+	}
+}
+
+func defaultPattermBin() (string, error) {
+	if p := os.Getenv("PATTERM_BIN"); p != "" {
+		return p, nil
+	}
+	return buildPattermBinary()
+}
+
+func buildPattermBinary() (string, error) {
+	root, err := repoRoot()
+	if err != nil {
+		return "", err
+	}
+	out := filepath.Join(os.TempDir(), "patterm-harness-bin", "patterm")
+	if err := os.MkdirAll(filepath.Dir(out), 0o700); err != nil {
+		return "", err
+	}
+	cmd := exec.Command("go", "build", "-o", out, "./cmd/patterm")
+	cmd.Dir = root
+	cmd.Env = os.Environ()
+	if b, err := cmd.CombinedOutput(); err != nil {
+		return "", fmt.Errorf("build patterm: %w\n%s", err, string(b))
+	}
+	return out, nil
+}
+
+func repoRoot() (string, error) {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		return "", fmt.Errorf("runtime.Caller failed")
+	}
+	return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")), nil
+}

internal/harness/input.go

@@ -0,0 +1,35 @@
+package harness
+
+import "fmt"
+
+func EncodeChord(name string) ([]byte, error) {
+	switch name {
+	case "ctrl-k":
+		return []byte{0x0b}, nil
+	case "ctrl-k-kitty":
+		return []byte("\x1b[107;5u"), nil
+	case "ctrl-k-xterm":
+		return []byte("\x1b[27;5;107~"), nil
+	case "enter":
+		return []byte{'\r'}, nil
+	case "escape":
+		return []byte{0x1b}, nil
+	case "backspace":
+		return []byte{0x7f}, nil
+	case "up":
+		return []byte("\x1b[A"), nil
+	case "down":
+		return []byte("\x1b[B"), nil
+	case "left":
+		return []byte("\x1b[D"), nil
+	case "right":
+		return []byte("\x1b[C"), nil
+	case "ctrl-n":
+		return []byte{0x0e}, nil
+	case "ctrl-p":
+		return []byte{0x10}, nil
+	case "ctrl-u":
+		return []byte{0x15}, nil
+	}
+	return nil, fmt.Errorf("unknown chord %q", name)
+}

internal/harness/input_test.go

@@ -0,0 +1,22 @@
+package harness
+
+import "testing"
+
+func TestEncodeChord(t *testing.T) {
+	tests := map[string]string{
+		"ctrl-k":       "\x0b",
+		"ctrl-k-kitty": "\x1b[107;5u",
+		"ctrl-k-xterm": "\x1b[27;5;107~",
+		"enter":        "\r",
+		"down":         "\x1b[B",
+	}
+	for name, want := range tests {
+		got, err := EncodeChord(name)
+		if err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if string(got) != want {
+			t.Fatalf("%s = %q, want %q", name, string(got), want)
+		}
+	}
+}

internal/harness/mcp_client.go

@@ -0,0 +1,82 @@
+package harness
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net"
+	"sync"
+)
+
+type MCPClient struct {
+	conn net.Conn
+	r    *bufio.Reader
+	mu   sync.Mutex
+	next int
+}
+
+type RPCError struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+	Data    any    `json:"data,omitempty"`
+}
+
+func (e *RPCError) Error() string {
+	return fmt.Sprintf("json-rpc error %d: %s", e.Code, e.Message)
+}
+
+func DialMCP(socket string) (*MCPClient, error) {
+	conn, err := net.Dial("unix", socket)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write([]byte("{\"patterm_identity\":\"harness\"}\n")); err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	return &MCPClient{conn: conn, r: bufio.NewReader(conn)}, nil
+}
+
+func (c *MCPClient) Close() error {
+	if c == nil || c.conn == nil {
+		return nil
+	}
+	return c.conn.Close()
+}
+
+func (c *MCPClient) Call(method string, params any) (json.RawMessage, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.next++
+	req := map[string]any{
+		"jsonrpc": "2.0",
+		"id":      c.next,
+		"method":  method,
+	}
+	if params != nil {
+		req["params"] = params
+	}
+	b, err := json.Marshal(req)
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, '\n')
+	if _, err := c.conn.Write(b); err != nil {
+		return nil, err
+	}
+	line, err := c.r.ReadBytes('\n')
+	if err != nil {
+		return nil, err
+	}
+	var resp struct {
+		Result json.RawMessage `json:"result"`
+		Error  *RPCError       `json:"error"`
+	}
+	if err := json.Unmarshal(line, &resp); err != nil {
+		return nil, fmt.Errorf("parse response: %w: %s", err, string(line))
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+	return resp.Result, nil
+}

internal/harness/observe.go

@@ -0,0 +1,92 @@
+package harness
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func timeoutMS(ms int) time.Duration {
+	if ms <= 0 {
+		ms = 5000
+	}
+	return time.Duration(ms) * time.Millisecond
+}
+
+func valueAt(v any, path string) (any, bool) {
+	if path == "" {
+		return v, true
+	}
+	cur := v
+	for _, part := range strings.Split(path, ".") {
+		switch x := cur.(type) {
+		case map[string]any:
+			next, ok := x[part]
+			if !ok {
+				return nil, false
+			}
+			cur = next
+		case []any:
+			i, err := strconv.Atoi(part)
+			if err != nil || i < 0 || i >= len(x) {
+				return nil, false
+			}
+			cur = x[i]
+		default:
+			return nil, false
+		}
+	}
+	return cur, true
+}
+
+func assertJSONValue(raw json.RawMessage, path string, equals any, contains string, allowSubstring bool) error {
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return err
+	}
+	got, ok := valueAt(v, path)
+	if !ok {
+		return fmt.Errorf("path %q not found in %s", path, string(raw))
+	}
+	if equals != nil {
+		if !reflect.DeepEqual(normalizeJSON(equals), normalizeJSON(got)) {
+			return fmt.Errorf("path %q = %#v, want %#v", path, got, equals)
+		}
+	}
+	if contains != "" {
+		switch x := got.(type) {
+		case string:
+			if !strings.Contains(x, contains) {
+				return fmt.Errorf("path %q = %q, does not contain %q", path, x, contains)
+			}
+		default:
+			b, _ := json.Marshal(x)
+			if allowSubstring && strings.Contains(string(b), contains) {
+				return nil
+			}
+			return fmt.Errorf("path %q is not a string: %#v", path, got)
+		}
+	}
+	return nil
+}
+
+func normalizeJSON(v any) any {
+	switch x := v.(type) {
+	case int:
+		return float64(x)
+	case int64:
+		return float64(x)
+	case []any:
+		for i := range x {
+			x[i] = normalizeJSON(x[i])
+		}
+	case map[string]any:
+		for k := range x {
+			x[k] = normalizeJSON(x[k])
+		}
+	}
+	return v
+}

internal/harness/recorder.go

@@ -0,0 +1,52 @@
+package harness
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type Artifact struct {
+	Dir string `json:"artifact_dir"`
+}
+
+func (s *Session) DumpArtifacts(sc *Scenario, failingStep int, cause error) (*Artifact, error) {
+	name := sc.Name
+	if name == "" {
+		name = strings.TrimSuffix(filepath.Base(sc.Path), filepath.Ext(sc.Path))
+	}
+	if name == "" {
+		name = "scenario"
+	}
+	dir := filepath.Join("internal", "harness", ".artifacts", fmt.Sprintf("%s-%d", name, time.Now().Unix()))
+	abs, _ := filepath.Abs(dir)
+	if err := os.MkdirAll(abs, 0o700); err != nil {
+		return nil, err
+	}
+	screen, _ := s.em.ScreenText()
+	_ = os.WriteFile(filepath.Join(abs, "grid.log"), []byte(screen), 0o600)
+	_ = os.WriteFile(filepath.Join(abs, "raw.bytes"), s.rawBytes(), 0o600)
+	if b, err := s.em.SerializeVT(); err == nil {
+		_ = os.WriteFile(filepath.Join(abs, "serialize.vt"), b, 0o600)
+	}
+	envBytes, _ := json.MarshalIndent(s.env, "", "  ")
+	_ = os.WriteFile(filepath.Join(abs, "env.json"), append(envBytes, '\n'), 0o600)
+	annotated := map[string]any{"scenario": sc, "failing_step": failingStep}
+	if cause != nil {
+		annotated["error"] = cause.Error()
+	}
+	ab, _ := json.MarshalIndent(annotated, "", "  ")
+	_ = os.WriteFile(filepath.Join(abs, "scenario.annotated.json"), append(ab, '\n'), 0o600)
+	snapshot := map[string]json.RawMessage{}
+	for _, method := range []string{"whoami", "list_processes", "get_project_status"} {
+		if raw, err := s.mcp.Call(method, map[string]any{}); err == nil {
+			snapshot[method] = raw
+		}
+	}
+	sb, _ := json.MarshalIndent(snapshot, "", "  ")
+	_ = os.WriteFile(filepath.Join(abs, "mcp-snapshot.json"), append(sb, '\n'), 0o600)
+	return &Artifact{Dir: abs}, nil
+}

internal/harness/runner.go

@@ -0,0 +1,205 @@
+package harness
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Event struct {
+	Step     int             `json:"step"`
+	Type     string          `json:"type"`
+	OK       bool            `json:"ok"`
+	Result   json.RawMessage `json:"result,omitempty"`
+	Error    string          `json:"error,omitempty"`
+	Artifact string          `json:"artifact_dir,omitempty"`
+}
+
+type RunResult struct {
+	OK       bool    `json:"ok"`
+	Steps    int     `json:"steps"`
+	Artifact string  `json:"artifact_dir,omitempty"`
+	Error    string  `json:"error,omitempty"`
+	Events   []Event `json:"-"`
+}
+
+func RunScenario(opts Options, emit func(Event)) (RunResult, error) {
+	sc := opts.Scenario
+	s, err := NewCLI(opts)
+	if err != nil {
+		return RunResult{OK: false, Error: err.Error()}, err
+	}
+	defer s.Close()
+	results := map[string]json.RawMessage{}
+	for i, step := range sc.Steps {
+		ev := Event{Step: i + 1, Type: step.Type}
+		err := runStep(s, step, results)
+		if err != nil {
+			ev.OK = false
+			ev.Error = err.Error()
+			art, aerr := s.DumpArtifacts(sc, i, err)
+			if aerr == nil {
+				ev.Artifact = art.Dir
+			}
+			if emit != nil {
+				emit(ev)
+			}
+			return RunResult{OK: false, Steps: i + 1, Error: err.Error(), Artifact: ev.Artifact}, err
+		}
+		ev.OK = true
+		if step.SaveAs != "" {
+			ev.Result = results[step.SaveAs]
+		}
+		if emit != nil {
+			emit(ev)
+		}
+	}
+	return RunResult{OK: true, Steps: len(sc.Steps)}, nil
+}
+
+func runStep(s *Session, step Step, results map[string]json.RawMessage) error {
+	switch step.Type {
+	case "wait_stable":
+		return s.WaitForStable(timeoutMS(step.TimeoutMS))
+	case "send_chord":
+		return s.SendChord(step.Chord)
+	case "send_text":
+		return s.SendText(step.Text)
+	case "assert_contains":
+		screen, err := s.Screen()
+		if err != nil {
+			return err
+		}
+		if !strings.Contains(screen, step.Contains) {
+			return fmt.Errorf("screen does not contain %q:\n%s", step.Contains, screen)
+		}
+		return nil
+	case "assert_regex":
+		return s.WaitForRegex(step.Regex, timeoutMS(step.TimeoutMS))
+	case "wait_text":
+		return s.WaitForText(step.Contains, timeoutMS(step.TimeoutMS))
+	case "assert_cursor":
+		cur, err := s.Cursor()
+		if err != nil {
+			return err
+		}
+		if step.CursorRow != nil && int(cur.Row) != *step.CursorRow {
+			return fmt.Errorf("cursor row = %d, want %d", cur.Row, *step.CursorRow)
+		}
+		if step.CursorCol != nil && int(cur.Col) != *step.CursorCol {
+			return fmt.Errorf("cursor col = %d, want %d", cur.Col, *step.CursorCol)
+		}
+		return nil
+	case "mcp_call":
+		params, err := resolveParams(step.Params, results)
+		if err != nil {
+			return err
+		}
+		raw, err := s.MCPCall(step.Method, params)
+		if err != nil {
+			return err
+		}
+		if step.SaveAs != "" {
+			results[step.SaveAs] = raw
+		}
+		return nil
+	case "assert_mcp":
+		params, perr := resolveParams(step.Params, results)
+		if perr != nil {
+			return perr
+		}
+		raw, err := s.MCPCall(step.Method, params)
+		if step.ErrorKind != "" {
+			if err == nil {
+				return fmt.Errorf("expected MCP error kind %q, got success %s", step.ErrorKind, string(raw))
+			}
+			rpcErr, ok := err.(*RPCError)
+			if !ok {
+				return err
+			}
+			if rpcErr.Data != nil {
+				if data, ok := rpcErr.Data.(map[string]any); ok && data["kind"] == step.ErrorKind {
+					return nil
+				}
+			}
+			if strings.Contains(rpcErr.Message, step.ErrorKind) || strings.Contains(rpcErr.Message, strings.ReplaceAll(step.ErrorKind, "_", " ")) {
+				return nil
+			}
+			return fmt.Errorf("MCP error = %#v, want kind %q", rpcErr, step.ErrorKind)
+		}
+		if err != nil {
+			return err
+		}
+		return assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring)
+	case "assert_saved":
+		raw, ok := results[step.From]
+		if !ok {
+			return fmt.Errorf("no saved result %q", step.From)
+		}
+		return assertJSONValue(raw, step.Path, step.Equals, step.Contains, step.AllowSubstring)
+	}
+	return fmt.Errorf("unknown step type %q", step.Type)
+}
+
+func resolveParams(raw json.RawMessage, saved map[string]json.RawMessage) (json.RawMessage, error) {
+	if len(raw) == 0 {
+		return raw, nil
+	}
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return nil, err
+	}
+	resolved, err := resolveValue(v, saved)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(resolved)
+}
+
+func resolveValue(v any, saved map[string]json.RawMessage) (any, error) {
+	switch x := v.(type) {
+	case string:
+		if strings.HasPrefix(x, "{{") && strings.HasSuffix(x, "}}") {
+			ref := strings.TrimSuffix(strings.TrimPrefix(x, "{{"), "}}")
+			parts := strings.SplitN(ref, ".", 2)
+			raw, ok := saved[parts[0]]
+			if !ok {
+				return nil, fmt.Errorf("unknown saved result %q", parts[0])
+			}
+			var sv any
+			if err := json.Unmarshal(raw, &sv); err != nil {
+				return nil, err
+			}
+			path := ""
+			if len(parts) == 2 {
+				path = parts[1]
+			}
+			got, ok := valueAt(sv, path)
+			if !ok {
+				return nil, fmt.Errorf("path %q not found in saved result %q", path, parts[0])
+			}
+			return got, nil
+		}
+		return x, nil
+	case []any:
+		for i := range x {
+			r, err := resolveValue(x[i], saved)
+			if err != nil {
+				return nil, err
+			}
+			x[i] = r
+		}
+		return x, nil
+	case map[string]any:
+		for k := range x {
+			r, err := resolveValue(x[k], saved)
+			if err != nil {
+				return nil, err
+			}
+			x[k] = r
+		}
+		return x, nil
+	default:
+		return x, nil
+	}
+}

internal/harness/scenario.go

@@ -0,0 +1,76 @@
+package harness
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+type Scenario struct {
+	Path       string            `json:"-"`
+	Name       string            `json:"name,omitempty"`
+	ProjectDir string            `json:"project_dir,omitempty"`
+	Cols       uint16            `json:"cols,omitempty"`
+	Rows       uint16            `json:"rows,omitempty"`
+	Env        map[string]string `json:"env,omitempty"`
+	Trust      []string          `json:"trust,omitempty"`
+	Presets    ScenarioPresets   `json:"presets,omitempty"`
+	Scripts    []ScenarioScript  `json:"scripts,omitempty"`
+	Steps      []Step            `json:"steps"`
+}
+
+type ScenarioPresets struct {
+	Agents    []ScenarioPreset `json:"agents,omitempty"`
+	Processes []ScenarioPreset `json:"processes,omitempty"`
+}
+
+type ScenarioPreset struct {
+	Name       string            `json:"name"`
+	Argv       []string          `json:"argv"`
+	Env        map[string]string `json:"env,omitempty"`
+	WorkingDir string            `json:"working_dir,omitempty"`
+	Shell      bool              `json:"shell,omitempty"`
+}
+
+type ScenarioScript struct {
+	Name string `json:"name"`
+	Body string `json:"body"`
+}
+
+type Step struct {
+	Type           string          `json:"type"`
+	Chord          string          `json:"chord,omitempty"`
+	Text           string          `json:"text,omitempty"`
+	Contains       string          `json:"contains,omitempty"`
+	Regex          string          `json:"regex,omitempty"`
+	TimeoutMS      int             `json:"timeout_ms,omitempty"`
+	Method         string          `json:"method,omitempty"`
+	Params         json.RawMessage `json:"params,omitempty"`
+	SaveAs         string          `json:"save_as,omitempty"`
+	From           string          `json:"from,omitempty"`
+	Path           string          `json:"path,omitempty"`
+	Equals         any             `json:"equals,omitempty"`
+	ErrorKind      string          `json:"error_kind,omitempty"`
+	CursorRow      *int            `json:"cursor_row,omitempty"`
+	CursorCol      *int            `json:"cursor_col,omitempty"`
+	AllowSubstring bool            `json:"allow_substring,omitempty"`
+}
+
+func LoadScenario(path string) (*Scenario, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var sc Scenario
+	if err := json.Unmarshal(b, &sc); err != nil {
+		return nil, fmt.Errorf("parse scenario %s: %w", path, err)
+	}
+	sc.Path = path
+	if sc.Cols == 0 {
+		sc.Cols = 120
+	}
+	if sc.Rows == 0 {
+		sc.Rows = 40
+	}
+	return &sc, nil
+}

internal/harness/scenario_test.go

@@ -0,0 +1,49 @@
+package harness
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestLoadScenario(t *testing.T) {
+	sc, err := LoadScenario(filepath.Join("scenarios", "spawn_process_via_palette.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if sc.Cols != 120 || sc.Rows != 40 {
+		t.Fatalf("default size = %dx%d, want 120x40", sc.Cols, sc.Rows)
+	}
+	if len(sc.Steps) == 0 {
+		t.Fatal("expected steps")
+	}
+}
+
+func TestHarnessScenarios(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping end-to-end harness scenarios in short mode")
+	}
+	paths, err := filepath.Glob(filepath.Join("scenarios", "*.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(paths) == 0 {
+		t.Fatal("no scenarios")
+	}
+	for _, path := range paths {
+		path := path
+		t.Run(filepath.Base(path), func(t *testing.T) {
+			sc, err := LoadScenario(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			res, err := RunScenario(Options{Scenario: sc, PattermBin: os.Getenv("PATTERM_BIN")}, nil)
+			if err != nil {
+				if res.Artifact != "" {
+					t.Fatalf("%v\nartifact: %s", err, res.Artifact)
+				}
+				t.Fatal(err)
+			}
+		})
+	}
+}

internal/harness/scenarios/child_exit.json

@@ -0,0 +1,23 @@
+{
+  "name": "child_exit",
+  "trust": ["exit0"],
+  "presets": {
+    "processes": [
+      {
+        "name": "exit0",
+        "argv": ["sh", "-lc", "exit 0"]
+      }
+    ]
+  },
+  "steps": [
+    {
+      "type": "mcp_call",
+      "method": "spawn_process",
+      "params": { "kind": "command", "preset": "exit0" },
+      "save_as": "proc"
+    },
+    { "type": "wait_stable", "timeout_ms": 3000 },
+    { "type": "assert_mcp", "method": "list_processes", "path": "0.status", "equals": "exited" },
+    { "type": "assert_mcp", "method": "list_processes", "path": "0.exit_code", "equals": 0 }
+  ]
+}

internal/harness/scenarios/mcp_send_input.json

@@ -0,0 +1,30 @@
+{
+  "name": "mcp_send_input",
+  "steps": [
+    {
+      "type": "mcp_call",
+      "method": "spawn_process",
+      "params": {
+        "kind": "command",
+        "argv": ["sh", "-lc", "read line; echo got:$line; sleep 5"],
+        "name": "reader"
+      },
+      "save_as": "proc"
+    },
+    {
+      "type": "mcp_call",
+      "method": "send_input",
+      "params": {
+        "process_id": "{{proc.process_id}}",
+        "kind": "text",
+        "text": "hello",
+        "submit": true,
+        "wait_ms": 250,
+        "tail_mode": "grid"
+      },
+      "save_as": "input"
+    },
+    { "type": "assert_saved", "from": "input", "path": "tail.content", "contains": "got:hello" },
+    { "type": "wait_text", "contains": "got:hello", "timeout_ms": 5000 }
+  ]
+}

internal/harness/scenarios/spawn_process_via_palette.json

@@ -0,0 +1,26 @@
+{
+  "name": "spawn_process_via_palette",
+  "presets": {
+    "processes": [
+      {
+        "name": "ready",
+        "argv": ["ready-script"]
+      }
+    ]
+  },
+  "scripts": [
+    {
+      "name": "ready-script",
+      "body": "#!/bin/sh\necho READY\nsleep 5\n"
+    }
+  ],
+  "steps": [
+    { "type": "wait_stable", "timeout_ms": 3000 },
+    { "type": "send_chord", "chord": "ctrl-k" },
+    { "type": "send_text", "text": "ready" },
+    { "type": "send_chord", "chord": "enter" },
+    { "type": "wait_text", "contains": "READY", "timeout_ms": 5000 },
+    { "type": "assert_mcp", "method": "list_processes", "path": "0.name", "equals": "ready" },
+    { "type": "assert_mcp", "method": "list_processes", "path": "0.status", "equals": "running" }
+  ]
+}

internal/harness/scenarios/switch_via_palette.json

@@ -0,0 +1,34 @@
+{
+  "name": "switch_via_palette",
+  "scripts": [
+    {
+      "name": "named-loop",
+      "body": "#!/bin/sh\necho \"$1 READY\"\nsleep 5\n"
+    }
+  ],
+  "steps": [
+    {
+      "type": "mcp_call",
+      "method": "spawn_process",
+      "params": { "kind": "command", "argv": ["named-loop", "first"], "name": "first" },
+      "save_as": "first"
+    },
+    {
+      "type": "mcp_call",
+      "method": "spawn_process",
+      "params": { "kind": "command", "argv": ["named-loop", "second"], "name": "second" },
+      "save_as": "second"
+    },
+    { "type": "wait_text", "contains": "second READY", "timeout_ms": 5000 },
+    { "type": "send_chord", "chord": "ctrl-k" },
+    { "type": "send_text", "text": "Switch to first" },
+    { "type": "send_chord", "chord": "enter" },
+    { "type": "wait_text", "contains": "first READY", "timeout_ms": 5000 },
+    {
+      "type": "assert_mcp",
+      "method": "get_project_status",
+      "path": "processes.0.name",
+      "equals": "first"
+    }
+  ]
+}

internal/harness/scenarios/trust_required.json

@@ -0,0 +1,19 @@
+{
+  "name": "trust_required",
+  "presets": {
+    "processes": [
+      {
+        "name": "untrusted",
+        "argv": ["sh", "-lc", "echo should-not-run"]
+      }
+    ]
+  },
+  "steps": [
+    {
+      "type": "assert_mcp",
+      "method": "spawn_process",
+      "params": { "kind": "command", "preset": "untrusted" },
+      "error_kind": "needs_trust"
+    }
+  ]
+}

internal/harness/session.go

@@ -0,0 +1,264 @@
+package harness
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	pkgpty "github.com/hjbdev/patterm/internal/pty"
+	"github.com/hjbdev/patterm/internal/vt"
+)
+
+type Session struct {
+	pty *pkgpty.PTY
+	em  *vt.GhosttyEmulator
+	mcp *MCPClient
+	env *testEnv
+
+	bytesMu sync.Mutex
+	bytes   []byte
+
+	lastWriteNS atomic.Int64
+	readerDone  chan struct{}
+	closeOnce   sync.Once
+	closeErr    error
+}
+
+func New(t testing.TB, opts Options) *Session {
+	t.Helper()
+	s, err := NewCLI(opts)
+	if err != nil {
+		t.Fatalf("harness New: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func NewCLI(opts Options) (*Session, error) {
+	if opts.Scenario == nil {
+		return nil, fmt.Errorf("harness: Scenario required")
+	}
+	env, childEnv, err := prepareEnv(opts)
+	if err != nil {
+		return nil, err
+	}
+	em, err := vt.NewGhosttyEmulator(env.Cols, env.Rows)
+	if err != nil {
+		return nil, err
+	}
+	p, err := pkgpty.Start([]string{env.PattermBin, "--project", env.ProjectDir}, childEnv, env.Cols, env.Rows)
+	if err != nil {
+		_ = em.Close()
+		return nil, err
+	}
+	em.OnWritePTY(func(b []byte) { _, _ = p.Write(b) })
+	s := &Session{pty: p, em: em, env: env, readerDone: make(chan struct{})}
+	go s.readLoop()
+	if err := s.bootstrapMCP(2 * time.Second); err != nil {
+		_ = s.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *Session) readLoop() {
+	defer close(s.readerDone)
+	buf := make([]byte, 64*1024)
+	for {
+		n, err := s.pty.Read(buf)
+		if n > 0 {
+			chunk := make([]byte, n)
+			copy(chunk, buf[:n])
+			_, _ = s.em.Write(chunk)
+			s.bytesMu.Lock()
+			s.bytes = append(s.bytes, chunk...)
+			s.bytesMu.Unlock()
+			s.lastWriteNS.Store(time.Now().UnixNano())
+		}
+		if err != nil {
+			return
+		}
+	}
+}
+
+func (s *Session) bootstrapMCP(timeout time.Duration) error {
+	socket := filepath.Join(s.env.RuntimeDir, "patterm", fmt.Sprintf("%d.sock", s.pty.Pid()))
+	deadline := time.Now().Add(timeout)
+	var last error
+	for time.Now().Before(deadline) {
+		if _, err := os.Stat(socket); err != nil {
+			last = err
+			time.Sleep(25 * time.Millisecond)
+			continue
+		}
+		c, err := DialMCP(socket)
+		if err != nil {
+			last = err
+			time.Sleep(25 * time.Millisecond)
+			continue
+		}
+		_, err = c.Call("whoami", map[string]any{})
+		if err == nil {
+			s.mcp = c
+			return nil
+		}
+		last = err
+		_ = c.Close()
+		if strings.Contains(err.Error(), "tool host not initialized") {
+			time.Sleep(25 * time.Millisecond)
+			continue
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	raw := strings.TrimSpace(string(s.rawBytes()))
+	if raw != "" {
+		return fmt.Errorf("mcp bootstrap timed out: %w; child output: %s", last, raw)
+	}
+	return fmt.Errorf("mcp bootstrap timed out: %w", last)
+}
+
+func (s *Session) Close() error {
+	s.closeOnce.Do(func() {
+		if s.mcp != nil {
+			_ = s.mcp.Close()
+		}
+		pid := s.pty.Pid()
+		if pid > 0 {
+			if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil {
+				_ = syscall.Kill(pid, syscall.SIGTERM)
+			}
+		}
+		done := make(chan error, 1)
+		go func() { done <- s.pty.Wait() }()
+		select {
+		case <-done:
+		case <-time.After(2 * time.Second):
+			if pid > 0 {
+				if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil {
+					_ = syscall.Kill(pid, syscall.SIGKILL)
+				}
+			}
+			select {
+			case <-done:
+			case <-time.After(500 * time.Millisecond):
+			}
+		}
+		select {
+		case <-s.readerDone:
+		case <-time.After(time.Second):
+			if err := s.pty.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
+				s.closeErr = err
+			}
+			select {
+			case <-s.readerDone:
+			case <-time.After(500 * time.Millisecond):
+			}
+		}
+		_ = s.em.Close()
+	})
+	return s.closeErr
+}
+
+func (s *Session) SendChord(name string) error {
+	b, err := EncodeChord(name)
+	if err != nil {
+		return err
+	}
+	_, err = s.pty.Write(b)
+	return err
+}
+
+func (s *Session) SendText(text string) error {
+	_, err := s.pty.Write([]byte(text))
+	return err
+}
+
+func (s *Session) Screen() (string, error) { return s.em.PlainText() }
+
+func (s *Session) Cursor() (vt.CursorState, error) { return s.em.Cursor() }
+
+func (s *Session) WaitForStable(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	tick := time.NewTicker(25 * time.Millisecond)
+	defer tick.Stop()
+	confirmed := false
+	for {
+		last := s.lastWriteNS.Load()
+		idle := last == 0 || time.Since(time.Unix(0, last)) >= time.Second
+		if idle {
+			if confirmed {
+				return nil
+			}
+			confirmed = true
+		} else {
+			confirmed = false
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("screen did not stabilize within %s", timeout)
+		}
+		<-tick.C
+	}
+}
+
+func (s *Session) WaitForText(text string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		screen, err := s.Screen()
+		if err != nil {
+			return err
+		}
+		if strings.Contains(screen, text) {
+			return nil
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	screen, _ := s.Screen()
+	return fmt.Errorf("text %q not found before timeout; screen:\n%s", text, screen)
+}
+
+func (s *Session) WaitForRegex(pattern string, timeout time.Duration) error {
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return err
+	}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		screen, err := s.Screen()
+		if err != nil {
+			return err
+		}
+		if re.MatchString(screen) {
+			return nil
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	screen, _ := s.Screen()
+	return fmt.Errorf("regex %q not found before timeout; screen:\n%s", pattern, screen)
+}
+
+func (s *Session) MCPCall(method string, params json.RawMessage) (json.RawMessage, error) {
+	var v any = map[string]any{}
+	if len(params) > 0 {
+		if err := json.Unmarshal(params, &v); err != nil {
+			return nil, err
+		}
+	}
+	return s.mcp.Call(method, v)
+}
+
+func (s *Session) rawBytes() []byte {
+	s.bytesMu.Lock()
+	defer s.bytesMu.Unlock()
+	out := make([]byte, len(s.bytes))
+	copy(out, s.bytes)
+	return out
+}