aggressive token saving attempts

This commit is contained in:
2026-05-29 14:23:09 +01:00
parent 51aac9f447
commit 45263d59f8
9 changed files with 481 additions and 38 deletions

View File

@@ -13,9 +13,10 @@ loosely follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Changed
- The tab bar now shows each visible agent tab's own summary instead
of only rendering the focused tab's summary.
- Grid-mode `get_process_output` now returns whitespace-normalized
text to avoid sending padded terminal rows and repeated blank lines
over MCP.
- `get_process_output` now returns aggressively canonical terminal text
by default, removing ANSI/control noise, decorative borders, duplicate
status churn, and volatile progress/timer fragments; raw PTY bytes are
opt-in with `raw:true`.
- MCP responses now use slimmer defaults: tool-call JSON is no longer
duplicated into text content, large output and scratchpad reads are
capped with truncation metadata, and `whoami` / `get_project_status`

View File

@@ -0,0 +1 @@
- [ ] Pasting into codex is no longer clean, it sends loads of messages rather than one clean paste.

143
internal/app/canonical.go Normal file
View File

@@ -0,0 +1,143 @@
package app
import (
"regexp"
"strings"
"unicode"
"unicode/utf8"
)
var (
statusVolatileRE = regexp.MustCompile(`\b(?:\d+h\s*)?\d+m\s*\d+s\b|\b\d{1,2}:\d{2}(?::\d{2})?\b|\b\d+(?:\.\d+)?s\b`)
counterRE = regexp.MustCompile(`\b\d+\s*/\s*\d+\b|\b\d{1,3}%`)
spinnerGlyphRE = regexp.MustCompile(`^[\s⠁⠂⠄⡀⢀⠠⠐⠈⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏•·∙◐◓◑◒]+`)
)
func canonicalizeTerminalText(s string, maxLines int) (string, bool, int) {
s = string(stripANSIBytes(nil, []byte(s)))
s = strings.ReplaceAll(s, "\r\n", "\n")
s = carriageReturnToLines(s)
s = strings.ReplaceAll(s, "\r", "\n")
lines := strings.Split(s, "\n")
out := make([]string, 0, len(lines))
pendingBlank := false
for _, raw := range lines {
line := strings.TrimRightFunc(stripControlRunes(raw), unicode.IsSpace)
if strings.TrimSpace(line) == "" {
if len(out) > 0 {
pendingBlank = true
}
continue
}
if isBorderOnlyLine(line) {
continue
}
line = canonicalStatusLine(line)
if len(out) > 0 && out[len(out)-1] == line {
pendingBlank = false
continue
}
if pendingBlank {
out = append(out, "")
pendingBlank = false
}
out = append(out, line)
}
if maxLines > 0 && len(out) > maxLines {
dropped := strings.Join(out[:len(out)-maxLines], "\n")
out = out[len(out)-maxLines:]
return strings.Join(out, "\n"), true, len(dropped)
}
return strings.Join(out, "\n"), false, 0
}
func carriageReturnToLines(s string) string {
var out []string
var current strings.Builder
flush := func() {
out = append(out, current.String())
current.Reset()
}
for len(s) > 0 {
r, size := utf8.DecodeRuneInString(s)
s = s[size:]
switch r {
case '\r':
current.Reset()
case '\n':
flush()
default:
current.WriteRune(r)
}
}
if current.Len() > 0 || len(out) == 0 {
flush()
}
return strings.Join(out, "\n")
}
func stripControlRunes(s string) string {
return strings.Map(func(r rune) rune {
if r == '\t' || r == '\n' {
return r
}
if unicode.IsControl(r) {
return -1
}
return r
}, s)
}
func isBorderOnlyLine(s string) bool {
trimmed := strings.TrimSpace(s)
if trimmed == "" {
return false
}
seenBox := false
for _, r := range trimmed {
if r >= 0x2500 && r <= 0x257f {
seenBox = true
continue
}
switch r {
case ' ', '\t', '-', '_', '=', '+', '|', ':', '.', '\'', '"', '`', '*':
continue
default:
return false
}
}
return seenBox
}
func canonicalStatusLine(s string) string {
if !looksStatusLike(s) {
return s
}
leading := len(s) - len(strings.TrimLeftFunc(s, unicode.IsSpace))
prefix := s[:leading]
body := s[leading:]
body = spinnerGlyphRE.ReplaceAllString(body, "")
body = statusVolatileRE.ReplaceAllString(body, "[time]")
body = counterRE.ReplaceAllString(body, "[count]")
return prefix + strings.TrimRightFunc(body, unicode.IsSpace)
}
func looksStatusLike(s string) bool {
lower := strings.ToLower(s)
for _, token := range []string{
"status", "running", "remaining", "progress", "loading",
"building", "installing", "downloading", "waiting", "working",
} {
if strings.Contains(lower, token) {
return true
}
}
trimmed := strings.TrimSpace(s)
if trimmed == "" {
return false
}
r, _ := utf8.DecodeRuneInString(trimmed)
return strings.ContainsRune("⠁⠂⠄⡀⢀⠠⠐⠈⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏•·∙◐◓◑◒", r)
}

View File

@@ -0,0 +1,167 @@
package app
import (
"strings"
"testing"
"github.com/hjbdev/patterm/internal/mcp"
"github.com/hjbdev/patterm/internal/preset"
)
func TestCanonicalizeTerminalText(t *testing.T) {
cases := []struct {
name string
in string
want string
}{
{
name: "ansi osc and controls",
in: "\x1b]0;title\x07\x1b[31mred\x1b[0m\x00\nok",
want: "red\nok",
},
{
name: "noisy harness stream",
in: "\x1b]0;noise\x07\x1b[31mStatus: running 12s\x1b[0m\nStatus: running 13s\n╭────╮\n│ │\nDownloading 10%\rDownloading 100%\nFINAL: deploy ready\n",
want: "Status: running [time]\nDownloading [count]\nFINAL: deploy ready",
},
{
name: "repeated blank collapse",
in: "one\n\n\n two\n \n\t\nthree",
want: "one\n\n two\n\nthree",
},
{
name: "border only box drawing removal",
in: "╭────────╮\n│ │\nimportant\n╰────────╯",
want: "important",
},
{
name: "carriage return progress coalesces final frame",
in: "Downloading 10%\rDownloading 20%\rDownloading 100%\nDone",
want: "Downloading [count]\nDone",
},
{
name: "volatile timer duplicate collapse",
in: "Status: running 12s\nStatus: running 13s\nStatus: running 01:23",
want: "Status: running [time]",
},
{
name: "duplicate status row collapse",
in: "⠋ Building 1/4\n⠙ Building 2/4\n⠹ Building 3/4\nready",
want: "Building [count]\nready",
},
{
name: "preserve meaningful indented code and tables",
in: " if elapsed == 12s {\n return value\n }\n| name | value |\n| a | 1 |",
want: " if elapsed == 12s {\n return value\n }\n| name | value |\n| a | 1 |",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, truncated, _ := canonicalizeTerminalText(tc.in, 120)
if truncated {
t.Fatalf("unexpected truncation")
}
if got != tc.want {
t.Fatalf("got %q want %q", got, tc.want)
}
})
}
}
func TestCanonicalizeTerminalTextMaxLines(t *testing.T) {
got, truncated, dropped := canonicalizeTerminalText("one\ntwo\nthree", 2)
if !truncated {
t.Fatalf("expected truncation")
}
if dropped == 0 {
t.Fatalf("expected dropped bytes")
}
if got != "two\nthree" {
t.Fatalf("got %q", got)
}
}
func TestGetProcessOutputStreamCanonicalByDefault(t *testing.T) {
sess := NewSession(t.TempDir(), "test")
c := newChildEntry("p1", "proc", KindCommand, nil, nil, "", "", "")
addChild(sess, c)
c.recordWrite([]byte("\x1b[31mStatus: running 12s\x1b[0m\nStatus: running 13s\nresult\n"))
host := newToolHost(sess, nil, nil, preset.Set{}, nil, 80, 24)
out, err := host.GetProcessOutput("", mcp.ProcessOutputArgs{ProcessID: c.ID, Mode: "stream"})
if err != nil {
t.Fatal(err)
}
if !out.Canonicalized {
t.Fatalf("expected canonicalized output")
}
if out.Content != "Status: running [time]\nresult" {
t.Fatalf("content = %q", out.Content)
}
if out.Cursor != nil || out.Rows != 0 || out.Cols != 0 || out.ScreenVersion != 0 || out.IdleMS != 0 {
t.Fatalf("default output should be metadata-light: %#v", out)
}
}
func TestGetProcessOutputRawReturnsStreamBytes(t *testing.T) {
sess := NewSession(t.TempDir(), "test")
c := newChildEntry("p1", "proc", KindCommand, nil, nil, "", "", "")
addChild(sess, c)
c.recordWrite([]byte("\x1b[31mred\x1b[0m"))
host := newToolHost(sess, nil, nil, preset.Set{}, nil, 80, 24)
out, err := host.GetProcessOutput("", mcp.ProcessOutputArgs{ProcessID: c.ID, Mode: "grid", Raw: true})
if err != nil {
t.Fatal(err)
}
if out.Mode != "stream" {
t.Fatalf("raw grid mode should report stream semantics, got %q", out.Mode)
}
if out.Canonicalized {
t.Fatalf("raw output should not be canonicalized")
}
if out.Content != "\x1b[31mred\x1b[0m" {
t.Fatalf("content = %q", out.Content)
}
if out.NewOffset != int64(len(out.Content)) {
t.Fatalf("new_offset=%d want %d", out.NewOffset, len(out.Content))
}
}
func TestGetProcessOutputCanonicalAfterRawRead(t *testing.T) {
sess := NewSession(t.TempDir(), "test")
c := newChildEntry("p1", "proc", KindCommand, nil, nil, "", "", "")
addChild(sess, c)
c.recordWrite([]byte("\x1b[31mStatus: running 12s\x1b[0m\nStatus: running 13s\nDownloading 10%\rDownloading 100%\nFINAL: deploy ready\n"))
host := newToolHost(sess, nil, nil, preset.Set{}, nil, 80, 24)
if _, err := host.GetProcessOutput("", mcp.ProcessOutputArgs{ProcessID: c.ID, Mode: "stream", Raw: true}); err != nil {
t.Fatal(err)
}
out, err := host.GetProcessOutput("", mcp.ProcessOutputArgs{ProcessID: c.ID, Mode: "stream", MaxLines: 20})
if err != nil {
t.Fatal(err)
}
if out.Content != "Status: running [time]\nDownloading [count]\nFINAL: deploy ready" {
t.Fatalf("content = %q", out.Content)
}
}
func TestGetProcessOutputIncludeMetaRestoresFields(t *testing.T) {
sess := NewSession(t.TempDir(), "test")
c := newChildEntry("p1", "proc", KindCommand, nil, nil, "", "", "")
addChild(sess, c)
c.recordWrite([]byte("ok"))
host := newToolHost(sess, nil, nil, preset.Set{}, nil, 80, 24)
out, err := host.GetProcessOutput("", mcp.ProcessOutputArgs{ProcessID: c.ID, Mode: "stream", IncludeMeta: true})
if err != nil {
t.Fatal(err)
}
if out.ScreenVersion == 0 {
t.Fatalf("screen_version missing with include_meta: %#v", out)
}
if !strings.Contains(out.Content, "ok") {
t.Fatalf("content = %q", out.Content)
}
}

View File

@@ -68,6 +68,8 @@ type toolHost struct {
const (
defaultMCPContentBytes = 12_000
maxMCPContentBytes = 65_536
defaultMCPCanonicalLines = 120
maxMCPCanonicalLines = 500
defaultMCPTailBytes = 8_000
defaultScratchpadReadBytes = 12_000
defaultSearchLineBytes = 2_000
@@ -380,22 +382,42 @@ func (h *toolHost) GetProcessOutput(callerID string, args mcp.ProcessOutputArgs)
if c == nil {
return mcp.ProcessOutput{}, mcp.Errorf(mcp.ErrorKindNotFound, "no such process %q", processID)
}
if mode == "" {
mode = "grid"
}
if args.Raw {
b, end := c.StreamRead(sinceOffset)
content, contentBytes, truncated, truncatedBytes := capBytesTail(b, capLimit(args.MaxBytes, defaultMCPContentBytes))
return mcp.ProcessOutput{
Content: content,
Mode: "stream",
NewOffset: end,
Status: string(c.Status()),
ContentBytes: contentBytes,
Truncated: truncated,
TruncatedBytes: truncatedBytes,
}, nil
}
out := mcp.ProcessOutput{
Mode: mode,
IdleMS: c.IdleMS(),
Status: string(c.Status()),
ScreenVersion: c.ScreenVersion(),
Canonicalized: true,
}
if args.IncludeMeta {
out.IdleMS = c.IdleMS()
out.ScreenVersion = c.ScreenVersion()
if em := c.Emulator(); em != nil {
if sc, err := em.ActiveScreen(); err == nil {
out.ActiveScreen = activeScreenName(sc)
}
if cur, err := em.Cursor(); err == nil {
out.Cursor = mcp.Cursor{X: int(cur.Col), Y: int(cur.Row)}
out.Cursor = &mcp.Cursor{X: int(cur.Col), Y: int(cur.Row)}
}
cols, rows := em.Size()
out.Cols, out.Rows = int(cols), int(rows)
}
}
maxLines := canonicalLineLimit(args.MaxLines)
switch mode {
case "grid":
em := c.Emulator()
@@ -409,12 +431,21 @@ func (h *toolHost) GetProcessOutput(callerID string, args mcp.ProcessOutputArgs)
if c.Kind == KindAgent {
txt = applyChromeTrim(txt, h.chromeHintsFor(c.PresetRef))
}
content := normalizeGridText(txt)
content, lineTruncated, lineDroppedBytes := canonicalizeTerminalText(txt, maxLines)
out.Content, out.ContentBytes, out.Truncated, out.TruncatedBytes = capTextMiddle(content, capLimit(args.MaxBytes, defaultMCPContentBytes))
if lineTruncated {
out.Truncated = true
out.TruncatedBytes += lineDroppedBytes
}
return out, nil
case "stream":
b, end := c.StreamRead(sinceOffset)
out.Content, out.ContentBytes, out.Truncated, out.TruncatedBytes = capBytesTail(stripANSIBytes(nil, b), capLimit(args.MaxBytes, defaultMCPContentBytes))
content, lineTruncated, lineDroppedBytes := canonicalizeTerminalText(string(b), maxLines)
out.Content, out.ContentBytes, out.Truncated, out.TruncatedBytes = capTextTail(content, capLimit(args.MaxBytes, defaultMCPContentBytes))
if lineTruncated {
out.Truncated = true
out.TruncatedBytes += lineDroppedBytes
}
out.NewOffset = end
return out, nil
default:
@@ -1064,11 +1095,10 @@ func activeScreenName(s pkgvt.Screen) string {
}
}
// ansiRegexp strips CSI escape sequences and common single-character
// controls (BEL, OSC terminators) from the stream. The vt emulator
// already handles full rendering for grid mode; this is only for
// stream-mode ANSI-stripped output.
var ansiRegexp = regexp.MustCompile(`\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]|\x1b[\x40-\x5f]|\x07`)
// ansiRegexp strips CSI/OSC escape sequences and common single-character
// controls from the stream. The vt emulator already handles full
// rendering for grid mode; this is only for stream-mode text output.
var ansiRegexp = regexp.MustCompile(`\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]|\x1b[\x40-\x5f]|\x07`)
func stripANSI(s string) string {
return ansiRegexp.ReplaceAllString(s, "")
@@ -1111,6 +1141,16 @@ func capLimit(requested, def int) int {
return requested
}
func canonicalLineLimit(requested int) int {
if requested <= 0 {
return defaultMCPCanonicalLines
}
if requested > maxMCPCanonicalLines {
return maxMCPCanonicalLines
}
return requested
}
func capBytesTail(b []byte, limit int) (string, int, bool, int) {
if limit <= 0 || len(b) <= limit {
return string(b), len(b), false, 0
@@ -1149,6 +1189,7 @@ func capTextMiddle(s string, limit int) (string, int, bool, int) {
// pattern match (WaitForPattern scrollback). Recognises the same
// shapes the regex did:
// - `\x1b[ <params> <intermediate> <final-byte>` (CSI / SGR)
// - `\x1b] ... (BEL|ST)` (OSC)
// - `\x1b<final-byte>` for `@..._` (one-byte escapes)
// - `\x07` (BEL)
//
@@ -1178,6 +1219,24 @@ func stripANSIBytes(dst, src []byte) []byte {
continue
}
next := src[i+1]
if next == ']' {
j := i + 2
for j < len(src) {
if src[j] == 0x07 {
i = j + 1
break
}
if src[j] == 0x1b && j+1 < len(src) && src[j+1] == '\\' {
i = j + 2
break
}
j++
}
if j >= len(src) {
i = len(src)
}
continue
}
if next != '[' {
// One-byte ESC sequence (`\x1b<final>` where final is
// `@..._` per the regex; we drop anything that follows).
@@ -1260,7 +1319,7 @@ func helpFor(topic string) mcp.HelpResponse {
case "inspection":
return mcp.HelpResponse{
Topic: "inspection",
Content: "get_process_output gives you the visible pane (grid mode) or a byte slice from since_offset (stream mode). list_processes is for the whole session. get_project_status batches everything you need to orient yourself.",
Content: "get_process_output gives you canonical terminal text by default: the visible pane (grid mode) or recent stream text from since_offset (stream mode), with ANSI/control noise, borders, duplicate status churn, and volatile timers removed. Use raw:true only when you need diagnostic PTY bytes; include_meta:true restores cursor, geometry, and screen-version fields. list_processes is for the whole session. get_project_status batches everything you need to orient yourself.",
RelatedTools: []string{"list_processes", "get_process_status", "get_process_output", "search_output", "wait_for_pattern", "get_project_status"},
}
case "io":

View File

@@ -90,6 +90,8 @@ func TestStripANSIBytesEquivalence(t *testing.T) {
cases := []string{
"hello world",
"\x1b[31mred\x1b[0m text",
"\x1b]0;title\x07after osc",
"\x1b]2;title\x1b\\after st",
"line1\nline2\r\nline3",
"bell\x07ish",
"weird \x1bA escape",

View File

@@ -0,0 +1,62 @@
{
"name": "canonical_output_noise",
"steps": [
{
"type": "mcp_call",
"method": "spawn_process",
"params": {
"kind": "command",
"argv": [
"sh",
"-lc",
"printf '\\033[31mStatus: running 12s\\033[0m\\nStatus: running 13s\\n╭────╮\\n│ │\\nDownloading 10%%\\rDownloading 100%%\\nFINAL: deploy ready\\n'; sleep 5"
],
"name": "noisy"
},
"save_as": "proc"
},
{
"type": "wait_until_mcp",
"method": "get_process_output",
"params": {
"process_id": "{{proc.process_id}}",
"mode": "stream",
"raw": true,
"max_lines": 20
},
"path": "content",
"contains": "FINAL: deploy ready",
"timeout_ms": 5000,
"save_as": "raw"
},
{
"type": "assert_saved",
"from": "raw",
"path": "content",
"contains": "FINAL: deploy ready"
},
{
"type": "mcp_call",
"method": "get_process_output",
"params": {
"process_id": "{{proc.process_id}}",
"mode": "stream",
"since_offset": 0,
"max_lines": 20
},
"save_as": "canonical"
},
{
"type": "assert_saved",
"from": "canonical",
"path": "content",
"equals": "Status: running [time]\nDownloading [count]\nFINAL: deploy ready"
},
{
"type": "assert_saved",
"from": "canonical",
"path": "canonicalized",
"equals": true
}
]
}

View File

@@ -200,17 +200,20 @@ func toolCatalog(role CallerRole) []toolDescriptor {
},
{
Name: "get_process_output",
Description: "Read rendered grid (\"grid\") or ANSI-stripped stream (\"stream\") output, with screen-version watermark.",
Description: "Read canonical terminal text by default: visible grid (\"grid\") or recent stream (\"stream\") with ANSI/control noise, borders, duplicate status churn, and volatile timers removed. Set raw=true only for diagnostic ANSI-preserved PTY bytes.",
InputSchema: objectSchema(map[string]any{
"process_id": stringProp("Target process id."),
"mode": stringProp("\"grid\" (default) or \"stream\"."),
"since_offset": integerProp("Watermark offset from a previous call."),
"max_bytes": integerProp("Maximum content bytes to return."),
"max_lines": integerProp("Maximum canonical lines to return (default 120, max 500)."),
"raw": booleanProp("Return raw ANSI-preserved stream bytes instead of canonical text."),
"include_meta": booleanProp("Include verbose cursor, geometry, active screen, idle, and screen-version metadata."),
}, []string{"process_id"}),
},
{
Name: "get_process_raw_output",
Description: "Read the raw ANSI byte stream since since_offset.",
Description: "Compatibility alias for raw=true get_process_output: read the raw ANSI byte stream since since_offset.",
InputSchema: objectSchema(map[string]any{
"process_id": stringProp("Target process id."),
"since_offset": integerProp("Byte offset from a previous call."),

View File

@@ -167,8 +167,9 @@ type ProjectMeta struct {
Key string `json:"key"`
}
// ProcessOutput is the get_process_output payload. SPEC §7 enriches
// the old read_output result with screen geometry + version.
// ProcessOutput is the get_process_output payload. By default it is
// canonical text with light metadata; include_meta restores screen
// geometry + version, and raw requests return stream bytes.
type ProcessOutput struct {
Content string `json:"content"`
Mode string `json:"mode"`
@@ -176,13 +177,14 @@ type ProcessOutput struct {
ActiveScreen string `json:"active_screen,omitempty"`
Rows int `json:"rows,omitempty"`
Cols int `json:"cols,omitempty"`
Cursor Cursor `json:"cursor"`
Cursor *Cursor `json:"cursor,omitempty"`
IdleMS int64 `json:"idle_ms,omitempty"`
Status string `json:"status,omitempty"`
ScreenVersion int64 `json:"screen_version,omitempty"`
ContentBytes int `json:"content_bytes,omitempty"`
Truncated bool `json:"truncated,omitempty"`
TruncatedBytes int `json:"truncated_bytes,omitempty"`
Canonicalized bool `json:"canonicalized,omitempty"`
}
type ProcessOutputArgs struct {
@@ -190,6 +192,9 @@ type ProcessOutputArgs struct {
Mode string `json:"mode"`
SinceOffset int64 `json:"since_offset"`
MaxBytes int `json:"max_bytes"`
MaxLines int `json:"max_lines"`
Raw bool `json:"raw"`
IncludeMeta bool `json:"include_meta"`
}
// RawOutput is the get_process_raw_output payload — ANSI preserved.