Merge feat/playback: session playback, synced audio and thumbnails
# Conflicts: # README.md # crates/clipbin-desktop/Cargo.toml # crates/clipbin-desktop/src/main.rs # crates/clipbin-media/src/lib.rs # docs/validation.md
This commit is contained in:
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env bash
|
||||
# Playback, active browsing, real thumbnails and audio in the private desktop rig.
|
||||
set -euo pipefail
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
out="${1:-${repo}/target/playback-validation}"
|
||||
mkdir -p "${out}"
|
||||
out="$(cd "${out}" && pwd)"
|
||||
cargo build --locked --workspace --manifest-path "${repo}/Cargo.toml"
|
||||
cargo build --locked -p clipbin-media --example playback-check --manifest-path "${repo}/Cargo.toml"
|
||||
export CLIPBIN_PLAYBACK_REPO="${repo}" CLIPBIN_PLAYBACK_OUT="${out}"
|
||||
"${repo}/scripts/rig/headless.sh" bash <<'INNER'
|
||||
set -euo pipefail
|
||||
repo="${CLIPBIN_PLAYBACK_REPO}" out="${CLIPBIN_PLAYBACK_OUT}"
|
||||
export XDG_DATA_HOME="${out}/data" XDG_CONFIG_HOME="${out}/config" CLIPBIN_TRACE_FRAMES=1
|
||||
mkdir -p "${XDG_DATA_HOME}" "${XDG_CONFIG_HOME}"
|
||||
# ALSA's PipeWire plugin is cpal's fallback on Linux; there is no hardware card.
|
||||
cat > "${out}/alsa.conf" <<'ALSA'
|
||||
</usr/share/alsa/alsa.conf>
|
||||
pcm.!default { type pipewire }
|
||||
ctl.!default { type pipewire }
|
||||
ALSA
|
||||
export ALSA_CONFIG_PATH="${out}/alsa.conf"
|
||||
swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null
|
||||
send_key() {
|
||||
# Destroying rig-pointer's virtual seat can clear keyboard focus in Sway.
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
sleep .2
|
||||
timeout 10 wtype -s 200 -P "$1" -s 100 -p "$1"
|
||||
}
|
||||
# Timeline bounds come from GPUI's trace rather than guessed screen positions.
|
||||
pointer_time() {
|
||||
python3 - "$1" "$2" "$3" "${XDG_DATA_HOME}" <<'PYPOINTER'
|
||||
import pathlib,re,json,sys
|
||||
log=pathlib.Path(sys.argv[1]).read_text()
|
||||
x,y,w,h=map(float,re.findall(r'playback_timeline x=([\d.]+) y=([\d.]+) width=([\d.]+) height=([\d.]+)',log)[-1])
|
||||
p=max(pathlib.Path(sys.argv[4]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());length=m['session']['length'];length=length['secs']+length['nanos']/1e9
|
||||
if sys.argv[2] in ('gap','after-gap'):
|
||||
segment=next(s for s in m['segments'] if s['complete'] and not s['available'])
|
||||
def seconds(d):return d['secs']+d['nanos']/1e9
|
||||
at=(seconds(segment['start'])+seconds(segment['end']))/2 if sys.argv[2]=='gap' else seconds(segment['end'])+.5
|
||||
else:at=float(sys.argv[2])
|
||||
print(f"move {round(280+x+w*at/length)} {round(110+y+h/2)}\nsleep .15\n{sys.argv[3]}\nsleep .15")
|
||||
PYPOINTER
|
||||
}
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] floating enable' > /dev/null
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] resize set width 1360 px height 860 px' > /dev/null
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] move position 280 110' > /dev/null
|
||||
"${repo}/target/debug/clipbin-desktop" > "${out}/active.log" 2>&1 & app=$!
|
||||
trap 'kill "${app:-}" "${game:-}" "${audio_capture:-}" 2>/dev/null || true' EXIT
|
||||
sleep 3
|
||||
swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null
|
||||
bash -c 'exec -a reaper python3 -c "import time; time.sleep(120)" SteamLaunch AppId=480 --' & game=$!
|
||||
for _ in $(seq 30); do if grep -q recorder_status=Recording "${out}/active.log"; then break; fi; sleep 1; done
|
||||
grep -q recorder_status=Recording "${out}/active.log"
|
||||
sleep 8
|
||||
grim "${out}/sessions-thumbnails.png"
|
||||
# Open the first session card, seek near the start, and play while recording continues.
|
||||
printf 'move 670 330\nsleep .2\ndown\nsleep .1\nup\nsleep 1\n' | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
grim "${out}/editor-open.png"
|
||||
pointer_time "${out}/active.log" 1 click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
# GPUI/Wayland needs the virtual keyboard keymap to settle and a held key;
|
||||
# instantaneous wtype press/release occasionally disappears before dispatch.
|
||||
send_key space
|
||||
sleep 1
|
||||
grim "${out}/editor-playing-1.png"
|
||||
sleep 2
|
||||
grim "${out}/editor-playing-2.png"
|
||||
python3 - "${XDG_DATA_HOME}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());assert m['session']['active'];assert len([s for s in m['segments'] if s['complete'] and s['available']])>=4
|
||||
assert list(p.parent.rglob('*.thumbnail.png')), 'thumbnail worker produced no cached frame'
|
||||
print('PASS playback left recording active and generated real thumbnails')
|
||||
PY
|
||||
kill "${game}"; wait "${game}" 2>/dev/null || true
|
||||
sleep 3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl
|
||||
for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done
|
||||
if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi
|
||||
wait "${app}"
|
||||
kill "$(cat "${CLIPBIN_RIG_DIR}/game.pid")"
|
||||
# Simulate a cleaned-up middle segment in this isolated test library. Keep its
|
||||
# timeline placement: the player must show the gap rather than concatenate it away.
|
||||
python3 - "${XDG_DATA_HOME}" "${out}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());segments=[s for s in m['segments'] if s['complete'] and s['available']];assert len(segments)>=4
|
||||
s=segments[2];s['available']=False
|
||||
m['session']['spans']=[dict(start=m['session']['spans'][0]['start'],end=s['start'],kind='Footage'),dict(start=s['start'],end=s['end'],kind='Removed'),dict(start=s['end'],end=m['session']['length'],kind='Footage')]
|
||||
p.write_text(json.dumps(m));pathlib.Path(sys.argv[2]+'/session-id').write_text(str(m['session']['id']))
|
||||
PY
|
||||
id="$(cat "${out}/session-id")"
|
||||
# The CLI exercises the same player with audio only; recording tone has stopped.
|
||||
ffmpeg -v error -y -f pulse -i clipbin-rig-sink.monitor -t 10 "${out}/audio-output.wav" & audio_capture=$!
|
||||
python3 "${repo}/scripts/validate/measure-cpu.py" "${out}/player-cpu.txt" "${repo}/target/debug/examples/playback-check" "${XDG_DATA_HOME}/clipbin/sessions" "${id}" > "${out}/player.log" 2>&1
|
||||
wait "${audio_capture}"
|
||||
CLIPBIN_ROUTE="session:${id}" "${repo}/target/debug/clipbin-desktop" > "${out}/playback.log" 2>&1 & app=$!
|
||||
sleep 3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null
|
||||
# Drag through several files, then seek into the removed interval.
|
||||
{ pointer_time "${out}/playback.log" 1 down; pointer_time "${out}/playback.log" 3 "sleep 0.5"; pointer_time "${out}/playback.log" after-gap up; } | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep 1
|
||||
grim "${out}/editor-seek.png"
|
||||
pointer_time "${out}/playback.log" gap click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep 1
|
||||
grim "${out}/editor-gap.png"
|
||||
pointer_time "${out}/playback.log" 1 $'scroll -1\nsleep 0.5\nscroll 1' | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
pointer_time "${out}/playback.log" 2.5 click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep .3
|
||||
send_key Right
|
||||
sleep .3
|
||||
send_key Left
|
||||
sleep .3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl
|
||||
for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done
|
||||
if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi
|
||||
wait "${app}"
|
||||
python3 - "${out}" <<'PY'
|
||||
import pathlib,subprocess,re,sys,math,array,statistics,json
|
||||
out=pathlib.Path(sys.argv[1])
|
||||
def rgb(path):return subprocess.check_output(['ffmpeg','-v','error','-i',str(path),'-vf','crop=700:300:555:260','-f','rawvideo','-pix_fmt','rgb24','-'])
|
||||
a,b=map(rgb,[out/'editor-playing-1.png',out/'editor-playing-2.png'])
|
||||
changed=sum(x!=y for x,y in zip(a,b))/len(a);assert changed>.02, f'preview frozen: {changed}'
|
||||
raw=subprocess.check_output(['ffmpeg','-v','error','-i',str(out/'audio-output.wav'),'-ac','1','-ar','48000','-f','f32le','-'])
|
||||
samples=array.array('f',raw);rms=math.sqrt(sum(x*x for x in samples)/len(samples));assert rms>.001, f'no device audio: {rms}'
|
||||
def window_rms(start,end):
|
||||
block=samples[int(start*48000):int(end*48000)]
|
||||
return math.sqrt(sum(x*x for x in block)/max(1,len(block)))
|
||||
full=window_rms(2,3);half=window_rms(7.9,8.4);muted=window_rms(9.5,9.9)
|
||||
assert full>.001 and .35<half/full<.65 and muted<full*.02, f'volume/mute wrong: {full}, {half}, {muted}'
|
||||
log=(out/'player.log').read_text();assert 'state=gap' in log and 'state=frame' in log
|
||||
assert 'Cannot play audio' not in log and 'Audio output failed' not in log
|
||||
rows=re.findall(r'playback time=([\d.]+) decode_ms=([\d.]+).*presented=true',log)
|
||||
times=[float(row[1]) for row in rows];assert times
|
||||
print(f'PASS moving preview ({changed:.1%} pixels changed), device audio RMS={rms:.5f}, gaps and boundary playback')
|
||||
print(f'decode_ms median={statistics.median(times):.3f} p95={sorted(times)[int(len(times)*.95)]:.3f} max={max(times):.3f}')
|
||||
ui_log=(out/'playback.log').read_text()
|
||||
metadata=json.loads(next((out/'data/clipbin/sessions').glob('*/session.json')).read_text())
|
||||
def seconds(d):return d['secs']+d['nanos']/1e9
|
||||
removed=next(s for s in metadata['segments'] if s['complete'] and not s['available'])
|
||||
gap=(seconds(removed['start'])+seconds(removed['end']))/2
|
||||
gap_times=[float(t) for t in re.findall(r'playback time=([\d.]+).*state=gap',ui_log)]
|
||||
assert any(abs(t-gap)<.1 for t in gap_times), f'UI seek never reached gap at {gap}'
|
||||
frame_times=[(float(t),float(v)) for t,v in re.findall(r'playback time=([\d.]+).*state=frame.*video_time=([\d.]+)',ui_log)]
|
||||
assert any(abs(t-(seconds(removed['end'])+.5))<.1 and abs(t-v)<.020 for t,v in frame_times), 'UI boundary seek showed the wrong frame/time'
|
||||
# Right then Left must actually move the selected frame, while remaining paused.
|
||||
steps=[v for t,v in frame_times if abs(t-2.5)<.15]
|
||||
assert len(set(steps))>=2, f'frame stepping did not change the video frame: {steps}'
|
||||
uploads=[float(x) for x in re.findall(r'playback_ui upload_ms=([\d.]+)',(out/'active.log').read_text())]
|
||||
assert len(uploads)>30, 'too few moving frames reached GPUI'
|
||||
print(f'GPUI frame upload median={statistics.median(uploads):.3f}ms p95={sorted(uploads)[int(len(uploads)*.95)]:.3f}ms')
|
||||
(out/'report.txt').write_text(f'changed_preview_pixels={changed:.4f}\naudio_rms={rms:.5f}\ndecode_median_ms={statistics.median(times):.3f}\ndecode_p95_ms={sorted(times)[int(len(times)*.95)]:.3f}\n')
|
||||
PY
|
||||
echo "PASS playback validation; screenshots and timing in ${out}"
|
||||
INNER
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure child CPU/wall time without depending on GNU time."""
|
||||
import pathlib
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
before = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(sys.argv[2:], check=False)
|
||||
elapsed = time.monotonic() - started
|
||||
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
user = usage.ru_utime - before.ru_utime
|
||||
system = usage.ru_stime - before.ru_stime
|
||||
pathlib.Path(sys.argv[1]).write_text(
|
||||
f"cpu_user={user:.3f} cpu_system={system:.3f} wall={elapsed:.3f} "
|
||||
f"cpu_percent={(user + system) / elapsed * 100:.1f}%\n"
|
||||
)
|
||||
sys.exit(result.returncode)
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deterministic, clock-labelled 1080p60 segments with independent file timestamps.
|
||||
set -euo pipefail
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
out="${1:-${repo}/target/playback-fixture}"
|
||||
mkdir -p "${out}/1"
|
||||
ffmpeg -v error -y -f lavfi -i 'testsrc2=size=1920x1080:rate=60' \
|
||||
-f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 6 \
|
||||
-vf "drawtext=fontfile=/usr/share/fonts/TTF/DejaVuSans.ttf:text='%{pts\:hms}':fontsize=64:fontcolor=white:box=1:boxcolor=black:x=100:y=100" \
|
||||
-c:v libx264 -preset ultrafast -crf 28 -threads 2 -g 60 -keyint_min 60 -sc_threshold 0 -bf 0 \
|
||||
-c:a aac -f segment -segment_time 2 -reset_timestamps 1 "${out}/1/segment-%03d.mkv"
|
||||
ffmpeg -v error -y -i "${out}/1/segment-000.mkv" \
|
||||
-f lavfi -i 'sine=frequency=220:sample_rate=48000' \
|
||||
-f lavfi -i 'sine=frequency=880:sample_rate=48000' \
|
||||
-f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 2 \
|
||||
-map 0:v -map 1:a -map 2:a -map 3:a -c:v copy -c:a aac "${out}/combined.mkv"
|
||||
python3 - "${out}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
root=pathlib.Path(sys.argv[1]); d=lambda s:dict(secs=s,nanos=0)
|
||||
segments=[dict(path=f'segment-{i:03}.mkv',start=d(t),end=d(t+2),complete=True,available=True,frame_start=i*120,frames_per_second=60,dropped_frames=0) for i,t in enumerate([0,2,6])]
|
||||
session=dict(id=1,game='Playback fixture',started=dict(secs_since_epoch=1791453600,nanos_since_epoch=0),length=d(8),active=False,spans=[dict(start=d(a),end=d(b),kind=k) for a,b,k in [(0,4,'Footage'),(4,6,'Missing'),(6,8,'Footage')]],bookmarks=[])
|
||||
(root/'1/session.json').write_text(json.dumps(dict(session=session,app_id=480,segments=segments)))
|
||||
PY
|
||||
CLIPBIN_TEST_MEDIA_DIR="${out}" cargo test --locked -p clipbin-media --manifest-path "${repo}/Cargo.toml"
|
||||
Reference in New Issue
Block a user