diff --git a/Cargo.lock b/Cargo.lock index 19781a6..18ff6e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,28 @@ dependencies = [ "equator", ] +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -877,8 +899,10 @@ name = "clipbin-desktop" version = "0.1.0" dependencies = [ "clipbin-core", + "clipbin-media", "clipbin-obs", "gpui", + "image", ] [[package]] @@ -886,7 +910,11 @@ name = "clipbin-media" version = "0.1.0" dependencies = [ "clipbin-core", + "cpal", "ffmpeg-next", + "image", + "rustix 1.1.5", + "windows-sys 0.61.2", ] [[package]] @@ -983,6 +1011,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "command-fds" version = "0.3.3" @@ -1193,6 +1231,20 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cosmic-text" version = "0.14.2" @@ -1216,6 +1268,39 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1" +dependencies = [ + "alsa", + "block2", + "coreaudio-rs", + "dasp_sample", + "futures", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "portable-atomic", + "pulseaudio", + "web-sys", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1319,6 +1404,12 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.2" @@ -1528,6 +1619,17 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn 2.0.119", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -2250,7 +2352,7 @@ dependencies = [ "wayland-protocols-plasma", "windows 0.61.3", "windows-core 0.61.2", - "windows-numerics", + "windows-numerics 0.2.0", "windows-registry 0.5.3", "x11-clipboard", "x11rb", @@ -2917,6 +3019,64 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -3159,6 +3319,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3282,6 +3448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -3336,6 +3503,35 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3521,6 +3717,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc" version = "0.2.7" @@ -3564,6 +3782,55 @@ dependencies = [ "objc2-quartz-core", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -3571,7 +3838,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.2", + "block2", "dispatch2", + "libc", "objc2", ] @@ -3588,6 +3857,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.13.2", + "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -3907,6 +4178,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "postage" version = "0.5.0" @@ -4044,6 +4321,22 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" +[[package]] +name = "pulseaudio" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70623bd7967a9ca4c2ae0e807fc380b291f98480fc037042305ec643a4d3373" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-primitive-derive", + "futures", + "log", + "mio", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -4965,6 +5258,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simd_helpers" version = "0.1.0" @@ -6334,11 +6637,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -6351,7 +6666,7 @@ dependencies = [ "rayon", "thiserror 2.0.21", "windows 0.61.3", - "windows-future", + "windows-future 0.2.1", ] [[package]] @@ -6363,6 +6678,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -6388,6 +6712,19 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-future" version = "0.2.1" @@ -6396,7 +6733,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -6465,6 +6813,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-registry" version = "0.4.0" @@ -6505,6 +6863,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.3.1" @@ -6523,6 +6890,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -6616,6 +6992,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" diff --git a/README.md b/README.md index def506e..4f019e1 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The recording foundation is being validated (see [docs/validation.md](docs/valid The desktop app detects Steam games, owns the recorder on a background thread, persists settings and sessions, and loads recorded sessions from disk. Linux detects Steam’s `reaper` processes and watches their pidfds; Windows reads Steam’s registry state and watches native process handles (cross-checked, not run). Game exit finalises a session; closing the window keeps the current session recording until the game exits, then exits the app. Ctrl+Q or Quit stops the output cleanly. -The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Playback, export, global hotkeys, storage cleanup, autostart and a tray are not implemented yet. After closing the only window, reopening it without a tray is a follow-up. +The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Playback and frame seeking work across completed segments, including active sessions and missing footage. The editor has combined audio playback, mute/volume, Space/K to play/pause and J/L or arrows to step frames. Sessions and clips use cached frame thumbnails when footage is available. Export, global hotkeys, storage cleanup, autostart and a tray are not implemented yet. After closing the only window, reopening it without a tray is a follow-up. Windows is a first-class target. Every change must keep the Windows code paths building (`cargo clippy --target x86_64-pc-windows-msvc`) and list what still needs validating on Windows. @@ -19,6 +19,7 @@ Windows is a first-class target. Every change must keep the Windows code paths b ```text crates/ clipbin-core/ Settings, Steam detection and session storage; independent of GPUI and OBS. + clipbin-media/ Shared FFmpeg probing, scaled video/audio playback and cached thumbnails. clipbin-obs/ Recording engine over the pinned libobs runtime; all unsafe code is in src/ffi.rs. clipbin-probe/ Validation probe for capture, encoders, segments and status. clipbin-desktop/ GPUI application and initial session-library view. @@ -41,7 +42,7 @@ Keep the lockfile checked in and use `--locked`. It currently pins `libc` to 0.2 On Arch/CachyOS, install the native build dependencies: ```sh -sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang +sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang ffmpeg alsa-lib pipewire-alsa libpulse ``` ### OBS runtime @@ -62,9 +63,11 @@ On Debian/Ubuntu: ```sh sudo apt-get update -sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev +sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libasound2-dev libpulse-dev pipewire-alsa ``` +Media dynamically links FFmpeg through `ffmpeg-next` 9. Linux uses system FFmpeg 8. On Windows, set `FFMPEG_DIR` to an FFmpeg **7.1 shared development** package with MSVC import libraries, matching the OBS-bundled `avcodec-61`, `avformat-61`, `avutil-59`, `swscale-8` and `swresample-5` DLLs. Keep those OBS DLLs alongside the desktop executable; do not mix FFmpeg major versions. Audio uses cpal (PulseAudio or ALSA/PipeWire on Linux, WASAPI on Windows). + For Windows, use the MSVC Rust toolchain and Visual Studio Build Tools with the Desktop development with C++ workload. Windows support needs validation on a Windows machine; see [docs/validation.md](docs/validation.md#windows). Run the desktop shell from an active graphical session: @@ -87,7 +90,7 @@ cargo test --locked --workspace cargo clippy --locked --target x86_64-pc-windows-msvc -p clipbin-core -p clipbin-obs -p clipbin-probe --all-targets -- -D warnings ``` -Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times. +Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Playback fixtures: `scripts/validate/playback-fixture.sh`. Playback/audio e2e: `scripts/validate/linux-playback.sh [output-dir]` (moving preview, active recording, gaps, thumbnails, audio monitor, timing and CPU report). Fake Steam rigs must run serially because discovery reads the machine’s `/proc`. Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times. Development builds optimise dependencies (`[profile.dev.package."*"]`): unoptimised GPUI made screens noticeably slow, with first frames of 200–300 ms. diff --git a/crates/clipbin-desktop/Cargo.toml b/crates/clipbin-desktop/Cargo.toml index 44aabd0..cf3c508 100644 --- a/crates/clipbin-desktop/Cargo.toml +++ b/crates/clipbin-desktop/Cargo.toml @@ -10,6 +10,8 @@ publish.workspace = true [dependencies] clipbin-core.workspace = true +clipbin-media.workspace = true +image = { version = "0.25.10", default-features = false } clipbin-obs.workspace = true gpui.workspace = true diff --git a/crates/clipbin-desktop/src/app.rs b/crates/clipbin-desktop/src/app.rs index 7c977ca..8cc12fc 100644 --- a/crates/clipbin-desktop/src/app.rs +++ b/crates/clipbin-desktop/src/app.rs @@ -62,6 +62,7 @@ pub struct ClipbinApp { pub settings: Settings, pub recorder: Recorder, pub editor: Editor, + pub playback: crate::playback::Playback, /// A short message for actions that are not connected yet. pub notice: Option, pub drag: Option, @@ -201,7 +202,7 @@ impl ClipbinApp { let library = if sample { Library::sample(SystemTime::now()) } else { - Library::empty(settings.unprotected_session_limit_bytes) + service.snapshot().library }; let recorder = match (sample, std::env::var("CLIPBIN_SAMPLE_STATUS").as_deref()) { (_, Ok("failed")) => Recorder { @@ -234,6 +235,7 @@ impl ClipbinApp { selection: (Duration::ZERO, Duration::ZERO), playing: false, }, + playback: crate::playback::Playback::default(), notice: None, drag: None, timeline_bounds: Rc::default(), @@ -287,6 +289,7 @@ impl ClipbinApp { }) .detach(); } + Self::start_playback_updates(window, cx); app } @@ -298,7 +301,11 @@ impl ClipbinApp { self.notice = None; match route { Route::Session(id) => self.open_session(id), - route => self.route = route, + route => { + self.playback.player.close(); + self.editor.playing = false; + self.route = route; + } } cx.notify(); } @@ -322,6 +329,19 @@ impl ClipbinApp { selection, playing: false, }; + if let Ok(store) = clipbin_core::store::Store::discover() { + let preview_time = playhead.min( + session + .length + .saturating_sub(Duration::from_nanos(16_666_667)), + ); + self.editor.playhead = preview_time; + if session.bookmarks.is_empty() { + self.playback.player.open_latest(store, id); + } else { + self.playback.player.open(store, id, preview_time); + } + } self.route = Route::Session(id); } diff --git a/crates/clipbin-desktop/src/main.rs b/crates/clipbin-desktop/src/main.rs index a769075..ca4c51c 100644 --- a/crates/clipbin-desktop/src/main.rs +++ b/crates/clipbin-desktop/src/main.rs @@ -6,6 +6,7 @@ use gpui::{ mod app; mod assets; mod components; +mod playback; mod recorder_service; mod screens; mod text_field; @@ -47,6 +48,7 @@ fn main() { }); cx.bind_keys([KeyBinding::new("ctrl-q", Quit, None)]); text_field::bind_keys(cx); + playback::bind_keys(cx); let bounds = Bounds::centered(None, size(px(1360.), px(860.)), cx); let result = cx.open_window( @@ -65,10 +67,13 @@ fn main() { }, ); - if let Err(error) = result { - eprintln!("Could not open Clipbin: {error}"); - cx.quit(); - return; + match result { + Ok(window) => playback::bind_actions(window, cx), + Err(error) => { + eprintln!("Could not open Clipbin: {error}"); + cx.quit(); + return; + } } cx.activate(true); diff --git a/crates/clipbin-desktop/src/playback.rs b/crates/clipbin-desktop/src/playback.rs new file mode 100644 index 0000000..adecc2a --- /dev/null +++ b/crates/clipbin-desktop/src/playback.rs @@ -0,0 +1,279 @@ +//! GPUI image upload and controls; file I/O and decoding belong to the player. +use crate::app::{ClipbinApp, Route}; +use clipbin_core::store::Store; +use clipbin_media::{ + player::{Picture, Player}, + thumbnail::Thumbnails, +}; +use gpui::{Context, RenderImage}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; + +pub struct Playback { + pub player: Player, + pub image: Option>, + pub message: Option, + revision: u64, + last_frame: Option>, + pub muted: bool, + pub volume: f32, + pub audio_error: Option, + pub preview_height: f32, + pub thumbnails: HashMap, + pub clip_thumbnails: HashMap, +} +impl Default for Playback { + fn default() -> Self { + Self { + player: Player::new(), + image: None, + message: Some("Loading footage…".into()), + revision: 0, + last_frame: None, + muted: false, + volume: 1., + audio_error: None, + preview_height: 416., + thumbnails: HashMap::new(), + clip_thumbnails: HashMap::new(), + } + } +} +impl ClipbinApp { + pub fn seek_playback(&mut self, time: Duration) { + self.editor.playhead = time; + self.playback.player.seek(time); + } + pub fn toggle_playback(&mut self) { + if let Route::Session(id) = self.route { + if self + .library + .session(id) + .is_some_and(|session| self.editor.playhead >= session.length) + { + self.seek_playback(Duration::ZERO); + } + self.editor.playing = !self.editor.playing; + self.playback.player.play(self.editor.playing); + } + } + pub fn start_playback_updates(window: &gpui::Window, cx: &mut Context) { + cx.spawn_in(window, async |entity, cx| { + loop { + gpui::Timer::after(Duration::from_millis(16)).await; + if entity + .update_in(cx, |app, window, cx| { + if !matches!(app.route, Route::Session(_)) { + if let Some(image) = app.playback.image.take() { + let _ = window.drop_image(image); + } + app.playback.last_frame = None; + return; + } + let snapshot = app.playback.player.snapshot(); + if snapshot.revision == app.playback.revision { + return; + } + app.playback.revision = snapshot.revision; + app.editor.playhead = snapshot.time; + app.editor.playing = snapshot.playing; + app.playback.audio_error = snapshot.audio_error; + match snapshot.picture { + Picture::Frame(frame) => { + // Swscale already produced GPUI-native BGRA off the UI thread. + if app + .playback + .last_frame + .as_ref() + .is_none_or(|last| !Arc::ptr_eq(last, &frame)) + { + let upload_started = std::time::Instant::now(); + let bytes = frame.bgra.clone(); + if let Some(buffer) = + image::RgbaImage::from_raw(frame.width, frame.height, bytes) + { + let image = + Arc::new(RenderImage::new(vec![image::Frame::new( + buffer, + )])); + if let Some(old) = app.playback.image.replace(image) { + let _ = window.drop_image(old); + } + if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() { + eprintln!( + "playback_ui upload_ms={:.3} width={} height={}", + upload_started.elapsed().as_secs_f64() * 1000., + frame.width, + frame.height + ); + } + } + } + app.playback.last_frame = Some(frame); + app.playback.message = None; + } + Picture::Loading => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = Some("Loading footage…".into()); + } + Picture::Gap => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = + Some("No footage (not captured / cleaned up)".into()); + } + Picture::Error(error) => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = Some(error); + } + } + cx.notify(); + }) + .is_err() + { + break; + } + } + }) + .detach(); + // Discovery and thumbnail work never touch the UI thread's filesystem. + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let thumbnails = Thumbnails::new(); + loop { + let mut ready = HashMap::new(); + let mut clips = HashMap::new(); + if let Ok(store) = Store::discover() + && let Ok(metadata) = store.read() + { + for metadata in metadata { + if let Some(segment) = metadata + .segments + .iter() + .find(|segment| segment.complete && segment.available) + && let Some(path) = thumbnails + .request(&store.directory(metadata.session.id).join(&segment.path)) + { + ready.insert(metadata.session.id, path); + } + } + } + if let Ok(store) = Store::discover() + && let Some(root) = store.root.parent() + && let Ok(entries) = std::fs::read_dir(root.join("clips")) + { + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_some_and(|extension| extension == "mp4") + && let Some(id) = path + .file_stem() + .and_then(|id| id.to_str()) + .and_then(|id| id.parse().ok()) + && let Some(thumbnail) = thumbnails.request(&path) + { + clips.insert(id, thumbnail); + } + } + } + if matches!( + sender.try_send((ready, clips)), + Err(std::sync::mpsc::TrySendError::Disconnected(_)) + ) { + break; + } + std::thread::sleep(Duration::from_secs(1)); + } + }); + cx.spawn(async move |entity, cx| { + loop { + gpui::Timer::after(Duration::from_millis(250)).await; + let ready = receiver.try_recv().ok(); + if entity + .update(cx, |app, cx| { + if let Some((sessions, clips)) = ready + && (app.playback.thumbnails != sessions + || app.playback.clip_thumbnails != clips) + { + app.playback.thumbnails = sessions; + app.playback.clip_thumbnails = clips; + cx.notify(); + } + }) + .is_err() + { + break; + } + } + }) + .detach(); + } +} + +gpui::actions!(playback, [TogglePlayback, StepBack, StepForward]); +pub fn bind_keys(cx: &mut gpui::App) { + cx.bind_keys([ + gpui::KeyBinding::new("space", TogglePlayback, None), + gpui::KeyBinding::new("k", TogglePlayback, None), + gpui::KeyBinding::new("left", StepBack, None), + gpui::KeyBinding::new("j", StepBack, None), + gpui::KeyBinding::new("right", StepForward, None), + gpui::KeyBinding::new("l", StepForward, None), + ]); + cx.bind_keys( + ["space", "k", "j", "l"] + .map(|key| gpui::KeyBinding::new(key, gpui::NoAction {}, Some("TextField"))), + ); +} +/// Window actions also work when a mouse click leaves no focused GPUI element. +/// Text-field contexts keep their own typing and arrow bindings. +pub fn bind_actions(window: gpui::WindowHandle, cx: &mut gpui::App) { + // The event dispatcher already borrows the window; update its entity directly. + let editor = window.entity(cx).expect("editor window").downgrade(); + let toggle = editor.clone(); + cx.on_action(move |_: &TogglePlayback, cx| { + let _ = toggle.update(cx, |app, cx| { + app.toggle_playback(); + cx.notify(); + }); + }); + let back = editor.clone(); + cx.on_action(move |_: &StepBack, cx| { + let _ = back.update(cx, |app, cx| { + app.step_playback(false); + cx.notify(); + }); + }); + cx.on_action(move |_: &StepForward, cx| { + let _ = editor.update(cx, |app, cx| { + app.step_playback(true); + cx.notify(); + }); + }); +} + +impl ClipbinApp { + pub fn step_playback(&mut self, forward: bool) { + if !matches!(self.route, Route::Session(_)) { + return; + } + self.editor.playing = false; + self.playback.player.play(false); + let snapshot = self.playback.player.snapshot(); + let step = Duration::from_secs_f64(1. / snapshot.frames_per_second.max(1) as f64); + let time = snapshot.video_time.unwrap_or(self.editor.playhead); + // Round towards the selected frame when nanoseconds cannot represent a frame period. + let at = if forward { + time + step + } else { + time.saturating_sub(step) + } + Duration::from_nanos(2); + self.seek_playback(at); + } +} diff --git a/crates/clipbin-desktop/src/screens/clips.rs b/crates/clipbin-desktop/src/screens/clips.rs index c9e4dc6..42b511f 100644 --- a/crates/clipbin-desktop/src/screens/clips.rs +++ b/crates/clipbin-desktop/src/screens/clips.rs @@ -61,7 +61,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context) -> AnyElement { .flex_col() .gap_3() .child( - thumbnail(&clip.game).child( + thumbnail(&clip.game, app.playback.clip_thumbnails.get(&clip.id)).child( corner() .bottom_2p5() .right_2p5() diff --git a/crates/clipbin-desktop/src/screens/editor.rs b/crates/clipbin-desktop/src/screens/editor.rs index 8749f59..02c1b2a 100644 --- a/crates/clipbin-desktop/src/screens/editor.rs +++ b/crates/clipbin-desktop/src/screens/editor.rs @@ -6,15 +6,14 @@ use clipbin_core::library::{ Session, SpanKind, export_range, format_ago, format_clock, format_length, }; use gpui::{ - AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, IntoElement, MouseButton, - MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, ScrollWheelEvent, canvas, div, - pattern_slash, prelude::*, px, relative, + AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, ImageSource, IntoElement, + MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ObjectFit, Pixels, Point, + ScrollWheelEvent, canvas, div, img, pattern_slash, prelude::*, px, relative, }; use crate::{ app::{ClipbinApp, Drag, Route}, components::{ButtonStyle, aspect_ratio, button, chip, icon, panel, row}, - screens::backdrop, theme, timeline_view::TimelineView, }; @@ -73,37 +72,54 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ); let preview = aspect_ratio(div(), 16. / 9.) + .h(px(app.playback.preview_height)) + .flex_shrink_0() .relative() .w_full() .overflow_hidden() .rounded_lg() .bg(gpui::black()) - .child(backdrop(&session.game).absolute().size_full().opacity(0.55)) - .child( + .children(app.playback.image.clone().map(|image| { + img(ImageSource::Render(image)) + .absolute() + .size_full() + .object_fit(ObjectFit::Contain) + })) + .children(app.playback.message.clone().map(|message| { div() .absolute() .size_full() .flex() - .flex_col() .items_center() .justify_center() - .gap_3() - .child( - div() - .flex() - .items_center() - .justify_center() - .size(px(56.)) - .rounded_full() - .bg(theme::overlay()) - .child(icon("play").size_6().text_color(theme::foreground())), - ) - .child( - div() - .text_xs() - .text_color(theme::muted()) - .child("Preview arrives with playback"), - ), + .text_xs() + .text_color(theme::muted()) + .child(message) + })) + .child( + canvas( + { + let player = cx.entity().downgrade(); + move |bounds, window, cx| { + let scale = window.scale_factor(); + let size = ( + (f32::from(bounds.size.width) * scale) as u32, + (f32::from(bounds.size.height) * scale) as u32, + ); + let _ = player.update(cx, |app, cx| { + app.playback.player.resize(size); + let height = f32::from(bounds.size.width) * 9. / 16.; + if (app.playback.preview_height - height).abs() > 1. { + app.playback.preview_height = height; + cx.notify(); + } + }); + } + }, + |_, _, _, _| {}, + ) + .absolute() + .size_full(), ) .child( div() @@ -126,6 +142,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .map(|b| b.at); let transport = div() .flex() + .flex_wrap() .items_center() .gap_1() .child( @@ -136,8 +153,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle None, ) .on_click(cx.listener(|app, _: &ClickEvent, _, cx| { - app.notice = - Some("Playback isn't connected yet; it comes with the decoder.".into()); + app.toggle_playback(); cx.notify(); })), ) @@ -150,7 +166,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { if let Some(at) = previous_bookmark { - app.editor.playhead = at; + app.seek_playback(at); cx.notify(); } })), @@ -164,7 +180,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { if let Some(at) = next_bookmark { - app.editor.playhead = at; + app.seek_playback(at); cx.notify(); } })), @@ -187,6 +203,52 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .child(format!("/ {}", format_clock(length))), ), ) + .child( + button( + "mute", + ButtonStyle::Ghost, + Some(if app.playback.muted { + "volume-x" + } else { + "volume-2" + }), + None, + ) + .on_click(cx.listener(|app, _: &ClickEvent, _, cx| { + app.playback.muted = !app.playback.muted; + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + })), + ) + .child( + button("volume-down", ButtonStyle::Ghost, None, Some("−")).on_click(cx.listener( + |app, _: &ClickEvent, _, cx| { + app.playback.volume = (app.playback.volume - 0.1).max(0.); + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + }, + )), + ) + .child( + div() + .text_xs() + .child(format!("{:.0}%", app.playback.volume * 100.)), + ) + .child( + button("volume-up", ButtonStyle::Ghost, None, Some("+")).on_click(cx.listener( + |app, _: &ClickEvent, _, cx| { + app.playback.volume = (app.playback.volume + 0.1).min(1.); + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + }, + )), + ) .child(div().flex_1()) .child( div() @@ -210,6 +272,12 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .gap_3() .child(preview) .child(transport) + .children( + app.playback + .audio_error + .clone() + .map(|error| div().text_xs().text_color(theme::warning()).child(error)), + ) .child(timeline(app, session, cx)); let (export_start, export_end) = export_range(start, end, KEYFRAME_INTERVAL, length); @@ -320,7 +388,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle }), ) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { - app.editor.playhead = at; + app.seek_playback(at); app.editor.selection = range; cx.notify(); })) @@ -529,9 +597,25 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - .size_full() .overflow_hidden() .child( - canvas(move |area, _, _| bounds.set(Some(area)), |_, _, _, _| {}) - .absolute() - .size_full(), + canvas( + move |area, _, _| { + if bounds.get() != Some(area) + && std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() + { + eprintln!( + "playback_timeline x={} y={} width={} height={}", + f32::from(area.left()), + f32::from(area.top()), + f32::from(area.size.width), + f32::from(area.size.height) + ); + } + bounds.set(Some(area)); + }, + |_, _, _, _| {}, + ) + .absolute() + .size_full(), ) .child( div() @@ -565,7 +649,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - } else if near(end) { Drag::SelectionEnd } else { - app.editor.playhead = time; + app.seek_playback(time); Drag::Playhead }); cx.notify(); @@ -583,7 +667,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - let (start, end) = app.editor.selection; let minimum = Duration::from_secs(1); match drag { - Drag::Playhead => app.editor.playhead = time, + Drag::Playhead => app.seek_playback(time), Drag::SelectionStart => { app.editor.selection.0 = time.min(end.saturating_sub(minimum)) } diff --git a/crates/clipbin-desktop/src/screens/mod.rs b/crates/clipbin-desktop/src/screens/mod.rs index f3f1700..7faafb2 100644 --- a/crates/clipbin-desktop/src/screens/mod.rs +++ b/crates/clipbin-desktop/src/screens/mod.rs @@ -9,7 +9,20 @@ use crate::{components::aspect_ratio, theme}; /// A 16:9 thumbnail. Until frames are decoded, each game gets its own colour /// with its initials, so cards stay recognisable. -pub fn thumbnail(game: &str) -> Div { +pub fn thumbnail(game: &str, path: Option<&std::path::PathBuf>) -> Div { + if let Some(path) = path { + return aspect_ratio(div(), 16. / 9.) + .relative() + .w_full() + .overflow_hidden() + .rounded_xl() + .child( + gpui::img(path.clone()) + .absolute() + .size_full() + .object_fit(gpui::ObjectFit::Cover), + ); + } let tint = game_tint(game); let initials: String = game .split_whitespace() diff --git a/crates/clipbin-desktop/src/screens/sessions.rs b/crates/clipbin-desktop/src/screens/sessions.rs index 46343d2..7932d1e 100644 --- a/crates/clipbin-desktop/src/screens/sessions.rs +++ b/crates/clipbin-desktop/src/screens/sessions.rs @@ -61,7 +61,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context) -> AnyElement { .cursor_pointer() .group("card") .child( - thumbnail(&session.game) + thumbnail(&session.game, app.playback.thumbnails.get(&session.id)) .border_1() .border_color(gpui::transparent_black()) .group_hover("card", |style| { diff --git a/crates/clipbin-media/Cargo.toml b/crates/clipbin-media/Cargo.toml index 12934c9..a23ed24 100644 --- a/crates/clipbin-media/Cargo.toml +++ b/crates/clipbin-media/Cargo.toml @@ -11,6 +11,14 @@ publish.workspace = true [dependencies] clipbin-core.workspace = true ffmpeg-next = { version = "9", default-features = false, features = ["codec", "format", "software-scaling", "software-resampling"] } +image = { version = "0.25.10", default-features = false, features = ["png"] } +cpal = { version = "0.18.2", features = ["pulseaudio"] } -[lints] -workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { version = "1", features = ["process", "thread"] } + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_System_Threading"] } + +[lints.rust] +unsafe_code = "deny" diff --git a/crates/clipbin-media/examples/playback-check.rs b/crates/clipbin-media/examples/playback-check.rs new file mode 100644 index 0000000..fe7c960 --- /dev/null +++ b/crates/clipbin-media/examples/playback-check.rs @@ -0,0 +1,67 @@ +//! Headless timing check against real segments; run inside the rig for device audio. +use clipbin_core::store::Store; +use clipbin_media::player::{Picture, Player}; +use std::{ + thread, + time::{Duration, Instant}, +}; +fn main() { + let directory = std::env::args().nth(1).expect("session root"); + let id = std::env::args() + .nth(2) + .unwrap_or_else(|| "1".into()) + .parse() + .unwrap(); + let player = Player::new(); + player.resize((800, 450)); + player.open( + Store { + root: directory.into(), + }, + id, + Duration::ZERO, + ); + thread::sleep(Duration::from_millis(250)); + player.play(true); + let begin = Instant::now(); + let mut last = None; + let mut frames = 0; + for _ in 0..700 { + thread::sleep(Duration::from_millis(10)); + let snapshot = player.snapshot(); + assert!(snapshot.audio_error.is_none(), "{:?}", snapshot.audio_error); + if let Picture::Frame(frame) = snapshot.picture { + if let Some(video_time) = snapshot.video_time { + assert!( + video_time.abs_diff(snapshot.time) <= Duration::from_millis(20), + "video out of sync: {video_time:?} vs {:?}", + snapshot.time + ); + } + if last + .as_ref() + .is_none_or(|previous| !std::sync::Arc::ptr_eq(previous, &frame)) + { + frames += 1; + } + last = Some(frame); + } + } + let snapshot = player.snapshot(); + println!( + "playback_check wall={:.3} session={:.3} displayed_frames={frames}", + begin.elapsed().as_secs_f64(), + snapshot.time.as_secs_f64() + ); + assert!( + snapshot.time >= Duration::from_secs(6), + "audio master clock stalled: {snapshot:?}" + ); + player.audio(false, 0.5); + thread::sleep(Duration::from_secs(1)); + player.audio(true, 0.5); + let before = snapshot.time; + thread::sleep(Duration::from_secs(1)); + assert!(player.snapshot().time > before, "muted wall clock stalled"); + player.close(); +} diff --git a/crates/clipbin-media/src/audio.rs b/crates/clipbin-media/src/audio.rs new file mode 100644 index 0000000..9580e90 --- /dev/null +++ b/crates/clipbin-media/src/audio.rs @@ -0,0 +1,454 @@ +//! Combined-track decoding and bounded device output. No decoder runs in a callback. +use crate::ffmpeg::{self, format, media, software::resampling, util::frame::Audio as AudioFrame}; +use clipbin_core::store::Metadata; +use cpal::{ + FromSample, SizedSample, + traits::{DeviceTrait, HostTrait, StreamTrait}, +}; +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +struct Block { + time: Duration, + samples: Vec, + consumed: usize, +} +struct Buffer { + blocks: VecDeque, + queued: usize, + playing: bool, + volume: f32, + // The time of the first sample actually scheduled for the device. + anchors: VecDeque<(Duration, Instant, Duration)>, + error: Option, +} +impl Buffer { + fn time(&mut self, now: Instant) -> Option { + while self.anchors.len() > 1 && self.anchors[1].1 <= now { + self.anchors.pop_front(); + } + self.anchors.front().and_then(|(time, begins, span)| { + (now <= *begins + *span + Duration::from_millis(100)) + .then(|| *time + now.saturating_duration_since(*begins).min(*span)) + }) + } +} +pub struct Audio { + _stream: cpal::Stream, + buffer: Arc>, + rate: u32, + channels: u16, + decoder: Option<(PathBuf, AudioDecoder)>, + next: Duration, +} +impl Audio { + pub fn new() -> Result { + #[cfg(target_os = "linux")] + if let Ok(host) = cpal::host_from_id(cpal::HostId::PulseAudio) { + match Self::with_host(host) { + Ok(audio) => return Ok(audio), + Err(error) => eprintln!("PulseAudio output unavailable: {error}; trying ALSA"), + } + } + Self::with_host(cpal::default_host()) + } + fn with_host(host: cpal::Host) -> Result { + let device = host + .default_output_device() + .ok_or("No audio output device")?; + let supported = device.default_output_config().map_err(|e| e.to_string())?; + let format = supported.sample_format(); + let config: cpal::StreamConfig = supported.into(); + let rate = config.sample_rate; + let channels = config.channels; + let buffer = Arc::new(Mutex::new(Buffer { + blocks: VecDeque::new(), + queued: 0, + playing: false, + volume: 1., + anchors: VecDeque::new(), + error: None, + })); + let stream = match format { + cpal::SampleFormat::F32 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::I16 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::U16 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::I32 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::F64 => output::(&device, config, buffer.clone()), + _ => return Err(format!("Unsupported audio device format: {format}")), + } + .map_err(|e| e.to_string())?; + stream.play().map_err(|e| e.to_string())?; + eprintln!("playback_audio rate={rate} channels={channels} format={format}"); + Ok(Self { + _stream: stream, + buffer, + rate, + channels, + decoder: None, + next: Duration::ZERO, + }) + } + pub fn reset(&mut self, time: Duration) { + let mut buffer = self.buffer.lock().unwrap(); + buffer.blocks.clear(); + buffer.queued = 0; + buffer.anchors.clear(); + self.decoder = None; + self.next = time; + } + pub fn controls(&self, playing: bool, volume: f32) { + let mut buffer = self.buffer.lock().unwrap(); + buffer.playing = playing; + buffer.volume = volume.clamp(0., 1.); + if !playing { + buffer.anchors.clear(); + } + } + /// Device time, corrected for callback-to-DAC latency; bounded by scheduled samples. + pub fn time(&self, now: Instant) -> Option { + self.buffer.lock().unwrap().time(now) + } + pub fn error(&self) -> Option { + self.buffer.lock().unwrap().error.clone() + } + pub fn catch_up(&mut self, time: Duration) { + if self.next + Duration::from_millis(50) < time && self.buffer.lock().unwrap().queued == 0 { + self.reset(time); + } + } + pub fn fill(&mut self, metadata: &Metadata) -> Result<(), ffmpeg::Error> { + // At most 250 ms buffered. All allocations/decoding happen on the player thread. + let limit = self.rate as usize * self.channels as usize / 4; + for _ in 0..16 { + if self.buffer.lock().unwrap().queued >= limit { + break; + } + let Some(at) = metadata.locate(self.next).filter(|at| at.segment.complete) else { + break; + }; + if self + .decoder + .as_ref() + .is_none_or(|(path, _)| path != &at.segment.path) + { + self.decoder = Some(( + at.segment.path.clone(), + AudioDecoder::open(&at.segment.path, self.rate, self.channels, at.offset)?, + )); + } + let (_, decoder) = self.decoder.as_mut().unwrap(); + let Some((offset, mut samples)) = decoder.next()? else { + self.next = at.segment.end; + self.decoder = None; + continue; + }; + let begin = at.segment.start + offset; + let skip = ((self.next.saturating_sub(begin).as_secs_f64() * self.rate as f64).round() + as usize) + .min(samples.len() / self.channels as usize); + samples.drain(..skip * self.channels as usize); + let time = begin + Duration::from_secs_f64(skip as f64 / self.rate as f64); + let keep = ((at.segment.end.saturating_sub(time).as_secs_f64() * self.rate as f64) + .round() as usize) + * self.channels as usize; + samples.truncate(keep); + if samples.is_empty() { + continue; + } + self.next = time + + Duration::from_secs_f64( + (samples.len() / self.channels as usize) as f64 / self.rate as f64, + ); + if at.segment.end.saturating_sub(self.next) + < Duration::from_nanos(1_000_000_000 / self.rate as u64) + { + self.next = at.segment.end; + } + let mut buffer = self.buffer.lock().unwrap(); + buffer.queued += samples.len(); + buffer.blocks.push_back(Block { + time, + samples, + consumed: 0, + }); + } + Ok(()) + } +} +fn output>( + device: &cpal::Device, + config: cpal::StreamConfig, + buffer: Arc>, +) -> Result { + let errors = buffer.clone(); + let rate = config.sample_rate; + let channels = config.channels as usize; + device.build_output_stream( + config, + move |data: &mut [T], info: &cpal::OutputCallbackInfo| { + // try_lock guarantees UI/decoder contention cannot block the device callback. + data.fill(T::from_sample(0.)); + let Ok(mut buffer) = buffer.try_lock() else { + return; + }; + if !buffer.playing { + return; + } + let latency = info + .timestamp() + .playback + .duration_since(info.timestamp().callback); + let mut scheduled = None; + for (frame_number, output) in data.chunks_exact_mut(channels).enumerate() { + let volume = buffer.volume; + let Some(block) = buffer.blocks.front_mut() else { + break; + }; + let time = block.time + + Duration::from_secs_f64((block.consumed / channels) as f64 / rate as f64); + scheduled.get_or_insert(( + time, + Instant::now() + + latency + + Duration::from_secs_f64(frame_number as f64 / rate as f64), + Duration::ZERO, + )); + for sample in output { + *sample = + T::from_sample((block.samples[block.consumed] * volume).clamp(-1., 1.)); + block.consumed += 1; + } + let done = block.consumed == block.samples.len(); + buffer.queued -= channels; + if let Some((_, _, span)) = scheduled.as_mut() { + *span += Duration::from_secs_f64(1. / rate as f64); + } + if done { + buffer.blocks.pop_front(); + } + } + if let Some(scheduled) = scheduled { + if buffer.anchors.len() == 32 { + buffer.anchors.pop_front(); + } + buffer.anchors.push_back(scheduled); + } + }, + move |error| { + if let Ok(mut buffer) = errors.lock() { + buffer.error = Some(format!("Audio output failed: {error}")); + } + }, + None, + ) +} + +struct AudioDecoder { + input: format::context::Input, + decoder: ffmpeg::decoder::Audio, + resampler: resampling::Context, + stream: usize, + time_base: f64, + origin: i64, + rate: u32, + channels: u16, + eof: bool, + next_time: Duration, + initialised_time: bool, + drained: bool, +} +impl AudioDecoder { + fn open(path: &Path, rate: u32, channels: u16, seek: Duration) -> Result { + crate::init(); + let mut input = format::input(path)?; + // OBS writes system, mic, combined, in that order; a single track is already mixed. + let tracks: Vec<_> = input + .streams() + .filter(|stream| stream.parameters().medium() == media::Type::Audio) + .collect(); + let track = if tracks.len() >= 3 { + &tracks[2] + } else { + tracks.first().ok_or(ffmpeg::Error::StreamNotFound)? + }; + let stream = track.index(); + let time_base = f64::from(track.time_base()); + let origin = track.start_time().max(0); + let decoder = ffmpeg::codec::context::Context::from_parameters(track.parameters())? + .decoder() + .audio()?; + let layout = if decoder.channel_layout().channels() == 0 { + ffmpeg::ChannelLayout::default(decoder.channels() as i32) + } else { + decoder.channel_layout() + }; + let resampler = resampling::Context::get( + decoder.format(), + layout, + decoder.rate(), + format::Sample::F32(format::sample::Type::Packed), + ffmpeg::ChannelLayout::default(channels as i32), + rate, + )?; + if seek > Duration::ZERO { + let timestamp = (seek.as_secs_f64() * 1_000_000.) as i64; + input.seek(timestamp, ..timestamp)?; + } + Ok(Self { + input, + decoder, + resampler, + stream, + time_base, + origin, + rate, + channels, + eof: false, + next_time: seek, + initialised_time: false, + drained: false, + }) + } + fn next(&mut self) -> Result)>, ffmpeg::Error> { + loop { + let mut frame = AudioFrame::empty(); + match self.decoder.receive_frame(&mut frame) { + Ok(()) => { + let timestamp = frame.timestamp().map_or(self.next_time, |pts| { + Duration::from_secs_f64((pts - self.origin).max(0) as f64 * self.time_base) + }); + let time = if self.initialised_time { + self.next_time + } else { + timestamp + }; + self.initialised_time = true; + // Allocate enough output for upsampling; ffmpeg-next's empty output uses input count. + let count = ((frame.samples() as u64 * self.rate as u64) + .div_ceil(frame.rate() as u64) + + 256) as usize; + let mut converted = AudioFrame::new( + format::Sample::F32(format::sample::Type::Packed), + count, + ffmpeg::ChannelLayout::default(self.channels as i32), + ); + converted.set_rate(self.rate); + self.resampler.run(&frame, &mut converted)?; + let samples = converted.plane::(0).to_vec(); + self.next_time = time + + Duration::from_secs_f64(converted.samples() as f64 / self.rate as f64); + return Ok(Some((time, samples))); + } + Err(ffmpeg::Error::Eof) => { + if self.drained { + return Ok(None); + } + self.drained = true; + let mut converted = AudioFrame::new( + format::Sample::F32(format::sample::Type::Packed), + 256, + ffmpeg::ChannelLayout::default(self.channels as i32), + ); + converted.set_rate(self.rate); + self.resampler.flush(&mut converted)?; + return if converted.samples() == 0 { + Ok(None) + } else { + Ok(Some((self.next_time, converted.plane::(0).to_vec()))) + }; + } + Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {} + Err(error) => return Err(error), + } + if self.eof { + return Ok(None); + } + let mut packet = ffmpeg::Packet::empty(); + loop { + match packet.read(&mut self.input) { + Ok(()) if packet.stream() == self.stream => { + self.decoder.send_packet(&packet)?; + break; + } + Ok(()) => {} + Err(ffmpeg::Error::Eof) => { + self.eof = true; + self.decoder.send_eof()?; + break; + } + Err(error) => return Err(error), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn device_clock_waits_for_latency_and_stops_at_scheduled_samples() { + let now = Instant::now(); + let mut buffer = Buffer { + blocks: VecDeque::new(), + queued: 0, + playing: true, + volume: 1., + anchors: VecDeque::new(), + error: None, + }; + buffer.anchors.push_back(( + Duration::from_secs(2), + now + Duration::from_millis(100), + Duration::from_millis(50), + )); + buffer.anchors.push_back(( + Duration::from_millis(2050), + now + Duration::from_millis(150), + Duration::from_millis(50), + )); + assert_eq!(buffer.time(now), Some(Duration::from_secs(2))); + assert_eq!( + buffer.time(now + Duration::from_millis(125)), + Some(Duration::from_millis(2025)) + ); + assert_eq!( + buffer.time(now + Duration::from_millis(200)), + Some(Duration::from_millis(2100)) + ); + assert_eq!(buffer.anchors.len(), 1); + assert_eq!(buffer.time(now + Duration::from_secs(5)), None); + } + #[test] + fn resampling_selects_combined_track_and_preserves_its_tone() { + let Some(directory) = std::env::var_os("CLIPBIN_TEST_MEDIA_DIR") else { + return; + }; + let mut decoder = AudioDecoder::open( + &PathBuf::from(directory).join("combined.mkv"), + 44_100, + 1, + Duration::ZERO, + ) + .unwrap(); + let mut samples = Vec::new(); + while let Some((_, block)) = decoder.next().unwrap() { + samples.extend(block); + } + assert!(samples.len() > 80_000 && samples.len() < 95_000); + let crossings = samples + .windows(2) + .filter(|pair| pair[0] <= 0. && pair[1] > 0.) + .count(); + let hz = crossings as f64 / (samples.len() as f64 / 44_100.); + assert!( + (435. ..445.).contains(&hz), + "combined track was not the 440 Hz tone: {hz}" + ); + } +} diff --git a/crates/clipbin-media/src/decode.rs b/crates/clipbin-media/src/decode.rs new file mode 100644 index 0000000..bab48a2 --- /dev/null +++ b/crates/clipbin-media/src/decode.rs @@ -0,0 +1,203 @@ +//! Single-threaded file decoding. Scale only frames which will be displayed. +use crate::ffmpeg::{self, codec, format, media, software::scaling, util::frame::video::Video}; +use std::{path::Path, time::Duration}; + +#[derive(Clone, Debug)] +pub struct Frame { + pub width: u32, + pub height: u32, + /// Packed BGRA, without FFmpeg's row padding. + pub bgra: Vec, +} + +pub struct Decoder { + input: format::context::Input, + decoder: ffmpeg::decoder::Video, + stream: usize, + time_base: f64, + origin: i64, + fps: u32, + next_index: u64, + eof: bool, + scaler: Option<(u32, u32, format::Pixel, u32, u32, scaling::context::Context)>, +} + +impl Decoder { + pub fn open(path: &Path, fps: u32) -> Result { + crate::init(); + let input = format::input(path)?; + let video = input + .streams() + .best(media::Type::Video) + .ok_or(ffmpeg::Error::StreamNotFound)?; + let stream = video.index(); + let time_base = f64::from(video.time_base()); + let origin = video.start_time().max(0); + let mut context = codec::context::Context::from_parameters(video.parameters())?; + // Never let FFmpeg create a pool competing with the recorder. + context.set_threading(codec::threading::Config::count(1)); + let decoder = context.decoder().video()?; + Ok(Self { + input, + decoder, + stream, + time_base, + origin, + fps: fps.max(1), + next_index: 0, + eof: false, + scaler: None, + }) + } + + /// Reuse swscale's setup while the preview dimensions stay unchanged. + pub fn scale(&mut self, video: &Video, target: (u32, u32)) -> Result { + let (width, height) = fitted_size((video.width(), video.height()), target); + if self + .scaler + .as_ref() + .is_none_or(|(w, h, pixel, source_w, source_h, _)| { + (*w, *h) != (width, height) + || *pixel != video.format() + || (*source_w, *source_h) != (video.width(), video.height()) + }) + { + self.scaler = Some(( + width, + height, + video.format(), + video.width(), + video.height(), + scaling::context::Context::get( + video.format(), + video.width(), + video.height(), + format::Pixel::BGRA, + width, + height, + scaling::flag::Flags::FAST_BILINEAR, + )?, + )); + } + let (_, _, _, _, _, scaler) = self.scaler.as_mut().unwrap(); + packed_frame(scaler, video, width, height) + } + + /// FFmpeg seeks backwards to a keyframe; callers decode forward by frame index. + pub fn seek(&mut self, time: Duration) -> Result<(), ffmpeg::Error> { + let timestamp = (time.as_secs_f64() * 1_000_000. + + self.origin as f64 * self.time_base * 1_000_000.) as i64; + self.input.seek(timestamp, ..timestamp)?; + self.decoder.flush(); + self.next_index = frame_index(time, self.fps); + self.eof = false; + Ok(()) + } + + pub fn next_frame(&mut self) -> Result, ffmpeg::Error> { + loop { + let mut frame = Video::empty(); + match self.decoder.receive_frame(&mut frame) { + Ok(()) => { + let index = frame.timestamp().map_or(self.next_index, |pts| { + (((pts - self.origin).max(0) as f64 * self.time_base * self.fps as f64) + .round()) as u64 + }); + self.next_index = index + 1; + return Ok(Some((index, frame))); + } + Err(ffmpeg::Error::Eof) => return Ok(None), + Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {} + Err(error) => return Err(error), + } + if self.eof { + return Ok(None); + } + let mut packet = ffmpeg::Packet::empty(); + loop { + match packet.read(&mut self.input) { + Ok(()) if packet.stream() == self.stream => { + self.decoder.send_packet(&packet)?; + break; + } + Ok(()) => {} + Err(ffmpeg::Error::Eof) => { + self.eof = true; + self.decoder.send_eof()?; + break; + } + Err(error) => return Err(error), + } + } + } + } +} + +pub fn frame_index(time: Duration, fps: u32) -> u64 { + (time.as_nanos() * u128::from(fps.max(1)) / 1_000_000_000) as u64 +} + +/// Preserve aspect ratio and cap allocation even on a very large display. +pub fn fitted_size(source: (u32, u32), target: (u32, u32)) -> (u32, u32) { + let ratio = (target.0.clamp(1, 1920) as f64 / source.0.max(1) as f64) + .min(target.1.clamp(1, 1080) as f64 / source.1.max(1) as f64) + .min(1.); + ( + (source.0 as f64 * ratio).round().max(1.) as u32, + (source.1 as f64 * ratio).round().max(1.) as u32, + ) +} + +pub fn scale(video: &Video, target: (u32, u32)) -> Result { + let (width, height) = fitted_size((video.width(), video.height()), target); + let mut scaler = scaling::context::Context::get( + video.format(), + video.width(), + video.height(), + format::Pixel::BGRA, + width, + height, + scaling::flag::Flags::FAST_BILINEAR, + )?; + packed_frame(&mut scaler, video, width, height) +} + +fn packed_frame( + scaler: &mut scaling::context::Context, + video: &Video, + width: u32, + height: u32, +) -> Result { + let mut output = Video::empty(); + scaler.run(video, &mut output)?; + let mut bgra = Vec::with_capacity(width as usize * height as usize * 4); + for row in 0..height as usize { + let start = row * output.stride(0); + bgra.extend_from_slice(&output.data(0)[start..start + width as usize * 4]); + } + Ok(Frame { + width, + height, + bgra, + }) +} + +/// A thumbnail from the first decoded frame of any MP4/MKV path. +pub fn first_frame(path: &Path, target: (u32, u32)) -> Result { + let mut decoder = Decoder::open(path, 60)?; + let (_, frame) = decoder.next_frame()?.ok_or(ffmpeg::Error::Eof)?; + scale(&frame, target) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn frame_exact_seek_and_bounded_aspect_ratio() { + assert_eq!(frame_index(Duration::from_millis(1999), 60), 119); + assert_eq!(frame_index(Duration::from_secs(2), 60), 120); + assert_eq!(fitted_size((2560, 1440), (800, 450)), (800, 450)); + assert_eq!(fitted_size((1440, 2560), (800, 450)), (253, 450)); + assert_eq!(fitted_size((8000, 8000), (8000, 8000)), (1080, 1080)); + } +} diff --git a/crates/clipbin-media/src/lib.rs b/crates/clipbin-media/src/lib.rs index f55d659..cafd8e0 100644 --- a/crates/clipbin-media/src/lib.rs +++ b/crates/clipbin-media/src/lib.rs @@ -67,3 +67,13 @@ mod tests { println!("{probe:?}"); } } + +pub mod audio; +pub mod decode; +pub mod player; +pub mod thumbnail; + +mod priority; +#[cfg(windows)] +#[allow(unsafe_code)] +mod windows_priority; diff --git a/crates/clipbin-media/src/player.rs b/crates/clipbin-media/src/player.rs new file mode 100644 index 0000000..38ec149 --- /dev/null +++ b/crates/clipbin-media/src/player.rs @@ -0,0 +1,459 @@ +//! On-demand background playback with a one-frame mailbox and coalesced controls. +use crate::decode::{self, Decoder, Frame}; +use clipbin_core::store::{Metadata, Store}; +use std::{ + path::PathBuf, + sync::{Arc, Condvar, Mutex}, + thread, + time::{Duration, Instant}, +}; + +#[derive(Clone, Debug)] +pub enum Picture { + Loading, + Frame(Arc), + Gap, + Error(String), +} +#[derive(Clone, Debug)] +pub struct Snapshot { + pub time: Duration, + pub playing: bool, + pub picture: Picture, + pub revision: u64, + pub audio_error: Option, + pub video_time: Option, + pub frames_per_second: u32, +} +struct Controls { + session: Option<(Store, u64)>, + seek: Option, + latest: bool, + playing: bool, + size: (u32, u32), + generation: u64, + muted: bool, + volume: f32, + quit: bool, + snapshot: Snapshot, +} +struct Shared { + state: Mutex, + wake: Condvar, +} + +pub struct Player { + shared: Arc, +} +impl Default for Player { + fn default() -> Self { + Self::new() + } +} +impl Player { + pub fn new() -> Self { + let shared = Arc::new(Shared { + state: Mutex::new(Controls { + session: None, + seek: None, + latest: false, + playing: false, + size: (800, 450), + generation: 0, + muted: false, + volume: 1., + quit: false, + snapshot: Snapshot { + time: Duration::ZERO, + playing: false, + picture: Picture::Loading, + revision: 0, + audio_error: None, + video_time: None, + frames_per_second: 60, + }, + }), + wake: Condvar::new(), + }); + let worker = shared.clone(); + thread::Builder::new() + .name("clipbin-player".into()) + .spawn(move || run(worker)) + .expect("player thread"); + Self { shared } + } + pub fn open(&self, store: Store, id: u64, time: Duration) { + self.open_session(store, id, time, false); + } + pub fn open_latest(&self, store: Store, id: u64) { + self.open_session(store, id, Duration::ZERO, true); + } + fn open_session(&self, store: Store, id: u64, time: Duration, latest: bool) { + let mut state = self.shared.state.lock().unwrap(); + state.session = Some((store, id)); + state.latest = latest; + state.seek = Some(time); + state.playing = false; + state.generation += 1; + state.snapshot.picture = Picture::Loading; + state.snapshot.revision += 1; + self.shared.wake.notify_one(); + } + pub fn close(&self) { + let mut state = self.shared.state.lock().unwrap(); + state.session = None; + state.playing = false; + state.generation += 1; + self.shared.wake.notify_one(); + } + pub fn seek(&self, time: Duration) { + let mut state = self.shared.state.lock().unwrap(); + state.seek = Some(time); + state.snapshot.time = time; + state.snapshot.picture = Picture::Loading; + state.snapshot.revision += 1; + state.generation += 1; + self.shared.wake.notify_one(); + } + pub fn play(&self, playing: bool) { + self.shared.state.lock().unwrap().playing = playing; + self.shared.wake.notify_one(); + } + pub fn audio(&self, muted: bool, volume: f32) { + let mut state = self.shared.state.lock().unwrap(); + if state.muted != muted { + state.seek = Some(state.snapshot.time); + state.generation += 1; + } + state.muted = muted; + state.volume = volume.clamp(0., 1.); + self.shared.wake.notify_one(); + } + pub fn resize(&self, size: (u32, u32)) { + let mut state = self.shared.state.lock().unwrap(); + if state.size != size { + state.size = size; + state.generation += 1; + self.shared.wake.notify_one(); + } + } + pub fn snapshot(&self) -> Snapshot { + self.shared.state.lock().unwrap().snapshot.clone() + } +} +impl Drop for Player { + fn drop(&mut self) { + self.shared.state.lock().unwrap().quit = true; + self.shared.wake.notify_one(); + } +} + +/// Monotonic session clock: decoding delays never accumulate into playback time. +#[derive(Debug)] +pub struct Clock { + base: Duration, + started: Instant, + playing: bool, +} +impl Clock { + pub fn new(now: Instant) -> Self { + Self { + base: Duration::ZERO, + started: now, + playing: false, + } + } + pub fn time(&self, now: Instant) -> Duration { + self.base + + if self.playing { + now.saturating_duration_since(self.started) + } else { + Duration::ZERO + } + } + pub fn set(&mut self, time: Duration, playing: bool, now: Instant) { + self.base = time; + self.started = now; + self.playing = playing; + } +} + +fn run(shared: Arc) { + crate::priority::lower(false); + let mut clock = Clock::new(Instant::now()); + let mut generation = u64::MAX; + let mut metadata: Option = None; + let mut refreshed = Instant::now(); + let mut decoder: Option<(PathBuf, Decoder, u64)> = None; + let mut audio: Option = None; + let mut audio_error = None; + let mut audio_attempted = false; + loop { + let (session, seek, playing, size, requested, muted, volume, latest) = { + let mut state = shared.state.lock().unwrap(); + if state.quit { + break; + } + ( + state.session.clone(), + state.seek.take(), + state.playing, + state.size, + state.generation, + state.muted, + state.volume, + std::mem::take(&mut state.latest), + ) + }; + let changed = requested != generation; + if changed { + generation = requested; + decoder = None; + } + let now = Instant::now(); + if changed || now.duration_since(refreshed) >= Duration::from_millis(250) { + metadata = session + .as_ref() + .and_then(|(store, id)| store.metadata(*id).ok()); + refreshed = now; + } + let seek = if latest { + metadata + .as_ref() + .and_then(|metadata| { + metadata + .segments + .iter() + .rev() + .find(|segment| { + segment.complete && segment.available && segment.end > segment.start + }) + .map(|segment| { + segment.end.saturating_sub(Duration::from_secs_f64( + 1. / segment.frames_per_second.max(1) as f64, + )) + }) + }) + .or(Some(Duration::ZERO)) + } else { + seek.map(|time| { + metadata + .as_ref() + .map_or(time, |metadata| time.min(metadata.session.length)) + }) + }; + let transport_changed = playing != clock.playing; + if seek.is_some() || transport_changed { + clock.set(seek.unwrap_or_else(|| clock.time(now)), playing, now); + } + if session.is_none() { + audio = None; + audio_attempted = false; + audio_error = None; + } + if session.is_some() && !audio_attempted && !muted { + audio_attempted = true; + match crate::audio::Audio::new() { + Ok(output) => audio = Some(output), + Err(error) => audio_error = Some(error), + } + } + if let Some(audio) = audio.as_mut() { + if changed || seek.is_some() || transport_changed { + audio.reset(clock.time(now)); + } + audio.controls(playing && !muted, volume); + } + let Some(metadata) = metadata.as_ref().filter(|_| session.is_some()) else { + wait(&shared, Duration::from_millis(100)); + continue; + }; + let mut time = clock.time(now).min(metadata.session.length); + if playing + && !muted + && let Some(audio) = audio.as_mut() + { + if let Some(device_time) = audio.time(now) + && metadata + .locate(device_time) + .is_some_and(|at| at.segment.complete) + { + time = device_time.min(metadata.session.length); + clock.set(time, true, now); + } + audio.catch_up(time); + if let Err(error) = audio.fill(metadata) { + audio_error = Some(format!("Cannot play audio: {error}")); + } + if let Some(error) = audio.error() { + audio_error = Some(error); + } + } + + if time >= metadata.session.length && playing { + clock.set(time, false, now); + shared.state.lock().unwrap().playing = false; + } + // Completed files only: active files may still lack seekable cues. + let located = metadata.locate(time).filter(|at| at.segment.complete); + let start = located + .as_ref() + .map_or(Duration::ZERO, |at| at.segment.start); + let frames_per_second = located + .as_ref() + .map_or(60, |at| at.segment.frames_per_second); + let mut picture = Picture::Gap; + let mut presented = false; + let started = Instant::now(); + if let Some(at) = located { + let target = decode::frame_index(at.offset, at.segment.frames_per_second); + let result = (|| { + if decoder.as_ref().is_none_or(|(path, _, index)| { + path != &at.segment.path + || target < *index + || target > index.saturating_add(120) + }) { + let mut opened = Decoder::open(&at.segment.path, at.segment.frames_per_second)?; + if at.offset > Duration::ZERO { + opened.seek(at.offset)?; + } + decoder = Some((at.segment.path.clone(), opened, u64::MAX)); + } + let (_, active, last) = decoder.as_mut().unwrap(); + if *last == target && !changed { + return Ok(None); + } + loop { + // A drag or navigation cancels decode-forward work between frames. + let state = shared.state.lock().unwrap(); + if state.quit || state.generation != generation { + return Ok(None); + } + drop(state); + let Some((index, frame)) = active.next_frame()? else { + return Ok(Some(Picture::Gap)); + }; + if index >= target { + *last = index; + return active + .scale(&frame, size) + .map(|frame| Some(Picture::Frame(Arc::new(frame)))); + } + } + })(); + match result { + Ok(Some(frame)) => { + presented = matches!(frame, Picture::Frame(_)); + picture = frame; + } + Ok(None) => picture = shared.state.lock().unwrap().snapshot.picture.clone(), + Err(error) => { + picture = Picture::Error(format!("Cannot read footage: {error}")); + decoder = None; + } + } + } else { + decoder = None; + } + let mut state = shared.state.lock().unwrap(); + if state.generation == generation { + let updated = time != state.snapshot.time + || clock.playing != state.snapshot.playing + || presented + || std::mem::discriminant(&picture) + != std::mem::discriminant(&state.snapshot.picture) + || audio_error != state.snapshot.audio_error; + let revision = state.snapshot.revision + u64::from(updated); + let video_time = matches!(picture, Picture::Frame(_)) + .then(|| { + decoder.as_ref().map(|(_, _, index)| { + start + clipbin_core::store::frame_offset(*index, frames_per_second) + }) + }) + .flatten(); + state.snapshot = Snapshot { + time, + playing: clock.playing, + picture, + revision, + audio_error: audio_error.clone(), + video_time, + frames_per_second, + }; + if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() && (playing || changed) { + eprintln!( + "playback time={:.3} decode_ms={:.3} generation={generation} presented={presented} state={} audio_time={:.3} video_time={:.3}", + time.as_secs_f64(), + started.elapsed().as_secs_f64() * 1000., + match &state.snapshot.picture { + Picture::Frame(_) => "frame", + Picture::Gap => "gap", + Picture::Loading => "loading", + Picture::Error(_) => "error", + }, + audio + .as_ref() + .and_then(|audio| audio.time(Instant::now())) + .map_or(-1., |time| time.as_secs_f64()), + video_time.map_or(-1., |time| time.as_secs_f64()) + ); + } + } + drop(state); + wait( + &shared, + if clock.playing { + Duration::from_millis(4) + } else { + Duration::from_millis(100) + }, + ); + } +} +fn wait(shared: &Shared, timeout: Duration) { + let state = shared.state.lock().unwrap(); + if !state.quit && state.seek.is_none() { + drop(shared.wake.wait_timeout(state, timeout).unwrap()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn a_decoder_stall_skips_overdue_frames_without_shifting_the_clock() { + let now = Instant::now(); + let mut clock = Clock::new(now); + clock.set(Duration::ZERO, true, now); + let first_target = decode::frame_index(clock.time(now + Duration::from_millis(16)), 60); + let after_stall = decode::frame_index(clock.time(now + Duration::from_millis(250)), 60); + assert_eq!(first_target, 0); + assert_eq!(after_stall, 15); + // Scaling only the target discards the intervening frames; their decoding + // cost must never be added to the clock's next presentation deadline. + assert_eq!( + decode::frame_index(clock.time(now + Duration::from_secs(1)), 60), + 60 + ); + } + #[test] + fn clock_pauses_and_seek_does_not_accumulate_decode_delay() { + let now = Instant::now(); + let mut clock = Clock::new(now); + clock.set(Duration::from_secs(2), true, now); + assert_eq!( + clock.time(now + Duration::from_secs(3)), + Duration::from_secs(5) + ); + clock.set(Duration::from_secs(5), false, now); + assert_eq!( + clock.time(now + Duration::from_secs(100)), + Duration::from_secs(5) + ); + clock.set(Duration::from_millis(1500), true, now); + assert_eq!( + decode::frame_index(clock.time(now + Duration::from_millis(100)), 60), + 96 + ); + } +} diff --git a/crates/clipbin-media/src/priority.rs b/crates/clipbin-media/src/priority.rs new file mode 100644 index 0000000..b078f89 --- /dev/null +++ b/crates/clipbin-media/src/priority.rs @@ -0,0 +1,14 @@ +//! Browsing runs below the recorder's normal scheduling priority. +pub fn lower(thumbnail: bool) { + #[cfg(target_os = "linux")] + if let Err(error) = rustix::process::setpriority_process( + Some(rustix::thread::gettid()), + if thumbnail { 10 } else { 3 }, + ) { + eprintln!("Cannot lower browsing thread priority: {error}"); + } + #[cfg(windows)] + crate::windows_priority::lower(thumbnail); + #[cfg(not(any(windows, target_os = "linux")))] + let _ = thumbnail; +} diff --git a/crates/clipbin-media/src/thumbnail.rs b/crates/clipbin-media/src/thumbnail.rs new file mode 100644 index 0000000..453ff74 --- /dev/null +++ b/crates/clipbin-media/src/thumbnail.rs @@ -0,0 +1,81 @@ +//! One background worker and a bounded queue for persistent PNG thumbnails. +use crate::decode; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::{Arc, Mutex, mpsc}, + thread, + time::Duration, +}; + +/// Deterministic cache path beside the source; exported MP4s use the same API. +pub fn cache_path(path: &Path) -> PathBuf { + path.with_extension("thumbnail.png") +} + +/// Decode any MP4/MKV and atomically publish a small thumbnail. +pub fn generate(path: &Path) -> Result { + let cache = cache_path(path); + if cache.is_file() { + return Ok(cache); + } + let mut frame = decode::first_frame(path, (320, 180)).map_err(|error| error.to_string())?; + let temporary = cache.with_extension("png.part"); + for pixel in frame.bgra.as_chunks_mut::<4>().0 { + pixel.swap(0, 2); + } + image::save_buffer_with_format( + &temporary, + &frame.bgra, + frame.width, + frame.height, + image::ColorType::Rgba8, + image::ImageFormat::Png, + ) + .map_err(|error| error.to_string())?; + std::fs::rename(&temporary, &cache).map_err(|error| error.to_string())?; + Ok(cache) +} + +pub struct Thumbnails { + sender: mpsc::SyncSender, + pending: Arc>>, +} +impl Default for Thumbnails { + fn default() -> Self { + Self::new() + } +} +impl Thumbnails { + pub fn new() -> Self { + let (sender, receiver) = mpsc::sync_channel::(16); + let pending = Arc::new(Mutex::new(HashSet::new())); + let work = pending.clone(); + thread::Builder::new() + .name("clipbin-thumbnails".into()) + .spawn(move || { + crate::priority::lower(true); + for path in receiver { + // One decoder thread, one frame, and a duty-cycle cap leave capture priority. + thread::sleep(Duration::from_millis(150)); + if let Err(error) = generate(&path) { + eprintln!("thumbnail {}: {error}", path.display()); + } + work.lock().unwrap().remove(&path); + } + }) + .expect("thumbnail thread"); + Self { sender, pending } + } + pub fn request(&self, path: &Path) -> Option { + let cache = cache_path(path); + if cache.is_file() { + return Some(cache); + } + let mut pending = self.pending.lock().unwrap(); + if pending.insert(path.to_owned()) && self.sender.try_send(path.to_owned()).is_err() { + pending.remove(path); + } + None + } +} diff --git a/crates/clipbin-media/src/windows_priority.rs b/crates/clipbin-media/src/windows_priority.rs new file mode 100644 index 0000000..2b3ffa5 --- /dev/null +++ b/crates/clipbin-media/src/windows_priority.rs @@ -0,0 +1,19 @@ +//! The only raw Windows calls in the media crate; no handles escape this bridge. +use windows_sys::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_LOWEST, +}; + +pub fn lower(thumbnail: bool) { + let priority = if thumbnail { + THREAD_PRIORITY_LOWEST + } else { + THREAD_PRIORITY_BELOW_NORMAL + }; + // GetCurrentThread returns a borrowed pseudo-handle, valid for this thread. + if unsafe { SetThreadPriority(GetCurrentThread(), priority) } == 0 { + eprintln!( + "Cannot lower browsing thread priority: {}", + std::io::Error::last_os_error() + ); + } +} diff --git a/crates/clipbin-media/tests/playback.rs b/crates/clipbin-media/tests/playback.rs new file mode 100644 index 0000000..63416f8 --- /dev/null +++ b/crates/clipbin-media/tests/playback.rs @@ -0,0 +1,106 @@ +//! Run through scripts/validate/playback-fixture.sh for real FFmpeg seek coverage. +use clipbin_core::store::Store; +use clipbin_media::{ + decode::{Decoder, scale}, + player::{Picture, Player}, +}; +use std::{ + path::PathBuf, + thread, + time::{Duration, Instant}, +}; + +fn fixture() -> Option { + std::env::var_os("CLIPBIN_TEST_MEDIA_DIR").map(PathBuf::from) +} +#[test] +fn seeking_matches_sequential_frames_in_each_independent_segment() { + let Some(directory) = fixture() else { + return; + }; + for segment in 0..3 { + let path = directory.join(format!("1/segment-{segment:03}.mkv")); + let mut sequential = Decoder::open(&path, 60).unwrap(); + let mut expected = Vec::new(); + while let Some((index, frame)) = sequential.next_frame().unwrap() { + expected.push((index, scale(&frame, (160, 90)).unwrap().bgra)); + } + assert_eq!(expected.len(), 120); + for target in [0, 1, 59, 60, 61, 119] { + let mut decoder = Decoder::open(&path, 60).unwrap(); + decoder + .seek(Duration::from_secs_f64(target as f64 / 60.)) + .unwrap(); + loop { + let (index, frame) = decoder.next_frame().unwrap().unwrap(); + if index >= target { + assert_eq!(index, target); + assert_eq!( + scale(&frame, (160, 90)).unwrap().bgra, + expected[target as usize].1 + ); + break; + } + } + } + } +} +fn wait(player: &Player, at: Duration) -> clipbin_media::player::Snapshot { + let until = Instant::now() + Duration::from_secs(5); + loop { + let snapshot = player.snapshot(); + if snapshot.time == at && !matches!(snapshot.picture, Picture::Loading) { + return snapshot; + } + assert!(Instant::now() < until, "seek timed out: {snapshot:?}"); + thread::sleep(Duration::from_millis(10)); + } +} +#[test] +fn player_crosses_boundaries_and_gaps_and_discards_superseded_seeks() { + let Some(directory) = fixture() else { + return; + }; + let player = Player::new(); + player.audio(true, 1.); + player.open(Store { root: directory }, 1, Duration::ZERO); + assert!(matches!( + wait(&player, Duration::ZERO).picture, + Picture::Frame(_) + )); + for millis in [1999, 2000, 2500, 4000, 5999, 6000, 7000] { + let at = Duration::from_millis(millis); + player.seek(at); + let snapshot = wait(&player, at); + assert_eq!( + matches!(snapshot.picture, Picture::Gap), + (4000..6000).contains(&millis), + "at {millis}" + ); + } + for i in 0..50 { + player.seek(Duration::from_millis(i * 40)); + } + player.seek(Duration::from_millis(6500)); + assert!(matches!( + wait(&player, Duration::from_millis(6500)).picture, + Picture::Frame(_) + )); + player.seek(Duration::from_millis(1900)); + wait(&player, Duration::from_millis(1900)); + player.play(true); + thread::sleep(Duration::from_millis(400)); + player.play(false); + let snapshot = player.snapshot(); + assert!((Duration::from_millis(2250)..Duration::from_millis(2500)).contains(&snapshot.time)); + assert!(matches!(snapshot.picture, Picture::Frame(_))); + assert!(snapshot.video_time.unwrap().abs_diff(snapshot.time) < Duration::from_millis(20)); + player.seek(Duration::from_millis(3900)); + wait(&player, Duration::from_millis(3900)); + player.play(true); + thread::sleep(Duration::from_millis(350)); + let snapshot = player.snapshot(); + assert!(snapshot.time >= Duration::from_millis(4200)); + assert!(matches!(snapshot.picture, Picture::Gap)); + player.close(); +} diff --git a/docs/validation.md b/docs/validation.md index cdb46ac..3049dd6 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -262,3 +262,99 @@ active, it exits immediately. It stops discovering new games once closed, so there is no indefinitely running invisible app. Reopening the window and persistent background discovery need the future tray integration. Explicit Quit/Ctrl+Q stops and finalises the current recording immediately. + +## Playback and thumbnails + +Playback uses the shared `clipbin-media` crate (`ffmpeg-next` 9) without GPUI. +One background decoder opens only completed, available segments. Session time +comes from `Metadata::locate`; each file's timestamps are normalised independently. +Seeking goes backwards to a keyframe, then decodes forward to the selected frame. +A one-frame mailbox and coalesced seeks bound work; overdue frames are decoded +but discarded before swscale. Playback threads have lower scheduling priority +than recording. Leaving the editor closes its decoder and audio stream. + +The editor uploads packed BGRA at its actual pixel size, maintains a 16:9 +preview, and releases old GPUI images. Space/K toggle playback; arrows/J/L step +one frame. The timeline seeks and drags across files and gaps; completed footage +from active sessions becomes browsable on the next 250 ms metadata refresh +within the recorder's segment completion delay. Missing/removed footage advances +on the wall clock and shows “No footage (not captured / cleaned up)”. There is +no live recording preview. + +Audio uses cpal and swresample at the output device's rate/channel count, with +sample conversion in the device callback. OBS's combined third audio track is +selected when three tracks exist; otherwise the first track is used. The queue +holds at most 250 ms plus one decoded block. Callback timestamps, including DAC +latency, drive the master clock; gaps and mute use the wall clock. Callbacks do +no decoding and never wait for the queue lock. Mute and volume are in the editor; +an unavailable device is shown explicitly while video remains usable. The Linux +rig uses its existing null sink and ALSA's PipeWire plugin as a fallback if cpal's +native PulseAudio backend refuses the device. Install/configure `pipewire-alsa` +or an ALSA default routed to PulseAudio on systems without a physical ALSA card. + +One low-priority worker produces 320×180 PNG thumbnails beside the first playable +session segment and exported MP4s. Its request queue holds 16 paths and duplicate +requests coalesce; card discovery and all filesystem work run off the UI thread. +`clipbin_media::thumbnail::{generate,cache_path,Thumbnails}` accept any MP4/MKV. +The gradient remains while a thumbnail is pending or footage is unavailable. + +`scripts/validate/playback-fixture.sh` generates independent 1080p60 H.264/AAC +segments with one-second keyframes and a three-tone audio fixture. It passed on +2026-10-08: exact frame pixels at frame indices 0/1/59/60/61/119 in every file, +real-time boundary crossing, gaps, superseded seeks, pause/seek clock maths, +bounded frame placement, resampling 48 kHz to 44.1 kHz and combined-track selection. +The ordinary workspace tests skip the generated-media assertions unless +`CLIPBIN_TEST_MEDIA_DIR` points to the fixture; use the script to exercise them. + +Windows: the cpal WASAPI dependency cross-check passed, as did the dedicated +Windows thread-priority bridge. Core/OBS/probe strict MSVC cross-lint passed. +The full media cross-lint is excluded on this Linux machine because the FFmpeg +Windows development import libraries are unavailable. On Windows set `FFMPEG_DIR` +to an FFmpeg **7.1 shared development** package with MSVC import libraries matching +the OBS-bundled avcodec-61/avformat-61/avutil-59/swscale-8/swresample-5 DLLs. Linux +links system FFmpeg 8. Actual Windows decoding, WASAPI device latency/seek behaviour, +GPUI image presentation and thumbnail caching need a Windows machine. Hardware +video decode is not used; HEVC/AV1 software performance and real output devices +other than the rig null sink still need measurement. + +Final playback validation passed on 2026-10-08, serially with other rigs stopped: + +- `cargo fmt --all -- --check`, strict locked workspace clippy, and locked + workspace tests (with `CLIPBIN_TEST_MEDIA_DIR=target/playback-fixture`). +- `env PATH=/usr/bin:/bin:$HOME/.cargo/bin scripts/validate/linux-rig.sh`: + all nine scenarios. An earlier concurrent run lost its H.264 probe to SIGKILL; + the serial rerun passed without timeout or sandbox changes. +- The same PATH with `scripts/validate/linux-session.sh`: automatic recording, + exact segment placement, close-window survival, game exit, library reload and + clean Quit. Repeated after the final keyboard integration, also passed. +- `scripts/validate/linux-playback.sh target/playback-e2e-final-6`: moving preview + (11.0% changed pixels inside the preview), active recording with zero reported + recording drops, completed-segment availability, real session thumbnail, + cross-file drag, exact post-boundary frame/time, cleaned-up gap, zoom, arrow + stepping, device audio, half-volume and mute. The audio monitor RMS was 0.08044; + the player asserts video time stays within 20 ms of the audio master clock. + Screenshots were inspected: `sessions-thumbnails.png`, `editor-playing-1.png`, + `editor-playing-2.png`, `editor-seek.png`, and `editor-gap.png` in that directory. + +The rig runs on an Intel i5-12500 (12 logical CPUs). These are development builds +with dependencies at opt-level 2, software video decoding, 1080p60 H.264 source +and an 800×450 target (the editor used 740×416). Five serial repetitions of +`target/debug/examples/playback-check ` with +`scripts/validate/measure-cpu.py`, inside the null-sink rig, measured process CPU +at **16.7% median / 17.7% p95 of one core** (12.2–17.7% range). The filesystem cache +was warm; each repetition reopened its decoders. Across presented frames, +decode/scale time was **2.39 ms median / 8.80 ms p95**; maximum **101.5 ms** included +opening files and decoding forward on seeks. This is a test pattern with gaps, +not a worst-case game or a release-build benchmark. Per-run logs and CPU reports +are in `target/playback-timing-final`; `report.json` stores the distributions. + +During the active-session e2e, the player presented **59.2 frames/s** while +recording remained at zero reported drops. GPUI upload/construction time was +**0.22 ms median / 0.48 ms p95**. UI frame intervals (including vsync, not CPU work) +were 16.2 ms median / 32.5 ms p95; the one-frame mailbox drops presentation work +rather than allowing delay to accumulate. `CLIPBIN_TRACE_FRAMES=1` logs decoded +video time, audio time, decode/scale work, uploads and actual timeline bounds. +The headless e2e restores keyboard focus after virtual-pointer commands and +holds virtual keys briefly: instantaneous virtual input can disappear before +GPUI/Wayland dispatch. Keyboard actions update the editor entity directly, +avoiding a nested window borrow during event dispatch. diff --git a/scripts/validate/linux-playback.sh b/scripts/validate/linux-playback.sh new file mode 100755 index 0000000..da176a3 --- /dev/null +++ b/scripts/validate/linux-playback.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Playback, active browsing, real thumbnails and audio in the private desktop rig. +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +out="${1:-${repo}/target/playback-validation}" +mkdir -p "${out}" +out="$(cd "${out}" && pwd)" +cargo build --locked --workspace --manifest-path "${repo}/Cargo.toml" +cargo build --locked -p clipbin-media --example playback-check --manifest-path "${repo}/Cargo.toml" +export CLIPBIN_PLAYBACK_REPO="${repo}" CLIPBIN_PLAYBACK_OUT="${out}" +"${repo}/scripts/rig/headless.sh" bash <<'INNER' +set -euo pipefail +repo="${CLIPBIN_PLAYBACK_REPO}" out="${CLIPBIN_PLAYBACK_OUT}" +export XDG_DATA_HOME="${out}/data" XDG_CONFIG_HOME="${out}/config" CLIPBIN_TRACE_FRAMES=1 +mkdir -p "${XDG_DATA_HOME}" "${XDG_CONFIG_HOME}" +# ALSA's PipeWire plugin is cpal's fallback on Linux; there is no hardware card. +cat > "${out}/alsa.conf" <<'ALSA' + +pcm.!default { type pipewire } +ctl.!default { type pipewire } +ALSA +export ALSA_CONFIG_PATH="${out}/alsa.conf" +swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null +send_key() { + # Destroying rig-pointer's virtual seat can clear keyboard focus in Sway. + swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null + sleep .2 + timeout 10 wtype -s 200 -P "$1" -s 100 -p "$1" +} +# Timeline bounds come from GPUI's trace rather than guessed screen positions. +pointer_time() { + python3 - "$1" "$2" "$3" "${XDG_DATA_HOME}" <<'PYPOINTER' +import pathlib,re,json,sys +log=pathlib.Path(sys.argv[1]).read_text() +x,y,w,h=map(float,re.findall(r'playback_timeline x=([\d.]+) y=([\d.]+) width=([\d.]+) height=([\d.]+)',log)[-1]) +p=max(pathlib.Path(sys.argv[4]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());length=m['session']['length'];length=length['secs']+length['nanos']/1e9 +if sys.argv[2] in ('gap','after-gap'): + segment=next(s for s in m['segments'] if s['complete'] and not s['available']) + def seconds(d):return d['secs']+d['nanos']/1e9 + at=(seconds(segment['start'])+seconds(segment['end']))/2 if sys.argv[2]=='gap' else seconds(segment['end'])+.5 +else:at=float(sys.argv[2]) +print(f"move {round(280+x+w*at/length)} {round(110+y+h/2)}\nsleep .15\n{sys.argv[3]}\nsleep .15") +PYPOINTER +} +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] floating enable' > /dev/null +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] resize set width 1360 px height 860 px' > /dev/null +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] move position 280 110' > /dev/null +"${repo}/target/debug/clipbin-desktop" > "${out}/active.log" 2>&1 & app=$! +trap 'kill "${app:-}" "${game:-}" "${audio_capture:-}" 2>/dev/null || true' EXIT +sleep 3 +swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null +swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null +bash -c 'exec -a reaper python3 -c "import time; time.sleep(120)" SteamLaunch AppId=480 --' & game=$! +for _ in $(seq 30); do if grep -q recorder_status=Recording "${out}/active.log"; then break; fi; sleep 1; done +grep -q recorder_status=Recording "${out}/active.log" +sleep 8 +grim "${out}/sessions-thumbnails.png" +# Open the first session card, seek near the start, and play while recording continues. +printf 'move 670 330\nsleep .2\ndown\nsleep .1\nup\nsleep 1\n' | "${repo}/target/debug/rig-pointer" 1920 1080 +grim "${out}/editor-open.png" +pointer_time "${out}/active.log" 1 click | "${repo}/target/debug/rig-pointer" 1920 1080 +# GPUI/Wayland needs the virtual keyboard keymap to settle and a held key; +# instantaneous wtype press/release occasionally disappears before dispatch. +send_key space +sleep 1 +grim "${out}/editor-playing-1.png" +sleep 2 +grim "${out}/editor-playing-2.png" +python3 - "${XDG_DATA_HOME}" <<'PY' +import pathlib,json,sys +p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());assert m['session']['active'];assert len([s for s in m['segments'] if s['complete'] and s['available']])>=4 +assert list(p.parent.rglob('*.thumbnail.png')), 'thumbnail worker produced no cached frame' +print('PASS playback left recording active and generated real thumbnails') +PY +kill "${game}"; wait "${game}" 2>/dev/null || true +sleep 3 +swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null +timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl +for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done +if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi +wait "${app}" +kill "$(cat "${CLIPBIN_RIG_DIR}/game.pid")" +# Simulate a cleaned-up middle segment in this isolated test library. Keep its +# timeline placement: the player must show the gap rather than concatenate it away. +python3 - "${XDG_DATA_HOME}" "${out}" <<'PY' +import pathlib,json,sys +p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());segments=[s for s in m['segments'] if s['complete'] and s['available']];assert len(segments)>=4 +s=segments[2];s['available']=False +m['session']['spans']=[dict(start=m['session']['spans'][0]['start'],end=s['start'],kind='Footage'),dict(start=s['start'],end=s['end'],kind='Removed'),dict(start=s['end'],end=m['session']['length'],kind='Footage')] +p.write_text(json.dumps(m));pathlib.Path(sys.argv[2]+'/session-id').write_text(str(m['session']['id'])) +PY +id="$(cat "${out}/session-id")" +# The CLI exercises the same player with audio only; recording tone has stopped. +ffmpeg -v error -y -f pulse -i clipbin-rig-sink.monitor -t 10 "${out}/audio-output.wav" & audio_capture=$! +python3 "${repo}/scripts/validate/measure-cpu.py" "${out}/player-cpu.txt" "${repo}/target/debug/examples/playback-check" "${XDG_DATA_HOME}/clipbin/sessions" "${id}" > "${out}/player.log" 2>&1 +wait "${audio_capture}" +CLIPBIN_ROUTE="session:${id}" "${repo}/target/debug/clipbin-desktop" > "${out}/playback.log" 2>&1 & app=$! +sleep 3 +swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null +# Drag through several files, then seek into the removed interval. +{ pointer_time "${out}/playback.log" 1 down; pointer_time "${out}/playback.log" 3 "sleep 0.5"; pointer_time "${out}/playback.log" after-gap up; } | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep 1 +grim "${out}/editor-seek.png" +pointer_time "${out}/playback.log" gap click | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep 1 +grim "${out}/editor-gap.png" +pointer_time "${out}/playback.log" 1 $'scroll -1\nsleep 0.5\nscroll 1' | "${repo}/target/debug/rig-pointer" 1920 1080 +pointer_time "${out}/playback.log" 2.5 click | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep .3 +send_key Right +sleep .3 +send_key Left +sleep .3 +swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null +timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl +for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done +if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi +wait "${app}" +python3 - "${out}" <<'PY' +import pathlib,subprocess,re,sys,math,array,statistics,json +out=pathlib.Path(sys.argv[1]) +def rgb(path):return subprocess.check_output(['ffmpeg','-v','error','-i',str(path),'-vf','crop=700:300:555:260','-f','rawvideo','-pix_fmt','rgb24','-']) +a,b=map(rgb,[out/'editor-playing-1.png',out/'editor-playing-2.png']) +changed=sum(x!=y for x,y in zip(a,b))/len(a);assert changed>.02, f'preview frozen: {changed}' +raw=subprocess.check_output(['ffmpeg','-v','error','-i',str(out/'audio-output.wav'),'-ac','1','-ar','48000','-f','f32le','-']) +samples=array.array('f',raw);rms=math.sqrt(sum(x*x for x in samples)/len(samples));assert rms>.001, f'no device audio: {rms}' +def window_rms(start,end): + block=samples[int(start*48000):int(end*48000)] + return math.sqrt(sum(x*x for x in block)/max(1,len(block))) +full=window_rms(2,3);half=window_rms(7.9,8.4);muted=window_rms(9.5,9.9) +assert full>.001 and .35=2, f'frame stepping did not change the video frame: {steps}' +uploads=[float(x) for x in re.findall(r'playback_ui upload_ms=([\d.]+)',(out/'active.log').read_text())] +assert len(uploads)>30, 'too few moving frames reached GPUI' +print(f'GPUI frame upload median={statistics.median(uploads):.3f}ms p95={sorted(uploads)[int(len(uploads)*.95)]:.3f}ms') +(out/'report.txt').write_text(f'changed_preview_pixels={changed:.4f}\naudio_rms={rms:.5f}\ndecode_median_ms={statistics.median(times):.3f}\ndecode_p95_ms={sorted(times)[int(len(times)*.95)]:.3f}\n') +PY +echo "PASS playback validation; screenshots and timing in ${out}" +INNER diff --git a/scripts/validate/measure-cpu.py b/scripts/validate/measure-cpu.py new file mode 100644 index 0000000..14557b9 --- /dev/null +++ b/scripts/validate/measure-cpu.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Measure child CPU/wall time without depending on GNU time.""" +import pathlib +import resource +import subprocess +import sys +import time + +before = resource.getrusage(resource.RUSAGE_CHILDREN) +started = time.monotonic() +result = subprocess.run(sys.argv[2:], check=False) +elapsed = time.monotonic() - started +usage = resource.getrusage(resource.RUSAGE_CHILDREN) +user = usage.ru_utime - before.ru_utime +system = usage.ru_stime - before.ru_stime +pathlib.Path(sys.argv[1]).write_text( + f"cpu_user={user:.3f} cpu_system={system:.3f} wall={elapsed:.3f} " + f"cpu_percent={(user + system) / elapsed * 100:.1f}%\n" +) +sys.exit(result.returncode) diff --git a/scripts/validate/playback-fixture.sh b/scripts/validate/playback-fixture.sh new file mode 100755 index 0000000..42eaddb --- /dev/null +++ b/scripts/validate/playback-fixture.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Deterministic, clock-labelled 1080p60 segments with independent file timestamps. +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +out="${1:-${repo}/target/playback-fixture}" +mkdir -p "${out}/1" +ffmpeg -v error -y -f lavfi -i 'testsrc2=size=1920x1080:rate=60' \ + -f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 6 \ + -vf "drawtext=fontfile=/usr/share/fonts/TTF/DejaVuSans.ttf:text='%{pts\:hms}':fontsize=64:fontcolor=white:box=1:boxcolor=black:x=100:y=100" \ + -c:v libx264 -preset ultrafast -crf 28 -threads 2 -g 60 -keyint_min 60 -sc_threshold 0 -bf 0 \ + -c:a aac -f segment -segment_time 2 -reset_timestamps 1 "${out}/1/segment-%03d.mkv" +ffmpeg -v error -y -i "${out}/1/segment-000.mkv" \ + -f lavfi -i 'sine=frequency=220:sample_rate=48000' \ + -f lavfi -i 'sine=frequency=880:sample_rate=48000' \ + -f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 2 \ + -map 0:v -map 1:a -map 2:a -map 3:a -c:v copy -c:a aac "${out}/combined.mkv" +python3 - "${out}" <<'PY' +import pathlib,json,sys +root=pathlib.Path(sys.argv[1]); d=lambda s:dict(secs=s,nanos=0) +segments=[dict(path=f'segment-{i:03}.mkv',start=d(t),end=d(t+2),complete=True,available=True,frame_start=i*120,frames_per_second=60,dropped_frames=0) for i,t in enumerate([0,2,6])] +session=dict(id=1,game='Playback fixture',started=dict(secs_since_epoch=1791453600,nanos_since_epoch=0),length=d(8),active=False,spans=[dict(start=d(a),end=d(b),kind=k) for a,b,k in [(0,4,'Footage'),(4,6,'Missing'),(6,8,'Footage')]],bookmarks=[]) +(root/'1/session.json').write_text(json.dumps(dict(session=session,app_id=480,segments=segments))) +PY +CLIPBIN_TEST_MEDIA_DIR="${out}" cargo test --locked -p clipbin-media --manifest-path "${repo}/Cargo.toml"