diff --git a/Cargo.lock b/Cargo.lock index e55467b..e0a0813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,28 @@ dependencies = [ "equator", ] +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -885,6 +907,7 @@ dependencies = [ "clipbin-media", "clipbin-obs", "gpui", + "image", ] [[package]] @@ -892,7 +915,11 @@ name = "clipbin-media" version = "0.1.0" dependencies = [ "clipbin-core", + "cpal", "ffmpeg-next", + "image", + "rustix 1.1.5", + "windows-sys 0.61.2", ] [[package]] @@ -989,6 +1016,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "command-fds" version = "0.3.3" @@ -1199,6 +1236,20 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cosmic-text" version = "0.14.2" @@ -1222,6 +1273,39 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1" +dependencies = [ + "alsa", + "block2", + "coreaudio-rs", + "dasp_sample", + "futures", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "portable-atomic", + "pulseaudio", + "web-sys", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1325,6 +1409,12 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.2" @@ -1534,6 +1624,17 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn 2.0.119", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -2256,7 +2357,7 @@ dependencies = [ "wayland-protocols-plasma", "windows 0.61.3", "windows-core 0.61.2", - "windows-numerics", + "windows-numerics 0.2.0", "windows-registry 0.5.3", "x11-clipboard", "x11rb", @@ -2923,6 +3024,64 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -3165,6 +3324,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3288,6 +3453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -3342,6 +3508,35 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3527,6 +3722,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc" version = "0.2.7" @@ -3570,6 +3787,55 @@ dependencies = [ "objc2-quartz-core", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -3577,7 +3843,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.2", + "block2", "dispatch2", + "libc", "objc2", ] @@ -3594,6 +3862,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.13.2", + "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -3913,6 +4183,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "postage" version = "0.5.0" @@ -4050,6 +4326,22 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" +[[package]] +name = "pulseaudio" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70623bd7967a9ca4c2ae0e807fc380b291f98480fc037042305ec643a4d3373" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-primitive-derive", + "futures", + "log", + "mio", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "pxfm" version = "0.1.30" @@ -4971,6 +5263,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simd_helpers" version = "0.1.0" @@ -6340,11 +6642,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -6357,7 +6671,7 @@ dependencies = [ "rayon", "thiserror 2.0.21", "windows 0.61.3", - "windows-future", + "windows-future 0.2.1", ] [[package]] @@ -6369,6 +6683,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -6394,6 +6717,19 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-future" version = "0.2.1" @@ -6402,7 +6738,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -6471,6 +6818,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-registry" version = "0.4.0" @@ -6511,6 +6868,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.3.1" @@ -6529,6 +6895,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -6622,6 +6997,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" diff --git a/README.md b/README.md index b6185ba..306a881 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The recording foundation is being validated (see [docs/validation.md](docs/valid The desktop app detects Steam games, owns the recorder on a background thread, persists settings and sessions, and loads recorded sessions from disk. Linux detects Steam’s `reaper` processes and watches their pidfds; Windows reads Steam’s registry state and watches native process handles (cross-checked, not run). Game exit finalises a session; closing the window keeps the current session recording until the game exits, then exits the app. Ctrl+Q or Quit stops the output cleanly. -The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Clip export, bookmark protection and storage cleanup are implemented. Playback, autostart and a tray are separate work. Exports copy the recorded codec to MP4 in a background job, save local clip metadata, and support opening/deleting clips. Ctrl+Shift+F9 clips the last configured interval; Ctrl+Shift+F10 protects a bookmark. Both also have sidebar buttons. Wayland uses the GlobalShortcuts portal (first-use approval, portal rebinding); X11 uses native key grabs and Windows uses RegisterHotKey. Settings offer independent alternate function keys. After closing the only window, reopening it without a tray is a follow-up. +The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Playback and frame seeking work across completed segments, including active sessions and missing footage, with combined audio, mute/volume, Space/K to play/pause and J/L or arrows to step frames. Sessions and clips use cached frame thumbnails. Exports copy the recorded codec to MP4 in a background job, save local clip metadata, and support opening/deleting clips. Ctrl+Shift+F9 clips the last configured interval; Ctrl+Shift+F10 protects a bookmark. Both also have sidebar buttons. Wayland uses the GlobalShortcuts portal (first-use approval, portal rebinding); X11 uses native key grabs and Windows uses RegisterHotKey. Settings offer independent alternate function keys. Storage cleanup keeps unprotected footage within the budget. Autostart and a tray are not implemented yet; after closing the only window, reopening it without a tray is a follow-up. Windows is a first-class target. Every change must keep the Windows code paths building (`cargo clippy --target x86_64-pc-windows-msvc`) and list what still needs validating on Windows. @@ -19,7 +19,7 @@ Windows is a first-class target. Every change must keep the Windows code paths b ```text crates/ clipbin-core/ Settings, Steam detection and session storage; independent of GPUI and OBS. - clipbin-media/ FFmpeg stream-copy exports and shared media reading. + clipbin-media/ FFmpeg: stream-copy clip export, scaled video/audio playback and cached thumbnails. clipbin-obs/ Recording engine over the pinned libobs runtime; all unsafe code is in src/ffi.rs. clipbin-probe/ Validation probe for capture, encoders, segments and status. clipbin-desktop/ GPUI application and initial session-library view. @@ -42,7 +42,7 @@ Keep the lockfile checked in and use `--locked`. It currently pins `libc` to 0.2 On Arch/CachyOS, install the native build dependencies: ```sh -sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang +sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang ffmpeg alsa-lib pipewire-alsa libpulse ``` ### OBS runtime @@ -63,9 +63,11 @@ On Debian/Ubuntu: ```sh sudo apt-get update -sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev +sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libasound2-dev libpulse-dev pipewire-alsa ``` +Media dynamically links FFmpeg through `ffmpeg-next` 9. Linux uses system FFmpeg 8. On Windows, set `FFMPEG_DIR` to an FFmpeg **7.1 shared development** package with MSVC import libraries, matching the OBS-bundled `avcodec-61`, `avformat-61`, `avutil-59`, `swscale-8` and `swresample-5` DLLs. Keep those OBS DLLs alongside the desktop executable; do not mix FFmpeg major versions. Audio uses cpal (PulseAudio or ALSA/PipeWire on Linux, WASAPI on Windows). + For Windows, use the MSVC Rust toolchain and Visual Studio Build Tools with the Desktop development with C++ workload. Windows support needs validation on a Windows machine; see [docs/validation.md](docs/validation.md#windows). Run the desktop shell from an active graphical session: @@ -90,7 +92,7 @@ cargo clippy --locked --target x86_64-pc-windows-msvc -p clipbin-core -p clipbin Clips/bookmarks/cleanup validation: `scripts/validate/linux-clips.sh`. Native X11 hotkey validation: `scripts/validate/linux-hotkeys.sh`. Both run in the headless rig. Real Hyprland portal hotkeys and Windows runtime behaviour still need validation. -Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times. +Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Playback fixtures: `scripts/validate/playback-fixture.sh`. Playback/audio e2e: `scripts/validate/linux-playback.sh [output-dir]` (moving preview, active recording, gaps, thumbnails, audio monitor, timing and CPU report). Fake Steam rigs must run serially because discovery reads the machine’s `/proc`. Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times. Development builds optimise dependencies (`[profile.dev.package."*"]`): unoptimised GPUI made screens noticeably slow, with first frames of 200–300 ms. diff --git a/crates/clipbin-desktop/Cargo.toml b/crates/clipbin-desktop/Cargo.toml index 52f7745..cf3c508 100644 --- a/crates/clipbin-desktop/Cargo.toml +++ b/crates/clipbin-desktop/Cargo.toml @@ -11,6 +11,7 @@ publish.workspace = true [dependencies] clipbin-core.workspace = true clipbin-media.workspace = true +image = { version = "0.25.10", default-features = false } clipbin-obs.workspace = true gpui.workspace = true diff --git a/crates/clipbin-desktop/src/app.rs b/crates/clipbin-desktop/src/app.rs index ac23053..73a1eb2 100644 --- a/crates/clipbin-desktop/src/app.rs +++ b/crates/clipbin-desktop/src/app.rs @@ -62,6 +62,7 @@ pub struct ClipbinApp { pub settings: Settings, pub recorder: Recorder, pub editor: Editor, + pub playback: crate::playback::Playback, /// A short message for actions that are not connected yet. pub notice: Option, pub drag: Option, @@ -204,7 +205,7 @@ impl ClipbinApp { let library = if sample { Library::sample(SystemTime::now()) } else { - Library::empty(settings.unprotected_session_limit_bytes) + service.snapshot().library }; let recorder = match (sample, std::env::var("CLIPBIN_SAMPLE_STATUS").as_deref()) { (_, Ok("failed")) => Recorder { @@ -237,6 +238,7 @@ impl ClipbinApp { selection: (Duration::ZERO, Duration::ZERO), playing: false, }, + playback: crate::playback::Playback::default(), notice: None, drag: None, timeline_bounds: Rc::default(), @@ -312,6 +314,7 @@ impl ClipbinApp { }) .detach(); } + Self::start_playback_updates(window, cx); app } @@ -323,7 +326,11 @@ impl ClipbinApp { self.notice = None; match route { Route::Session(id) => self.open_session(id), - route => self.route = route, + route => { + self.playback.player.close(); + self.editor.playing = false; + self.route = route; + } } cx.notify(); } @@ -347,6 +354,19 @@ impl ClipbinApp { selection, playing: false, }; + if let Ok(store) = clipbin_core::store::Store::discover() { + let preview_time = playhead.min( + session + .length + .saturating_sub(Duration::from_nanos(16_666_667)), + ); + self.editor.playhead = preview_time; + if session.bookmarks.is_empty() { + self.playback.player.open_latest(store, id); + } else { + self.playback.player.open(store, id, preview_time); + } + } self.route = Route::Session(id); } diff --git a/crates/clipbin-desktop/src/main.rs b/crates/clipbin-desktop/src/main.rs index 0e886e1..4b75c1c 100644 --- a/crates/clipbin-desktop/src/main.rs +++ b/crates/clipbin-desktop/src/main.rs @@ -7,6 +7,7 @@ mod app; mod assets; mod components; mod exports; +mod playback; mod recorder_service; mod screens; mod storage; @@ -49,6 +50,7 @@ fn main() { }); cx.bind_keys([KeyBinding::new("ctrl-q", Quit, None)]); text_field::bind_keys(cx); + playback::bind_keys(cx); let bounds = Bounds::centered(None, size(px(1360.), px(860.)), cx); let result = cx.open_window( @@ -67,10 +69,13 @@ fn main() { }, ); - if let Err(error) = result { - eprintln!("Could not open Clipbin: {error}"); - cx.quit(); - return; + match result { + Ok(window) => playback::bind_actions(window, cx), + Err(error) => { + eprintln!("Could not open Clipbin: {error}"); + cx.quit(); + return; + } } cx.activate(true); diff --git a/crates/clipbin-desktop/src/playback.rs b/crates/clipbin-desktop/src/playback.rs new file mode 100644 index 0000000..adecc2a --- /dev/null +++ b/crates/clipbin-desktop/src/playback.rs @@ -0,0 +1,279 @@ +//! GPUI image upload and controls; file I/O and decoding belong to the player. +use crate::app::{ClipbinApp, Route}; +use clipbin_core::store::Store; +use clipbin_media::{ + player::{Picture, Player}, + thumbnail::Thumbnails, +}; +use gpui::{Context, RenderImage}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; + +pub struct Playback { + pub player: Player, + pub image: Option>, + pub message: Option, + revision: u64, + last_frame: Option>, + pub muted: bool, + pub volume: f32, + pub audio_error: Option, + pub preview_height: f32, + pub thumbnails: HashMap, + pub clip_thumbnails: HashMap, +} +impl Default for Playback { + fn default() -> Self { + Self { + player: Player::new(), + image: None, + message: Some("Loading footage…".into()), + revision: 0, + last_frame: None, + muted: false, + volume: 1., + audio_error: None, + preview_height: 416., + thumbnails: HashMap::new(), + clip_thumbnails: HashMap::new(), + } + } +} +impl ClipbinApp { + pub fn seek_playback(&mut self, time: Duration) { + self.editor.playhead = time; + self.playback.player.seek(time); + } + pub fn toggle_playback(&mut self) { + if let Route::Session(id) = self.route { + if self + .library + .session(id) + .is_some_and(|session| self.editor.playhead >= session.length) + { + self.seek_playback(Duration::ZERO); + } + self.editor.playing = !self.editor.playing; + self.playback.player.play(self.editor.playing); + } + } + pub fn start_playback_updates(window: &gpui::Window, cx: &mut Context) { + cx.spawn_in(window, async |entity, cx| { + loop { + gpui::Timer::after(Duration::from_millis(16)).await; + if entity + .update_in(cx, |app, window, cx| { + if !matches!(app.route, Route::Session(_)) { + if let Some(image) = app.playback.image.take() { + let _ = window.drop_image(image); + } + app.playback.last_frame = None; + return; + } + let snapshot = app.playback.player.snapshot(); + if snapshot.revision == app.playback.revision { + return; + } + app.playback.revision = snapshot.revision; + app.editor.playhead = snapshot.time; + app.editor.playing = snapshot.playing; + app.playback.audio_error = snapshot.audio_error; + match snapshot.picture { + Picture::Frame(frame) => { + // Swscale already produced GPUI-native BGRA off the UI thread. + if app + .playback + .last_frame + .as_ref() + .is_none_or(|last| !Arc::ptr_eq(last, &frame)) + { + let upload_started = std::time::Instant::now(); + let bytes = frame.bgra.clone(); + if let Some(buffer) = + image::RgbaImage::from_raw(frame.width, frame.height, bytes) + { + let image = + Arc::new(RenderImage::new(vec![image::Frame::new( + buffer, + )])); + if let Some(old) = app.playback.image.replace(image) { + let _ = window.drop_image(old); + } + if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() { + eprintln!( + "playback_ui upload_ms={:.3} width={} height={}", + upload_started.elapsed().as_secs_f64() * 1000., + frame.width, + frame.height + ); + } + } + } + app.playback.last_frame = Some(frame); + app.playback.message = None; + } + Picture::Loading => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = Some("Loading footage…".into()); + } + Picture::Gap => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = + Some("No footage (not captured / cleaned up)".into()); + } + Picture::Error(error) => { + if let Some(old) = app.playback.image.take() { + let _ = window.drop_image(old); + } + app.playback.last_frame = None; + app.playback.message = Some(error); + } + } + cx.notify(); + }) + .is_err() + { + break; + } + } + }) + .detach(); + // Discovery and thumbnail work never touch the UI thread's filesystem. + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let thumbnails = Thumbnails::new(); + loop { + let mut ready = HashMap::new(); + let mut clips = HashMap::new(); + if let Ok(store) = Store::discover() + && let Ok(metadata) = store.read() + { + for metadata in metadata { + if let Some(segment) = metadata + .segments + .iter() + .find(|segment| segment.complete && segment.available) + && let Some(path) = thumbnails + .request(&store.directory(metadata.session.id).join(&segment.path)) + { + ready.insert(metadata.session.id, path); + } + } + } + if let Ok(store) = Store::discover() + && let Some(root) = store.root.parent() + && let Ok(entries) = std::fs::read_dir(root.join("clips")) + { + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_some_and(|extension| extension == "mp4") + && let Some(id) = path + .file_stem() + .and_then(|id| id.to_str()) + .and_then(|id| id.parse().ok()) + && let Some(thumbnail) = thumbnails.request(&path) + { + clips.insert(id, thumbnail); + } + } + } + if matches!( + sender.try_send((ready, clips)), + Err(std::sync::mpsc::TrySendError::Disconnected(_)) + ) { + break; + } + std::thread::sleep(Duration::from_secs(1)); + } + }); + cx.spawn(async move |entity, cx| { + loop { + gpui::Timer::after(Duration::from_millis(250)).await; + let ready = receiver.try_recv().ok(); + if entity + .update(cx, |app, cx| { + if let Some((sessions, clips)) = ready + && (app.playback.thumbnails != sessions + || app.playback.clip_thumbnails != clips) + { + app.playback.thumbnails = sessions; + app.playback.clip_thumbnails = clips; + cx.notify(); + } + }) + .is_err() + { + break; + } + } + }) + .detach(); + } +} + +gpui::actions!(playback, [TogglePlayback, StepBack, StepForward]); +pub fn bind_keys(cx: &mut gpui::App) { + cx.bind_keys([ + gpui::KeyBinding::new("space", TogglePlayback, None), + gpui::KeyBinding::new("k", TogglePlayback, None), + gpui::KeyBinding::new("left", StepBack, None), + gpui::KeyBinding::new("j", StepBack, None), + gpui::KeyBinding::new("right", StepForward, None), + gpui::KeyBinding::new("l", StepForward, None), + ]); + cx.bind_keys( + ["space", "k", "j", "l"] + .map(|key| gpui::KeyBinding::new(key, gpui::NoAction {}, Some("TextField"))), + ); +} +/// Window actions also work when a mouse click leaves no focused GPUI element. +/// Text-field contexts keep their own typing and arrow bindings. +pub fn bind_actions(window: gpui::WindowHandle, cx: &mut gpui::App) { + // The event dispatcher already borrows the window; update its entity directly. + let editor = window.entity(cx).expect("editor window").downgrade(); + let toggle = editor.clone(); + cx.on_action(move |_: &TogglePlayback, cx| { + let _ = toggle.update(cx, |app, cx| { + app.toggle_playback(); + cx.notify(); + }); + }); + let back = editor.clone(); + cx.on_action(move |_: &StepBack, cx| { + let _ = back.update(cx, |app, cx| { + app.step_playback(false); + cx.notify(); + }); + }); + cx.on_action(move |_: &StepForward, cx| { + let _ = editor.update(cx, |app, cx| { + app.step_playback(true); + cx.notify(); + }); + }); +} + +impl ClipbinApp { + pub fn step_playback(&mut self, forward: bool) { + if !matches!(self.route, Route::Session(_)) { + return; + } + self.editor.playing = false; + self.playback.player.play(false); + let snapshot = self.playback.player.snapshot(); + let step = Duration::from_secs_f64(1. / snapshot.frames_per_second.max(1) as f64); + let time = snapshot.video_time.unwrap_or(self.editor.playhead); + // Round towards the selected frame when nanoseconds cannot represent a frame period. + let at = if forward { + time + step + } else { + time.saturating_sub(step) + } + Duration::from_nanos(2); + self.seek_playback(at); + } +} diff --git a/crates/clipbin-desktop/src/screens/clips.rs b/crates/clipbin-desktop/src/screens/clips.rs index 0f2b17b..fa0b449 100644 --- a/crates/clipbin-desktop/src/screens/clips.rs +++ b/crates/clipbin-desktop/src/screens/clips.rs @@ -92,7 +92,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context) -> AnyElement { .flex_col() .gap_3() .child( - thumbnail(&clip.game).child( + thumbnail(&clip.game, app.playback.clip_thumbnails.get(&clip.id)).child( corner() .bottom_2p5() .right_2p5() diff --git a/crates/clipbin-desktop/src/screens/editor.rs b/crates/clipbin-desktop/src/screens/editor.rs index 6e1fb9d..f249c41 100644 --- a/crates/clipbin-desktop/src/screens/editor.rs +++ b/crates/clipbin-desktop/src/screens/editor.rs @@ -6,15 +6,14 @@ use clipbin_core::library::{ Session, SpanKind, export_range, format_ago, format_clock, format_length, }; use gpui::{ - AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, IntoElement, MouseButton, - MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, ScrollWheelEvent, canvas, div, - pattern_slash, prelude::*, px, relative, + AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, ImageSource, IntoElement, + MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ObjectFit, Pixels, Point, + ScrollWheelEvent, canvas, div, img, pattern_slash, prelude::*, px, relative, }; use crate::{ app::{ClipbinApp, Drag, Route}, components::{ButtonStyle, aspect_ratio, button, chip, icon, panel, row}, - screens::backdrop, theme, timeline_view::TimelineView, }; @@ -73,37 +72,54 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ); let preview = aspect_ratio(div(), 16. / 9.) + .h(px(app.playback.preview_height)) + .flex_shrink_0() .relative() .w_full() .overflow_hidden() .rounded_lg() .bg(gpui::black()) - .child(backdrop(&session.game).absolute().size_full().opacity(0.55)) - .child( + .children(app.playback.image.clone().map(|image| { + img(ImageSource::Render(image)) + .absolute() + .size_full() + .object_fit(ObjectFit::Contain) + })) + .children(app.playback.message.clone().map(|message| { div() .absolute() .size_full() .flex() - .flex_col() .items_center() .justify_center() - .gap_3() - .child( - div() - .flex() - .items_center() - .justify_center() - .size(px(56.)) - .rounded_full() - .bg(theme::overlay()) - .child(icon("play").size_6().text_color(theme::foreground())), - ) - .child( - div() - .text_xs() - .text_color(theme::muted()) - .child("Preview arrives with playback"), - ), + .text_xs() + .text_color(theme::muted()) + .child(message) + })) + .child( + canvas( + { + let player = cx.entity().downgrade(); + move |bounds, window, cx| { + let scale = window.scale_factor(); + let size = ( + (f32::from(bounds.size.width) * scale) as u32, + (f32::from(bounds.size.height) * scale) as u32, + ); + let _ = player.update(cx, |app, cx| { + app.playback.player.resize(size); + let height = f32::from(bounds.size.width) * 9. / 16.; + if (app.playback.preview_height - height).abs() > 1. { + app.playback.preview_height = height; + cx.notify(); + } + }); + } + }, + |_, _, _, _| {}, + ) + .absolute() + .size_full(), ) .child( div() @@ -126,6 +142,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .map(|b| b.at); let transport = div() .flex() + .flex_wrap() .items_center() .gap_1() .child( @@ -136,8 +153,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle None, ) .on_click(cx.listener(|app, _: &ClickEvent, _, cx| { - app.notice = - Some("Playback isn't connected yet; it comes with the decoder.".into()); + app.toggle_playback(); cx.notify(); })), ) @@ -150,7 +166,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { if let Some(at) = previous_bookmark { - app.editor.playhead = at; + app.seek_playback(at); cx.notify(); } })), @@ -164,7 +180,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { if let Some(at) = next_bookmark { - app.editor.playhead = at; + app.seek_playback(at); cx.notify(); } })), @@ -187,6 +203,52 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .child(format!("/ {}", format_clock(length))), ), ) + .child( + button( + "mute", + ButtonStyle::Ghost, + Some(if app.playback.muted { + "volume-x" + } else { + "volume-2" + }), + None, + ) + .on_click(cx.listener(|app, _: &ClickEvent, _, cx| { + app.playback.muted = !app.playback.muted; + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + })), + ) + .child( + button("volume-down", ButtonStyle::Ghost, None, Some("−")).on_click(cx.listener( + |app, _: &ClickEvent, _, cx| { + app.playback.volume = (app.playback.volume - 0.1).max(0.); + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + }, + )), + ) + .child( + div() + .text_xs() + .child(format!("{:.0}%", app.playback.volume * 100.)), + ) + .child( + button("volume-up", ButtonStyle::Ghost, None, Some("+")).on_click(cx.listener( + |app, _: &ClickEvent, _, cx| { + app.playback.volume = (app.playback.volume + 0.1).min(1.); + app.playback + .player + .audio(app.playback.muted, app.playback.volume); + cx.notify(); + }, + )), + ) .child(div().flex_1()) .child( div() @@ -210,6 +272,12 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle .gap_3() .child(preview) .child(transport) + .children( + app.playback + .audio_error + .clone() + .map(|error| div().text_xs().text_color(theme::warning()).child(error)), + ) .child(timeline(app, session, cx)); let (export_start, export_end) = app @@ -337,7 +405,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context) -> AnyEle ) }) .on_click(cx.listener(move |app, _: &ClickEvent, _, cx| { - app.editor.playhead = at; + app.seek_playback(at); app.editor.selection = range; cx.notify(); })) @@ -546,9 +614,25 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - .size_full() .overflow_hidden() .child( - canvas(move |area, _, _| bounds.set(Some(area)), |_, _, _, _| {}) - .absolute() - .size_full(), + canvas( + move |area, _, _| { + if bounds.get() != Some(area) + && std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() + { + eprintln!( + "playback_timeline x={} y={} width={} height={}", + f32::from(area.left()), + f32::from(area.top()), + f32::from(area.size.width), + f32::from(area.size.height) + ); + } + bounds.set(Some(area)); + }, + |_, _, _, _| {}, + ) + .absolute() + .size_full(), ) .child( div() @@ -582,7 +666,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - } else if near(end) { Drag::SelectionEnd } else { - app.editor.playhead = time; + app.seek_playback(time); Drag::Playhead }); cx.notify(); @@ -600,7 +684,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context) - let (start, end) = app.editor.selection; let minimum = Duration::from_secs(1); match drag { - Drag::Playhead => app.editor.playhead = time, + Drag::Playhead => app.seek_playback(time), Drag::SelectionStart => { app.editor.selection.0 = time.min(end.saturating_sub(minimum)) } diff --git a/crates/clipbin-desktop/src/screens/mod.rs b/crates/clipbin-desktop/src/screens/mod.rs index f3f1700..7faafb2 100644 --- a/crates/clipbin-desktop/src/screens/mod.rs +++ b/crates/clipbin-desktop/src/screens/mod.rs @@ -9,7 +9,20 @@ use crate::{components::aspect_ratio, theme}; /// A 16:9 thumbnail. Until frames are decoded, each game gets its own colour /// with its initials, so cards stay recognisable. -pub fn thumbnail(game: &str) -> Div { +pub fn thumbnail(game: &str, path: Option<&std::path::PathBuf>) -> Div { + if let Some(path) = path { + return aspect_ratio(div(), 16. / 9.) + .relative() + .w_full() + .overflow_hidden() + .rounded_xl() + .child( + gpui::img(path.clone()) + .absolute() + .size_full() + .object_fit(gpui::ObjectFit::Cover), + ); + } let tint = game_tint(game); let initials: String = game .split_whitespace() diff --git a/crates/clipbin-desktop/src/screens/sessions.rs b/crates/clipbin-desktop/src/screens/sessions.rs index 46343d2..7932d1e 100644 --- a/crates/clipbin-desktop/src/screens/sessions.rs +++ b/crates/clipbin-desktop/src/screens/sessions.rs @@ -61,7 +61,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context) -> AnyElement { .cursor_pointer() .group("card") .child( - thumbnail(&session.game) + thumbnail(&session.game, app.playback.thumbnails.get(&session.id)) .border_1() .border_color(gpui::transparent_black()) .group_hover("card", |style| { diff --git a/crates/clipbin-media/Cargo.toml b/crates/clipbin-media/Cargo.toml index 12934c9..a23ed24 100644 --- a/crates/clipbin-media/Cargo.toml +++ b/crates/clipbin-media/Cargo.toml @@ -11,6 +11,14 @@ publish.workspace = true [dependencies] clipbin-core.workspace = true ffmpeg-next = { version = "9", default-features = false, features = ["codec", "format", "software-scaling", "software-resampling"] } +image = { version = "0.25.10", default-features = false, features = ["png"] } +cpal = { version = "0.18.2", features = ["pulseaudio"] } -[lints] -workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { version = "1", features = ["process", "thread"] } + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_System_Threading"] } + +[lints.rust] +unsafe_code = "deny" diff --git a/crates/clipbin-media/examples/playback-check.rs b/crates/clipbin-media/examples/playback-check.rs new file mode 100644 index 0000000..fe7c960 --- /dev/null +++ b/crates/clipbin-media/examples/playback-check.rs @@ -0,0 +1,67 @@ +//! Headless timing check against real segments; run inside the rig for device audio. +use clipbin_core::store::Store; +use clipbin_media::player::{Picture, Player}; +use std::{ + thread, + time::{Duration, Instant}, +}; +fn main() { + let directory = std::env::args().nth(1).expect("session root"); + let id = std::env::args() + .nth(2) + .unwrap_or_else(|| "1".into()) + .parse() + .unwrap(); + let player = Player::new(); + player.resize((800, 450)); + player.open( + Store { + root: directory.into(), + }, + id, + Duration::ZERO, + ); + thread::sleep(Duration::from_millis(250)); + player.play(true); + let begin = Instant::now(); + let mut last = None; + let mut frames = 0; + for _ in 0..700 { + thread::sleep(Duration::from_millis(10)); + let snapshot = player.snapshot(); + assert!(snapshot.audio_error.is_none(), "{:?}", snapshot.audio_error); + if let Picture::Frame(frame) = snapshot.picture { + if let Some(video_time) = snapshot.video_time { + assert!( + video_time.abs_diff(snapshot.time) <= Duration::from_millis(20), + "video out of sync: {video_time:?} vs {:?}", + snapshot.time + ); + } + if last + .as_ref() + .is_none_or(|previous| !std::sync::Arc::ptr_eq(previous, &frame)) + { + frames += 1; + } + last = Some(frame); + } + } + let snapshot = player.snapshot(); + println!( + "playback_check wall={:.3} session={:.3} displayed_frames={frames}", + begin.elapsed().as_secs_f64(), + snapshot.time.as_secs_f64() + ); + assert!( + snapshot.time >= Duration::from_secs(6), + "audio master clock stalled: {snapshot:?}" + ); + player.audio(false, 0.5); + thread::sleep(Duration::from_secs(1)); + player.audio(true, 0.5); + let before = snapshot.time; + thread::sleep(Duration::from_secs(1)); + assert!(player.snapshot().time > before, "muted wall clock stalled"); + player.close(); +} diff --git a/crates/clipbin-media/src/audio.rs b/crates/clipbin-media/src/audio.rs new file mode 100644 index 0000000..9580e90 --- /dev/null +++ b/crates/clipbin-media/src/audio.rs @@ -0,0 +1,454 @@ +//! Combined-track decoding and bounded device output. No decoder runs in a callback. +use crate::ffmpeg::{self, format, media, software::resampling, util::frame::Audio as AudioFrame}; +use clipbin_core::store::Metadata; +use cpal::{ + FromSample, SizedSample, + traits::{DeviceTrait, HostTrait, StreamTrait}, +}; +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +struct Block { + time: Duration, + samples: Vec, + consumed: usize, +} +struct Buffer { + blocks: VecDeque, + queued: usize, + playing: bool, + volume: f32, + // The time of the first sample actually scheduled for the device. + anchors: VecDeque<(Duration, Instant, Duration)>, + error: Option, +} +impl Buffer { + fn time(&mut self, now: Instant) -> Option { + while self.anchors.len() > 1 && self.anchors[1].1 <= now { + self.anchors.pop_front(); + } + self.anchors.front().and_then(|(time, begins, span)| { + (now <= *begins + *span + Duration::from_millis(100)) + .then(|| *time + now.saturating_duration_since(*begins).min(*span)) + }) + } +} +pub struct Audio { + _stream: cpal::Stream, + buffer: Arc>, + rate: u32, + channels: u16, + decoder: Option<(PathBuf, AudioDecoder)>, + next: Duration, +} +impl Audio { + pub fn new() -> Result { + #[cfg(target_os = "linux")] + if let Ok(host) = cpal::host_from_id(cpal::HostId::PulseAudio) { + match Self::with_host(host) { + Ok(audio) => return Ok(audio), + Err(error) => eprintln!("PulseAudio output unavailable: {error}; trying ALSA"), + } + } + Self::with_host(cpal::default_host()) + } + fn with_host(host: cpal::Host) -> Result { + let device = host + .default_output_device() + .ok_or("No audio output device")?; + let supported = device.default_output_config().map_err(|e| e.to_string())?; + let format = supported.sample_format(); + let config: cpal::StreamConfig = supported.into(); + let rate = config.sample_rate; + let channels = config.channels; + let buffer = Arc::new(Mutex::new(Buffer { + blocks: VecDeque::new(), + queued: 0, + playing: false, + volume: 1., + anchors: VecDeque::new(), + error: None, + })); + let stream = match format { + cpal::SampleFormat::F32 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::I16 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::U16 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::I32 => output::(&device, config, buffer.clone()), + cpal::SampleFormat::F64 => output::(&device, config, buffer.clone()), + _ => return Err(format!("Unsupported audio device format: {format}")), + } + .map_err(|e| e.to_string())?; + stream.play().map_err(|e| e.to_string())?; + eprintln!("playback_audio rate={rate} channels={channels} format={format}"); + Ok(Self { + _stream: stream, + buffer, + rate, + channels, + decoder: None, + next: Duration::ZERO, + }) + } + pub fn reset(&mut self, time: Duration) { + let mut buffer = self.buffer.lock().unwrap(); + buffer.blocks.clear(); + buffer.queued = 0; + buffer.anchors.clear(); + self.decoder = None; + self.next = time; + } + pub fn controls(&self, playing: bool, volume: f32) { + let mut buffer = self.buffer.lock().unwrap(); + buffer.playing = playing; + buffer.volume = volume.clamp(0., 1.); + if !playing { + buffer.anchors.clear(); + } + } + /// Device time, corrected for callback-to-DAC latency; bounded by scheduled samples. + pub fn time(&self, now: Instant) -> Option { + self.buffer.lock().unwrap().time(now) + } + pub fn error(&self) -> Option { + self.buffer.lock().unwrap().error.clone() + } + pub fn catch_up(&mut self, time: Duration) { + if self.next + Duration::from_millis(50) < time && self.buffer.lock().unwrap().queued == 0 { + self.reset(time); + } + } + pub fn fill(&mut self, metadata: &Metadata) -> Result<(), ffmpeg::Error> { + // At most 250 ms buffered. All allocations/decoding happen on the player thread. + let limit = self.rate as usize * self.channels as usize / 4; + for _ in 0..16 { + if self.buffer.lock().unwrap().queued >= limit { + break; + } + let Some(at) = metadata.locate(self.next).filter(|at| at.segment.complete) else { + break; + }; + if self + .decoder + .as_ref() + .is_none_or(|(path, _)| path != &at.segment.path) + { + self.decoder = Some(( + at.segment.path.clone(), + AudioDecoder::open(&at.segment.path, self.rate, self.channels, at.offset)?, + )); + } + let (_, decoder) = self.decoder.as_mut().unwrap(); + let Some((offset, mut samples)) = decoder.next()? else { + self.next = at.segment.end; + self.decoder = None; + continue; + }; + let begin = at.segment.start + offset; + let skip = ((self.next.saturating_sub(begin).as_secs_f64() * self.rate as f64).round() + as usize) + .min(samples.len() / self.channels as usize); + samples.drain(..skip * self.channels as usize); + let time = begin + Duration::from_secs_f64(skip as f64 / self.rate as f64); + let keep = ((at.segment.end.saturating_sub(time).as_secs_f64() * self.rate as f64) + .round() as usize) + * self.channels as usize; + samples.truncate(keep); + if samples.is_empty() { + continue; + } + self.next = time + + Duration::from_secs_f64( + (samples.len() / self.channels as usize) as f64 / self.rate as f64, + ); + if at.segment.end.saturating_sub(self.next) + < Duration::from_nanos(1_000_000_000 / self.rate as u64) + { + self.next = at.segment.end; + } + let mut buffer = self.buffer.lock().unwrap(); + buffer.queued += samples.len(); + buffer.blocks.push_back(Block { + time, + samples, + consumed: 0, + }); + } + Ok(()) + } +} +fn output>( + device: &cpal::Device, + config: cpal::StreamConfig, + buffer: Arc>, +) -> Result { + let errors = buffer.clone(); + let rate = config.sample_rate; + let channels = config.channels as usize; + device.build_output_stream( + config, + move |data: &mut [T], info: &cpal::OutputCallbackInfo| { + // try_lock guarantees UI/decoder contention cannot block the device callback. + data.fill(T::from_sample(0.)); + let Ok(mut buffer) = buffer.try_lock() else { + return; + }; + if !buffer.playing { + return; + } + let latency = info + .timestamp() + .playback + .duration_since(info.timestamp().callback); + let mut scheduled = None; + for (frame_number, output) in data.chunks_exact_mut(channels).enumerate() { + let volume = buffer.volume; + let Some(block) = buffer.blocks.front_mut() else { + break; + }; + let time = block.time + + Duration::from_secs_f64((block.consumed / channels) as f64 / rate as f64); + scheduled.get_or_insert(( + time, + Instant::now() + + latency + + Duration::from_secs_f64(frame_number as f64 / rate as f64), + Duration::ZERO, + )); + for sample in output { + *sample = + T::from_sample((block.samples[block.consumed] * volume).clamp(-1., 1.)); + block.consumed += 1; + } + let done = block.consumed == block.samples.len(); + buffer.queued -= channels; + if let Some((_, _, span)) = scheduled.as_mut() { + *span += Duration::from_secs_f64(1. / rate as f64); + } + if done { + buffer.blocks.pop_front(); + } + } + if let Some(scheduled) = scheduled { + if buffer.anchors.len() == 32 { + buffer.anchors.pop_front(); + } + buffer.anchors.push_back(scheduled); + } + }, + move |error| { + if let Ok(mut buffer) = errors.lock() { + buffer.error = Some(format!("Audio output failed: {error}")); + } + }, + None, + ) +} + +struct AudioDecoder { + input: format::context::Input, + decoder: ffmpeg::decoder::Audio, + resampler: resampling::Context, + stream: usize, + time_base: f64, + origin: i64, + rate: u32, + channels: u16, + eof: bool, + next_time: Duration, + initialised_time: bool, + drained: bool, +} +impl AudioDecoder { + fn open(path: &Path, rate: u32, channels: u16, seek: Duration) -> Result { + crate::init(); + let mut input = format::input(path)?; + // OBS writes system, mic, combined, in that order; a single track is already mixed. + let tracks: Vec<_> = input + .streams() + .filter(|stream| stream.parameters().medium() == media::Type::Audio) + .collect(); + let track = if tracks.len() >= 3 { + &tracks[2] + } else { + tracks.first().ok_or(ffmpeg::Error::StreamNotFound)? + }; + let stream = track.index(); + let time_base = f64::from(track.time_base()); + let origin = track.start_time().max(0); + let decoder = ffmpeg::codec::context::Context::from_parameters(track.parameters())? + .decoder() + .audio()?; + let layout = if decoder.channel_layout().channels() == 0 { + ffmpeg::ChannelLayout::default(decoder.channels() as i32) + } else { + decoder.channel_layout() + }; + let resampler = resampling::Context::get( + decoder.format(), + layout, + decoder.rate(), + format::Sample::F32(format::sample::Type::Packed), + ffmpeg::ChannelLayout::default(channels as i32), + rate, + )?; + if seek > Duration::ZERO { + let timestamp = (seek.as_secs_f64() * 1_000_000.) as i64; + input.seek(timestamp, ..timestamp)?; + } + Ok(Self { + input, + decoder, + resampler, + stream, + time_base, + origin, + rate, + channels, + eof: false, + next_time: seek, + initialised_time: false, + drained: false, + }) + } + fn next(&mut self) -> Result)>, ffmpeg::Error> { + loop { + let mut frame = AudioFrame::empty(); + match self.decoder.receive_frame(&mut frame) { + Ok(()) => { + let timestamp = frame.timestamp().map_or(self.next_time, |pts| { + Duration::from_secs_f64((pts - self.origin).max(0) as f64 * self.time_base) + }); + let time = if self.initialised_time { + self.next_time + } else { + timestamp + }; + self.initialised_time = true; + // Allocate enough output for upsampling; ffmpeg-next's empty output uses input count. + let count = ((frame.samples() as u64 * self.rate as u64) + .div_ceil(frame.rate() as u64) + + 256) as usize; + let mut converted = AudioFrame::new( + format::Sample::F32(format::sample::Type::Packed), + count, + ffmpeg::ChannelLayout::default(self.channels as i32), + ); + converted.set_rate(self.rate); + self.resampler.run(&frame, &mut converted)?; + let samples = converted.plane::(0).to_vec(); + self.next_time = time + + Duration::from_secs_f64(converted.samples() as f64 / self.rate as f64); + return Ok(Some((time, samples))); + } + Err(ffmpeg::Error::Eof) => { + if self.drained { + return Ok(None); + } + self.drained = true; + let mut converted = AudioFrame::new( + format::Sample::F32(format::sample::Type::Packed), + 256, + ffmpeg::ChannelLayout::default(self.channels as i32), + ); + converted.set_rate(self.rate); + self.resampler.flush(&mut converted)?; + return if converted.samples() == 0 { + Ok(None) + } else { + Ok(Some((self.next_time, converted.plane::(0).to_vec()))) + }; + } + Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {} + Err(error) => return Err(error), + } + if self.eof { + return Ok(None); + } + let mut packet = ffmpeg::Packet::empty(); + loop { + match packet.read(&mut self.input) { + Ok(()) if packet.stream() == self.stream => { + self.decoder.send_packet(&packet)?; + break; + } + Ok(()) => {} + Err(ffmpeg::Error::Eof) => { + self.eof = true; + self.decoder.send_eof()?; + break; + } + Err(error) => return Err(error), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn device_clock_waits_for_latency_and_stops_at_scheduled_samples() { + let now = Instant::now(); + let mut buffer = Buffer { + blocks: VecDeque::new(), + queued: 0, + playing: true, + volume: 1., + anchors: VecDeque::new(), + error: None, + }; + buffer.anchors.push_back(( + Duration::from_secs(2), + now + Duration::from_millis(100), + Duration::from_millis(50), + )); + buffer.anchors.push_back(( + Duration::from_millis(2050), + now + Duration::from_millis(150), + Duration::from_millis(50), + )); + assert_eq!(buffer.time(now), Some(Duration::from_secs(2))); + assert_eq!( + buffer.time(now + Duration::from_millis(125)), + Some(Duration::from_millis(2025)) + ); + assert_eq!( + buffer.time(now + Duration::from_millis(200)), + Some(Duration::from_millis(2100)) + ); + assert_eq!(buffer.anchors.len(), 1); + assert_eq!(buffer.time(now + Duration::from_secs(5)), None); + } + #[test] + fn resampling_selects_combined_track_and_preserves_its_tone() { + let Some(directory) = std::env::var_os("CLIPBIN_TEST_MEDIA_DIR") else { + return; + }; + let mut decoder = AudioDecoder::open( + &PathBuf::from(directory).join("combined.mkv"), + 44_100, + 1, + Duration::ZERO, + ) + .unwrap(); + let mut samples = Vec::new(); + while let Some((_, block)) = decoder.next().unwrap() { + samples.extend(block); + } + assert!(samples.len() > 80_000 && samples.len() < 95_000); + let crossings = samples + .windows(2) + .filter(|pair| pair[0] <= 0. && pair[1] > 0.) + .count(); + let hz = crossings as f64 / (samples.len() as f64 / 44_100.); + assert!( + (435. ..445.).contains(&hz), + "combined track was not the 440 Hz tone: {hz}" + ); + } +} diff --git a/crates/clipbin-media/src/decode.rs b/crates/clipbin-media/src/decode.rs new file mode 100644 index 0000000..bab48a2 --- /dev/null +++ b/crates/clipbin-media/src/decode.rs @@ -0,0 +1,203 @@ +//! Single-threaded file decoding. Scale only frames which will be displayed. +use crate::ffmpeg::{self, codec, format, media, software::scaling, util::frame::video::Video}; +use std::{path::Path, time::Duration}; + +#[derive(Clone, Debug)] +pub struct Frame { + pub width: u32, + pub height: u32, + /// Packed BGRA, without FFmpeg's row padding. + pub bgra: Vec, +} + +pub struct Decoder { + input: format::context::Input, + decoder: ffmpeg::decoder::Video, + stream: usize, + time_base: f64, + origin: i64, + fps: u32, + next_index: u64, + eof: bool, + scaler: Option<(u32, u32, format::Pixel, u32, u32, scaling::context::Context)>, +} + +impl Decoder { + pub fn open(path: &Path, fps: u32) -> Result { + crate::init(); + let input = format::input(path)?; + let video = input + .streams() + .best(media::Type::Video) + .ok_or(ffmpeg::Error::StreamNotFound)?; + let stream = video.index(); + let time_base = f64::from(video.time_base()); + let origin = video.start_time().max(0); + let mut context = codec::context::Context::from_parameters(video.parameters())?; + // Never let FFmpeg create a pool competing with the recorder. + context.set_threading(codec::threading::Config::count(1)); + let decoder = context.decoder().video()?; + Ok(Self { + input, + decoder, + stream, + time_base, + origin, + fps: fps.max(1), + next_index: 0, + eof: false, + scaler: None, + }) + } + + /// Reuse swscale's setup while the preview dimensions stay unchanged. + pub fn scale(&mut self, video: &Video, target: (u32, u32)) -> Result { + let (width, height) = fitted_size((video.width(), video.height()), target); + if self + .scaler + .as_ref() + .is_none_or(|(w, h, pixel, source_w, source_h, _)| { + (*w, *h) != (width, height) + || *pixel != video.format() + || (*source_w, *source_h) != (video.width(), video.height()) + }) + { + self.scaler = Some(( + width, + height, + video.format(), + video.width(), + video.height(), + scaling::context::Context::get( + video.format(), + video.width(), + video.height(), + format::Pixel::BGRA, + width, + height, + scaling::flag::Flags::FAST_BILINEAR, + )?, + )); + } + let (_, _, _, _, _, scaler) = self.scaler.as_mut().unwrap(); + packed_frame(scaler, video, width, height) + } + + /// FFmpeg seeks backwards to a keyframe; callers decode forward by frame index. + pub fn seek(&mut self, time: Duration) -> Result<(), ffmpeg::Error> { + let timestamp = (time.as_secs_f64() * 1_000_000. + + self.origin as f64 * self.time_base * 1_000_000.) as i64; + self.input.seek(timestamp, ..timestamp)?; + self.decoder.flush(); + self.next_index = frame_index(time, self.fps); + self.eof = false; + Ok(()) + } + + pub fn next_frame(&mut self) -> Result, ffmpeg::Error> { + loop { + let mut frame = Video::empty(); + match self.decoder.receive_frame(&mut frame) { + Ok(()) => { + let index = frame.timestamp().map_or(self.next_index, |pts| { + (((pts - self.origin).max(0) as f64 * self.time_base * self.fps as f64) + .round()) as u64 + }); + self.next_index = index + 1; + return Ok(Some((index, frame))); + } + Err(ffmpeg::Error::Eof) => return Ok(None), + Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {} + Err(error) => return Err(error), + } + if self.eof { + return Ok(None); + } + let mut packet = ffmpeg::Packet::empty(); + loop { + match packet.read(&mut self.input) { + Ok(()) if packet.stream() == self.stream => { + self.decoder.send_packet(&packet)?; + break; + } + Ok(()) => {} + Err(ffmpeg::Error::Eof) => { + self.eof = true; + self.decoder.send_eof()?; + break; + } + Err(error) => return Err(error), + } + } + } + } +} + +pub fn frame_index(time: Duration, fps: u32) -> u64 { + (time.as_nanos() * u128::from(fps.max(1)) / 1_000_000_000) as u64 +} + +/// Preserve aspect ratio and cap allocation even on a very large display. +pub fn fitted_size(source: (u32, u32), target: (u32, u32)) -> (u32, u32) { + let ratio = (target.0.clamp(1, 1920) as f64 / source.0.max(1) as f64) + .min(target.1.clamp(1, 1080) as f64 / source.1.max(1) as f64) + .min(1.); + ( + (source.0 as f64 * ratio).round().max(1.) as u32, + (source.1 as f64 * ratio).round().max(1.) as u32, + ) +} + +pub fn scale(video: &Video, target: (u32, u32)) -> Result { + let (width, height) = fitted_size((video.width(), video.height()), target); + let mut scaler = scaling::context::Context::get( + video.format(), + video.width(), + video.height(), + format::Pixel::BGRA, + width, + height, + scaling::flag::Flags::FAST_BILINEAR, + )?; + packed_frame(&mut scaler, video, width, height) +} + +fn packed_frame( + scaler: &mut scaling::context::Context, + video: &Video, + width: u32, + height: u32, +) -> Result { + let mut output = Video::empty(); + scaler.run(video, &mut output)?; + let mut bgra = Vec::with_capacity(width as usize * height as usize * 4); + for row in 0..height as usize { + let start = row * output.stride(0); + bgra.extend_from_slice(&output.data(0)[start..start + width as usize * 4]); + } + Ok(Frame { + width, + height, + bgra, + }) +} + +/// A thumbnail from the first decoded frame of any MP4/MKV path. +pub fn first_frame(path: &Path, target: (u32, u32)) -> Result { + let mut decoder = Decoder::open(path, 60)?; + let (_, frame) = decoder.next_frame()?.ok_or(ffmpeg::Error::Eof)?; + scale(&frame, target) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn frame_exact_seek_and_bounded_aspect_ratio() { + assert_eq!(frame_index(Duration::from_millis(1999), 60), 119); + assert_eq!(frame_index(Duration::from_secs(2), 60), 120); + assert_eq!(fitted_size((2560, 1440), (800, 450)), (800, 450)); + assert_eq!(fitted_size((1440, 2560), (800, 450)), (253, 450)); + assert_eq!(fitted_size((8000, 8000), (8000, 8000)), (1080, 1080)); + } +} diff --git a/crates/clipbin-media/src/lib.rs b/crates/clipbin-media/src/lib.rs index 7a6ff00..33affe0 100644 --- a/crates/clipbin-media/src/lib.rs +++ b/crates/clipbin-media/src/lib.rs @@ -68,4 +68,13 @@ mod tests { } } +pub mod audio; +pub mod decode; pub mod export; +pub mod player; +pub mod thumbnail; + +mod priority; +#[cfg(windows)] +#[allow(unsafe_code)] +mod windows_priority; diff --git a/crates/clipbin-media/src/player.rs b/crates/clipbin-media/src/player.rs new file mode 100644 index 0000000..38ec149 --- /dev/null +++ b/crates/clipbin-media/src/player.rs @@ -0,0 +1,459 @@ +//! On-demand background playback with a one-frame mailbox and coalesced controls. +use crate::decode::{self, Decoder, Frame}; +use clipbin_core::store::{Metadata, Store}; +use std::{ + path::PathBuf, + sync::{Arc, Condvar, Mutex}, + thread, + time::{Duration, Instant}, +}; + +#[derive(Clone, Debug)] +pub enum Picture { + Loading, + Frame(Arc), + Gap, + Error(String), +} +#[derive(Clone, Debug)] +pub struct Snapshot { + pub time: Duration, + pub playing: bool, + pub picture: Picture, + pub revision: u64, + pub audio_error: Option, + pub video_time: Option, + pub frames_per_second: u32, +} +struct Controls { + session: Option<(Store, u64)>, + seek: Option, + latest: bool, + playing: bool, + size: (u32, u32), + generation: u64, + muted: bool, + volume: f32, + quit: bool, + snapshot: Snapshot, +} +struct Shared { + state: Mutex, + wake: Condvar, +} + +pub struct Player { + shared: Arc, +} +impl Default for Player { + fn default() -> Self { + Self::new() + } +} +impl Player { + pub fn new() -> Self { + let shared = Arc::new(Shared { + state: Mutex::new(Controls { + session: None, + seek: None, + latest: false, + playing: false, + size: (800, 450), + generation: 0, + muted: false, + volume: 1., + quit: false, + snapshot: Snapshot { + time: Duration::ZERO, + playing: false, + picture: Picture::Loading, + revision: 0, + audio_error: None, + video_time: None, + frames_per_second: 60, + }, + }), + wake: Condvar::new(), + }); + let worker = shared.clone(); + thread::Builder::new() + .name("clipbin-player".into()) + .spawn(move || run(worker)) + .expect("player thread"); + Self { shared } + } + pub fn open(&self, store: Store, id: u64, time: Duration) { + self.open_session(store, id, time, false); + } + pub fn open_latest(&self, store: Store, id: u64) { + self.open_session(store, id, Duration::ZERO, true); + } + fn open_session(&self, store: Store, id: u64, time: Duration, latest: bool) { + let mut state = self.shared.state.lock().unwrap(); + state.session = Some((store, id)); + state.latest = latest; + state.seek = Some(time); + state.playing = false; + state.generation += 1; + state.snapshot.picture = Picture::Loading; + state.snapshot.revision += 1; + self.shared.wake.notify_one(); + } + pub fn close(&self) { + let mut state = self.shared.state.lock().unwrap(); + state.session = None; + state.playing = false; + state.generation += 1; + self.shared.wake.notify_one(); + } + pub fn seek(&self, time: Duration) { + let mut state = self.shared.state.lock().unwrap(); + state.seek = Some(time); + state.snapshot.time = time; + state.snapshot.picture = Picture::Loading; + state.snapshot.revision += 1; + state.generation += 1; + self.shared.wake.notify_one(); + } + pub fn play(&self, playing: bool) { + self.shared.state.lock().unwrap().playing = playing; + self.shared.wake.notify_one(); + } + pub fn audio(&self, muted: bool, volume: f32) { + let mut state = self.shared.state.lock().unwrap(); + if state.muted != muted { + state.seek = Some(state.snapshot.time); + state.generation += 1; + } + state.muted = muted; + state.volume = volume.clamp(0., 1.); + self.shared.wake.notify_one(); + } + pub fn resize(&self, size: (u32, u32)) { + let mut state = self.shared.state.lock().unwrap(); + if state.size != size { + state.size = size; + state.generation += 1; + self.shared.wake.notify_one(); + } + } + pub fn snapshot(&self) -> Snapshot { + self.shared.state.lock().unwrap().snapshot.clone() + } +} +impl Drop for Player { + fn drop(&mut self) { + self.shared.state.lock().unwrap().quit = true; + self.shared.wake.notify_one(); + } +} + +/// Monotonic session clock: decoding delays never accumulate into playback time. +#[derive(Debug)] +pub struct Clock { + base: Duration, + started: Instant, + playing: bool, +} +impl Clock { + pub fn new(now: Instant) -> Self { + Self { + base: Duration::ZERO, + started: now, + playing: false, + } + } + pub fn time(&self, now: Instant) -> Duration { + self.base + + if self.playing { + now.saturating_duration_since(self.started) + } else { + Duration::ZERO + } + } + pub fn set(&mut self, time: Duration, playing: bool, now: Instant) { + self.base = time; + self.started = now; + self.playing = playing; + } +} + +fn run(shared: Arc) { + crate::priority::lower(false); + let mut clock = Clock::new(Instant::now()); + let mut generation = u64::MAX; + let mut metadata: Option = None; + let mut refreshed = Instant::now(); + let mut decoder: Option<(PathBuf, Decoder, u64)> = None; + let mut audio: Option = None; + let mut audio_error = None; + let mut audio_attempted = false; + loop { + let (session, seek, playing, size, requested, muted, volume, latest) = { + let mut state = shared.state.lock().unwrap(); + if state.quit { + break; + } + ( + state.session.clone(), + state.seek.take(), + state.playing, + state.size, + state.generation, + state.muted, + state.volume, + std::mem::take(&mut state.latest), + ) + }; + let changed = requested != generation; + if changed { + generation = requested; + decoder = None; + } + let now = Instant::now(); + if changed || now.duration_since(refreshed) >= Duration::from_millis(250) { + metadata = session + .as_ref() + .and_then(|(store, id)| store.metadata(*id).ok()); + refreshed = now; + } + let seek = if latest { + metadata + .as_ref() + .and_then(|metadata| { + metadata + .segments + .iter() + .rev() + .find(|segment| { + segment.complete && segment.available && segment.end > segment.start + }) + .map(|segment| { + segment.end.saturating_sub(Duration::from_secs_f64( + 1. / segment.frames_per_second.max(1) as f64, + )) + }) + }) + .or(Some(Duration::ZERO)) + } else { + seek.map(|time| { + metadata + .as_ref() + .map_or(time, |metadata| time.min(metadata.session.length)) + }) + }; + let transport_changed = playing != clock.playing; + if seek.is_some() || transport_changed { + clock.set(seek.unwrap_or_else(|| clock.time(now)), playing, now); + } + if session.is_none() { + audio = None; + audio_attempted = false; + audio_error = None; + } + if session.is_some() && !audio_attempted && !muted { + audio_attempted = true; + match crate::audio::Audio::new() { + Ok(output) => audio = Some(output), + Err(error) => audio_error = Some(error), + } + } + if let Some(audio) = audio.as_mut() { + if changed || seek.is_some() || transport_changed { + audio.reset(clock.time(now)); + } + audio.controls(playing && !muted, volume); + } + let Some(metadata) = metadata.as_ref().filter(|_| session.is_some()) else { + wait(&shared, Duration::from_millis(100)); + continue; + }; + let mut time = clock.time(now).min(metadata.session.length); + if playing + && !muted + && let Some(audio) = audio.as_mut() + { + if let Some(device_time) = audio.time(now) + && metadata + .locate(device_time) + .is_some_and(|at| at.segment.complete) + { + time = device_time.min(metadata.session.length); + clock.set(time, true, now); + } + audio.catch_up(time); + if let Err(error) = audio.fill(metadata) { + audio_error = Some(format!("Cannot play audio: {error}")); + } + if let Some(error) = audio.error() { + audio_error = Some(error); + } + } + + if time >= metadata.session.length && playing { + clock.set(time, false, now); + shared.state.lock().unwrap().playing = false; + } + // Completed files only: active files may still lack seekable cues. + let located = metadata.locate(time).filter(|at| at.segment.complete); + let start = located + .as_ref() + .map_or(Duration::ZERO, |at| at.segment.start); + let frames_per_second = located + .as_ref() + .map_or(60, |at| at.segment.frames_per_second); + let mut picture = Picture::Gap; + let mut presented = false; + let started = Instant::now(); + if let Some(at) = located { + let target = decode::frame_index(at.offset, at.segment.frames_per_second); + let result = (|| { + if decoder.as_ref().is_none_or(|(path, _, index)| { + path != &at.segment.path + || target < *index + || target > index.saturating_add(120) + }) { + let mut opened = Decoder::open(&at.segment.path, at.segment.frames_per_second)?; + if at.offset > Duration::ZERO { + opened.seek(at.offset)?; + } + decoder = Some((at.segment.path.clone(), opened, u64::MAX)); + } + let (_, active, last) = decoder.as_mut().unwrap(); + if *last == target && !changed { + return Ok(None); + } + loop { + // A drag or navigation cancels decode-forward work between frames. + let state = shared.state.lock().unwrap(); + if state.quit || state.generation != generation { + return Ok(None); + } + drop(state); + let Some((index, frame)) = active.next_frame()? else { + return Ok(Some(Picture::Gap)); + }; + if index >= target { + *last = index; + return active + .scale(&frame, size) + .map(|frame| Some(Picture::Frame(Arc::new(frame)))); + } + } + })(); + match result { + Ok(Some(frame)) => { + presented = matches!(frame, Picture::Frame(_)); + picture = frame; + } + Ok(None) => picture = shared.state.lock().unwrap().snapshot.picture.clone(), + Err(error) => { + picture = Picture::Error(format!("Cannot read footage: {error}")); + decoder = None; + } + } + } else { + decoder = None; + } + let mut state = shared.state.lock().unwrap(); + if state.generation == generation { + let updated = time != state.snapshot.time + || clock.playing != state.snapshot.playing + || presented + || std::mem::discriminant(&picture) + != std::mem::discriminant(&state.snapshot.picture) + || audio_error != state.snapshot.audio_error; + let revision = state.snapshot.revision + u64::from(updated); + let video_time = matches!(picture, Picture::Frame(_)) + .then(|| { + decoder.as_ref().map(|(_, _, index)| { + start + clipbin_core::store::frame_offset(*index, frames_per_second) + }) + }) + .flatten(); + state.snapshot = Snapshot { + time, + playing: clock.playing, + picture, + revision, + audio_error: audio_error.clone(), + video_time, + frames_per_second, + }; + if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() && (playing || changed) { + eprintln!( + "playback time={:.3} decode_ms={:.3} generation={generation} presented={presented} state={} audio_time={:.3} video_time={:.3}", + time.as_secs_f64(), + started.elapsed().as_secs_f64() * 1000., + match &state.snapshot.picture { + Picture::Frame(_) => "frame", + Picture::Gap => "gap", + Picture::Loading => "loading", + Picture::Error(_) => "error", + }, + audio + .as_ref() + .and_then(|audio| audio.time(Instant::now())) + .map_or(-1., |time| time.as_secs_f64()), + video_time.map_or(-1., |time| time.as_secs_f64()) + ); + } + } + drop(state); + wait( + &shared, + if clock.playing { + Duration::from_millis(4) + } else { + Duration::from_millis(100) + }, + ); + } +} +fn wait(shared: &Shared, timeout: Duration) { + let state = shared.state.lock().unwrap(); + if !state.quit && state.seek.is_none() { + drop(shared.wake.wait_timeout(state, timeout).unwrap()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn a_decoder_stall_skips_overdue_frames_without_shifting_the_clock() { + let now = Instant::now(); + let mut clock = Clock::new(now); + clock.set(Duration::ZERO, true, now); + let first_target = decode::frame_index(clock.time(now + Duration::from_millis(16)), 60); + let after_stall = decode::frame_index(clock.time(now + Duration::from_millis(250)), 60); + assert_eq!(first_target, 0); + assert_eq!(after_stall, 15); + // Scaling only the target discards the intervening frames; their decoding + // cost must never be added to the clock's next presentation deadline. + assert_eq!( + decode::frame_index(clock.time(now + Duration::from_secs(1)), 60), + 60 + ); + } + #[test] + fn clock_pauses_and_seek_does_not_accumulate_decode_delay() { + let now = Instant::now(); + let mut clock = Clock::new(now); + clock.set(Duration::from_secs(2), true, now); + assert_eq!( + clock.time(now + Duration::from_secs(3)), + Duration::from_secs(5) + ); + clock.set(Duration::from_secs(5), false, now); + assert_eq!( + clock.time(now + Duration::from_secs(100)), + Duration::from_secs(5) + ); + clock.set(Duration::from_millis(1500), true, now); + assert_eq!( + decode::frame_index(clock.time(now + Duration::from_millis(100)), 60), + 96 + ); + } +} diff --git a/crates/clipbin-media/src/priority.rs b/crates/clipbin-media/src/priority.rs new file mode 100644 index 0000000..b078f89 --- /dev/null +++ b/crates/clipbin-media/src/priority.rs @@ -0,0 +1,14 @@ +//! Browsing runs below the recorder's normal scheduling priority. +pub fn lower(thumbnail: bool) { + #[cfg(target_os = "linux")] + if let Err(error) = rustix::process::setpriority_process( + Some(rustix::thread::gettid()), + if thumbnail { 10 } else { 3 }, + ) { + eprintln!("Cannot lower browsing thread priority: {error}"); + } + #[cfg(windows)] + crate::windows_priority::lower(thumbnail); + #[cfg(not(any(windows, target_os = "linux")))] + let _ = thumbnail; +} diff --git a/crates/clipbin-media/src/thumbnail.rs b/crates/clipbin-media/src/thumbnail.rs new file mode 100644 index 0000000..453ff74 --- /dev/null +++ b/crates/clipbin-media/src/thumbnail.rs @@ -0,0 +1,81 @@ +//! One background worker and a bounded queue for persistent PNG thumbnails. +use crate::decode; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::{Arc, Mutex, mpsc}, + thread, + time::Duration, +}; + +/// Deterministic cache path beside the source; exported MP4s use the same API. +pub fn cache_path(path: &Path) -> PathBuf { + path.with_extension("thumbnail.png") +} + +/// Decode any MP4/MKV and atomically publish a small thumbnail. +pub fn generate(path: &Path) -> Result { + let cache = cache_path(path); + if cache.is_file() { + return Ok(cache); + } + let mut frame = decode::first_frame(path, (320, 180)).map_err(|error| error.to_string())?; + let temporary = cache.with_extension("png.part"); + for pixel in frame.bgra.as_chunks_mut::<4>().0 { + pixel.swap(0, 2); + } + image::save_buffer_with_format( + &temporary, + &frame.bgra, + frame.width, + frame.height, + image::ColorType::Rgba8, + image::ImageFormat::Png, + ) + .map_err(|error| error.to_string())?; + std::fs::rename(&temporary, &cache).map_err(|error| error.to_string())?; + Ok(cache) +} + +pub struct Thumbnails { + sender: mpsc::SyncSender, + pending: Arc>>, +} +impl Default for Thumbnails { + fn default() -> Self { + Self::new() + } +} +impl Thumbnails { + pub fn new() -> Self { + let (sender, receiver) = mpsc::sync_channel::(16); + let pending = Arc::new(Mutex::new(HashSet::new())); + let work = pending.clone(); + thread::Builder::new() + .name("clipbin-thumbnails".into()) + .spawn(move || { + crate::priority::lower(true); + for path in receiver { + // One decoder thread, one frame, and a duty-cycle cap leave capture priority. + thread::sleep(Duration::from_millis(150)); + if let Err(error) = generate(&path) { + eprintln!("thumbnail {}: {error}", path.display()); + } + work.lock().unwrap().remove(&path); + } + }) + .expect("thumbnail thread"); + Self { sender, pending } + } + pub fn request(&self, path: &Path) -> Option { + let cache = cache_path(path); + if cache.is_file() { + return Some(cache); + } + let mut pending = self.pending.lock().unwrap(); + if pending.insert(path.to_owned()) && self.sender.try_send(path.to_owned()).is_err() { + pending.remove(path); + } + None + } +} diff --git a/crates/clipbin-media/src/windows_priority.rs b/crates/clipbin-media/src/windows_priority.rs new file mode 100644 index 0000000..2b3ffa5 --- /dev/null +++ b/crates/clipbin-media/src/windows_priority.rs @@ -0,0 +1,19 @@ +//! The only raw Windows calls in the media crate; no handles escape this bridge. +use windows_sys::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_LOWEST, +}; + +pub fn lower(thumbnail: bool) { + let priority = if thumbnail { + THREAD_PRIORITY_LOWEST + } else { + THREAD_PRIORITY_BELOW_NORMAL + }; + // GetCurrentThread returns a borrowed pseudo-handle, valid for this thread. + if unsafe { SetThreadPriority(GetCurrentThread(), priority) } == 0 { + eprintln!( + "Cannot lower browsing thread priority: {}", + std::io::Error::last_os_error() + ); + } +} diff --git a/crates/clipbin-media/tests/playback.rs b/crates/clipbin-media/tests/playback.rs new file mode 100644 index 0000000..63416f8 --- /dev/null +++ b/crates/clipbin-media/tests/playback.rs @@ -0,0 +1,106 @@ +//! Run through scripts/validate/playback-fixture.sh for real FFmpeg seek coverage. +use clipbin_core::store::Store; +use clipbin_media::{ + decode::{Decoder, scale}, + player::{Picture, Player}, +}; +use std::{ + path::PathBuf, + thread, + time::{Duration, Instant}, +}; + +fn fixture() -> Option { + std::env::var_os("CLIPBIN_TEST_MEDIA_DIR").map(PathBuf::from) +} +#[test] +fn seeking_matches_sequential_frames_in_each_independent_segment() { + let Some(directory) = fixture() else { + return; + }; + for segment in 0..3 { + let path = directory.join(format!("1/segment-{segment:03}.mkv")); + let mut sequential = Decoder::open(&path, 60).unwrap(); + let mut expected = Vec::new(); + while let Some((index, frame)) = sequential.next_frame().unwrap() { + expected.push((index, scale(&frame, (160, 90)).unwrap().bgra)); + } + assert_eq!(expected.len(), 120); + for target in [0, 1, 59, 60, 61, 119] { + let mut decoder = Decoder::open(&path, 60).unwrap(); + decoder + .seek(Duration::from_secs_f64(target as f64 / 60.)) + .unwrap(); + loop { + let (index, frame) = decoder.next_frame().unwrap().unwrap(); + if index >= target { + assert_eq!(index, target); + assert_eq!( + scale(&frame, (160, 90)).unwrap().bgra, + expected[target as usize].1 + ); + break; + } + } + } + } +} +fn wait(player: &Player, at: Duration) -> clipbin_media::player::Snapshot { + let until = Instant::now() + Duration::from_secs(5); + loop { + let snapshot = player.snapshot(); + if snapshot.time == at && !matches!(snapshot.picture, Picture::Loading) { + return snapshot; + } + assert!(Instant::now() < until, "seek timed out: {snapshot:?}"); + thread::sleep(Duration::from_millis(10)); + } +} +#[test] +fn player_crosses_boundaries_and_gaps_and_discards_superseded_seeks() { + let Some(directory) = fixture() else { + return; + }; + let player = Player::new(); + player.audio(true, 1.); + player.open(Store { root: directory }, 1, Duration::ZERO); + assert!(matches!( + wait(&player, Duration::ZERO).picture, + Picture::Frame(_) + )); + for millis in [1999, 2000, 2500, 4000, 5999, 6000, 7000] { + let at = Duration::from_millis(millis); + player.seek(at); + let snapshot = wait(&player, at); + assert_eq!( + matches!(snapshot.picture, Picture::Gap), + (4000..6000).contains(&millis), + "at {millis}" + ); + } + for i in 0..50 { + player.seek(Duration::from_millis(i * 40)); + } + player.seek(Duration::from_millis(6500)); + assert!(matches!( + wait(&player, Duration::from_millis(6500)).picture, + Picture::Frame(_) + )); + player.seek(Duration::from_millis(1900)); + wait(&player, Duration::from_millis(1900)); + player.play(true); + thread::sleep(Duration::from_millis(400)); + player.play(false); + let snapshot = player.snapshot(); + assert!((Duration::from_millis(2250)..Duration::from_millis(2500)).contains(&snapshot.time)); + assert!(matches!(snapshot.picture, Picture::Frame(_))); + assert!(snapshot.video_time.unwrap().abs_diff(snapshot.time) < Duration::from_millis(20)); + player.seek(Duration::from_millis(3900)); + wait(&player, Duration::from_millis(3900)); + player.play(true); + thread::sleep(Duration::from_millis(350)); + let snapshot = player.snapshot(); + assert!(snapshot.time >= Duration::from_millis(4200)); + assert!(matches!(snapshot.picture, Picture::Gap)); + player.close(); +} diff --git a/docs/validation.md b/docs/validation.md index d8ec477..9af3a31 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -370,3 +370,99 @@ AV1 export is unvalidated because this GPU has no usable AV1 encoder. B-frame export and ranges containing an internal timestamp gap need additional media fixtures; dropped-frame ranges are explicitly rejected. Linux default-player and file-manager associations still need testing on a real desktop. + +## Playback and thumbnails + +Playback uses the shared `clipbin-media` crate (`ffmpeg-next` 9) without GPUI. +One background decoder opens only completed, available segments. Session time +comes from `Metadata::locate`; each file's timestamps are normalised independently. +Seeking goes backwards to a keyframe, then decodes forward to the selected frame. +A one-frame mailbox and coalesced seeks bound work; overdue frames are decoded +but discarded before swscale. Playback threads have lower scheduling priority +than recording. Leaving the editor closes its decoder and audio stream. + +The editor uploads packed BGRA at its actual pixel size, maintains a 16:9 +preview, and releases old GPUI images. Space/K toggle playback; arrows/J/L step +one frame. The timeline seeks and drags across files and gaps; completed footage +from active sessions becomes browsable on the next 250 ms metadata refresh +within the recorder's segment completion delay. Missing/removed footage advances +on the wall clock and shows “No footage (not captured / cleaned up)”. There is +no live recording preview. + +Audio uses cpal and swresample at the output device's rate/channel count, with +sample conversion in the device callback. OBS's combined third audio track is +selected when three tracks exist; otherwise the first track is used. The queue +holds at most 250 ms plus one decoded block. Callback timestamps, including DAC +latency, drive the master clock; gaps and mute use the wall clock. Callbacks do +no decoding and never wait for the queue lock. Mute and volume are in the editor; +an unavailable device is shown explicitly while video remains usable. The Linux +rig uses its existing null sink and ALSA's PipeWire plugin as a fallback if cpal's +native PulseAudio backend refuses the device. Install/configure `pipewire-alsa` +or an ALSA default routed to PulseAudio on systems without a physical ALSA card. + +One low-priority worker produces 320×180 PNG thumbnails beside the first playable +session segment and exported MP4s. Its request queue holds 16 paths and duplicate +requests coalesce; card discovery and all filesystem work run off the UI thread. +`clipbin_media::thumbnail::{generate,cache_path,Thumbnails}` accept any MP4/MKV. +The gradient remains while a thumbnail is pending or footage is unavailable. + +`scripts/validate/playback-fixture.sh` generates independent 1080p60 H.264/AAC +segments with one-second keyframes and a three-tone audio fixture. It passed on +2026-10-08: exact frame pixels at frame indices 0/1/59/60/61/119 in every file, +real-time boundary crossing, gaps, superseded seeks, pause/seek clock maths, +bounded frame placement, resampling 48 kHz to 44.1 kHz and combined-track selection. +The ordinary workspace tests skip the generated-media assertions unless +`CLIPBIN_TEST_MEDIA_DIR` points to the fixture; use the script to exercise them. + +Windows: the cpal WASAPI dependency cross-check passed, as did the dedicated +Windows thread-priority bridge. Core/OBS/probe strict MSVC cross-lint passed. +The full media cross-lint is excluded on this Linux machine because the FFmpeg +Windows development import libraries are unavailable. On Windows set `FFMPEG_DIR` +to an FFmpeg **7.1 shared development** package with MSVC import libraries matching +the OBS-bundled avcodec-61/avformat-61/avutil-59/swscale-8/swresample-5 DLLs. Linux +links system FFmpeg 8. Actual Windows decoding, WASAPI device latency/seek behaviour, +GPUI image presentation and thumbnail caching need a Windows machine. Hardware +video decode is not used; HEVC/AV1 software performance and real output devices +other than the rig null sink still need measurement. + +Final playback validation passed on 2026-10-08, serially with other rigs stopped: + +- `cargo fmt --all -- --check`, strict locked workspace clippy, and locked + workspace tests (with `CLIPBIN_TEST_MEDIA_DIR=target/playback-fixture`). +- `env PATH=/usr/bin:/bin:$HOME/.cargo/bin scripts/validate/linux-rig.sh`: + all nine scenarios. An earlier concurrent run lost its H.264 probe to SIGKILL; + the serial rerun passed without timeout or sandbox changes. +- The same PATH with `scripts/validate/linux-session.sh`: automatic recording, + exact segment placement, close-window survival, game exit, library reload and + clean Quit. Repeated after the final keyboard integration, also passed. +- `scripts/validate/linux-playback.sh target/playback-e2e-final-6`: moving preview + (11.0% changed pixels inside the preview), active recording with zero reported + recording drops, completed-segment availability, real session thumbnail, + cross-file drag, exact post-boundary frame/time, cleaned-up gap, zoom, arrow + stepping, device audio, half-volume and mute. The audio monitor RMS was 0.08044; + the player asserts video time stays within 20 ms of the audio master clock. + Screenshots were inspected: `sessions-thumbnails.png`, `editor-playing-1.png`, + `editor-playing-2.png`, `editor-seek.png`, and `editor-gap.png` in that directory. + +The rig runs on an Intel i5-12500 (12 logical CPUs). These are development builds +with dependencies at opt-level 2, software video decoding, 1080p60 H.264 source +and an 800×450 target (the editor used 740×416). Five serial repetitions of +`target/debug/examples/playback-check ` with +`scripts/validate/measure-cpu.py`, inside the null-sink rig, measured process CPU +at **16.7% median / 17.7% p95 of one core** (12.2–17.7% range). The filesystem cache +was warm; each repetition reopened its decoders. Across presented frames, +decode/scale time was **2.39 ms median / 8.80 ms p95**; maximum **101.5 ms** included +opening files and decoding forward on seeks. This is a test pattern with gaps, +not a worst-case game or a release-build benchmark. Per-run logs and CPU reports +are in `target/playback-timing-final`; `report.json` stores the distributions. + +During the active-session e2e, the player presented **59.2 frames/s** while +recording remained at zero reported drops. GPUI upload/construction time was +**0.22 ms median / 0.48 ms p95**. UI frame intervals (including vsync, not CPU work) +were 16.2 ms median / 32.5 ms p95; the one-frame mailbox drops presentation work +rather than allowing delay to accumulate. `CLIPBIN_TRACE_FRAMES=1` logs decoded +video time, audio time, decode/scale work, uploads and actual timeline bounds. +The headless e2e restores keyboard focus after virtual-pointer commands and +holds virtual keys briefly: instantaneous virtual input can disappear before +GPUI/Wayland dispatch. Keyboard actions update the editor entity directly, +avoiding a nested window borrow during event dispatch. diff --git a/scripts/validate/linux-playback.sh b/scripts/validate/linux-playback.sh new file mode 100755 index 0000000..da176a3 --- /dev/null +++ b/scripts/validate/linux-playback.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Playback, active browsing, real thumbnails and audio in the private desktop rig. +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +out="${1:-${repo}/target/playback-validation}" +mkdir -p "${out}" +out="$(cd "${out}" && pwd)" +cargo build --locked --workspace --manifest-path "${repo}/Cargo.toml" +cargo build --locked -p clipbin-media --example playback-check --manifest-path "${repo}/Cargo.toml" +export CLIPBIN_PLAYBACK_REPO="${repo}" CLIPBIN_PLAYBACK_OUT="${out}" +"${repo}/scripts/rig/headless.sh" bash <<'INNER' +set -euo pipefail +repo="${CLIPBIN_PLAYBACK_REPO}" out="${CLIPBIN_PLAYBACK_OUT}" +export XDG_DATA_HOME="${out}/data" XDG_CONFIG_HOME="${out}/config" CLIPBIN_TRACE_FRAMES=1 +mkdir -p "${XDG_DATA_HOME}" "${XDG_CONFIG_HOME}" +# ALSA's PipeWire plugin is cpal's fallback on Linux; there is no hardware card. +cat > "${out}/alsa.conf" <<'ALSA' + +pcm.!default { type pipewire } +ctl.!default { type pipewire } +ALSA +export ALSA_CONFIG_PATH="${out}/alsa.conf" +swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null +send_key() { + # Destroying rig-pointer's virtual seat can clear keyboard focus in Sway. + swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null + sleep .2 + timeout 10 wtype -s 200 -P "$1" -s 100 -p "$1" +} +# Timeline bounds come from GPUI's trace rather than guessed screen positions. +pointer_time() { + python3 - "$1" "$2" "$3" "${XDG_DATA_HOME}" <<'PYPOINTER' +import pathlib,re,json,sys +log=pathlib.Path(sys.argv[1]).read_text() +x,y,w,h=map(float,re.findall(r'playback_timeline x=([\d.]+) y=([\d.]+) width=([\d.]+) height=([\d.]+)',log)[-1]) +p=max(pathlib.Path(sys.argv[4]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());length=m['session']['length'];length=length['secs']+length['nanos']/1e9 +if sys.argv[2] in ('gap','after-gap'): + segment=next(s for s in m['segments'] if s['complete'] and not s['available']) + def seconds(d):return d['secs']+d['nanos']/1e9 + at=(seconds(segment['start'])+seconds(segment['end']))/2 if sys.argv[2]=='gap' else seconds(segment['end'])+.5 +else:at=float(sys.argv[2]) +print(f"move {round(280+x+w*at/length)} {round(110+y+h/2)}\nsleep .15\n{sys.argv[3]}\nsleep .15") +PYPOINTER +} +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] floating enable' > /dev/null +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] resize set width 1360 px height 860 px' > /dev/null +swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] move position 280 110' > /dev/null +"${repo}/target/debug/clipbin-desktop" > "${out}/active.log" 2>&1 & app=$! +trap 'kill "${app:-}" "${game:-}" "${audio_capture:-}" 2>/dev/null || true' EXIT +sleep 3 +swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null +swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null +bash -c 'exec -a reaper python3 -c "import time; time.sleep(120)" SteamLaunch AppId=480 --' & game=$! +for _ in $(seq 30); do if grep -q recorder_status=Recording "${out}/active.log"; then break; fi; sleep 1; done +grep -q recorder_status=Recording "${out}/active.log" +sleep 8 +grim "${out}/sessions-thumbnails.png" +# Open the first session card, seek near the start, and play while recording continues. +printf 'move 670 330\nsleep .2\ndown\nsleep .1\nup\nsleep 1\n' | "${repo}/target/debug/rig-pointer" 1920 1080 +grim "${out}/editor-open.png" +pointer_time "${out}/active.log" 1 click | "${repo}/target/debug/rig-pointer" 1920 1080 +# GPUI/Wayland needs the virtual keyboard keymap to settle and a held key; +# instantaneous wtype press/release occasionally disappears before dispatch. +send_key space +sleep 1 +grim "${out}/editor-playing-1.png" +sleep 2 +grim "${out}/editor-playing-2.png" +python3 - "${XDG_DATA_HOME}" <<'PY' +import pathlib,json,sys +p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());assert m['session']['active'];assert len([s for s in m['segments'] if s['complete'] and s['available']])>=4 +assert list(p.parent.rglob('*.thumbnail.png')), 'thumbnail worker produced no cached frame' +print('PASS playback left recording active and generated real thumbnails') +PY +kill "${game}"; wait "${game}" 2>/dev/null || true +sleep 3 +swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null +timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl +for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done +if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi +wait "${app}" +kill "$(cat "${CLIPBIN_RIG_DIR}/game.pid")" +# Simulate a cleaned-up middle segment in this isolated test library. Keep its +# timeline placement: the player must show the gap rather than concatenate it away. +python3 - "${XDG_DATA_HOME}" "${out}" <<'PY' +import pathlib,json,sys +p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name)) +m=json.loads(p.read_text());segments=[s for s in m['segments'] if s['complete'] and s['available']];assert len(segments)>=4 +s=segments[2];s['available']=False +m['session']['spans']=[dict(start=m['session']['spans'][0]['start'],end=s['start'],kind='Footage'),dict(start=s['start'],end=s['end'],kind='Removed'),dict(start=s['end'],end=m['session']['length'],kind='Footage')] +p.write_text(json.dumps(m));pathlib.Path(sys.argv[2]+'/session-id').write_text(str(m['session']['id'])) +PY +id="$(cat "${out}/session-id")" +# The CLI exercises the same player with audio only; recording tone has stopped. +ffmpeg -v error -y -f pulse -i clipbin-rig-sink.monitor -t 10 "${out}/audio-output.wav" & audio_capture=$! +python3 "${repo}/scripts/validate/measure-cpu.py" "${out}/player-cpu.txt" "${repo}/target/debug/examples/playback-check" "${XDG_DATA_HOME}/clipbin/sessions" "${id}" > "${out}/player.log" 2>&1 +wait "${audio_capture}" +CLIPBIN_ROUTE="session:${id}" "${repo}/target/debug/clipbin-desktop" > "${out}/playback.log" 2>&1 & app=$! +sleep 3 +swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null +# Drag through several files, then seek into the removed interval. +{ pointer_time "${out}/playback.log" 1 down; pointer_time "${out}/playback.log" 3 "sleep 0.5"; pointer_time "${out}/playback.log" after-gap up; } | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep 1 +grim "${out}/editor-seek.png" +pointer_time "${out}/playback.log" gap click | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep 1 +grim "${out}/editor-gap.png" +pointer_time "${out}/playback.log" 1 $'scroll -1\nsleep 0.5\nscroll 1' | "${repo}/target/debug/rig-pointer" 1920 1080 +pointer_time "${out}/playback.log" 2.5 click | "${repo}/target/debug/rig-pointer" 1920 1080 +sleep .3 +send_key Right +sleep .3 +send_key Left +sleep .3 +swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null +timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl +for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done +if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi +wait "${app}" +python3 - "${out}" <<'PY' +import pathlib,subprocess,re,sys,math,array,statistics,json +out=pathlib.Path(sys.argv[1]) +def rgb(path):return subprocess.check_output(['ffmpeg','-v','error','-i',str(path),'-vf','crop=700:300:555:260','-f','rawvideo','-pix_fmt','rgb24','-']) +a,b=map(rgb,[out/'editor-playing-1.png',out/'editor-playing-2.png']) +changed=sum(x!=y for x,y in zip(a,b))/len(a);assert changed>.02, f'preview frozen: {changed}' +raw=subprocess.check_output(['ffmpeg','-v','error','-i',str(out/'audio-output.wav'),'-ac','1','-ar','48000','-f','f32le','-']) +samples=array.array('f',raw);rms=math.sqrt(sum(x*x for x in samples)/len(samples));assert rms>.001, f'no device audio: {rms}' +def window_rms(start,end): + block=samples[int(start*48000):int(end*48000)] + return math.sqrt(sum(x*x for x in block)/max(1,len(block))) +full=window_rms(2,3);half=window_rms(7.9,8.4);muted=window_rms(9.5,9.9) +assert full>.001 and .35=2, f'frame stepping did not change the video frame: {steps}' +uploads=[float(x) for x in re.findall(r'playback_ui upload_ms=([\d.]+)',(out/'active.log').read_text())] +assert len(uploads)>30, 'too few moving frames reached GPUI' +print(f'GPUI frame upload median={statistics.median(uploads):.3f}ms p95={sorted(uploads)[int(len(uploads)*.95)]:.3f}ms') +(out/'report.txt').write_text(f'changed_preview_pixels={changed:.4f}\naudio_rms={rms:.5f}\ndecode_median_ms={statistics.median(times):.3f}\ndecode_p95_ms={sorted(times)[int(len(times)*.95)]:.3f}\n') +PY +echo "PASS playback validation; screenshots and timing in ${out}" +INNER diff --git a/scripts/validate/measure-cpu.py b/scripts/validate/measure-cpu.py new file mode 100644 index 0000000..14557b9 --- /dev/null +++ b/scripts/validate/measure-cpu.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Measure child CPU/wall time without depending on GNU time.""" +import pathlib +import resource +import subprocess +import sys +import time + +before = resource.getrusage(resource.RUSAGE_CHILDREN) +started = time.monotonic() +result = subprocess.run(sys.argv[2:], check=False) +elapsed = time.monotonic() - started +usage = resource.getrusage(resource.RUSAGE_CHILDREN) +user = usage.ru_utime - before.ru_utime +system = usage.ru_stime - before.ru_stime +pathlib.Path(sys.argv[1]).write_text( + f"cpu_user={user:.3f} cpu_system={system:.3f} wall={elapsed:.3f} " + f"cpu_percent={(user + system) / elapsed * 100:.1f}%\n" +) +sys.exit(result.returncode) diff --git a/scripts/validate/playback-fixture.sh b/scripts/validate/playback-fixture.sh new file mode 100755 index 0000000..42eaddb --- /dev/null +++ b/scripts/validate/playback-fixture.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Deterministic, clock-labelled 1080p60 segments with independent file timestamps. +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +out="${1:-${repo}/target/playback-fixture}" +mkdir -p "${out}/1" +ffmpeg -v error -y -f lavfi -i 'testsrc2=size=1920x1080:rate=60' \ + -f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 6 \ + -vf "drawtext=fontfile=/usr/share/fonts/TTF/DejaVuSans.ttf:text='%{pts\:hms}':fontsize=64:fontcolor=white:box=1:boxcolor=black:x=100:y=100" \ + -c:v libx264 -preset ultrafast -crf 28 -threads 2 -g 60 -keyint_min 60 -sc_threshold 0 -bf 0 \ + -c:a aac -f segment -segment_time 2 -reset_timestamps 1 "${out}/1/segment-%03d.mkv" +ffmpeg -v error -y -i "${out}/1/segment-000.mkv" \ + -f lavfi -i 'sine=frequency=220:sample_rate=48000' \ + -f lavfi -i 'sine=frequency=880:sample_rate=48000' \ + -f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 2 \ + -map 0:v -map 1:a -map 2:a -map 3:a -c:v copy -c:a aac "${out}/combined.mkv" +python3 - "${out}" <<'PY' +import pathlib,json,sys +root=pathlib.Path(sys.argv[1]); d=lambda s:dict(secs=s,nanos=0) +segments=[dict(path=f'segment-{i:03}.mkv',start=d(t),end=d(t+2),complete=True,available=True,frame_start=i*120,frames_per_second=60,dropped_frames=0) for i,t in enumerate([0,2,6])] +session=dict(id=1,game='Playback fixture',started=dict(secs_since_epoch=1791453600,nanos_since_epoch=0),length=d(8),active=False,spans=[dict(start=d(a),end=d(b),kind=k) for a,b,k in [(0,4,'Footage'),(4,6,'Missing'),(6,8,'Footage')]],bookmarks=[]) +(root/'1/session.json').write_text(json.dumps(dict(session=session,app_id=480,segments=segments))) +PY +CLIPBIN_TEST_MEDIA_DIR="${out}" cargo test --locked -p clipbin-media --manifest-path "${repo}/Cargo.toml"