Merge feat/playback: session playback, synced audio and thumbnails
# Conflicts: # README.md # crates/clipbin-desktop/Cargo.toml # crates/clipbin-desktop/src/main.rs # crates/clipbin-media/src/lib.rs # docs/validation.md
This commit is contained in:
Generated
+390
-6
@@ -60,6 +60,28 @@ dependencies = [
|
||||
"equator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
|
||||
dependencies = [
|
||||
"alsa-sys",
|
||||
"bitflags 2.13.2",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa-sys"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
@@ -885,6 +907,7 @@ dependencies = [
|
||||
"clipbin-media",
|
||||
"clipbin-obs",
|
||||
"gpui",
|
||||
"image",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -892,7 +915,11 @@ name = "clipbin-media"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clipbin-core",
|
||||
"cpal",
|
||||
"ffmpeg-next",
|
||||
"image",
|
||||
"rustix 1.1.5",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -989,6 +1016,16 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "command-fds"
|
||||
version = "0.3.3"
|
||||
@@ -1199,6 +1236,20 @@ dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"libc",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cosmic-text"
|
||||
version = "0.14.2"
|
||||
@@ -1222,6 +1273,39 @@ dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1"
|
||||
dependencies = [
|
||||
"alsa",
|
||||
"block2",
|
||||
"coreaudio-rs",
|
||||
"dasp_sample",
|
||||
"futures",
|
||||
"jni",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"mach2",
|
||||
"ndk",
|
||||
"ndk-context",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"objc2",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-avf-audio",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
"portable-atomic",
|
||||
"pulseaudio",
|
||||
"web-sys",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -1325,6 +1409,12 @@ version = "0.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2"
|
||||
|
||||
[[package]]
|
||||
name = "dasp_sample"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
|
||||
|
||||
[[package]]
|
||||
name = "data-url"
|
||||
version = "0.3.2"
|
||||
@@ -1534,6 +1624,17 @@ version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099"
|
||||
|
||||
[[package]]
|
||||
name = "enum-primitive-derive"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enumflags2"
|
||||
version = "0.7.12"
|
||||
@@ -2256,7 +2357,7 @@ dependencies = [
|
||||
"wayland-protocols-plasma",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
"windows-numerics",
|
||||
"windows-numerics 0.2.0",
|
||||
"windows-registry 0.5.3",
|
||||
"x11-clipboard",
|
||||
"x11rb",
|
||||
@@ -2923,6 +3024,64 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"combine",
|
||||
"jni-macros",
|
||||
"jni-sys 0.4.1",
|
||||
"log",
|
||||
"simd_cesu8",
|
||||
"thiserror 2.0.21",
|
||||
"walkdir",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-macros"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"simd_cesu8",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
|
||||
dependencies = [
|
||||
"jni-sys 0.4.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
|
||||
dependencies = [
|
||||
"jni-sys-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys-macros"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.35"
|
||||
@@ -3165,6 +3324,12 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b"
|
||||
|
||||
[[package]]
|
||||
name = "malloc_buf"
|
||||
version = "0.0.6"
|
||||
@@ -3288,6 +3453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -3342,6 +3508,35 @@ dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"ndk-sys",
|
||||
"num_enum",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk-context"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
|
||||
|
||||
[[package]]
|
||||
name = "ndk-sys"
|
||||
version = "0.6.0+11769913"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873"
|
||||
dependencies = [
|
||||
"jni-sys 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
@@ -3527,6 +3722,28 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
|
||||
dependencies = [
|
||||
"num_enum_derive",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc"
|
||||
version = "0.2.7"
|
||||
@@ -3570,6 +3787,55 @@ dependencies = [
|
||||
"objc2-quartz-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-audio-toolbox"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"libc",
|
||||
"objc2",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-avf-audio"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-audio"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2"
|
||||
dependencies = [
|
||||
"dispatch2",
|
||||
"objc2",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-audio-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.2"
|
||||
@@ -3577,7 +3843,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"block2",
|
||||
"dispatch2",
|
||||
"libc",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
@@ -3594,6 +3862,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"block2",
|
||||
"libc",
|
||||
"objc2",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
@@ -3913,6 +4183,12 @@ version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "postage"
|
||||
version = "0.5.0"
|
||||
@@ -4050,6 +4326,22 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740"
|
||||
|
||||
[[package]]
|
||||
name = "pulseaudio"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d70623bd7967a9ca4c2ae0e807fc380b291f98480fc037042305ec643a4d3373"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"byteorder",
|
||||
"enum-primitive-derive",
|
||||
"futures",
|
||||
"log",
|
||||
"mio",
|
||||
"num-traits",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.30"
|
||||
@@ -4971,6 +5263,16 @@ version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "simd_cesu8"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd_helpers"
|
||||
version = "0.1.0"
|
||||
@@ -6340,11 +6642,23 @@ version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-collections 0.2.0",
|
||||
"windows-core 0.61.2",
|
||||
"windows-future",
|
||||
"windows-future 0.2.1",
|
||||
"windows-link 0.1.3",
|
||||
"windows-numerics",
|
||||
"windows-numerics 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
|
||||
dependencies = [
|
||||
"windows-collections 0.3.2",
|
||||
"windows-core 0.62.2",
|
||||
"windows-future 0.3.2",
|
||||
"windows-numerics 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6357,7 +6671,7 @@ dependencies = [
|
||||
"rayon",
|
||||
"thiserror 2.0.21",
|
||||
"windows 0.61.3",
|
||||
"windows-future",
|
||||
"windows-future 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6369,6 +6683,15 @@ dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.57.0"
|
||||
@@ -6394,6 +6717,19 @@ dependencies = [
|
||||
"windows-strings 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement 0.60.2",
|
||||
"windows-interface 0.59.3",
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
"windows-strings 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.2.1"
|
||||
@@ -6402,7 +6738,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core 0.61.2",
|
||||
"windows-link 0.1.3",
|
||||
"windows-threading",
|
||||
"windows-threading 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
"windows-threading 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6471,6 +6818,16 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
|
||||
dependencies = [
|
||||
"windows-core 0.62.2",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.4.0"
|
||||
@@ -6511,6 +6868,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.3.1"
|
||||
@@ -6529,6 +6895,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
@@ -6622,6 +6997,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
|
||||
@@ -10,7 +10,7 @@ The recording foundation is being validated (see [docs/validation.md](docs/valid
|
||||
|
||||
The desktop app detects Steam games, owns the recorder on a background thread, persists settings and sessions, and loads recorded sessions from disk. Linux detects Steam’s `reaper` processes and watches their pidfds; Windows reads Steam’s registry state and watches native process handles (cross-checked, not run). Game exit finalises a session; closing the window keeps the current session recording until the game exits, then exits the app. Ctrl+Q or Quit stops the output cleanly.
|
||||
|
||||
The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Clip export, bookmark protection and storage cleanup are implemented. Playback, autostart and a tray are separate work. Exports copy the recorded codec to MP4 in a background job, save local clip metadata, and support opening/deleting clips. Ctrl+Shift+F9 clips the last configured interval; Ctrl+Shift+F10 protects a bookmark. Both also have sidebar buttons. Wayland uses the GlobalShortcuts portal (first-use approval, portal rebinding); X11 uses native key grabs and Windows uses RegisterHotKey. Settings offer independent alternate function keys. After closing the only window, reopening it without a tray is a follow-up.
|
||||
The desktop interface follows the web Clipbin's look (zinc dark theme, purple accent, Geist Mono): a session library, a session timeline with bookmarks and a clip selection, clips, and settings. It shows sessions from disk; `CLIPBIN_SAMPLE_LIBRARY=1` fills it with example sessions for interface work. Playback and frame seeking work across completed segments, including active sessions and missing footage, with combined audio, mute/volume, Space/K to play/pause and J/L or arrows to step frames. Sessions and clips use cached frame thumbnails. Exports copy the recorded codec to MP4 in a background job, save local clip metadata, and support opening/deleting clips. Ctrl+Shift+F9 clips the last configured interval; Ctrl+Shift+F10 protects a bookmark. Both also have sidebar buttons. Wayland uses the GlobalShortcuts portal (first-use approval, portal rebinding); X11 uses native key grabs and Windows uses RegisterHotKey. Settings offer independent alternate function keys. Storage cleanup keeps unprotected footage within the budget. Autostart and a tray are not implemented yet; after closing the only window, reopening it without a tray is a follow-up.
|
||||
|
||||
Windows is a first-class target. Every change must keep the Windows code paths building (`cargo clippy --target x86_64-pc-windows-msvc`) and list what still needs validating on Windows.
|
||||
|
||||
@@ -19,7 +19,7 @@ Windows is a first-class target. Every change must keep the Windows code paths b
|
||||
```text
|
||||
crates/
|
||||
clipbin-core/ Settings, Steam detection and session storage; independent of GPUI and OBS.
|
||||
clipbin-media/ FFmpeg stream-copy exports and shared media reading.
|
||||
clipbin-media/ FFmpeg: stream-copy clip export, scaled video/audio playback and cached thumbnails.
|
||||
clipbin-obs/ Recording engine over the pinned libobs runtime; all unsafe code is in src/ffi.rs.
|
||||
clipbin-probe/ Validation probe for capture, encoders, segments and status.
|
||||
clipbin-desktop/ GPUI application and initial session-library view.
|
||||
@@ -42,7 +42,7 @@ Keep the lockfile checked in and use `--locked`. It currently pins `libc` to 0.2
|
||||
On Arch/CachyOS, install the native build dependencies:
|
||||
|
||||
```sh
|
||||
sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang
|
||||
sudo pacman -S --needed base-devel pkgconf fontconfig freetype2 wayland libxcb libxkbcommon libxkbcommon-x11 openssl clang ffmpeg alsa-lib pipewire-alsa libpulse
|
||||
```
|
||||
|
||||
### OBS runtime
|
||||
@@ -63,9 +63,11 @@ On Debian/Ubuntu:
|
||||
|
||||
```sh
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev
|
||||
sudo apt-get install build-essential pkg-config libfontconfig1-dev libfreetype6-dev libwayland-dev libxcb1-dev libxkbcommon-dev libxkbcommon-x11-dev libssl-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libasound2-dev libpulse-dev pipewire-alsa
|
||||
```
|
||||
|
||||
Media dynamically links FFmpeg through `ffmpeg-next` 9. Linux uses system FFmpeg 8. On Windows, set `FFMPEG_DIR` to an FFmpeg **7.1 shared development** package with MSVC import libraries, matching the OBS-bundled `avcodec-61`, `avformat-61`, `avutil-59`, `swscale-8` and `swresample-5` DLLs. Keep those OBS DLLs alongside the desktop executable; do not mix FFmpeg major versions. Audio uses cpal (PulseAudio or ALSA/PipeWire on Linux, WASAPI on Windows).
|
||||
|
||||
For Windows, use the MSVC Rust toolchain and Visual Studio Build Tools with the Desktop development with C++ workload. Windows support needs validation on a Windows machine; see [docs/validation.md](docs/validation.md#windows).
|
||||
|
||||
Run the desktop shell from an active graphical session:
|
||||
@@ -90,7 +92,7 @@ cargo clippy --locked --target x86_64-pc-windows-msvc -p clipbin-core -p clipbin
|
||||
|
||||
Clips/bookmarks/cleanup validation: `scripts/validate/linux-clips.sh`. Native X11 hotkey validation: `scripts/validate/linux-hotkeys.sh`. Both run in the headless rig. Real Hyprland portal hotkeys and Windows runtime behaviour still need validation.
|
||||
|
||||
Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times.
|
||||
Recording validation on Linux, headless: `scripts/validate/linux-rig.sh`. Automatic desktop sessions: `scripts/validate/linux-session.sh` (fake Steam reaper, recording and library screenshots, close-window survival, game exit and explicit Quit). Playback fixtures: `scripts/validate/playback-fixture.sh`. Playback/audio e2e: `scripts/validate/linux-playback.sh [output-dir]` (moving preview, active recording, gaps, thumbnails, audio monitor, timing and CPU report). Fake Steam rigs must run serially because discovery reads the machine’s `/proc`. Interface screenshots, headless: `scripts/rig/screenshot-ui.sh` (screens: sessions, session, clips, settings, failed, empty). The rig has no input devices; `rig-pointer` drives a virtual pointer (`move`, `click`, `down`/`up`, `scroll`, `sleep` on stdin) and `wtype` types. `CLIPBIN_TRACE_FRAMES=1` logs frame times.
|
||||
|
||||
Development builds optimise dependencies (`[profile.dev.package."*"]`): unoptimised GPUI made screens noticeably slow, with first frames of 200–300 ms.
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ publish.workspace = true
|
||||
[dependencies]
|
||||
clipbin-core.workspace = true
|
||||
clipbin-media.workspace = true
|
||||
image = { version = "0.25.10", default-features = false }
|
||||
clipbin-obs.workspace = true
|
||||
gpui.workspace = true
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ pub struct ClipbinApp {
|
||||
pub settings: Settings,
|
||||
pub recorder: Recorder,
|
||||
pub editor: Editor,
|
||||
pub playback: crate::playback::Playback,
|
||||
/// A short message for actions that are not connected yet.
|
||||
pub notice: Option<SharedString>,
|
||||
pub drag: Option<Drag>,
|
||||
@@ -204,7 +205,7 @@ impl ClipbinApp {
|
||||
let library = if sample {
|
||||
Library::sample(SystemTime::now())
|
||||
} else {
|
||||
Library::empty(settings.unprotected_session_limit_bytes)
|
||||
service.snapshot().library
|
||||
};
|
||||
let recorder = match (sample, std::env::var("CLIPBIN_SAMPLE_STATUS").as_deref()) {
|
||||
(_, Ok("failed")) => Recorder {
|
||||
@@ -237,6 +238,7 @@ impl ClipbinApp {
|
||||
selection: (Duration::ZERO, Duration::ZERO),
|
||||
playing: false,
|
||||
},
|
||||
playback: crate::playback::Playback::default(),
|
||||
notice: None,
|
||||
drag: None,
|
||||
timeline_bounds: Rc::default(),
|
||||
@@ -312,6 +314,7 @@ impl ClipbinApp {
|
||||
})
|
||||
.detach();
|
||||
}
|
||||
Self::start_playback_updates(window, cx);
|
||||
app
|
||||
}
|
||||
|
||||
@@ -323,7 +326,11 @@ impl ClipbinApp {
|
||||
self.notice = None;
|
||||
match route {
|
||||
Route::Session(id) => self.open_session(id),
|
||||
route => self.route = route,
|
||||
route => {
|
||||
self.playback.player.close();
|
||||
self.editor.playing = false;
|
||||
self.route = route;
|
||||
}
|
||||
}
|
||||
cx.notify();
|
||||
}
|
||||
@@ -347,6 +354,19 @@ impl ClipbinApp {
|
||||
selection,
|
||||
playing: false,
|
||||
};
|
||||
if let Ok(store) = clipbin_core::store::Store::discover() {
|
||||
let preview_time = playhead.min(
|
||||
session
|
||||
.length
|
||||
.saturating_sub(Duration::from_nanos(16_666_667)),
|
||||
);
|
||||
self.editor.playhead = preview_time;
|
||||
if session.bookmarks.is_empty() {
|
||||
self.playback.player.open_latest(store, id);
|
||||
} else {
|
||||
self.playback.player.open(store, id, preview_time);
|
||||
}
|
||||
}
|
||||
self.route = Route::Session(id);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ mod app;
|
||||
mod assets;
|
||||
mod components;
|
||||
mod exports;
|
||||
mod playback;
|
||||
mod recorder_service;
|
||||
mod screens;
|
||||
mod storage;
|
||||
@@ -49,6 +50,7 @@ fn main() {
|
||||
});
|
||||
cx.bind_keys([KeyBinding::new("ctrl-q", Quit, None)]);
|
||||
text_field::bind_keys(cx);
|
||||
playback::bind_keys(cx);
|
||||
|
||||
let bounds = Bounds::centered(None, size(px(1360.), px(860.)), cx);
|
||||
let result = cx.open_window(
|
||||
@@ -67,11 +69,14 @@ fn main() {
|
||||
},
|
||||
);
|
||||
|
||||
if let Err(error) = result {
|
||||
match result {
|
||||
Ok(window) => playback::bind_actions(window, cx),
|
||||
Err(error) => {
|
||||
eprintln!("Could not open Clipbin: {error}");
|
||||
cx.quit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
cx.activate(true);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
//! GPUI image upload and controls; file I/O and decoding belong to the player.
|
||||
use crate::app::{ClipbinApp, Route};
|
||||
use clipbin_core::store::Store;
|
||||
use clipbin_media::{
|
||||
player::{Picture, Player},
|
||||
thumbnail::Thumbnails,
|
||||
};
|
||||
use gpui::{Context, RenderImage};
|
||||
use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration};
|
||||
|
||||
pub struct Playback {
|
||||
pub player: Player,
|
||||
pub image: Option<Arc<RenderImage>>,
|
||||
pub message: Option<String>,
|
||||
revision: u64,
|
||||
last_frame: Option<Arc<clipbin_media::decode::Frame>>,
|
||||
pub muted: bool,
|
||||
pub volume: f32,
|
||||
pub audio_error: Option<String>,
|
||||
pub preview_height: f32,
|
||||
pub thumbnails: HashMap<u64, PathBuf>,
|
||||
pub clip_thumbnails: HashMap<u64, PathBuf>,
|
||||
}
|
||||
impl Default for Playback {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
player: Player::new(),
|
||||
image: None,
|
||||
message: Some("Loading footage…".into()),
|
||||
revision: 0,
|
||||
last_frame: None,
|
||||
muted: false,
|
||||
volume: 1.,
|
||||
audio_error: None,
|
||||
preview_height: 416.,
|
||||
thumbnails: HashMap::new(),
|
||||
clip_thumbnails: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl ClipbinApp {
|
||||
pub fn seek_playback(&mut self, time: Duration) {
|
||||
self.editor.playhead = time;
|
||||
self.playback.player.seek(time);
|
||||
}
|
||||
pub fn toggle_playback(&mut self) {
|
||||
if let Route::Session(id) = self.route {
|
||||
if self
|
||||
.library
|
||||
.session(id)
|
||||
.is_some_and(|session| self.editor.playhead >= session.length)
|
||||
{
|
||||
self.seek_playback(Duration::ZERO);
|
||||
}
|
||||
self.editor.playing = !self.editor.playing;
|
||||
self.playback.player.play(self.editor.playing);
|
||||
}
|
||||
}
|
||||
pub fn start_playback_updates(window: &gpui::Window, cx: &mut Context<Self>) {
|
||||
cx.spawn_in(window, async |entity, cx| {
|
||||
loop {
|
||||
gpui::Timer::after(Duration::from_millis(16)).await;
|
||||
if entity
|
||||
.update_in(cx, |app, window, cx| {
|
||||
if !matches!(app.route, Route::Session(_)) {
|
||||
if let Some(image) = app.playback.image.take() {
|
||||
let _ = window.drop_image(image);
|
||||
}
|
||||
app.playback.last_frame = None;
|
||||
return;
|
||||
}
|
||||
let snapshot = app.playback.player.snapshot();
|
||||
if snapshot.revision == app.playback.revision {
|
||||
return;
|
||||
}
|
||||
app.playback.revision = snapshot.revision;
|
||||
app.editor.playhead = snapshot.time;
|
||||
app.editor.playing = snapshot.playing;
|
||||
app.playback.audio_error = snapshot.audio_error;
|
||||
match snapshot.picture {
|
||||
Picture::Frame(frame) => {
|
||||
// Swscale already produced GPUI-native BGRA off the UI thread.
|
||||
if app
|
||||
.playback
|
||||
.last_frame
|
||||
.as_ref()
|
||||
.is_none_or(|last| !Arc::ptr_eq(last, &frame))
|
||||
{
|
||||
let upload_started = std::time::Instant::now();
|
||||
let bytes = frame.bgra.clone();
|
||||
if let Some(buffer) =
|
||||
image::RgbaImage::from_raw(frame.width, frame.height, bytes)
|
||||
{
|
||||
let image =
|
||||
Arc::new(RenderImage::new(vec![image::Frame::new(
|
||||
buffer,
|
||||
)]));
|
||||
if let Some(old) = app.playback.image.replace(image) {
|
||||
let _ = window.drop_image(old);
|
||||
}
|
||||
if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() {
|
||||
eprintln!(
|
||||
"playback_ui upload_ms={:.3} width={} height={}",
|
||||
upload_started.elapsed().as_secs_f64() * 1000.,
|
||||
frame.width,
|
||||
frame.height
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
app.playback.last_frame = Some(frame);
|
||||
app.playback.message = None;
|
||||
}
|
||||
Picture::Loading => {
|
||||
if let Some(old) = app.playback.image.take() {
|
||||
let _ = window.drop_image(old);
|
||||
}
|
||||
app.playback.last_frame = None;
|
||||
app.playback.message = Some("Loading footage…".into());
|
||||
}
|
||||
Picture::Gap => {
|
||||
if let Some(old) = app.playback.image.take() {
|
||||
let _ = window.drop_image(old);
|
||||
}
|
||||
app.playback.last_frame = None;
|
||||
app.playback.message =
|
||||
Some("No footage (not captured / cleaned up)".into());
|
||||
}
|
||||
Picture::Error(error) => {
|
||||
if let Some(old) = app.playback.image.take() {
|
||||
let _ = window.drop_image(old);
|
||||
}
|
||||
app.playback.last_frame = None;
|
||||
app.playback.message = Some(error);
|
||||
}
|
||||
}
|
||||
cx.notify();
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
// Discovery and thumbnail work never touch the UI thread's filesystem.
|
||||
let (sender, receiver) = std::sync::mpsc::sync_channel(1);
|
||||
std::thread::spawn(move || {
|
||||
let thumbnails = Thumbnails::new();
|
||||
loop {
|
||||
let mut ready = HashMap::new();
|
||||
let mut clips = HashMap::new();
|
||||
if let Ok(store) = Store::discover()
|
||||
&& let Ok(metadata) = store.read()
|
||||
{
|
||||
for metadata in metadata {
|
||||
if let Some(segment) = metadata
|
||||
.segments
|
||||
.iter()
|
||||
.find(|segment| segment.complete && segment.available)
|
||||
&& let Some(path) = thumbnails
|
||||
.request(&store.directory(metadata.session.id).join(&segment.path))
|
||||
{
|
||||
ready.insert(metadata.session.id, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(store) = Store::discover()
|
||||
&& let Some(root) = store.root.parent()
|
||||
&& let Ok(entries) = std::fs::read_dir(root.join("clips"))
|
||||
{
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.extension().is_some_and(|extension| extension == "mp4")
|
||||
&& let Some(id) = path
|
||||
.file_stem()
|
||||
.and_then(|id| id.to_str())
|
||||
.and_then(|id| id.parse().ok())
|
||||
&& let Some(thumbnail) = thumbnails.request(&path)
|
||||
{
|
||||
clips.insert(id, thumbnail);
|
||||
}
|
||||
}
|
||||
}
|
||||
if matches!(
|
||||
sender.try_send((ready, clips)),
|
||||
Err(std::sync::mpsc::TrySendError::Disconnected(_))
|
||||
) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
});
|
||||
cx.spawn(async move |entity, cx| {
|
||||
loop {
|
||||
gpui::Timer::after(Duration::from_millis(250)).await;
|
||||
let ready = receiver.try_recv().ok();
|
||||
if entity
|
||||
.update(cx, |app, cx| {
|
||||
if let Some((sessions, clips)) = ready
|
||||
&& (app.playback.thumbnails != sessions
|
||||
|| app.playback.clip_thumbnails != clips)
|
||||
{
|
||||
app.playback.thumbnails = sessions;
|
||||
app.playback.clip_thumbnails = clips;
|
||||
cx.notify();
|
||||
}
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
}
|
||||
}
|
||||
|
||||
gpui::actions!(playback, [TogglePlayback, StepBack, StepForward]);
|
||||
pub fn bind_keys(cx: &mut gpui::App) {
|
||||
cx.bind_keys([
|
||||
gpui::KeyBinding::new("space", TogglePlayback, None),
|
||||
gpui::KeyBinding::new("k", TogglePlayback, None),
|
||||
gpui::KeyBinding::new("left", StepBack, None),
|
||||
gpui::KeyBinding::new("j", StepBack, None),
|
||||
gpui::KeyBinding::new("right", StepForward, None),
|
||||
gpui::KeyBinding::new("l", StepForward, None),
|
||||
]);
|
||||
cx.bind_keys(
|
||||
["space", "k", "j", "l"]
|
||||
.map(|key| gpui::KeyBinding::new(key, gpui::NoAction {}, Some("TextField"))),
|
||||
);
|
||||
}
|
||||
/// Window actions also work when a mouse click leaves no focused GPUI element.
|
||||
/// Text-field contexts keep their own typing and arrow bindings.
|
||||
pub fn bind_actions(window: gpui::WindowHandle<ClipbinApp>, cx: &mut gpui::App) {
|
||||
// The event dispatcher already borrows the window; update its entity directly.
|
||||
let editor = window.entity(cx).expect("editor window").downgrade();
|
||||
let toggle = editor.clone();
|
||||
cx.on_action(move |_: &TogglePlayback, cx| {
|
||||
let _ = toggle.update(cx, |app, cx| {
|
||||
app.toggle_playback();
|
||||
cx.notify();
|
||||
});
|
||||
});
|
||||
let back = editor.clone();
|
||||
cx.on_action(move |_: &StepBack, cx| {
|
||||
let _ = back.update(cx, |app, cx| {
|
||||
app.step_playback(false);
|
||||
cx.notify();
|
||||
});
|
||||
});
|
||||
cx.on_action(move |_: &StepForward, cx| {
|
||||
let _ = editor.update(cx, |app, cx| {
|
||||
app.step_playback(true);
|
||||
cx.notify();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
impl ClipbinApp {
|
||||
pub fn step_playback(&mut self, forward: bool) {
|
||||
if !matches!(self.route, Route::Session(_)) {
|
||||
return;
|
||||
}
|
||||
self.editor.playing = false;
|
||||
self.playback.player.play(false);
|
||||
let snapshot = self.playback.player.snapshot();
|
||||
let step = Duration::from_secs_f64(1. / snapshot.frames_per_second.max(1) as f64);
|
||||
let time = snapshot.video_time.unwrap_or(self.editor.playhead);
|
||||
// Round towards the selected frame when nanoseconds cannot represent a frame period.
|
||||
let at = if forward {
|
||||
time + step
|
||||
} else {
|
||||
time.saturating_sub(step)
|
||||
} + Duration::from_nanos(2);
|
||||
self.seek_playback(at);
|
||||
}
|
||||
}
|
||||
@@ -92,7 +92,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context<ClipbinApp>) -> AnyElement {
|
||||
.flex_col()
|
||||
.gap_3()
|
||||
.child(
|
||||
thumbnail(&clip.game).child(
|
||||
thumbnail(&clip.game, app.playback.clip_thumbnails.get(&clip.id)).child(
|
||||
corner()
|
||||
.bottom_2p5()
|
||||
.right_2p5()
|
||||
|
||||
@@ -6,15 +6,14 @@ use clipbin_core::library::{
|
||||
Session, SpanKind, export_range, format_ago, format_clock, format_length,
|
||||
};
|
||||
use gpui::{
|
||||
AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, IntoElement, MouseButton,
|
||||
MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, ScrollWheelEvent, canvas, div,
|
||||
pattern_slash, prelude::*, px, relative,
|
||||
AnyElement, Bounds, ClickEvent, Context, FontWeight, Hsla, ImageSource, IntoElement,
|
||||
MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ObjectFit, Pixels, Point,
|
||||
ScrollWheelEvent, canvas, div, img, pattern_slash, prelude::*, px, relative,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
app::{ClipbinApp, Drag, Route},
|
||||
components::{ButtonStyle, aspect_ratio, button, chip, icon, panel, row},
|
||||
screens::backdrop,
|
||||
theme,
|
||||
timeline_view::TimelineView,
|
||||
};
|
||||
@@ -73,37 +72,54 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
);
|
||||
|
||||
let preview = aspect_ratio(div(), 16. / 9.)
|
||||
.h(px(app.playback.preview_height))
|
||||
.flex_shrink_0()
|
||||
.relative()
|
||||
.w_full()
|
||||
.overflow_hidden()
|
||||
.rounded_lg()
|
||||
.bg(gpui::black())
|
||||
.child(backdrop(&session.game).absolute().size_full().opacity(0.55))
|
||||
.child(
|
||||
.children(app.playback.image.clone().map(|image| {
|
||||
img(ImageSource::Render(image))
|
||||
.absolute()
|
||||
.size_full()
|
||||
.object_fit(ObjectFit::Contain)
|
||||
}))
|
||||
.children(app.playback.message.clone().map(|message| {
|
||||
div()
|
||||
.absolute()
|
||||
.size_full()
|
||||
.flex()
|
||||
.flex_col()
|
||||
.items_center()
|
||||
.justify_center()
|
||||
.gap_3()
|
||||
.child(
|
||||
div()
|
||||
.flex()
|
||||
.items_center()
|
||||
.justify_center()
|
||||
.size(px(56.))
|
||||
.rounded_full()
|
||||
.bg(theme::overlay())
|
||||
.child(icon("play").size_6().text_color(theme::foreground())),
|
||||
)
|
||||
.child(
|
||||
div()
|
||||
.text_xs()
|
||||
.text_color(theme::muted())
|
||||
.child("Preview arrives with playback"),
|
||||
),
|
||||
.child(message)
|
||||
}))
|
||||
.child(
|
||||
canvas(
|
||||
{
|
||||
let player = cx.entity().downgrade();
|
||||
move |bounds, window, cx| {
|
||||
let scale = window.scale_factor();
|
||||
let size = (
|
||||
(f32::from(bounds.size.width) * scale) as u32,
|
||||
(f32::from(bounds.size.height) * scale) as u32,
|
||||
);
|
||||
let _ = player.update(cx, |app, cx| {
|
||||
app.playback.player.resize(size);
|
||||
let height = f32::from(bounds.size.width) * 9. / 16.;
|
||||
if (app.playback.preview_height - height).abs() > 1. {
|
||||
app.playback.preview_height = height;
|
||||
cx.notify();
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
|_, _, _, _| {},
|
||||
)
|
||||
.absolute()
|
||||
.size_full(),
|
||||
)
|
||||
.child(
|
||||
div()
|
||||
@@ -126,6 +142,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
.map(|b| b.at);
|
||||
let transport = div()
|
||||
.flex()
|
||||
.flex_wrap()
|
||||
.items_center()
|
||||
.gap_1()
|
||||
.child(
|
||||
@@ -136,8 +153,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
None,
|
||||
)
|
||||
.on_click(cx.listener(|app, _: &ClickEvent, _, cx| {
|
||||
app.notice =
|
||||
Some("Playback isn't connected yet; it comes with the decoder.".into());
|
||||
app.toggle_playback();
|
||||
cx.notify();
|
||||
})),
|
||||
)
|
||||
@@ -150,7 +166,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
)
|
||||
.on_click(cx.listener(move |app, _: &ClickEvent, _, cx| {
|
||||
if let Some(at) = previous_bookmark {
|
||||
app.editor.playhead = at;
|
||||
app.seek_playback(at);
|
||||
cx.notify();
|
||||
}
|
||||
})),
|
||||
@@ -164,7 +180,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
)
|
||||
.on_click(cx.listener(move |app, _: &ClickEvent, _, cx| {
|
||||
if let Some(at) = next_bookmark {
|
||||
app.editor.playhead = at;
|
||||
app.seek_playback(at);
|
||||
cx.notify();
|
||||
}
|
||||
})),
|
||||
@@ -187,6 +203,52 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
.child(format!("/ {}", format_clock(length))),
|
||||
),
|
||||
)
|
||||
.child(
|
||||
button(
|
||||
"mute",
|
||||
ButtonStyle::Ghost,
|
||||
Some(if app.playback.muted {
|
||||
"volume-x"
|
||||
} else {
|
||||
"volume-2"
|
||||
}),
|
||||
None,
|
||||
)
|
||||
.on_click(cx.listener(|app, _: &ClickEvent, _, cx| {
|
||||
app.playback.muted = !app.playback.muted;
|
||||
app.playback
|
||||
.player
|
||||
.audio(app.playback.muted, app.playback.volume);
|
||||
cx.notify();
|
||||
})),
|
||||
)
|
||||
.child(
|
||||
button("volume-down", ButtonStyle::Ghost, None, Some("−")).on_click(cx.listener(
|
||||
|app, _: &ClickEvent, _, cx| {
|
||||
app.playback.volume = (app.playback.volume - 0.1).max(0.);
|
||||
app.playback
|
||||
.player
|
||||
.audio(app.playback.muted, app.playback.volume);
|
||||
cx.notify();
|
||||
},
|
||||
)),
|
||||
)
|
||||
.child(
|
||||
div()
|
||||
.text_xs()
|
||||
.child(format!("{:.0}%", app.playback.volume * 100.)),
|
||||
)
|
||||
.child(
|
||||
button("volume-up", ButtonStyle::Ghost, None, Some("+")).on_click(cx.listener(
|
||||
|app, _: &ClickEvent, _, cx| {
|
||||
app.playback.volume = (app.playback.volume + 0.1).min(1.);
|
||||
app.playback
|
||||
.player
|
||||
.audio(app.playback.muted, app.playback.volume);
|
||||
cx.notify();
|
||||
},
|
||||
)),
|
||||
)
|
||||
.child(div().flex_1())
|
||||
.child(
|
||||
div()
|
||||
@@ -210,6 +272,12 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
.gap_3()
|
||||
.child(preview)
|
||||
.child(transport)
|
||||
.children(
|
||||
app.playback
|
||||
.audio_error
|
||||
.clone()
|
||||
.map(|error| div().text_xs().text_color(theme::warning()).child(error)),
|
||||
)
|
||||
.child(timeline(app, session, cx));
|
||||
|
||||
let (export_start, export_end) = app
|
||||
@@ -337,7 +405,7 @@ pub fn render(app: &ClipbinApp, id: u64, cx: &mut Context<ClipbinApp>) -> AnyEle
|
||||
)
|
||||
})
|
||||
.on_click(cx.listener(move |app, _: &ClickEvent, _, cx| {
|
||||
app.editor.playhead = at;
|
||||
app.seek_playback(at);
|
||||
app.editor.selection = range;
|
||||
cx.notify();
|
||||
}))
|
||||
@@ -546,7 +614,23 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context<ClipbinApp>) -
|
||||
.size_full()
|
||||
.overflow_hidden()
|
||||
.child(
|
||||
canvas(move |area, _, _| bounds.set(Some(area)), |_, _, _, _| {})
|
||||
canvas(
|
||||
move |area, _, _| {
|
||||
if bounds.get() != Some(area)
|
||||
&& std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some()
|
||||
{
|
||||
eprintln!(
|
||||
"playback_timeline x={} y={} width={} height={}",
|
||||
f32::from(area.left()),
|
||||
f32::from(area.top()),
|
||||
f32::from(area.size.width),
|
||||
f32::from(area.size.height)
|
||||
);
|
||||
}
|
||||
bounds.set(Some(area));
|
||||
},
|
||||
|_, _, _, _| {},
|
||||
)
|
||||
.absolute()
|
||||
.size_full(),
|
||||
)
|
||||
@@ -582,7 +666,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context<ClipbinApp>) -
|
||||
} else if near(end) {
|
||||
Drag::SelectionEnd
|
||||
} else {
|
||||
app.editor.playhead = time;
|
||||
app.seek_playback(time);
|
||||
Drag::Playhead
|
||||
});
|
||||
cx.notify();
|
||||
@@ -600,7 +684,7 @@ fn timeline(app: &ClipbinApp, session: &Session, cx: &mut Context<ClipbinApp>) -
|
||||
let (start, end) = app.editor.selection;
|
||||
let minimum = Duration::from_secs(1);
|
||||
match drag {
|
||||
Drag::Playhead => app.editor.playhead = time,
|
||||
Drag::Playhead => app.seek_playback(time),
|
||||
Drag::SelectionStart => {
|
||||
app.editor.selection.0 = time.min(end.saturating_sub(minimum))
|
||||
}
|
||||
|
||||
@@ -9,7 +9,20 @@ use crate::{components::aspect_ratio, theme};
|
||||
|
||||
/// A 16:9 thumbnail. Until frames are decoded, each game gets its own colour
|
||||
/// with its initials, so cards stay recognisable.
|
||||
pub fn thumbnail(game: &str) -> Div {
|
||||
pub fn thumbnail(game: &str, path: Option<&std::path::PathBuf>) -> Div {
|
||||
if let Some(path) = path {
|
||||
return aspect_ratio(div(), 16. / 9.)
|
||||
.relative()
|
||||
.w_full()
|
||||
.overflow_hidden()
|
||||
.rounded_xl()
|
||||
.child(
|
||||
gpui::img(path.clone())
|
||||
.absolute()
|
||||
.size_full()
|
||||
.object_fit(gpui::ObjectFit::Cover),
|
||||
);
|
||||
}
|
||||
let tint = game_tint(game);
|
||||
let initials: String = game
|
||||
.split_whitespace()
|
||||
|
||||
@@ -61,7 +61,7 @@ pub fn render(app: &ClipbinApp, cx: &mut Context<ClipbinApp>) -> AnyElement {
|
||||
.cursor_pointer()
|
||||
.group("card")
|
||||
.child(
|
||||
thumbnail(&session.game)
|
||||
thumbnail(&session.game, app.playback.thumbnails.get(&session.id))
|
||||
.border_1()
|
||||
.border_color(gpui::transparent_black())
|
||||
.group_hover("card", |style| {
|
||||
|
||||
@@ -11,6 +11,14 @@ publish.workspace = true
|
||||
[dependencies]
|
||||
clipbin-core.workspace = true
|
||||
ffmpeg-next = { version = "9", default-features = false, features = ["codec", "format", "software-scaling", "software-resampling"] }
|
||||
image = { version = "0.25.10", default-features = false, features = ["png"] }
|
||||
cpal = { version = "0.18.2", features = ["pulseaudio"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rustix = { version = "1", features = ["process", "thread"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_System_Threading"] }
|
||||
|
||||
[lints.rust]
|
||||
unsafe_code = "deny"
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Headless timing check against real segments; run inside the rig for device audio.
|
||||
use clipbin_core::store::Store;
|
||||
use clipbin_media::player::{Picture, Player};
|
||||
use std::{
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
fn main() {
|
||||
let directory = std::env::args().nth(1).expect("session root");
|
||||
let id = std::env::args()
|
||||
.nth(2)
|
||||
.unwrap_or_else(|| "1".into())
|
||||
.parse()
|
||||
.unwrap();
|
||||
let player = Player::new();
|
||||
player.resize((800, 450));
|
||||
player.open(
|
||||
Store {
|
||||
root: directory.into(),
|
||||
},
|
||||
id,
|
||||
Duration::ZERO,
|
||||
);
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
player.play(true);
|
||||
let begin = Instant::now();
|
||||
let mut last = None;
|
||||
let mut frames = 0;
|
||||
for _ in 0..700 {
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
let snapshot = player.snapshot();
|
||||
assert!(snapshot.audio_error.is_none(), "{:?}", snapshot.audio_error);
|
||||
if let Picture::Frame(frame) = snapshot.picture {
|
||||
if let Some(video_time) = snapshot.video_time {
|
||||
assert!(
|
||||
video_time.abs_diff(snapshot.time) <= Duration::from_millis(20),
|
||||
"video out of sync: {video_time:?} vs {:?}",
|
||||
snapshot.time
|
||||
);
|
||||
}
|
||||
if last
|
||||
.as_ref()
|
||||
.is_none_or(|previous| !std::sync::Arc::ptr_eq(previous, &frame))
|
||||
{
|
||||
frames += 1;
|
||||
}
|
||||
last = Some(frame);
|
||||
}
|
||||
}
|
||||
let snapshot = player.snapshot();
|
||||
println!(
|
||||
"playback_check wall={:.3} session={:.3} displayed_frames={frames}",
|
||||
begin.elapsed().as_secs_f64(),
|
||||
snapshot.time.as_secs_f64()
|
||||
);
|
||||
assert!(
|
||||
snapshot.time >= Duration::from_secs(6),
|
||||
"audio master clock stalled: {snapshot:?}"
|
||||
);
|
||||
player.audio(false, 0.5);
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
player.audio(true, 0.5);
|
||||
let before = snapshot.time;
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
assert!(player.snapshot().time > before, "muted wall clock stalled");
|
||||
player.close();
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
//! Combined-track decoding and bounded device output. No decoder runs in a callback.
|
||||
use crate::ffmpeg::{self, format, media, software::resampling, util::frame::Audio as AudioFrame};
|
||||
use clipbin_core::store::Metadata;
|
||||
use cpal::{
|
||||
FromSample, SizedSample,
|
||||
traits::{DeviceTrait, HostTrait, StreamTrait},
|
||||
};
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
struct Block {
|
||||
time: Duration,
|
||||
samples: Vec<f32>,
|
||||
consumed: usize,
|
||||
}
|
||||
struct Buffer {
|
||||
blocks: VecDeque<Block>,
|
||||
queued: usize,
|
||||
playing: bool,
|
||||
volume: f32,
|
||||
// The time of the first sample actually scheduled for the device.
|
||||
anchors: VecDeque<(Duration, Instant, Duration)>,
|
||||
error: Option<String>,
|
||||
}
|
||||
impl Buffer {
|
||||
fn time(&mut self, now: Instant) -> Option<Duration> {
|
||||
while self.anchors.len() > 1 && self.anchors[1].1 <= now {
|
||||
self.anchors.pop_front();
|
||||
}
|
||||
self.anchors.front().and_then(|(time, begins, span)| {
|
||||
(now <= *begins + *span + Duration::from_millis(100))
|
||||
.then(|| *time + now.saturating_duration_since(*begins).min(*span))
|
||||
})
|
||||
}
|
||||
}
|
||||
pub struct Audio {
|
||||
_stream: cpal::Stream,
|
||||
buffer: Arc<Mutex<Buffer>>,
|
||||
rate: u32,
|
||||
channels: u16,
|
||||
decoder: Option<(PathBuf, AudioDecoder)>,
|
||||
next: Duration,
|
||||
}
|
||||
impl Audio {
|
||||
pub fn new() -> Result<Self, String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Ok(host) = cpal::host_from_id(cpal::HostId::PulseAudio) {
|
||||
match Self::with_host(host) {
|
||||
Ok(audio) => return Ok(audio),
|
||||
Err(error) => eprintln!("PulseAudio output unavailable: {error}; trying ALSA"),
|
||||
}
|
||||
}
|
||||
Self::with_host(cpal::default_host())
|
||||
}
|
||||
fn with_host(host: cpal::Host) -> Result<Self, String> {
|
||||
let device = host
|
||||
.default_output_device()
|
||||
.ok_or("No audio output device")?;
|
||||
let supported = device.default_output_config().map_err(|e| e.to_string())?;
|
||||
let format = supported.sample_format();
|
||||
let config: cpal::StreamConfig = supported.into();
|
||||
let rate = config.sample_rate;
|
||||
let channels = config.channels;
|
||||
let buffer = Arc::new(Mutex::new(Buffer {
|
||||
blocks: VecDeque::new(),
|
||||
queued: 0,
|
||||
playing: false,
|
||||
volume: 1.,
|
||||
anchors: VecDeque::new(),
|
||||
error: None,
|
||||
}));
|
||||
let stream = match format {
|
||||
cpal::SampleFormat::F32 => output::<f32>(&device, config, buffer.clone()),
|
||||
cpal::SampleFormat::I16 => output::<i16>(&device, config, buffer.clone()),
|
||||
cpal::SampleFormat::U16 => output::<u16>(&device, config, buffer.clone()),
|
||||
cpal::SampleFormat::I32 => output::<i32>(&device, config, buffer.clone()),
|
||||
cpal::SampleFormat::F64 => output::<f64>(&device, config, buffer.clone()),
|
||||
_ => return Err(format!("Unsupported audio device format: {format}")),
|
||||
}
|
||||
.map_err(|e| e.to_string())?;
|
||||
stream.play().map_err(|e| e.to_string())?;
|
||||
eprintln!("playback_audio rate={rate} channels={channels} format={format}");
|
||||
Ok(Self {
|
||||
_stream: stream,
|
||||
buffer,
|
||||
rate,
|
||||
channels,
|
||||
decoder: None,
|
||||
next: Duration::ZERO,
|
||||
})
|
||||
}
|
||||
pub fn reset(&mut self, time: Duration) {
|
||||
let mut buffer = self.buffer.lock().unwrap();
|
||||
buffer.blocks.clear();
|
||||
buffer.queued = 0;
|
||||
buffer.anchors.clear();
|
||||
self.decoder = None;
|
||||
self.next = time;
|
||||
}
|
||||
pub fn controls(&self, playing: bool, volume: f32) {
|
||||
let mut buffer = self.buffer.lock().unwrap();
|
||||
buffer.playing = playing;
|
||||
buffer.volume = volume.clamp(0., 1.);
|
||||
if !playing {
|
||||
buffer.anchors.clear();
|
||||
}
|
||||
}
|
||||
/// Device time, corrected for callback-to-DAC latency; bounded by scheduled samples.
|
||||
pub fn time(&self, now: Instant) -> Option<Duration> {
|
||||
self.buffer.lock().unwrap().time(now)
|
||||
}
|
||||
pub fn error(&self) -> Option<String> {
|
||||
self.buffer.lock().unwrap().error.clone()
|
||||
}
|
||||
pub fn catch_up(&mut self, time: Duration) {
|
||||
if self.next + Duration::from_millis(50) < time && self.buffer.lock().unwrap().queued == 0 {
|
||||
self.reset(time);
|
||||
}
|
||||
}
|
||||
pub fn fill(&mut self, metadata: &Metadata) -> Result<(), ffmpeg::Error> {
|
||||
// At most 250 ms buffered. All allocations/decoding happen on the player thread.
|
||||
let limit = self.rate as usize * self.channels as usize / 4;
|
||||
for _ in 0..16 {
|
||||
if self.buffer.lock().unwrap().queued >= limit {
|
||||
break;
|
||||
}
|
||||
let Some(at) = metadata.locate(self.next).filter(|at| at.segment.complete) else {
|
||||
break;
|
||||
};
|
||||
if self
|
||||
.decoder
|
||||
.as_ref()
|
||||
.is_none_or(|(path, _)| path != &at.segment.path)
|
||||
{
|
||||
self.decoder = Some((
|
||||
at.segment.path.clone(),
|
||||
AudioDecoder::open(&at.segment.path, self.rate, self.channels, at.offset)?,
|
||||
));
|
||||
}
|
||||
let (_, decoder) = self.decoder.as_mut().unwrap();
|
||||
let Some((offset, mut samples)) = decoder.next()? else {
|
||||
self.next = at.segment.end;
|
||||
self.decoder = None;
|
||||
continue;
|
||||
};
|
||||
let begin = at.segment.start + offset;
|
||||
let skip = ((self.next.saturating_sub(begin).as_secs_f64() * self.rate as f64).round()
|
||||
as usize)
|
||||
.min(samples.len() / self.channels as usize);
|
||||
samples.drain(..skip * self.channels as usize);
|
||||
let time = begin + Duration::from_secs_f64(skip as f64 / self.rate as f64);
|
||||
let keep = ((at.segment.end.saturating_sub(time).as_secs_f64() * self.rate as f64)
|
||||
.round() as usize)
|
||||
* self.channels as usize;
|
||||
samples.truncate(keep);
|
||||
if samples.is_empty() {
|
||||
continue;
|
||||
}
|
||||
self.next = time
|
||||
+ Duration::from_secs_f64(
|
||||
(samples.len() / self.channels as usize) as f64 / self.rate as f64,
|
||||
);
|
||||
if at.segment.end.saturating_sub(self.next)
|
||||
< Duration::from_nanos(1_000_000_000 / self.rate as u64)
|
||||
{
|
||||
self.next = at.segment.end;
|
||||
}
|
||||
let mut buffer = self.buffer.lock().unwrap();
|
||||
buffer.queued += samples.len();
|
||||
buffer.blocks.push_back(Block {
|
||||
time,
|
||||
samples,
|
||||
consumed: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn output<T: SizedSample + FromSample<f32>>(
|
||||
device: &cpal::Device,
|
||||
config: cpal::StreamConfig,
|
||||
buffer: Arc<Mutex<Buffer>>,
|
||||
) -> Result<cpal::Stream, cpal::Error> {
|
||||
let errors = buffer.clone();
|
||||
let rate = config.sample_rate;
|
||||
let channels = config.channels as usize;
|
||||
device.build_output_stream(
|
||||
config,
|
||||
move |data: &mut [T], info: &cpal::OutputCallbackInfo| {
|
||||
// try_lock guarantees UI/decoder contention cannot block the device callback.
|
||||
data.fill(T::from_sample(0.));
|
||||
let Ok(mut buffer) = buffer.try_lock() else {
|
||||
return;
|
||||
};
|
||||
if !buffer.playing {
|
||||
return;
|
||||
}
|
||||
let latency = info
|
||||
.timestamp()
|
||||
.playback
|
||||
.duration_since(info.timestamp().callback);
|
||||
let mut scheduled = None;
|
||||
for (frame_number, output) in data.chunks_exact_mut(channels).enumerate() {
|
||||
let volume = buffer.volume;
|
||||
let Some(block) = buffer.blocks.front_mut() else {
|
||||
break;
|
||||
};
|
||||
let time = block.time
|
||||
+ Duration::from_secs_f64((block.consumed / channels) as f64 / rate as f64);
|
||||
scheduled.get_or_insert((
|
||||
time,
|
||||
Instant::now()
|
||||
+ latency
|
||||
+ Duration::from_secs_f64(frame_number as f64 / rate as f64),
|
||||
Duration::ZERO,
|
||||
));
|
||||
for sample in output {
|
||||
*sample =
|
||||
T::from_sample((block.samples[block.consumed] * volume).clamp(-1., 1.));
|
||||
block.consumed += 1;
|
||||
}
|
||||
let done = block.consumed == block.samples.len();
|
||||
buffer.queued -= channels;
|
||||
if let Some((_, _, span)) = scheduled.as_mut() {
|
||||
*span += Duration::from_secs_f64(1. / rate as f64);
|
||||
}
|
||||
if done {
|
||||
buffer.blocks.pop_front();
|
||||
}
|
||||
}
|
||||
if let Some(scheduled) = scheduled {
|
||||
if buffer.anchors.len() == 32 {
|
||||
buffer.anchors.pop_front();
|
||||
}
|
||||
buffer.anchors.push_back(scheduled);
|
||||
}
|
||||
},
|
||||
move |error| {
|
||||
if let Ok(mut buffer) = errors.lock() {
|
||||
buffer.error = Some(format!("Audio output failed: {error}"));
|
||||
}
|
||||
},
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
struct AudioDecoder {
|
||||
input: format::context::Input,
|
||||
decoder: ffmpeg::decoder::Audio,
|
||||
resampler: resampling::Context,
|
||||
stream: usize,
|
||||
time_base: f64,
|
||||
origin: i64,
|
||||
rate: u32,
|
||||
channels: u16,
|
||||
eof: bool,
|
||||
next_time: Duration,
|
||||
initialised_time: bool,
|
||||
drained: bool,
|
||||
}
|
||||
impl AudioDecoder {
|
||||
fn open(path: &Path, rate: u32, channels: u16, seek: Duration) -> Result<Self, ffmpeg::Error> {
|
||||
crate::init();
|
||||
let mut input = format::input(path)?;
|
||||
// OBS writes system, mic, combined, in that order; a single track is already mixed.
|
||||
let tracks: Vec<_> = input
|
||||
.streams()
|
||||
.filter(|stream| stream.parameters().medium() == media::Type::Audio)
|
||||
.collect();
|
||||
let track = if tracks.len() >= 3 {
|
||||
&tracks[2]
|
||||
} else {
|
||||
tracks.first().ok_or(ffmpeg::Error::StreamNotFound)?
|
||||
};
|
||||
let stream = track.index();
|
||||
let time_base = f64::from(track.time_base());
|
||||
let origin = track.start_time().max(0);
|
||||
let decoder = ffmpeg::codec::context::Context::from_parameters(track.parameters())?
|
||||
.decoder()
|
||||
.audio()?;
|
||||
let layout = if decoder.channel_layout().channels() == 0 {
|
||||
ffmpeg::ChannelLayout::default(decoder.channels() as i32)
|
||||
} else {
|
||||
decoder.channel_layout()
|
||||
};
|
||||
let resampler = resampling::Context::get(
|
||||
decoder.format(),
|
||||
layout,
|
||||
decoder.rate(),
|
||||
format::Sample::F32(format::sample::Type::Packed),
|
||||
ffmpeg::ChannelLayout::default(channels as i32),
|
||||
rate,
|
||||
)?;
|
||||
if seek > Duration::ZERO {
|
||||
let timestamp = (seek.as_secs_f64() * 1_000_000.) as i64;
|
||||
input.seek(timestamp, ..timestamp)?;
|
||||
}
|
||||
Ok(Self {
|
||||
input,
|
||||
decoder,
|
||||
resampler,
|
||||
stream,
|
||||
time_base,
|
||||
origin,
|
||||
rate,
|
||||
channels,
|
||||
eof: false,
|
||||
next_time: seek,
|
||||
initialised_time: false,
|
||||
drained: false,
|
||||
})
|
||||
}
|
||||
fn next(&mut self) -> Result<Option<(Duration, Vec<f32>)>, ffmpeg::Error> {
|
||||
loop {
|
||||
let mut frame = AudioFrame::empty();
|
||||
match self.decoder.receive_frame(&mut frame) {
|
||||
Ok(()) => {
|
||||
let timestamp = frame.timestamp().map_or(self.next_time, |pts| {
|
||||
Duration::from_secs_f64((pts - self.origin).max(0) as f64 * self.time_base)
|
||||
});
|
||||
let time = if self.initialised_time {
|
||||
self.next_time
|
||||
} else {
|
||||
timestamp
|
||||
};
|
||||
self.initialised_time = true;
|
||||
// Allocate enough output for upsampling; ffmpeg-next's empty output uses input count.
|
||||
let count = ((frame.samples() as u64 * self.rate as u64)
|
||||
.div_ceil(frame.rate() as u64)
|
||||
+ 256) as usize;
|
||||
let mut converted = AudioFrame::new(
|
||||
format::Sample::F32(format::sample::Type::Packed),
|
||||
count,
|
||||
ffmpeg::ChannelLayout::default(self.channels as i32),
|
||||
);
|
||||
converted.set_rate(self.rate);
|
||||
self.resampler.run(&frame, &mut converted)?;
|
||||
let samples = converted.plane::<f32>(0).to_vec();
|
||||
self.next_time = time
|
||||
+ Duration::from_secs_f64(converted.samples() as f64 / self.rate as f64);
|
||||
return Ok(Some((time, samples)));
|
||||
}
|
||||
Err(ffmpeg::Error::Eof) => {
|
||||
if self.drained {
|
||||
return Ok(None);
|
||||
}
|
||||
self.drained = true;
|
||||
let mut converted = AudioFrame::new(
|
||||
format::Sample::F32(format::sample::Type::Packed),
|
||||
256,
|
||||
ffmpeg::ChannelLayout::default(self.channels as i32),
|
||||
);
|
||||
converted.set_rate(self.rate);
|
||||
self.resampler.flush(&mut converted)?;
|
||||
return if converted.samples() == 0 {
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some((self.next_time, converted.plane::<f32>(0).to_vec())))
|
||||
};
|
||||
}
|
||||
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
if self.eof {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut packet = ffmpeg::Packet::empty();
|
||||
loop {
|
||||
match packet.read(&mut self.input) {
|
||||
Ok(()) if packet.stream() == self.stream => {
|
||||
self.decoder.send_packet(&packet)?;
|
||||
break;
|
||||
}
|
||||
Ok(()) => {}
|
||||
Err(ffmpeg::Error::Eof) => {
|
||||
self.eof = true;
|
||||
self.decoder.send_eof()?;
|
||||
break;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn device_clock_waits_for_latency_and_stops_at_scheduled_samples() {
|
||||
let now = Instant::now();
|
||||
let mut buffer = Buffer {
|
||||
blocks: VecDeque::new(),
|
||||
queued: 0,
|
||||
playing: true,
|
||||
volume: 1.,
|
||||
anchors: VecDeque::new(),
|
||||
error: None,
|
||||
};
|
||||
buffer.anchors.push_back((
|
||||
Duration::from_secs(2),
|
||||
now + Duration::from_millis(100),
|
||||
Duration::from_millis(50),
|
||||
));
|
||||
buffer.anchors.push_back((
|
||||
Duration::from_millis(2050),
|
||||
now + Duration::from_millis(150),
|
||||
Duration::from_millis(50),
|
||||
));
|
||||
assert_eq!(buffer.time(now), Some(Duration::from_secs(2)));
|
||||
assert_eq!(
|
||||
buffer.time(now + Duration::from_millis(125)),
|
||||
Some(Duration::from_millis(2025))
|
||||
);
|
||||
assert_eq!(
|
||||
buffer.time(now + Duration::from_millis(200)),
|
||||
Some(Duration::from_millis(2100))
|
||||
);
|
||||
assert_eq!(buffer.anchors.len(), 1);
|
||||
assert_eq!(buffer.time(now + Duration::from_secs(5)), None);
|
||||
}
|
||||
#[test]
|
||||
fn resampling_selects_combined_track_and_preserves_its_tone() {
|
||||
let Some(directory) = std::env::var_os("CLIPBIN_TEST_MEDIA_DIR") else {
|
||||
return;
|
||||
};
|
||||
let mut decoder = AudioDecoder::open(
|
||||
&PathBuf::from(directory).join("combined.mkv"),
|
||||
44_100,
|
||||
1,
|
||||
Duration::ZERO,
|
||||
)
|
||||
.unwrap();
|
||||
let mut samples = Vec::new();
|
||||
while let Some((_, block)) = decoder.next().unwrap() {
|
||||
samples.extend(block);
|
||||
}
|
||||
assert!(samples.len() > 80_000 && samples.len() < 95_000);
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| pair[0] <= 0. && pair[1] > 0.)
|
||||
.count();
|
||||
let hz = crossings as f64 / (samples.len() as f64 / 44_100.);
|
||||
assert!(
|
||||
(435. ..445.).contains(&hz),
|
||||
"combined track was not the 440 Hz tone: {hz}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
//! Single-threaded file decoding. Scale only frames which will be displayed.
|
||||
use crate::ffmpeg::{self, codec, format, media, software::scaling, util::frame::video::Video};
|
||||
use std::{path::Path, time::Duration};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Frame {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Packed BGRA, without FFmpeg's row padding.
|
||||
pub bgra: Vec<u8>,
|
||||
}
|
||||
|
||||
pub struct Decoder {
|
||||
input: format::context::Input,
|
||||
decoder: ffmpeg::decoder::Video,
|
||||
stream: usize,
|
||||
time_base: f64,
|
||||
origin: i64,
|
||||
fps: u32,
|
||||
next_index: u64,
|
||||
eof: bool,
|
||||
scaler: Option<(u32, u32, format::Pixel, u32, u32, scaling::context::Context)>,
|
||||
}
|
||||
|
||||
impl Decoder {
|
||||
pub fn open(path: &Path, fps: u32) -> Result<Self, ffmpeg::Error> {
|
||||
crate::init();
|
||||
let input = format::input(path)?;
|
||||
let video = input
|
||||
.streams()
|
||||
.best(media::Type::Video)
|
||||
.ok_or(ffmpeg::Error::StreamNotFound)?;
|
||||
let stream = video.index();
|
||||
let time_base = f64::from(video.time_base());
|
||||
let origin = video.start_time().max(0);
|
||||
let mut context = codec::context::Context::from_parameters(video.parameters())?;
|
||||
// Never let FFmpeg create a pool competing with the recorder.
|
||||
context.set_threading(codec::threading::Config::count(1));
|
||||
let decoder = context.decoder().video()?;
|
||||
Ok(Self {
|
||||
input,
|
||||
decoder,
|
||||
stream,
|
||||
time_base,
|
||||
origin,
|
||||
fps: fps.max(1),
|
||||
next_index: 0,
|
||||
eof: false,
|
||||
scaler: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reuse swscale's setup while the preview dimensions stay unchanged.
|
||||
pub fn scale(&mut self, video: &Video, target: (u32, u32)) -> Result<Frame, ffmpeg::Error> {
|
||||
let (width, height) = fitted_size((video.width(), video.height()), target);
|
||||
if self
|
||||
.scaler
|
||||
.as_ref()
|
||||
.is_none_or(|(w, h, pixel, source_w, source_h, _)| {
|
||||
(*w, *h) != (width, height)
|
||||
|| *pixel != video.format()
|
||||
|| (*source_w, *source_h) != (video.width(), video.height())
|
||||
})
|
||||
{
|
||||
self.scaler = Some((
|
||||
width,
|
||||
height,
|
||||
video.format(),
|
||||
video.width(),
|
||||
video.height(),
|
||||
scaling::context::Context::get(
|
||||
video.format(),
|
||||
video.width(),
|
||||
video.height(),
|
||||
format::Pixel::BGRA,
|
||||
width,
|
||||
height,
|
||||
scaling::flag::Flags::FAST_BILINEAR,
|
||||
)?,
|
||||
));
|
||||
}
|
||||
let (_, _, _, _, _, scaler) = self.scaler.as_mut().unwrap();
|
||||
packed_frame(scaler, video, width, height)
|
||||
}
|
||||
|
||||
/// FFmpeg seeks backwards to a keyframe; callers decode forward by frame index.
|
||||
pub fn seek(&mut self, time: Duration) -> Result<(), ffmpeg::Error> {
|
||||
let timestamp = (time.as_secs_f64() * 1_000_000.
|
||||
+ self.origin as f64 * self.time_base * 1_000_000.) as i64;
|
||||
self.input.seek(timestamp, ..timestamp)?;
|
||||
self.decoder.flush();
|
||||
self.next_index = frame_index(time, self.fps);
|
||||
self.eof = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn next_frame(&mut self) -> Result<Option<(u64, Video)>, ffmpeg::Error> {
|
||||
loop {
|
||||
let mut frame = Video::empty();
|
||||
match self.decoder.receive_frame(&mut frame) {
|
||||
Ok(()) => {
|
||||
let index = frame.timestamp().map_or(self.next_index, |pts| {
|
||||
(((pts - self.origin).max(0) as f64 * self.time_base * self.fps as f64)
|
||||
.round()) as u64
|
||||
});
|
||||
self.next_index = index + 1;
|
||||
return Ok(Some((index, frame)));
|
||||
}
|
||||
Err(ffmpeg::Error::Eof) => return Ok(None),
|
||||
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::error::EAGAIN => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
if self.eof {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut packet = ffmpeg::Packet::empty();
|
||||
loop {
|
||||
match packet.read(&mut self.input) {
|
||||
Ok(()) if packet.stream() == self.stream => {
|
||||
self.decoder.send_packet(&packet)?;
|
||||
break;
|
||||
}
|
||||
Ok(()) => {}
|
||||
Err(ffmpeg::Error::Eof) => {
|
||||
self.eof = true;
|
||||
self.decoder.send_eof()?;
|
||||
break;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn frame_index(time: Duration, fps: u32) -> u64 {
|
||||
(time.as_nanos() * u128::from(fps.max(1)) / 1_000_000_000) as u64
|
||||
}
|
||||
|
||||
/// Preserve aspect ratio and cap allocation even on a very large display.
|
||||
pub fn fitted_size(source: (u32, u32), target: (u32, u32)) -> (u32, u32) {
|
||||
let ratio = (target.0.clamp(1, 1920) as f64 / source.0.max(1) as f64)
|
||||
.min(target.1.clamp(1, 1080) as f64 / source.1.max(1) as f64)
|
||||
.min(1.);
|
||||
(
|
||||
(source.0 as f64 * ratio).round().max(1.) as u32,
|
||||
(source.1 as f64 * ratio).round().max(1.) as u32,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn scale(video: &Video, target: (u32, u32)) -> Result<Frame, ffmpeg::Error> {
|
||||
let (width, height) = fitted_size((video.width(), video.height()), target);
|
||||
let mut scaler = scaling::context::Context::get(
|
||||
video.format(),
|
||||
video.width(),
|
||||
video.height(),
|
||||
format::Pixel::BGRA,
|
||||
width,
|
||||
height,
|
||||
scaling::flag::Flags::FAST_BILINEAR,
|
||||
)?;
|
||||
packed_frame(&mut scaler, video, width, height)
|
||||
}
|
||||
|
||||
fn packed_frame(
|
||||
scaler: &mut scaling::context::Context,
|
||||
video: &Video,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<Frame, ffmpeg::Error> {
|
||||
let mut output = Video::empty();
|
||||
scaler.run(video, &mut output)?;
|
||||
let mut bgra = Vec::with_capacity(width as usize * height as usize * 4);
|
||||
for row in 0..height as usize {
|
||||
let start = row * output.stride(0);
|
||||
bgra.extend_from_slice(&output.data(0)[start..start + width as usize * 4]);
|
||||
}
|
||||
Ok(Frame {
|
||||
width,
|
||||
height,
|
||||
bgra,
|
||||
})
|
||||
}
|
||||
|
||||
/// A thumbnail from the first decoded frame of any MP4/MKV path.
|
||||
pub fn first_frame(path: &Path, target: (u32, u32)) -> Result<Frame, ffmpeg::Error> {
|
||||
let mut decoder = Decoder::open(path, 60)?;
|
||||
let (_, frame) = decoder.next_frame()?.ok_or(ffmpeg::Error::Eof)?;
|
||||
scale(&frame, target)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn frame_exact_seek_and_bounded_aspect_ratio() {
|
||||
assert_eq!(frame_index(Duration::from_millis(1999), 60), 119);
|
||||
assert_eq!(frame_index(Duration::from_secs(2), 60), 120);
|
||||
assert_eq!(fitted_size((2560, 1440), (800, 450)), (800, 450));
|
||||
assert_eq!(fitted_size((1440, 2560), (800, 450)), (253, 450));
|
||||
assert_eq!(fitted_size((8000, 8000), (8000, 8000)), (1080, 1080));
|
||||
}
|
||||
}
|
||||
@@ -68,4 +68,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
pub mod audio;
|
||||
pub mod decode;
|
||||
pub mod export;
|
||||
pub mod player;
|
||||
pub mod thumbnail;
|
||||
|
||||
mod priority;
|
||||
#[cfg(windows)]
|
||||
#[allow(unsafe_code)]
|
||||
mod windows_priority;
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
//! On-demand background playback with a one-frame mailbox and coalesced controls.
|
||||
use crate::decode::{self, Decoder, Frame};
|
||||
use clipbin_core::store::{Metadata, Store};
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{Arc, Condvar, Mutex},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Picture {
|
||||
Loading,
|
||||
Frame(Arc<Frame>),
|
||||
Gap,
|
||||
Error(String),
|
||||
}
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Snapshot {
|
||||
pub time: Duration,
|
||||
pub playing: bool,
|
||||
pub picture: Picture,
|
||||
pub revision: u64,
|
||||
pub audio_error: Option<String>,
|
||||
pub video_time: Option<Duration>,
|
||||
pub frames_per_second: u32,
|
||||
}
|
||||
struct Controls {
|
||||
session: Option<(Store, u64)>,
|
||||
seek: Option<Duration>,
|
||||
latest: bool,
|
||||
playing: bool,
|
||||
size: (u32, u32),
|
||||
generation: u64,
|
||||
muted: bool,
|
||||
volume: f32,
|
||||
quit: bool,
|
||||
snapshot: Snapshot,
|
||||
}
|
||||
struct Shared {
|
||||
state: Mutex<Controls>,
|
||||
wake: Condvar,
|
||||
}
|
||||
|
||||
pub struct Player {
|
||||
shared: Arc<Shared>,
|
||||
}
|
||||
impl Default for Player {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
impl Player {
|
||||
pub fn new() -> Self {
|
||||
let shared = Arc::new(Shared {
|
||||
state: Mutex::new(Controls {
|
||||
session: None,
|
||||
seek: None,
|
||||
latest: false,
|
||||
playing: false,
|
||||
size: (800, 450),
|
||||
generation: 0,
|
||||
muted: false,
|
||||
volume: 1.,
|
||||
quit: false,
|
||||
snapshot: Snapshot {
|
||||
time: Duration::ZERO,
|
||||
playing: false,
|
||||
picture: Picture::Loading,
|
||||
revision: 0,
|
||||
audio_error: None,
|
||||
video_time: None,
|
||||
frames_per_second: 60,
|
||||
},
|
||||
}),
|
||||
wake: Condvar::new(),
|
||||
});
|
||||
let worker = shared.clone();
|
||||
thread::Builder::new()
|
||||
.name("clipbin-player".into())
|
||||
.spawn(move || run(worker))
|
||||
.expect("player thread");
|
||||
Self { shared }
|
||||
}
|
||||
pub fn open(&self, store: Store, id: u64, time: Duration) {
|
||||
self.open_session(store, id, time, false);
|
||||
}
|
||||
pub fn open_latest(&self, store: Store, id: u64) {
|
||||
self.open_session(store, id, Duration::ZERO, true);
|
||||
}
|
||||
fn open_session(&self, store: Store, id: u64, time: Duration, latest: bool) {
|
||||
let mut state = self.shared.state.lock().unwrap();
|
||||
state.session = Some((store, id));
|
||||
state.latest = latest;
|
||||
state.seek = Some(time);
|
||||
state.playing = false;
|
||||
state.generation += 1;
|
||||
state.snapshot.picture = Picture::Loading;
|
||||
state.snapshot.revision += 1;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
pub fn close(&self) {
|
||||
let mut state = self.shared.state.lock().unwrap();
|
||||
state.session = None;
|
||||
state.playing = false;
|
||||
state.generation += 1;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
pub fn seek(&self, time: Duration) {
|
||||
let mut state = self.shared.state.lock().unwrap();
|
||||
state.seek = Some(time);
|
||||
state.snapshot.time = time;
|
||||
state.snapshot.picture = Picture::Loading;
|
||||
state.snapshot.revision += 1;
|
||||
state.generation += 1;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
pub fn play(&self, playing: bool) {
|
||||
self.shared.state.lock().unwrap().playing = playing;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
pub fn audio(&self, muted: bool, volume: f32) {
|
||||
let mut state = self.shared.state.lock().unwrap();
|
||||
if state.muted != muted {
|
||||
state.seek = Some(state.snapshot.time);
|
||||
state.generation += 1;
|
||||
}
|
||||
state.muted = muted;
|
||||
state.volume = volume.clamp(0., 1.);
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
pub fn resize(&self, size: (u32, u32)) {
|
||||
let mut state = self.shared.state.lock().unwrap();
|
||||
if state.size != size {
|
||||
state.size = size;
|
||||
state.generation += 1;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
}
|
||||
pub fn snapshot(&self) -> Snapshot {
|
||||
self.shared.state.lock().unwrap().snapshot.clone()
|
||||
}
|
||||
}
|
||||
impl Drop for Player {
|
||||
fn drop(&mut self) {
|
||||
self.shared.state.lock().unwrap().quit = true;
|
||||
self.shared.wake.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
/// Monotonic session clock: decoding delays never accumulate into playback time.
|
||||
#[derive(Debug)]
|
||||
pub struct Clock {
|
||||
base: Duration,
|
||||
started: Instant,
|
||||
playing: bool,
|
||||
}
|
||||
impl Clock {
|
||||
pub fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
base: Duration::ZERO,
|
||||
started: now,
|
||||
playing: false,
|
||||
}
|
||||
}
|
||||
pub fn time(&self, now: Instant) -> Duration {
|
||||
self.base
|
||||
+ if self.playing {
|
||||
now.saturating_duration_since(self.started)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
}
|
||||
pub fn set(&mut self, time: Duration, playing: bool, now: Instant) {
|
||||
self.base = time;
|
||||
self.started = now;
|
||||
self.playing = playing;
|
||||
}
|
||||
}
|
||||
|
||||
fn run(shared: Arc<Shared>) {
|
||||
crate::priority::lower(false);
|
||||
let mut clock = Clock::new(Instant::now());
|
||||
let mut generation = u64::MAX;
|
||||
let mut metadata: Option<Metadata> = None;
|
||||
let mut refreshed = Instant::now();
|
||||
let mut decoder: Option<(PathBuf, Decoder, u64)> = None;
|
||||
let mut audio: Option<crate::audio::Audio> = None;
|
||||
let mut audio_error = None;
|
||||
let mut audio_attempted = false;
|
||||
loop {
|
||||
let (session, seek, playing, size, requested, muted, volume, latest) = {
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
if state.quit {
|
||||
break;
|
||||
}
|
||||
(
|
||||
state.session.clone(),
|
||||
state.seek.take(),
|
||||
state.playing,
|
||||
state.size,
|
||||
state.generation,
|
||||
state.muted,
|
||||
state.volume,
|
||||
std::mem::take(&mut state.latest),
|
||||
)
|
||||
};
|
||||
let changed = requested != generation;
|
||||
if changed {
|
||||
generation = requested;
|
||||
decoder = None;
|
||||
}
|
||||
let now = Instant::now();
|
||||
if changed || now.duration_since(refreshed) >= Duration::from_millis(250) {
|
||||
metadata = session
|
||||
.as_ref()
|
||||
.and_then(|(store, id)| store.metadata(*id).ok());
|
||||
refreshed = now;
|
||||
}
|
||||
let seek = if latest {
|
||||
metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| {
|
||||
metadata
|
||||
.segments
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|segment| {
|
||||
segment.complete && segment.available && segment.end > segment.start
|
||||
})
|
||||
.map(|segment| {
|
||||
segment.end.saturating_sub(Duration::from_secs_f64(
|
||||
1. / segment.frames_per_second.max(1) as f64,
|
||||
))
|
||||
})
|
||||
})
|
||||
.or(Some(Duration::ZERO))
|
||||
} else {
|
||||
seek.map(|time| {
|
||||
metadata
|
||||
.as_ref()
|
||||
.map_or(time, |metadata| time.min(metadata.session.length))
|
||||
})
|
||||
};
|
||||
let transport_changed = playing != clock.playing;
|
||||
if seek.is_some() || transport_changed {
|
||||
clock.set(seek.unwrap_or_else(|| clock.time(now)), playing, now);
|
||||
}
|
||||
if session.is_none() {
|
||||
audio = None;
|
||||
audio_attempted = false;
|
||||
audio_error = None;
|
||||
}
|
||||
if session.is_some() && !audio_attempted && !muted {
|
||||
audio_attempted = true;
|
||||
match crate::audio::Audio::new() {
|
||||
Ok(output) => audio = Some(output),
|
||||
Err(error) => audio_error = Some(error),
|
||||
}
|
||||
}
|
||||
if let Some(audio) = audio.as_mut() {
|
||||
if changed || seek.is_some() || transport_changed {
|
||||
audio.reset(clock.time(now));
|
||||
}
|
||||
audio.controls(playing && !muted, volume);
|
||||
}
|
||||
let Some(metadata) = metadata.as_ref().filter(|_| session.is_some()) else {
|
||||
wait(&shared, Duration::from_millis(100));
|
||||
continue;
|
||||
};
|
||||
let mut time = clock.time(now).min(metadata.session.length);
|
||||
if playing
|
||||
&& !muted
|
||||
&& let Some(audio) = audio.as_mut()
|
||||
{
|
||||
if let Some(device_time) = audio.time(now)
|
||||
&& metadata
|
||||
.locate(device_time)
|
||||
.is_some_and(|at| at.segment.complete)
|
||||
{
|
||||
time = device_time.min(metadata.session.length);
|
||||
clock.set(time, true, now);
|
||||
}
|
||||
audio.catch_up(time);
|
||||
if let Err(error) = audio.fill(metadata) {
|
||||
audio_error = Some(format!("Cannot play audio: {error}"));
|
||||
}
|
||||
if let Some(error) = audio.error() {
|
||||
audio_error = Some(error);
|
||||
}
|
||||
}
|
||||
|
||||
if time >= metadata.session.length && playing {
|
||||
clock.set(time, false, now);
|
||||
shared.state.lock().unwrap().playing = false;
|
||||
}
|
||||
// Completed files only: active files may still lack seekable cues.
|
||||
let located = metadata.locate(time).filter(|at| at.segment.complete);
|
||||
let start = located
|
||||
.as_ref()
|
||||
.map_or(Duration::ZERO, |at| at.segment.start);
|
||||
let frames_per_second = located
|
||||
.as_ref()
|
||||
.map_or(60, |at| at.segment.frames_per_second);
|
||||
let mut picture = Picture::Gap;
|
||||
let mut presented = false;
|
||||
let started = Instant::now();
|
||||
if let Some(at) = located {
|
||||
let target = decode::frame_index(at.offset, at.segment.frames_per_second);
|
||||
let result = (|| {
|
||||
if decoder.as_ref().is_none_or(|(path, _, index)| {
|
||||
path != &at.segment.path
|
||||
|| target < *index
|
||||
|| target > index.saturating_add(120)
|
||||
}) {
|
||||
let mut opened = Decoder::open(&at.segment.path, at.segment.frames_per_second)?;
|
||||
if at.offset > Duration::ZERO {
|
||||
opened.seek(at.offset)?;
|
||||
}
|
||||
decoder = Some((at.segment.path.clone(), opened, u64::MAX));
|
||||
}
|
||||
let (_, active, last) = decoder.as_mut().unwrap();
|
||||
if *last == target && !changed {
|
||||
return Ok(None);
|
||||
}
|
||||
loop {
|
||||
// A drag or navigation cancels decode-forward work between frames.
|
||||
let state = shared.state.lock().unwrap();
|
||||
if state.quit || state.generation != generation {
|
||||
return Ok(None);
|
||||
}
|
||||
drop(state);
|
||||
let Some((index, frame)) = active.next_frame()? else {
|
||||
return Ok(Some(Picture::Gap));
|
||||
};
|
||||
if index >= target {
|
||||
*last = index;
|
||||
return active
|
||||
.scale(&frame, size)
|
||||
.map(|frame| Some(Picture::Frame(Arc::new(frame))));
|
||||
}
|
||||
}
|
||||
})();
|
||||
match result {
|
||||
Ok(Some(frame)) => {
|
||||
presented = matches!(frame, Picture::Frame(_));
|
||||
picture = frame;
|
||||
}
|
||||
Ok(None) => picture = shared.state.lock().unwrap().snapshot.picture.clone(),
|
||||
Err(error) => {
|
||||
picture = Picture::Error(format!("Cannot read footage: {error}"));
|
||||
decoder = None;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
decoder = None;
|
||||
}
|
||||
let mut state = shared.state.lock().unwrap();
|
||||
if state.generation == generation {
|
||||
let updated = time != state.snapshot.time
|
||||
|| clock.playing != state.snapshot.playing
|
||||
|| presented
|
||||
|| std::mem::discriminant(&picture)
|
||||
!= std::mem::discriminant(&state.snapshot.picture)
|
||||
|| audio_error != state.snapshot.audio_error;
|
||||
let revision = state.snapshot.revision + u64::from(updated);
|
||||
let video_time = matches!(picture, Picture::Frame(_))
|
||||
.then(|| {
|
||||
decoder.as_ref().map(|(_, _, index)| {
|
||||
start + clipbin_core::store::frame_offset(*index, frames_per_second)
|
||||
})
|
||||
})
|
||||
.flatten();
|
||||
state.snapshot = Snapshot {
|
||||
time,
|
||||
playing: clock.playing,
|
||||
picture,
|
||||
revision,
|
||||
audio_error: audio_error.clone(),
|
||||
video_time,
|
||||
frames_per_second,
|
||||
};
|
||||
if std::env::var_os("CLIPBIN_TRACE_FRAMES").is_some() && (playing || changed) {
|
||||
eprintln!(
|
||||
"playback time={:.3} decode_ms={:.3} generation={generation} presented={presented} state={} audio_time={:.3} video_time={:.3}",
|
||||
time.as_secs_f64(),
|
||||
started.elapsed().as_secs_f64() * 1000.,
|
||||
match &state.snapshot.picture {
|
||||
Picture::Frame(_) => "frame",
|
||||
Picture::Gap => "gap",
|
||||
Picture::Loading => "loading",
|
||||
Picture::Error(_) => "error",
|
||||
},
|
||||
audio
|
||||
.as_ref()
|
||||
.and_then(|audio| audio.time(Instant::now()))
|
||||
.map_or(-1., |time| time.as_secs_f64()),
|
||||
video_time.map_or(-1., |time| time.as_secs_f64())
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(state);
|
||||
wait(
|
||||
&shared,
|
||||
if clock.playing {
|
||||
Duration::from_millis(4)
|
||||
} else {
|
||||
Duration::from_millis(100)
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
fn wait(shared: &Shared, timeout: Duration) {
|
||||
let state = shared.state.lock().unwrap();
|
||||
if !state.quit && state.seek.is_none() {
|
||||
drop(shared.wake.wait_timeout(state, timeout).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn a_decoder_stall_skips_overdue_frames_without_shifting_the_clock() {
|
||||
let now = Instant::now();
|
||||
let mut clock = Clock::new(now);
|
||||
clock.set(Duration::ZERO, true, now);
|
||||
let first_target = decode::frame_index(clock.time(now + Duration::from_millis(16)), 60);
|
||||
let after_stall = decode::frame_index(clock.time(now + Duration::from_millis(250)), 60);
|
||||
assert_eq!(first_target, 0);
|
||||
assert_eq!(after_stall, 15);
|
||||
// Scaling only the target discards the intervening frames; their decoding
|
||||
// cost must never be added to the clock's next presentation deadline.
|
||||
assert_eq!(
|
||||
decode::frame_index(clock.time(now + Duration::from_secs(1)), 60),
|
||||
60
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn clock_pauses_and_seek_does_not_accumulate_decode_delay() {
|
||||
let now = Instant::now();
|
||||
let mut clock = Clock::new(now);
|
||||
clock.set(Duration::from_secs(2), true, now);
|
||||
assert_eq!(
|
||||
clock.time(now + Duration::from_secs(3)),
|
||||
Duration::from_secs(5)
|
||||
);
|
||||
clock.set(Duration::from_secs(5), false, now);
|
||||
assert_eq!(
|
||||
clock.time(now + Duration::from_secs(100)),
|
||||
Duration::from_secs(5)
|
||||
);
|
||||
clock.set(Duration::from_millis(1500), true, now);
|
||||
assert_eq!(
|
||||
decode::frame_index(clock.time(now + Duration::from_millis(100)), 60),
|
||||
96
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//! Browsing runs below the recorder's normal scheduling priority.
|
||||
pub fn lower(thumbnail: bool) {
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Err(error) = rustix::process::setpriority_process(
|
||||
Some(rustix::thread::gettid()),
|
||||
if thumbnail { 10 } else { 3 },
|
||||
) {
|
||||
eprintln!("Cannot lower browsing thread priority: {error}");
|
||||
}
|
||||
#[cfg(windows)]
|
||||
crate::windows_priority::lower(thumbnail);
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let _ = thumbnail;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! One background worker and a bounded queue for persistent PNG thumbnails.
|
||||
use crate::decode;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex, mpsc},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Deterministic cache path beside the source; exported MP4s use the same API.
|
||||
pub fn cache_path(path: &Path) -> PathBuf {
|
||||
path.with_extension("thumbnail.png")
|
||||
}
|
||||
|
||||
/// Decode any MP4/MKV and atomically publish a small thumbnail.
|
||||
pub fn generate(path: &Path) -> Result<PathBuf, String> {
|
||||
let cache = cache_path(path);
|
||||
if cache.is_file() {
|
||||
return Ok(cache);
|
||||
}
|
||||
let mut frame = decode::first_frame(path, (320, 180)).map_err(|error| error.to_string())?;
|
||||
let temporary = cache.with_extension("png.part");
|
||||
for pixel in frame.bgra.as_chunks_mut::<4>().0 {
|
||||
pixel.swap(0, 2);
|
||||
}
|
||||
image::save_buffer_with_format(
|
||||
&temporary,
|
||||
&frame.bgra,
|
||||
frame.width,
|
||||
frame.height,
|
||||
image::ColorType::Rgba8,
|
||||
image::ImageFormat::Png,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
std::fs::rename(&temporary, &cache).map_err(|error| error.to_string())?;
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
pub struct Thumbnails {
|
||||
sender: mpsc::SyncSender<PathBuf>,
|
||||
pending: Arc<Mutex<HashSet<PathBuf>>>,
|
||||
}
|
||||
impl Default for Thumbnails {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
impl Thumbnails {
|
||||
pub fn new() -> Self {
|
||||
let (sender, receiver) = mpsc::sync_channel::<PathBuf>(16);
|
||||
let pending = Arc::new(Mutex::new(HashSet::new()));
|
||||
let work = pending.clone();
|
||||
thread::Builder::new()
|
||||
.name("clipbin-thumbnails".into())
|
||||
.spawn(move || {
|
||||
crate::priority::lower(true);
|
||||
for path in receiver {
|
||||
// One decoder thread, one frame, and a duty-cycle cap leave capture priority.
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
if let Err(error) = generate(&path) {
|
||||
eprintln!("thumbnail {}: {error}", path.display());
|
||||
}
|
||||
work.lock().unwrap().remove(&path);
|
||||
}
|
||||
})
|
||||
.expect("thumbnail thread");
|
||||
Self { sender, pending }
|
||||
}
|
||||
pub fn request(&self, path: &Path) -> Option<PathBuf> {
|
||||
let cache = cache_path(path);
|
||||
if cache.is_file() {
|
||||
return Some(cache);
|
||||
}
|
||||
let mut pending = self.pending.lock().unwrap();
|
||||
if pending.insert(path.to_owned()) && self.sender.try_send(path.to_owned()).is_err() {
|
||||
pending.remove(path);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//! The only raw Windows calls in the media crate; no handles escape this bridge.
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_LOWEST,
|
||||
};
|
||||
|
||||
pub fn lower(thumbnail: bool) {
|
||||
let priority = if thumbnail {
|
||||
THREAD_PRIORITY_LOWEST
|
||||
} else {
|
||||
THREAD_PRIORITY_BELOW_NORMAL
|
||||
};
|
||||
// GetCurrentThread returns a borrowed pseudo-handle, valid for this thread.
|
||||
if unsafe { SetThreadPriority(GetCurrentThread(), priority) } == 0 {
|
||||
eprintln!(
|
||||
"Cannot lower browsing thread priority: {}",
|
||||
std::io::Error::last_os_error()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! Run through scripts/validate/playback-fixture.sh for real FFmpeg seek coverage.
|
||||
use clipbin_core::store::Store;
|
||||
use clipbin_media::{
|
||||
decode::{Decoder, scale},
|
||||
player::{Picture, Player},
|
||||
};
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
fn fixture() -> Option<PathBuf> {
|
||||
std::env::var_os("CLIPBIN_TEST_MEDIA_DIR").map(PathBuf::from)
|
||||
}
|
||||
#[test]
|
||||
fn seeking_matches_sequential_frames_in_each_independent_segment() {
|
||||
let Some(directory) = fixture() else {
|
||||
return;
|
||||
};
|
||||
for segment in 0..3 {
|
||||
let path = directory.join(format!("1/segment-{segment:03}.mkv"));
|
||||
let mut sequential = Decoder::open(&path, 60).unwrap();
|
||||
let mut expected = Vec::new();
|
||||
while let Some((index, frame)) = sequential.next_frame().unwrap() {
|
||||
expected.push((index, scale(&frame, (160, 90)).unwrap().bgra));
|
||||
}
|
||||
assert_eq!(expected.len(), 120);
|
||||
for target in [0, 1, 59, 60, 61, 119] {
|
||||
let mut decoder = Decoder::open(&path, 60).unwrap();
|
||||
decoder
|
||||
.seek(Duration::from_secs_f64(target as f64 / 60.))
|
||||
.unwrap();
|
||||
loop {
|
||||
let (index, frame) = decoder.next_frame().unwrap().unwrap();
|
||||
if index >= target {
|
||||
assert_eq!(index, target);
|
||||
assert_eq!(
|
||||
scale(&frame, (160, 90)).unwrap().bgra,
|
||||
expected[target as usize].1
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn wait(player: &Player, at: Duration) -> clipbin_media::player::Snapshot {
|
||||
let until = Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let snapshot = player.snapshot();
|
||||
if snapshot.time == at && !matches!(snapshot.picture, Picture::Loading) {
|
||||
return snapshot;
|
||||
}
|
||||
assert!(Instant::now() < until, "seek timed out: {snapshot:?}");
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn player_crosses_boundaries_and_gaps_and_discards_superseded_seeks() {
|
||||
let Some(directory) = fixture() else {
|
||||
return;
|
||||
};
|
||||
let player = Player::new();
|
||||
player.audio(true, 1.);
|
||||
player.open(Store { root: directory }, 1, Duration::ZERO);
|
||||
assert!(matches!(
|
||||
wait(&player, Duration::ZERO).picture,
|
||||
Picture::Frame(_)
|
||||
));
|
||||
for millis in [1999, 2000, 2500, 4000, 5999, 6000, 7000] {
|
||||
let at = Duration::from_millis(millis);
|
||||
player.seek(at);
|
||||
let snapshot = wait(&player, at);
|
||||
assert_eq!(
|
||||
matches!(snapshot.picture, Picture::Gap),
|
||||
(4000..6000).contains(&millis),
|
||||
"at {millis}"
|
||||
);
|
||||
}
|
||||
for i in 0..50 {
|
||||
player.seek(Duration::from_millis(i * 40));
|
||||
}
|
||||
player.seek(Duration::from_millis(6500));
|
||||
assert!(matches!(
|
||||
wait(&player, Duration::from_millis(6500)).picture,
|
||||
Picture::Frame(_)
|
||||
));
|
||||
player.seek(Duration::from_millis(1900));
|
||||
wait(&player, Duration::from_millis(1900));
|
||||
player.play(true);
|
||||
thread::sleep(Duration::from_millis(400));
|
||||
player.play(false);
|
||||
let snapshot = player.snapshot();
|
||||
assert!((Duration::from_millis(2250)..Duration::from_millis(2500)).contains(&snapshot.time));
|
||||
assert!(matches!(snapshot.picture, Picture::Frame(_)));
|
||||
assert!(snapshot.video_time.unwrap().abs_diff(snapshot.time) < Duration::from_millis(20));
|
||||
player.seek(Duration::from_millis(3900));
|
||||
wait(&player, Duration::from_millis(3900));
|
||||
player.play(true);
|
||||
thread::sleep(Duration::from_millis(350));
|
||||
let snapshot = player.snapshot();
|
||||
assert!(snapshot.time >= Duration::from_millis(4200));
|
||||
assert!(matches!(snapshot.picture, Picture::Gap));
|
||||
player.close();
|
||||
}
|
||||
@@ -370,3 +370,99 @@ AV1 export is unvalidated because this GPU has no usable AV1 encoder. B-frame
|
||||
export and ranges containing an internal timestamp gap need additional media
|
||||
fixtures; dropped-frame ranges are explicitly rejected. Linux default-player
|
||||
and file-manager associations still need testing on a real desktop.
|
||||
|
||||
## Playback and thumbnails
|
||||
|
||||
Playback uses the shared `clipbin-media` crate (`ffmpeg-next` 9) without GPUI.
|
||||
One background decoder opens only completed, available segments. Session time
|
||||
comes from `Metadata::locate`; each file's timestamps are normalised independently.
|
||||
Seeking goes backwards to a keyframe, then decodes forward to the selected frame.
|
||||
A one-frame mailbox and coalesced seeks bound work; overdue frames are decoded
|
||||
but discarded before swscale. Playback threads have lower scheduling priority
|
||||
than recording. Leaving the editor closes its decoder and audio stream.
|
||||
|
||||
The editor uploads packed BGRA at its actual pixel size, maintains a 16:9
|
||||
preview, and releases old GPUI images. Space/K toggle playback; arrows/J/L step
|
||||
one frame. The timeline seeks and drags across files and gaps; completed footage
|
||||
from active sessions becomes browsable on the next 250 ms metadata refresh
|
||||
within the recorder's segment completion delay. Missing/removed footage advances
|
||||
on the wall clock and shows “No footage (not captured / cleaned up)”. There is
|
||||
no live recording preview.
|
||||
|
||||
Audio uses cpal and swresample at the output device's rate/channel count, with
|
||||
sample conversion in the device callback. OBS's combined third audio track is
|
||||
selected when three tracks exist; otherwise the first track is used. The queue
|
||||
holds at most 250 ms plus one decoded block. Callback timestamps, including DAC
|
||||
latency, drive the master clock; gaps and mute use the wall clock. Callbacks do
|
||||
no decoding and never wait for the queue lock. Mute and volume are in the editor;
|
||||
an unavailable device is shown explicitly while video remains usable. The Linux
|
||||
rig uses its existing null sink and ALSA's PipeWire plugin as a fallback if cpal's
|
||||
native PulseAudio backend refuses the device. Install/configure `pipewire-alsa`
|
||||
or an ALSA default routed to PulseAudio on systems without a physical ALSA card.
|
||||
|
||||
One low-priority worker produces 320×180 PNG thumbnails beside the first playable
|
||||
session segment and exported MP4s. Its request queue holds 16 paths and duplicate
|
||||
requests coalesce; card discovery and all filesystem work run off the UI thread.
|
||||
`clipbin_media::thumbnail::{generate,cache_path,Thumbnails}` accept any MP4/MKV.
|
||||
The gradient remains while a thumbnail is pending or footage is unavailable.
|
||||
|
||||
`scripts/validate/playback-fixture.sh` generates independent 1080p60 H.264/AAC
|
||||
segments with one-second keyframes and a three-tone audio fixture. It passed on
|
||||
2026-10-08: exact frame pixels at frame indices 0/1/59/60/61/119 in every file,
|
||||
real-time boundary crossing, gaps, superseded seeks, pause/seek clock maths,
|
||||
bounded frame placement, resampling 48 kHz to 44.1 kHz and combined-track selection.
|
||||
The ordinary workspace tests skip the generated-media assertions unless
|
||||
`CLIPBIN_TEST_MEDIA_DIR` points to the fixture; use the script to exercise them.
|
||||
|
||||
Windows: the cpal WASAPI dependency cross-check passed, as did the dedicated
|
||||
Windows thread-priority bridge. Core/OBS/probe strict MSVC cross-lint passed.
|
||||
The full media cross-lint is excluded on this Linux machine because the FFmpeg
|
||||
Windows development import libraries are unavailable. On Windows set `FFMPEG_DIR`
|
||||
to an FFmpeg **7.1 shared development** package with MSVC import libraries matching
|
||||
the OBS-bundled avcodec-61/avformat-61/avutil-59/swscale-8/swresample-5 DLLs. Linux
|
||||
links system FFmpeg 8. Actual Windows decoding, WASAPI device latency/seek behaviour,
|
||||
GPUI image presentation and thumbnail caching need a Windows machine. Hardware
|
||||
video decode is not used; HEVC/AV1 software performance and real output devices
|
||||
other than the rig null sink still need measurement.
|
||||
|
||||
Final playback validation passed on 2026-10-08, serially with other rigs stopped:
|
||||
|
||||
- `cargo fmt --all -- --check`, strict locked workspace clippy, and locked
|
||||
workspace tests (with `CLIPBIN_TEST_MEDIA_DIR=target/playback-fixture`).
|
||||
- `env PATH=/usr/bin:/bin:$HOME/.cargo/bin scripts/validate/linux-rig.sh`:
|
||||
all nine scenarios. An earlier concurrent run lost its H.264 probe to SIGKILL;
|
||||
the serial rerun passed without timeout or sandbox changes.
|
||||
- The same PATH with `scripts/validate/linux-session.sh`: automatic recording,
|
||||
exact segment placement, close-window survival, game exit, library reload and
|
||||
clean Quit. Repeated after the final keyboard integration, also passed.
|
||||
- `scripts/validate/linux-playback.sh target/playback-e2e-final-6`: moving preview
|
||||
(11.0% changed pixels inside the preview), active recording with zero reported
|
||||
recording drops, completed-segment availability, real session thumbnail,
|
||||
cross-file drag, exact post-boundary frame/time, cleaned-up gap, zoom, arrow
|
||||
stepping, device audio, half-volume and mute. The audio monitor RMS was 0.08044;
|
||||
the player asserts video time stays within 20 ms of the audio master clock.
|
||||
Screenshots were inspected: `sessions-thumbnails.png`, `editor-playing-1.png`,
|
||||
`editor-playing-2.png`, `editor-seek.png`, and `editor-gap.png` in that directory.
|
||||
|
||||
The rig runs on an Intel i5-12500 (12 logical CPUs). These are development builds
|
||||
with dependencies at opt-level 2, software video decoding, 1080p60 H.264 source
|
||||
and an 800×450 target (the editor used 740×416). Five serial repetitions of
|
||||
`target/debug/examples/playback-check <sessions-root> <id>` with
|
||||
`scripts/validate/measure-cpu.py`, inside the null-sink rig, measured process CPU
|
||||
at **16.7% median / 17.7% p95 of one core** (12.2–17.7% range). The filesystem cache
|
||||
was warm; each repetition reopened its decoders. Across presented frames,
|
||||
decode/scale time was **2.39 ms median / 8.80 ms p95**; maximum **101.5 ms** included
|
||||
opening files and decoding forward on seeks. This is a test pattern with gaps,
|
||||
not a worst-case game or a release-build benchmark. Per-run logs and CPU reports
|
||||
are in `target/playback-timing-final`; `report.json` stores the distributions.
|
||||
|
||||
During the active-session e2e, the player presented **59.2 frames/s** while
|
||||
recording remained at zero reported drops. GPUI upload/construction time was
|
||||
**0.22 ms median / 0.48 ms p95**. UI frame intervals (including vsync, not CPU work)
|
||||
were 16.2 ms median / 32.5 ms p95; the one-frame mailbox drops presentation work
|
||||
rather than allowing delay to accumulate. `CLIPBIN_TRACE_FRAMES=1` logs decoded
|
||||
video time, audio time, decode/scale work, uploads and actual timeline bounds.
|
||||
The headless e2e restores keyboard focus after virtual-pointer commands and
|
||||
holds virtual keys briefly: instantaneous virtual input can disappear before
|
||||
GPUI/Wayland dispatch. Keyboard actions update the editor entity directly,
|
||||
avoiding a nested window borrow during event dispatch.
|
||||
|
||||
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env bash
|
||||
# Playback, active browsing, real thumbnails and audio in the private desktop rig.
|
||||
set -euo pipefail
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
out="${1:-${repo}/target/playback-validation}"
|
||||
mkdir -p "${out}"
|
||||
out="$(cd "${out}" && pwd)"
|
||||
cargo build --locked --workspace --manifest-path "${repo}/Cargo.toml"
|
||||
cargo build --locked -p clipbin-media --example playback-check --manifest-path "${repo}/Cargo.toml"
|
||||
export CLIPBIN_PLAYBACK_REPO="${repo}" CLIPBIN_PLAYBACK_OUT="${out}"
|
||||
"${repo}/scripts/rig/headless.sh" bash <<'INNER'
|
||||
set -euo pipefail
|
||||
repo="${CLIPBIN_PLAYBACK_REPO}" out="${CLIPBIN_PLAYBACK_OUT}"
|
||||
export XDG_DATA_HOME="${out}/data" XDG_CONFIG_HOME="${out}/config" CLIPBIN_TRACE_FRAMES=1
|
||||
mkdir -p "${XDG_DATA_HOME}" "${XDG_CONFIG_HOME}"
|
||||
# ALSA's PipeWire plugin is cpal's fallback on Linux; there is no hardware card.
|
||||
cat > "${out}/alsa.conf" <<'ALSA'
|
||||
</usr/share/alsa/alsa.conf>
|
||||
pcm.!default { type pipewire }
|
||||
ctl.!default { type pipewire }
|
||||
ALSA
|
||||
export ALSA_CONFIG_PATH="${out}/alsa.conf"
|
||||
swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null
|
||||
send_key() {
|
||||
# Destroying rig-pointer's virtual seat can clear keyboard focus in Sway.
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
sleep .2
|
||||
timeout 10 wtype -s 200 -P "$1" -s 100 -p "$1"
|
||||
}
|
||||
# Timeline bounds come from GPUI's trace rather than guessed screen positions.
|
||||
pointer_time() {
|
||||
python3 - "$1" "$2" "$3" "${XDG_DATA_HOME}" <<'PYPOINTER'
|
||||
import pathlib,re,json,sys
|
||||
log=pathlib.Path(sys.argv[1]).read_text()
|
||||
x,y,w,h=map(float,re.findall(r'playback_timeline x=([\d.]+) y=([\d.]+) width=([\d.]+) height=([\d.]+)',log)[-1])
|
||||
p=max(pathlib.Path(sys.argv[4]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());length=m['session']['length'];length=length['secs']+length['nanos']/1e9
|
||||
if sys.argv[2] in ('gap','after-gap'):
|
||||
segment=next(s for s in m['segments'] if s['complete'] and not s['available'])
|
||||
def seconds(d):return d['secs']+d['nanos']/1e9
|
||||
at=(seconds(segment['start'])+seconds(segment['end']))/2 if sys.argv[2]=='gap' else seconds(segment['end'])+.5
|
||||
else:at=float(sys.argv[2])
|
||||
print(f"move {round(280+x+w*at/length)} {round(110+y+h/2)}\nsleep .15\n{sys.argv[3]}\nsleep .15")
|
||||
PYPOINTER
|
||||
}
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] floating enable' > /dev/null
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] resize set width 1360 px height 860 px' > /dev/null
|
||||
swaymsg 'for_window [app_id="dev.clipbin.Clipbin"] move position 280 110' > /dev/null
|
||||
"${repo}/target/debug/clipbin-desktop" > "${out}/active.log" 2>&1 & app=$!
|
||||
trap 'kill "${app:-}" "${game:-}" "${audio_capture:-}" 2>/dev/null || true' EXIT
|
||||
sleep 3
|
||||
swaymsg '[title="clipbin-test-game"] fullscreen disable' > /dev/null
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null
|
||||
bash -c 'exec -a reaper python3 -c "import time; time.sleep(120)" SteamLaunch AppId=480 --' & game=$!
|
||||
for _ in $(seq 30); do if grep -q recorder_status=Recording "${out}/active.log"; then break; fi; sleep 1; done
|
||||
grep -q recorder_status=Recording "${out}/active.log"
|
||||
sleep 8
|
||||
grim "${out}/sessions-thumbnails.png"
|
||||
# Open the first session card, seek near the start, and play while recording continues.
|
||||
printf 'move 670 330\nsleep .2\ndown\nsleep .1\nup\nsleep 1\n' | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
grim "${out}/editor-open.png"
|
||||
pointer_time "${out}/active.log" 1 click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
# GPUI/Wayland needs the virtual keyboard keymap to settle and a held key;
|
||||
# instantaneous wtype press/release occasionally disappears before dispatch.
|
||||
send_key space
|
||||
sleep 1
|
||||
grim "${out}/editor-playing-1.png"
|
||||
sleep 2
|
||||
grim "${out}/editor-playing-2.png"
|
||||
python3 - "${XDG_DATA_HOME}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());assert m['session']['active'];assert len([s for s in m['segments'] if s['complete'] and s['available']])>=4
|
||||
assert list(p.parent.rglob('*.thumbnail.png')), 'thumbnail worker produced no cached frame'
|
||||
print('PASS playback left recording active and generated real thumbnails')
|
||||
PY
|
||||
kill "${game}"; wait "${game}" 2>/dev/null || true
|
||||
sleep 3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl
|
||||
for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done
|
||||
if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi
|
||||
wait "${app}"
|
||||
kill "$(cat "${CLIPBIN_RIG_DIR}/game.pid")"
|
||||
# Simulate a cleaned-up middle segment in this isolated test library. Keep its
|
||||
# timeline placement: the player must show the gap rather than concatenate it away.
|
||||
python3 - "${XDG_DATA_HOME}" "${out}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
p=max(pathlib.Path(sys.argv[1]).glob('clipbin/sessions/*/session.json'),key=lambda p:int(p.parent.name))
|
||||
m=json.loads(p.read_text());segments=[s for s in m['segments'] if s['complete'] and s['available']];assert len(segments)>=4
|
||||
s=segments[2];s['available']=False
|
||||
m['session']['spans']=[dict(start=m['session']['spans'][0]['start'],end=s['start'],kind='Footage'),dict(start=s['start'],end=s['end'],kind='Removed'),dict(start=s['end'],end=m['session']['length'],kind='Footage')]
|
||||
p.write_text(json.dumps(m));pathlib.Path(sys.argv[2]+'/session-id').write_text(str(m['session']['id']))
|
||||
PY
|
||||
id="$(cat "${out}/session-id")"
|
||||
# The CLI exercises the same player with audio only; recording tone has stopped.
|
||||
ffmpeg -v error -y -f pulse -i clipbin-rig-sink.monitor -t 10 "${out}/audio-output.wav" & audio_capture=$!
|
||||
python3 "${repo}/scripts/validate/measure-cpu.py" "${out}/player-cpu.txt" "${repo}/target/debug/examples/playback-check" "${XDG_DATA_HOME}/clipbin/sessions" "${id}" > "${out}/player.log" 2>&1
|
||||
wait "${audio_capture}"
|
||||
CLIPBIN_ROUTE="session:${id}" "${repo}/target/debug/clipbin-desktop" > "${out}/playback.log" 2>&1 & app=$!
|
||||
sleep 3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] floating enable, resize set width 1360 px height 860 px, move position 280 110, focus' > /dev/null
|
||||
# Drag through several files, then seek into the removed interval.
|
||||
{ pointer_time "${out}/playback.log" 1 down; pointer_time "${out}/playback.log" 3 "sleep 0.5"; pointer_time "${out}/playback.log" after-gap up; } | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep 1
|
||||
grim "${out}/editor-seek.png"
|
||||
pointer_time "${out}/playback.log" gap click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep 1
|
||||
grim "${out}/editor-gap.png"
|
||||
pointer_time "${out}/playback.log" 1 $'scroll -1\nsleep 0.5\nscroll 1' | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
pointer_time "${out}/playback.log" 2.5 click | "${repo}/target/debug/rig-pointer" 1920 1080
|
||||
sleep .3
|
||||
send_key Right
|
||||
sleep .3
|
||||
send_key Left
|
||||
sleep .3
|
||||
swaymsg '[app_id="dev.clipbin.Clipbin"] focus' > /dev/null
|
||||
timeout 10 wtype -s 150 -M ctrl -s 100 -P q -s 100 -p q -m ctrl
|
||||
for _ in $(seq 20); do if ! kill -0 "${app}" 2>/dev/null; then break; fi; sleep 1; done
|
||||
if kill -0 "${app}" 2>/dev/null; then echo 'FAIL Ctrl+Q did not exit' >&2; exit 1; fi
|
||||
wait "${app}"
|
||||
python3 - "${out}" <<'PY'
|
||||
import pathlib,subprocess,re,sys,math,array,statistics,json
|
||||
out=pathlib.Path(sys.argv[1])
|
||||
def rgb(path):return subprocess.check_output(['ffmpeg','-v','error','-i',str(path),'-vf','crop=700:300:555:260','-f','rawvideo','-pix_fmt','rgb24','-'])
|
||||
a,b=map(rgb,[out/'editor-playing-1.png',out/'editor-playing-2.png'])
|
||||
changed=sum(x!=y for x,y in zip(a,b))/len(a);assert changed>.02, f'preview frozen: {changed}'
|
||||
raw=subprocess.check_output(['ffmpeg','-v','error','-i',str(out/'audio-output.wav'),'-ac','1','-ar','48000','-f','f32le','-'])
|
||||
samples=array.array('f',raw);rms=math.sqrt(sum(x*x for x in samples)/len(samples));assert rms>.001, f'no device audio: {rms}'
|
||||
def window_rms(start,end):
|
||||
block=samples[int(start*48000):int(end*48000)]
|
||||
return math.sqrt(sum(x*x for x in block)/max(1,len(block)))
|
||||
full=window_rms(2,3);half=window_rms(7.9,8.4);muted=window_rms(9.5,9.9)
|
||||
assert full>.001 and .35<half/full<.65 and muted<full*.02, f'volume/mute wrong: {full}, {half}, {muted}'
|
||||
log=(out/'player.log').read_text();assert 'state=gap' in log and 'state=frame' in log
|
||||
assert 'Cannot play audio' not in log and 'Audio output failed' not in log
|
||||
rows=re.findall(r'playback time=([\d.]+) decode_ms=([\d.]+).*presented=true',log)
|
||||
times=[float(row[1]) for row in rows];assert times
|
||||
print(f'PASS moving preview ({changed:.1%} pixels changed), device audio RMS={rms:.5f}, gaps and boundary playback')
|
||||
print(f'decode_ms median={statistics.median(times):.3f} p95={sorted(times)[int(len(times)*.95)]:.3f} max={max(times):.3f}')
|
||||
ui_log=(out/'playback.log').read_text()
|
||||
metadata=json.loads(next((out/'data/clipbin/sessions').glob('*/session.json')).read_text())
|
||||
def seconds(d):return d['secs']+d['nanos']/1e9
|
||||
removed=next(s for s in metadata['segments'] if s['complete'] and not s['available'])
|
||||
gap=(seconds(removed['start'])+seconds(removed['end']))/2
|
||||
gap_times=[float(t) for t in re.findall(r'playback time=([\d.]+).*state=gap',ui_log)]
|
||||
assert any(abs(t-gap)<.1 for t in gap_times), f'UI seek never reached gap at {gap}'
|
||||
frame_times=[(float(t),float(v)) for t,v in re.findall(r'playback time=([\d.]+).*state=frame.*video_time=([\d.]+)',ui_log)]
|
||||
assert any(abs(t-(seconds(removed['end'])+.5))<.1 and abs(t-v)<.020 for t,v in frame_times), 'UI boundary seek showed the wrong frame/time'
|
||||
# Right then Left must actually move the selected frame, while remaining paused.
|
||||
steps=[v for t,v in frame_times if abs(t-2.5)<.15]
|
||||
assert len(set(steps))>=2, f'frame stepping did not change the video frame: {steps}'
|
||||
uploads=[float(x) for x in re.findall(r'playback_ui upload_ms=([\d.]+)',(out/'active.log').read_text())]
|
||||
assert len(uploads)>30, 'too few moving frames reached GPUI'
|
||||
print(f'GPUI frame upload median={statistics.median(uploads):.3f}ms p95={sorted(uploads)[int(len(uploads)*.95)]:.3f}ms')
|
||||
(out/'report.txt').write_text(f'changed_preview_pixels={changed:.4f}\naudio_rms={rms:.5f}\ndecode_median_ms={statistics.median(times):.3f}\ndecode_p95_ms={sorted(times)[int(len(times)*.95)]:.3f}\n')
|
||||
PY
|
||||
echo "PASS playback validation; screenshots and timing in ${out}"
|
||||
INNER
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure child CPU/wall time without depending on GNU time."""
|
||||
import pathlib
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
before = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(sys.argv[2:], check=False)
|
||||
elapsed = time.monotonic() - started
|
||||
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
user = usage.ru_utime - before.ru_utime
|
||||
system = usage.ru_stime - before.ru_stime
|
||||
pathlib.Path(sys.argv[1]).write_text(
|
||||
f"cpu_user={user:.3f} cpu_system={system:.3f} wall={elapsed:.3f} "
|
||||
f"cpu_percent={(user + system) / elapsed * 100:.1f}%\n"
|
||||
)
|
||||
sys.exit(result.returncode)
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deterministic, clock-labelled 1080p60 segments with independent file timestamps.
|
||||
set -euo pipefail
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
out="${1:-${repo}/target/playback-fixture}"
|
||||
mkdir -p "${out}/1"
|
||||
ffmpeg -v error -y -f lavfi -i 'testsrc2=size=1920x1080:rate=60' \
|
||||
-f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 6 \
|
||||
-vf "drawtext=fontfile=/usr/share/fonts/TTF/DejaVuSans.ttf:text='%{pts\:hms}':fontsize=64:fontcolor=white:box=1:boxcolor=black:x=100:y=100" \
|
||||
-c:v libx264 -preset ultrafast -crf 28 -threads 2 -g 60 -keyint_min 60 -sc_threshold 0 -bf 0 \
|
||||
-c:a aac -f segment -segment_time 2 -reset_timestamps 1 "${out}/1/segment-%03d.mkv"
|
||||
ffmpeg -v error -y -i "${out}/1/segment-000.mkv" \
|
||||
-f lavfi -i 'sine=frequency=220:sample_rate=48000' \
|
||||
-f lavfi -i 'sine=frequency=880:sample_rate=48000' \
|
||||
-f lavfi -i 'sine=frequency=440:sample_rate=48000' -t 2 \
|
||||
-map 0:v -map 1:a -map 2:a -map 3:a -c:v copy -c:a aac "${out}/combined.mkv"
|
||||
python3 - "${out}" <<'PY'
|
||||
import pathlib,json,sys
|
||||
root=pathlib.Path(sys.argv[1]); d=lambda s:dict(secs=s,nanos=0)
|
||||
segments=[dict(path=f'segment-{i:03}.mkv',start=d(t),end=d(t+2),complete=True,available=True,frame_start=i*120,frames_per_second=60,dropped_frames=0) for i,t in enumerate([0,2,6])]
|
||||
session=dict(id=1,game='Playback fixture',started=dict(secs_since_epoch=1791453600,nanos_since_epoch=0),length=d(8),active=False,spans=[dict(start=d(a),end=d(b),kind=k) for a,b,k in [(0,4,'Footage'),(4,6,'Missing'),(6,8,'Footage')]],bookmarks=[])
|
||||
(root/'1/session.json').write_text(json.dumps(dict(session=session,app_id=480,segments=segments)))
|
||||
PY
|
||||
CLIPBIN_TEST_MEDIA_DIR="${out}" cargo test --locked -p clipbin-media --manifest-path "${repo}/Cargo.toml"
|
||||
Reference in New Issue
Block a user