From 1a469894c2737b5593289b2a584cff76c582a267 Mon Sep 17 00:00:00 2001 From: Troy Benson Date: Fri, 23 Jun 2023 21:53:27 +0000 Subject: [PATCH] feat: improve player (#85) - Added ABR (#85 point 1) Adaptive bitrate streaming is the ability for the player to dynamically change quality based on network conditions - Added Codec Support (#85 point 3) We now check if the browser supports AV1 / HEVC / OPUS before we list the quality as a potential variant. Note: We added some unsafe code. You should read the safety comment on what it is and how it works. --- .github/workflows/commitlint.yml | 2 +- Cargo.lock | 10 +- frontend/player/Cargo.lock | 215 +-- frontend/player/Cargo.toml | 4 + frontend/player/demo/index.ts | 134 +- frontend/player/index.html | 59 +- frontend/player/package.json | 2 +- frontend/player/src/hls/playlist/master.rs | 94 +- frontend/player/src/hls/playlist/media.rs | 52 + frontend/player/src/hls/playlist/utils.rs | 10 + frontend/player/src/player/bandwidth.rs | 82 ++ frontend/player/src/player/blank.rs | 30 +- frontend/player/src/player/events.rs | 3 +- frontend/player/src/player/fetch.rs | 163 ++- frontend/player/src/player/inner.rs | 172 +-- frontend/player/src/player/mod.rs | 126 +- frontend/player/src/player/runner.rs | 1180 ---------------- frontend/player/src/player/runner/events.rs | 18 + frontend/player/src/player/runner/mod.rs | 1200 +++++++++++++++++ .../player/src/player/runner/source_buffer.rs | 87 ++ frontend/player/src/player/runner/util.rs | 154 +++ frontend/player/src/player/track.rs | 427 ++++-- frontend/player/src/player/util.rs | 26 +- video/edge/src/edge/mod.rs | 6 + video/edge/src/edge/stream.rs | 20 +- video/transcoder/src/config.rs | 2 +- video/transcoder/src/transcoder/job/mod.rs | 91 +- .../src/transcoder/job/renditions.rs | 51 + .../src/transcoder/job/variant/mod.rs | 50 +- 29 files changed, 2738 insertions(+), 1732 deletions(-) create mode 100644 frontend/player/src/player/bandwidth.rs delete mode 100644 frontend/player/src/player/runner.rs create mode 100644 frontend/player/src/player/runner/events.rs create mode 100644 frontend/player/src/player/runner/mod.rs create mode 100644 frontend/player/src/player/runner/source_buffer.rs create mode 100644 frontend/player/src/player/runner/util.rs create mode 100644 video/transcoder/src/transcoder/job/renditions.rs diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index b5a44998..8fc88fff 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -19,7 +19,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - cache: 'pnpm' + cache: "pnpm" node-version: 18.x - name: Install dependencies diff --git a/Cargo.lock b/Cargo.lock index 94161188..779af0a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3992,7 +3992,7 @@ dependencies = [ [[package]] name = "tracing" version = "0.2.0" -source = "git+https://github.com/ScuffleTV/tracing#8ce22527ce1f76937f86c71461a19a7edc57427c" +source = "git+https://github.com/ScuffleTV/tracing#e2dd0f6e0e31896f41607717159c71fd24f26b4f" dependencies = [ "pin-project-lite", "tracing-core 0.2.0 (git+https://github.com/ScuffleTV/tracing)", @@ -4021,7 +4021,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.2.0" -source = "git+https://github.com/ScuffleTV/tracing#8ce22527ce1f76937f86c71461a19a7edc57427c" +source = "git+https://github.com/ScuffleTV/tracing#e2dd0f6e0e31896f41607717159c71fd24f26b4f" dependencies = [ "once_cell", ] @@ -4049,7 +4049,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/ScuffleTV/tracing#8ce22527ce1f76937f86c71461a19a7edc57427c" +source = "git+https://github.com/ScuffleTV/tracing#e2dd0f6e0e31896f41607717159c71fd24f26b4f" dependencies = [ "log", "once_cell", @@ -4070,7 +4070,7 @@ dependencies = [ [[package]] name = "tracing-serde" version = "0.2.0" -source = "git+https://github.com/ScuffleTV/tracing#8ce22527ce1f76937f86c71461a19a7edc57427c" +source = "git+https://github.com/ScuffleTV/tracing#e2dd0f6e0e31896f41607717159c71fd24f26b4f" dependencies = [ "serde", "tracing-core 0.2.0 (git+https://github.com/ScuffleTV/tracing)", @@ -4079,7 +4079,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.0" -source = "git+https://github.com/ScuffleTV/tracing#8ce22527ce1f76937f86c71461a19a7edc57427c" +source = "git+https://github.com/ScuffleTV/tracing#e2dd0f6e0e31896f41607717159c71fd24f26b4f" dependencies = [ "matchers", "nu-ansi-term", diff --git a/frontend/player/Cargo.lock b/frontend/player/Cargo.lock index b227de78..42ab8fe4 100644 --- a/frontend/player/Cargo.lock +++ b/frontend/player/Cargo.lock @@ -13,6 +13,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "anyhow" version = "1.0.71" @@ -40,6 +55,21 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.21.2" @@ -87,6 +117,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -194,7 +230,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] @@ -227,6 +263,12 @@ dependencies = [ "slab", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -274,10 +316,11 @@ dependencies = [ [[package]] name = "half" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" dependencies = [ + "cfg-if", "crunchy", ] @@ -293,9 +336,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" [[package]] name = "js-sys" @@ -312,6 +355,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + [[package]] name = "log" version = "0.4.19" @@ -324,6 +373,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mp4" version = "0.0.1" @@ -352,13 +410,13 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.23", ] [[package]] @@ -370,6 +428,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -384,9 +451,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "paste" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" [[package]] name = "percent-encoding" @@ -396,9 +463,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -437,27 +504,33 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" [[package]] name = "scoped-tls" @@ -467,9 +540,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.166" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "d01b7404f9d441d3ad40e6a636a7782c377d2abdbe4fa2440e2edcc2f4f10db8" dependencies = [ "serde_derive", ] @@ -487,13 +560,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.166" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "5dd83d6dde2b6b2d466e14d9d1acce8816dedee94f735eac6395808b3483c6d6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] @@ -504,14 +577,14 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c" dependencies = [ "itoa", "ryu", @@ -555,9 +628,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" dependencies = [ "proc-macro2", "quote", @@ -591,14 +664,14 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "pin-project-lite", "tokio-macros", - "windows-sys", ] [[package]] @@ -609,7 +682,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] @@ -643,7 +716,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] @@ -703,7 +776,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.18", + "syn 2.0.23", ] [[package]] @@ -720,9 +793,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" [[package]] name = "unicode-normalization" @@ -772,7 +845,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", "wasm-bindgen-shared", ] @@ -806,7 +879,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.23", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -872,69 +945,3 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/frontend/player/Cargo.toml b/frontend/player/Cargo.toml index 4f672f3b..261523ff 100644 --- a/frontend/player/Cargo.toml +++ b/frontend/player/Cargo.toml @@ -67,6 +67,10 @@ features = [ "Window", "XmlHttpRequest", "XmlHttpRequestResponseType", + "PerformanceResourceTiming", + "PerformanceObserver", + "PerformanceObserverInit", + "PerformanceObserverEntryList", ] [dev-dependencies] diff --git a/frontend/player/demo/index.ts b/frontend/player/demo/index.ts index 198bb872..6346f4e1 100644 --- a/frontend/player/demo/index.ts +++ b/frontend/player/demo/index.ts @@ -10,70 +10,38 @@ const video = document.getElementById("video") as HTMLVideoElement; const bufferSize = document.getElementById("buffer-size") as HTMLElement; const videoTime = document.getElementById("video-time") as HTMLElement; const frameRate = document.getElementById("frame-rate") as HTMLElement; +const resolution = document.getElementById("resolution") as HTMLElement; +const droppedFrames = document.getElementById("dropped-frames") as HTMLElement; -const selectTrack0 = document.getElementById("select-track-0") as HTMLButtonElement; -const selectTrack1 = document.getElementById("select-track-1") as HTMLButtonElement; -const selectTrack2 = document.getElementById("select-track-2") as HTMLButtonElement; -const selectTrack3 = document.getElementById("select-track-3") as HTMLButtonElement; -const selectTrack4 = document.getElementById("select-track-4") as HTMLButtonElement; - -const forceTrack0 = document.getElementById("force-track-0") as HTMLButtonElement; -const forceTrack1 = document.getElementById("force-track-1") as HTMLButtonElement; -const forceTrack2 = document.getElementById("force-track-2") as HTMLButtonElement; -const forceTrack3 = document.getElementById("force-track-3") as HTMLButtonElement; -const forceTrack4 = document.getElementById("force-track-4") as HTMLButtonElement; +const selectTracksDiv = document.getElementById("select-tracks") as HTMLDivElement; +const forceTracksDiv = document.getElementById("force-tracks") as HTMLDivElement; const toggleLowLatency = document.getElementById("toggle-low-latency") as HTMLButtonElement; +const toggleAbr = document.getElementById("toggle-abr") as HTMLButtonElement; const jumpToLive = document.getElementById("jump-to-live") as HTMLButtonElement; let lastFrameTime = 0; let frameCount = 0; -selectTrack0.addEventListener("click", () => { - player.nextTrackId = 0; -}); +player.lowLatency = true; +player.abrEnabled = true; -selectTrack1.addEventListener("click", () => { - player.nextTrackId = 1; -}); - -selectTrack2.addEventListener("click", () => { - player.nextTrackId = 2; -}); - -selectTrack3.addEventListener("click", () => { - player.nextTrackId = 3; -}); - -selectTrack4.addEventListener("click", () => { - player.nextTrackId = 4; -}); - -forceTrack0.addEventListener("click", () => { - player.forceTrackId = 0; -}); - -forceTrack1.addEventListener("click", () => { - player.forceTrackId = 1; -}); - -forceTrack2.addEventListener("click", () => { - player.forceTrackId = 2; -}); - -forceTrack3.addEventListener("click", () => { - player.forceTrackId = 3; -}); - -forceTrack4.addEventListener("click", () => { - player.forceTrackId = 4; -}); +toggleAbr.innerText = player.abrEnabled ? "Disable ABR" : "Enable ABR"; +toggleLowLatency.innerText = player.lowLatency ? "Disable Low Latency" : "Enable Low Latency"; toggleLowLatency.addEventListener("click", () => { player.lowLatency = !player.lowLatency; + toggleLowLatency.innerText = player.lowLatency ? "Disable Low Latency" : "Enable Low Latency"; +}); + +toggleAbr.addEventListener("click", () => { + player.abrEnabled = !player.abrEnabled; + toggleAbr.innerText = player.abrEnabled ? "Disable ABR" : "Enable ABR"; }); jumpToLive.addEventListener("click", () => { + if (!video.buffered.length) return; + if (player.lowLatency) { video.currentTime = video.buffered.end(video.buffered.length - 1) - 0.5; } else { @@ -81,35 +49,73 @@ jumpToLive.addEventListener("click", () => { } }); -video.addEventListener("timeupdate", () => { - bufferSize.innerText = video.buffered.end(video.buffered.length - 1) - video.currentTime + ""; - videoTime.innerText = video.currentTime + ""; +const loop = () => { + if (video.buffered.length) { + bufferSize.innerText = `${( + video.buffered.end(video.buffered.length - 1) - video.currentTime + ).toFixed(3)}`; + } else { + bufferSize.innerText = "0"; + } + + videoTime.innerText = `${video.currentTime.toFixed(3)}`; + resolution.innerText = `${video.videoWidth}x${video.videoHeight}`; const quality = video.getVideoPlaybackQuality(); + droppedFrames.innerText = `${quality.droppedVideoFrames}`; + const now = performance.now(); - if (now - lastFrameTime > 1000) { - frameRate.innerText = quality.totalVideoFrames - frameCount + ""; + if (now - lastFrameTime >= 1000) { + frameRate.innerText = `${quality.totalVideoFrames - frameCount}`; frameCount = quality.totalVideoFrames; lastFrameTime = now; } -}); -player.lowLatency = false; + window.requestAnimationFrame(loop); +}; + +window.requestAnimationFrame(loop); player.onerror = (evt) => { console.error(evt); }; player.onmanifestloaded = (evt) => { - console.log(evt); + selectTracksDiv.innerHTML = ""; + forceTracksDiv.innerHTML = ""; + + evt.variants.forEach((variant) => { + const button = document.createElement("button"); + button.innerText = `${variant.group} - ${variant.name}`; + button.addEventListener("click", () => { + player.nextVariantId = variant.id; + }); + selectTracksDiv.appendChild(button); + + const forceButton = document.createElement("button"); + forceButton.innerText = `${variant.group} - ${variant.name}`; + forceButton.addEventListener("click", () => { + player.variantId = variant.id; + }); + forceTracksDiv.appendChild(forceButton); + }); }; -player.load( - // "http://192.168.2.177:9080/4f75cb30-6acf-4b1f-a91d-d9ae2c72c0cd/master.m3u8", - "https://troy-edge.scuffle.tv/4f75cb30-6acf-4b1f-a91d-d9ae2c72c0cd/master.m3u8", - // "http://192.168.2.177:9080/51636c0f-a2f1-46d6-9da1-07b386efff7a/03f92acb-fd92-4fb5-9023-5e27b82ba987/index.m3u8", - // "http://192.168.2.177:9080/4def6aa7-6ae2-4a35-a473-d346de345e54/041c0b21-972d-4992-aca5-f010c01067c5/index.m3u8", -); +const loadButton = document.getElementById("load") as HTMLButtonElement; +const inputUrl = document.getElementById("input-url") as HTMLInputElement; + +loadButton.addEventListener("click", () => { + player.load(inputUrl.value); + player.attach(video); +}); + +// Get URL fragment for the predefined url +const url = new URL(window.location.href); + +const urlFragment = url.hash.slice(1); -await player.attach(video); +if (urlFragment) { + inputUrl.value = urlFragment; + loadButton.click(); +} diff --git a/frontend/player/index.html b/frontend/player/index.html index 44c1c984..3829b3d8 100644 --- a/frontend/player/index.html +++ b/frontend/player/index.html @@ -7,40 +7,53 @@
-
-

Select

- - - - - +
+

Input

+ + +
+
+

Select

+

Force

- - - - - -
-
-

Toggle

- - +
+
+

Toggle

+ + + +
+

Video Time:

+

Buffer Size:

+

Frame Rate:

+

Dropped Frames:

+

Resolution:

-

Video Time

-

Buffer Size

-

Frame Rate

diff --git a/frontend/player/package.json b/frontend/player/package.json index 6f3ac551..dd24b013 100644 --- a/frontend/player/package.json +++ b/frontend/player/package.json @@ -13,7 +13,7 @@ "format": "prettier --plugin-search-dir . --write \"**/*\" -u && cargo fmt && cargo clippy --fix --allow-dirty --allow-staged", "demo:dev": "pnpm run wasm:watch & vite", "demo:build": "pnpm run wasm:build --release && tsc && vite build -c vite.demo.config.ts", - "demo:preview": "vite preview", + "demo:preview": "vite preview -c vite.demo.config.ts", "clean": "rimraf dist pkg" }, "module": "./dist/player.js", diff --git a/frontend/player/src/hls/playlist/master.rs b/frontend/player/src/hls/playlist/master.rs index a31ea098..1b8625e4 100644 --- a/frontend/player/src/hls/playlist/master.rs +++ b/frontend/player/src/hls/playlist/master.rs @@ -8,6 +8,12 @@ use super::utils::Tag; pub struct MasterPlaylist { pub streams: Vec, pub groups: HashMap>, + pub scuf_groups: HashMap, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ScufGroup { + pub priority: i32, } #[derive(Debug, Clone, Serialize)] @@ -23,24 +29,27 @@ pub struct Media { pub media_type: MediaType, pub uri: String, pub name: String, + pub group_id: String, + pub codecs: String, + pub bandwidth: u32, pub autoselect: bool, pub default: bool, pub forced: bool, + pub resolution: Option<(u32, u32)>, + pub frame_rate: Option, } #[derive(Debug, Clone, Serialize)] pub struct Stream { pub uri: String, pub bandwidth: u32, - pub average_bandwidth: Option, - pub codecs: Option, + pub name: String, + pub group: String, + pub codecs: String, pub resolution: Option<(u32, u32)>, pub frame_rate: Option, - pub hdcp_level: Option, pub audio: Option, pub video: Option, - pub subtitles: Option, - pub closed_captions: Option, } impl MasterPlaylist { @@ -50,18 +59,15 @@ impl MasterPlaylist { .filter_map(|t| match t { Tag::ExtXStreamInf(attributes, uri) => Some(Stream { uri: uri.clone(), + name: attributes.get("NAME").map(|s| s.to_string())?, + group: attributes.get("GROUP").map(|s| s.to_string())?, audio: attributes.get("AUDIO").map(|s| s.to_string()), video: attributes.get("VIDEO").map(|s| s.to_string()), - subtitles: attributes.get("SUBTITLES").map(|s| s.to_string()), - closed_captions: attributes.get("CLOSED-CAPTIONS").map(|s| s.to_string()), bandwidth: attributes .get("BANDWIDTH") .and_then(|s| s.parse().ok()) .unwrap_or(0), - average_bandwidth: attributes - .get("AVERAGE-BANDWIDTH") - .and_then(|s| s.parse().ok()), - codecs: attributes.get("CODECS").map(|s| s.to_string()), + codecs: attributes.get("CODECS").map(|s| s.to_string())?, resolution: attributes.get("RESOLUTION").and_then(|s| { let mut parts = s.split('x'); let width = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); @@ -73,7 +79,6 @@ impl MasterPlaylist { } }), frame_rate: attributes.get("FRAME-RATE").and_then(|s| s.parse().ok()), - hdcp_level: attributes.get("HDCP-LEVEL").map(|s| s.to_string()), }), _ => None, }) @@ -112,11 +117,36 @@ impl MasterPlaylist { .map(|s| s == "YES") .unwrap_or_default(); + let group_id = attributes + .get("GROUP-ID") + .map(|s| s.to_string()) + .unwrap_or_default(); + + let bandwidth = attributes + .get("BANDWIDTH") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let codecs = attributes + .get("CODECS") + .map(|s| s.to_string()) + .unwrap_or_default(); + + let resolution = attributes.get("RESOLUTION").and_then(|s| { + let mut parts = s.split('x'); + let width = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + let height = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + if width == 0 || height == 0 { + None + } else { + Some((width, height)) + } + }); + + let frame_rate = attributes.get("FRAME-RATE").and_then(|s| s.parse().ok()); + Some(( - attributes - .get("GROUP-ID") - .map(|s| s.to_string()) - .unwrap_or_default(), + group_id.clone(), Media { media_type, uri, @@ -124,6 +154,11 @@ impl MasterPlaylist { autoselect, default, forced, + group_id, + bandwidth, + codecs, + resolution, + frame_rate, }, )) } @@ -137,6 +172,31 @@ impl MasterPlaylist { }, ); - Ok(Self { streams, groups }) + let scuf_groups = tags + .iter() + .filter_map(|t| match t { + Tag::ExtXScufGroup(attributes) => { + let priority = attributes + .get("PRIORITY") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + Some(( + attributes + .get("GROUP") + .map(|s| s.to_string()) + .unwrap_or_default(), + ScufGroup { priority }, + )) + } + _ => None, + }) + .collect(); + + Ok(Self { + streams, + groups, + scuf_groups, + }) } } diff --git a/frontend/player/src/hls/playlist/media.rs b/frontend/player/src/hls/playlist/media.rs index 823ad9c6..567e8c10 100644 --- a/frontend/player/src/hls/playlist/media.rs +++ b/frontend/player/src/hls/playlist/media.rs @@ -6,12 +6,21 @@ use super::utils::Tag; pub struct MediaPlaylist { pub version: u8, pub target_duration: u32, + pub part_target_duration: Option, pub media_sequence: u32, pub discontinuity_sequence: u32, pub end_list: bool, pub server_control: Option, pub segments: Vec, pub preload_hint: Vec, + pub rendition_reports: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RenditionReport { + pub uri: String, + pub last_msn: u32, + pub last_part: u32, } #[derive(Debug, Clone, Serialize)] @@ -122,6 +131,7 @@ impl MediaPlaylist { let mut current_segment = None; let mut program_date_time = None; let mut discontinuity = false; + let mut part_target_duration = None; for tag in tags.iter() { match tag { @@ -191,6 +201,16 @@ impl MediaPlaylist { current_segment.parts.push(part); } + Tag::ExtXPartInf(attributes) => { + let duration = attributes + .get("PART-TARGET") + .ok_or("no DURATION attribute found")?; + let duration = duration + .parse() + .map_err(|_| "DURATION attribute is not a number")?; + + part_target_duration = Some(duration); + } _ => {} } } @@ -219,6 +239,36 @@ impl MediaPlaylist { }) .collect::, _>>()?; + let rendition_reports = tags + .iter() + .filter_map(|t| match t { + Tag::ExtXRenditionReport(attributes) => { + let Some(uri) = attributes.get("URI") else { + return Some(Err("no URI attribute found")); + }; + let Some(last_msn) = attributes.get("LAST-MSN") else { + return Some(Err("no LAST-MSN attribute found")); + }; + let Ok(last_msn) = last_msn.parse() else { + return Some(Err("LAST-MSN attribute is not a number")); + }; + let Some(last_part) = attributes.get("LAST-PART") else { + return Some(Err("no LAST-PART attribute found")); + }; + let Ok(last_part) = last_part.parse() else { + return Some(Err("LAST-PART attribute is not a number")); + }; + + Some(Ok(RenditionReport { + uri: uri.clone(), + last_msn, + last_part, + })) + } + _ => None, + }) + .collect::, _>>()?; + Ok(Self { version, target_duration, @@ -228,6 +278,8 @@ impl MediaPlaylist { server_control, segments, preload_hint, + part_target_duration, + rendition_reports, }) } } diff --git a/frontend/player/src/hls/playlist/utils.rs b/frontend/player/src/hls/playlist/utils.rs index 65a556b6..1040357d 100644 --- a/frontend/player/src/hls/playlist/utils.rs +++ b/frontend/player/src/hls/playlist/utils.rs @@ -34,6 +34,7 @@ pub enum Tag { ExtXSessionData(HashMap), ExtXSessionKey(HashMap), ExtXContentSteering(HashMap), + ExtXScufGroup(HashMap), Unknown(String), } @@ -52,6 +53,7 @@ impl Tag { | Tag::ExtXSessionData(_) | Tag::ExtXSessionKey(_) | Tag::ExtXContentSteering(_) + | Tag::ExtXScufGroup(_) | Tag::Unknown(_) ) } @@ -398,6 +400,14 @@ fn parse_tag<'a>(lines: &mut impl Iterator) -> Result { + let attributes = parse_attributes( + line.strip_prefix("#EXT-X-SCUF-GROUP:") + .ok_or("invalid scuf group")?, + )?; + + Ok(Some(Tag::ExtXScufGroup(attributes))) + } line => Ok(Some(Tag::Unknown(line.into()))), } } diff --git a/frontend/player/src/player/bandwidth.rs b/frontend/player/src/player/bandwidth.rs new file mode 100644 index 00000000..10dbc94f --- /dev/null +++ b/frontend/player/src/player/bandwidth.rs @@ -0,0 +1,82 @@ +use std::{cell::RefCell, rc::Rc, sync::atomic::AtomicUsize}; + +use super::fetch::Metrics; + +#[derive(Debug)] +struct Report { + total_bytes: u32, + bandwidth: u32, +} + +#[derive(Clone)] +pub struct Bandwidth { + bandwidth: Rc>>, + max_count: Rc, +} + +impl Default for Bandwidth { + fn default() -> Self { + Self { + bandwidth: Rc::new(RefCell::new(Vec::new())), + max_count: Rc::new(AtomicUsize::new(10)), + } + } +} + +impl std::fmt::Debug for Bandwidth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Bandwidth") + .field("bandwidth", &self.get()) + .field("reports", &self.bandwidth.borrow().as_slice()) + .finish() + } +} + +impl Bandwidth { + pub fn new() -> Self { + Self::default() + } + + pub fn get(&self) -> Option { + let inner = self.bandwidth.borrow(); + if inner.is_empty() { + return None; + } + + let total = inner.iter().map(|r| r.total_bytes).sum::() as f64; + Some( + inner + .iter() + .map(|r| r.bandwidth as f64 * r.total_bytes as f64 / total) + .sum::() as u32, + ) + } + + pub fn set_max_count(&self, max_count: usize) { + self.max_count + .store(max_count, std::sync::atomic::Ordering::Relaxed); + } + + pub fn report_download(&self, metrics: &Metrics) { + if metrics.download_size == 0 { + return; + } + + let real_download_time = metrics.download_time / 1000.0; + let real_bandwidth = metrics.download_size as f64 / real_download_time; + + { + let mut inner = self.bandwidth.borrow_mut(); + let size = inner.len(); + let max_count = self.max_count.load(std::sync::atomic::Ordering::Relaxed); + if size > max_count { + inner.drain(0..(size - max_count)); + } + + inner.push(Report { + total_bytes: metrics.download_size, + bandwidth: real_bandwidth as u32, + }); + } + } +} diff --git a/frontend/player/src/player/blank.rs b/frontend/player/src/player/blank.rs index bfd72d37..c400d147 100644 --- a/frontend/player/src/player/blank.rs +++ b/frontend/player/src/player/blank.rs @@ -1,5 +1,5 @@ use bytes::Bytes; -use h264::{AVCDecoderConfigurationRecord, AvccExtendedConfig}; +use h264::AVCDecoderConfigurationRecord; use mp4::{ types::{ avc1::Avc1, @@ -18,6 +18,7 @@ use mp4::{ stco::Stco, stsc::Stsc, stsd::{SampleEntry, Stsd, VisualSampleEntry}, + stsz::Stsz, stts::Stts, tfdt::Tfdt, tfhd::Tfhd, @@ -34,6 +35,7 @@ use mp4::{ pub struct VideoFactory { timescale: u32, sequence_number: u32, + stop_at: Option, } impl VideoFactory { @@ -41,9 +43,18 @@ impl VideoFactory { Self { sequence_number: 0, timescale, + stop_at: None, } } + pub fn stop_at(&self) -> Option { + self.stop_at + } + + pub fn set_stop_at(&mut self, stop_at: Option) { + self.stop_at = stop_at; + } + pub fn moov(&self) -> Moov { Moov::new( Mvhd::new(0, 0, 1000, 0, 2), @@ -53,7 +64,7 @@ impl VideoFactory { 0, 1, 0, - Some((2, 2)), + Some((100, 100)), ), None, Mdia::new( @@ -69,21 +80,16 @@ impl VideoFactory { profile_compatibility: 0, level_indication: 10, length_size_minus_one: 3, - sps: vec![Bytes::from_static(b"gd\0\n\xac\xd9_\x88\x88\xc0D\0\0\x03\0\x04\0\0\x03\0\x08 (Moof, Mdat) { self.sequence_number += 1; - let mdat = Mdat::new(vec![Bytes::from_static(b"\0\0\x02\xad\x06\x05\xff\xff\xa9\xdcE\xe9\xbd\xe6\xd9H\xb7\x96,\xd8 \xd9#\xee\xefx264 - core 163 r3060 5db6aa6 - H.264/MPEG-4 AVC codec - Copyleft 2003-2021 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=1 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=1 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00\0\x80\0\0\0\x10e\x88\x84\0\x15\xff\xfe\xf7\xc9\xef\xc0\xa6\xeb\xdb\xdf\x81")]); + let mdat = Mdat::new(vec![Bytes::from_static(b"\x00\x00\x02\xad\x06\x05\xff\xff\xa9\xdc\x45\xe9\xbd\xe6\xd9\x48\xb7\x96\x2c\xd8\x20\xd9\x23\xee\xef\x78\x32\x36\x34\x20\x2d\x20\x63\x6f\x72\x65\x20\x31\x36\x33\x20\x72\x33\x30\x36\x30\x20\x35\x64\x62\x36\x61\x61\x36\x20\x2d\x20\x48\x2e\x32\x36\x34\x2f\x4d\x50\x45\x47\x2d\x34\x20\x41\x56\x43\x20\x63\x6f\x64\x65\x63\x20\x2d\x20\x43\x6f\x70\x79\x6c\x65\x66\x74\x20\x32\x30\x30\x33\x2d\x32\x30\x32\x31\x20\x2d\x20\x68\x74\x74\x70\x3a\x2f\x2f\x77\x77\x77\x2e\x76\x69\x64\x65\x6f\x6c\x61\x6e\x2e\x6f\x72\x67\x2f\x78\x32\x36\x34\x2e\x68\x74\x6d\x6c\x20\x2d\x20\x6f\x70\x74\x69\x6f\x6e\x73\x3a\x20\x63\x61\x62\x61\x63\x3d\x31\x20\x72\x65\x66\x3d\x33\x20\x64\x65\x62\x6c\x6f\x63\x6b\x3d\x31\x3a\x30\x3a\x30\x20\x61\x6e\x61\x6c\x79\x73\x65\x3d\x30\x78\x33\x3a\x30\x78\x31\x31\x33\x20\x6d\x65\x3d\x68\x65\x78\x20\x73\x75\x62\x6d\x65\x3d\x37\x20\x70\x73\x79\x3d\x31\x20\x70\x73\x79\x5f\x72\x64\x3d\x31\x2e\x30\x30\x3a\x30\x2e\x30\x30\x20\x6d\x69\x78\x65\x64\x5f\x72\x65\x66\x3d\x31\x20\x6d\x65\x5f\x72\x61\x6e\x67\x65\x3d\x31\x36\x20\x63\x68\x72\x6f\x6d\x61\x5f\x6d\x65\x3d\x31\x20\x74\x72\x65\x6c\x6c\x69\x73\x3d\x31\x20\x38\x78\x38\x64\x63\x74\x3d\x31\x20\x63\x71\x6d\x3d\x30\x20\x64\x65\x61\x64\x7a\x6f\x6e\x65\x3d\x32\x31\x2c\x31\x31\x20\x66\x61\x73\x74\x5f\x70\x73\x6b\x69\x70\x3d\x31\x20\x63\x68\x72\x6f\x6d\x61\x5f\x71\x70\x5f\x6f\x66\x66\x73\x65\x74\x3d\x2d\x32\x20\x74\x68\x72\x65\x61\x64\x73\x3d\x33\x20\x6c\x6f\x6f\x6b\x61\x68\x65\x61\x64\x5f\x74\x68\x72\x65\x61\x64\x73\x3d\x31\x20\x73\x6c\x69\x63\x65\x64\x5f\x74\x68\x72\x65\x61\x64\x73\x3d\x30\x20\x6e\x72\x3d\x30\x20\x64\x65\x63\x69\x6d\x61\x74\x65\x3d\x31\x20\x69\x6e\x74\x65\x72\x6c\x61\x63\x65\x64\x3d\x30\x20\x62\x6c\x75\x72\x61\x79\x5f\x63\x6f\x6d\x70\x61\x74\x3d\x30\x20\x63\x6f\x6e\x73\x74\x72\x61\x69\x6e\x65\x64\x5f\x69\x6e\x74\x72\x61\x3d\x30\x20\x62\x66\x72\x61\x6d\x65\x73\x3d\x33\x20\x62\x5f\x70\x79\x72\x61\x6d\x69\x64\x3d\x32\x20\x62\x5f\x61\x64\x61\x70\x74\x3d\x31\x20\x62\x5f\x62\x69\x61\x73\x3d\x30\x20\x64\x69\x72\x65\x63\x74\x3d\x31\x20\x77\x65\x69\x67\x68\x74\x62\x3d\x31\x20\x6f\x70\x65\x6e\x5f\x67\x6f\x70\x3d\x30\x20\x77\x65\x69\x67\x68\x74\x70\x3d\x32\x20\x6b\x65\x79\x69\x6e\x74\x3d\x32\x35\x30\x20\x6b\x65\x79\x69\x6e\x74\x5f\x6d\x69\x6e\x3d\x31\x20\x73\x63\x65\x6e\x65\x63\x75\x74\x3d\x34\x30\x20\x69\x6e\x74\x72\x61\x5f\x72\x65\x66\x72\x65\x73\x68\x3d\x30\x20\x72\x63\x5f\x6c\x6f\x6f\x6b\x61\x68\x65\x61\x64\x3d\x34\x30\x20\x72\x63\x3d\x63\x72\x66\x20\x6d\x62\x74\x72\x65\x65\x3d\x31\x20\x63\x72\x66\x3d\x32\x33\x2e\x30\x20\x71\x63\x6f\x6d\x70\x3d\x30\x2e\x36\x30\x20\x71\x70\x6d\x69\x6e\x3d\x30\x20\x71\x70\x6d\x61\x78\x3d\x36\x39\x20\x71\x70\x73\x74\x65\x70\x3d\x34\x20\x69\x70\x5f\x72\x61\x74\x69\x6f\x3d\x31\x2e\x34\x30\x20\x61\x71\x3d\x31\x3a\x31\x2e\x30\x30\x00\x80\x00\x00\x00\x2e\x65\x88\x84\x00\x15\xff\xfe\xf7\xc9\xef\xc0\xa6\xeb\xdb\xde\xb5\xbf\x93\xcf\x48\xfc\x2c\xb7\x3e\xca\xf4\x4d\xb5\x40\x78\x78\xd4\x35\xda\xfb\xf5\xfb\x25\x80\x10\xa0\x06\xb8\x55\x69\xc1")]); let mut moof = Moof::new( Mfhd::new(self.sequence_number), diff --git a/frontend/player/src/player/events.rs b/frontend/player/src/player/events.rs index 3917cdf4..23d49d54 100644 --- a/frontend/player/src/player/events.rs +++ b/frontend/player/src/player/events.rs @@ -2,7 +2,7 @@ use serde::Serialize; use tsify::Tsify; use wasm_bindgen::prelude::*; -use super::track::Track; +use super::track::{Track, Variant}; #[wasm_bindgen(getter_with_clone)] pub struct EventError { @@ -40,6 +40,7 @@ type OnManifestLoadedFunction = (this: null, evt: EventManifestLoaded) => void; pub struct EventManifestLoaded { pub is_master_playlist: bool, pub tracks: Vec, + pub variants: Vec, } #[wasm_bindgen] diff --git a/frontend/player/src/player/fetch.rs b/frontend/player/src/player/fetch.rs index 2a8b2edc..131c48ee 100644 --- a/frontend/player/src/player/fetch.rs +++ b/frontend/player/src/player/fetch.rs @@ -1,18 +1,84 @@ -use std::collections::HashMap; +use std::{cell::RefCell, collections::HashMap, rc::Rc}; +use gloo_timers::future::TimeoutFuture; use js_sys::ArrayBuffer; use tokio::sync::mpsc; +use url::Url; use wasm_bindgen::{prelude::Closure, JsCast, JsValue}; -use web_sys::{XmlHttpRequest, XmlHttpRequestResponseType}; +use web_sys::{ + PerformanceObserver, PerformanceObserverEntryList, PerformanceObserverInit, + PerformanceResourceTiming, XmlHttpRequest, XmlHttpRequestResponseType, +}; + +use super::util::{register_events, Holder}; + +thread_local!(static PERFORMANCE_OBSERVER: PerformanceObserverHolder = PerformanceObserverHolder::new()); + +struct PerformanceObserverHolder { + _obs: PerformanceObserver, + _cb: Closure, + entries: Rc>>, +} + +impl PerformanceObserverHolder { + fn new() -> Self { + let entries = Rc::new(RefCell::new(Vec::new())); + + tracing::info!("creating performance observer"); + + let cb = { + let entries = entries.clone(); + Closure::::new(move |items: PerformanceObserverEntryList| { + let mut entries = entries.borrow_mut(); + entries.extend( + items + .get_entries() + .iter() + .map(|i| i.unchecked_into::()), + ); + + let size = entries.len(); + if size > 250 { + entries.drain(0..size - 250); + } + }) + }; + + let obs = PerformanceObserver::new(cb.as_ref().unchecked_ref()).unwrap(); + + obs.observe(&PerformanceObserverInit::new(&js_sys::Array::of1( + &"resource".into(), + ))); + + Self { + _obs: obs, + entries, + _cb: cb, + } + } + + fn get_entries_by_name(&self, name: &str) -> Vec { + let entries = self.entries.borrow(); + let entries = entries + .iter() + .filter(|e| e.name() == name) + .cloned() + .collect(); + + entries + } +} pub struct FetchRequest { - url: String, + url: Url, headers: HashMap, method: String, timeout: Option, } pub struct InflightRequest { - xhr: XmlHttpRequest, + url: Url, + xhr: Holder, + rx: mpsc::Receiver<()>, } #[derive(Debug, Default, Clone)] @@ -21,14 +87,12 @@ pub struct Metrics { pub ttfb: f64, pub download_time: f64, pub download_size: u32, - pub size: u32, - pub cached: bool, } impl FetchRequest { - pub fn new(method: impl ToString, url: impl ToString) -> Self { + pub fn new(method: &str, url: Url) -> Self { Self { - url: url.to_string(), + url, headers: HashMap::new(), method: method.to_string(), timeout: None, @@ -46,6 +110,9 @@ impl FetchRequest { } pub fn start(&self) -> Result { + // Make sure the performance observer is initialized + PERFORMANCE_OBSERVER.with(|_| {}); + let req = XmlHttpRequest::new()?; req.set_response_type(XmlHttpRequestResponseType::Arraybuffer); @@ -54,7 +121,7 @@ impl FetchRequest { req.set_timeout(timeout); } - req.open(&self.method, &self.url)?; + req.open(&self.method, self.url.as_str())?; for (key, value) in &self.headers { req.set_request_header(key, value)?; @@ -62,25 +129,41 @@ impl FetchRequest { req.send()?; - Ok(InflightRequest { xhr: req }) - } -} + // We need a large enough buffer to hold all events that can be fired + // This is becasue we might read the events until the request is done + let (tx, rx) = mpsc::channel(4); + + let cb = register_events!(req, { + "loadend" => { + move |_| { + if tx.try_send(()).is_err() { + tracing::warn!("fetch event queue full"); + } + } + }, + }); -impl InflightRequest { - pub async fn wait_result(&self) -> Result, JsValue> { - let (tx, mut rx) = mpsc::channel(1); + let xhr = Holder::new(req, cb); - let closure = Closure::::new(move || { - tx.try_send(()).ok(); - }); + Ok(InflightRequest { + url: self.url.clone(), + xhr, + rx, + }) + } - self.xhr - .set_onloadend(Some(closure.as_ref().unchecked_ref())); + pub fn url(&self) -> &Url { + &self.url + } +} - rx.recv().await; +impl InflightRequest { + pub async fn wait_result(&mut self) -> Result, JsValue> { + self.rx.recv().await; - self.xhr.set_onloadend(None); - drop(closure); + while !self.is_done() { + TimeoutFuture::new(0).await; + } let result = self.result()?; @@ -88,10 +171,30 @@ impl InflightRequest { } pub fn is_done(&self) -> bool { - self.xhr.ready_state() == XmlHttpRequest::DONE + self.xhr.ready_state() == XmlHttpRequest::DONE && self.metrics().is_some() } - pub fn result(&self) -> Result>, JsValue> { + pub fn metrics(&self) -> Option { + let entity = PERFORMANCE_OBSERVER.with(|p| p.get_entries_by_name(self.url.as_str())); + + let Some(entity) = entity.first() else { + return None; + }; + + let start_time = entity.fetch_start(); + let ttfb = entity.response_start() - start_time; + let download_time = entity.response_end() - entity.response_start(); + let download_size = entity.transfer_size() as u32; + + Some(Metrics { + start_time, + ttfb, + download_size, + download_time, + }) + } + + pub fn result(&mut self) -> Result>, JsValue> { if !self.is_done() { return Ok(None); } @@ -103,8 +206,12 @@ impl InflightRequest { } let resp = self.xhr.response()?; + if resp.is_null() { + return Err("request aborted or no response".into()); + } + let Some(buf) = resp.dyn_ref::() else { - return Err(resp); + return Err("response is not an ArrayBuffer".into()); }; Ok(Some(js_sys::Uint8Array::new(buf).to_vec())) @@ -113,6 +220,10 @@ impl InflightRequest { pub fn abort(&self) { self.xhr.abort().ok(); } + + pub fn url(&self) -> &Url { + &self.url + } } impl Drop for InflightRequest { diff --git a/frontend/player/src/player/inner.rs b/frontend/player/src/player/inner.rs index bfbd2551..b1194278 100644 --- a/frontend/player/src/player/inner.rs +++ b/frontend/player/src/player/inner.rs @@ -1,6 +1,6 @@ use std::{ - cell::{Cell, Ref, RefCell, RefMut}, - panic::Location, + cell::UnsafeCell, + ops::{Deref, DerefMut}, rc::Rc, }; @@ -9,33 +9,34 @@ use web_sys::HtmlVideoElement; use super::{ events::{EventError, EventManifestLoaded, OnErrorFunction, OnManifestLoadedFunction}, - track::Track, + track::{Track, Variant}, }; pub struct PlayerInner { url: String, low_latency: bool, + abr_enabled: bool, is_master_playlist: bool, - abr_estimate: Option, video_element: Option, on_error: Option, on_manifest_loaded: Option, tracks: Vec, + variants: Vec, - active_track_id: u32, - active_reference_track_ids: Vec, + active_group_track_ids: Vec, - next_track_id: Option, + active_variant_id: u32, + next_variant_id: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NextTrack { +pub enum NextVariant { Switch(u32), Force(u32), } -impl NextTrack { - pub fn track_id(&self) -> u32 { +impl NextVariant { + pub fn variant_id(&self) -> u32 { match self { Self::Switch(id) | Self::Force(id) => *id, } @@ -51,64 +52,70 @@ impl Default for PlayerInner { Self { url: String::new(), low_latency: true, + abr_enabled: true, is_master_playlist: false, - abr_estimate: None, on_error: None, on_manifest_loaded: None, tracks: Vec::new(), video_element: None, - active_reference_track_ids: Vec::new(), - active_track_id: 0, - next_track_id: None, + variants: Vec::new(), + active_group_track_ids: Vec::new(), + next_variant_id: None, + active_variant_id: 0, } } } #[derive(Default, Clone)] -pub struct PlayerInnerHolder { - inner: Rc>, - previous_holder: Cell>>, -} - -impl PlayerInnerHolder { - const AQUIRE_ERROR: &'static str = r#"We failed to borrow the inner state, this is a bug! -Likely caused by holidng a reference to the inner state across an await point. -If you see this error, please file a bug report at https://github.com/scuffletv/scuffle"#; - - #[track_caller] - pub fn aquire(&self) -> Ref<'_, PlayerInner> { - let Ok(inner) = self.inner.try_borrow() else { - tracing::error!("{}\nPrevious hold at: {}\nNew hold at: {}", Self::AQUIRE_ERROR, self.previous_holder.get().unwrap(), Location::caller()); - unreachable!("{}", Self::AQUIRE_ERROR) - }; - - self.previous_holder.set(Some(Location::caller())); - - inner +pub struct PlayerInnerHolder(Rc>); + +impl Deref for PlayerInnerHolder { + type Target = PlayerInner; + + fn deref(&self) -> &Self::Target { + // Safety: PlayerInner does not return any references to its fields. It also requires that the caller does not create references to PlayerInner. + // Ie. do not manually dereference the holder, or pass a dereferenced value to another function. + // Always use the PlayerInnerHolder as the type and pass that around cloning it. + // Without this unsafe block it becomes very complex to interop with the JS side. + // TLDR: + // 1. PlayerInner does not return references to its fields. + // 2. Requires that the caller does not create references to PlayerInner. + // 3. Always use the PlayerInnerHolder as the type and pass that around cloning it. + unsafe { &*self.0.get() } } +} - #[track_caller] - pub fn aquire_mut(&self) -> RefMut<'_, PlayerInner> { - let Ok(inner) = self.inner.try_borrow_mut() else { - tracing::error!("{}\nPrevious hold at: {}\nNew hold at: {}", Self::AQUIRE_ERROR, self.previous_holder.get().unwrap(), Location::caller()); - unreachable!("{}", Self::AQUIRE_ERROR) - }; - - self.previous_holder.set(Some(Location::caller())); - - inner +impl DerefMut for PlayerInnerHolder { + fn deref_mut(&mut self) -> &mut Self::Target { + // Safety: PlayerInner does not return any references to its fields. It also requires that the caller does not create references to PlayerInner. + // Ie. do not manually dereference the holder, or pass a dereferenced value to another function. + // Always use the PlayerInnerHolder as the type and pass that around cloning it. + // Without this unsafe block it becomes very complex to interop with the JS side. + // TLDR: + // 1. PlayerInner does not return references to its fields. + // 2. Requires that the caller does not create references to PlayerInner. + // 3. Always use the PlayerInnerHolder as the type and pass that around cloning it. + unsafe { &mut *self.0.get() } } } impl PlayerInner { - pub fn url(&self) -> &str { - &self.url + pub fn url(&self) -> String { + self.url.clone() } pub fn low_latency(&self) -> bool { self.low_latency } + pub fn abr_enabled(&self) -> bool { + self.abr_enabled + } + + pub fn set_abr_enabled(&mut self, abr_enabled: bool) { + self.abr_enabled = abr_enabled; + } + pub fn video_element(&self) -> Option { self.video_element.clone() } @@ -117,8 +124,12 @@ impl PlayerInner { self.video_element = element; } - pub fn tracks(&self) -> &[Track] { - &self.tracks + pub fn tracks(&self) -> Vec { + self.tracks.clone() + } + + pub fn variants(&self) -> Vec { + self.variants.clone() } pub fn set_url(&mut self, url: impl ToString) { @@ -149,64 +160,63 @@ impl PlayerInner { self.low_latency = low_latency; } - pub fn set_abr_estimate(&mut self, abr_estimate: Option) { - self.abr_estimate = abr_estimate; - } - - pub fn set_active_track_id(&mut self, track_id: u32) { - self.active_track_id = track_id; + pub fn set_active_variant_id(&mut self, variant_id: u32) { + self.active_variant_id = variant_id; } - pub fn active_track_id(&self) -> u32 { - self.active_track_id + pub fn active_variant_id(&self) -> u32 { + self.active_variant_id } - pub fn set_active_reference_track_ids(&mut self, groups: Vec) { - self.active_reference_track_ids = groups; + pub fn set_active_group_track_ids(&mut self, track_ids: Vec) { + self.active_group_track_ids = track_ids; } - pub fn set_next_track_id(&mut self, track_id: Option) { - self.next_track_id = track_id; + pub fn set_next_variant_id(&mut self, variant_id: Option) { + self.next_variant_id = variant_id; } - pub fn next_track_id(&self) -> Option { - self.next_track_id + pub fn next_variant_id(&self) -> Option { + self.next_variant_id } - pub fn set_tracks(&mut self, tracks: Vec, master_playlist: bool) -> impl FnOnce() { + pub fn set_tracks( + &mut self, + tracks: Vec, + variants: Vec, + master_playlist: bool, + ) { self.tracks = tracks; self.is_master_playlist = master_playlist; + self.variants = variants; self.send_manifest_loaded(EventManifestLoaded { is_master_playlist: self.is_master_playlist, + variants: self.variants.clone(), tracks: self.tracks.clone(), }) } - pub fn send_error(&self, error: EventError) -> impl FnOnce() { + pub fn send_error(&self, error: EventError) { let js_fn = self.on_error(); - move || { - if let Some(f) = js_fn { - if let Err(err) = f.call(JsValue::null(), error) { - tracing::error!("Error in on_error callback: {:?}", err); - } + if let Some(f) = js_fn { + if let Err(err) = f.call(JsValue::null(), error) { + tracing::error!("Error in on_error callback: {:?}", err); } } } - pub fn send_manifest_loaded(&self, evt: EventManifestLoaded) -> impl FnOnce() { + pub fn send_manifest_loaded(&self, evt: EventManifestLoaded) { let js_fn = self.on_manifest_loaded(); - move || { - if let Some(f) = js_fn { - if let Err(err) = f - .dyn_ref::() - .unwrap() - .clone() - .unchecked_into::() - .call(JsValue::null(), evt) - { - tracing::error!("Error in on_manifest_loaded callback: {:?}", err); - } + if let Some(f) = js_fn { + if let Err(err) = f + .dyn_ref::() + .unwrap() + .clone() + .unchecked_into::() + .call(JsValue::null(), evt) + { + tracing::error!("Error in on_manifest_loaded callback: {:?}", err); } } } diff --git a/frontend/player/src/player/mod.rs b/frontend/player/src/player/mod.rs index 57553072..7c9e5bc6 100644 --- a/frontend/player/src/player/mod.rs +++ b/frontend/player/src/player/mod.rs @@ -6,10 +6,11 @@ use web_sys::HtmlVideoElement; use self::{ events::{OnErrorFunction, OnManifestLoadedFunction}, - inner::{NextTrack, PlayerInnerHolder}, + inner::{NextVariant, PlayerInnerHolder}, runner::PlayerRunner, }; +mod bandwidth; mod blank; mod events; mod fetch; @@ -36,6 +37,12 @@ extern "C" { pub type VectorTracks; } +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "Variant[]")] + pub type VectorVariant; +} + #[wasm_bindgen] impl Player { #[wasm_bindgen(constructor)] @@ -50,25 +57,19 @@ impl Player { } #[wasm_bindgen(setter = lowLatency)] - pub fn set_low_latency(&self, low_latency: bool) { - self.inner.aquire_mut().set_low_latency(low_latency); + pub fn set_low_latency(&mut self, low_latency: bool) { + self.inner.set_low_latency(low_latency); } #[wasm_bindgen(getter = lowLatency)] pub fn low_latency(&self) -> bool { - self.inner.aquire().low_latency() + self.inner.low_latency() } - pub fn set_abr_estimate(&self, abr_estimate: Option) { - self.inner.aquire_mut().set_abr_estimate(abr_estimate); - } - - pub fn load(&self, url: &str) -> Result<(), JsValue> { - let mut inner = self.inner.aquire_mut(); - - inner.set_url(url); + pub fn load(&mut self, url: &str) -> Result<(), JsValue> { + self.inner.set_url(url); - if inner.video_element().is_none() { + if self.inner.video_element().is_none() { return Ok(()); } @@ -79,19 +80,18 @@ impl Player { } #[wasm_bindgen(setter = onerror)] - pub fn set_on_error(&self, f: Option) { - self.inner.aquire_mut().set_on_error(f); + pub fn set_on_error(&mut self, f: Option) { + self.inner.set_on_error(f); } #[wasm_bindgen(getter = onerror)] pub fn on_error(&self) -> Option { - self.inner.aquire().on_error() + self.inner.on_error() } #[wasm_bindgen(getter = tracks)] pub fn tracks(&self) -> VectorTracks { self.inner - .aquire() .tracks() .iter() .map(JsValue::from_serde) @@ -100,30 +100,51 @@ impl Player { .unchecked_into() } + #[wasm_bindgen(getter = variants)] + pub fn variants(&self) -> VectorVariant { + self.inner + .variants() + .iter() + .map(JsValue::from_serde) + .collect::>() + .unwrap() + .unchecked_into() + } + #[wasm_bindgen(setter = onmanifestloaded)] - pub fn set_on_manifest_loaded(&self, f: Option) { - self.inner.aquire_mut().set_on_manifest_loaded(f); + pub fn set_on_manifest_loaded(&mut self, f: Option) { + self.inner.set_on_manifest_loaded(f); } #[wasm_bindgen(getter = onmanifestloaded)] pub fn on_manifest_loaded(&self) -> Option { - self.inner.aquire().on_manifest_loaded() + self.inner.on_manifest_loaded() + } + + #[wasm_bindgen(setter = abrEnabled)] + pub fn set_abr_enabled(&mut self, abr_enabled: bool) { + self.inner.set_abr_enabled(abr_enabled); } - pub fn attach(&self, el: HtmlVideoElement) -> Result<(), JsValue> { + #[wasm_bindgen(getter = abrEnabled)] + pub fn abr_enabled(&self) -> bool { + self.inner.abr_enabled() + } + + pub fn attach(&mut self, el: HtmlVideoElement) -> Result<(), JsValue> { let Ok(element) = el.dyn_into::() else { return Err(JsValue::from_str("element is not a video element")); }; - if let Some(el) = self.inner.aquire().video_element() { + if let Some(el) = self.inner.video_element() { if el.is_same_node(Some(&element)) { return Err(JsValue::from_str("element is already attached")); } } - self.inner.aquire_mut().set_video_element(Some(element)); + self.inner.set_video_element(Some(element)); - if self.inner.aquire().url().is_empty() { + if self.inner.url().is_empty() { return Ok(()); } @@ -137,45 +158,46 @@ impl Player { self.shutdown_sender.send(()).ok(); } - /// Gracefully switch to this track id when the current segment is finished. - #[wasm_bindgen(setter = nextTrackId)] - pub fn set_next_track_id(&self, track_id: Option) { + /// Gracefully switch to this variant id when the current segment is finished. + #[wasm_bindgen(setter = nextVariantId)] + pub fn set_next_variant_id(&mut self, track_id: Option) { + if let Some(track_id) = track_id { + self.inner.set_abr_enabled(false); + if self.inner.variants().len() as u32 <= track_id { + return; + } + } + self.inner - .aquire_mut() - .set_next_track_id(track_id.map(NextTrack::Switch)) + .set_next_variant_id(track_id.map(NextVariant::Switch)) } - /// Get the next track id that will be switched to. - #[wasm_bindgen(getter = nextTrackId)] - pub fn next_track_id(&self) -> Option { - match self.inner.aquire().next_track_id() { - Some(NextTrack::Switch(track_id)) | Some(NextTrack::Force(track_id)) => Some(track_id), + /// Get the variant track id that will be switched to. + #[wasm_bindgen(getter = nextVariantId)] + pub fn next_variant_id(&self) -> Option { + match self.inner.next_variant_id() { + Some(NextVariant::Switch(track_id)) | Some(NextVariant::Force(track_id)) => { + Some(track_id) + } None => None, } } - /// Force switch to this track id immediately. - #[wasm_bindgen(setter = forceTrackId)] - pub fn set_force_track_id(&self, track_id: Option) { - self.inner - .aquire_mut() - .set_next_track_id(track_id.map(NextTrack::Force)) + /// Get the variant id that is currently active. + #[wasm_bindgen(getter = variantId)] + pub fn variant_id(&self) -> u32 { + self.inner.active_variant_id() } - /// Get the track id that will be forced to switch to. - #[wasm_bindgen(getter = forceTrackId)] - pub fn force_track_id(&self) -> Option { - match self.inner.aquire().next_track_id() { - Some(NextTrack::Force(track_id)) => Some(track_id), - _ => None, + /// Force switch to this variant id immediately. + #[wasm_bindgen(setter = variantId)] + pub fn set_variant_id(&mut self, track_id: u32) { + if self.inner.variants().len() as u32 > track_id { + self.inner.set_abr_enabled(false); + self.inner + .set_next_variant_id(Some(NextVariant::Force(track_id))); } } - - /// Get the current track id. - #[wasm_bindgen(getter = trackId)] - pub fn track_id(&self) -> u32 { - self.inner.aquire().active_track_id() - } } impl Player { diff --git a/frontend/player/src/player/runner.rs b/frontend/player/src/player/runner.rs deleted file mode 100644 index 3210057b..00000000 --- a/frontend/player/src/player/runner.rs +++ /dev/null @@ -1,1180 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - pin::pin, -}; - -use crate::{ - hls::{ - self, - master::{MasterPlaylist, Media}, - media::MediaPlaylist, - }, - player::{ - fetch::FetchRequest, - track::{Fragment, ReferenceTrack, TrackResult}, - }, -}; - -use gloo_timers::future::TimeoutFuture; -use mp4::{ - types::{ - ftyp::{FourCC, Ftyp}, - moov::Moov, - }, - BoxType, -}; -use tokio::{ - select, - sync::{broadcast, mpsc}, -}; -use url::Url; -use wasm_bindgen::{prelude::Closure, JsCast, JsValue}; -use wasm_bindgen_futures::JsFuture; -use web_sys::{HtmlVideoElement, MediaSource, SourceBuffer}; - -use super::{ - blank::VideoFactory, - inner::PlayerInnerHolder, - track::{Track, TrackState}, - util::{register_events, Holder}, -}; - -struct SourceBufferHolder { - sb: Holder, - rx: mpsc::Receiver<()>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum TrackMapping { - Audio, - Video, - AudioVideo, - Buffer, -} - -impl SourceBufferHolder { - fn new(media_source: &MediaSource, codec: &str) -> Result { - let sb = media_source.add_source_buffer(codec)?; - let (tx, rx) = mpsc::channel(128); - - let cleanup = register_events!(sb, { - "updateend" => move |_| { - if tx.try_send(()).is_err() { - tracing::warn!("failed to send updateend event"); - } - } - }); - - Ok(Self { - sb: Holder::new(sb, cleanup), - rx, - }) - } - - fn change_type(&self, codec: &str) -> Result<(), JsValue> { - self.sb.change_type(codec)?; - Ok(()) - } - - async fn append_buffer(&mut self, mut data: Vec) -> Result<(), JsValue> { - self.sb.append_buffer_with_u8_array(data.as_mut_slice())?; - self.rx.recv().await; - Ok(()) - } - - async fn remove(&mut self, start: f64, end: f64) -> Result<(), JsValue> { - self.sb.remove(start, end)?; - self.rx.recv().await; - Ok(()) - } -} - -pub struct PlayerRunner { - inner: PlayerInnerHolder, - track_states: Vec, - - active_track_id: u32, - next_track_id: Option, - - active_reference_track_ids: Vec, - fragment_buffer: HashMap>, - - track_mapping: HashMap, - - moov_map: HashMap, - - force_mapping: HashMap, - - init: bool, - shutdown_recv: broadcast::Receiver<()>, - - low_latency: bool, - - video: Option, - audio: Option, - audiovideo: Option, - - media_source: Holder, - video_element: Holder, - - video_factory: Option, - - evt_recv: mpsc::Receiver<(RunnerEvent, web_sys::Event)>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RunnerEvent { - VideoError, - VideoPlay, - VideoPause, - VideoSuspend, - VideoStalled, - VideoWaiting, - VideoSeeking, - VideoSeeked, - VideoTimeUpdate, - VideoVolumeChange, - VideoRateChange, - MediaSourceOpen, - MediaSourceClose, - MediaSourceEnded, -} - -fn make_video_holder( - element: HtmlVideoElement, - tx: &mpsc::Sender<(RunnerEvent, web_sys::Event)>, -) -> Holder { - let cleanup = register_events!(element, { - "error" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoError, evt)).is_err() { - tracing::warn!("Video error event dropped"); - } - } - }, - "pause" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoPause, evt)).is_err() { - tracing::warn!("Video pause event dropped"); - } - } - }, - "play" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoPlay, evt)).is_err() { - tracing::warn!("Video play event dropped"); - } - } - }, - "ratechange" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoRateChange, evt)).is_err() { - tracing::warn!("Video ratechange event dropped"); - } - } - }, - "seeked" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoSeeked, evt)).is_err() { - tracing::warn!("Video seeked event dropped"); - } - } - }, - "seeking" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoSeeking, evt)).is_err() { - tracing::warn!("Video seeking event dropped"); - } - } - }, - "stalled" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoStalled, evt)).is_err() { - tracing::warn!("Video stalled event dropped"); - } - } - }, - "suspend" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoSuspend, evt)).is_err() { - tracing::warn!("Video suspend event dropped"); - } - } - }, - "timeupdate" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoTimeUpdate, evt)).is_err() { - tracing::warn!("Video timeupdate event dropped"); - } - } - }, - "volumechange" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoVolumeChange, evt)).is_err() { - tracing::warn!("Video volumechange event dropped"); - } - } - }, - "waiting" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::VideoWaiting, evt)).is_err() { - tracing::warn!("Video waiting event dropped"); - } - } - }, - }); - - Holder::new(element, cleanup) -} - -fn make_media_source_holder( - media_source: MediaSource, - tx: &mpsc::Sender<(RunnerEvent, web_sys::Event)>, -) -> Holder { - let cleanup = register_events!(media_source, { - "sourceclose" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::MediaSourceClose, evt)).is_err() { - tracing::warn!("MediaSource close event dropped") - } - } - }, - "sourceended" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::MediaSourceEnded, evt)).is_err() { - tracing::warn!("MediaSource ended event dropped") - } - } - }, - "sourceopen" => { - let tx = tx.clone(); - move |evt| { - if tx.try_send((RunnerEvent::MediaSourceOpen, evt)).is_err() { - tracing::warn!("MediaSource open event dropped") - } - } - }, - }); - - Holder::new(media_source, cleanup) -} - -impl PlayerRunner { - pub fn new(inner: PlayerInnerHolder, shutdown_recv: broadcast::Receiver<()>) -> Self { - let ms = MediaSource::new().unwrap(); - - let (tx, rx) = mpsc::channel(128); - - let video_element = make_video_holder(inner.aquire().video_element().unwrap(), &tx); - let media_source = make_media_source_holder(ms, &tx); - - Self { - inner, - track_states: Vec::new(), - shutdown_recv, - active_track_id: 0, - next_track_id: None, - moov_map: HashMap::new(), - init: false, - fragment_buffer: HashMap::new(), - active_reference_track_ids: Vec::new(), - track_mapping: HashMap::new(), - force_mapping: HashMap::new(), - low_latency: false, - audio: None, - video: None, - audiovideo: None, - media_source, - video_element, - video_factory: None, - evt_recv: rx, - } - } - - pub async fn start(mut self) { - match self.bind_element().await { - Err(err) => { - tracing::error!("failed to bind element: {:?}", err); - self.inner.aquire().send_error(err.into())(); - return; - } - Ok(true) => {} - Ok(false) => return, - } - - match self.fetch_playlist().await { - Err(err) => { - tracing::error!("failed to handle playlist: {:?}", err); - self.inner.aquire().send_error(err.into())(); - return; - } - Ok(true) => {} - Ok(false) => return, - } - - self.active_track_id = self.inner.aquire().active_track_id(); - - tracing::info!("starting playback"); - - for tid in self.active_track_ids() { - self.track_states.get_mut(tid as usize).unwrap().start(); - } - - 'running: loop { - self.set_low_latency(!self.init); - - let next_track_id = self.inner.aquire().next_track_id(); - if let Some(next_track_id) = next_track_id { - if Some(next_track_id.track_id()) != self.next_track_id { - if self.next_track_id.is_some() { - self.active_track_ids() - .difference(&self.next_track_ids()) - .for_each(|tid| { - self.track_states - .get_mut(*tid as usize) - .unwrap() - .set_stop_at(None); - tracing::trace!("resuming track: {}", tid); - }); - self.next_track_ids() - .difference(&self.active_track_ids()) - .for_each(|tid| { - self.track_states.get_mut(*tid as usize).unwrap().stop(); - tracing::trace!("stopped track: {}", tid); - }); - } - - if next_track_id.track_id() != self.active_track_id { - self.next_track_id = Some(next_track_id.track_id()); - self.next_track_ids() - .difference(&self.active_track_ids()) - .for_each(|tid| { - self.track_states.get_mut(*tid as usize).unwrap().start(); - tracing::trace!("starting track: {}", tid); - }); - } else { - self.next_track_id = None; - self.inner.aquire_mut().set_next_track_id(None); - self.inner - .aquire_mut() - .set_active_track_id(self.active_track_id); - self.fragment_buffer.clear(); - } - } - - if next_track_id.is_force() && self.next_track_id.is_some() { - self.active_track_ids() - .difference(&self.next_track_ids()) - .for_each(|tid| { - self.track_states.get_mut(*tid as usize).unwrap().stop(); - tracing::trace!("stopping track: {}", tid); - match self.track_mapping.get(tid) { - Some(TrackMapping::Audio) => { - self.force_mapping.insert(TrackMapping::Audio, ()) - } - Some(TrackMapping::Video) => { - self.force_mapping.insert(TrackMapping::Video, ()) - } - Some(TrackMapping::AudioVideo) => { - self.force_mapping.insert(TrackMapping::AudioVideo, ()) - } - _ => None, - }; - }); - } - } - - if let Some(next_track_id) = self.next_track_id { - if !self - .track_states - .get(self.active_track_id as usize) - .unwrap() - .running() - || self.active_track_ids().len() == 1 - { - self.active_track_id = next_track_id; - self.next_track_id = None; - self.make_init_seq(None).await.unwrap(); - self.inner - .aquire_mut() - .set_active_track_id(self.active_track_id); - self.inner.aquire_mut().set_next_track_id(None); - tracing::trace!("switched to track: {}", self.active_track_id); - } - } - - for tid in self.active_track_ids().union(&self.next_track_ids()) { - match self.track_states.get_mut(*tid as usize).unwrap().run() { - Ok(Some(result)) => match result { - TrackResult::Init { moov } => { - self.moov_map.insert(*tid, moov); - if let Err(err) = self.make_init_seq(Some(*tid)).await { - tracing::error!("failed to make init seq: {:?}", err); - self.inner.aquire().send_error(err.into())(); - break 'running; - } - } - TrackResult::Media { - fragments, - start_time, - end_time, - } => { - if let Err(err) = self - .handle_fragments(*tid, fragments, start_time, end_time) - .await - { - tracing::error!("failed to handle media: {:?}", err); - self.inner.aquire().send_error(err.into())(); - break 'running; - } - } - }, - Ok(None) => {} - Err(err) => { - tracing::error!("failed to run track: {:?}", err); - self.inner.aquire().send_error(err.into())(); - break 'running; - } - } - } - - let mut loop_timer = pin!(TimeoutFuture::new(0)); - loop { - select! { - _ = &mut loop_timer => { - break; - } - _ = self.shutdown_recv.recv() => { - break 'running; - } - evt = self.evt_recv.recv() => { - tracing::info!("got event: {:?}", evt); - } - } - } - } - - tracing::info!("playback stopped"); - } - - async fn handle_fragments( - &mut self, - tid: u32, - fragments: Vec, - start_time: f64, - end_time: f64, - ) -> Result<(), JsValue> { - if self.next_track_id == Some(tid) { - // We have the next track data from start_time so we can stop using the old track - self.active_track_ids() - .difference(&self.next_track_ids()) - .for_each(|tid| { - let track = self.track_states.get_mut(*tid as usize).unwrap(); - if track.stop_at().is_none() { - track.set_stop_at(Some(start_time)); - } - }); - } - - tracing::trace!( - "tid: {} start_time: {} end_time: {}", - tid, - start_time, - end_time - ); - - // If the track is not active we are going to buffer the fragments. - if !self.track_mapping.contains_key(&tid) { - return Err(JsValue::from_str(&format!("track: {} is not active", tid))); - } - - if matches!(self.track_mapping.get(&tid).unwrap(), TrackMapping::Buffer) { - self.fragment_buffer - .entry(tid) - .or_insert_with(Vec::new) - .extend(fragments); - return Ok(()); - } - - let mut data = Vec::new(); - fragments.iter().for_each(|fragment| { - fragment.moof.mux(&mut data).unwrap(); - fragment.mdat.mux(&mut data).unwrap(); - }); - - let mut forced = false; - - match self.track_mapping.get(&tid).unwrap() { - TrackMapping::Audio => { - if self.force_mapping.remove(&TrackMapping::Audio).is_some() { - self.audio.as_mut().unwrap().remove(0.0, start_time).await?; - forced = true; - } else { - self.audio - .as_mut() - .unwrap() - .remove(0.0, start_time - 30.0) - .await?; - } - self.audio.as_mut().unwrap().append_buffer(data).await?; - - if let Some(video_factory) = &mut self.video_factory { - let mut data = Vec::new(); - fragments.iter().for_each(|fragment| { - let decode_time = fragment.moof.traf[0] - .tfdt - .as_ref() - .unwrap() - .base_media_decode_time; - let duration = fragment.moof.traf[0].duration(); - - let (moof, mdat) = video_factory.moof_mdat(decode_time, duration); - moof.mux(&mut data).unwrap(); - mdat.mux(&mut data).unwrap(); - }); - - self.video.as_mut().unwrap().append_buffer(data).await?; - } - } - TrackMapping::Video => { - if self.force_mapping.remove(&TrackMapping::Video).is_some() { - self.video.as_mut().unwrap().remove(0.0, start_time).await?; - forced = true; - } else { - self.video - .as_mut() - .unwrap() - .remove(0.0, start_time - 30.0) - .await?; - } - self.video.as_mut().unwrap().append_buffer(data).await?; - } - TrackMapping::AudioVideo => { - if self - .force_mapping - .remove(&TrackMapping::AudioVideo) - .is_some() - { - self.audiovideo - .as_mut() - .unwrap() - .remove(0.0, start_time) - .await?; - forced = true; - } else { - self.audiovideo - .as_mut() - .unwrap() - .remove(0.0, start_time - 30.0) - .await?; - } - self.audiovideo - .as_mut() - .unwrap() - .append_buffer(data) - .await?; - } - TrackMapping::Buffer => unreachable!(), - } - - if forced { - let current_time = self.inner.aquire().video_element().unwrap().current_time(); - - if current_time > start_time && current_time < end_time - 0.1 { - // Slight hack to push the video forward and prevent it from getting stuck - self.video_element.set_current_time(current_time + 0.1); - } else { - self.inner - .aquire() - .video_element() - .unwrap() - .set_current_time(start_time); - } - } - - self.autoplay().await; - - Ok(()) - } - - async fn fetch_playlist(&mut self) -> Result { - let Ok(input_url) = Url::parse(self.inner.aquire().url()) else { - return Err(JsValue::from_str(&format!("failed to parse url: {}", self.inner.aquire().url()))); - }; - - let req = FetchRequest::new("GET", input_url.as_str()) - .header("Accept", "application/vnd.apple.mpegurl") - .set_timeout(2000) - .start()?; - - let data = select! { - r = req.wait_result() => { - r? - } - _ = self.shutdown_recv.recv() => { - return Ok(false); - } - }; - - let playlist = match hls::Playlist::try_from(data.as_slice()) { - Ok(playlist) => playlist, - Err(err) => return Err(JsValue::from_str(&err)), - }; - - // We now need to determine what kind of playlist we have, if we have a master playlist we need to do some ABR logic to determine what variant to use - // If we have a media playlist we can just start playing it directly. - match playlist { - hls::Playlist::Master(playlist) => self.handle_master_playlist(input_url, playlist)?, - hls::Playlist::Media(playlist) => self.handle_media_playlist(input_url, playlist)?, - } - - Ok(true) - } - - async fn make_init_seq(&mut self, for_tid: Option) -> Result<(), JsValue> { - let active_tracks = self.active_track_ids(); - if active_tracks.len() > 2 { - return Err(JsValue::from_str( - "too many active tracks, currently only 2 are supported", - )); - } - - let next_tracks = self.next_track_ids(); - if next_tracks.len() > 2 { - return Err(JsValue::from_str( - "too many next tracks, currently only 2 are supported", - )); - } - - tracing::trace!( - "active_tracks: {:?} next_tracks: {:?}, for_tid: {:?}", - active_tracks, - next_tracks, - for_tid - ); - - let diff = next_tracks - .difference(&active_tracks) - .collect::>(); - - for (tid, moov) in self - .moov_map - .clone() - .iter() - .filter(|(tid, _)| active_tracks.contains(tid) || next_tracks.contains(tid)) - { - if let Some(for_tid) = for_tid { - if *tid != for_tid { - continue; - } - } - - if diff.contains(tid) { - self.track_mapping.insert(*tid, TrackMapping::Buffer); - continue; - } - - let track = self.track_states.get(*tid as usize).unwrap().track(); - - let (sb, mapping) = if moov.traks.is_empty() { - return Err(JsValue::from_str("no tracks in moov")); - } else if moov.traks.len() == 1 - && (!track.referenced_group_ids.is_empty() || track.reference.is_some()) - { - if self.audiovideo.is_some() { - return Err(JsValue::from_str("audiovideo track already exists")); - } - - let trak = moov.traks.get(0).unwrap(); - let codecs = trak.mdia.minf.stbl.stsd.get_codecs().collect::>(); - if trak.mdia.minf.stbl.stsd.is_audio() { - // We have an audio track - let codec = format!("audio/mp4; codecs=\"{}\"", &codecs.join(",")); - if self.audio.is_none() { - self.audio = Some(SourceBufferHolder::new(&self.media_source, &codec)?); - self.video = Some(SourceBufferHolder::new( - &self.media_source, - "video/mp4; codecs=\"avc1.4d002a\"", - )?); - } - - if self.active_track_ids().len() == 1 { - let video_factory = VideoFactory::new(trak.mdia.mdhd.timescale); - - let codecs = video_factory.moov().traks[0] - .mdia - .minf - .stbl - .stsd - .get_codecs() - .collect::>(); - - let video = self.video.as_mut().unwrap(); - video - .change_type(&format!("video/mp4; codecs=\"{}\"", codecs.join(",")))?; - - self.video_factory = Some(video_factory); - } else { - self.video_factory = None; - } - - let audio = self.audio.as_mut().unwrap(); - audio.change_type(&codec)?; - (audio, TrackMapping::Audio) - } else if trak.mdia.minf.stbl.stsd.is_video() { - // We have a video track - let codec = format!("video/mp4; codecs=\"{}\"", &codecs.join(",")); - if self.video.is_none() { - self.video = Some(SourceBufferHolder::new(&self.media_source, &codec)?); - self.audio = Some(SourceBufferHolder::new( - &self.media_source, - "audio/mp4; codecs=\"mp4a.40.2\"", - )?); - } - - if self.active_track_ids().len() == 1 { - return Err(JsValue::from_str( - "video track must be paired with audio track", - )); - } else { - self.video_factory = None; - } - - let video = self.video.as_mut().unwrap(); - video.change_type(&codec)?; - (video, TrackMapping::Video) - } else { - return Err(JsValue::from_str("unsupported track type")); - } - } else { - if self.video.is_some() || self.audio.is_some() { - return Err(JsValue::from_str("audio or video track already exists")); - } - - self.video_factory = None; - - // We have both audio and video tracks - let audio_trak = moov - .traks - .iter() - .find(|trak| trak.mdia.minf.stbl.stsd.is_audio()); - let video_trak = moov - .traks - .iter() - .find(|trak| trak.mdia.minf.stbl.stsd.is_video()); - - if audio_trak.is_none() && video_trak.is_none() { - return Err(JsValue::from_str("missing audio and video track")); - } - - let mut codecs = Vec::new(); - - if let Some(audio_trak) = audio_trak { - let audio_codecs = audio_trak.mdia.minf.stbl.stsd.get_codecs(); - codecs.extend(audio_codecs); - } - - if let Some(video_trak) = video_trak { - let video_codecs = video_trak.mdia.minf.stbl.stsd.get_codecs(); - codecs.extend(video_codecs); - } - - let codec = format!("video/mp4; codecs=\"{}\"", &codecs.join(",")); - - if self.audiovideo.is_none() { - self.audiovideo = Some(SourceBufferHolder::new(&self.media_source, &codec)?); - } - - let audiovideo = self.audiovideo.as_mut().unwrap(); - audiovideo.change_type(&codec)?; - (audiovideo, TrackMapping::AudioVideo) - }; - - // Construct a moov segment - let mut data = Vec::new(); - Ftyp::new(FourCC::Iso5, 512, vec![FourCC::Iso5, FourCC::Iso6]) - .mux(&mut data) - .unwrap(); - moov.mux(&mut data).unwrap(); - - sb.append_buffer(data).await?; - - if let Some(video_factory) = &self.video_factory { - let mut data = Vec::new(); - Ftyp::new(FourCC::Iso5, 512, vec![FourCC::Iso5, FourCC::Iso6]) - .mux(&mut data) - .unwrap(); - video_factory.moov().mux(&mut data).unwrap(); - - self.video.as_mut().unwrap().append_buffer(data).await?; - } - - if matches!( - self.track_mapping.insert(*tid, mapping), - Some(TrackMapping::Buffer) - ) { - let fragments = self.fragment_buffer.remove(tid).unwrap_or_default(); - let start_time = fragments.first().map(|f| f.start_time).unwrap_or_default(); - let end_time = fragments.last().map(|f| f.end_time).unwrap_or_default(); - - self.handle_fragments(*tid, fragments, start_time, end_time) - .await?; - } - } - - Ok(()) - } - - async fn autoplay(&mut self) { - if self.init { - return; - } - - let fut = { - let inner = self.inner.aquire(); - let element = inner.video_element().unwrap(); - let Ok(start) = element.buffered().start(0) else { - return; - }; - - self.init = true; - - element.set_current_time(start); - element.play().map(JsFuture::from) - }; - - if let Ok(fut) = fut { - fut.await.ok(); - } - } - - async fn bind_element(&mut self) -> Result { - let url = web_sys::Url::create_object_url_with_source(&self.media_source)?; - - self.video_element.set_src(&url); - - let mut result = Ok(true); - - let mut global_evt = self.shutdown_recv.resubscribe(); - - 'l: loop { - select! { - _ = global_evt.recv() => { - result = Ok(false); - break 'l; - } - evt = self.evt_recv.recv() => { - match evt { - Some((RunnerEvent::MediaSourceOpen, _)) => { - break 'l; - } - Some((RunnerEvent::MediaSourceClose, _)) => { - result = Err(JsValue::from_str("media source closed")); - break 'l; - } - Some((RunnerEvent::MediaSourceEnded, _)) => { - result = Err(JsValue::from_str("media source ended")); - break 'l; - } - None => unreachable!(), - _ => {} - } - } - } - } - - web_sys::Url::revoke_object_url(&url)?; - - result - } - - fn set_low_latency(&mut self, force: bool) { - let low_latency = self.inner.aquire().low_latency(); - if self.low_latency != low_latency || force { - self.low_latency = low_latency; - self.track_states.iter_mut().for_each(|track| { - track.set_low_latency(low_latency); - }); - - let buffered = self.inner.aquire().video_element().unwrap().buffered(); - if buffered.length() != 0 { - self.inner - .aquire() - .video_element() - .unwrap() - .set_current_time( - (if low_latency { - buffered - .end(buffered.length() - 1) - .map(|t| t - 0.1) - .unwrap_or_default() - } else { - buffered - .end(buffered.length() - 1) - .map(|t| t - 2.0) - .unwrap_or_default() - }) - .max(0.0), - ) - } - } - } - - fn active_track_ids(&self) -> HashSet { - self.track_ids(self.active_track_id) - } - - fn next_track_ids(&self) -> HashSet { - self.next_track_id - .map(|id| self.track_ids(id)) - .unwrap_or_default() - } - - fn track_ids(&self, track_id: u32) -> HashSet { - let active_track = self.track_states.get(track_id as usize).unwrap(); - - let mut track_ids = active_track - .track() - .referenced_group_ids - .iter() - .map(|id| *self.active_reference_track_ids.get(*id as usize).unwrap()) - .collect::>(); - - track_ids.insert(track_id); - - track_ids - } - - fn handle_master_playlist( - &mut self, - input_url: Url, - mut playlist: MasterPlaylist, - ) -> Result<(), JsValue> { - let mut inner = self.inner.aquire_mut(); - - let mut reference_streams = HashSet::new(); - - for stream in playlist.streams.iter() { - if let Some(audio) = stream.audio.as_ref() { - reference_streams.insert(audio); - } - - if let Some(video) = stream.video.as_ref() { - reference_streams.insert(video); - } - } - - let mut m3u8_url_to_track = HashMap::new(); - - enum TrackReference { - Flat(Media), - Reference(u32), - } - - let mut reference_tracks = HashMap::new(); - let mut current_track_idx = 0; - let mut group_id = 0; - - for stream in reference_streams.into_iter() { - let Some(groups) = playlist.groups.get_mut(stream) else { - return Err(JsValue::from_str(&format!("failed to find group for stream: {}", stream))); - }; - - let pos = groups.iter().position(|item| item.default).unwrap_or(0); - groups.iter_mut().for_each(|item| item.default = false); - - let default_item = &mut groups[pos]; - if default_item.uri.is_empty() { - // This is a reference track but is not really a reference track - reference_tracks.insert(stream.clone(), TrackReference::Flat(default_item.clone())); - continue; - } - - default_item.default = true; - - // Otherwise this is actually a reference track - // So we need to generate a new track id for it. - let mut ids = HashSet::new(); - for track in groups { - let url = match Url::parse(&track.uri).or_else(|_| input_url.join(&track.uri)) { - Ok(url) => url, - Err(err) => { - return Err(JsValue::from_str(&format!("failed to parse url: {}", err))); - } - }; - - let track_id = m3u8_url_to_track - .entry(url.clone()) - .or_insert_with(|| { - let t = Track { - id: current_track_idx, - is_variant_track: false, - playlist_url: url.clone(), - referenced_group_ids: Vec::new(), - name: Some(track.name.clone()), - bandwidth: None, - codecs: None, - frame_rate: None, - height: None, - width: None, - reference: Some(ReferenceTrack { - group_id: group_id as u32, - is_default: track.default, - }), - }; - - current_track_idx += 1; - - t - }) - .id; - - if track.default { - self.active_reference_track_ids.push(track_id); - } - - ids.insert(track_id); - } - - reference_tracks.insert(stream.clone(), TrackReference::Reference(group_id as u32)); - group_id += 1; - } - - for stream in playlist.streams.iter() { - let url = match Url::parse(&stream.uri).or_else(|_| input_url.join(&stream.uri)) { - Ok(url) => url, - Err(err) => { - return Err(JsValue::from_str(&format!("failed to parse url: {}", err))); - } - }; - - let track = m3u8_url_to_track.entry(url.clone()).or_insert_with(|| { - let t = Track { - id: current_track_idx, - is_variant_track: true, - playlist_url: url.clone(), - referenced_group_ids: Vec::new(), - name: None, - reference: None, - bandwidth: Some(stream.bandwidth), - codecs: stream.codecs.clone(), - frame_rate: stream.frame_rate, - width: stream.resolution.map(|r| r.0), - height: stream.resolution.map(|r| r.1), - }; - - current_track_idx += 1; - - t - }); - - track.bandwidth = Some(stream.bandwidth); - track.codecs = stream.codecs.clone(); - track.frame_rate = stream.frame_rate; - track.width = stream.resolution.map(|r| r.0); - track.height = stream.resolution.map(|r| r.1); - track.is_variant_track = true; - - if let Some(audio) = stream.audio.as_ref() { - match reference_tracks.get(audio) { - Some(TrackReference::Flat(media)) => { - track.name = Some(media.name.clone()); - } - Some(TrackReference::Reference(group_id)) => { - if track.reference.as_ref().map(|t| t.group_id) != Some(*group_id) { - track.referenced_group_ids.push(*group_id); - } - } - None => { - return Err(JsValue::from_str(&format!( - "failed to find reference track for audio: {}", - audio - ))); - } - } - } - - if let Some(video) = stream.video.as_ref() { - match reference_tracks.get(video) { - Some(TrackReference::Flat(media)) => { - track.name = Some(media.name.clone()); - } - Some(TrackReference::Reference(group_id)) => { - if track.reference.as_ref().map(|t| t.group_id) != Some(*group_id) { - track.referenced_group_ids.push(*group_id); - } - } - None => { - return Err(JsValue::from_str(&format!( - "failed to find reference track for video: {}", - video - ))); - } - } - } - } - - let mut tracks = m3u8_url_to_track.into_values().collect::>(); - tracks.sort_by(|a, b| a.id.cmp(&b.id)); - - self.track_states = tracks.clone().into_iter().map(TrackState::new).collect(); - - let fire_event = inner.set_tracks(tracks, true); - inner.set_active_reference_track_ids(self.active_reference_track_ids.clone()); - inner.set_active_track_id(0); - - drop(inner); - - fire_event(); - - Ok(()) - } - - fn handle_media_playlist( - &mut self, - input_url: Url, - playlist: MediaPlaylist, - ) -> Result<(), JsValue> { - let mut inner = self.inner.aquire_mut(); - - let track = Track { - id: 0, - bandwidth: None, - is_variant_track: true, - name: None, - playlist_url: input_url, - referenced_group_ids: Vec::new(), - reference: None, - codecs: None, - frame_rate: None, - height: None, - width: None, - }; - - let mut track_state = TrackState::new(track.clone()); - track_state.set_playlist(playlist); - - self.track_states = vec![track_state]; - - let fire_event = inner.set_tracks(vec![track], false); - inner.set_active_track_id(0); - - drop(inner); - fire_event(); - - Ok(()) - } -} diff --git a/frontend/player/src/player/runner/events.rs b/frontend/player/src/player/runner/events.rs new file mode 100644 index 00000000..3b559d08 --- /dev/null +++ b/frontend/player/src/player/runner/events.rs @@ -0,0 +1,18 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunnerEvent { + VideoError, + VideoPlay, + VideoPause, + VideoSuspend, + VideoStalled, + VideoWaiting, + VideoSeeking, + VideoSeeked, + VideoTimeUpdate, + VideoVolumeChange, + VideoRateChange, + MediaSourceOpen, + MediaSourceClose, + MediaSourceEnded, + DocumentVisibilityChange, +} diff --git a/frontend/player/src/player/runner/mod.rs b/frontend/player/src/player/runner/mod.rs new file mode 100644 index 00000000..a8a663f7 --- /dev/null +++ b/frontend/player/src/player/runner/mod.rs @@ -0,0 +1,1200 @@ +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + pin::pin, +}; + +use crate::{ + hls::{self, master::MasterPlaylist, media::MediaPlaylist}, + player::{ + fetch::FetchRequest, + inner::NextVariant, + runner::source_buffer::SourceBufferHolder, + track::{Fragment, TrackResult}, + util::now, + }, +}; + +use gloo_timers::future::TimeoutFuture; +use mp4::{ + types::{ + ftyp::{FourCC, Ftyp}, + moov::Moov, + }, + BoxType, +}; +use tokio::{ + select, + sync::{broadcast, mpsc}, +}; +use url::Url; +use wasm_bindgen::JsValue; +use wasm_bindgen_futures::JsFuture; +use web_sys::{Document, HtmlVideoElement, MediaSource}; + +mod events; +mod source_buffer; +mod util; + +use self::{ + events::RunnerEvent, + source_buffer::SourceBuffers, + util::{make_document_holder, make_media_source_holder, make_video_holder}, +}; + +use super::{ + bandwidth::Bandwidth, + blank::VideoFactory, + inner::PlayerInnerHolder, + track::{Track, TrackState, Variant}, + util::Holder, +}; + +pub struct PlayerRunner { + inner: PlayerInnerHolder, + + /// Track states (offset by track id) + track_states: Vec, + /// Variants (offset by variant id) + variants: Vec, + + active_variant_id: u32, + next_variant_id: Option, + + /// Group Id -> Track Id + active_group_track_ids: Vec, + + /// Track Id -> Vec + /// This is used to buffer fragments for tracks that are not active + fragment_buffer: HashMap>, + + /// Track Id -> Moov + /// This is a map of the last moov we got for a track, this is used to generate init segments + moov_map: HashMap, + + /// If we have initialized the player + init: bool, + + /// Used to shutdown all the tasks + shutdown_recv: broadcast::Receiver<()>, + + /// If we are in low latency mode + low_latency: bool, + + /// The source buffers + source_buffers: SourceBuffers, + + /// The media source + media_source: Holder, + + /// The video element + video_element: Holder, + + /// The dom document + document: Holder, + + /// Video factory, used to generate black video frames + video_factory: Option, + + /// Last iteration time + last_iteration: f64, + /// Last time we switched variants due to ABR + last_abr_switch: f64, + + /// The moment the document was hidden and what variant was active at that time (since when we switch to the background we switch to audio only) + document_hidden_at: Option, + // The variant id that was active when we switched to audio only mode. + document_hidden_variant_id: Option, + + /// The bandwidth estimator (used for ABR) + bandwidth: Bandwidth, + + /// The event receiver + evt_recv: mpsc::Receiver<(RunnerEvent, web_sys::Event)>, +} + +impl PlayerRunner { + pub fn new(inner: PlayerInnerHolder, shutdown_recv: broadcast::Receiver<()>) -> Self { + let ms = MediaSource::new().unwrap(); + + let (tx, rx) = mpsc::channel(128); + + let video_element = make_video_holder(inner.video_element().unwrap(), &tx); + let media_source = make_media_source_holder(ms, &tx); + let document = make_document_holder(&tx); + + Self { + track_states: Vec::new(), + shutdown_recv, + active_variant_id: 0, + next_variant_id: None, + moov_map: HashMap::new(), + init: false, + fragment_buffer: HashMap::new(), + variants: Vec::new(), + active_group_track_ids: Vec::new(), + low_latency: false, + source_buffers: SourceBuffers::None, + media_source, + video_element, + document_hidden_at: None, + document_hidden_variant_id: None, + document, + video_factory: None, + evt_recv: rx, + last_iteration: 0.0, + last_abr_switch: 0.0, + bandwidth: Bandwidth::new(), + inner, + } + } + + pub async fn start(mut self) { + match self.bind_element().await { + Err(err) => { + tracing::error!("failed to bind element: {:?}", err); + self.inner.send_error(err.into()); + return; + } + Ok(true) => {} + Ok(false) => return, + } + + match self.fetch_playlist().await { + Err(err) => { + tracing::error!("failed to handle playlist: {:?}", err); + self.inner.send_error(err.into()); + return; + } + Ok(true) => {} + Ok(false) => return, + } + + self.active_variant_id = self.inner.active_variant_id(); + + tracing::info!("starting playback"); + + for tid in self.active_track_ids() { + self.track_states.get_mut(tid as usize).unwrap().start(); + } + + self.last_iteration = now(); + + 'running: loop { + self.set_low_latency(!self.init); + + if self.document.hidden() && self.document_hidden_at.is_none() { + self.document_hidden_at = Some(now()); + } else if !self.document.hidden() && self.document_hidden_variant_id.is_some() { + self.document_hidden_at = None; + let mut variant_id = self.document_hidden_variant_id.take().unwrap(); + if self.active_variant_id != variant_id { + if self.inner.abr_enabled() { + variant_id = self.abr_variant_id().unwrap_or(variant_id); + } + + self.inner + .set_next_variant_id(Some(NextVariant::Force(variant_id))); + } + } else if self.document.hidden() + && self.document_hidden_at.is_some() + && self.document_hidden_variant_id.is_none() + { + if let Some(document_hidden_at) = self.document_hidden_at { + let audio_variant = self + .variants + .iter() + .find(|v| v.video_track.is_none()) + .unwrap() + .id; + + if now() - document_hidden_at > 5000.0 + && self.next_variant_id.is_none() + && self.active_variant_id != audio_variant + { + self.document_hidden_variant_id = Some(self.active_variant_id); + self.inner + .set_next_variant_id(Some(NextVariant::Switch(audio_variant))) + } + } + } + + let current_time = now(); + let delta = current_time - self.last_iteration; + self.last_iteration = current_time; + // If the delta was bigger than 500ms we need to seek forward because there was likely a player stall (if we were playing) + + if self.document_hidden_at.is_none() { + self.inner.video_element().and_then(|el| { + if el.paused() || !self.low_latency { + return None; + } + + let buffered_range = el.buffered(); + let length = buffered_range.length(); + if length == 0 { + return None; + } + + let end = buffered_range.end(length - 1).unwrap(); + + let delta = end - el.current_time(); + if delta < 1.5 { + return None; + } + + el.set_current_time(end - 0.5); + + Some(()) + }); + } + + // If we took more then 5s to loop we need to refresh the playlists before requesting fragments. + if delta > 2000.0 { + self.active_track_ids() + .union(&self.next_track_ids()) + .collect::>() + .into_iter() + .for_each(|tid| { + self.track_states.get_mut(*tid as usize).unwrap().stop(); + }); + + let nvid = self.inner.next_variant_id(); + if let Some(nvid) = nvid { + self.inner.set_next_variant_id(None); + self.inner.set_active_variant_id(nvid.variant_id()); + self.active_variant_id = nvid.variant_id(); + } + + self.active_track_ids().into_iter().for_each(|tid| { + self.track_states.get_mut(tid as usize).unwrap().start(); + }); + + tracing::info!("refreshing playlists"); + } + + if self.inner.abr_enabled() + && self.inner.next_variant_id().is_none() + && self.document_hidden_at.is_none() + { + if let Some(variant_id) = self.abr_variant_id() { + if variant_id != self.active_variant_id && self.last_abr_switch + 5000.0 < now() + { + self.last_abr_switch = now(); + self.inner + .set_next_variant_id(Some(NextVariant::Switch(variant_id))); + } + } + } + + if let Some(next_variant_id) = self.inner.next_variant_id() { + if Some(next_variant_id.variant_id()) != self.next_variant_id { + if self.next_variant_id.is_some() { + self.active_track_ids() + .difference(&self.next_track_ids()) + .for_each(|tid| { + self.track_states + .get_mut(*tid as usize) + .unwrap() + .set_stop_at(None); + tracing::trace!("resuming track: {}", tid); + }); + self.next_track_ids() + .difference(&self.active_track_ids()) + .for_each(|tid| { + self.track_states.get_mut(*tid as usize).unwrap().stop(); + tracing::trace!("stopped track: {}", tid); + }); + } + + self.fragment_buffer.clear(); + + if next_variant_id.variant_id() != self.active_variant_id { + self.next_variant_id = Some(next_variant_id.variant_id()); + self.next_track_ids() + .difference(&self.active_track_ids()) + .for_each(|tid| { + tracing::trace!("starting track: {}", tid); + let new_track_url = + self.track_states.get(*tid as usize).unwrap().url(); + if self.low_latency && !next_variant_id.is_force() { + if let Some(id) = self.active_track_ids().iter().next() { + if let Some(report) = self + .track_states + .get(*id as usize) + .unwrap() + .rendition_reports(new_track_url) + { + tracing::info!("rendition report: {:?}", report); + self.track_states + .get_mut(*tid as usize) + .unwrap() + .start_at( + report.last_msn + + if report.last_part == 0 { 0 } else { 1 }, + ); + return; + } + } + } + + self.track_states.get_mut(*tid as usize).unwrap().start(); + }); + } else { + self.next_variant_id = None; + self.inner.set_next_variant_id(None); + self.inner.set_active_variant_id(self.active_variant_id); + } + + // Audio only case + if self.next_track_ids().len() == 1 { + self.active_track_ids() + .difference(&self.next_track_ids()) + .for_each(|t| self.track_states[*t as usize].stop()); + } + + if next_variant_id.is_force() && self.next_variant_id.is_some() { + self.active_track_ids() + .difference(&self.next_track_ids()) + .for_each(|tid| { + self.track_states.get_mut(*tid as usize).unwrap().stop(); + tracing::trace!("stopping track: {}", tid); + }); + } + } + } + + if let Some(next_variant_id) = self.next_variant_id { + let next_variant = self.variants.get(next_variant_id as usize).unwrap(); + let current_variant = self.variants.get(self.active_variant_id as usize).unwrap(); + + let next_video_track_id = next_variant + .video_track + .map(|id| self.active_group_track_ids[id as usize] as usize); + + let current_video_track_id = current_variant + .video_track + .map(|id| self.active_group_track_ids[id as usize] as usize); + + if next_video_track_id != current_video_track_id + && !current_video_track_id + .map(|id| self.track_states[id].running()) + .unwrap_or_else(|| self.video_factory.is_some()) + { + self.active_track_ids() + .difference(&self.next_track_ids()) + .for_each(|tid| { + self.track_states.get_mut(*tid as usize).unwrap().stop(); + tracing::trace!("stopping track: {}", tid); + }); + + self.active_variant_id = next_variant_id; + self.next_variant_id = None; + self.make_init_seq(None).await.unwrap(); + self.inner.set_active_variant_id(self.active_variant_id); + self.inner.set_next_variant_id(None); + + tracing::trace!("switched to variant: {}", self.active_variant_id); + } + } + + for tid in self.active_track_ids().union(&self.next_track_ids()) { + match self.track_states.get_mut(*tid as usize).unwrap().run() { + Ok(Some(result)) => match result { + TrackResult::Init { moov } => { + self.moov_map.insert(*tid, moov); + if let Err(err) = self.make_init_seq(Some(*tid)).await { + tracing::error!("failed to make init seq: {:?}", err); + self.inner.send_error(err.into()); + break 'running; + } + } + TrackResult::Media { + fragments, + start_time, + end_time, + } => { + if let Err(err) = self + .handle_fragments(*tid, fragments, start_time, end_time) + .await + { + tracing::error!("failed to handle media: {:?}", err); + self.inner.send_error(err.into()); + break 'running; + } + } + }, + Ok(None) => {} + Err(err) => { + tracing::error!("failed to run track: {:?}", err); + self.inner.send_error(err.into()); + break 'running; + } + } + } + + let mut loop_timer = pin!(TimeoutFuture::new(0)); + loop { + select! { + _ = &mut loop_timer => { + break; + } + _ = self.shutdown_recv.recv() => { + break 'running; + } + evt = self.evt_recv.recv() => { + let (evt, js_evt) = evt.unwrap(); + match evt { + RunnerEvent::VideoError => { + tracing::error!("video error: {:?}", js_evt); + self.inner.send_error(JsValue::from(js_evt).into()); + break 'running; + } + RunnerEvent::DocumentVisibilityChange => { + if self.document.hidden() { + self.document_hidden_at = Some(now()); + } else if self.document_hidden_at.is_some() { + self.document_hidden_at = None; + if let Some(mut variant_id) = self.document_hidden_variant_id.take() { + if self.active_variant_id != variant_id { + if self.inner.abr_enabled() { + variant_id = self.abr_variant_id().unwrap_or(variant_id); + } + + self.inner.set_next_variant_id(Some(NextVariant::Force(variant_id))); + } + } + } + } + _ => {} + } + } + } + } + } + + tracing::info!("playback stopped"); + } + + fn abr_variant_id(&self) -> Option { + let bandwidth = self.bandwidth.get()? * 8; + + // We have some bandwidth estimation, so we should try do some ABR to get the best quality + let best_variant = self.variants.iter().find(|v| { + let bandwidth = match self.active_variant_id.cmp(&v.id) { + // Discourage switching to a higher quality unless we can get a 25% increase in bandwidth + Ordering::Greater => v.bandwidth as f64 * 1.5 <= bandwidth as f64, + // Incourage staying on the same quality unless our bandwidth is 5% lower then the current quality + Ordering::Equal => v.bandwidth as f64 * 0.95 <= bandwidth as f64, + // Incourage switching to a lower quality. + Ordering::Less => v.bandwidth as f64 <= bandwidth as f64, + }; + + bandwidth && v.audio_track.is_some() && v.video_track.is_some() + })?; + + Some(best_variant.id) + } + + async fn handle_fragments( + &mut self, + tid: u32, + fragments: Vec, + start_time: f64, + end_time: f64, + ) -> Result<(), JsValue> { + if !self.active_track_ids().contains(&tid) && self.next_track_ids().contains(&tid) { + tracing::debug!("Stopping old tracks"); + // We have the next track data from start_time so we can stop using the old track + self.active_track_ids() + .difference(&self.next_track_ids()) + .for_each(|tid| { + let track = self.track_states.get_mut(*tid as usize).unwrap(); + if track.stop_at().is_none() { + track.set_stop_at(Some(start_time)); + } + }); + + if let Some(video_factory) = &mut self.video_factory { + video_factory.set_stop_at(Some(start_time)); + } + } + + tracing::trace!( + "tid: {} start_time: {} end_time: {}", + tid, + start_time, + end_time + ); + + if !self.active_track_ids().contains(&tid) && self.next_track_ids().contains(&tid) { + tracing::debug!( + "Buffering fragments for track: {} ({} - {})", + tid, + start_time, + end_time + ); + self.fragment_buffer + .entry(tid) + .or_insert_with(Vec::new) + .extend(fragments); + return Ok(()); + } else if !self.active_track_ids().contains(&tid) { + tracing::warn!( + "Got fragments for track that is not active or next: {}", + tid + ); + return Ok(()); + } + + let mut data = Vec::new(); + fragments.iter().for_each(|fragment| { + fragment.moof.mux(&mut data).unwrap(); + fragment.mdat.mux(&mut data).unwrap(); + }); + + let buffer = match &mut self.source_buffers { + SourceBuffers::AudioVideoCombined(av) => av, + SourceBuffers::AudioVideoSplit { audio, video } => { + let variant = self.variants.get(self.active_variant_id as usize).unwrap(); + if let Some(group_id) = variant.audio_track { + if self.active_group_track_ids[group_id as usize] == tid { + audio + } else { + video + } + } else { + video + } + } + SourceBuffers::None => { + return Err(JsValue::from_str("no source buffers")); + } + }; + + let current_time = self.inner.video_element().unwrap().current_time(); + let buffered_ranges = buffer.buffered().unwrap(); + let duration = if buffered_ranges.length() != 0 { + buffered_ranges.end(buffered_ranges.length() - 1).unwrap() + } else { + 0.0 + }; + + // We have already buffered some data in this region so we must remove it. + buffer.remove(0.0, current_time - 30.0).await?; + buffer.remove(start_time, duration).await?; + buffer.append_buffer(data).await?; + + // If we have a video factory we must be in split mode and also we need to buffer black frames + if let Some(video_factory) = &mut self.video_factory { + let mut data = Vec::new(); + tracing::trace!( + "Generating black frames for track: {} ({} - {})", + tid, + start_time, + end_time + ); + + fragments.iter().for_each(|fragment| { + let decode_time = fragment.moof.traf[0] + .tfdt + .as_ref() + .unwrap() + .base_media_decode_time; + let duration = fragment.moof.traf[0].duration(); + + let (moof, mdat) = video_factory.moof_mdat(decode_time, duration); + moof.mux(&mut data).unwrap(); + mdat.mux(&mut data).unwrap(); + }); + + self.source_buffers + .video() + .unwrap() + .append_buffer(data) + .await?; + + if let Some(stop_at) = video_factory.stop_at() { + if end_time >= stop_at { + tracing::info!("stopping video factory"); + self.video_factory = None; + } + } + } + + TimeoutFuture::new(0).await; + + let buffered_ranges = self.inner.video_element().unwrap().buffered(); + if buffered_ranges.length() != 0 { + let current_time = self.inner.video_element().unwrap().current_time(); + let last_range = ( + buffered_ranges.start(buffered_ranges.length() - 1).unwrap(), + buffered_ranges.end(buffered_ranges.length() - 1).unwrap(), + ); + if last_range.0 > current_time { + tracing::info!( + "seeking ahead to last buffered range: {} - {:?}", + current_time, + last_range + ); + self.inner + .video_element() + .unwrap() + .set_current_time(last_range.0); + } else if last_range.1 < current_time { + tracing::info!( + "seeking back to last buffered range: {} - {:?}", + current_time, + last_range + ); + self.inner + .video_element() + .unwrap() + .set_current_time(last_range.1); + } + } + + self.autoplay().await; + + Ok(()) + } + + async fn fetch_playlist(&mut self) -> Result { + let Ok(input_url) = Url::parse(&self.inner.url()) else { + return Err(JsValue::from_str(&format!("failed to parse url: {}", self.inner.url()))); + }; + + let mut req = FetchRequest::new("GET", input_url.clone()) + .header("Accept", "application/vnd.apple.mpegurl") + .set_timeout(2000) + .start()?; + + let data = select! { + r = req.wait_result() => { + r? + } + _ = self.shutdown_recv.recv() => { + return Ok(false); + } + }; + + let playlist = match hls::Playlist::try_from(data.as_slice()) { + Ok(playlist) => playlist, + Err(err) => return Err(JsValue::from_str(&err)), + }; + + // We now need to determine what kind of playlist we have, if we have a master playlist we need to do some ABR logic to determine what variant to use + // If we have a media playlist we can just start playing it directly. + match playlist { + hls::Playlist::Master(playlist) => self.handle_master_playlist(input_url, playlist)?, + hls::Playlist::Media(playlist) => self.handle_media_playlist(input_url, playlist)?, + } + + Ok(true) + } + + async fn make_init_seq(&mut self, for_tid: Option) -> Result<(), JsValue> { + let active_tracks = self.active_track_ids(); + if active_tracks.len() > 2 { + return Err(JsValue::from_str( + "too many active tracks, currently only 2 are supported", + )); + } + + let next_tracks = self.next_track_ids(); + if next_tracks.len() > 2 { + return Err(JsValue::from_str( + "too many next tracks, currently only 2 are supported", + )); + } + + tracing::trace!( + "active_tracks: {:?} next_tracks: {:?}, for_tid: {:?}", + active_tracks, + next_tracks, + for_tid + ); + + let diff = next_tracks + .difference(&active_tracks) + .collect::>(); + + for (tid, moov) in self + .moov_map + .clone() + .iter() + .filter(|(tid, _)| active_tracks.contains(tid) || next_tracks.contains(tid)) + { + if let Some(for_tid) = for_tid { + if *tid != for_tid { + continue; + } + } + + if diff.contains(tid) { + continue; + } + + let sb = if moov.traks.is_empty() { + return Err(JsValue::from_str("no tracks in moov")); + } else if moov.traks.len() == 1 { + if self.source_buffers.audiovideo().is_some() { + return Err(JsValue::from_str("audiovideo track already exists")); + } + + let trak = moov.traks.get(0).unwrap(); + let codecs = trak.mdia.minf.stbl.stsd.get_codecs().collect::>(); + if trak.mdia.minf.stbl.stsd.is_audio() { + // We have an audio track + let codec = format!("audio/mp4; codecs=\"{}\"", &codecs.join(",")); + if self.source_buffers.audio().is_none() { + self.source_buffers = SourceBuffers::AudioVideoSplit { + audio: SourceBufferHolder::new(&self.media_source, &codec)?, + video: SourceBufferHolder::new( + &self.media_source, + "video/mp4; codecs=\"avc1.4d002a\"", // This is a generic codec we will change it later when we have more information about the video track + )?, + }; + } + + // If we only have 1 track but we are using a split source buffer we need to make a dummy video track + // We use video factory to generate dummy frames. + if self.active_track_ids().len() == 1 { + let video_factory = VideoFactory::new(trak.mdia.mdhd.timescale); + + let codecs = video_factory.moov().traks[0] + .mdia + .minf + .stbl + .stsd + .get_codecs() + .collect::>(); + + self.source_buffers + .video() + .unwrap() + .change_type(&format!("video/mp4; codecs=\"{}\"", codecs.join(",")))?; + + tracing::info!("using video factory to generate dummy video track"); + + self.video_factory = Some(video_factory); + } + + let audio = self.source_buffers.audio().unwrap(); + audio.change_type(&codec)?; + audio + } else if trak.mdia.minf.stbl.stsd.is_video() { + // We have a video track + let codec = format!("video/mp4; codecs=\"{}\"", &codecs.join(",")); + if self.source_buffers.video().is_none() { + self.source_buffers = SourceBuffers::AudioVideoSplit { + audio: SourceBufferHolder::new( + &self.media_source, + "audio/mp4; codecs=\"mp4a.40.2\"", + )?, + video: SourceBufferHolder::new(&self.media_source, &codec)?, + }; + } + + if self.active_track_ids().len() == 1 { + return Err(JsValue::from_str( + "video track must be paired with audio track", + )); + } + + // Since we have a real video track we don't need the video factory anymore + self.video_factory = None; + + let video = self.source_buffers.video().unwrap(); + video.change_type(&codec)?; + video + } else { + return Err(JsValue::from_str("unsupported track type")); + } + } else { + if self.source_buffers.audio().is_some() || self.source_buffers.video().is_some() { + return Err(JsValue::from_str("audio or video track already exists")); + } + + self.video_factory = None; + + // We have both audio and video tracks + let audio_trak = moov + .traks + .iter() + .find(|trak| trak.mdia.minf.stbl.stsd.is_audio()); + let video_trak = moov + .traks + .iter() + .find(|trak| trak.mdia.minf.stbl.stsd.is_video()); + + if audio_trak.is_none() && video_trak.is_none() { + return Err(JsValue::from_str("missing audio and video track")); + } + + let mut codecs = Vec::new(); + + if let Some(audio_trak) = audio_trak { + let audio_codecs = audio_trak.mdia.minf.stbl.stsd.get_codecs(); + codecs.extend(audio_codecs); + } + + if let Some(video_trak) = video_trak { + let video_codecs = video_trak.mdia.minf.stbl.stsd.get_codecs(); + codecs.extend(video_codecs); + } + + let codec = format!("video/mp4; codecs=\"{}\"", &codecs.join(",")); + + if self.source_buffers.audiovideo().is_none() { + self.source_buffers = SourceBuffers::AudioVideoCombined( + SourceBufferHolder::new(&self.media_source, &codec)?, + ); + } + + let audiovideo = self.source_buffers.audiovideo().unwrap(); + audiovideo.change_type(&codec)?; + audiovideo + }; + + // Construct a moov segment + let mut data = Vec::new(); + Ftyp::new(FourCC::Iso5, 512, vec![FourCC::Iso5, FourCC::Iso6]) + .mux(&mut data) + .unwrap(); + moov.mux(&mut data).unwrap(); + + sb.append_buffer(data).await?; + + // If we have a video factory we need to generate a dummy init segment + if let Some(video_factory) = &self.video_factory { + let mut data = Vec::new(); + Ftyp::new(FourCC::Iso5, 512, vec![FourCC::Iso5, FourCC::Iso6]) + .mux(&mut data) + .unwrap(); + video_factory.moov().mux(&mut data).unwrap(); + + self.source_buffers + .video() + .unwrap() + .append_buffer(data) + .await?; + } + + // If we had buffered fragments we need to handle them now + if let Some(fragments) = self.fragment_buffer.remove(tid) { + let start_time = fragments.first().map(|f| f.start_time).unwrap_or_default(); + let end_time = fragments.last().map(|f| f.end_time).unwrap_or_default(); + + tracing::debug!("Handling buffered fragments: {} - {}", start_time, end_time); + + self.handle_fragments(*tid, fragments, start_time, end_time) + .await?; + } + } + + Ok(()) + } + + async fn autoplay(&mut self) { + if self.init { + return; + } + + let element = self.inner.video_element().unwrap(); + let buffered = element.buffered(); + if buffered.length() == 0 { + return; + } + + let Ok(start) = buffered.start(buffered.length() - 1) else { + return; + }; + + self.init = true; + + element.set_current_time(start); + + if let Ok(fut) = element.play().map(JsFuture::from) { + fut.await.ok(); + } + } + + async fn bind_element(&mut self) -> Result { + let url = web_sys::Url::create_object_url_with_source(&self.media_source)?; + + self.video_element.set_src(&url); + + let mut result = Ok(true); + + let mut global_evt = self.shutdown_recv.resubscribe(); + + 'l: loop { + select! { + _ = global_evt.recv() => { + result = Ok(false); + break 'l; + } + evt = self.evt_recv.recv() => { + match evt { + Some((RunnerEvent::MediaSourceOpen, _)) => { + break 'l; + } + Some((RunnerEvent::MediaSourceClose, _)) => { + result = Err(JsValue::from_str("media source closed")); + break 'l; + } + Some((RunnerEvent::MediaSourceEnded, _)) => { + result = Err(JsValue::from_str("media source ended")); + break 'l; + } + None => unreachable!(), + _ => {} + } + } + } + } + + web_sys::Url::revoke_object_url(&url)?; + + result + } + + fn set_low_latency(&mut self, force: bool) { + let low_latency = self.inner.low_latency(); + if self.low_latency != low_latency || force { + self.low_latency = low_latency; + self.track_states.iter_mut().for_each(|track| { + track.set_low_latency(low_latency); + }); + + let buffered = self.inner.video_element().unwrap().buffered(); + if buffered.length() != 0 { + self.inner.video_element().unwrap().set_current_time( + (if low_latency { + buffered + .end(buffered.length() - 1) + .map(|t| t - 0.1) + .unwrap_or_default() + } else { + buffered + .end(buffered.length() - 1) + .map(|t| t - 2.0) + .unwrap_or_default() + }) + .max(0.0), + ) + } + } + + self.bandwidth + .set_max_count(if low_latency { 15 } else { 5 }) + } + + fn active_track_ids(&self) -> HashSet { + self.track_ids(self.active_variant_id) + } + + fn next_track_ids(&self) -> HashSet { + self.next_variant_id + .map(|id| self.track_ids(id)) + .unwrap_or_default() + } + + fn track_ids(&self, variant_id: u32) -> HashSet { + let Some(active_track) = self.variants.get(variant_id as usize) else { + return HashSet::new(); + }; + + active_track + .audio_track + .iter() + .chain(active_track.video_track.iter()) + .map(|id| self.active_group_track_ids[*id as usize]) + .collect::>() + } + + fn handle_master_playlist( + &mut self, + input_url: Url, + mut playlist: MasterPlaylist, + ) -> Result<(), JsValue> { + let mut m3u8_url_to_track = HashMap::new(); + + let mut track_idx = 0; + + let reference_streams = playlist + .streams + .iter() + .flat_map(|stream| { + stream + .audio + .as_ref() + .into_iter() + .chain(stream.video.as_ref()) + .map(|group| group.as_str()) + }) + .collect::>() + .into_iter(); + + let mut group_to_id = HashMap::new(); + for (group_idx, stream) in reference_streams.enumerate() { + let Some(groups) = playlist.groups.get_mut(stream) else { + return Err(JsValue::from_str(&format!("failed to find group for stream: {}", stream))); + }; + + // If we have a default track we need to make sure only 1 track is default + let pos = groups.iter().position(|item| item.default).unwrap_or(0); + groups.iter_mut().for_each(|item| item.default = false); + groups[pos].default = true; + + // Otherwise this is actually a reference track + // So we need to generate a new track id for it. + for track in groups { + let url = match Url::parse(&track.uri).or_else(|_| input_url.join(&track.uri)) { + Ok(url) => url, + Err(err) => { + return Err(JsValue::from_str(&format!("failed to parse url: {}", err))); + } + }; + + let track_id = m3u8_url_to_track + .entry(url.clone()) + .or_insert_with(|| { + let t = Track { + id: track_idx, + group_id: group_idx as u32, + playlist_url: url.clone(), + bandwidth: track.bandwidth, + codecs: track.codecs.clone(), + frame_rate: track.frame_rate, + width: track.resolution.map(|r| r.0), + height: track.resolution.map(|r| r.1), + }; + + track_idx += 1; + + t + }) + .id; + + if track.default { + self.active_group_track_ids.push(track_id); + } + } + + group_to_id.insert(stream, group_idx as u32); + } + + let mut variants_map = HashMap::new(); + + for (id, stream) in playlist.streams.iter().enumerate() { + let variant = Variant { + audio_track: stream + .audio + .as_ref() + .and_then(|id| group_to_id.get(id.as_str()).cloned()), + video_track: stream + .video + .as_ref() + .and_then(|id| group_to_id.get(id.as_str()).cloned()), + group: stream.group.clone(), + name: stream.name.clone(), + bandwidth: stream.bandwidth, + id: id as u32, + }; + + let codec = format!( + "video/mp4; codecs=\"{}\"", + stream + .video + .iter() + .chain(stream.audio.iter()) + .filter_map(|id| playlist.groups.get(id)) + .filter_map(|group| group.first()) + .map(|track| track.codecs.as_str()) + .collect::>() + .join(",") + ); + + if MediaSource::is_type_supported(&codec) { + variants_map + .entry(stream.name.as_str()) + .or_insert_with(Vec::new) + .push(variant); + } + } + + let mut scuf_groups = playlist.scuf_groups.iter().collect::>(); + scuf_groups.sort_by(|(_, a), (_, b)| a.priority.cmp(&b.priority)); + + let mut variants = variants_map + .into_iter() + .filter_map(|(_, variants)| { + scuf_groups.iter().find_map(move |(group, _)| { + variants.iter().find(|v| v.group == group.as_str()).cloned() + }) + }) + .collect::>(); + + variants.sort_by(|a, b| b.bandwidth.cmp(&a.bandwidth)); + + variants.iter_mut().enumerate().for_each(|(id, variant)| { + variant.id = id as u32; + }); + + self.variants = variants.clone(); + + let mut tracks = m3u8_url_to_track.into_values().collect::>(); + tracks.sort_by(|a, b| a.id.cmp(&b.id)); + + self.track_states = tracks + .clone() + .into_iter() + .map(|t| TrackState::new(t, self.bandwidth.clone())) + .collect(); + + self.inner.set_tracks(tracks, variants, true); + self.inner + .set_active_group_track_ids(self.active_group_track_ids.clone()); + self.inner.set_active_variant_id(1); + + Ok(()) + } + + fn handle_media_playlist( + &mut self, + input_url: Url, + playlist: MediaPlaylist, + ) -> Result<(), JsValue> { + let track = Track { + id: 0, + group_id: 0, + bandwidth: 0, + playlist_url: input_url, + codecs: "".to_string(), + frame_rate: None, + height: None, + width: None, + }; + + let variant = Variant { + audio_track: Some(0), + video_track: Some(0), + group: "default".to_string(), + name: "default".to_string(), + bandwidth: 0, + id: 0, + }; + + let mut track_state = TrackState::new(track.clone(), self.bandwidth.clone()); + track_state.set_playlist(playlist); + + self.track_states = vec![track_state]; + + self.inner.set_tracks(vec![track], vec![variant], false); + self.inner.set_active_variant_id(0); + + Ok(()) + } +} diff --git a/frontend/player/src/player/runner/source_buffer.rs b/frontend/player/src/player/runner/source_buffer.rs new file mode 100644 index 00000000..5e10d7e2 --- /dev/null +++ b/frontend/player/src/player/runner/source_buffer.rs @@ -0,0 +1,87 @@ +use tokio::sync::mpsc; +use wasm_bindgen::JsValue; +use web_sys::{MediaSource, SourceBuffer, TimeRanges}; + +use crate::player::util::{register_events, Holder}; + +pub struct SourceBufferHolder { + sb: Holder, + rx: mpsc::Receiver<()>, +} + +impl SourceBufferHolder { + pub fn new(media_source: &MediaSource, codec: &str) -> Result { + let sb = media_source.add_source_buffer(codec)?; + let (tx, rx) = mpsc::channel(128); + + let cleanup = register_events!(sb, { + "updateend" => move |_| { + if tx.try_send(()).is_err() { + tracing::warn!("failed to send updateend event"); + } + } + }); + + Ok(Self { + sb: Holder::new(sb, cleanup), + rx, + }) + } + + pub fn buffered(&self) -> Result { + self.sb.buffered() + } + + pub fn change_type(&self, codec: &str) -> Result<(), JsValue> { + self.sb.change_type(codec)?; + Ok(()) + } + + pub async fn append_buffer(&mut self, mut data: Vec) -> Result<(), JsValue> { + self.sb.append_buffer_with_u8_array(data.as_mut_slice())?; + self.rx.recv().await; + Ok(()) + } + + pub async fn remove(&mut self, start: f64, end: f64) -> Result<(), JsValue> { + if start >= end { + return Ok(()); + } + + self.sb.remove(start, end)?; + self.rx.recv().await; + Ok(()) + } +} + +pub enum SourceBuffers { + AudioVideoSplit { + audio: SourceBufferHolder, + video: SourceBufferHolder, + }, + None, + AudioVideoCombined(SourceBufferHolder), +} + +impl SourceBuffers { + pub fn audio(&mut self) -> Option<&mut SourceBufferHolder> { + match self { + Self::AudioVideoSplit { audio, .. } => Some(audio), + _ => None, + } + } + + pub fn video(&mut self) -> Option<&mut SourceBufferHolder> { + match self { + Self::AudioVideoSplit { video, .. } => Some(video), + _ => None, + } + } + + pub fn audiovideo(&mut self) -> Option<&mut SourceBufferHolder> { + match self { + Self::AudioVideoCombined(audiovideo) => Some(audiovideo), + _ => None, + } + } +} diff --git a/frontend/player/src/player/runner/util.rs b/frontend/player/src/player/runner/util.rs new file mode 100644 index 00000000..d480853b --- /dev/null +++ b/frontend/player/src/player/runner/util.rs @@ -0,0 +1,154 @@ +use tokio::sync::mpsc; +use web_sys::{Document, HtmlVideoElement, MediaSource}; + +use crate::player::util::{register_events, Holder}; + +use super::events::RunnerEvent; + +pub fn make_video_holder( + element: HtmlVideoElement, + tx: &mpsc::Sender<(RunnerEvent, web_sys::Event)>, +) -> Holder { + let cleanup = register_events!(element, { + "error" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoError, evt)).is_err() { + tracing::warn!("Video error event dropped"); + } + } + }, + "pause" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoPause, evt)).is_err() { + tracing::warn!("Video pause event dropped"); + } + } + }, + "play" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoPlay, evt)).is_err() { + tracing::warn!("Video play event dropped"); + } + } + }, + "ratechange" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoRateChange, evt)).is_err() { + tracing::warn!("Video ratechange event dropped"); + } + } + }, + "seeked" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoSeeked, evt)).is_err() { + tracing::warn!("Video seeked event dropped"); + } + } + }, + "seeking" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoSeeking, evt)).is_err() { + tracing::warn!("Video seeking event dropped"); + } + } + }, + "stalled" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoStalled, evt)).is_err() { + tracing::warn!("Video stalled event dropped"); + } + } + }, + "suspend" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoSuspend, evt)).is_err() { + tracing::warn!("Video suspend event dropped"); + } + } + }, + "timeupdate" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoTimeUpdate, evt)).is_err() { + tracing::warn!("Video timeupdate event dropped"); + } + } + }, + "volumechange" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoVolumeChange, evt)).is_err() { + tracing::warn!("Video volumechange event dropped"); + } + } + }, + "waiting" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::VideoWaiting, evt)).is_err() { + tracing::warn!("Video waiting event dropped"); + } + } + }, + }); + + Holder::new(element, cleanup) +} + +pub fn make_media_source_holder( + media_source: MediaSource, + tx: &mpsc::Sender<(RunnerEvent, web_sys::Event)>, +) -> Holder { + let cleanup = register_events!(media_source, { + "sourceclose" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::MediaSourceClose, evt)).is_err() { + tracing::warn!("MediaSource close event dropped") + } + } + }, + "sourceended" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::MediaSourceEnded, evt)).is_err() { + tracing::warn!("MediaSource ended event dropped") + } + } + }, + "sourceopen" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::MediaSourceOpen, evt)).is_err() { + tracing::warn!("MediaSource open event dropped") + } + } + }, + }); + + Holder::new(media_source, cleanup) +} + +pub fn make_document_holder(tx: &mpsc::Sender<(RunnerEvent, web_sys::Event)>) -> Holder { + let document = web_sys::window().unwrap().document().unwrap(); + let cleanup = register_events!(document, { + "visibilitychange" => { + let tx = tx.clone(); + move |evt| { + if tx.try_send((RunnerEvent::DocumentVisibilityChange, evt)).is_err() { + tracing::warn!("Document visibilitychange event dropped") + } + } + }, + }); + + Holder::new(document, cleanup) +} diff --git a/frontend/player/src/player/track.rs b/frontend/player/src/player/track.rs index 8446a1b0..fa7effa2 100644 --- a/frontend/player/src/player/track.rs +++ b/frontend/player/src/player/track.rs @@ -1,4 +1,7 @@ -use std::{collections::VecDeque, io}; +use std::{ + collections::{HashSet, VecDeque}, + io, +}; use bytes::{Buf, Bytes}; use mp4::{ @@ -9,38 +12,73 @@ use serde::Serialize; use tsify::Tsify; use url::Url; use wasm_bindgen::JsValue; -use web_sys::window; -use crate::hls::{self, media::MediaPlaylist}; +use crate::hls::{ + self, + media::{MediaPlaylist, RenditionReport}, +}; -use super::fetch::{FetchRequest, InflightRequest}; +use super::{ + bandwidth::Bandwidth, + fetch::{FetchRequest, InflightRequest}, + util::now, +}; #[derive(Tsify, Debug, Clone, Serialize)] #[tsify(into_wasm_abi)] pub struct Track { + /// The track id (unique for all tracks) pub id: u32, - pub bandwidth: Option, - pub name: Option, + /// The group this track belongs to + pub group_id: u32, + /// The bandwidth estimate for this track + pub bandwidth: u32, + /// The url to the playlist for this track pub playlist_url: Url, - pub referenced_group_ids: Vec, - pub is_variant_track: bool, - pub codecs: Option, + /// The codecs for this track + pub codecs: String, + + /// The width of this track (if video) pub width: Option, + /// The height of this track (if video) pub height: Option, + /// The frame rate of this track (if video) pub frame_rate: Option, - pub reference: Option, } #[derive(Tsify, Debug, Clone, Serialize)] #[tsify(into_wasm_abi)] -pub struct ReferenceTrack { - pub group_id: u32, - pub is_default: bool, +pub struct Variant { + /// The variant id (unique for all variants) + pub id: u32, + /// The name of this variant + pub name: String, + /// The scuffle group this variant belongs to + pub group: String, + /// The group id of the audio track + pub audio_track: Option, + /// The group id of the video track + pub video_track: Option, + /// The bandwidth estimation for this variant + pub bandwidth: u32, } pub struct TrackRequest { - req: InflightRequest, + inflight: Option, + request: FetchRequest, is_init: bool, + is_preload: bool, +} + +impl TrackRequest { + fn new(req: FetchRequest, is_init: bool, is_preload: bool) -> Self { + Self { + inflight: None, + request: req, + is_init, + is_preload, + } + } } pub struct TrackState { @@ -48,11 +86,12 @@ pub struct TrackState { playlist_req: Option, playlist: Option, - requests: VecDeque, + requests: Requests, current_sn: u32, current_part: u32, current_map_sn: u32, + was_rendition: bool, running: bool, low_latency: bool, @@ -60,15 +99,95 @@ pub struct TrackState { last_fetch_delay: f64, last_playlist_fetch: f64, + preloaded_map: HashSet, + last_end_time: f64, stop_at: Option, + bandwidth: Bandwidth, + track_info: Option, } -fn now() -> f64 { - window().unwrap().performance().unwrap().now() +struct Requests { + max_concurrent_requests: usize, + inflight: VecDeque, + inflight_urls: HashSet, +} + +impl Requests { + fn new(max_concurrent_requests: usize) -> Self { + Self { + max_concurrent_requests, + inflight: VecDeque::new(), + inflight_urls: HashSet::new(), + } + } + + fn set_max_concurrent_requests(&mut self, max_concurrent_requests: usize) { + self.max_concurrent_requests = max_concurrent_requests; + } + + fn push(&mut self, mut req: TrackRequest) -> Result<(), JsValue> { + if self.inflight_urls.contains(req.request.url()) { + return Ok(()); + } + + if self.active_count() < self.max_concurrent_requests && req.inflight.is_none() { + req.inflight = Some(req.request.start()?); + } + + self.inflight_urls.insert(req.request.url().clone()); + self.inflight.push_back(req); + + Ok(()) + } + + fn pop(&mut self) -> Result, JsValue> { + let Some(req) = self.inflight.front_mut() else { + return Ok(None); + }; + + if req.inflight.is_none() { + req.inflight = Some(req.request.start()?); + } + + if req.inflight.as_mut().unwrap().is_done() { + self.inflight_urls.remove(req.request.url()); + + let old = self.inflight.pop_front(); + + while self.active_count() < self.max_concurrent_requests { + if let Some(req) = self.inflight.iter_mut().find(|r| r.inflight.is_none()) { + req.inflight = Some(req.request.start()?); + } else { + break; + } + } + + return Ok(old); + } + + Ok(None) + } + + fn active_count(&mut self) -> usize { + self.inflight + .iter() + .filter(|r| { + r.inflight + .as_ref() + .map(|i| !i.is_done()) + .unwrap_or_default() + }) + .count() + } + + fn clear(&mut self) { + self.inflight.clear(); + self.inflight_urls.clear(); + } } fn get_url(playlist_url: &str, url: &str) -> Result { @@ -113,17 +232,18 @@ fn demux_mp4_boxes(mut cursor: io::Cursor) -> Result, JsValue }) .take_while(|r| r.is_ok()) .collect::, _>>() - .map_err(|_| "invalid init segment")?) + .map_err(|e| e.to_string())?) } impl TrackState { - pub fn new(track: Track) -> Self { + pub fn new(track: Track, bandwidth: Bandwidth) -> Self { Self { track, playlist_req: None, running: false, + was_rendition: false, playlist: None, - requests: VecDeque::new(), + requests: Requests::new(1), current_sn: 0, current_map_sn: 0, next_playlist_req_time: 0.0, @@ -134,22 +254,61 @@ impl TrackState { last_end_time: 0.0, stop_at: None, + preloaded_map: HashSet::new(), + + bandwidth, + track_info: None, } } + pub fn url(&self) -> &Url { + &self.track.playlist_url + } + pub fn running(&self) -> bool { self.running } + pub fn rendition_reports(&self, url: &Url) -> Option { + let playlist = self.playlist.as_ref()?; + + playlist + .rendition_reports + .iter() + .find(|r| { + let r_url = get_url(self.track.playlist_url.as_str(), &r.uri).unwrap(); + &r_url == url + }) + .cloned() + } + pub fn run(&mut self) -> Result, JsValue> { if !self.running { return Ok(None); } - if let Some(req) = self.handle_requests()? { + if let Some(mut req) = self.requests.pop()? { + let inflight = req.inflight.as_mut().unwrap(); + // We have something to yeild to the caller. - let data = req.req.result()?.expect("request should be done"); + let data = match inflight.result() { + Ok(Some(data)) => data, + Ok(None) => { + return Err("request should be done".into()); + } + Err(err) => { + if req.is_preload { + self.preloaded_map.remove(inflight.url()); + return Ok(None); + } else { + return Err(err); + } + } + }; + + self.bandwidth.report_download(&inflight.metrics().unwrap()); + if req.is_init { let boxes = demux_mp4_boxes(io::Cursor::new(Bytes::from(data)))?; @@ -190,8 +349,6 @@ impl TrackState { let mut fragments = Vec::new(); - let mut keyframe = 0; - // Convert the boxes vector into a tuple of moof and mdat let boxes = boxes .into_iter() @@ -230,10 +387,6 @@ impl TrackState { return Err("invalid media segment, missing traf".into()); }; - if traf.contains_keyframe() { - keyframe += 1; - } - // This will tell us when the fragment starts let base_decode_time = traf .tfdt @@ -260,10 +413,8 @@ impl TrackState { self.last_end_time = end_time; - if let Some(stop_at) = self.stop_at { - if end_time >= stop_at && keyframe > 0 { - self.stop(); - } + if self.stop_at.map(|s| s <= end_time).unwrap_or_default() { + self.stop(); } Ok(Some(TrackResult::Media { @@ -280,24 +431,19 @@ impl TrackState { } } - fn handle_requests(&mut self) -> Result, JsValue> { - let Some(req) = self.requests.front() else { - return Ok(None); - }; - - if req.req.is_done() { - Ok(self.requests.pop_front()) - } else { - Ok(None) - } - } - pub fn set_stop_at(&mut self, stop_at: Option) { self.stop_at = stop_at; + if let Some(stop_at) = self.stop_at { + if stop_at <= self.last_end_time { + self.stop(); + } + } } pub fn set_low_latency(&mut self, low_latency: bool) { self.low_latency = low_latency; + self.requests + .set_max_concurrent_requests(if low_latency { 3 } else { 1 }) } pub fn stop_at(&self) -> Option { @@ -315,13 +461,13 @@ impl TrackState { self.track.playlist_url.as_str(), segment.map.as_deref().unwrap(), )?; - self.requests.push_back(TrackRequest { - req: FetchRequest::new("GET", url.as_str()) + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) .header("Accept", "video/mp4") - .set_timeout(2000) - .start()?, - is_init: true, - }); + .set_timeout(2000), + true, + false, + ))?; self.current_map_sn = segment.sn; } @@ -356,72 +502,144 @@ impl TrackState { if let Some(map) = &segment.map { if self.current_map_sn < segment.sn { let url = get_url(self.track.playlist_url.as_str(), map)?; - self.requests.push_back(TrackRequest { - req: FetchRequest::new("GET", url.as_str()) + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) .header("Accept", "video/mp4") - .set_timeout(2000) - .start()?, - is_init: true, - }); + .set_timeout(2000), + true, + false, + ))?; } + } else if self.current_map_sn < segment.sn.checked_sub(1).unwrap_or_default() { + // Since its now possible to start at any segment, we need to make sure we have + // the map for the previous segment + let url = get_url( + self.track.playlist_url.as_str(), + playlist + .segments + .iter() + .rev() + .find_map(|s| s.map.as_ref()) + .ok_or("missing map")?, + )?; + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) + .header("Accept", "video/mp4") + .set_timeout(2000), + true, + false, + ))?; } self.current_map_sn = segment.sn; - if segment.url.is_empty() && (!self.low_latency || segment.parts.is_empty()) { - continue; - } + if self.low_latency { + if segment.parts.is_empty() && segment.url.is_empty() { + break; + } - let url = if self.low_latency && segment.parts.len() > self.current_part as usize { - let part = &segment.parts[self.current_part as usize]; - self.current_part += 1; - self.last_fetch_delay = part.duration * 1000.0 / 2.0; - part.uri.as_str() - } else if !segment.url.is_empty() { - self.current_part = 0; - self.current_sn = segment.sn + 1; - self.last_fetch_delay = segment.duration / 2.0 * 1000.0; - segment.url.as_str() - } else { - continue; - }; + if !segment.url.is_empty() && self.current_part == 0 { + // We havent loaded this segment yet and it has a completed url + // So we just request that instead + let url = get_url(self.track.playlist_url.as_str(), &segment.url)?; + self.last_fetch_delay = segment.duration / 2.0 * 1000.0; + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) + .header("Accept", "video/mp4") + .set_timeout((segment.duration * 1000.0) as u32 + 1500), + false, + false, + ))?; + } else { + for part in segment.parts.iter().skip(self.current_part as usize) { + let url = get_url(self.track.playlist_url.as_str(), &part.uri)?; + if !self.preloaded_map.remove(&url) { + self.last_fetch_delay = part.duration * 1000.0 / 2.0; + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) + .header("Accept", "video/mp4") + .set_timeout((part.duration * 1000.0) as u32 + 1500), + false, + false, + ))?; + } + + self.current_part += 1; + } + } - let url = get_url(self.track.playlist_url.as_str(), url)?; - self.requests.push_back(TrackRequest { - req: FetchRequest::new("GET", url.as_str()) - .header("Accept", "video/mp4") - .set_timeout(2000) - .start()?, - is_init: false, - }); + if !segment.url.is_empty() { + self.current_sn = segment.sn + 1; + self.current_part = 0; + } + } else { + if segment.url.is_empty() { + break; + } - if self.low_latency - && segment.parts.len() == self.current_part as usize - && !segment.url.is_empty() - { - // We are finished with this segment self.current_sn = segment.sn + 1; self.current_part = 0; - continue; + let url = get_url(self.track.playlist_url.as_str(), &segment.url)?; + self.last_fetch_delay = segment.duration / 2.0 * 1000.0; + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) + .header("Accept", "video/mp4") + .set_timeout((segment.duration * 1000.0) as u32 + 1500), + false, + false, + ))?; } } - // If the playlist has an end list tag we don't need to request it again if playlist.end_list { self.next_playlist_req_time = -1.0; } else if self.next_playlist_req_time == -1.0 { self.next_playlist_req_time = now() + self.last_fetch_delay; } + let msn = playlist + .segments + .last() + .map(|s| s.sn) + .unwrap_or(playlist.media_sequence); + if msn < self.current_sn { + return Ok(()); + } + + if self.low_latency { + for hint in &playlist.preload_hint { + let is_init = hint.hint_type.to_uppercase() == "MAP"; + let url = get_url(self.track.playlist_url.as_str(), &hint.uri)?; + if self.preloaded_map.insert(url.clone()) { + self.requests.push(TrackRequest::new( + FetchRequest::new("GET", url) + .header("Accept", "video/mp4") + .set_timeout( + (playlist.part_target_duration.unwrap_or(0.5) * 1000.0) as u32 + + 1500, + ), + is_init, + true, + ))?; + } + } + } + Ok(()) } fn request_playlist(&mut self) -> Result<(), JsValue> { - if let Some(req) = self.playlist_req.as_ref() { + if let Some(req) = self.playlist_req.as_mut() { if let Some(result) = req.result()? { - match hls::Playlist::try_from(result.as_slice())? { - hls::Playlist::Media(playlist) => { + match hls::Playlist::try_from(result.as_slice()) { + Ok(hls::Playlist::Media(playlist)) => { self.playlist = Some(playlist); + self.next_playlist_req_time = -1.0; + self.last_playlist_fetch = now(); + } + Err(err) => { + tracing::error!("failed to parse playlist: {}", err); + self.next_playlist_req_time = now(); } _ => { return Err("invalid playlist".into()); @@ -429,34 +647,37 @@ impl TrackState { } self.playlist_req = None; - self.next_playlist_req_time = -1.0; - self.last_playlist_fetch = now(); } - } else if self.next_playlist_req_time != -1.0 { + } + + if self.next_playlist_req_time != -1.0 && self.playlist_req.is_none() { if self.low_latency - && self + && (self .playlist .as_ref() .and_then(|p| p.server_control.as_ref().map(|s| s.can_block_reload)) .unwrap_or_default() + || self.was_rendition) { // Low latency request let mut url = self.track.playlist_url.clone(); + self.was_rendition = false; + url.query_pairs_mut() .append_pair("_HLS_msn", self.current_sn.to_string().as_str()); url.query_pairs_mut() .append_pair("_HLS_part", self.current_part.to_string().as_str()); self.playlist_req = Some( - FetchRequest::new("GET", url.as_str()) + FetchRequest::new("GET", url) .header("Accept", "application/vnd.apple.mpegurl") - .set_timeout(2000) + .set_timeout(5000) .start()?, ); } else if now() >= self.next_playlist_req_time { self.playlist_req = Some( - FetchRequest::new("GET", self.track.playlist_url.as_str()) + FetchRequest::new("GET", self.track.playlist_url.clone()) .header("Accept", "application/vnd.apple.mpegurl") .set_timeout(2000) .start()?, @@ -467,12 +688,17 @@ impl TrackState { Ok(()) } - pub fn track(&self) -> &Track { - &self.track + pub fn start(&mut self) { + self.running = true; + self.stop_at = None; + self.was_rendition = false; } - pub fn start(&mut self) { + pub fn start_at(&mut self, current_sn: u32) { self.running = true; + self.current_sn = current_sn; + self.current_part = 0; + self.was_rendition = true; self.stop_at = None; } @@ -488,6 +714,7 @@ impl TrackState { self.last_playlist_fetch = 0.0; self.next_playlist_req_time = 0.0; self.requests.clear(); + self.preloaded_map.clear(); } pub fn set_playlist(&mut self, playlist: MediaPlaylist) { diff --git a/frontend/player/src/player/util.rs b/frontend/player/src/player/util.rs index c00d041f..e75c8d0f 100644 --- a/frontend/player/src/player/util.rs +++ b/frontend/player/src/player/util.rs @@ -2,18 +2,23 @@ use std::ops::{Deref, DerefMut}; use wasm_bindgen::JsCast; -type Cleanup = Box; +type BoxedCleanup = Box; +/// A holder is a wrapper around an event target which implements JsCast. +/// This is because we want to be able to remove the event listeners when the holder is dropped. +/// This is done by calling the cleanup function. +/// This is really convenient because we can just pass the holder around and not worry about removing the event listeners. +/// The cleanup function is only called once. pub struct Holder { inner: T, - cleanup: Option, + cleanup: Option, } impl Holder { - pub fn new(inner: T, cleanup: Cleanup) -> Self { + pub fn new(inner: T, cleanup: impl FnOnce(&web_sys::EventTarget) + 'static) -> Self { Self { inner, - cleanup: Some(cleanup), + cleanup: Some(Box::new(cleanup)), } } } @@ -47,23 +52,30 @@ macro_rules! register_events { ),* $(,)? }) => { { + use wasm_bindgen::JsCast; + let mut handlers = std::collections::VecDeque::new(); $( - handlers.push_back((vec![$($evt.to_string()),+], Closure::::new($body))); + handlers.push_back((vec![$($evt.to_string()),+], ::wasm_bindgen::closure::Closure::::new($body))); $( $ob.add_event_listener_with_callback($evt, handlers.back().unwrap().1.as_ref().unchecked_ref()).unwrap(); )* )* - Box::new(move |val: &web_sys::EventTarget| { + move |val: &web_sys::EventTarget| { handlers.drain(..).for_each(|(evts, cb)| { for evt in evts { val.remove_event_listener_with_callback(&evt, cb.as_ref().unchecked_ref()).unwrap(); } }); - }) as Box + } } }; } pub(super) use register_events; +use web_sys::window; + +pub fn now() -> f64 { + window().unwrap().performance().unwrap().now() +} diff --git a/video/edge/src/edge/mod.rs b/video/edge/src/edge/mod.rs index 97be735f..039f7f3e 100644 --- a/video/edge/src/edge/mod.rs +++ b/video/edge/src/edge/mod.rs @@ -56,6 +56,12 @@ pub fn cors_middleware(_: &Arc) -> Middleware { .insert(header::ACCESS_CONTROL_ALLOW_METHODS, "*".parse().unwrap()); resp.headers_mut() .insert(header::ACCESS_CONTROL_ALLOW_HEADERS, "*".parse().unwrap()); + resp.headers_mut().insert( + header::ACCESS_CONTROL_EXPOSE_HEADERS, + "Date".parse().unwrap(), + ); + resp.headers_mut() + .insert("Timing-Allow-Origin", "*".parse().unwrap()); resp.headers_mut().insert( header::ACCESS_CONTROL_MAX_AGE, Duration::from_secs(86400) diff --git a/video/edge/src/edge/stream.rs b/video/edge/src/edge/stream.rs index 96204d00..e936d706 100644 --- a/video/edge/src/edge/stream.rs +++ b/video/edge/src/edge/stream.rs @@ -50,7 +50,7 @@ pub async fn variant_playlist(req: Request) -> Result> { let mut count = 0; loop { - if count > 10 { + if count > 100 { return Err((StatusCode::BAD_REQUEST, "Bad Request").into()); } @@ -75,14 +75,6 @@ pub async fn variant_playlist(req: Request) -> Result> { let current_segment_idx: u64 = fields[0].parse::().unwrap_or_default(); let current_fragment_idx: u64 = fields[1].parse::().unwrap_or_default(); - tracing::info!( - sequence_number = sequence_number, - current_segment_idx = current_segment_idx, - part_number = part_number, - current_fragment_idx = current_fragment_idx, - "waiting for sequence number" - ); - if sequence_number > current_segment_idx + 3 { return Err((StatusCode::BAD_REQUEST, "Bad Request").into()); } @@ -94,7 +86,7 @@ pub async fn variant_playlist(req: Request) -> Result> { } count += 1; - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; } } @@ -214,14 +206,6 @@ pub async fn segment(req: Request) -> Result> { let current_segment_idx: u64 = fields[0].parse::().unwrap_or_default(); let current_fragment_idx: u64 = fields[1].parse::().unwrap_or_default(); - tracing::info!( - sequence_number = sequence_number, - current_segment_idx = current_segment_idx, - part_number = part_number, - current_fragment_idx = current_fragment_idx, - "waiting for sequence number" - ); - if sequence_number > current_segment_idx + 3 { return Err((StatusCode::BAD_REQUEST, "Bad Request").into()); } diff --git a/video/transcoder/src/config.rs b/video/transcoder/src/config.rs index 07ffca96..7c16a686 100644 --- a/video/transcoder/src/config.rs +++ b/video/transcoder/src/config.rs @@ -175,7 +175,7 @@ pub struct TranscoderConfig { impl Default for TranscoderConfig { fn default() -> Self { Self { - socket_dir: "/tmp".to_string(), + socket_dir: format!("/tmp/{}", std::process::id()), uid: 1000, gid: 1000, } diff --git a/video/transcoder/src/transcoder/job/mod.rs b/video/transcoder/src/transcoder/job/mod.rs index 37c9a517..3b5c0e14 100644 --- a/video/transcoder/src/transcoder/job/mod.rs +++ b/video/transcoder/src/transcoder/job/mod.rs @@ -43,6 +43,9 @@ use crate::{ }; use fred::interfaces::KeysInterface; +use self::renditions::RenditionMap; + +mod renditions; mod track_parser; mod utils; pub(crate) mod variant; @@ -111,18 +114,18 @@ struct Job { } #[inline(always)] -fn redis_mutex_key(stream_id: &str) -> String { +fn redis_mutex_key(stream_id: impl std::fmt::Display) -> String { format!("transcoder:{}:mutex", stream_id) } #[inline(always)] -fn redis_master_playlist_key(stream_id: &str) -> String { +fn redis_master_playlist_key(stream_id: impl std::fmt::Display) -> String { format!("transcoder:{}:playlist", stream_id) } fn set_master_playlist( global: Arc, - stream_id: &str, + stream_id: impl std::fmt::Display, state: &StreamState, lock: CancellationToken, ) -> impl futures::Future> + Send + 'static { @@ -135,14 +138,35 @@ fn set_master_playlist( let mut state_map = HashMap::new(); for transcode_state in state.transcodes.iter() { - playlist.push_str(format!("#EXT-X-MEDIA:TYPE={},GROUP-ID=\"{}\",NAME=\"{}\",AUTOSELECT=YES,DEFAULT=YES,URI=\"{}/index.m3u8\"\n", match transcode_state.settings.as_ref().unwrap() { - stream_state::transcode::Settings::Video(_) => { - "VIDEO" - }, - stream_state::transcode::Settings::Audio(_) => { - "AUDIO" - }, - }, transcode_state.id, transcode_state.id, transcode_state.id).as_str()); + let mut tags = vec![ + format!( + "TYPE={}", + match transcode_state.settings.as_ref().unwrap() { + stream_state::transcode::Settings::Video(_) => { + "VIDEO" + } + stream_state::transcode::Settings::Audio(_) => { + "AUDIO" + } + } + ), + "AUTOSELECT=YES".to_string(), + "DEFAULT=YES".to_string(), + format!("GROUP-ID=\"{}\"", transcode_state.id), + format!("NAME=\"{}\"", transcode_state.id), + format!("BANDWIDTH={}", transcode_state.bitrate), + format!("CODECS=\"{}\"", transcode_state.codec), + format!("URI=\"{}/index.m3u8\"", transcode_state.id), + ]; + + if let Some(stream_state::transcode::Settings::Video(settings)) = + transcode_state.settings.as_ref() + { + tags.push(format!("FRAME-RATE={}", settings.framerate)); + tags.push(format!("RESOLUTION={}x{}", settings.width, settings.height)); + } + + playlist.push_str(format!("#EXT-X-MEDIA:{}\n", tags.join(",")).as_str()); state_map.insert(transcode_state.id.as_str(), transcode_state); } @@ -246,33 +270,22 @@ fn set_master_playlist( } fn report_to_ingest( - global: Arc, mut client: IngestClient, mut channel: mpsc::Receiver, ) -> impl Stream> + Send + 'static { stream!({ - loop { - select! { - msg = channel.recv() => { - match msg { - Some(msg) => { - match client.transcoder_event(msg).timeout(Duration::from_secs(5)).await { - Ok(Ok(_)) => {}, - Ok(Err(e)) => { - yield Err(e.into()); - } - Err(e) => { - yield Err(e.into()); - } - } - }, - None => { - break; - } - } - }, - _ = global.ctx.done() => { - break; + while let Some(msg) = channel.recv().await { + match client + .transcoder_event(msg) + .timeout(Duration::from_secs(5)) + .await + { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + yield Err(e.into()); + } + Err(e) => { + yield Err(e.into()); } } } @@ -295,7 +308,7 @@ impl Job { let mut update_playlist_fut = pin!(set_master_playlist( global.clone(), - &self.req.stream_id, + self.req.stream_id.clone(), self.stream_state(), self.lock_owner.child_token(), )); @@ -314,6 +327,11 @@ impl Job { let stream_state = self.stream_state(); let (ready_tx, mut ready_recv) = mpsc::channel(16); + let mut rendition_map = RenditionMap::new(); + for transcode_state in stream_state.transcodes.iter() { + rendition_map.insert(transcode_state.id.clone()); + } + let rendition_map = Arc::new(rendition_map); for transcode_state in stream_state.transcodes.iter() { let sock_path = socket_dir.join(format!("{}.sock", transcode_state.id)); @@ -344,6 +362,7 @@ impl Job { transcode_state.id.clone(), self.req.request_id.clone(), socket, + rendition_map.clone(), )); } @@ -607,7 +626,7 @@ impl Job { let mut ready_count = 0; let (report, rx) = mpsc::channel(10); - let mut report_fut = pin!(report_to_ingest(global.clone(), self.client.clone(), rx)); + let mut report_fut = pin!(report_to_ingest(self.client.clone(), rx)); while select! { r = report_fut.next() => { diff --git a/video/transcoder/src/transcoder/job/renditions.rs b/video/transcoder/src/transcoder/job/renditions.rs new file mode 100644 index 00000000..123772c3 --- /dev/null +++ b/video/transcoder/src/transcoder/job/renditions.rs @@ -0,0 +1,51 @@ +use std::{collections::HashMap, sync::atomic::AtomicU32}; + +#[derive(Debug, Default)] +pub struct RenditionMap { + map: HashMap, +} + +impl RenditionMap { + pub fn new() -> Self { + Self::default() + } + + pub fn renditions(&self) -> Vec { + self.map + .iter() + .map(|(id, r)| Rendition { + id: id.clone(), + last_msn: r.last_msn.load(std::sync::atomic::Ordering::Relaxed), + last_part: r.last_part.load(std::sync::atomic::Ordering::Relaxed), + }) + .collect() + } + + pub fn set(&self, id: &str, last_msn: u32, last_part: u32) -> bool { + if let Some(r) = self.map.get(id) { + r.last_msn + .store(last_msn, std::sync::atomic::Ordering::Relaxed); + r.last_part + .store(last_part, std::sync::atomic::Ordering::Relaxed); + true + } else { + false + } + } + + pub fn insert(&mut self, id: String) { + self.map.insert(id, AtomicRendition::default()); + } +} + +#[derive(Debug, Default)] +struct AtomicRendition { + last_msn: AtomicU32, + last_part: AtomicU32, +} + +pub struct Rendition { + pub id: String, + pub last_msn: u32, + pub last_part: u32, +} diff --git a/video/transcoder/src/transcoder/job/variant/mod.rs b/video/transcoder/src/transcoder/job/variant/mod.rs index fe7468ca..0ca63d5a 100644 --- a/video/transcoder/src/transcoder/job/variant/mod.rs +++ b/video/transcoder/src/transcoder/job/variant/mod.rs @@ -31,6 +31,7 @@ use tokio::{net::UnixListener, select, sync::mpsc}; use tokio_util::sync::CancellationToken; use super::{ + renditions::RenditionMap, track_parser::{track_parser, TrackOut, TrackSample}, utils::{release_lock, set_lock, unix_stream}, }; @@ -63,6 +64,7 @@ struct Variant { segment_state: HashMap)>, ready: mpsc::Sender<()>, is_ready: bool, + renditions: Arc, } pub async fn handle_variant( @@ -72,8 +74,9 @@ pub async fn handle_variant( variant_id: String, request_id: String, track: UnixListener, + renditions: Arc, ) -> Result { - let mut variant = Variant::new(ready, 1, stream_id, variant_id, request_id); + let mut variant = Variant::new(ready, 1, stream_id, variant_id, request_id, renditions); variant .run( @@ -92,6 +95,7 @@ impl Variant { stream_id: String, variant_id: String, request_id: String, + renditions: Arc, ) -> Self { Self { stream_id, @@ -104,6 +108,7 @@ impl Variant { segment_state: HashMap::new(), should_discontinuity: false, ready, + renditions, is_ready: false, } } @@ -116,7 +121,7 @@ impl Variant { ) -> Result<(), ()> { let mut set_lock_fut = pin!(set_lock( global.clone(), - consts::redis_mutex_key(&self.stream_id, &self.variant_id), + consts::redis_mutex_key(&self.stream_id.to_string(), &self.variant_id.to_string()), self.request_id.clone(), self.lock_owner.clone(), )); @@ -273,6 +278,7 @@ impl Variant { } Operation::Fragments => { self.handle_sample(global).await?; + self.update_renditions(); let pipeline = global.redis.pipeline(); if self.update_keys(&pipeline).await? { @@ -290,6 +296,30 @@ impl Variant { Ok(()) } + fn update_renditions(&self) { + let current_segment_idx = self.redis_state.current_segment_idx() + - if self.redis_state.current_fragment_idx() == 0 + && self.redis_state.current_segment_idx() != 0 + { + 1 + } else { + 0 + }; + let current_fragment_idx = self + .segment_state + .get(¤t_segment_idx) + .map(|(segment, _)| segment.fragments().len()) + .unwrap_or(0) as u32; + let current_fragment_idx = if current_fragment_idx != 0 { + current_fragment_idx - 1 + } else { + 0 + }; + + self.renditions + .set(&self.variant_id, current_segment_idx, current_fragment_idx); + } + async fn construct_init(&mut self, global: &Arc) -> Result<()> { if self.tracks.iter().any(|track| track.moov.is_none()) { return Ok(()); @@ -884,7 +914,7 @@ impl Variant { } segment_data.push_str(&format!( - "#EXT-X-PRELOAD-HINT:TYPE=PART,URI=\"{}.{}.mp4\"", + "#EXT-X-PRELOAD-HINT:TYPE=PART,URI=\"{}.{}.mp4\"\n", self.redis_state.current_segment_idx(), self.redis_state.current_fragment_idx() )); @@ -913,6 +943,20 @@ impl Variant { playlist.push_str(&segment_data); + playlist.push('\n'); + + for rendition in self + .renditions + .renditions() + .into_iter() + .filter(|rendition| rendition.id != self.variant_id) + { + playlist.push_str(&format!( + "#EXT-X-RENDITION-REPORT:URI=\"../{}/index.m3u8\",LAST-MSN={},LAST-PART={}\n", + rendition.id, rendition.last_msn, rendition.last_part + )); + } + self.redis_state.set_playlist(playlist); Ok(())