diff --git a/.gitignore b/.gitignore index 3df91dd..535cac8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ +.vscode/ /target *.sock diff --git a/Cargo.lock b/Cargo.lock index d407d96..a39ea14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1551,6 +1551,7 @@ dependencies = [ "regex", "reqwest", "rgb", + "thiserror 2.0.3", "tokio", ] @@ -1689,7 +1690,7 @@ dependencies = [ "rand_chacha", "simd_helpers", "system-deps", - "thiserror", + "thiserror 1.0.69", "v_frame", "wasm-bindgen", ] @@ -2176,7 +2177,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", ] [[package]] @@ -2190,6 +2200,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.36" @@ -2288,7 +2309,7 @@ checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", - "thiserror", + "thiserror 1.0.69", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 21902e0..84b906a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ blake3 = { version = "1.5.1", optional = true } bytes = "1.6.0" futures-util = "0.3.30" listenfd = "1.0.1" +thiserror = { version = "2.0.3", optional = true } [features] default = ["webp", "mimalloc", "reqwest-rustls", "qhash"] @@ -34,8 +35,8 @@ default = ["webp", "mimalloc", "reqwest-rustls", "qhash"] reqwest-rustls = ["reqwest/rustls-tls"] reqwest-native-tls = ["reqwest/default-tls"] -avif = ["dep:ravif", "dep:rgb", "dep:image"] -webp = ["dep:libwebp-sys", "dep:image"] +avif = ["dep:ravif", "dep:rgb", "dep:image", "dep:thiserror"] +webp = ["dep:libwebp-sys", "dep:image", "dep:thiserror"] mimalloc = ["dep:mimalloc"] @@ -43,5 +44,7 @@ optimized = ["libwebp-sys?/sse41", "libwebp-sys?/avx2", "libwebp-sys?/neon"] qhash = ["blake3"] +invidious = [] + [profile.release] lto = true diff --git a/README.md b/README.md index 9b719b6..22bfe56 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # piped-proxy -A proxy for Piped written in Rust, meant to superseed [http3-ytproxy](https://github.com/TeamPiped/http3-ytproxy). +A proxy for Piped written in Rust, meant to supersede [http3-ytproxy](https://github.com/TeamPiped/http3-ytproxy). diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 0000000..c75fa29 --- /dev/null +++ b/src/client.rs @@ -0,0 +1,51 @@ +use actix_web::HttpRequest; +use once_cell::sync::Lazy; +use reqwest::{Client, Method, Request, Url}; +use std::env; +use std::net::{IpAddr, Ipv4Addr}; + +use crate::headers::is_header_allowed; + +pub static CLIENT: Lazy = Lazy::new(|| { + let builder = Client::builder() + .user_agent("Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"); + + let proxy = if let Ok(proxy) = env::var("PROXY") { + reqwest::Proxy::all(proxy).ok() + } else { + None + }; + + let builder = if let Some(proxy) = proxy { + // proxy basic auth + if let Ok(proxy_auth_user) = env::var("PROXY_USER") { + let proxy_auth_pass = env::var("PROXY_PASS").unwrap_or_default(); + builder.proxy(proxy.basic_auth(&proxy_auth_user, &proxy_auth_pass)) + } else { + builder.proxy(proxy) + } + } else { + builder + }; + + if crate::utils::get_env_bool("IPV4_ONLY") { + builder.local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) + } else { + builder + } + .build() + .unwrap() +}); + +pub fn create_request(req: &HttpRequest, method: Method, url: Url) -> Request { + let mut request = Request::new(method, url); + let request_headers = request.headers_mut(); + + for (key, value) in req.headers() { + if is_header_allowed(key.as_str()) { + request_headers.insert(key, value.clone()); + } + } + + request +} diff --git a/src/headers.rs b/src/headers.rs new file mode 100644 index 0000000..5f77cd4 --- /dev/null +++ b/src/headers.rs @@ -0,0 +1,52 @@ +use actix_web::HttpResponseBuilder; +use reqwest::{header::HeaderMap, Response}; + +pub fn add_headers(response: &mut HttpResponseBuilder) { + response + .append_header(("Access-Control-Allow-Origin", "*")) + .append_header(("Access-Control-Allow-Headers", "*")) + .append_header(("Access-Control-Allow-Methods", "*")) + .append_header(("Access-Control-Max-Age", "1728000")); +} + +pub fn is_header_allowed(header: &str) -> bool { + if header.starts_with("access-control") { + return false; + } + + !matches!( + header, + "host" + | "content-length" + | "set-cookie" + | "alt-svc" + | "accept-ch" + | "report-to" + | "strict-transport-security" + | "user-agent" + | "range" + | "transfer-encoding" + | "x-real-ip" + | "origin" + | "referer" + // the 'x-title' header contains non-ascii characters which is not allowed on some HTTP clients + | "x-title" + ) +} + +pub fn get_content_length(headers: &HeaderMap) -> Option { + headers + .get("content-length") + .and_then(|cl| cl.to_str().ok()) + .and_then(|cl| str::parse::(cl).ok()) +} + +pub fn copy_response_headers(req_resp: &Response, http_resp: &mut HttpResponseBuilder) { + add_headers(http_resp); + + for (key, value) in req_resp.headers() { + if is_header_allowed(key.as_str()) { + http_resp.append_header((key.as_str(), value.as_bytes())); + } + } +} diff --git a/src/main.rs b/src/main.rs index d2b51dd..e67429a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,22 @@ +mod client; +mod headers; +#[cfg(feature = "invidious")] +mod proxy_image; +#[cfg(any(feature = "webp", feature = "avif"))] +mod transcode_image; mod ump_stream; mod utils; use actix_web::http::{Method, StatusCode}; -use actix_web::{web, App, HttpRequest, HttpResponse, HttpResponseBuilder, HttpServer}; +#[cfg(any(feature = "webp", feature = "avif"))] +use actix_web::Either; +use actix_web::{web, App, HttpRequest, HttpResponse, HttpServer}; +use client::create_request; use listenfd::ListenFd; use once_cell::sync::Lazy; use qstring::QString; use regex::Regex; -use reqwest::{Body, Client, Request, Url}; +use reqwest::{Body, Url}; use std::error::Error; use std::io::ErrorKind; use std::net::TcpListener; @@ -22,6 +31,11 @@ use futures_util::TryStreamExt; use tokio::task::spawn_blocking; use ump_stream::UmpTransformStream; +use crate::client::CLIENT; +use crate::headers::{add_headers, copy_response_headers, get_content_length}; +#[cfg(any(feature = "webp", feature = "avif"))] +use crate::transcode_image::{transcode_image, DISALLOW_IMAGE_TRANSCODING}; + #[cfg(feature = "mimalloc")] #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; @@ -96,37 +110,6 @@ static RE_MANIFEST: Lazy = Lazy::new(|| Regex::new("(?m)URI=\"([^\"]+)\"" static RE_DASH_MANIFEST: Lazy = Lazy::new(|| Regex::new("BaseURL>(https://[^<]+) = Lazy::new(|| { - let builder = Client::builder() - .user_agent("Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"); - - let proxy = if let Ok(proxy) = env::var("PROXY") { - reqwest::Proxy::all(proxy).ok() - } else { - None - }; - - let builder = if let Some(proxy) = proxy { - // proxy basic auth - if let Ok(proxy_auth_user) = env::var("PROXY_USER") { - let proxy_auth_pass = env::var("PROXY_PASS").unwrap_or_default(); - builder.proxy(proxy.basic_auth(&proxy_auth_user, &proxy_auth_pass)) - } else { - builder.proxy(proxy) - } - } else { - builder - }; - - if utils::get_env_bool("IPV4_ONLY") { - builder.local_address("0.0.0.0".parse().ok()) - } else { - builder - } - .build() - .unwrap() -}); - const ANDROID_USER_AGENT: &str = "com.google.android.youtube/1537338816 (Linux; U; Android 13; en_US; ; Build/TQ2A.230505.002; Cronet/113.0.5672.24)"; const ALLOWED_DOMAINS: [&str; 8] = [ "youtube.com", @@ -139,39 +122,6 @@ const ALLOWED_DOMAINS: [&str; 8] = [ "ajay.app", ]; -fn add_headers(response: &mut HttpResponseBuilder) { - response - .append_header(("Access-Control-Allow-Origin", "*")) - .append_header(("Access-Control-Allow-Headers", "*")) - .append_header(("Access-Control-Allow-Methods", "*")) - .append_header(("Access-Control-Max-Age", "1728000")); -} - -fn is_header_allowed(header: &str) -> bool { - if header.starts_with("access-control") { - return false; - } - - !matches!( - header, - "host" - | "content-length" - | "set-cookie" - | "alt-svc" - | "accept-ch" - | "report-to" - | "strict-transport-security" - | "user-agent" - | "range" - | "transfer-encoding" - | "x-real-ip" - | "origin" - | "referer" - // the 'x-title' header contains non-ascii characters which is not allowed on some HTTP clients - | "x-title" - ) -} - async fn index(req: HttpRequest) -> Result> { if req.method() == Method::OPTIONS { let mut response = HttpResponse::Ok(); @@ -183,6 +133,13 @@ async fn index(req: HttpRequest) -> Result> { return Ok(response.finish()); } + #[cfg(feature = "invidious")] + if matches!(req.path().get(0..4), Some("/vi/") | Some("/sb/")) { + return proxy_image::proxy(req, proxy_image::ImageSource::YtImg).await; + } else if req.path().starts_with("/ggpht/") { + return proxy_image::proxy(req, proxy_image::ImageSource::GgPht).await; + } + // parse query string let mut query = QString::from(req.query_string()); @@ -256,14 +213,8 @@ async fn index(req: HttpRequest) -> Result> { return Err("No host provided".into()); }; - #[cfg(any(feature = "webp", feature = "avif"))] - let disallow_image_transcoding = utils::get_env_bool("DISALLOW_IMAGE_TRANSCODING"); - let rewrite = query.get("rewrite") != Some("false"); - #[cfg(feature = "avif")] - let avif = query.get("avif") == Some("true"); - let Some(domain) = RE_DOMAIN .captures(host.as_str()) .map(|domain| domain.get(1).unwrap().as_str()) @@ -313,6 +264,9 @@ async fn index(req: HttpRequest) -> Result> { } } + #[cfg(feature = "avif")] + let avif = query.get("avif") == Some("true"); + let range = query.get("range").map(|s| s.to_string()); let qs = { @@ -335,116 +289,42 @@ async fn index(req: HttpRequest) -> Result> { } }; - let mut request = Request::new(method, url); + let mut request = create_request(&req, method, url); if is_web && video_playback { request.body_mut().replace(Body::from("x\0")); } - let request_headers = request.headers_mut(); - - for (key, value) in req.headers() { - if is_header_allowed(key.as_str()) { - request_headers.insert(key, value.clone()); - } - } - if is_android { - request_headers.insert("User-Agent", ANDROID_USER_AGENT.parse().unwrap()); + request + .headers_mut() + .insert("User-Agent", ANDROID_USER_AGENT.parse().unwrap()); } - let resp = CLIENT.execute(request).await?; + #[cfg_attr(not(any(feature = "webp", feature = "avif")), allow(unused_mut))] + let mut resp = CLIENT.execute(request).await?; let mut response = HttpResponse::build(resp.status()); - add_headers(&mut response); - - for (key, value) in resp.headers() { - if is_header_allowed(key.as_str()) { - response.append_header((key.as_str(), value.as_bytes())); - } - } + copy_response_headers(&resp, &mut response); if rewrite { - if let Some(content_type) = resp.headers().get("content-type") { - #[cfg(feature = "avif")] - if !disallow_image_transcoding - && (content_type == "image/webp" || content_type == "image/jpeg" && avif) + #[cfg(any(feature = "webp", feature = "avif"))] + if !*DISALLOW_IMAGE_TRANSCODING { + match transcode_image( + resp, + &mut response, + #[cfg(feature = "avif")] + avif, + ) + .await? { - let resp_bytes = resp.bytes().await.unwrap(); - let (body, content_type) = spawn_blocking(|| { - use ravif::{Encoder, Img}; - use rgb::FromSlice; - - let image = image::load_from_memory(&resp_bytes).unwrap(); - - let width = image.width() as usize; - let height = image.height() as usize; - - let buf = image.into_rgb8(); - let buf = buf.as_raw().as_rgb(); - - let buffer = Img::new(buf, width, height); - - let res = Encoder::new() - .with_quality(80f32) - .with_speed(7) - .encode_rgb(buffer); - - if let Ok(res) = res { - (res.avif_file.to_vec(), "image/avif") - } else { - (resp_bytes.into(), "image/jpeg") - } - }) - .await - .unwrap(); - response.content_type(content_type); - return Ok(response.body(body)); - } - - #[cfg(feature = "webp")] - if !disallow_image_transcoding && content_type == "image/jpeg" { - let resp_bytes = resp.bytes().await.unwrap(); - let (body, content_type) = spawn_blocking(|| { - use libwebp_sys::{WebPEncodeRGB, WebPFree}; - - let image = image::load_from_memory(&resp_bytes).unwrap(); - let width = image.width(); - let height = image.height(); - - let quality = 85; - - let data = image.as_rgb8().unwrap().as_raw(); - - let bytes: Vec = unsafe { - let mut out_buf = std::ptr::null_mut(); - let stride = width as i32 * 3; - let len: usize = WebPEncodeRGB( - data.as_ptr(), - width as i32, - height as i32, - stride, - quality as f32, - &mut out_buf, - ); - let vec = std::slice::from_raw_parts(out_buf, len).into(); - WebPFree(out_buf as *mut _); - vec - }; - - if bytes.len() < resp_bytes.len() { - (bytes, "image/webp") - } else { - (resp_bytes.into(), "image/jpeg") - } - }) - .await - .unwrap(); - response.content_type(content_type); - return Ok(response.body(body)); + Either::Left(http_response) => return Ok(http_response), + Either::Right(image_response) => resp = image_response, } + } + if let Some(content_type) = resp.headers().get("content-type") { if content_type == "application/x-mpegurl" || content_type == "application/vnd.apple.mpegurl" { @@ -470,6 +350,7 @@ async fn index(req: HttpRequest) -> Result> { return Ok(response.body(modified)); } + if content_type == "video/vnd.mpeg.dash.mpd" || content_type == "application/dash+xml" { let resp_str = resp.text().await.unwrap(); let mut new_resp = resp_str.clone(); @@ -485,8 +366,8 @@ async fn index(req: HttpRequest) -> Result> { } } - if let Some(content_length) = resp.headers().get("content-length") { - response.no_chunking(content_length.to_str().unwrap().parse::().unwrap()); + if let Some(content_length) = get_content_length(resp.headers()) { + response.no_chunking(content_length); } if is_ump && resp.status().is_success() { diff --git a/src/proxy_image.rs b/src/proxy_image.rs new file mode 100644 index 0000000..a69a3c3 --- /dev/null +++ b/src/proxy_image.rs @@ -0,0 +1,148 @@ +#[cfg(any(feature = "webp", feature = "avif"))] +use actix_web::Either; +use actix_web::{HttpRequest, HttpResponse}; +use once_cell::sync::Lazy; +use reqwest::Url; + +use crate::client::{create_request, CLIENT}; +use crate::headers::{copy_response_headers, get_content_length}; +#[cfg(any(feature = "webp", feature = "avif"))] +use crate::transcode_image::{transcode_image, DISALLOW_IMAGE_TRANSCODING}; + +pub async fn proxy( + req: HttpRequest, + src: ImageSource, +) -> Result> { + let mut resp = get_image(&req, src).await?; + + let mut response = HttpResponse::build(resp.status()); + + copy_response_headers(&resp, &mut response); + + if !*DISALLOW_IMAGE_TRANSCODING { + match transcode_image( + resp, + &mut response, + #[cfg(feature = "avif")] + true, + ) + .await? + { + Either::Left(http_response) => return Ok(http_response), + Either::Right(image_response) => resp = image_response, + } + } + + if let Some(content_length) = get_content_length(resp.headers()) { + response.no_chunking(content_length); + } + + Ok(response.streaming(resp.bytes_stream())) +} + +#[derive(PartialEq)] +pub enum ImageSource { + YtImg, + GgPht, +} + +impl ImageSource { + fn get_base_url(&self) -> Url { + static YTIMG_URL: Lazy = + Lazy::new(|| Url::parse("https://i.ytimg.com").expect("Invalid ytimg URL")); + static GGPHT_URL: Lazy = + Lazy::new(|| Url::parse("https://yt3.ggpht.com").expect("Invalid ggpht URL")); + + match self { + Self::YtImg => YTIMG_URL.clone(), + Self::GgPht => GGPHT_URL.clone(), + } + } + + fn strip_path_prefix<'p>(&self, path: &'p str) -> &'p str { + const GGPHT_PREFIX_LEN: usize = "/ggpht".len(); + + match self { + Self::YtImg => path, + Self::GgPht => &path[GGPHT_PREFIX_LEN..], + } + } +} + +const MAX_RES_SEGMENT: &str = "/maxres.jpg"; + +async fn get_image(req: &HttpRequest, src: ImageSource) -> reqwest::Result { + let req_uri = req.uri(); + + let mut url = src.get_base_url(); + url.set_query(req_uri.query()); + + let path = src.strip_path_prefix(req_uri.path()); + + if src == ImageSource::YtImg && path.ends_with(MAX_RES_SEGMENT) { + get_max_res_thumbnail(req, path, url).await + } else { + url.set_path(path); + CLIENT + .execute(create_request(req, req.method().clone(), url)) + .await + } +} + +async fn get_max_res_thumbnail( + req: &HttpRequest, + req_path: &str, + mut proxy_url: Url, +) -> reqwest::Result { + const FORMATS: &[&str] = &["/maxresdefault.jpg", "/sddefault.jpg", "/hqdefault.jpg"]; + const DEFAULT_FORMAT: &str = "/mqdefault.jpg"; + const FORMAT_MAX_LENGTH: usize = get_formats_max_length(FORMATS, DEFAULT_FORMAT); + + let path_without_format_len = req_path.len() - MAX_RES_SEGMENT.len(); + + let mut path = String::with_capacity(path_without_format_len + FORMAT_MAX_LENGTH); + path.push_str(&req_path[..path_without_format_len]); + + for format in FORMATS { + path.push_str(format); + + let mut url = proxy_url.clone(); + url.set_path(&path); + + if let Ok(res) = CLIENT + .execute(create_request(req, req.method().clone(), url)) + .await + { + if res.status() == 200 { + return Ok(res); + } + } + + path.truncate(path_without_format_len); + } + + path.push_str(DEFAULT_FORMAT); + proxy_url.set_path(&path); + + CLIENT + .execute(create_request(req, req.method().clone(), proxy_url)) + .await +} + +const fn get_formats_max_length(formats: &[&str], default_format: &str) -> usize { + let mut max_len = default_format.len(); + let mut i = 0; + let size = formats.len(); + + while i < size { + let len = formats[i].len(); + + if len > max_len { + max_len = len; + } + + i += 1; + } + + max_len +} diff --git a/src/transcode_image.rs b/src/transcode_image.rs new file mode 100644 index 0000000..ef8cbfe --- /dev/null +++ b/src/transcode_image.rs @@ -0,0 +1,114 @@ +use actix_web::{Either, HttpResponse, HttpResponseBuilder}; +use image::ImageError; +use once_cell::sync::Lazy; +use reqwest::Response; +use tokio::task::spawn_blocking; + +pub static DISALLOW_IMAGE_TRANSCODING: Lazy = + Lazy::new(|| crate::utils::get_env_bool("DISALLOW_IMAGE_TRANSCODING")); + +#[derive(Debug, thiserror::Error)] +enum ImageTranscodingError { + #[error("Image loading error: {0}")] + ImageLoadingError(#[from] ImageError), + #[cfg(feature = "webp")] + #[error("Image is not an 8bit RGB")] + NotAnRgb8Image, +} + +pub async fn transcode_image( + image_response: Response, + http_response: &mut HttpResponseBuilder, + #[cfg(feature = "avif")] avif: bool, +) -> Result, Box> { + let Some(content_type) = image_response.headers().get("content-type") else { + return Ok(Either::Right(image_response)); + }; + + #[cfg(feature = "avif")] + if content_type == "image/webp" || content_type == "image/jpeg" && avif { + let resp_bytes = image_response.bytes().await?; + + let (body, content_type) = spawn_blocking( + || -> Result<(Vec, &'static str), ImageTranscodingError> { + use ravif::{Encoder, Img}; + use rgb::FromSlice; + + let image = image::load_from_memory(&resp_bytes)?; + + let width = image.width() as usize; + let height = image.height() as usize; + + let buf = image.into_rgb8(); + let buf = buf.as_raw().as_rgb(); + + let buffer = Img::new(buf, width, height); + + let res = Encoder::new() + .with_quality(80f32) + .with_speed(7) + .encode_rgb(buffer); + + if let Ok(res) = res { + Ok((res.avif_file, "image/avif")) + } else { + Ok((resp_bytes.into(), "image/jpeg")) + } + }, + ) + .await??; + + http_response.content_type(content_type); + return Ok(Either::Left(http_response.body(body))); + } + + #[cfg(feature = "webp")] + if content_type == "image/jpeg" { + let resp_bytes = image_response.bytes().await?; + + let (body, content_type) = spawn_blocking( + || -> Result<(Vec, &'static str), ImageTranscodingError> { + use libwebp_sys::{WebPEncodeRGB, WebPFree}; + + let image = image::load_from_memory(&resp_bytes)?; + let width = image.width(); + let height = image.height(); + + let quality = 85; + + let data = image + .as_rgb8() + .ok_or(ImageTranscodingError::NotAnRgb8Image)? + .as_raw(); + + let bytes: Vec = unsafe { + let mut out_buf = std::ptr::null_mut(); + let stride = width as i32 * 3; + let len: usize = WebPEncodeRGB( + data.as_ptr(), + width as i32, + height as i32, + stride, + quality as f32, + &mut out_buf, + ); + let vec = std::slice::from_raw_parts(out_buf, len).into(); + WebPFree(out_buf as *mut _); + vec + }; + + if bytes.len() < resp_bytes.len() { + Ok((bytes, "image/webp")) + } else { + Ok((resp_bytes.into(), "image/jpeg")) + } + }, + ) + .await??; + + http_response.content_type(content_type); + return Ok(Either::Left(http_response.body(body))); + } + + Ok(Either::Right(image_response)) +}