Replace reqwest with hyper

Cargo.toml

@@ -47,6 +47,7 @@ async-smtp = { version = "0.9", default-features = false, features = ["runtime-t
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
 base64 = { workspace = true }
 brotli = { version = "6", default-features=false, features = ["std"] }
+bytes = "1"
 chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
@@ -57,7 +58,10 @@ futures = { workspace = true }
 futures-lite = { workspace = true }
 hex = "0.4.0"
 hickory-resolver = "0.24"
+http-body-util = "0.1.2"
 humansize = "2"
+hyper = "1"
+hyper-util = "0.1.7"
 image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 iroh_old = { version = "0.4.2", default-features = false, package = "iroh"}
 iroh-net = { version = "0.22.0", default-features = false }

src/net/http.rs

@@ -2,12 +2,16 @@
 
 use std::sync::Arc;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Context as _, Result};
+use http_body_util::BodyExt;
+use hyper_util::rt::TokioIo;
 use mime::Mime;
 use once_cell::sync::Lazy;
 
 use crate::context::Context;
 use crate::net::lookup_host_with_cache;
+use crate::net::session::SessionStream;
+use crate::net::tls::wrap_tls;
 use crate::socks::Socks5Config;
 
 static LETSENCRYPT_ROOT: Lazy<reqwest::tls::Certificate> = Lazy::new(|| {
@@ -32,47 +36,77 @@ pub struct Response {
 
 /// Retrieves the text contents of URL using HTTP GET request.
 pub async fn read_url(context: &Context, url: &str) -> Result<String> {
-    Ok(read_url_inner(context, url).await?.text().await?)
+    let response = read_url_blob(context, url).await?;
+    let text = String::from_utf8_lossy(&response.blob);
+    Ok(text.to_string())
 }
 
-/// Retrieves the binary contents of URL using HTTP GET request.
-pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
-    let response = read_url_inner(context, url).await?;
-    let content_type = response
-        .headers()
-        .get(reqwest::header::CONTENT_TYPE)
-        .and_then(|value| value.to_str().ok())
-        .and_then(|value| value.parse::<Mime>().ok());
-    let mimetype = content_type
-        .as_ref()
-        .map(|mime| mime.essence_str().to_string());
-    let encoding = content_type.as_ref().and_then(|mime| {
-        mime.get_param(mime::CHARSET)
-            .map(|charset| charset.as_str().to_string())
-    });
-    let blob: Vec<u8> = response.bytes().await?.into();
-    Ok(Response {
-        blob,
-        mimetype,
-        encoding,
-    })
-}
+async fn get_http_sender<B>(
+    context: &Context,
+    parsed_url: hyper::Uri,
+) -> Result<hyper::client::conn::http1::SendRequest<B>>
+where
+    B: hyper::body::Body + 'static + Send,
+    B::Data: Send,
+    B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
+{
+    let scheme = parsed_url.scheme_str().context("URL has no scheme")?;
+    let host = parsed_url.host().context("URL has no host")?;
+
+    let stream: Box<dyn SessionStream> = match scheme {
+        "http" => {
+            let port = parsed_url.port_u16().unwrap_or(80);
+
+            // It is safe to use cached IP addresses
+            // for HTTPS URLs, but for HTTP URLs
+            // better resolve from scratch each time to prevent
+            // cache poisoning attacks from having lasting effects.
+            let load_cache = false;
+            let tcp_stream = crate::net::connect_tcp(context, host, port, load_cache).await?;
+            Box::new(tcp_stream)
+        }
+        "https" => {
+            let port = parsed_url.port_u16().unwrap_or(443);
+            let load_cache = true;
+            let tcp_stream = crate::net::connect_tcp(context, host, port, load_cache).await?;
+            let strict_tls = true;
+            let tls_stream = wrap_tls(strict_tls, host, &[], tcp_stream).await?;
+            Box::new(tls_stream)
+        }
+        _ => bail!("Unknown URL scheme"),
+    };
+
+    let io = TokioIo::new(stream);
+    let (sender, conn) = hyper::client::conn::http1::handshake(io).await?;
+    tokio::task::spawn(conn);
 
-async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Response> {
-    // It is safe to use cached IP addresses
-    // for HTTPS URLs, but for HTTP URLs
-    // better resolve from scratch each time to prevent
-    // cache poisoning attacks from having lasting effects.
-    let load_cache = url.starts_with("https://");
+    Ok(sender)
+}
 
-    let client = get_client(context, load_cache).await?;
+/// Retrieves the binary contents of URL using HTTP GET request.
+pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
     let mut url = url.to_string();
 
     // Follow up to 10 http-redirects
     for _i in 0..10 {
-        let response = client.get(&url).send().await?;
+        let parsed_url = url
+            .parse::<hyper::Uri>()
+            .with_context(|| format!("Failed to parse URL {url:?}"))?;
+
+        let mut sender = get_http_sender(context, parsed_url.clone()).await?;
+        let authority = parsed_url
+            .authority()
+            .context("URL has no authority")?
+            .clone();
+
+        let req = hyper::Request::builder()
+            .uri(parsed_url.path())
+            .header(hyper::header::HOST, authority.as_str())
+            .body(http_body_util::Empty::<bytes::Bytes>::new())?;
+        let response = sender.send_request(req).await?;
+
+        let headers = response.headers();
         if response.status().is_redirection() {
-            let headers = response.headers();
             let header = headers
                 .get_all("location")
                 .iter()
@@ -84,7 +118,25 @@ async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Respons
             continue;
         }
 
-        return Ok(response);
+        let content_type = response
+            .headers()
+            .get("content-type")
+            .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.parse::<Mime>().ok());
+        let mimetype = content_type
+            .as_ref()
+            .map(|mime| mime.essence_str().to_string());
+        let encoding = content_type.as_ref().and_then(|mime| {
+            mime.get_param(mime::CHARSET)
+                .map(|charset| charset.as_str().to_string())
+        });
+        let body = response.collect().await?.to_bytes();
+        let blob: Vec<u8> = body.to_vec();
+        return Ok(Response {
+            blob,
+            mimetype,
+            encoding,
+        });
     }
 
     Err(anyhow!("Followed 10 redirections"))