diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ae4d36c..6911490e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,8 @@ set(LIBPOWERLOADER_SRCS ${POWERLOADER_SOURCE_DIR}/target.cpp ${POWERLOADER_SOURCE_DIR}/url.cpp ${POWERLOADER_SOURCE_DIR}/utils.cpp + ${POWERLOADER_SOURCE_DIR}/target.hpp + ${POWERLOADER_SOURCE_DIR}/curl_internal.hpp ) if (WITH_ZCHUNK) list(APPEND LIBPOWERLOADER_SRCS ${POWERLOADER_SOURCE_DIR}/zck.cpp diff --git a/include/powerloader/context.hpp b/include/powerloader/context.hpp index 5676422a..fe1fb29a 100644 --- a/include/powerloader/context.hpp +++ b/include/powerloader/context.hpp @@ -11,13 +11,14 @@ #include #include +#include namespace powerloader { namespace fs = std::filesystem; class Context; - struct Mirror; + class Mirror; using mirror_set = std::vector>; // TODO: replace by std::flat_set once available. @@ -86,6 +87,15 @@ namespace powerloader void reset(mirror_map_base new_values = {}); }; + using proxy_map_type = std::map; + + // Options provided when starting a powerloader context. + struct ContextOptions + { + // If set, specifies which SSL backend to use with CURL. + std::optional ssl_backend; + }; + class POWERLOADER_API Context { public: @@ -122,19 +132,24 @@ namespace powerloader std::chrono::steady_clock::duration retry_default_timeout = std::chrono::seconds(2); mirror_map_type mirror_map; + proxy_map_type proxy_map; std::vector additional_httpheaders; void set_verbosity(int v); // Throws if another instance already exists: there can only be one at any time! - Context(); + Context(ContextOptions options = {}); ~Context(); Context(const Context&) = delete; Context& operator=(const Context&) = delete; Context(Context&&) = delete; Context& operator=(Context&&) = delete; + + private: + struct Impl; + std::unique_ptr impl; // Private implementation details }; } diff --git a/include/powerloader/curl.hpp b/include/powerloader/curl.hpp index 79508a23..6f9d38d0 100644 --- a/include/powerloader/curl.hpp +++ b/include/powerloader/curl.hpp @@ -13,6 +13,11 @@ #include #include +extern "C" +{ +#include +} + #include #include #include @@ -21,8 +26,25 @@ namespace powerloader { class Context; class CURLHandle; + using proxy_map_type = std::map; -#include + enum class ssl_backend_t + { + none = CURLSSLBACKEND_NONE, + openssl = CURLSSLBACKEND_OPENSSL, + gnutls = CURLSSLBACKEND_GNUTLS, + nss = CURLSSLBACKEND_NSS, + gskit = CURLSSLBACKEND_GSKIT, + // polarssl = CURLSSLBACKEND_POLARSSL /* deprecated by curl */, + wolfssl = CURLSSLBACKEND_WOLFSSL, + schannel = CURLSSLBACKEND_SCHANNEL, + securetransport = CURLSSLBACKEND_SECURETRANSPORT, + // axtls = CURLSSLBACKEND_AXTLS, /* deprecated by curl */ + mbedtls = CURLSSLBACKEND_MBEDTLS, + // mesalink = CURLSSLBACKEND_MESALINK, /* deprecated by curl */ + bearssl = CURLSSLBACKEND_BEARSSL, + rustls = CURLSSLBACKEND_RUSTLS, + }; class POWERLOADER_API curl_error : public std::runtime_error { @@ -66,7 +88,7 @@ namespace powerloader CURLHandle(const Context& ctx, const std::string& url); ~CURLHandle(); - CURLHandle& url(const std::string& url); + CURLHandle& url(const std::string& url, const proxy_map_type& proxies); CURLHandle& accept_encoding(); CURLHandle& user_agent(const std::string& user_agent); @@ -134,6 +156,8 @@ namespace powerloader } return *this; } + + std::optional proxy_match(const proxy_map_type& ctx, const std::string& url); } #endif diff --git a/include/powerloader/mirror.hpp b/include/powerloader/mirror.hpp index b29b4459..27c2f8b8 100644 --- a/include/powerloader/mirror.hpp +++ b/include/powerloader/mirror.hpp @@ -62,11 +62,29 @@ namespace powerloader } }; + inline std::string strip_trailing_slash(const std::string& s) + { + if (s.size() > 0 && s.back() == '/' && s != "file://") + { + return s.substr(0, s.size() - 1); + } + return s; + } + // mirrors should be dict -> urls mapping - struct POWERLOADER_API Mirror + class POWERLOADER_API Mirror { - Mirror(MirrorID id, const Context& ctx, const std::string& url); - Mirror(const Context& ctx, const std::string& url); + public: + Mirror(const MirrorID& id, const Context& ctx, const std::string& url) + : m_id(id) + , m_url(strip_trailing_slash(url)) + { + if (ctx.max_downloads_per_mirror > 0) + { + m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror; + } + } + virtual ~Mirror(); Mirror(const Mirror&) = delete; @@ -156,8 +174,8 @@ namespace powerloader } private: - std::string m_url; const MirrorID m_id; + const std::string m_url; Protocol m_protocol = Protocol::kHTTP; MirrorState m_state = MirrorState::READY; @@ -180,6 +198,28 @@ namespace powerloader std::size_t m_retry_counter = 0; }; + class POWERLOADER_API HTTPMirror : public Mirror + { + public: + HTTPMirror(const Context& ctx, const std::string& url) + : Mirror(HTTPMirror::id(url), ctx, url) + { + } + + static MirrorID id(const std::string& url) + { + return MirrorID{ fmt::format("HTTPMirror[{}]", url) }; + } + + void set_auth(const std::string& user, const std::string& password); + + bool authenticate(CURLHandle& handle, const std::string& path) override; + + private: + std::string m_auth_user; + std::string m_auth_password; + }; + bool sort_mirrors(std::vector>& mirrors, const std::shared_ptr& mirror, bool success, diff --git a/include/powerloader/mirrors/oci.hpp b/include/powerloader/mirrors/oci.hpp index 538a1e2a..7297b66b 100644 --- a/include/powerloader/mirrors/oci.hpp +++ b/include/powerloader/mirrors/oci.hpp @@ -69,6 +69,9 @@ namespace powerloader std::string m_password; split_function_type m_split_func; + // we copy over the proxy map from the context, otherwise we can't set new + // proxy options for each curl handle + proxy_map_type m_proxy_map; std::pair split_path_tag(const std::string& path) const; diff --git a/src/cli/main.cpp b/src/cli/main.cpp index 89acd5cd..6947b381 100644 --- a/src/cli/main.cpp +++ b/src/cli/main.cpp @@ -392,7 +392,7 @@ parse_mirrors(const Context& ctx, const YAML::Node& node) else if (kof == KindOf::kHTTP) { spdlog::info("Adding HTTP mirror: {} -> {}", mirror_name, creds.url.url()); - result.create_unique_mirror(mirror_name, ctx, creds.url.url()); + result.create_unique_mirror(mirror_name, ctx, creds.url.url()); } } } diff --git a/src/context.cpp b/src/context.cpp index 2ab6baaa..83e86525 100644 --- a/src/context.cpp +++ b/src/context.cpp @@ -14,18 +14,31 @@ extern "C" #include +#include "./curl_internal.hpp" + namespace powerloader { + struct Context::Impl + { + std::optional curl_setup; + }; + static std::atomic is_context_alive{ false }; - Context::Context() + Context::Context(ContextOptions options) + : impl(new Impl) { bool expected = false; if (!is_context_alive.compare_exchange_strong(expected, true)) throw std::runtime_error( "powerloader::Context created more than once - instance must be unique"); + if (options.ssl_backend) + { + impl->curl_setup = details::CURLSetup{ options.ssl_backend.value() }; + } + cache_dir = fs::absolute(fs::path(".pdcache")); if (!fs::exists(cache_dir)) { diff --git a/src/curl.cpp b/src/curl.cpp index 9d402643..f88df136 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace powerloader { @@ -24,6 +25,7 @@ namespace powerloader return m_serious; } + /************** * CURLHandle* **************/ @@ -54,11 +56,18 @@ namespace powerloader if (ctx.disable_ssl) { + spdlog::warn("SSL verification is disabled"); setopt(CURLOPT_SSL_VERIFYHOST, 0); setopt(CURLOPT_SSL_VERIFYPEER, 0); + + // also disable proxy SSL verification + setopt(CURLOPT_PROXY_SSL_VERIFYPEER, 0L); + setopt(CURLOPT_PROXY_SSL_VERIFYHOST, 0L); } else { + spdlog::warn("SSL verification is ENABLED"); + setopt(CURLOPT_SSL_VERIFYHOST, 2); setopt(CURLOPT_SSL_VERIFYPEER, 1); @@ -66,16 +75,16 @@ namespace powerloader CURLcode verifystatus = curl_easy_setopt(m_handle, CURLOPT_SSL_VERIFYSTATUS, 0); if (verifystatus != CURLE_OK && verifystatus != CURLE_NOT_BUILT_IN) throw curl_error("Could not initialize CURL handle"); - } - if (!ctx.ssl_ca_info.empty()) - { - setopt(CURLOPT_CAINFO, ctx.ssl_ca_info.c_str()); - } + if (!ctx.ssl_ca_info.empty()) + { + setopt(CURLOPT_CAINFO, ctx.ssl_ca_info.c_str()); + } - if (ctx.ssl_no_revoke) - { - setopt(CURLOPT_SSL_OPTIONS, ctx.ssl_no_revoke); + if (ctx.ssl_no_revoke) + { + setopt(CURLOPT_SSL_OPTIONS, ctx.ssl_no_revoke); + } } setopt(CURLOPT_FTP_USE_EPSV, (long) ctx.ftp_use_seepsv); @@ -88,7 +97,7 @@ namespace powerloader CURLHandle::CURLHandle(const Context& ctx, const std::string& url) : CURLHandle(ctx) { - this->url(url); + this->url(url, ctx.proxy_map); } CURLHandle::~CURLHandle() @@ -123,9 +132,18 @@ namespace powerloader return *this; } - CURLHandle& CURLHandle::url(const std::string& url) + CURLHandle& CURLHandle::url(const std::string& url, const proxy_map_type& proxies) { setopt(CURLOPT_URL, url.c_str()); + const auto match = proxy_match(proxies, url); + if (match) + { + setopt(CURLOPT_PROXY, match.value().c_str()); + } + else + { + setopt(CURLOPT_PROXY, nullptr); + } return *this; } @@ -373,4 +391,42 @@ namespace powerloader downloaded_size = handle.getinfo(CURLINFO_SIZE_DOWNLOAD_T).value(); } + + std::optional proxy_match(const proxy_map_type& proxies, const std::string& url) + { + // This is a reimplementation of requests.utils.select_proxy() + // of the python requests library used by conda + if (proxies.empty()) + { + return std::nullopt; + } + + auto handler = URLHandler(url); + auto scheme = handler.scheme(); + auto host = handler.host(); + std::vector options; + + if (host.empty()) + { + options = { + scheme, + "all", + }; + } + else + { + options = { scheme + "://" + host, scheme, "all://" + host, "all" }; + } + + for (auto& option : options) + { + auto proxy = proxies.find(option); + if (proxy != proxies.end()) + { + return proxy->second; + } + } + + return std::nullopt; + } } diff --git a/src/curl_internal.hpp b/src/curl_internal.hpp new file mode 100644 index 00000000..1e957f98 --- /dev/null +++ b/src/curl_internal.hpp @@ -0,0 +1,43 @@ +#ifndef POWERLOADER_SRC_CURL_INTERNAL_HPP +#define POWERLOADER_SRC_CURL_INTERNAL_HPP + +#include + +namespace powerloader::details +{ + // Scoped initialization and termination of CURL. + // This should never have more than one instance live at any time. + class CURLSetup final + { + public: + explicit CURLSetup(const ssl_backend_t& ssl_backend) + { + const auto res = curl_global_sslset((curl_sslbackend) ssl_backend, nullptr, nullptr); + if (res == CURLSSLSET_UNKNOWN_BACKEND) + { + throw curl_error("unknown curl ssl backend"); + } + else if (res == CURLSSLSET_NO_BACKENDS) + { + throw curl_error("no curl ssl backend available"); + } + else if (res == CURLSSLSET_TOO_LATE) + { + throw curl_error("curl ssl backend set too late"); + } + else if (res != CURLSSLSET_OK) + { + throw curl_error("failed to set curl ssl backend"); + } + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) + throw curl_error("failed to initialize curl"); + } + + ~CURLSetup() + { + curl_global_cleanup(); + } + }; +} +#endif diff --git a/src/download_target.cpp b/src/download_target.cpp index 3c955c70..5e9c4861 100644 --- a/src/download_target.cpp +++ b/src/download_target.cpp @@ -46,7 +46,7 @@ namespace powerloader URLHandler uh{ target_url }; if (uh.scheme() == "file") { - ctx.mirror_map.create_unique_mirror("[file]", ctx, "file://"); + ctx.mirror_map.create_unique_mirror("[file]", ctx, "file://"); return std::make_shared(uh.path(), "[file]", destination_path); } @@ -57,7 +57,12 @@ namespace powerloader const fs::path dst = destination_path.empty() ? fs::path{ rsplit(path, "/", 1).back() } : destination_path; - ctx.mirror_map.create_unique_mirror(host, ctx, mirror_url); + if (!ctx.mirror_map.has_mirrors(host)) + { + // Only when there is zero mirrors associated with the host, we still need at least + // one mirror to exist, so we create one as a fallback. + ctx.mirror_map.create_unique_mirror(host, ctx, mirror_url); + } return std::make_shared(path.substr(1, std::string::npos), host, dst); } else diff --git a/src/downloader.cpp b/src/downloader.cpp index 4911c18f..1a181543 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -571,7 +571,6 @@ namespace powerloader bool retry = false; spdlog::error("Error during transfer"); - // Call mirrorfailure callback // LrMirrorFailureCb mf_cb = target->target().mirrorfailurecb; // if (mf_cb) diff --git a/src/fastest_mirror.cpp b/src/fastest_mirror.cpp index db238f40..6a3555dd 100644 --- a/src/fastest_mirror.cpp +++ b/src/fastest_mirror.cpp @@ -29,8 +29,7 @@ namespace powerloader std::vector check_mirrors; for (const std::string& u : urls) { - CURLHandle handle(ctx); - handle.url(u); + CURLHandle handle(ctx, u); handle.setopt(CURLOPT_CONNECT_ONLY, 1L); check_mirrors.push_back(detail::InternalMirror{ u, std::move(handle), -1 }); } diff --git a/src/mirror.cpp b/src/mirror.cpp index 9981be37..85441e79 100644 --- a/src/mirror.cpp +++ b/src/mirror.cpp @@ -7,24 +7,6 @@ namespace powerloader { - Mirror::Mirror(MirrorID id, const Context& ctx, const std::string& url) - : m_url(url) - , m_id(id) - { - if (url.back() == '/' && url != "file://") - m_url = m_url.substr(0, m_url.size() - 1); - - if (ctx.max_downloads_per_mirror > 0) - { - m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror; - } - } - - Mirror::Mirror(const Context& ctx, const std::string& url) - : Mirror(Mirror::id(url), ctx, url) - { - } - Mirror::~Mirror() = default; void Mirror::change_max_ranges(int new_value) @@ -202,4 +184,22 @@ namespace powerloader return true; } + + bool HTTPMirror::authenticate(CURLHandle& handle, const std::string& path) + { + if (!m_auth_password.empty()) + { + spdlog::warn( + "Setting HTTP authentication for {} to {}:{}", path, m_auth_user, m_auth_password); + handle.setopt(CURLOPT_USERNAME, m_auth_user.c_str()); + handle.setopt(CURLOPT_PASSWORD, m_auth_password.c_str()); + } + return true; + } + + void HTTPMirror::set_auth(const std::string& user, const std::string& password) + { + m_auth_user = user; + m_auth_password = password; + } } diff --git a/src/mirrors/oci.cpp b/src/mirrors/oci.cpp index 952278cd..c887a93f 100644 --- a/src/mirrors/oci.cpp +++ b/src/mirrors/oci.cpp @@ -43,6 +43,7 @@ namespace powerloader : Mirror(OCIMirror::id(host, repo_prefix), ctx, host) , m_repo_prefix(repo_prefix) , m_scope("pull") + , m_proxy_map(ctx.proxy_map) { } @@ -57,6 +58,7 @@ namespace powerloader , m_scope(scope) , m_username(username) , m_password(password) + , m_proxy_map(ctx.proxy_map) { } @@ -143,7 +145,7 @@ namespace powerloader if (cbdata->token.empty() && need_auth()) { std::string auth_url = get_auth_url(split_path, m_scope); - handle.url(auth_url); + handle.url(auth_url, m_proxy_map); handle.set_default_callbacks(); @@ -178,7 +180,7 @@ namespace powerloader std::string manifest_url = get_manifest_url(split_path, split_tag); - handle.url(manifest_url) + handle.url(manifest_url, m_proxy_map) .add_headers(get_auth_headers(path)) .add_header("Accept: application/vnd.oci.image.manifest.v1+json"); diff --git a/src/python/main.cpp b/src/python/main.cpp index aee61157..3bb0a22b 100644 --- a/src/python/main.cpp +++ b/src/python/main.cpp @@ -24,13 +24,14 @@ PYBIND11_MODULE(pypowerloader, m) py::class_>(m, "DownloadTarget") .def(py::init()) - .def_property_readonly("complete_url", &DownloadTarget::complete_url) .def_property("progress_callback", &DownloadTarget::progress_callback, &DownloadTarget::set_progress_callback); + py::class_(m, "MirrorID").def(py::init()); + py::class_>(m, "Mirror") - .def(py::init()); + .def(py::init()); py::class_(m, "MirrorMap") .def(py::init<>()) diff --git a/src/target.cpp b/src/target.cpp index cb6f24e1..08b98ccd 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -38,6 +38,7 @@ namespace powerloader void Target::reset() { + spdlog::warn("Resetting target {}", m_target->destination_path().string()); if (m_target->outfile() && !zck_running()) { std::error_code ec; @@ -108,15 +109,29 @@ namespace powerloader bool Target::truncate_transfer_file() { + spdlog::warn("Truncating transfer file ... "); std::ptrdiff_t offset = 0; std::error_code ec; - if (!m_target->outfile() || !m_target->outfile()->open()) + spdlog::warn("NOT TRUNCATING BECAUSE FILE IS {} or {}", + (bool) m_target->outfile(), + m_target->outfile()->open()); + if (!m_target->outfile()) + { return true; + } + + if (!m_target->outfile()->open()) + { + fs::remove(m_target->outfile()->path()); + return true; + } if (m_original_offset >= 0) offset = m_original_offset; + spdlog::warn("Truncating transfer file offset {}", offset); + m_target->outfile()->truncate(offset, ec); if (ec) { @@ -545,7 +560,7 @@ namespace powerloader } // Set URL - h.url(full_url); + h.url(full_url, m_ctx.proxy_map); if (m_target->head_only()) { diff --git a/test.py b/test.py index 2811fd4b..fcccce34 100644 --- a/test.py +++ b/test.py @@ -10,7 +10,6 @@ baseurl = "https://conda.anaconda.org/conda-forge" filename = "python3.9_test" downTarg = pypowerloader.DownloadTarget(path, baseurl, filename) -print("complete url: " + downTarg.complete_url) def progress(total, done):