Only create fallback mirrors for targets when there is no mirror existing for it.

CMakeLists.txt

@@ -109,6 +109,8 @@ set(LIBPOWERLOADER_SRCS
     ${POWERLOADER_SOURCE_DIR}/target.cpp
     ${POWERLOADER_SOURCE_DIR}/url.cpp
     ${POWERLOADER_SOURCE_DIR}/utils.cpp
+    ${POWERLOADER_SOURCE_DIR}/target.hpp
+    ${POWERLOADER_SOURCE_DIR}/curl_internal.hpp
 )
 if (WITH_ZCHUNK)
     list(APPEND LIBPOWERLOADER_SRCS ${POWERLOADER_SOURCE_DIR}/zck.cpp

include/powerloader/context.hpp

@@ -11,13 +11,14 @@
 
 #include <powerloader/export.hpp>
 #include <powerloader/mirrorid.hpp>
+#include <powerloader/curl.hpp>
 
 namespace powerloader
 {
     namespace fs = std::filesystem;
 
     class Context;
-    struct Mirror;
+    class Mirror;
 
     using mirror_set
         = std::vector<std::shared_ptr<Mirror>>;  // TODO: replace by std::flat_set once available.
@@ -86,6 +87,15 @@ namespace powerloader
         void reset(mirror_map_base new_values = {});
     };
 
+    using proxy_map_type = std::map<std::string, std::string>;
+
+    // Options provided when starting a powerloader context.
+    struct ContextOptions
+    {
+        // If set, specifies which SSL backend to use with CURL.
+        std::optional<ssl_backend_t> ssl_backend;
+    };
+
     class POWERLOADER_API Context
     {
     public:
@@ -122,19 +132,24 @@ namespace powerloader
         std::chrono::steady_clock::duration retry_default_timeout = std::chrono::seconds(2);
 
         mirror_map_type mirror_map;
+        proxy_map_type proxy_map;
 
         std::vector<std::string> additional_httpheaders;
 
         void set_verbosity(int v);
 
         // Throws if another instance already exists: there can only be one at any time!
-        Context();
+        Context(ContextOptions options = {});
         ~Context();
 
         Context(const Context&) = delete;
         Context& operator=(const Context&) = delete;
         Context(Context&&) = delete;
         Context& operator=(Context&&) = delete;
+
+    private:
+        struct Impl;
+        std::unique_ptr<Impl> impl;  // Private implementation details
     };
 
 }

include/powerloader/curl.hpp

@@ -13,6 +13,11 @@
 #include <nlohmann/json.hpp>
 #include <tl/expected.hpp>
 
+extern "C"
+{
+#include <curl/curl.h>
+}
+
 #include <powerloader/export.hpp>
 #include <powerloader/utils.hpp>
 #include <powerloader/enums.hpp>
@@ -21,8 +26,25 @@ namespace powerloader
 {
     class Context;
     class CURLHandle;
+    using proxy_map_type = std::map<std::string, std::string>;
 
-#include <curl/curl.h>
+    enum class ssl_backend_t
+    {
+        none = CURLSSLBACKEND_NONE,
+        openssl = CURLSSLBACKEND_OPENSSL,
+        gnutls = CURLSSLBACKEND_GNUTLS,
+        nss = CURLSSLBACKEND_NSS,
+        gskit = CURLSSLBACKEND_GSKIT,
+        // polarssl = CURLSSLBACKEND_POLARSSL /* deprecated by curl */,
+        wolfssl = CURLSSLBACKEND_WOLFSSL,
+        schannel = CURLSSLBACKEND_SCHANNEL,
+        securetransport = CURLSSLBACKEND_SECURETRANSPORT,
+        // axtls = CURLSSLBACKEND_AXTLS, /* deprecated by curl */
+        mbedtls = CURLSSLBACKEND_MBEDTLS,
+        // mesalink = CURLSSLBACKEND_MESALINK, /* deprecated by curl */
+        bearssl = CURLSSLBACKEND_BEARSSL,
+        rustls = CURLSSLBACKEND_RUSTLS,
+    };
 
     class POWERLOADER_API curl_error : public std::runtime_error
     {
@@ -66,7 +88,7 @@ namespace powerloader
         CURLHandle(const Context& ctx, const std::string& url);
         ~CURLHandle();
 
-        CURLHandle& url(const std::string& url);
+        CURLHandle& url(const std::string& url, const proxy_map_type& proxies);
         CURLHandle& accept_encoding();
         CURLHandle& user_agent(const std::string& user_agent);
 
@@ -134,6 +156,8 @@ namespace powerloader
         }
         return *this;
     }
+
+    std::optional<std::string> proxy_match(const proxy_map_type& ctx, const std::string& url);
 }
 
 #endif

include/powerloader/mirror.hpp

@@ -62,11 +62,29 @@ namespace powerloader
         }
     };
 
+    inline std::string strip_trailing_slash(const std::string& s)
+    {
+        if (s.size() > 0 && s.back() == '/' && s != "file://")
+        {
+            return s.substr(0, s.size() - 1);
+        }
+        return s;
+    }
+
     // mirrors should be dict -> urls mapping
-    struct POWERLOADER_API Mirror
+    class POWERLOADER_API Mirror
     {
-        Mirror(MirrorID id, const Context& ctx, const std::string& url);
-        Mirror(const Context& ctx, const std::string& url);
+    public:
+        Mirror(const MirrorID& id, const Context& ctx, const std::string& url)
+            : m_id(id)
+            , m_url(strip_trailing_slash(url))
+        {
+            if (ctx.max_downloads_per_mirror > 0)
+            {
+                m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror;
+            }
+        }
+
         virtual ~Mirror();
 
         Mirror(const Mirror&) = delete;
@@ -156,8 +174,8 @@ namespace powerloader
         }
 
     private:
-        std::string m_url;
         const MirrorID m_id;
+        const std::string m_url;
 
         Protocol m_protocol = Protocol::kHTTP;
         MirrorState m_state = MirrorState::READY;
@@ -180,6 +198,28 @@ namespace powerloader
         std::size_t m_retry_counter = 0;
     };
 
+    class POWERLOADER_API HTTPMirror : public Mirror
+    {
+    public:
+        HTTPMirror(const Context& ctx, const std::string& url)
+            : Mirror(HTTPMirror::id(url), ctx, url)
+        {
+        }
+
+        static MirrorID id(const std::string& url)
+        {
+            return MirrorID{ fmt::format("HTTPMirror[{}]", url) };
+        }
+
+        void set_auth(const std::string& user, const std::string& password);
+
+        bool authenticate(CURLHandle& handle, const std::string& path) override;
+
+    private:
+        std::string m_auth_user;
+        std::string m_auth_password;
+    };
+
     bool sort_mirrors(std::vector<std::shared_ptr<Mirror>>& mirrors,
                       const std::shared_ptr<Mirror>& mirror,
                       bool success,

include/powerloader/mirrors/oci.hpp

@@ -69,6 +69,9 @@ namespace powerloader
         std::string m_password;
         split_function_type m_split_func;
 
+        // we copy over the proxy map from the context, otherwise we can't set new
+        // proxy options for each curl handle
+        proxy_map_type m_proxy_map;
 
         std::pair<std::string, std::string> split_path_tag(const std::string& path) const;
 

src/cli/main.cpp

@@ -392,7 +392,7 @@ parse_mirrors(const Context& ctx, const YAML::Node& node)
             else if (kof == KindOf::kHTTP)
             {
                 spdlog::info("Adding HTTP mirror: {} -> {}", mirror_name, creds.url.url());
-                result.create_unique_mirror<Mirror>(mirror_name, ctx, creds.url.url());
+                result.create_unique_mirror<HTTPMirror>(mirror_name, ctx, creds.url.url());
             }
         }
     }

src/context.cpp

@@ -14,18 +14,31 @@ extern "C"
 
 #include <powerloader/mirror.hpp>
 
+#include "./curl_internal.hpp"
+
 
 namespace powerloader
 {
+    struct Context::Impl
+    {
+        std::optional<details::CURLSetup> curl_setup;
+    };
+
     static std::atomic<bool> is_context_alive{ false };
 
-    Context::Context()
+    Context::Context(ContextOptions options)
+        : impl(new Impl)
     {
         bool expected = false;
         if (!is_context_alive.compare_exchange_strong(expected, true))
             throw std::runtime_error(
                 "powerloader::Context created more than once - instance must be unique");
 
+        if (options.ssl_backend)
+        {
+            impl->curl_setup = details::CURLSetup{ options.ssl_backend.value() };
+        }
+
         cache_dir = fs::absolute(fs::path(".pdcache"));
         if (!fs::exists(cache_dir))
         {

src/curl.cpp

@@ -6,6 +6,7 @@
 #include <powerloader/curl.hpp>
 #include <powerloader/utils.hpp>
 #include <powerloader/context.hpp>
+#include <powerloader/url.hpp>
 
 namespace powerloader
 {
@@ -24,6 +25,7 @@ namespace powerloader
         return m_serious;
     }
 
+
     /**************
      * CURLHandle*
      **************/
@@ -54,28 +56,35 @@ namespace powerloader
 
         if (ctx.disable_ssl)
         {
+            spdlog::warn("SSL verification is disabled");
             setopt(CURLOPT_SSL_VERIFYHOST, 0);
             setopt(CURLOPT_SSL_VERIFYPEER, 0);
+
+            // also disable proxy SSL verification
+            setopt(CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
+            setopt(CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
         }
         else
         {
+            spdlog::warn("SSL verification is ENABLED");
+
             setopt(CURLOPT_SSL_VERIFYHOST, 2);
             setopt(CURLOPT_SSL_VERIFYPEER, 1);
 
             // Windows SSL backend doesn't support this
             CURLcode verifystatus = curl_easy_setopt(m_handle, CURLOPT_SSL_VERIFYSTATUS, 0);
             if (verifystatus != CURLE_OK && verifystatus != CURLE_NOT_BUILT_IN)
                 throw curl_error("Could not initialize CURL handle");
-        }
 
-        if (!ctx.ssl_ca_info.empty())
-        {
-            setopt(CURLOPT_CAINFO, ctx.ssl_ca_info.c_str());
-        }
+            if (!ctx.ssl_ca_info.empty())
+            {
+                setopt(CURLOPT_CAINFO, ctx.ssl_ca_info.c_str());
+            }
 
-        if (ctx.ssl_no_revoke)
-        {
-            setopt(CURLOPT_SSL_OPTIONS, ctx.ssl_no_revoke);
+            if (ctx.ssl_no_revoke)
+            {
+                setopt(CURLOPT_SSL_OPTIONS, ctx.ssl_no_revoke);
+            }
         }
 
         setopt(CURLOPT_FTP_USE_EPSV, (long) ctx.ftp_use_seepsv);
@@ -88,7 +97,7 @@ namespace powerloader
     CURLHandle::CURLHandle(const Context& ctx, const std::string& url)
         : CURLHandle(ctx)
     {
-        this->url(url);
+        this->url(url, ctx.proxy_map);
     }
 
     CURLHandle::~CURLHandle()
@@ -123,9 +132,18 @@ namespace powerloader
         return *this;
     }
 
-    CURLHandle& CURLHandle::url(const std::string& url)
+    CURLHandle& CURLHandle::url(const std::string& url, const proxy_map_type& proxies)
     {
         setopt(CURLOPT_URL, url.c_str());
+        const auto match = proxy_match(proxies, url);
+        if (match)
+        {
+            setopt(CURLOPT_PROXY, match.value().c_str());
+        }
+        else
+        {
+            setopt(CURLOPT_PROXY, nullptr);
+        }
         return *this;
     }
 
@@ -373,4 +391,42 @@ namespace powerloader
         downloaded_size
             = handle.getinfo<decltype(downloaded_size)>(CURLINFO_SIZE_DOWNLOAD_T).value();
     }
+
+    std::optional<std::string> proxy_match(const proxy_map_type& proxies, const std::string& url)
+    {
+        // This is a reimplementation of requests.utils.select_proxy()
+        // of the python requests library used by conda
+        if (proxies.empty())
+        {
+            return std::nullopt;
+        }
+
+        auto handler = URLHandler(url);
+        auto scheme = handler.scheme();
+        auto host = handler.host();
+        std::vector<std::string> options;
+
+        if (host.empty())
+        {
+            options = {
+                scheme,
+                "all",
+            };
+        }
+        else
+        {
+            options = { scheme + "://" + host, scheme, "all://" + host, "all" };
+        }
+
+        for (auto& option : options)
+        {
+            auto proxy = proxies.find(option);
+            if (proxy != proxies.end())
+            {
+                return proxy->second;
+            }
+        }
+
+        return std::nullopt;
+    }
 }