Https proxy (via CONNECT) support

README.md

@@ -246,7 +246,7 @@ Please refer to the [tutorial](doc/Tutorial.md) for more examples.
   - Write directly to the outgoing DataWriter when data is required.
   - Just provide a C++ object and let the library serialize it directly to the wire.
 - HTTP Proxy support
-- SOCKS5 Proxy support (naive implementatin for now, no support for authentication).
+- HTTPS Proxy (using CONNECT method), SOCKS5 Proxy support (naive implementatin for now, no support for authentication).
 
 # Current Status
 The project has been in public BETA since April 11th 2017.
@@ -262,7 +262,7 @@ These are the operating systems where my Continues Integration (Jenkins) servers
  - Ubuntu Xenial (LTS)
  - Ubuntu Jammy (LTS)
 
-Support for MacOS has been removed after Apples announcement that their love for privacy was just 
+Support for MacOS has been removed after Apples announcement that their love for privacy was just
 a marketing gimmick.
 
 Fedora is currently disabled in my CI because of failures to start their Docker containers. (Work in progress). 

ci/mock-backends/docker-compose.yml

@@ -8,6 +8,7 @@ services:
         build: nginx
         ports:
             - "3001:80"
+            - "3002:443"
         links:
             - "json:api"
     squid:

ci/mock-backends/nginx/Dockerfile

@@ -1,7 +1,10 @@
 FROM nginx
 RUN rm /etc/nginx/conf.d/default.conf
 COPY proxy.conf.bin /etc/nginx/conf.d/proxy.conf
+COPY proxy-https.conf.bin /etc/nginx/conf.d/proxy-https.conf
 COPY htpasswd.bin /etc/nginx/htpasswd
+COPY test.pem /etc/nginx/ssl/test.pem
+COPY test-key.pem /etc/nginx/ssl/test-key.pem
 RUN mkdir -p /etc/nginx/html/upload
 RUN chmod 777 /etc/nginx/html/upload
 

ci/mock-backends/nginx/create_cert.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+openssl req -x509 -newkey rsa:4096 -keyout test-key.pem -out test.pem -sha256 -days 3650 -nodes -subj "/C=XX/ST=Test/L=Test/OU=Test/O=Test/CN=localhost"
+
+#openssl x509 -in test.pem -text

ci/mock-backends/nginx/proxy-https.conf.bin

@@ -0,0 +1,62 @@
+server {
+    listen       443 ssl;
+    server_name  localhost;
+    gzip on;
+    gzip_proxied any;
+    ssl_certificate /etc/nginx/ssl/test.pem;
+    ssl_certificate_key /etc/nginx/ssl/test-key.pem;
+    ssl_protocols TLSv1.2 TLSv1.3;
+
+
+    location ~ ^/cookies(/?)(.*)$ {
+        add_header Content-Type "application/json; charset=utf-8";
+        add_header Set-Cookie test1=yes;
+        add_header Set-Cookie test2=maybe;
+        add_header Set-Cookie test3=no;
+        return 200 '{}';
+    }
+
+    location ~ ^/normal(/?)(.*)$ {
+        proxy_pass http://json/$2$is_args$args;
+    }
+
+    location  ~ ^/close(/?)(.*)$ {
+        proxy_pass http://json/$2$is_args$args;
+        keepalive_timeout 0;
+    }
+
+    # Force nginx to resolve 'json'
+    location /dns_workaround {
+        proxy_pass http://json;
+    }
+
+    location ~ ^/loop(/?)(.*)$ {
+        return 301 $scheme://$host:3001/loop/$2$is_args$args;
+    }
+
+    location ~ ^/redirect(/?)(.*)$ {
+        return 301 $scheme://$host:3001/normal/$2$is_args$args;
+    }
+
+    location ~ ^/reredirect(/?)(.*)$ {
+        return 301 $scheme://$host:3001/redirect/$2$is_args$args;
+    }
+
+    location ~ ^/restricted(/?)(.*)$ {
+        auth_basic           "Restricted Area";
+        auth_basic_user_file /etc/nginx/htpasswd;
+        proxy_pass http://json/$2$is_args$args;
+    }
+
+    location ~ ^/upload_raw(/?)(.*)$ {
+        limit_except POST          { deny all; }
+        return 200 "OK";
+    }
+
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}
+

ci/mock-backends/nginx/test-key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDPlSJHl/g59nyi
+HvVblS5eWCTRxFFueCIEbYGfUdeL+Jc+YP20fJs3u5Tq7gCvuYqGTMztXfERRk3H
+NrHy1uXndh0fGVUusn5hYz4XNB8AmRfZf1wDTpd4SjVe26Da52sIGppj1bGgPJWR
+9KK+7zDPNeEVQ4HPYnh8rTfHBG0vYlfHnHR4EGwS6znKA3YIgwHqHIrdm/cOmMK3
+hDMmjG8l01U5YHIBOadTh1QZvqhpEToJXnV949CvN12HiyawSQ3nG3HTC94PnrV7
+UtOLGf3rh+OxN8D9d+QgBVSAI8d7VqT+7xxU8wvzL5gWcQLuhY208cZW3Y6gC4U6
+ZpfHxeqUtAUw4ZK4bJmKr6HFdasXaPkKPsUZsppeiGYtXRwt6yuHJuI3I1JTwcuA
+SYCHEhus5biMyRZWZo5PLbCX1sXYbbAEt/qVpg1jeTwrkeZ62lC79lAKjd0runTE
+NMvZYKZwnJntSUz6vq6uMYoM0BzSMSioFu2fqWBFH9263Tje/wwz/aNNAKx2+3SN
+avC7//gSM/0gl/9bVRgsWLyxc5qxCTU8uQ1P5zNx07YNmv6C8UI44craA7KdxcYg
+qgdUStKDzpDB48eOvvbRRAZO55KAYzCS/1dgfFUcsugP7DklblI3kTNHTIjarYYX
+JjUNl0Tpcx3Tk6hfgBBMX1Re/GtvTwIDAQABAoICABjbVdXW3R2BEN7k8CJ9qaYY
+eJ2PHDqAiNTuO/r/n4eeRpYXTSoDUHQ1sm5d/kJh6F7j7BdxtqqH4vZmQUa/9EW+
+1LE4T6/zexi0UTy8EV5OoWwk87gIUtuk9IvIUZRaPmx5AGB7YqAPwSVPcvwuw4cn
+Ki2+qK5UWfMsAXrZDS4DC68rt/Kh8h8N1cdamhQI2VOBrss8oDKPmPlwC3lOkFyq
+LWayetRUurRQ3IGSCAk/doClXqeq06liQCvj6MfBPQ3zMQfBlbSvH4eF8oOR+DWa
+TxSV2uE/Laz7674bCrRlOrAK9+HgPLUWej1ts5krmugTmi7QAa0pKVSbRl+LU+cw
+CzH/7rSzI3AigNs7eA7tw+6J/0ZkC3VHqStVIC5vzOvOYqSNm1W2koBnZwMYGhZM
+A8CT/FiHRhuRSt2GrIKmLGtvommhaEDs/rMc/qaOxd3TyjO8Wi9+vohtI3PJLlgK
+sqIWtPn1J+Mw6bLElvC0Ljdhs9rb/cUqm0gmbXDi0/7iKYZF7GN2eLoJ7lImjAAW
+sPyjvKxgbuPxCGOrhT8SgLDtTTGlFGSOf+t2mOY+NRugOOsvjsYUjB5SRDMuRq36
+Jp/TcdprdVZ9DqIkCXQp/1vww5Lkk6eMicjF76PoMbSMXSyVnC95foc9wGWgdYMk
+djAbUqJgdQHhmj1vuvJxAoIBAQDnUhAn1uHwXMG/4cQh+Y+Wqiidx2yR1zxUee9y
+igKNs7z5lloX9bSPIQhGAToCFXwgd6dzGEeE/i46fkqm5iDQ4waWN+OE8fE/g4qm
+HpTjG7LhNC7AJWS2bq3RWCSMrWEvFs77yVvLb6MNhgyJDOc+4llXW3zr8B1aFsjB
+q3bA/hA3ulk67zBYJaAbDdi6kq+Bi+qiwVdmrAJNUf7rujjqYiH7dmw0wjdMy+sN
+MoOizoSg5OFpiTr8kLdrk2N5PWq9IY5FtsqE5sckhoSTf/i+eaXLkM8HWyx/rDkf
+VRiVZ4So4Iin3hEGh7xCtjsVVG8KA88zML1qT3q9baSxqfbJAoIBAQDluromSOoj
+okV0Epa+M/VvMt8hVupFd0oo7PtC+rOtvKes7aUJXVyZcrsbjuCzxsihLEKLCXKo
+7wSZmssCz9/jYTxqLzfAYN0xaj9OhZ2nCEJ5av3EEIif5cw9jNaG2ivv8/vSbrBl
+77rrv2uJlhQ63sV983sM7RxrQrJU9QuW60yDAy0jcDkYBUjNyAbrYOKETXVDTiuE
+4mSRyI7jwGp4oKWyM1/5iJCKqU6btBHHBOrOm89U7hTgTEWetjkimJ+llH/OrnYt
+CzlmZFEecg9PnFIS4HKt7QbuSJKp83EN10lrVr93Poaz0Efk3c2CpwoIEIyeZs94
+CgJ9an6CX4lXAoIBAQCumJgtGdnrjHeJFyTs5+rjM4f4nx9pbOXSdT6wW07WGcYX
+NM7HquMf7TTLcf2QuRq5ftba3oaM8TV/XPeHxccbI2BDXefS3rLS17x86jRCvxNj
+O/nVeePsdtmnWzorHGpwGm0cSr2IbbjKalVn1F9ubXY1o45EnzXoW64nz/2QabNf
+/L2A6Cy7O5r/EJJ3MGRcCXmOYxRPIKGULsGUtzhiYLN5k8bUg4st4fSGP4xwBCTD
+ND6XY8cr/ycSgWrhhePc4Uj7gZ6WdYH2JbpHgp4DVto3LhO1X7HUo+9xoM8vZbUR
+qng7DDgZj7YfPGCYFuTA0GNCJhWx+k+QTwOyPbFRAoIBAGzhHBrLEhWDcjF6IfHR
+xHBIhxJRFEWKLR7KeqebFI+ySzIdi8utcRbVFrMP+5WZEDu7M2qcNri0V9TJVZBm
+n3EwA6c768uE3TDvb0Oy9i5VLtRHDjDfuTE3g55kYsSVIJ/gXii1B2u4vDnBhqE1
+/S6NqMJyJI7SzlZTzRuQ7EZCDQhG+BzEsnqc/o1xUT47tAAKiho1MVEQz6N8j6SH
+7K5xTTbxPHqS7Bab+cK4DHjr7rGvjQtur3xDCfgX22p3NasPf6egbigZGsJZp0yr
+uG/94bRKpm+iWFeVE9XyqFFsCMMT4TkN7F/KxlhFe4KB0rJRzaPBjHETJWz1jTIT
+P48CggEBAL0X4oyLJvdYFh41yBOSMmOg7ulUyqEZRgKrHQQ75/qb6lxa4FcGymiH
+iNXsU4/o3PwJcUg0zEilkllYyVDKOXtjHEMipONifPgI2eaEPLdA97QmGVFRNFTk
+gWXMxMVRL6+TSPSnOU/PuOPbRTMIDMFdnZj2VeDf0M4uHwPRPwBGPYa8jW9m+Vz2
+senoxaJnwakywphSQWbr2HcS0PzJhGhyKBcbX2SYK0esPjLILvTkbyzYfgT3SJnH
+Ybu1KPcUH7sZKqn7KLDLzCSNq1pDM4asj8i3gGCjRAA3mPyDHrZ83L1LYVUE+gMh
+stBsKJU1QibpCwO3/NmBYwjRitw1E2c=
+-----END PRIVATE KEY-----

ci/mock-backends/nginx/test.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFmzCCA4OgAwIBAgIUJFXivxxXEIY3rp6fz2wgE3rsQzYwDQYJKoZIhvcNAQEL
+BQAwXTELMAkGA1UEBhMCWFgxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx
+DTALBgNVBAsMBFRlc3QxDTALBgNVBAoMBFRlc3QxEjAQBgNVBAMMCWxvY2FsaG9z
+dDAeFw0yMzA4MzAxMTM2MjJaFw0zMzA4MjcxMTM2MjJaMF0xCzAJBgNVBAYTAlhY
+MQ0wCwYDVQQIDARUZXN0MQ0wCwYDVQQHDARUZXN0MQ0wCwYDVQQLDARUZXN0MQ0w
+CwYDVQQKDARUZXN0MRIwEAYDVQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQDPlSJHl/g59nyiHvVblS5eWCTRxFFueCIEbYGfUdeL
++Jc+YP20fJs3u5Tq7gCvuYqGTMztXfERRk3HNrHy1uXndh0fGVUusn5hYz4XNB8A
+mRfZf1wDTpd4SjVe26Da52sIGppj1bGgPJWR9KK+7zDPNeEVQ4HPYnh8rTfHBG0v
+YlfHnHR4EGwS6znKA3YIgwHqHIrdm/cOmMK3hDMmjG8l01U5YHIBOadTh1QZvqhp
+EToJXnV949CvN12HiyawSQ3nG3HTC94PnrV7UtOLGf3rh+OxN8D9d+QgBVSAI8d7
+VqT+7xxU8wvzL5gWcQLuhY208cZW3Y6gC4U6ZpfHxeqUtAUw4ZK4bJmKr6HFdasX
+aPkKPsUZsppeiGYtXRwt6yuHJuI3I1JTwcuASYCHEhus5biMyRZWZo5PLbCX1sXY
+bbAEt/qVpg1jeTwrkeZ62lC79lAKjd0runTENMvZYKZwnJntSUz6vq6uMYoM0BzS
+MSioFu2fqWBFH9263Tje/wwz/aNNAKx2+3SNavC7//gSM/0gl/9bVRgsWLyxc5qx
+CTU8uQ1P5zNx07YNmv6C8UI44craA7KdxcYgqgdUStKDzpDB48eOvvbRRAZO55KA
+YzCS/1dgfFUcsugP7DklblI3kTNHTIjarYYXJjUNl0Tpcx3Tk6hfgBBMX1Re/Gtv
+TwIDAQABo1MwUTAdBgNVHQ4EFgQUtH5zl0rC27e1JxtdMbIxwqkg0NQwHwYDVR0j
+BBgwFoAUtH5zl0rC27e1JxtdMbIxwqkg0NQwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAgEAw7sQ1KIyHCYHJUzYdX9I/l0ylbzrKpfJUABq3OOmgjUM
+k6InH6bXvAEP7bj4zC3UDKj1hf5WVHQM0yUeLx8ZmzHzsgfS6C8MiSt+47V0NboQ
+5aGHY21ceMivlNDihf0cjPg+h4NsAZkzMWR771Nxe3WLT20Wo1ehKqCjIC7HnCJ3
+IAKbholWJwaiu1Y9tLn21sNMC0Z75hx/JWokP8OuZUyLxsp5RDLLvPA4N41z1Tep
+CXpBct6jdRVzq7b5bUgPIQJ4XVdnaIYtdWIZVavulbr0vo4P7dayhf7ZWGcFVyPh
+GmjHsPiurZy5BShEOgyYlwrSf9guDJfLR0gclKysKF1I4O9EWysRntvxuDMg0bys
+jdUkp25EjQzIP/gA5J6NhTUFYnt04cLIebxs8CU/ZqZ0QZ11wT3VnvTm9ukTsaT4
+lKzmBjlejZVLwlki6r3IAj/CD2v9pJddt950usj9/I9C6quw1XpWG6TYZlCyM9lb
+ucUfPttVknOM8c/eenXYAECR/f9MqlFBHO3Bj9YfO4hGTZtE7qaiY2f6yhcLRCOt
+Bd649G/bfKT0KvLp+6SdTJxMYFC70FUqoqniEfxT+AhsKP+2k0MfU4MvJ/neLcGA
+l/fM+TeIJWTzadl/DCZeE0FyQXAzeqHICnNQOieH6spMpjiXd6cv0aPO1zSqpV4=
+-----END CERTIFICATE-----

ci/mock-backends/squid/squid.conf.bin

@@ -1,5 +1,9 @@
-
+reply_header_add Test-proxy "THIS IS TEST PROXY" all
+acl CONNECT method CONNECT
+acl connect_denied_ports port 444
 http_access allow localhost manager
 http_access deny manager
+http_access deny CONNECT connect_denied_ports
 http_access allow all
 http_port 3128
+

create-and-run-containers.sh

@@ -13,8 +13,8 @@ else
     exit -1
 fi
 
-docker-compose stop
-docker-compose build
-docker-compose up -d
+docker compose stop
+docker compose build
+docker compose up -d
 docker ps
 popd

doc/Tutorial.md

@@ -288,10 +288,12 @@ use the *RequestBuilder*'s *Argument()* method for this.
 
 ```
 
-## Send a request going trough a HTTP Proxy
+## Send a request going trough a HTTP or HTTPS Proxy
 ```C++
     // Add the proxy information to the properties used by the client
     Request::Properties properties;
+    // Use proxy type Request::Proxy::Type::HTTP for simple HTTP proxing,
+    //   Request::Proxy::Type::HTTPS for HTTPS proxing over CONNECT method
     properties.proxy.type = Request::Proxy::Type::HTTP;
     properties.proxy.address = "http://127.0.0.1:3003";
 
@@ -313,7 +315,7 @@ use the *RequestBuilder*'s *Argument()* method for this.
     }).get();
 ```
 
-## Use an existing thread in stead of a new worker thread
+## Use an existing thread instead of a new worker thread
 
 This example is slightly more advanced. Here we take
 responsibility to run the io-service used internally by

include/restc-cpp/logging.h

@@ -143,10 +143,6 @@ inline void RestcCppTestStartLogger(const std::string& level = "info") {
 
 #elif defined RESTC_CPP_LOG_WITH_BOOST_LOG
 
-#ifndef WIN32
-#	define BOOST_LOG_DYN_LINK 1
-#endif
-
 #include <iostream>
 #include <boost/log/core.hpp>
 #include <boost/log/trivial.hpp>

include/restc-cpp/restc-cpp.h

@@ -66,8 +66,6 @@ class RequestBody;
 class Connection;
 class ConnectionPool;
 class Socket;
-class Request;
-class Reply;
 class Context;
 class DataWriter;
 
@@ -136,11 +134,12 @@ class Request {
     };
 
     struct Proxy {
-        enum class Type { NONE, HTTP, SOCKS5 };
+        enum class Type { NONE, HTTP, HTTPS, SOCKS5 };
         Type type = Type::NONE;
         std::string address;
 
         const std::string& GetName();
+        Type detect();
     };
 
     using args_t = std::deque<Arg>;
@@ -160,7 +159,7 @@ class Request {
     class Properties {
     public:
         using ptr_t = std::shared_ptr<Properties>;
-        using redirect_fn_t = std::function<void (int code, std::string& url, 
+        using redirect_fn_t = std::function<void (int code, std::string& url,
                                                   const Reply& reply)>;
         using general_callback_t = std::function<void()>;
 
@@ -417,7 +416,7 @@ class RestClient {
             done_handler.reset();
         });
 
-        return move(future);
+        return future;
     }
 
     /*! Process from within an existing coroutine */

include/restc-cpp/url_encode.h

@@ -1,8 +1,5 @@
 #pragma once
 
-#ifndef RESTC_CPP_URL_ENCODE_H_
-#define RESTC_CPP_URL_ENCODE_H_
-
 #include "restc-cpp.h"
 
 #include <boost/utility/string_ref.hpp>
@@ -13,4 +10,3 @@ std::string url_encode(const boost::string_ref& src);
 
 } // namespace
 
-#endif // RESTC_CPP_URL_ENCODE_H_

src/ConnectionPoolImpl.cpp

@@ -240,7 +240,7 @@ class ConnectionPoolImpl
                     idle_.erase(current);
                 } else {
                     RESTC_CPP_LOG_TRACE_("Keeping << " << *current->second->GetConnection()
-                        << " expieres in "
+                        << " expires in "
                         << std::chrono::duration_cast<std::chrono::seconds>(expires - now).count()
                         << " seconds ");
                 }
@@ -365,7 +365,9 @@ class ConnectionPoolImpl
         }
         else {
 #ifdef RESTC_CPP_WITH_TLS
-            socket = make_unique<TlsSocketImpl>(owner_.GetIoService(), owner_.GetTLSContext());
+            socket = make_unique<TlsSocketImpl>(owner_.GetIoService(), owner_.GetTLSContext(),
+                /*allow_sending_over_unupgraded_socket to send plain data over tls socket before handshake*/
+                properties_->proxy.type == Request::Proxy::Type::HTTPS);
 #else
             throw NotImplementedException(
                 "restc_cpp is compiled without TLS support");

src/DataReaderStream.cpp

@@ -71,6 +71,7 @@ DataReaderStream::GetData(size_t maxBytes) {
 
 void DataReaderStream::ReadServerResponse(Reply::HttpResponse& response)
 {
+    static const string http_1_0{"HTTP/1.0"}; //some proxies use HTTP/1.0
     static const string http_1_1{"HTTP/1.1"};
     constexpr size_t max_version_len = 16;
     constexpr size_t max_phrase_len = 256;
@@ -91,7 +92,8 @@ void DataReaderStream::ReadServerResponse(Reply::HttpResponse& response)
     if (value.empty()) {
         throw ProtocolException("ReadHeaders(): No HTTP version");
     }
-    if (ciEqLibC()(value, http_1_1)) {
+    if (ciEqLibC()(value, http_1_1) ||
+        ciEqLibC()(value, http_1_0)) {
         ; // Do nothing HTTP 1.1 is the default value
     } else {
         throw ProtocolException(