From b7b2f5946496fc10e855e9559638d723375b5ccf Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 8 Jul 2015 23:16:47 -0700 Subject: [PATCH 01/46] starting on s2n --- coro/ssl/s2n/s2n.pyx | 112 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 coro/ssl/s2n/s2n.pyx diff --git a/coro/ssl/s2n/s2n.pyx b/coro/ssl/s2n/s2n.pyx new file mode 100644 index 0000000..92c3312 --- /dev/null +++ b/coro/ssl/s2n/s2n.pyx @@ -0,0 +1,112 @@ +# -*- Mode: Cython -*- + +from libc.stdint cimport uint64_t, uint32_t, uint16_t, uint8_t + +#define S2N_SSLv2 20 +#define S2N_SSLv3 30 +#define S2N_TLS10 31 +#define S2N_TLS11 32 +#define S2N_TLS12 33 + +class S2N: + SSLv2 = 20 + SSLv3 = 30 + TLS10 = 31 + TLS11 = 32 + TLS12 = 33 + +cdef extern from "s2n.h": + + struct s2n_config + + int s2n_errno + + int s2n_init () + int s2n_cleanup () + s2n_config *s2n_config_new () + int s2n_config_free (s2n_config *config) + int s2n_config_free_dhparams (s2n_config *config) + int s2n_config_free_cert_chain_and_key (s2n_config *config) + const char *s2n_strerror (int error, const char *lang) + + int s2n_config_add_cert_chain_and_key (s2n_config *config, char *cert_chain_pem, char *private_key_pem) + int s2n_config_add_cert_chain_and_key_with_status ( + s2n_config *config, + char *cert_chain_pem, + char *private_key_pem, + const uint8_t *status, + uint32_t length + ) + int s2n_config_add_dhparams (s2n_config *config, char *dhparams_pem) + int s2n_config_set_key_exchange_preferences (s2n_config *config, const char *preferences) + int s2n_config_set_cipher_preferences (s2n_config *config, const char *version) + int s2n_config_set_protocol_preferences (s2n_config *config, const char * const *protocols, int protocol_count) + ctypedef enum s2n_status_request_type: + S2N_STATUS_REQUEST_NONE = 0, + S2N_STATUS_REQUEST_OCSP = 1 + int s2n_config_set_status_request_type (s2n_config *config, s2n_status_request_type type) + + struct s2n_connection + ctypedef enum s2n_mode: + S2N_SERVER, + S2N_CLIENT + s2n_connection *s2n_connection_new (s2n_mode mode) + int s2n_connection_set_config (s2n_connection *conn, s2n_config *config) + + int s2n_connection_set_fd (s2n_connection *conn, int readfd) + int s2n_connection_set_read_fd (s2n_connection *conn, int readfd) + int s2n_connection_set_write_fd (s2n_connection *conn, int readfd) + + ctypedef enum s2n_blinding: + S2N_BUILT_IN_BLINDING, + S2N_SELF_SERVICE_BLINDING + int s2n_connection_set_blinding (s2n_connection *conn, s2n_blinding blinding) + int s2n_connection_get_delay (s2n_connection *conn) + + int s2n_set_server_name (s2n_connection *conn, const char *server_name) + const char *s2n_get_server_name (s2n_connection *conn) + const char *s2n_get_application_protocol (s2n_connection *conn) + const uint8_t *s2n_connection_get_ocsp_response (s2n_connection *conn, uint32_t *length) + + int s2n_negotiate (s2n_connection *conn, int *more) + ssize_t s2n_send (s2n_connection *conn, void *buf, ssize_t size, int *more) + ssize_t s2n_recv (s2n_connection *conn, void *buf, ssize_t size, int *more) + + int s2n_connection_wipe (s2n_connection *conn) + int s2n_connection_free (s2n_connection *conn) + int s2n_shutdown (s2n_connection *conn, int *more) + + uint64_t s2n_connection_get_wire_bytes_in (s2n_connection *conn) + uint64_t s2n_connection_get_wire_bytes_out (s2n_connection *conn) + int s2n_connection_get_client_protocol_version (s2n_connection *conn) + int s2n_connection_get_server_protocol_version (s2n_connection *conn) + int s2n_connection_get_actual_protocol_version (s2n_connection *conn) + int s2n_connection_get_client_hello_version (s2n_connection *conn) + const char *s2n_connection_get_cipher (s2n_connection *conn) + int s2n_connection_get_alert (s2n_connection *conn) + +class Error (Exception): + pass + +cdef raise_s2n_error(): + raise Error (s2n_strerror (s2n_errno, "EN")) + +cdef check (int n): + if n != 0: + raise_s2n_error() + +def init(): + check (s2n_init()) + +def cleanup(): + check (s2n_cleanup()) + +cdef class config: + cdef s2n_config * c + def __init__ (self): + self.c = s2n_config_new() + if not self.c: + raise_s2n_error() + + + From 800092be7145645cdcf88844b0ed8f79400eed82 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 8 Jul 2015 23:17:12 -0700 Subject: [PATCH 02/46] starting on s2n --- setup.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/setup.py b/setup.py index 60b1625..ed28523 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,24 @@ def O (path): include_dirs=[O('include')], cython_compile_time_env={'NPN': USE_NPN}, ) + +# -------------------------------------------------------------------------- + +def path_join (*parts): + return os.path.join (*parts) + +s2n_base = '/Users/rushing/src/s2n/' +s2n_lib = path_join (s2n_base, 'lib') + +s2n_Extension = Extension ( + 'coro.ssl.s2n', + ['coro/ssl/s2n/s2n.pyx'], + libraries = ['s2n'], + #extra_link_args = ['-L', s2n_lib], + #include_dirs = [path_join (s2n_base, 'api')], + #extra_link_args = ['-L', s2n_lib, '-Wl,-rpath,%s' % (s2n_lib,)], + extra_link_args = ['-lgcc_eh'], + ) # -------------------------------------------------------------------------- setup ( @@ -184,6 +202,7 @@ def O (path): ), # the pre-computed openssl extension from above OpenSSL_Extension, + s2n_Extension, ], packages= find_packages(), py_modules = ['backdoor', 'coro.read_stream', 'coro_process', 'coro_unittest', ], From d34ad04dc34b014e1716b224d0c603eb59cb9b92 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 8 Jul 2015 23:18:11 -0700 Subject: [PATCH 03/46] #delete-trailing-whitespace --- coro/ssl/s2n/s2n.pyx | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/coro/ssl/s2n/s2n.pyx b/coro/ssl/s2n/s2n.pyx index 92c3312..83690d1 100644 --- a/coro/ssl/s2n/s2n.pyx +++ b/coro/ssl/s2n/s2n.pyx @@ -16,7 +16,7 @@ class S2N: TLS12 = 33 cdef extern from "s2n.h": - + struct s2n_config int s2n_errno @@ -32,9 +32,9 @@ cdef extern from "s2n.h": int s2n_config_add_cert_chain_and_key (s2n_config *config, char *cert_chain_pem, char *private_key_pem) int s2n_config_add_cert_chain_and_key_with_status ( s2n_config *config, - char *cert_chain_pem, - char *private_key_pem, - const uint8_t *status, + char *cert_chain_pem, + char *private_key_pem, + const uint8_t *status, uint32_t length ) int s2n_config_add_dhparams (s2n_config *config, char *dhparams_pem) @@ -42,13 +42,13 @@ cdef extern from "s2n.h": int s2n_config_set_cipher_preferences (s2n_config *config, const char *version) int s2n_config_set_protocol_preferences (s2n_config *config, const char * const *protocols, int protocol_count) ctypedef enum s2n_status_request_type: - S2N_STATUS_REQUEST_NONE = 0, - S2N_STATUS_REQUEST_OCSP = 1 + S2N_STATUS_REQUEST_NONE = 0, + S2N_STATUS_REQUEST_OCSP = 1 int s2n_config_set_status_request_type (s2n_config *config, s2n_status_request_type type) struct s2n_connection ctypedef enum s2n_mode: - S2N_SERVER, + S2N_SERVER, S2N_CLIENT s2n_connection *s2n_connection_new (s2n_mode mode) int s2n_connection_set_config (s2n_connection *conn, s2n_config *config) @@ -107,6 +107,3 @@ cdef class config: self.c = s2n_config_new() if not self.c: raise_s2n_error() - - - From fee507e7af6984e3b09c740351c3d4e3cbf31225 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sat, 11 Jul 2015 15:16:27 -0700 Subject: [PATCH 04/46] it's limping, but it's working. the semantics around non-blocking I/O are very confusing. --- coro/ssl/s2n/__init__.py | 110 +++++++++++++++ coro/ssl/s2n/_s2n.pyx | 279 +++++++++++++++++++++++++++++++++++++ coro/ssl/s2n/s2n.pyx | 109 --------------- coro/ssl/s2n/test/serve.py | 91 ++++++++++++ coro/ssl/s2n/test/t0.py | 91 ++++++++++++ setup.py | 15 +- 6 files changed, 579 insertions(+), 116 deletions(-) create mode 100644 coro/ssl/s2n/__init__.py create mode 100644 coro/ssl/s2n/_s2n.pyx delete mode 100644 coro/ssl/s2n/s2n.pyx create mode 100644 coro/ssl/s2n/test/serve.py create mode 100644 coro/ssl/s2n/test/t0.py diff --git a/coro/ssl/s2n/__init__.py b/coro/ssl/s2n/__init__.py new file mode 100644 index 0000000..dc28282 --- /dev/null +++ b/coro/ssl/s2n/__init__.py @@ -0,0 +1,110 @@ +# -*- Mode: Python -*- + +import coro + +from ._s2n import Config, Connection, S2N, MODE + +Config = _s2n.Config + +class sock (coro.sock): + + # XXX maybe delay creating connection object until either accept or connect in order to + # set mode automatically? + def __init__ (self, cfg, fd=-1, verify=False, domain=coro.AF.INET, mode=MODE.SERVER): + coro.sock.__init__ (self, fd=fd, domain=domain) + print 'fd=', self.fd + self.cfg = cfg + self.conn_ob = _s2n.Connection(mode) + self.conn_ob.set_config (cfg) + self.conn_ob.set_fd (fd) + # XXX verify + + def __repr__ (self): + return '' % (self.fd, id (self)) + + def accept (self): + conn, addr = coro.sock.accept (self) + try: + new = self.__class__ (self.cfg, domain=conn.domain, fd=conn.fd, mode=MODE.SERVER) + # ...avoid having socket.pyx close the fd + conn.fd = -1 + while 1: + more = new.conn_ob.negotiate() + if not more: + break + return new, addr + except: + conn.close() + raise + + def connect (self, addr): + coro.sock.connect (self, addr) + while 1: + more = new.conn_ob.negotiate() + if not more: + break + + def recv (self, block_size): + r = [] + left = block_size + while left: + b, more = self.conn_ob.recv (left) + print 'more, b', more, repr(b) + r.append (b) + if not more: + break + else: + left -= len(b) + self.wait_for_read() + return ''.join (r) + + read = recv + + def recv_exact (self, size): + left = size + r = [] + while left: + block = self.recv (left) + if not block: + break + else: + r.append (block) + left -= len (block) + return ''.join (r) + + def send (self, data): + pos = 0 + left = len(data) + while left: + n, more = self.conn_ob.send (data, pos) + print 'n, more', n, more + pos += n + if not more: + break + else: + self.wait_for_write() + left -= n + return pos + + write = send + + # XXX verify this + sendall = send + + # XXX writev + + def readv (self, _ignore): + raise NotImplementedError + + def shutdown (self, how=None): + more = 1 + while more: + more = self.conn_ob.shutdown() + + def close (self): + try: + coro.with_timeout (1, self.shutdown) + except coro.TimeoutError: + pass + finally: + coro.sock.close (self) diff --git a/coro/ssl/s2n/_s2n.pyx b/coro/ssl/s2n/_s2n.pyx new file mode 100644 index 0000000..c1d5a7a --- /dev/null +++ b/coro/ssl/s2n/_s2n.pyx @@ -0,0 +1,279 @@ +# -*- Mode: Cython -*- + +from cpython.bytes cimport PyBytes_FromStringAndSize +from libc.stdint cimport uint64_t, uint32_t, uint16_t, uint8_t + +class S2N: + SSLv2 = 20 + SSLv3 = 30 + TLS10 = 31 + TLS11 = 32 + TLS12 = 33 + +cdef extern from "s2n.h": + struct s2n_config + int s2n_errno + int s2n_init () + int s2n_cleanup () + + s2n_config *s2n_config_new () + int s2n_config_free (s2n_config *config) + int s2n_config_free_dhparams (s2n_config *config) + int s2n_config_free_cert_chain_and_key (s2n_config *config) + const char *s2n_strerror (int error, const char *lang) + int s2n_config_add_cert_chain_and_key ( + s2n_config *config, + char *cert_chain_pem, + char *private_key_pem + ) + int s2n_config_add_cert_chain_and_key_with_status ( + s2n_config *config, + char *cert_chain_pem, + char *private_key_pem, + const uint8_t *status, + uint32_t length + ) + int s2n_config_add_dhparams (s2n_config *config, char *dhparams_pem) + int s2n_config_set_key_exchange_preferences (s2n_config *config, const char *preferences) + int s2n_config_set_cipher_preferences ( + s2n_config *config, + const char *version + ) + int s2n_config_set_protocol_preferences ( + s2n_config *config, + const char * const *protocols, + int protocol_count + ) + + ctypedef enum s2n_status_request_type: + S2N_STATUS_REQUEST_NONE = 0, + S2N_STATUS_REQUEST_OCSP = 1 + + int s2n_config_set_status_request_type (s2n_config *config, s2n_status_request_type type) + + struct s2n_connection + + ctypedef enum s2n_mode: + S2N_SERVER, + S2N_CLIENT + + s2n_connection *s2n_connection_new (s2n_mode mode) + int s2n_connection_set_config (s2n_connection *conn, s2n_config *config) + + int s2n_connection_set_fd (s2n_connection *conn, int readfd) + int s2n_connection_set_read_fd (s2n_connection *conn, int readfd) + int s2n_connection_set_write_fd (s2n_connection *conn, int readfd) + + ctypedef enum s2n_blinding: + S2N_BUILT_IN_BLINDING, + S2N_SELF_SERVICE_BLINDING + + int s2n_connection_set_blinding (s2n_connection *conn, s2n_blinding blinding) + int s2n_connection_get_delay (s2n_connection *conn) + + int s2n_set_server_name (s2n_connection *conn, const char *server_name) + const char *s2n_get_server_name (s2n_connection *conn) + const char *s2n_get_application_protocol (s2n_connection *conn) + const uint8_t *s2n_connection_get_ocsp_response (s2n_connection *conn, uint32_t *length) + + int s2n_negotiate (s2n_connection *conn, int *more) + ssize_t s2n_send (s2n_connection *conn, void *buf, ssize_t size, int *more) + ssize_t s2n_recv (s2n_connection *conn, void *buf, ssize_t size, int *more) + + int s2n_connection_wipe (s2n_connection *conn) + int s2n_connection_free (s2n_connection *conn) + int s2n_shutdown (s2n_connection *conn, int *more) + + uint64_t s2n_connection_get_wire_bytes_in (s2n_connection *conn) + uint64_t s2n_connection_get_wire_bytes_out (s2n_connection *conn) + int s2n_connection_get_client_protocol_version (s2n_connection *conn) + int s2n_connection_get_server_protocol_version (s2n_connection *conn) + int s2n_connection_get_actual_protocol_version (s2n_connection *conn) + int s2n_connection_get_client_hello_version (s2n_connection *conn) + const char *s2n_connection_get_cipher (s2n_connection *conn) + int s2n_connection_get_alert (s2n_connection *conn) + +class MODE: + SERVER = S2N_SERVER + CLIENT = S2N_CLIENT + +class Error (Exception): + pass + +class Want (Exception): + pass + +class WantRead (Want): + pass + +class WantWrite (Want): + pass + +cdef raise_s2n_error(): + raise Error (s2n_strerror (s2n_errno, "EN")) + +cdef check (int n): + if n != 0: + raise_s2n_error() + +def init(): + check (s2n_init()) + +def cleanup(): + check (s2n_cleanup()) + +init() + +cdef class Config: + + cdef s2n_config * c + + def __init__ (self): + self.c = s2n_config_new() + if not self.c: + raise_s2n_error() + + def __del__ (self): + if self.c: + check (s2n_config_free (self.c)) + + def set_cipher_preferences (self, bytes version): + check (s2n_config_set_cipher_preferences (self.c, version)) + + def add_cert_chain_and_key (self, bytes chain_pem, bytes skey_pem): + check (s2n_config_add_cert_chain_and_key (self.c, chain_pem, skey_pem)) + + def add_cert_chain_and_key_with_status (self, bytes chain_pem, bytes skey_pem): + cdef uint8_t status[512] + check (s2n_config_add_cert_chain_and_key_with_status (self.c, chain_pem, skey_pem, &status[0], 512)) + return status + + def add_dhparams (self, bytes dhparams_pem): + check (s2n_config_add_dhparams (self.c, dhparams_pem)) + + def set_protocol_preferences (self, protocols): + cdef char * protos[50] + cdef int count = 0 + assert (len(protocols) < 50) + for i, proto in enumerate (protocols): + protos[i] = proto + count = i + check (s2n_config_set_protocol_preferences (self.c, protos, count)) + + def set_status_request_type (self, s2n_status_request_type stype): + check (s2n_config_set_status_request_type (self.c, stype)) + +cdef class Connection: + + cdef s2n_connection * conn + + def __init__ (self, s2n_mode mode): + self.conn = s2n_connection_new (mode) + if not self.conn: + raise_s2n_error() + + def __del__ (self): + if self.conn: + check (s2n_connection_free (self.conn)) + + def set_config (self, Config cfg): + check (s2n_connection_set_config (self.conn, cfg.c)) + + def set_fd (self, int readfd): + check (s2n_connection_set_fd (self.conn, readfd)) + + def set_read_fd (self, int readfd): + check (s2n_connection_set_read_fd (self.conn, readfd)) + + def set_write_fd (self, int readfd): + check (s2n_connection_set_write_fd (self.conn, readfd)) + + def set_server_name (self, bytes server_name): + check (s2n_set_server_name (self.conn, server_name)) + + def get_server_name (self): + cdef char * name = s2n_get_server_name (self.conn) + if name is not NULL: + return name + else: + return None + + def set_blinding (self, s2n_blinding blinding): + check (s2n_connection_set_blinding (self.conn, blinding)) + + def get_delay (self): + return s2n_connection_get_delay (self.conn) + + def get_wire_bytes (self): + return ( + s2n_connection_get_wire_bytes_in (self.conn), + s2n_connection_get_wire_bytes_out (self.conn), + ) + + def get_client_hello_version (self): + return s2n_connection_get_client_hello_version (self.conn) + + def get_client_protocol_version (self): + return s2n_connection_get_client_protocol_version (self.conn) + + def get_server_protocol_version (self): + return s2n_connection_get_server_protocol_version (self.conn) + + def get_actual_protocol_version (self): + return s2n_connection_get_actual_protocol_version (self.conn) + + def get_application_protocol (self): + return s2n_get_application_protocol (self.conn) + + def get_ocsp_response (self): + cdef uint8_t * r + cdef uint32_t length + r = s2n_connection_get_ocsp_response (self.conn, &length) + return r[:length] + + def get_alert (self): + return s2n_connection_get_alert (self.conn) + + def get_cipher (self): + return s2n_connection_get_cipher (self.conn) + + # I/O + + def negotiate (self): + cdef int more + cdef int r = s2n_negotiate (self.conn, &more) + if more: + return more + else: + check (r) + return more + + def send (self, bytes data, int pos=0): + cdef int more + cdef ssize_t n + assert (pos < len(data)) + n = s2n_send (self.conn, (data) + pos, len(data) - pos, &more) + if n < 0: + if more: + return 0, more + else: + raise_s2n_error() + else: + return n, more + + def recv (self, ssize_t size): + cdef int more + cdef bytes result = PyBytes_FromStringAndSize (NULL, size) + cdef ssize_t n = s2n_recv (self.conn, result, size, &more) + if n < 0: + if more: + return b'', more + else: + raise_s2n_error() + else: + return result[:n], more + + def shutdown (self): + cdef int more + check (s2n_shutdown (self.conn, &more)) + return more diff --git a/coro/ssl/s2n/s2n.pyx b/coro/ssl/s2n/s2n.pyx deleted file mode 100644 index 83690d1..0000000 --- a/coro/ssl/s2n/s2n.pyx +++ /dev/null @@ -1,109 +0,0 @@ -# -*- Mode: Cython -*- - -from libc.stdint cimport uint64_t, uint32_t, uint16_t, uint8_t - -#define S2N_SSLv2 20 -#define S2N_SSLv3 30 -#define S2N_TLS10 31 -#define S2N_TLS11 32 -#define S2N_TLS12 33 - -class S2N: - SSLv2 = 20 - SSLv3 = 30 - TLS10 = 31 - TLS11 = 32 - TLS12 = 33 - -cdef extern from "s2n.h": - - struct s2n_config - - int s2n_errno - - int s2n_init () - int s2n_cleanup () - s2n_config *s2n_config_new () - int s2n_config_free (s2n_config *config) - int s2n_config_free_dhparams (s2n_config *config) - int s2n_config_free_cert_chain_and_key (s2n_config *config) - const char *s2n_strerror (int error, const char *lang) - - int s2n_config_add_cert_chain_and_key (s2n_config *config, char *cert_chain_pem, char *private_key_pem) - int s2n_config_add_cert_chain_and_key_with_status ( - s2n_config *config, - char *cert_chain_pem, - char *private_key_pem, - const uint8_t *status, - uint32_t length - ) - int s2n_config_add_dhparams (s2n_config *config, char *dhparams_pem) - int s2n_config_set_key_exchange_preferences (s2n_config *config, const char *preferences) - int s2n_config_set_cipher_preferences (s2n_config *config, const char *version) - int s2n_config_set_protocol_preferences (s2n_config *config, const char * const *protocols, int protocol_count) - ctypedef enum s2n_status_request_type: - S2N_STATUS_REQUEST_NONE = 0, - S2N_STATUS_REQUEST_OCSP = 1 - int s2n_config_set_status_request_type (s2n_config *config, s2n_status_request_type type) - - struct s2n_connection - ctypedef enum s2n_mode: - S2N_SERVER, - S2N_CLIENT - s2n_connection *s2n_connection_new (s2n_mode mode) - int s2n_connection_set_config (s2n_connection *conn, s2n_config *config) - - int s2n_connection_set_fd (s2n_connection *conn, int readfd) - int s2n_connection_set_read_fd (s2n_connection *conn, int readfd) - int s2n_connection_set_write_fd (s2n_connection *conn, int readfd) - - ctypedef enum s2n_blinding: - S2N_BUILT_IN_BLINDING, - S2N_SELF_SERVICE_BLINDING - int s2n_connection_set_blinding (s2n_connection *conn, s2n_blinding blinding) - int s2n_connection_get_delay (s2n_connection *conn) - - int s2n_set_server_name (s2n_connection *conn, const char *server_name) - const char *s2n_get_server_name (s2n_connection *conn) - const char *s2n_get_application_protocol (s2n_connection *conn) - const uint8_t *s2n_connection_get_ocsp_response (s2n_connection *conn, uint32_t *length) - - int s2n_negotiate (s2n_connection *conn, int *more) - ssize_t s2n_send (s2n_connection *conn, void *buf, ssize_t size, int *more) - ssize_t s2n_recv (s2n_connection *conn, void *buf, ssize_t size, int *more) - - int s2n_connection_wipe (s2n_connection *conn) - int s2n_connection_free (s2n_connection *conn) - int s2n_shutdown (s2n_connection *conn, int *more) - - uint64_t s2n_connection_get_wire_bytes_in (s2n_connection *conn) - uint64_t s2n_connection_get_wire_bytes_out (s2n_connection *conn) - int s2n_connection_get_client_protocol_version (s2n_connection *conn) - int s2n_connection_get_server_protocol_version (s2n_connection *conn) - int s2n_connection_get_actual_protocol_version (s2n_connection *conn) - int s2n_connection_get_client_hello_version (s2n_connection *conn) - const char *s2n_connection_get_cipher (s2n_connection *conn) - int s2n_connection_get_alert (s2n_connection *conn) - -class Error (Exception): - pass - -cdef raise_s2n_error(): - raise Error (s2n_strerror (s2n_errno, "EN")) - -cdef check (int n): - if n != 0: - raise_s2n_error() - -def init(): - check (s2n_init()) - -def cleanup(): - check (s2n_cleanup()) - -cdef class config: - cdef s2n_config * c - def __init__ (self): - self.c = s2n_config_new() - if not self.c: - raise_s2n_error() diff --git a/coro/ssl/s2n/test/serve.py b/coro/ssl/s2n/test/serve.py new file mode 100644 index 0000000..85f63ab --- /dev/null +++ b/coro/ssl/s2n/test/serve.py @@ -0,0 +1,91 @@ +# -*- Mode: Python -*- + +import coro + +from coro.ssl.s2n import sock, Config, MODE + +# EC doesn't work with s2n? +key = """-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEjlUGmDtQ6knDdlAJSNkkBsrscdSAwCzwIIlf46CIgYoAoGCCqGSM49 +AwEHoUQDQgAEmscdfOltpZJIAiDBfCXEIiG98Dhlqh6uUkj0+NkmE4WNrJpMDYD/ +ZEcJ0pVEwjIZyAcb5Sl6tk4FuCkIfPmPRA== +-----END EC PRIVATE KEY----- +""" + +crt = """-----BEGIN CERTIFICATE----- +MIICCDCCAbCgAwIBAgIJAPMOQCT1kfmpMAkGByqGSM49BAEwOzEUMBIGA1UEAxML +ZXhhbXBsZS5jb20xFjAUBgNVBAoTDUV4YW1wbGUsIEluYy4xCzAJBgNVBAYTAlVT +MB4XDTE1MDcxMTIxMDczOFoXDTE1MDgxMDIxMDczOFowOzEUMBIGA1UEAxMLZXhh +bXBsZS5jb20xFjAUBgNVBAoTDUV4YW1wbGUsIEluYy4xCzAJBgNVBAYTAlVTMFkw +EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEmscdfOltpZJIAiDBfCXEIiG98Dhlqh6u +Ukj0+NkmE4WNrJpMDYD/ZEcJ0pVEwjIZyAcb5Sl6tk4FuCkIfPmPRKOBnTCBmjAd +BgNVHQ4EFgQUQR8Rw6gWKfmS8aag5pfjsNHblqMwawYDVR0jBGQwYoAUQR8Rw6gW +KfmS8aag5pfjsNHblqOhP6Q9MDsxFDASBgNVBAMTC2V4YW1wbGUuY29tMRYwFAYD +VQQKEw1FeGFtcGxlLCBJbmMuMQswCQYDVQQGEwJVU4IJAPMOQCT1kfmpMAwGA1Ud +EwQFMAMBAf8wCQYHKoZIzj0EAQNHADBEAiBXAW8xR517l9RbtvMt27lGGR8dcAcP +Tr9bHXvHzaDwLQIgAqbJQAkEhPdNZPzuxbtdTSMM1jijO+l+2hwJMScCpBM= +-----END CERTIFICATE----- +""" + +key = """-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDbXarRaFWNbH7zt7qNLoTV02lEv5FyF0cOFuLXp2uSTpvlfQje +6CCJ4KvEE3cbRr3XReWn9TOLCidvnrIUZ61EpzJ0hNpRoAOb9zzHAxmxrvwRP+xz +KvR57bgsq44p7mZ97N453HFz54mIaLTsAR93qPP5Ao4z3kQi8IOKLP9UHQIDAQAB +AoGAJ4phsO9ShHRrCbkzWiFpdjVuQyMYr2z8tNBxQRf/btbWiO4ZvDwxKUkjDOvJ +S1RcAcKqm7S5/rTs2NTNGpp5g5HfqaqfnRvRTSUwrY9Y7qoouPn61KiDcBgghTL9 +GH0v7pOvF0pOpaL/Q5jVv8tWjWXahtpmgHwsBIE44cavyc0CQQD7TfEvG+s17m+e +x0BZV98VpZ6mtP2oC0scgCn+GQ1ZNnhEfFawpY1gYxcQ2H7YP1xCNX0mHg52daum ++pdzXG/bAkEA33b0RzfUJa4Vu2QmuZD50lL6M13u1w60NkHtwQUghVC1BzNLktJs +uQRzHjrxHNy13RzpMWyYND1+Y+DpMwDpZwJAWyG6stC3DUm4JKYxCbU56wmybNX5 +nnTp+h3oHINNOers1jkY3tpKWIfWl39LEHR5qnDnP2lq6T5mzxjUzzrYPQJBALjF +VfRxMCQ7zlJU3ERBoJ+M5r6EY9FEojPezaT1BU/WTOj4O/vZq/ZLvJf5apZP1LxQ +hGzOewdu9UvGk2wNy+8CQQDRsF/US06+PrSsWn0uiu+1Fy80PO5SoQmqNDL/olHw +61trvDaDJPIWzqxXuFwhVXZlTFlnaLxg3gAG+bMEqtSg +-----END RSA PRIVATE KEY----- +""" + +crt = """-----BEGIN CERTIFICATE----- +MIICjzCCAfigAwIBAgIJAOFA3myvL0dJMA0GCSqGSIb3DQEBBQUAMDoxFDASBgNV +BAMTC2V4YW1wbGUuY29tMRUwEwYDVQQKEwxFeGFtcGxlLCBJbmMxCzAJBgNVBAYT +AlVTMB4XDTE1MDcxMTIxMjExOVoXDTE2MDcxMDIxMjExOVowOjEUMBIGA1UEAxML +ZXhhbXBsZS5jb20xFTATBgNVBAoTDEV4YW1wbGUsIEluYzELMAkGA1UEBhMCVVMw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANtdqtFoVY1sfvO3uo0uhNXTaUS/ +kXIXRw4W4tena5JOm+V9CN7oIIngq8QTdxtGvddF5af1M4sKJ2+eshRnrUSnMnSE +2lGgA5v3PMcDGbGu/BE/7HMq9HntuCyrjinuZn3s3jnccXPniYhotOwBH3eo8/kC +jjPeRCLwg4os/1QdAgMBAAGjgZwwgZkwHQYDVR0OBBYEFIj+uemrlTyRZdHTwx8c +dvw6lXj+MGoGA1UdIwRjMGGAFIj+uemrlTyRZdHTwx8cdvw6lXj+oT6kPDA6MRQw +EgYDVQQDEwtleGFtcGxlLmNvbTEVMBMGA1UEChMMRXhhbXBsZSwgSW5jMQswCQYD +VQQGEwJVU4IJAOFA3myvL0dJMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +gYEAIxmqzFn2JD4+Yp+wr2P+KiqCeP1NeNuDUsfqbx4p5xgM9fEMX3lnZsWeiCkX +2uv5idrZoUfBAkt1ao4xRAlRjc2TClwK7pNj3JKQQ0PdHVmPsJRQwxafcB1taXFS +14bcAwcAimx5zqYfMZho7tBUxpRd5vp6UVi1nR/9pIlZ7wE= +-----END CERTIFICATE----- +""" + + +cfg = Config() +cfg.add_cert_chain_and_key (crt, key) + +def echo (conn): + conn.send ('Howdy!\r\n') + while 1: + block = conn.recv (1024) + if not block: + break + else: + conn.send (block) + +def serve (port): + s = sock (cfg, mode=MODE.SERVER) + s.bind (('', port)) + s.listen (10) + while 1: + conn, addr = s.accept() + coro.spawn (echo, conn) + +coro.spawn (serve, 7777) +coro.event_loop() + diff --git a/coro/ssl/s2n/test/t0.py b/coro/ssl/s2n/test/t0.py new file mode 100644 index 0000000..1514b3c --- /dev/null +++ b/coro/ssl/s2n/test/t0.py @@ -0,0 +1,91 @@ +# -*- Mode: Python -*- + +import coro + +from coro.ssl.s2n import sock, Config, MODE + +# EC doesn't work with s2n? +key = """-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEjlUGmDtQ6knDdlAJSNkkBsrscdSAwCzwIIlf46CIgYoAoGCCqGSM49 +AwEHoUQDQgAEmscdfOltpZJIAiDBfCXEIiG98Dhlqh6uUkj0+NkmE4WNrJpMDYD/ +ZEcJ0pVEwjIZyAcb5Sl6tk4FuCkIfPmPRA== +-----END EC PRIVATE KEY----- +""" + +crt = """-----BEGIN CERTIFICATE----- +MIICCDCCAbCgAwIBAgIJAPMOQCT1kfmpMAkGByqGSM49BAEwOzEUMBIGA1UEAxML +ZXhhbXBsZS5jb20xFjAUBgNVBAoTDUV4YW1wbGUsIEluYy4xCzAJBgNVBAYTAlVT +MB4XDTE1MDcxMTIxMDczOFoXDTE1MDgxMDIxMDczOFowOzEUMBIGA1UEAxMLZXhh +bXBsZS5jb20xFjAUBgNVBAoTDUV4YW1wbGUsIEluYy4xCzAJBgNVBAYTAlVTMFkw +EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEmscdfOltpZJIAiDBfCXEIiG98Dhlqh6u +Ukj0+NkmE4WNrJpMDYD/ZEcJ0pVEwjIZyAcb5Sl6tk4FuCkIfPmPRKOBnTCBmjAd +BgNVHQ4EFgQUQR8Rw6gWKfmS8aag5pfjsNHblqMwawYDVR0jBGQwYoAUQR8Rw6gW +KfmS8aag5pfjsNHblqOhP6Q9MDsxFDASBgNVBAMTC2V4YW1wbGUuY29tMRYwFAYD +VQQKEw1FeGFtcGxlLCBJbmMuMQswCQYDVQQGEwJVU4IJAPMOQCT1kfmpMAwGA1Ud +EwQFMAMBAf8wCQYHKoZIzj0EAQNHADBEAiBXAW8xR517l9RbtvMt27lGGR8dcAcP +Tr9bHXvHzaDwLQIgAqbJQAkEhPdNZPzuxbtdTSMM1jijO+l+2hwJMScCpBM= +-----END CERTIFICATE----- +""" + +key = """-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDbXarRaFWNbH7zt7qNLoTV02lEv5FyF0cOFuLXp2uSTpvlfQje +6CCJ4KvEE3cbRr3XReWn9TOLCidvnrIUZ61EpzJ0hNpRoAOb9zzHAxmxrvwRP+xz +KvR57bgsq44p7mZ97N453HFz54mIaLTsAR93qPP5Ao4z3kQi8IOKLP9UHQIDAQAB +AoGAJ4phsO9ShHRrCbkzWiFpdjVuQyMYr2z8tNBxQRf/btbWiO4ZvDwxKUkjDOvJ +S1RcAcKqm7S5/rTs2NTNGpp5g5HfqaqfnRvRTSUwrY9Y7qoouPn61KiDcBgghTL9 +GH0v7pOvF0pOpaL/Q5jVv8tWjWXahtpmgHwsBIE44cavyc0CQQD7TfEvG+s17m+e +x0BZV98VpZ6mtP2oC0scgCn+GQ1ZNnhEfFawpY1gYxcQ2H7YP1xCNX0mHg52daum ++pdzXG/bAkEA33b0RzfUJa4Vu2QmuZD50lL6M13u1w60NkHtwQUghVC1BzNLktJs +uQRzHjrxHNy13RzpMWyYND1+Y+DpMwDpZwJAWyG6stC3DUm4JKYxCbU56wmybNX5 +nnTp+h3oHINNOers1jkY3tpKWIfWl39LEHR5qnDnP2lq6T5mzxjUzzrYPQJBALjF +VfRxMCQ7zlJU3ERBoJ+M5r6EY9FEojPezaT1BU/WTOj4O/vZq/ZLvJf5apZP1LxQ +hGzOewdu9UvGk2wNy+8CQQDRsF/US06+PrSsWn0uiu+1Fy80PO5SoQmqNDL/olHw +61trvDaDJPIWzqxXuFwhVXZlTFlnaLxg3gAG+bMEqtSg +-----END RSA PRIVATE KEY----- +""" + +crt = """-----BEGIN CERTIFICATE----- +MIICjzCCAfigAwIBAgIJAOFA3myvL0dJMA0GCSqGSIb3DQEBBQUAMDoxFDASBgNV +BAMTC2V4YW1wbGUuY29tMRUwEwYDVQQKEwxFeGFtcGxlLCBJbmMxCzAJBgNVBAYT +AlVTMB4XDTE1MDcxMTIxMjExOVoXDTE2MDcxMDIxMjExOVowOjEUMBIGA1UEAxML +ZXhhbXBsZS5jb20xFTATBgNVBAoTDEV4YW1wbGUsIEluYzELMAkGA1UEBhMCVVMw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANtdqtFoVY1sfvO3uo0uhNXTaUS/ +kXIXRw4W4tena5JOm+V9CN7oIIngq8QTdxtGvddF5af1M4sKJ2+eshRnrUSnMnSE +2lGgA5v3PMcDGbGu/BE/7HMq9HntuCyrjinuZn3s3jnccXPniYhotOwBH3eo8/kC +jjPeRCLwg4os/1QdAgMBAAGjgZwwgZkwHQYDVR0OBBYEFIj+uemrlTyRZdHTwx8c +dvw6lXj+MGoGA1UdIwRjMGGAFIj+uemrlTyRZdHTwx8cdvw6lXj+oT6kPDA6MRQw +EgYDVQQDEwtleGFtcGxlLmNvbTEVMBMGA1UEChMMRXhhbXBsZSwgSW5jMQswCQYD +VQQGEwJVU4IJAOFA3myvL0dJMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +gYEAIxmqzFn2JD4+Yp+wr2P+KiqCeP1NeNuDUsfqbx4p5xgM9fEMX3lnZsWeiCkX +2uv5idrZoUfBAkt1ao4xRAlRjc2TClwK7pNj3JKQQ0PdHVmPsJRQwxafcB1taXFS +14bcAwcAimx5zqYfMZho7tBUxpRd5vp6UVi1nR/9pIlZ7wE= +-----END CERTIFICATE----- +""" + + +cfg = Config() +cfg.add_cert_chain_and_key (crt, key) + +def go (conn): + try: + for i in range (10): + n = conn.send (crt) + print 'n, len(crt)', n, len(crt) + conn.shutdown() + finally: + conn.close() + +def serve (port): + s = sock (cfg, mode=MODE.SERVER) + s.bind (('', port)) + s.listen (10) + while 1: + conn, addr = s.accept() + coro.spawn (go, conn) + +coro.spawn (serve, 7777) +coro.event_loop() + diff --git a/setup.py b/setup.py index ed28523..b4ca9ed 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ def check_linux_aio(): # OS X: as of 10.9, openssl seems to have been completely removed. You'll need # to install from the sources. Once this is done, use '/usr/local/ssl/' for ossl_base. -ossl_base = '/usr' +ossl_base = '/usr/local/ssl' # Since openssl is deprecated on MacOSX 10.7+, look for homebrew installs homebrew_ossl_base = '/usr/local/opt/openssl' @@ -105,15 +105,16 @@ def O (path): ['coro/ssl/openssl.pyx'], depends=['coro/ssl/openssl.pxi'], # manual static link - # extra_link_args = [O('libcrypto.a'), O('libssl.a')], + #extra_link_args = [O('libcrypto.a'), O('libssl.a')], # link to an absolute location - # extra_link_args = ['-L %s -lcrypto -lssl' % (ossl_base,)] + extra_link_args = ['-L', '%s/lib' % (ossl_base), '-lcrypto', '-lssl'], # 'normal' link - libraries=['crypto', 'ssl'], + #libraries=['crypto', 'ssl'], include_dirs=[O('include')], cython_compile_time_env={'NPN': USE_NPN}, ) + # -------------------------------------------------------------------------- def path_join (*parts): @@ -123,13 +124,13 @@ def path_join (*parts): s2n_lib = path_join (s2n_base, 'lib') s2n_Extension = Extension ( - 'coro.ssl.s2n', - ['coro/ssl/s2n/s2n.pyx'], + 'coro.ssl.s2n._s2n', + ['coro/ssl/s2n/_s2n.pyx'], libraries = ['s2n'], #extra_link_args = ['-L', s2n_lib], #include_dirs = [path_join (s2n_base, 'api')], #extra_link_args = ['-L', s2n_lib, '-Wl,-rpath,%s' % (s2n_lib,)], - extra_link_args = ['-lgcc_eh'], + #extra_link_args = ['-lgcc_eh'], ) # -------------------------------------------------------------------------- From 2db4f8034621d7b598d8e87cc439383274643dc6 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sat, 11 Jul 2015 15:22:56 -0700 Subject: [PATCH 05/46] do a better job with openssl on darwin, and clean up the s2n_Extension. --- setup.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index b4ca9ed..5a0036f 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,9 @@ def check_linux_aio(): 'CORO_DEBUG': False, } +def path_join (*parts): + return os.path.join (*parts) + # -------------------------------------------------------------------------- # OpenSSL support # -------------------------------------------------------------------------- @@ -82,12 +85,21 @@ def check_linux_aio(): # OS X: as of 10.9, openssl seems to have been completely removed. You'll need # to install from the sources. Once this is done, use '/usr/local/ssl/' for ossl_base. -ossl_base = '/usr/local/ssl' +ossl_base = '/usr' # Since openssl is deprecated on MacOSX 10.7+, look for homebrew installs homebrew_ossl_base = '/usr/local/opt/openssl' -if os.uname()[0] == 'Darwin' and os.path.exists(homebrew_ossl_base): - ossl_base = homebrew_ossl_base +standard_ossl_base = '/usr/local/ssl' + +if os.uname()[0] == 'Darwin': + if os.path.exists (homebrew_ossl_base): + ossl_base = homebrew_ossl_base + elif os.path.exists (standard_ossl_base): + ossl_base = standard_ossl_base + else: + raise ValueError ("can't find an openssl installation") + +# look for generic 'make install' in /usr/local/ def O (path): return os.path.join (ossl_base, path) @@ -107,31 +119,24 @@ def O (path): # manual static link #extra_link_args = [O('libcrypto.a'), O('libssl.a')], # link to an absolute location - extra_link_args = ['-L', '%s/lib' % (ossl_base), '-lcrypto', '-lssl'], + #extra_link_args = ['-L', '%s/lib' % (ossl_base), '-lcrypto', '-lssl'], # 'normal' link - #libraries=['crypto', 'ssl'], + libraries=['crypto', 'ssl'], include_dirs=[O('include')], cython_compile_time_env={'NPN': USE_NPN}, ) # -------------------------------------------------------------------------- - -def path_join (*parts): - return os.path.join (*parts) - -s2n_base = '/Users/rushing/src/s2n/' -s2n_lib = path_join (s2n_base, 'lib') +# S2N support. +# -------------------------------------------------------------------------- s2n_Extension = Extension ( 'coro.ssl.s2n._s2n', ['coro/ssl/s2n/_s2n.pyx'], libraries = ['s2n'], - #extra_link_args = ['-L', s2n_lib], - #include_dirs = [path_join (s2n_base, 'api')], - #extra_link_args = ['-L', s2n_lib, '-Wl,-rpath,%s' % (s2n_lib,)], - #extra_link_args = ['-lgcc_eh'], ) + # -------------------------------------------------------------------------- setup ( From e80422320c27099e4d53778f364662a7e2542ad4 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 24 Aug 2015 15:10:05 -0700 Subject: [PATCH 06/46] remove debug prints. --- coro/ssl/s2n/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/coro/ssl/s2n/__init__.py b/coro/ssl/s2n/__init__.py index dc28282..8bccd99 100644 --- a/coro/ssl/s2n/__init__.py +++ b/coro/ssl/s2n/__init__.py @@ -12,7 +12,6 @@ class sock (coro.sock): # set mode automatically? def __init__ (self, cfg, fd=-1, verify=False, domain=coro.AF.INET, mode=MODE.SERVER): coro.sock.__init__ (self, fd=fd, domain=domain) - print 'fd=', self.fd self.cfg = cfg self.conn_ob = _s2n.Connection(mode) self.conn_ob.set_config (cfg) @@ -49,7 +48,6 @@ def recv (self, block_size): left = block_size while left: b, more = self.conn_ob.recv (left) - print 'more, b', more, repr(b) r.append (b) if not more: break @@ -77,7 +75,6 @@ def send (self, data): left = len(data) while left: n, more = self.conn_ob.send (data, pos) - print 'n, more', n, more pos += n if not more: break From bf2d6483c03b7e93b8e3d5c7fb48ce4695961455 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 24 Aug 2015 15:15:39 -0700 Subject: [PATCH 07/46] client test against serve.py using openssl. --- coro/ssl/s2n/test/openssl_client.py | 110 ++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 coro/ssl/s2n/test/openssl_client.py diff --git a/coro/ssl/s2n/test/openssl_client.py b/coro/ssl/s2n/test/openssl_client.py new file mode 100644 index 0000000..250c504 --- /dev/null +++ b/coro/ssl/s2n/test/openssl_client.py @@ -0,0 +1,110 @@ +# -*- Mode: Python -*- + +import coro +from coro.ssl import openssl +import coro.ssl +import coro.backdoor +import os +import random + +from coro.log import NoFacility +from hashlib import sha512 + +LOG = NoFacility() + +# use sha512 as a random-data generator. +# pre-generate 10MB of random data to iterate through + +def gen_random_data (seed='fnord'): + LOG ('generate data', 'start') + blocks = [] + h = sha512() + data = seed + nblocks = (1024 * 1024 * 10) / 64 + for i in range (nblocks): + h.update (data) + data = h.digest() + blocks.append (data) + LOG ('generate data', 'stop') + return blocks + +random_blocks = gen_random_data() + +class random_char_gen: + + def __init__ (self): + self.index = 0 + self.blocks = random_blocks + + def next (self): + result = self.blocks[self.index] + self.index += 1 + if self.index == len(self.blocks): + self.index = 0 + return result + +class random_block: + + def __init__ (self, gen): + self.gen = gen + self.buffer = self.gen.next() + self.random = random.Random (3141) + + def next (self, size): + while 1: + if size <= len(self.buffer): + result, self.buffer = self.buffer[:size], self.buffer[size:] + return result + else: + self.buffer += self.gen.next() + + def random_size (self): + size = self.random.randrange (10, 500) + return self.next (size) + +def feed (s): + global wbytes + blockgen = random_block (random_char_gen()) + while 1: + block = blockgen.random_size() + #LOG ('=>', block.encode ('hex')) + s.send (block) + wbytes += len (block) + #coro.sleep_relative (3) + +def session (addr): + LOG ('pid', os.getpid()) + global rbytes + ctx = openssl.ssl_ctx() + s = coro.ssl.sock (ctx) + s.connect (addr) + assert (s.recv (1024) == 'Howdy!\r\n') + coro.spawn (feed, s) + # generates the same stream of characters we sent + blockgen = random_block (random_char_gen()) + while 1: + data = s.recv (50) + #LOG ('<=', data.encode ('hex')) + rbytes += len (data) + assert (data == blockgen.next (len (data))) + +rbytes = 0 +wbytes = 0 + +def monitor (interval=10): + global rbytes, wbytes + while 1: + r0, w0 = rbytes, wbytes + coro.sleep_relative (interval) + r = rbytes - r0 + w = wbytes - w0 + LOG ('throughput', r / interval, w / interval) + +if __name__ == '__main__': + coro.spawn (session, ('127.0.0.1', 7777)) + coro.spawn (coro.backdoor.serve, unix_path='/tmp/openssl_client.bd', global_dict=globals()) + coro.spawn (monitor) + try: + coro.event_loop() + finally: + LOG ('done') From f08950cd19fcb0794a6c1cf417cebb0e698c9066 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 24 Aug 2015 15:16:11 -0700 Subject: [PATCH 08/46] add_cert_chain_and_key_with_status: use sizeof(status). --- coro/ssl/s2n/_s2n.pyx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coro/ssl/s2n/_s2n.pyx b/coro/ssl/s2n/_s2n.pyx index c1d5a7a..d5c04ef 100644 --- a/coro/ssl/s2n/_s2n.pyx +++ b/coro/ssl/s2n/_s2n.pyx @@ -3,6 +3,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize from libc.stdint cimport uint64_t, uint32_t, uint16_t, uint8_t + class S2N: SSLv2 = 20 SSLv3 = 30 @@ -10,6 +11,7 @@ class S2N: TLS11 = 32 TLS12 = 33 + cdef extern from "s2n.h": struct s2n_config int s2n_errno @@ -145,7 +147,7 @@ cdef class Config: def add_cert_chain_and_key_with_status (self, bytes chain_pem, bytes skey_pem): cdef uint8_t status[512] - check (s2n_config_add_cert_chain_and_key_with_status (self.c, chain_pem, skey_pem, &status[0], 512)) + check (s2n_config_add_cert_chain_and_key_with_status (self.c, chain_pem, skey_pem, &status[0], sizeof(status))) return status def add_dhparams (self, bytes dhparams_pem): From 739f351b1a0fdac2b97701e4b80bb8a95bb8606b Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 24 Aug 2015 15:16:46 -0700 Subject: [PATCH 09/46] track throughput. --- coro/ssl/s2n/test/serve.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/coro/ssl/s2n/test/serve.py b/coro/ssl/s2n/test/serve.py index 85f63ab..639cb60 100644 --- a/coro/ssl/s2n/test/serve.py +++ b/coro/ssl/s2n/test/serve.py @@ -70,13 +70,16 @@ cfg.add_cert_chain_and_key (crt, key) def echo (conn): + global rbytes, wbytes conn.send ('Howdy!\r\n') while 1: block = conn.recv (1024) if not block: break else: + rbytes += len(block) conn.send (block) + wbytes += len(block) def serve (port): s = sock (cfg, mode=MODE.SERVER) @@ -86,6 +89,22 @@ def serve (port): conn, addr = s.accept() coro.spawn (echo, conn) +from coro.log import NoFacility +LOG = NoFacility() + +rbytes = 0 +wbytes = 0 + +def monitor (interval=10): + global rbytes, wbytes + while 1: + r0, w0 = rbytes, wbytes + coro.sleep_relative (interval) + r = rbytes - r0 + w = wbytes - w0 + LOG ('throughput', r / interval, w / interval) + coro.spawn (serve, 7777) +coro.spawn (monitor) coro.event_loop() From a50d48ee555e4e5a5c23f2e5660dfa05a8705bea Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 14 Sep 2015 15:19:45 -0700 Subject: [PATCH 10/46] shutdown(): give a default of SHUT_WR. --- coro/socket.pxd | 2 +- coro/socket.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coro/socket.pxd b/coro/socket.pxd index 94118e8..89eaf24 100644 --- a/coro/socket.pxd +++ b/coro/socket.pxd @@ -197,7 +197,7 @@ cdef public class sock [ object sock_object, type sock_type ]: cpdef listen (self, int backlog) cpdef accept (self) cpdef accept_many (self, int max=?) - cpdef shutdown (self, int how) + cpdef shutdown (self, int how=?) cpdef getpeername (self) cpdef getsockname (self) cpdef dup(self) diff --git a/coro/socket.pyx b/coro/socket.pyx index 34aca0b..39a5b03 100644 --- a/coro/socket.pyx +++ b/coro/socket.pyx @@ -1104,7 +1104,7 @@ cdef public class sock [ object sock_object, type sock_type ]: count = count + 1 return result - cpdef shutdown (self, int how): + cpdef shutdown (self, int how=SHUT_WR): """Shutdown the socket. :param how: How to shut down the socket (see the shutdown(2) manpage). From 24384625e3e78b40d6fccf743fbbbae75ec872f6 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Mon, 14 Sep 2015 15:21:16 -0700 Subject: [PATCH 11/46] use larger buffer (for more reasonable performance numbers). don't send "howdy", this allows testing between coro++, blocking-mode s2n, and shrapnel. --- coro/ssl/s2n/test/openssl_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coro/ssl/s2n/test/openssl_client.py b/coro/ssl/s2n/test/openssl_client.py index 250c504..1931a1f 100644 --- a/coro/ssl/s2n/test/openssl_client.py +++ b/coro/ssl/s2n/test/openssl_client.py @@ -78,12 +78,11 @@ def session (addr): ctx = openssl.ssl_ctx() s = coro.ssl.sock (ctx) s.connect (addr) - assert (s.recv (1024) == 'Howdy!\r\n') coro.spawn (feed, s) # generates the same stream of characters we sent blockgen = random_block (random_char_gen()) while 1: - data = s.recv (50) + data = s.recv (500) #LOG ('<=', data.encode ('hex')) rbytes += len (data) assert (data == blockgen.next (len (data))) From 4aab70bfa3aebce8e26ba09c2e54888b6e281849 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 15 Sep 2015 15:43:09 -0700 Subject: [PATCH 12/46] probe for the cys2n module. --- setup.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 5a0036f..dbe1b8d 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,8 @@ def check_linux_aio(): def path_join (*parts): return os.path.join (*parts) +probed_extensions = [] + # -------------------------------------------------------------------------- # OpenSSL support # -------------------------------------------------------------------------- @@ -131,11 +133,18 @@ def O (path): # S2N support. # -------------------------------------------------------------------------- -s2n_Extension = Extension ( - 'coro.ssl.s2n._s2n', - ['coro/ssl/s2n/_s2n.pyx'], - libraries = ['s2n'], - ) +# probe for cys2n +try: + import cys2n + s2n_Extension = Extension ( + 'coro.ssl.s2n._s2n', + ['coro/ssl/s2n/_s2n.pyx'], + libraries = ['s2n'], + cython_compile_time_env=compile_time_env, + ) + probed_extensions.append (s2n_Extension) +except ImportError: + pass # -------------------------------------------------------------------------- @@ -208,8 +217,7 @@ def O (path): ), # the pre-computed openssl extension from above OpenSSL_Extension, - s2n_Extension, - ], + ] + probed_extensions, packages= find_packages(), py_modules = ['backdoor', 'coro.read_stream', 'coro_process', 'coro_unittest', ], scripts=['coro/log/catlog'], From d3fde61140f31095b5ddb2eb809c6ba0ce6fef67 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 15 Sep 2015 15:43:31 -0700 Subject: [PATCH 13/46] wrap cys2n's Connection object. --- coro/ssl/s2n/_s2n.pyx | 286 ++---------------------------------------- 1 file changed, 13 insertions(+), 273 deletions(-) diff --git a/coro/ssl/s2n/_s2n.pyx b/coro/ssl/s2n/_s2n.pyx index d5c04ef..635f62f 100644 --- a/coro/ssl/s2n/_s2n.pyx +++ b/coro/ssl/s2n/_s2n.pyx @@ -1,281 +1,21 @@ # -*- Mode: Cython -*- -from cpython.bytes cimport PyBytes_FromStringAndSize -from libc.stdint cimport uint64_t, uint32_t, uint16_t, uint8_t +from cys2n.cys2n cimport * +from coro._coro cimport sock +# here we override the default behavior of cys2n (to raise WantRead/WantWrite upon EAGAIN), +# hooking into shrapnel's event mechanism instead. -class S2N: - SSLv2 = 20 - SSLv3 = 30 - TLS10 = 31 - TLS11 = 32 - TLS12 = 33 +cdef class NonBlockingConnection (Connection): + cdef sock coro_sock -cdef extern from "s2n.h": - struct s2n_config - int s2n_errno - int s2n_init () - int s2n_cleanup () + def __init__ (self, s2n_mode mode, sock s): + Connection.__init__ (self, mode) + self.coro_sock = s - s2n_config *s2n_config_new () - int s2n_config_free (s2n_config *config) - int s2n_config_free_dhparams (s2n_config *config) - int s2n_config_free_cert_chain_and_key (s2n_config *config) - const char *s2n_strerror (int error, const char *lang) - int s2n_config_add_cert_chain_and_key ( - s2n_config *config, - char *cert_chain_pem, - char *private_key_pem - ) - int s2n_config_add_cert_chain_and_key_with_status ( - s2n_config *config, - char *cert_chain_pem, - char *private_key_pem, - const uint8_t *status, - uint32_t length - ) - int s2n_config_add_dhparams (s2n_config *config, char *dhparams_pem) - int s2n_config_set_key_exchange_preferences (s2n_config *config, const char *preferences) - int s2n_config_set_cipher_preferences ( - s2n_config *config, - const char *version - ) - int s2n_config_set_protocol_preferences ( - s2n_config *config, - const char * const *protocols, - int protocol_count - ) + cdef want_read (self): + self.coro_sock._wait_for_read() - ctypedef enum s2n_status_request_type: - S2N_STATUS_REQUEST_NONE = 0, - S2N_STATUS_REQUEST_OCSP = 1 - - int s2n_config_set_status_request_type (s2n_config *config, s2n_status_request_type type) - - struct s2n_connection - - ctypedef enum s2n_mode: - S2N_SERVER, - S2N_CLIENT - - s2n_connection *s2n_connection_new (s2n_mode mode) - int s2n_connection_set_config (s2n_connection *conn, s2n_config *config) - - int s2n_connection_set_fd (s2n_connection *conn, int readfd) - int s2n_connection_set_read_fd (s2n_connection *conn, int readfd) - int s2n_connection_set_write_fd (s2n_connection *conn, int readfd) - - ctypedef enum s2n_blinding: - S2N_BUILT_IN_BLINDING, - S2N_SELF_SERVICE_BLINDING - - int s2n_connection_set_blinding (s2n_connection *conn, s2n_blinding blinding) - int s2n_connection_get_delay (s2n_connection *conn) - - int s2n_set_server_name (s2n_connection *conn, const char *server_name) - const char *s2n_get_server_name (s2n_connection *conn) - const char *s2n_get_application_protocol (s2n_connection *conn) - const uint8_t *s2n_connection_get_ocsp_response (s2n_connection *conn, uint32_t *length) - - int s2n_negotiate (s2n_connection *conn, int *more) - ssize_t s2n_send (s2n_connection *conn, void *buf, ssize_t size, int *more) - ssize_t s2n_recv (s2n_connection *conn, void *buf, ssize_t size, int *more) - - int s2n_connection_wipe (s2n_connection *conn) - int s2n_connection_free (s2n_connection *conn) - int s2n_shutdown (s2n_connection *conn, int *more) - - uint64_t s2n_connection_get_wire_bytes_in (s2n_connection *conn) - uint64_t s2n_connection_get_wire_bytes_out (s2n_connection *conn) - int s2n_connection_get_client_protocol_version (s2n_connection *conn) - int s2n_connection_get_server_protocol_version (s2n_connection *conn) - int s2n_connection_get_actual_protocol_version (s2n_connection *conn) - int s2n_connection_get_client_hello_version (s2n_connection *conn) - const char *s2n_connection_get_cipher (s2n_connection *conn) - int s2n_connection_get_alert (s2n_connection *conn) - -class MODE: - SERVER = S2N_SERVER - CLIENT = S2N_CLIENT - -class Error (Exception): - pass - -class Want (Exception): - pass - -class WantRead (Want): - pass - -class WantWrite (Want): - pass - -cdef raise_s2n_error(): - raise Error (s2n_strerror (s2n_errno, "EN")) - -cdef check (int n): - if n != 0: - raise_s2n_error() - -def init(): - check (s2n_init()) - -def cleanup(): - check (s2n_cleanup()) - -init() - -cdef class Config: - - cdef s2n_config * c - - def __init__ (self): - self.c = s2n_config_new() - if not self.c: - raise_s2n_error() - - def __del__ (self): - if self.c: - check (s2n_config_free (self.c)) - - def set_cipher_preferences (self, bytes version): - check (s2n_config_set_cipher_preferences (self.c, version)) - - def add_cert_chain_and_key (self, bytes chain_pem, bytes skey_pem): - check (s2n_config_add_cert_chain_and_key (self.c, chain_pem, skey_pem)) - - def add_cert_chain_and_key_with_status (self, bytes chain_pem, bytes skey_pem): - cdef uint8_t status[512] - check (s2n_config_add_cert_chain_and_key_with_status (self.c, chain_pem, skey_pem, &status[0], sizeof(status))) - return status - - def add_dhparams (self, bytes dhparams_pem): - check (s2n_config_add_dhparams (self.c, dhparams_pem)) - - def set_protocol_preferences (self, protocols): - cdef char * protos[50] - cdef int count = 0 - assert (len(protocols) < 50) - for i, proto in enumerate (protocols): - protos[i] = proto - count = i - check (s2n_config_set_protocol_preferences (self.c, protos, count)) - - def set_status_request_type (self, s2n_status_request_type stype): - check (s2n_config_set_status_request_type (self.c, stype)) - -cdef class Connection: - - cdef s2n_connection * conn - - def __init__ (self, s2n_mode mode): - self.conn = s2n_connection_new (mode) - if not self.conn: - raise_s2n_error() - - def __del__ (self): - if self.conn: - check (s2n_connection_free (self.conn)) - - def set_config (self, Config cfg): - check (s2n_connection_set_config (self.conn, cfg.c)) - - def set_fd (self, int readfd): - check (s2n_connection_set_fd (self.conn, readfd)) - - def set_read_fd (self, int readfd): - check (s2n_connection_set_read_fd (self.conn, readfd)) - - def set_write_fd (self, int readfd): - check (s2n_connection_set_write_fd (self.conn, readfd)) - - def set_server_name (self, bytes server_name): - check (s2n_set_server_name (self.conn, server_name)) - - def get_server_name (self): - cdef char * name = s2n_get_server_name (self.conn) - if name is not NULL: - return name - else: - return None - - def set_blinding (self, s2n_blinding blinding): - check (s2n_connection_set_blinding (self.conn, blinding)) - - def get_delay (self): - return s2n_connection_get_delay (self.conn) - - def get_wire_bytes (self): - return ( - s2n_connection_get_wire_bytes_in (self.conn), - s2n_connection_get_wire_bytes_out (self.conn), - ) - - def get_client_hello_version (self): - return s2n_connection_get_client_hello_version (self.conn) - - def get_client_protocol_version (self): - return s2n_connection_get_client_protocol_version (self.conn) - - def get_server_protocol_version (self): - return s2n_connection_get_server_protocol_version (self.conn) - - def get_actual_protocol_version (self): - return s2n_connection_get_actual_protocol_version (self.conn) - - def get_application_protocol (self): - return s2n_get_application_protocol (self.conn) - - def get_ocsp_response (self): - cdef uint8_t * r - cdef uint32_t length - r = s2n_connection_get_ocsp_response (self.conn, &length) - return r[:length] - - def get_alert (self): - return s2n_connection_get_alert (self.conn) - - def get_cipher (self): - return s2n_connection_get_cipher (self.conn) - - # I/O - - def negotiate (self): - cdef int more - cdef int r = s2n_negotiate (self.conn, &more) - if more: - return more - else: - check (r) - return more - - def send (self, bytes data, int pos=0): - cdef int more - cdef ssize_t n - assert (pos < len(data)) - n = s2n_send (self.conn, (data) + pos, len(data) - pos, &more) - if n < 0: - if more: - return 0, more - else: - raise_s2n_error() - else: - return n, more - - def recv (self, ssize_t size): - cdef int more - cdef bytes result = PyBytes_FromStringAndSize (NULL, size) - cdef ssize_t n = s2n_recv (self.conn, result, size, &more) - if n < 0: - if more: - return b'', more - else: - raise_s2n_error() - else: - return result[:n], more - - def shutdown (self): - cdef int more - check (s2n_shutdown (self.conn, &more)) - return more + cdef want_write (self): + self.coro_sock._wait_for_write() From dd48873481e6652fa585affaca431bed5b0eb3b3 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 15 Sep 2015 15:43:52 -0700 Subject: [PATCH 14/46] rewritten using cys2n. --- coro/ssl/s2n/__init__.py | 51 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/coro/ssl/s2n/__init__.py b/coro/ssl/s2n/__init__.py index 8bccd99..e8fb34e 100644 --- a/coro/ssl/s2n/__init__.py +++ b/coro/ssl/s2n/__init__.py @@ -1,36 +1,42 @@ # -*- Mode: Python -*- import coro +from cys2n import MODE +from ._s2n import * -from ._s2n import Config, Connection, S2N, MODE +# Note: the plan is to push this class into _s2n.pyx as well. -Config = _s2n.Config +class S2NSocket (coro.sock): -class sock (coro.sock): - - # XXX maybe delay creating connection object until either accept or connect in order to - # set mode automatically? - def __init__ (self, cfg, fd=-1, verify=False, domain=coro.AF.INET, mode=MODE.SERVER): - coro.sock.__init__ (self, fd=fd, domain=domain) + def __init__ (self, cfg, fd=-1, verify=False, mode=MODE.SERVER): + coro.sock.__init__ (self, fd=fd) self.cfg = cfg - self.conn_ob = _s2n.Connection(mode) - self.conn_ob.set_config (cfg) - self.conn_ob.set_fd (fd) + self.s2n_conn = NonBlockingConnection (mode, self) + self.s2n_conn.set_config (cfg) + self.s2n_conn.set_fd (fd) + self.negotiated = False # XXX verify def __repr__ (self): return '' % (self.fd, id (self)) + def _check_negotiated (self): + if not self.negotiated: + self.negotiate() + + def negotiate (self): + while 1: + blocked = self.s2n_conn.negotiate() + if not blocked: + break + self.negotiated = True + def accept (self): conn, addr = coro.sock.accept (self) try: - new = self.__class__ (self.cfg, domain=conn.domain, fd=conn.fd, mode=MODE.SERVER) + new = self.__class__ (self.cfg, fd=conn.fd, mode=MODE.SERVER) # ...avoid having socket.pyx close the fd conn.fd = -1 - while 1: - more = new.conn_ob.negotiate() - if not more: - break return new, addr except: conn.close() @@ -38,16 +44,14 @@ def accept (self): def connect (self, addr): coro.sock.connect (self, addr) - while 1: - more = new.conn_ob.negotiate() - if not more: - break + self.negotiate() def recv (self, block_size): + self._check_negotiated() r = [] left = block_size while left: - b, more = self.conn_ob.recv (left) + b, more = self.s2n_conn.recv (left) r.append (b) if not more: break @@ -71,10 +75,11 @@ def recv_exact (self, size): return ''.join (r) def send (self, data): + self._check_negotiated() pos = 0 left = len(data) while left: - n, more = self.conn_ob.send (data, pos) + n, more = self.s2n_conn.send (data, pos) pos += n if not more: break @@ -96,7 +101,7 @@ def readv (self, _ignore): def shutdown (self, how=None): more = 1 while more: - more = self.conn_ob.shutdown() + more = self.s2n_conn.shutdown() def close (self): try: From 078f9735d58401925e739378f7279d7f18fffd37 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 15 Sep 2015 15:44:41 -0700 Subject: [PATCH 15/46] use cys2n, test alpn. --- coro/ssl/s2n/test/t0.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/coro/ssl/s2n/test/t0.py b/coro/ssl/s2n/test/t0.py index 1514b3c..eedfb16 100644 --- a/coro/ssl/s2n/test/t0.py +++ b/coro/ssl/s2n/test/t0.py @@ -1,8 +1,16 @@ # -*- Mode: Python -*- +# test this with: +# openssl s_client -connect 127.0.0.1:7777 -alpn spdy/3,nork/1 -crlf + import coro -from coro.ssl.s2n import sock, Config, MODE +from cys2n import Config, MODE, PROTOCOL +from coro.ssl.s2n import S2NSocket + +from coro.log import NoFacility + +LOG = NoFacility() # EC doesn't work with s2n? key = """-----BEGIN EC PARAMETERS----- @@ -68,24 +76,39 @@ cfg = Config() cfg.add_cert_chain_and_key (crt, key) +cfg.set_protocol_preferences (['nork/1']) + +def unproto (n): + return PROTOCOL.reverse_map.get (n, "unknown") def go (conn): + s2n = conn.s2n_conn + s2n.negotiate() + LOG ('client_hello_version', unproto (s2n.get_client_hello_version())) + LOG ('client_protocol_version', unproto (s2n.get_client_protocol_version())) + LOG ('server_protocol_version', unproto (s2n.get_server_protocol_version())) + LOG ('actual_protocol_version', unproto (s2n.get_actual_protocol_version())) + LOG ('application_protocol', s2n.get_application_protocol()) + LOG ('cipher', s2n.get_cipher()) try: - for i in range (10): - n = conn.send (crt) - print 'n, len(crt)', n, len(crt) - conn.shutdown() + #for i in range (10): + # n = conn.send (crt) + # print 'n, len(crt)', n, len(crt) + block = conn.recv (1024) + print repr(block) finally: + print 'bytes:', conn.s2n_conn.get_wire_bytes() conn.close() def serve (port): - s = sock (cfg, mode=MODE.SERVER) + s = S2NSocket (cfg, mode=MODE.SERVER) s.bind (('', port)) s.listen (10) while 1: conn, addr = s.accept() coro.spawn (go, conn) +import coro.backdoor +coro.spawn (coro.backdoor.serve, unix_path='/tmp/t0.bd') coro.spawn (serve, 7777) coro.event_loop() - From 8fc3fc28932d6b65d3ee3c457a8141af412a7a87 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 15 Sep 2015 15:45:00 -0700 Subject: [PATCH 16/46] use cys2n, test alpn. --- coro/ssl/s2n/test/serve.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/coro/ssl/s2n/test/serve.py b/coro/ssl/s2n/test/serve.py index 639cb60..b3fc102 100644 --- a/coro/ssl/s2n/test/serve.py +++ b/coro/ssl/s2n/test/serve.py @@ -2,7 +2,12 @@ import coro -from coro.ssl.s2n import sock, Config, MODE +from cys2n import Config, MODE, PROTOCOL +from coro.ssl.s2n import S2NSocket + +from coro.log import NoFacility + +LOG = NoFacility() # EC doesn't work with s2n? key = """-----BEGIN EC PARAMETERS----- @@ -68,10 +73,10 @@ cfg = Config() cfg.add_cert_chain_and_key (crt, key) +cfg.set_protocol_preferences (['nork/1']) def echo (conn): global rbytes, wbytes - conn.send ('Howdy!\r\n') while 1: block = conn.recv (1024) if not block: @@ -82,7 +87,7 @@ def echo (conn): wbytes += len(block) def serve (port): - s = sock (cfg, mode=MODE.SERVER) + s = S2NSocket (cfg, mode=MODE.SERVER) s.bind (('', port)) s.listen (10) while 1: From 58f2a210eec011c72a9927730bb08b3008f2ad96 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 17 Sep 2015 16:45:22 -0700 Subject: [PATCH 17/46] ALPN support. Written to be separate from the NPN support so that NPN can be easily removed. [tested on server side, but not client side yet]. --- coro/ssl/__init__.py | 4 +++- coro/ssl/openssl.pxi | 30 ++++++++++++++++++++++++++++++ coro/ssl/openssl.pyx | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/coro/ssl/__init__.py b/coro/ssl/__init__.py index 12d4d85..b210c6f 100644 --- a/coro/ssl/__init__.py +++ b/coro/ssl/__init__.py @@ -15,7 +15,7 @@ from openssl import x509, pkey, dh_param -def new_ctx (cert=None, chain=(), key=None, proto=None, ciphers=None, dhparam=None, next_protos=None): +def new_ctx (cert=None, chain=(), key=None, proto=None, ciphers=None, dhparam=None, next_protos=None, alpn_protos=None): ctx = openssl.ssl_ctx() if cert: ctx.use_cert (cert, chain) @@ -29,6 +29,8 @@ def new_ctx (cert=None, chain=(), key=None, proto=None, ciphers=None, dhparam=No ctx.set_tmp_dh (dhparam) if next_protos: ctx.set_next_protos (next_protos) + if alpn_protos: + ctx.set_alpn_protos (alpn_protos) return ctx class sock (coro.sock): diff --git a/coro/ssl/openssl.pxi b/coro/ssl/openssl.pxi index 4b491ea..17f273c 100644 --- a/coro/ssl/openssl.pxi +++ b/coro/ssl/openssl.pxi @@ -319,6 +319,36 @@ cdef extern from "openssl/ssl.h": unsigned char *, unsigned int ) + IF ALPN: + # application layer protocol negotiation ('ALPN') support + int SSL_CTX_set_alpn_protos ( + SSL_CTX *ctx, + const unsigned char *protos, + unsigned protos_len + ) + int SSL_set_alpn_protos ( + SSL *ssl, + const unsigned char *protos, + unsigned protos_len + ) + void SSL_CTX_set_alpn_select_cb ( + SSL_CTX *ctx, + int (*cb) ( + SSL *ssl, + const unsigned char **out, + unsigned char *outlen, + const unsigned char *, + unsigned int inlen, + void *arg + ), + void *arg + ) + void SSL_get0_alpn_selected ( + const SSL *ssl, + const unsigned char **data, + unsigned *len + ) + X509_STORE * SSL_CTX_get_cert_store (SSL_CTX *) int X509_STORE_CTX_init (X509_STORE_CTX *, X509_STORE *, X509 *, void *) void X509_STORE_CTX_free (X509_STORE_CTX *) diff --git a/coro/ssl/openssl.pyx b/coro/ssl/openssl.pyx index 3c932a1..1c7998a 100644 --- a/coro/ssl/openssl.pyx +++ b/coro/ssl/openssl.pyx @@ -1057,7 +1057,17 @@ cdef class ssl: cdef unsigned int len SSL_get0_next_proto_negotiated (self.ssl, &data, &len) if data: - return PyBytes_FromStringAndSize (data, len) + return data[:len] + else: + return None + + IF ALPN: + def get_alpn_selected (self): + cdef unsigned char * data = NULL + cdef unsigned int dlen = 0 + SSL_get0_alpn_selected (self.ssl, &data, &dlen) + if data: + return data[:dlen] else: return None @@ -1070,6 +1080,7 @@ cdef class ssl_ctx: cdef readonly pkey key cdef readonly dh_param dh cdef bytes next_protos + cdef bytes alpn_protos def __init__ (self): self.ctx = SSL_CTX_new (SSLv23_method()) @@ -1174,6 +1185,26 @@ cdef class ssl_ctx: SSL_select_next_proto (out, outlen, server, server_len, self.next_protos, len (self.next_protos)) return SSL_TLSEXT_ERR_OK + IF ALPN: + def set_alpn_protos (self, list protos): + encoded = [] + for proto in protos: + encoded.append (chr (len (proto))) + encoded.append (proto) + self.alpn_protos = b''.join (encoded) + if SSL_CTX_set_alpn_protos (self.ctx, self.alpn_protos, len(self.alpn_protos)): + raise_ssl_error() + else: + SSL_CTX_set_alpn_select_cb (self.ctx, alpn_select_callback, self) + + cdef alpn_select_callback (self, + const unsigned char **out, unsigned char *outlen, + const unsigned char *xin, unsigned int inlen): + # in CPython's _ssl.c and in openssl's ssltest.c, this function is called here. + # can we assume its name will change once NPN is deprecated? + SSL_select_next_proto (out, outlen, xin, inlen, self.alpn_protos, len(self.alpn_protos)) + return SSL_TLSEXT_ERR_OK + IF NPN: cdef int next_protos_server_callback (SSL *ssl, unsigned char **out, unsigned int *outlen, void *arg): cdef ssl_ctx ctx = arg @@ -1184,6 +1215,13 @@ IF NPN: cdef ssl_ctx ctx = arg return ctx.next_protos_client_callback (out, outlen, server, server_len) +IF ALPN: + + cdef int alpn_select_callback (SSL *ssl, unsigned char **out, unsigned char *outlen, + unsigned char * server, unsigned int server_len, void *arg): + cdef ssl_ctx ctx = arg + return ctx.alpn_select_callback (out, outlen, server, server_len) + # ================================================================================ cdef class cipher: From 700759447268a70d79a59e046f43b662e747d509 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 17 Sep 2015 16:45:48 -0700 Subject: [PATCH 18/46] probe for ALPN support in openssl. --- setup.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index dbe1b8d..a6e75ec 100644 --- a/setup.py +++ b/setup.py @@ -108,12 +108,19 @@ def O (path): # cheap probe for npn support USE_NPN = (open (O('include/openssl/ssl.h')).read().find ('next_protos') != -1) +# cheap probe for alpn support +USE_ALPN = (open (O('include/openssl/ssl.h')).read().find ('SSL_CTX_set_alpn_protos') != -1) if USE_NPN: sys.stderr.write ('detected NPN-capable OpenSSL\n') else: sys.stderr.write ('NPN support disabled. Needs OpenSSL-1.0.1+\n') +if USE_ALPN: + sys.stderr.write ('detected ALPN-capable OpenSSL\n') +else: + sys.stderr.write ('ALPN support disabled. Needs OpenSSL-1.0.2+\n') + OpenSSL_Extension = Extension ( 'coro.ssl.openssl', ['coro/ssl/openssl.pyx'], @@ -121,11 +128,11 @@ def O (path): # manual static link #extra_link_args = [O('libcrypto.a'), O('libssl.a')], # link to an absolute location - #extra_link_args = ['-L', '%s/lib' % (ossl_base), '-lcrypto', '-lssl'], + extra_link_args = ['-L', '%s/lib' % (ossl_base), '-lcrypto', '-lssl'], # 'normal' link - libraries=['crypto', 'ssl'], + #libraries=['crypto', 'ssl'], include_dirs=[O('include')], - cython_compile_time_env={'NPN': USE_NPN}, + cython_compile_time_env={'NPN': USE_NPN, 'ALPN': USE_ALPN}, ) From 60152f0b740bc3315305063a1c5e23e7dfd60a9f Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:18:35 -0700 Subject: [PATCH 19/46] new test keys, and a small to convert private key format for s2n. --- coro/http/cert/convert.py | 36 +++++++++++++++++++++++++++++ coro/http/cert/dhparam.pem | 8 +++++++ coro/http/cert/server.crt | 31 ++++++++++++++----------- coro/http/cert/server.key | 43 +++++++++++++++++++++++------------ coro/http/cert/server.raw.key | 27 ++++++++++++++++++++++ 5 files changed, 117 insertions(+), 28 deletions(-) create mode 100644 coro/http/cert/convert.py create mode 100644 coro/http/cert/dhparam.pem create mode 100644 coro/http/cert/server.raw.key diff --git a/coro/http/cert/convert.py b/coro/http/cert/convert.py new file mode 100644 index 0000000..1876107 --- /dev/null +++ b/coro/http/cert/convert.py @@ -0,0 +1,36 @@ +# -*- Mode: Python -*- + +# utility to convert a modern rfc5280 rsa key file to +# an older-style rsa-only key file (for s2n). I expect +# s2n to will soon handle rfc5280, so this should be temporary. + +import base64 +from coro.asn1.ber import * + +# can't use base64 module output directly, +# PEM requires lines of length 64. +def b64_at_width (data, width=64): + encoded = base64.encodestring (data) + lines = encoded.split ('\n') + oneline = ''.join (lines) + result = [] + for i in range (0, len(oneline), 64): + result.append (oneline[i:i+64] + '\n') + return ''.join (result) + +lines = open ('server.key', 'rb').readlines() +assert (lines[0] == '-----BEGIN PRIVATE KEY-----\n') +assert (lines[-1] == '-----END PRIVATE KEY-----\n') +der = base64.decodestring (''.join (lines[1:-1])) +d0, size = decode (der) +assert len(der) == size +assert d0[0] == 0 +assert d0[1] == [('oid', [1, 2, 840, 113549, 1, 1, 1]), None] +f = open ('server.raw.key', 'wb') +f2 = open ('/tmp/x.der', 'wb') +f.write ('-----BEGIN RSA PRIVATE KEY-----\n') +f.write (b64_at_width (d0[2])) +f2.write (d0[2]) +f.write ('-----END RSA PRIVATE KEY-----\n') +f.close() +f2.close() diff --git a/coro/http/cert/dhparam.pem b/coro/http/cert/dhparam.pem new file mode 100644 index 0000000..7c7b57f --- /dev/null +++ b/coro/http/cert/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAkcTH+esuX5lJyGsS9v+FhAZI7kgefoFjLIH41H+zQzf9YjnCB9Jq +h6snUFuckh2kJVDfR0Jft92ZdwkCyvoGahVAd4tWDCAFQfPnS9aRnc9Nx4qlZayD ++6gtUF0ZSI01Y26PdtVvjndf6uf8o4wnK3SCiC3KbA/3MEvsTdcUi7W3VzV1tQGw +Q6xmIIWBOi5E/foJLW/c0Ns/EzHPoV48YGZb6yKl8LsX2RzFZTOz0GGF6+o/wrCr +N7ev3D0TeQgjIdKdz2pjz9fzUhhCi/gfQ6uHGisIPx4uYxH5OODEjkuVWvz2rb0x +O/1ZmXWXYPhZAmvuttm67fPW3CHjdCEaQwIBAg== +-----END DH PARAMETERS----- diff --git a/coro/http/cert/server.crt b/coro/http/cert/server.crt index dcbcd22..e619119 100644 --- a/coro/http/cert/server.crt +++ b/coro/http/cert/server.crt @@ -1,15 +1,20 @@ -----BEGIN CERTIFICATE----- -MIICQTCCAaoCCQDQvDSH5q2j7jANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV -UzETMBEGA1UECBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxETAPBgNV -BAoTCFNocmFwbmVsMRswGQYDVQQDExJzaHJhcG5lbCB0ZXN0IGNlcnQwHhcNMTIw -MzEzMjIwNjEyWhcNMTMwMzEzMjIwNjEyWjBlMQswCQYDVQQGEwJVUzETMBEGA1UE -CBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxETAPBgNVBAoTCFNocmFw -bmVsMRswGQYDVQQDExJzaHJhcG5lbCB0ZXN0IGNlcnQwgZ8wDQYJKoZIhvcNAQEB -BQADgY0AMIGJAoGBAKEEpkHzToFxX4ui0GxsrscU73VZ/KpfS13oSZ+3m3bjg0Fu -Y8Gird/IYqKCPoGSxcNOijoNGCe9G3Jtfo387yOGlQF8scYrv9nBihIk1X20GGyq -3iIGV8cgG3Yc+IwsZMA3rAjv6edx4TL8q1rBKNjbk6mcgKpFzi24Vh2Jdx41AgMB -AAEwDQYJKoZIhvcNAQEFBQADgYEAFR3lhtBGvYDc5xA8rnUJh8z/LAI17aVpeb4B -HArVnK7rB2OiN7voMwKMDD3F9X0czbieyoCocEXPkL/QpITRDnRklWIWAKpktZ1U -xdX7TaYBVSK96R9r1tuNkKRNNtrdiesMt4JTrjU0ORwXaEX4WUCiqSOr1O0fPI69 -En3Ip7k= +MIIDUzCCAjugAwIBAgIJAPP2xS7wk9YuMA0GCSqGSIb3DQEBCwUAMEAxFDASBgNV +BAMMC2V4YW1wbGUuY29tMRswGQYDVQQKDBJzaHJhcG5lbCBkZW1vIGNlcnQxCzAJ +BgNVBAYTAlVTMB4XDTE1MDkxNTIzMzU1MloXDTE2MDkxNDIzMzU1MlowQDEUMBIG +A1UEAwwLZXhhbXBsZS5jb20xGzAZBgNVBAoMEnNocmFwbmVsIGRlbW8gY2VydDEL +MAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtMhYt +XogpUX1ibFcvgcDI5hKmiqUVwiU5MpXpDrrmFYAveTTsnEisIUo34JnqkK+omG6i +fa+j1BaiYs/TgxFrD8OW4vX9sAnE5LbhNNPWc3F8edvPqtWjfi7nH3btHe9f09p+ +fFt8EkyUge0RYGjxdMrfGAV7T19lKtESKIut7FHPkITV64qFoIcOEHK6PTJ6u7gO +afBWhmJ5q+yE/c+Pq+dkm4oN529UnR1isvFDaqhgxGL3r6Sm632ujLKlW5370F0X +YkFR8wEYi29pEE5lvsootRlb0FwJcl2tzy2LXugZ9bUXW4vqGjkJqc5fO/zYcwgK +sBxQR4lKDpucXDjbAgMBAAGjUDBOMB0GA1UdDgQWBBS+gAbkAILVhOIExdEm5h2N +/aF9QzAfBgNVHSMEGDAWgBS+gAbkAILVhOIExdEm5h2N/aF9QzAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCrtuE10LWbTGuo7dfWSEiIf0rF6lq36yos +0Qy9ECo9Pdc5LU0MShnbzkmfjkfggQOwstrITDQB9T8FpzmNcnqm8kxdNs7c4Z4a ++u4niI3rHcZBb0dqThQ8sH/eFjMKJmXlSS9wg4e7N7QtVwhDZzyptaUccjEq8IQm +uVpVOK2X54vnuKtmlUxqQ57OPTMqNEESkT9zZtLrZ0qKceyM9jCoXAVMCa7iuuKb +P2T2cJEhKpUHochpOiepWfHx9nJUu8Ki6pWdhFsytLKHgc2Kh61sSDqxabxSDesT +gSmBsKvtn7BIuJTp7JSVxqUyuBc30LvDxBH3wPg2ygmlTfo2MYds -----END CERTIFICATE----- diff --git a/coro/http/cert/server.key b/coro/http/cert/server.key index ac915b2..71e9260 100644 --- a/coro/http/cert/server.key +++ b/coro/http/cert/server.key @@ -1,15 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQChBKZB806BcV+LotBsbK7HFO91WfyqX0td6Emft5t244NBbmPB -oq3fyGKigj6BksXDToo6DRgnvRtybX6N/O8jhpUBfLHGK7/ZwYoSJNV9tBhsqt4i -BlfHIBt2HPiMLGTAN6wI7+nnceEy/KtawSjY25OpnICqRc4tuFYdiXceNQIDAQAB -AoGBAJTwpQ1KDusjQLr8Loise3sBPYKya0n4/dDuhfOsNaziRE4o2zOI1Aa056/k -hAb9Cdtf8fJCnH5dqV7OM4sJVPVyqZcqlVElaFtJR0skcztmv+h2mDdwyK1D6crl -x+hgKdur7uBDzg7CV7BttynNjD1a3gZ+swZ9ddY4K14Xv5adAkEA0yx2jctDKu49 -SSbAcf3hl1huJafrl/OWaucHVSm0S0BgJ3QQ+5v+I9tF9MZ5znKNFStJ56oEQ3MX -wZlexhICcwJBAMMypmgz8Noa1IXsoHqe0fFDzfZwaLjxZ+MBvKZB5JYG/y9x8dp+ -BFR8kiarIbvVpcyz2WnKQexLLvjfIsZtqrcCQDw7kXEuSfFD1N05nWimNqNZiMla -1RsZUo0ZaoEDDTbtnL+EHpf1zY5iq9h+iB23lMA2AbV/TAoFGQCSg3LRjjsCQHI1 -S2oCofRq6FfniEnWbQ3V10dOo+c5z8fhd0hrm1wwgdR3vcNSIiRwsm6PiHBHY/fu -btHX9lRT9QrGO6mP9ucCQGMncWmOaDfi/MjunZmxPwE0bbnClGAPOoZmP73baX61 -r3vmoTzLzrvcmPBo4mBDu6bP25wHcuQNSvyUBwFHhLs= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCtMhYtXogpUX1i +bFcvgcDI5hKmiqUVwiU5MpXpDrrmFYAveTTsnEisIUo34JnqkK+omG6ifa+j1Bai +Ys/TgxFrD8OW4vX9sAnE5LbhNNPWc3F8edvPqtWjfi7nH3btHe9f09p+fFt8EkyU +ge0RYGjxdMrfGAV7T19lKtESKIut7FHPkITV64qFoIcOEHK6PTJ6u7gOafBWhmJ5 +q+yE/c+Pq+dkm4oN529UnR1isvFDaqhgxGL3r6Sm632ujLKlW5370F0XYkFR8wEY +i29pEE5lvsootRlb0FwJcl2tzy2LXugZ9bUXW4vqGjkJqc5fO/zYcwgKsBxQR4lK +DpucXDjbAgMBAAECggEAPWwH5WsjCtQ1jVQyz0xK3HSLpLiCt2a04MYJ2C87JSH2 +5d8sX8VFGJZtxcdHjqJlSXtVsRa11Xn/1PMKFU7kPH3ItZgj+SYXMNaT7OZZQjKK +ysREqi9BuMulQp4sBQeavvdZA4aCw0uSERDKzGDOluL+l4PjXcnWYQCASNpMPw98 +sWCkIVYoZ4S5LxGS7u2qr0Tq2yxSYRZmYxWwlXFmB8TDJiXlQ2/cgoOopn4gH+F/ +RpUOGGNfysxDzACiLkHs64MYcmJQUCI57deXNR6bQcSqUOxCkTr8FZcCnw7mYyJl +SOjo4FneRfDIlkIe9V++OnweVp9uD6i9/YgAV2wPcQKBgQDbbHEi2L3KFl1fo+q6 +uJg9ct1BUR1OgEt/bmWOHX+FNBfvvOSZB7JiHT93jvVKgIJTsA/eS7bwGEyHmy/I +anzK3opDYZgvDM0L1UmUO0e72YcAWNpiAUwQh1wDjaQ1d6eM0okbsh9VAyOdIjYP +GO6Tw1BPjfL4n0epfXjRgbi1jwKBgQDKEPB8/zPEu933dakDGRlGZVfBLLtEZV3P +DBxzeT+70RKJUs2SZQexv1PU/V4EL8eJ7wWT7Am1s2MJGxcllqoYmawB5wQWmtuF +ZfVWbHr1t/JOwxI2jh0j6mIKqn5mONlBNEdfVb2Vm3rjSfVaV8km+IQaJ/OQ1WH5 +PV/GiqyZ9QKBgBf8VrG0d6qrnzFhPbuDikDNWZpWP5nhNF+NtdQ/LT1mYGd4gpSn +3rwS7mknW3D1c0mqqVFnfWvpfBQmxqZl3ZZflUWgWX5rK87rVcu6XzENqlBDZjvo +YGQ+J7TLuvONTOd77Dj20637Vd1LbBViaFIGu7S2k6TR5IeGi7p7L7HJAoGAYuEj +LaecZ5sfJWb4S8HOcnpJFQiUowWPgDAHBCLDI19N2NEiM48o0rwzg7hwd1ACLuc+ +LYKFxdqAjgYpr4Uou10HpO6tO3qQDZk1ExOGoBNhiVU/5l5ouBiL3XhM3izXc2bn +vikw2rL40ZxxacInduCJlFsUfz1L8jEsWvWlPLkCgYBMn1ZEHGlekaw8xkdmBbqH +g4ssQ6SWza/VRDwq8hTCBqVGXWE/XClAlGirB2wanl6J/XcvBO1UH1I8UmJoE+/2 +v0CiJ1UHH3UY/4n3KQK0iKVdewJrJfqUG1iRZ8wPKJ2E4Y3mYG0UGTDo1GIyoJyQ +FeXqsyNrI7zrmAbdgVwQfw== +-----END PRIVATE KEY----- diff --git a/coro/http/cert/server.raw.key b/coro/http/cert/server.raw.key new file mode 100644 index 0000000..ae68c10 --- /dev/null +++ b/coro/http/cert/server.raw.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEArTIWLV6IKVF9YmxXL4HAyOYSpoqlFcIlOTKV6Q665hWAL3k0 +7JxIrCFKN+CZ6pCvqJhuon2vo9QWomLP04MRaw/DluL1/bAJxOS24TTT1nNxfHnb +z6rVo34u5x927R3vX9PafnxbfBJMlIHtEWBo8XTK3xgFe09fZSrREiiLrexRz5CE +1euKhaCHDhByuj0yeru4DmnwVoZieavshP3Pj6vnZJuKDedvVJ0dYrLxQ2qoYMRi +96+kput9royypVud+9BdF2JBUfMBGItvaRBOZb7KKLUZW9BcCXJdrc8ti17oGfW1 +F1uL6ho5CanOXzv82HMICrAcUEeJSg6bnFw42wIDAQABAoIBAD1sB+VrIwrUNY1U +Ms9MStx0i6S4grdmtODGCdgvOyUh9uXfLF/FRRiWbcXHR46iZUl7VbEWtdV5/9Tz +ChVO5Dx9yLWYI/kmFzDWk+zmWUIyisrERKovQbjLpUKeLAUHmr73WQOGgsNLkhEQ +ysxgzpbi/peD413J1mEAgEjaTD8PfLFgpCFWKGeEuS8Rku7tqq9E6tssUmEWZmMV +sJVxZgfEwyYl5UNv3IKDqKZ+IB/hf0aVDhhjX8rMQ8wAoi5B7OuDGHJiUFAiOe3X +lzUem0HEqlDsQpE6/BWXAp8O5mMiZUjo6OBZ3kXwyJZCHvVfvjp8Hlafbg+ovf2I +AFdsD3ECgYEA22xxIti9yhZdX6PquriYPXLdQVEdToBLf25ljh1/hTQX77zkmQey +Yh0/d471SoCCU7AP3ku28BhMh5svyGp8yt6KQ2GYLwzNC9VJlDtHu9mHAFjaYgFM +EIdcA42kNXenjNKJG7IfVQMjnSI2Dxjuk8NQT43y+J9HqX140YG4tY8CgYEAyhDw +fP8zxLvd93WpAxkZRmVXwSy7RGVdzwwcc3k/u9ESiVLNkmUHsb9T1P1eBC/Hie8F +k+wJtbNjCRsXJZaqGJmsAecEFprbhWX1Vmx69bfyTsMSNo4dI+piCqp+ZjjZQTRH +X1W9lZt640n1WlfJJviEGifzkNVh+T1fxoqsmfUCgYAX/FaxtHeqq58xYT27g4pA +zVmaVj+Z4TRfjbXUPy09ZmBneIKUp968Eu5pJ1tw9XNJqqlRZ31r6XwUJsamZd2W +X5VFoFl+ayvO61XLul8xDapQQ2Y76GBkPie0y7rzjUzne+w49tOt+1XdS2wVYmhS +Bru0tpOk0eSHhou6ey+xyQKBgGLhIy2nnGebHyVm+EvBznJ6SRUIlKMFj4AwBwQi +wyNfTdjRIjOPKNK8M4O4cHdQAi7nPi2ChcXagI4GKa+FKLtdB6TurTt6kA2ZNRMT +hqATYYlVP+ZeaLgYi914TN4s13Nm574pMNqy+NGccWnCJ3bgiZRbFH89S/IxLFr1 +pTy5AoGATJ9WRBxpXpGsPMZHZgW6h4OLLEOkls2v1UQ8KvIUwgalRl1hP1wpQJRo +qwdsGp5eif13LwTtVB9SPFJiaBPv9r9AoidVBx91GP+J9ykCtIilXXsCayX6lBtY +kWfMDyidhOGN5mBtFBkw6NRiMqCckBXl6rMjayO865gG3YFcEH8= +-----END RSA PRIVATE KEY----- From b1305130193aabf980a80e8005c03cc3be7e8570 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:19:24 -0700 Subject: [PATCH 20/46] switching from NPN -> ALPN. --- coro/http/demo/spdy_client.py | 9 ++++----- coro/http/demo/spdy_server.py | 8 +++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/coro/http/demo/spdy_client.py b/coro/http/demo/spdy_client.py index f794ab1..187cf13 100644 --- a/coro/http/demo/spdy_client.py +++ b/coro/http/demo/spdy_client.py @@ -10,15 +10,13 @@ W = coro.write_stderr ctx = coro.ssl.new_ctx ( - # cert=coro.ssl.x509 (open ('cert/server.crt').read()), - # key=coro.ssl.pkey (open ('cert/server.key').read(), '', True), - next_protos=['spdy/2', 'http/1.1'], - proto='tlsv1', + #next_protos=['spdy/3.1', 'http/1.1'], + alpn_protos=['spdy/3.1', 'http/1.1'], + proto='tlsv2', ) def t0(): global ctx, s, c - # ctx = coro.ssl.new_ctx (proto='tlsv1', next_protos=['spdy/2', 'http/1.1']) s = coro.ssl.sock (ctx) c = spdy_client ('127.0.0.1', 9443, s) W ('negotiated: %r\n' % (s.ssl.get_next_protos_negotiated(),)) @@ -31,4 +29,5 @@ def t0(): if __name__ == '__main__': coro.spawn (coro.backdoor.serve, unix_path='/tmp/spdy_client.bd') + coro.spawn (t0) coro.event_loop (30.0) diff --git a/coro/http/demo/spdy_server.py b/coro/http/demo/spdy_server.py index dc23c09..4d65455 100644 --- a/coro/http/demo/spdy_server.py +++ b/coro/http/demo/spdy_server.py @@ -6,13 +6,15 @@ ctx = coro.ssl.new_ctx ( cert=coro.ssl.x509 (open ('cert/server.crt').read()), key=coro.ssl.pkey (open ('cert/server.key').read(), '', True), - next_protos=['spdy/3', 'http/1.1'], - proto='tlsv1', + #next_protos=['spdy/3.1', 'http/1.1'], + alpn_protos=['h2'], #'http/1.1'], + proto='tlsv2', ) server = coro.http.spdy.spdy_openssl_server (ctx) server.push_handler (coro.http.handlers.favicon_handler()) server.push_handler (coro.http.handlers.coro_status_handler()) -coro.spawn (server.start, ('0.0.0.0', 9443)) +server.push_handler (coro.http.handlers.file_handler ('.')) +coro.spawn (server.start, ('0.0.0.0', 8443)) coro.spawn (coro.backdoor.serve, unix_path='/tmp/spdys.bd') coro.event_loop (30.0) From 6a8718fac1702e1c9f4bde40b2ae922c86ed6ed9 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:20:52 -0700 Subject: [PATCH 21/46] S2N/ALPN support, logging changes. --- coro/http/spdy.py | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/coro/http/spdy.py b/coro/http/spdy.py index 563b88e..b134669 100644 --- a/coro/http/spdy.py +++ b/coro/http/spdy.py @@ -4,7 +4,7 @@ import coro import sys -from coro.http import connection, tlslite_server, openssl_server, http_request +from coro.http import connection, tlslite_server, openssl_server, s2n_server, http_request from coro.http.protocol import header_set, http_file from coro.http.zspdy import inflator, deflator, unpack_control_frame, pack_control_frame from coro.http.zspdy import pack_data_frame, pack_http_header, unpack_http_header @@ -173,7 +173,7 @@ def read_frames (self): def read_control_frame (self, head): fversion, ftype, flags, length = unpack_control_frame (head) data = self.read_exact (length) - # W ('control: version=%d type=%d flags=%x length=%d\n' % (fversion, ftype, flags, length, )) + LOG ('control: version=%d type=%d flags=%x length=%d' % (fversion, ftype, flags, length, )) assert (fversion == 3) method_name = 'frame_%s' % (self.frame_types.get (ftype, ''),) if method_name == 'frame_': @@ -185,7 +185,7 @@ def read_control_frame (self, head): def read_data_frame (self, head): stream_id, flags, length = unpack_data_frame (head) data = self.read_exact (length) - # W ('data: stream_id=%d flags=%x length=%d\n' % (stream_id, flags, length)) + LOG ('data: stream_id=%d flags=%x length=%d' % (stream_id, flags, length)) self.handle_data_frame (stream_id, flags, data) spdy_version = 3 @@ -240,6 +240,7 @@ def send_thread (self): if block is None: break else: + #LOG ('send', block) self.conn.send (block) self.obuf.release (len(block)) @@ -264,9 +265,9 @@ def push_syn_reply (self, req, has_data): flags = 0x01 else: flags = 0x00 - # W ('req.reply_headers=%r\n' % (str(req.reply_headers),)) + #LOG ('req.reply_headers=%r' % (str(req.reply_headers),)) name_vals = self.pack_http_header (req.reply_headers) - # W ('compressed name_vals=%r\n' % (name_vals,)) + #LOG ('compressed name_vals=%r' % (name_vals,)) frame = self.pack_control_frame ( 0x02, flags, ''.join ([struct.pack ('>L', req.stream_id), name_vals]) @@ -279,10 +280,10 @@ def frame_syn_stream (self, flags, data): # XXX do something with priority sid &= 0x7fffffff asid &= 0x7fffffff - # W ('syn_stream: sid=%d asid=%d pri=%x ' % (sid, asid, pri)) + #LOG ('syn_stream: sid=%d asid=%d pri=%x' % (sid, asid, pri)) headers = self.unpack_http_header (data[10:]) req = spdy_server_request (flags, sid, self, headers) - # W ('%s\n' % req.request,) + #LOG (req.request) self.streams[sid] = req coro.spawn (self.handle_request, req) @@ -311,30 +312,29 @@ def handle_request (self, req): def frame_rst_stream (self, flags, data): stream_id, status_code = struct.unpack ('>LL', data) - # W ('reset: %x status=%d %s\n' % (stream_id, status_code, self.status_codes.get (status_code, 'unknown'))) + #LOG ('reset: %x status=%d %s' % (stream_id, status_code, self.status_codes.get (status_code, 'unknown'))) del self.streams[stream_id] def frame_goaway (self, flags, data): last_stream_id, = struct.unpack ('>L', data) - # W ('goaway last_stream_id=%d\n' % (last_stream_id,)) + #LOG ('goaway last_stream_id=%d' % (last_stream_id,)) # XXX arrange for the connection to close self.close() def frame_ping (self, flags, data): ping_id, = struct.unpack ('>L', data) - # W ('ping_id=%x\n' % (ping_id,)) + #LOG ('ping', flags, ping_id) self.send_frame (self.pack_control_frame (6, 0, data)) def frame_settings (self, flags, data): - # self.log ('SPDY settings frame received [ignored]') + self.log ('SPDY settings frame received [ignored]') pass def frame_headers (self, flags, data): - # self.log ('SPDY headers frame received [ignored]') + self.log ('SPDY headers frame received [ignored]') pass def frame_window_update (self, flags, data): - # self.log ('SPDY window_update frame received [ignored]') stream_id, delta_window_size = struct.unpack ('>LL', data) self.log ('spdy window update', stream_id, delta_window_size) @@ -362,6 +362,24 @@ def create_connection (self, conn, addr): else: return connection (self, conn, addr) +class spdy_s2n_server (s2n_server): + + protocol = 'spdy' + + def create_connection (self, conn, addr): + + def unproto (n): + from coro.ssl.s2n import PROTOCOL + return PROTOCOL.reverse_map.get (n, "unknown") + + conn._check_negotiated() + s2n = conn.s2n_conn + #LOG ('ALPN', s2n.get_application_protocol()) + if conn.s2n_conn.get_application_protocol() == b'spdy/3.1': + return spdy_connection (self, conn, addr) + else: + return connection (self, conn, addr) + # -------------------------------------------------------------------------------- # spdy client # -------------------------------------------------------------------------------- From 0f5e10fae6288b1a7611188870a7c3a36088feeb Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:22:27 -0700 Subject: [PATCH 22/46] switch to cys2n. negotiate loop is in cys2n. implement writev. --- coro/ssl/s2n/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/coro/ssl/s2n/__init__.py b/coro/ssl/s2n/__init__.py index e8fb34e..692924d 100644 --- a/coro/ssl/s2n/__init__.py +++ b/coro/ssl/s2n/__init__.py @@ -1,9 +1,13 @@ # -*- Mode: Python -*- import coro -from cys2n import MODE +from cys2n import MODE, PROTOCOL, Config from ._s2n import * +from coro.log import Facility + +LOG = Facility ('s2n') + # Note: the plan is to push this class into _s2n.pyx as well. class S2NSocket (coro.sock): @@ -25,10 +29,7 @@ def _check_negotiated (self): self.negotiate() def negotiate (self): - while 1: - blocked = self.s2n_conn.negotiate() - if not blocked: - break + self.s2n_conn.negotiate() self.negotiated = True def accept (self): @@ -80,6 +81,7 @@ def send (self, data): left = len(data) while left: n, more = self.s2n_conn.send (data, pos) + #LOG ('send', data, pos, n, more) pos += n if not more: break @@ -93,7 +95,11 @@ def send (self, data): # XXX verify this sendall = send - # XXX writev + def writev (self, list_of_data): + _sum = 0 + for data in list_of_data: + _sum += self.send (data) + return _sum def readv (self, _ignore): raise NotImplementedError From ea329df773184ad2354358083f9b32db04cc0e70 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:23:07 -0700 Subject: [PATCH 23/46] s2n_server. --- coro/http/server.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/coro/http/server.py b/coro/http/server.py index 592f195..01c1f51 100644 --- a/coro/http/server.py +++ b/coro/http/server.py @@ -587,3 +587,16 @@ def create_sock (self): else: domain = socket.AF_INET return coro.ssl.sock (self.ctx, domain=domain) + +class s2n_server (server): + + def __init__ (self, cfg): + self.cfg = cfg + server.__init__ (self) + + def create_sock (self): + return coro.ssl.s2n.S2NSocket (self.cfg) + + def create_connection (self, conn, addr): + LOG ('create_connection', repr(conn), addr) + return self From 11372d6fc40e13728bcd79ff40ea8a52a25a8f71 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:23:49 -0700 Subject: [PATCH 24/46] header_set: set_one, __iter__. --- coro/http/protocol.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/coro/http/protocol.py b/coro/http/protocol.py index 73c0ea3..0bc015b 100644 --- a/coro/http/protocol.py +++ b/coro/http/protocol.py @@ -146,6 +146,10 @@ def crack (self, h): coro.write_stderr ('dropping bogus header %r\n' % (h,)) pass + def __iter__ (self): + for k, v in self.headers.items(): + yield k, v + def get_one (self, key): """Get the value of a header expected to have at most one value. If not present, return None. If more than one, raise ValueError.""" @@ -157,6 +161,15 @@ def get_one (self, key): else: return r[0] + def set_one (self, key, val): + """Set the value of a header expected to have at most one value. + If a value is already present, raise ValueError.""" + r = self.headers.get (key, None) + if r is None: + self.headers[key] = val + else: + raise ValueError ("header %r already has a value: %r" % (key, r)) + def has_key (self, key): "Is this header present?" return self.headers.has_key (key.lower()) From f76ee6c061747935ca03da89ad5e17027ef5559c Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:24:17 -0700 Subject: [PATCH 25/46] pull in s2n_server. --- coro/http/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coro/http/__init__.py b/coro/http/__init__.py index 513cce4..d916c3a 100644 --- a/coro/http/__init__.py +++ b/coro/http/__init__.py @@ -1,6 +1,6 @@ # -*- Mode: Python -*- -from server import server, tlslite_server, openssl_server, connection, http_request +from server import server, tlslite_server, openssl_server, s2n_server, connection, http_request import handlers import coro from coro import read_stream From f54a478c6856e58ead9ee10e29db8c24d83d960f Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Sun, 20 Sep 2015 17:26:00 -0700 Subject: [PATCH 26/46] HTTP/2 server. --- coro/http/__init__.py | 1 + coro/http/demo/h2_s2n_server.py | 22 ++ coro/http/demo/h2_server.py | 24 ++ coro/http/demo/s2n_server.py | 21 ++ coro/http/h2.py | 532 ++++++++++++++++++++++++++++++++ coro/http/hpack.py | 370 ++++++++++++++++++++++ 6 files changed, 970 insertions(+) create mode 100644 coro/http/demo/h2_s2n_server.py create mode 100644 coro/http/demo/h2_server.py create mode 100644 coro/http/demo/s2n_server.py create mode 100644 coro/http/h2.py create mode 100644 coro/http/hpack.py diff --git a/coro/http/__init__.py b/coro/http/__init__.py index d916c3a..818cd5d 100644 --- a/coro/http/__init__.py +++ b/coro/http/__init__.py @@ -7,3 +7,4 @@ import http_date import session_handler import spdy +import h2 diff --git a/coro/http/demo/h2_s2n_server.py b/coro/http/demo/h2_s2n_server.py new file mode 100644 index 0000000..b0c3371 --- /dev/null +++ b/coro/http/demo/h2_s2n_server.py @@ -0,0 +1,22 @@ +import coro +import coro.ssl.s2n +import coro.http.h2 +import coro.backdoor + +cfg = coro.ssl.s2n.Config() +cfg.add_cert_chain_and_key ( + open ('cert/server.crt').read(), + open ('cert/server.raw.key').read() +) +cfg.add_dhparams (open ('cert/dhparam.pem').read()) +cfg.set_cipher_preferences ('default') +cfg.set_protocol_preferences (['h2', 'http/1.1']) +#cfg.set_protocol_preferences (['http/1.1']) + +server = coro.http.h2.h2_s2n_server (cfg) +server.push_handler (coro.http.handlers.favicon_handler()) +server.push_handler (coro.http.handlers.coro_status_handler()) +server.push_handler (coro.http.handlers.file_handler ('.')) +coro.spawn (server.start, ('0.0.0.0', 8443)) +coro.spawn (coro.backdoor.serve, unix_path='/tmp/h2s.bd') +coro.event_loop (30.0) diff --git a/coro/http/demo/h2_server.py b/coro/http/demo/h2_server.py new file mode 100644 index 0000000..281eddc --- /dev/null +++ b/coro/http/demo/h2_server.py @@ -0,0 +1,24 @@ +import coro +import coro.ssl +import coro.http.spdy +import coro.backdoor + +# god how I love openssl. +cipher_suite = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK" + +ctx = coro.ssl.new_ctx ( + cert=coro.ssl.x509 (open ('cert/server.crt').read()), + key=coro.ssl.pkey (open ('cert/server.key').read(), '', True), + alpn_protos=['h2'], #, 'http/1.1'], + proto='tlsv2', + ciphers = cipher_suite, + dhparam=coro.ssl.dh_param (open ('cert/dhparam.pem').read()), +) + +server = coro.http.h2.h2_openssl_server (ctx) +server.push_handler (coro.http.handlers.favicon_handler()) +server.push_handler (coro.http.handlers.coro_status_handler()) +server.push_handler (coro.http.handlers.file_handler ('.')) +coro.spawn (server.start, ('0.0.0.0', 8443)) +coro.spawn (coro.backdoor.serve, unix_path='/tmp/h2s.bd') +coro.event_loop (30.0) diff --git a/coro/http/demo/s2n_server.py b/coro/http/demo/s2n_server.py new file mode 100644 index 0000000..4336955 --- /dev/null +++ b/coro/http/demo/s2n_server.py @@ -0,0 +1,21 @@ +import coro +import coro.ssl.s2n +import coro.http.spdy +import coro.backdoor + +cfg = coro.ssl.s2n.Config() +cfg.add_cert_chain_and_key ( + open ('cert/server.crt').read(), + open ('cert/server.raw.key').read() +) +cfg.add_dhparams (open ('cert/dhparam.pem').read()) +cfg.set_cipher_preferences ('default') +cfg.set_protocol_preferences (['spdy/3.1', 'http/1.1']) + +server = coro.http.spdy.spdy_s2n_server (cfg) +server.push_handler (coro.http.handlers.favicon_handler()) +server.push_handler (coro.http.handlers.coro_status_handler()) +server.push_handler (coro.http.handlers.file_handler ('.')) +coro.spawn (server.start, ('0.0.0.0', 7443)) +coro.spawn (coro.backdoor.serve, unix_path='/tmp/spdys.bd') +coro.event_loop (30.0) diff --git a/coro/http/h2.py b/coro/http/h2.py new file mode 100644 index 0000000..c48e65a --- /dev/null +++ b/coro/http/h2.py @@ -0,0 +1,532 @@ +# -*- Mode: Python -*- + +import struct +import coro +import sys +import os + +# Note: not trying to share code with spdy, since the plan is to deprecate it completely. + +from coro.http import connection, tlslite_server, openssl_server, s2n_server, http_request +from coro.http.protocol import header_set, http_file +from coro.http.hpack import Encoder, Decoder + +from coro.log import Facility +LOG = Facility ('h2') + +# When a reply is large (say >1MB) we still get a form of head-blocking behavior +# unless we chop it up into bits. Think about an architecture that would +# automatically do that. [i.e., a configurable max size for data frames] + +class h2_file (http_file): + + # override http_file's content generator (which is a 'pull' generator) + # with this coro.fifo-based 'push' generator. + + def get_content_gen (self, headers): + self.content_fifo = coro.fifo() + return self._gen_h2() + + def _gen_h2 (self): + while 1: + block = self.content_fifo.pop() + if block is None: + # LOG ('gen_h2: end of content') + self.done_cv.wake_all() + break + else: + yield block + +FLAG_FIN = 0x01 +FLAG_UNIDIRECTIONAL = 0x02 + +class h2_server_request (http_request): + + def __init__ (self, flags, stream_id, client, headers): + self.fin_sent = False + self.flags = flags + self.stream_id = stream_id + self.pending_data_frame = None + method = headers.get_one (':method').lower() + scheme = headers.get_one (':scheme') + host = headers.get_one (':host') + path = headers.get_one (':path') + version = headers.get_one (':version') + # left off by chrome now? + headers['host'] = host + # XXX proxy + # url = '%s://%s/%s' % (scheme, host, path) + url = path + # XXX consider changing the api to take these as separate arguments + request = '%s %s %s' % (method, url, version) + # XXX consider removing method/url/version? + http_request.__init__ (self, client, request, headers) + + def can_deflate (self): + return True + + def has_body (self): + return not (self.flags & FLAG_FIN) + + def make_content_file (self): + # XXX probably untested... + self.file = h2_file (self.request_headers, self.client.stream) + + def push_headers (self, has_data=False): + reason = self.responses[self.reply_code] + #self.reply_headers[':status'] = '%d %s' % (self.reply_code, reason) + self.reply_headers[':status'] = '%d' % (self.reply_code,) + #self.reply_headers[':version'] = 'HTTP/1.1' + self.client.push_headers (self, has_data) + self.sent_headers = True + + def push_data (self, data, last=False): + # we hold back one frame in order to be able to set FLAGS_END_STREAM on the last one. + if self.pending_data_frame is None: + self.pending_data_frame = data + else: + self.pending_data_frame, data = data, self.pending_data_frame + self.client.push_data (self, data, last) + + def push (self, data, flush=False): + "push output data for this request." + if not self.sent_headers: + self.push_headers (has_data=data) + if self.deflate: + if data: + data = self.deflate.compress (data) + if data: + self.push_data (data) + + def done (self): + if not self.sent_headers: + self.push_syn_reply (has_data=False) + else: + if self.deflate: + self.push_data (self.deflate.flush()) + self.push_data (None, last=True) + http_request.done (self) + +def unpack_frame_header (head): + lentype, flags, stream_id = struct.unpack ('>LBl', head) + assert stream_id >= 0 + return lentype >> 8, lentype & 0xff, flags, stream_id + +def pack_frame_header (length, ftype, flags, stream_id): + lentype = (length << 8) | (ftype & 0xff) + return struct.pack ('>LBL', lentype, flags, stream_id) + +# this is a mixin class used for both server and client. + +class h2_protocol: + + frame_types = { + 0: 'data', + 1: 'headers', + 2: 'priority', + 3: 'rst_stream', + 4: 'settings', + 5: 'push_promise', + 6: 'ping', + 7: 'goaway', + 8: 'window_update', + 9: 'continuation', + } + + protocol = 'h2' + preface = 'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' + is_server = True + + # for those socket types not implementing read_exact. + def read_exact (self, size): + try: + return self.conn.read_exact (size) + except AttributeError: + left = size + r = [] + while left: + block = self.conn.recv (left) + if not block: + break + else: + r.append (block) + left -= len (block) + return ''.join (r) + + def read_frames (self): + if self.is_server: + preface = self.read_exact (len (self.preface)) + if not preface: + self.close() + return + else: + LOG ('preface', preface) + assert (preface == self.preface) + while 1: + head = self.read_exact (9) + if not head: + self.close() + return + flen, ftype, flags, stream_id = unpack_frame_header (head) + if flen: + payload = self.read_exact (flen) + else: + payload = None + method_name = 'frame_%s' % (self.frame_types.get (ftype, '')) + if method_name == 'frame_': + self.log ('unknown h2 frame type: %d' % (ftype,)) + else: + LOG ('frame', method_name, flags, stream_id, payload) + method = getattr (self, method_name) + method (flags, stream_id, payload) + + def unpack_http_header (self, data): + hs = header_set() + hs.headers = unpack_http_header (self.inflate (data)) + return hs + + def pack_http_header (self, hset): + return self.deflate (pack_http_header (hset.headers)) + +# -------------------------------------------------------------------------------- +# h2 server +# -------------------------------------------------------------------------------- + +# XXX not a fan of multiple inheritance, but this seems to be the cleanest way to share +# XXX the code between server and client... + +class h2_connection (h2_protocol, connection): + + protocol = 'h2' + # default to 400K buffered output + output_buffer_size = 400 * 1024 + h2_settings = None + last_ping = None + + def run (self): + self.streams = {} + self.encoder = Encoder() + self.decoder = Decoder() + self.ofifo = coro.fifo() + self.obuf = coro.semaphore (self.output_buffer_size) + coro.spawn (self.send_thread) + try: + self.read_frames() + finally: + self.ofifo.push (None) + + def close (self): + self.ofifo.push (None) + self.conn.close() + + def send_thread (self): + while 1: + block = self.ofifo.pop() + if block is None: + break + else: + LOG ('send', block) + self.conn.send (block) + self.obuf.release (len(block)) + + def push_frame (self, frame): + self.obuf.acquire (len(frame)) + self.ofifo.push (frame) + return len(frame) + + def send_frame (self, ftype, flags, stream_id, data): + dlen = len(data) + head = pack_frame_header (dlen, ftype, flags, stream_id) + return self.push_frame (head + data) + + def push_headers (self, req, has_data): + LOG ('push_headers', req, not not has_data) + flags = self.FLAGS_END_HEADERS + if not has_data: + flags |= self.FLAGS_END_STREAM + hdata = self.pack_http_header (req.reply_headers) + LOG ('reply_headers', req.reply_headers) + LOG ('push_headers', hdata) + req.output.sent += self.send_frame (0x01, flags, req.stream_id, hdata) + + def push_ping (self, flags=0, data=None): + if data is None: + data = os.urandom (8) + assert len(data) == 8 + self.last_ping = data + self.send_frame (0x06, flags, 0, data) + + def push_data (self, req, data, last): + if last: + flags = self.FLAGS_END_STREAM + else: + flags = 0 + LOG ('push_data', len(data)) + req.output.sent += self.send_frame (0x00, flags, req.stream_id, data) + + def frame_settings (self, flags, stream_id, payload): + plen = len(payload) + n, check = divmod (plen, 6) + assert check == 0 + self.h2_settings = {} + for i in range (0, 6, plen): + ident, value = struct.unpack ('>HL', payload[i:i+6]) + self.h2_settings[ident] = value + LOG ('settings', self.h2_settings) + # ack it. + self.send_frame (0x04, 0x01, 0, '') + self.push_ping() + + FLAGS_PING_ACK = 0x01 + def frame_ping (self, flags, stream_id, payload): + assert len(payload) == 8 + if flags & self.FLAGS_PING_ACK: + assert payload == self.last_ping + else: + assert len(payload) == 8 + self.push_ping (0x01, payload) + + def frame_window_update (self, flags, stream_id, payload): + increment, = struct.unpack ('>l', payload) + assert increment >= 0 + LOG ('window_update', increment) + + FLAGS_END_STREAM = 0x01 + FLAGS_END_HEADERS = 0x04 + FLAGS_PADDED = 0x08 + FLAGS_PRIORITY = 0x20 + + def frame_headers (self, flags, stream_id, payload): + pos = 0 + pad_len = 0 + stream_dep = 0 + weight = 0 + assert stream_id > 0 + if flags & self.FLAGS_PADDED: + pad_len, = struct.unpack ('>B', payload[pos:pos+1]) + pos += 1 + LOG ('headers', 'padded', pad_len) + if flags & self.FLAGS_PRIORITY: + stream_dep, weight = struct.unpack ('>lB', payload[:5]) + pos += 5 + LOG ('headers', 'priority', stream_dep, weight) + if flags & self.FLAGS_END_STREAM: + LOG ('headers', 'end_stream') + if flags & self.FLAGS_END_HEADERS: + LOG ('headers', 'end_headers') + else: + raise NotImplementedError + if pad_len: + header_block = payload[pos:-pad_len] + else: + header_block = payload[pos:] + headers = self.unpack_http_header (header_block) + req = h2_server_request (flags, stream_id, self, headers) + self.streams[stream_id] = req + coro.spawn (self.handle_request, req) + + def unpack_http_header (self, header_block): + self.decoder.feed (header_block) + hs = header_set() + while not self.decoder.done: + hname, hval = self.decoder.get_header() + hs[hname] = hval + LOG ('headers', hs.headers) + return hs + + def pack_http_header (self, hset): + return self.encoder (hset) + + def frame_rst_stream (self, flags, stream_id, payload): + LOG ('frame_rst_stream', stream_id) + del self.streams[stream_id] + + def frame_data (self, flags, stream_id, payload): + import pdb; pdb.set_trace() + def frame_priority (self, flags, stream_id, payload): + import pdb; pdb.set_trace() + def frame_push_promise (self, flags, stream_id, payload): + import pdb; pdb.set_trace() + def frame_goaway (self, flags, stream_id, payload): + import pdb; pdb.set_trace() + def frame_continuation (self, flags, stream_id, payload): + import pdb; pdb.set_trace() + + def handle_request (self, req): + try: + handler = self.pick_handler (req) + LOG ('handler', repr(handler)) + if handler: + # XXX with_timeout() + handler.handle_request (req) + else: + req.error (404) + except: + tb = coro.compact_traceback() + req.error (500, tb) + self.log ('error: %r request=%r tb=%r' % (self.peer, req, tb)) + + # def frame_rst_stream (self, flags, data): + # stream_id, status_code = struct.unpack ('>LL', data) + # LOG ('reset: %x status=%d %s' % (stream_id, status_code, self.status_codes.get (status_code, 'unknown'))) + # del self.streams[stream_id] + + # def frame_goaway (self, flags, data): + # last_stream_id, = struct.unpack ('>L', data) + # LOG ('goaway last_stream_id=%d' % (last_stream_id,)) + # # XXX arrange for the connection to close + # self.close() + + # def frame_ping (self, flags, data): + # ping_id, = struct.unpack ('>L', data) + # LOG ('ping', flags, ping_id) + # self.send_frame (self.pack_control_frame (6, 0, data)) + + # def frame_settings (self, flags, data): + # self.log ('H2 settings frame received [ignored]') + # pass + + # def frame_headers (self, flags, data): + # self.log ('H2 headers frame received [ignored]') + # pass + + # def frame_window_update (self, flags, data): + # stream_id, delta_window_size = struct.unpack ('>LL', data) + # self.log ('h2 window update', stream_id, delta_window_size) + +class h2_tlslite_server (tlslite_server): + + protocol = 'h2' + + def __init__ (self, addr, cert_path, key_path, settings=None): + tlslite_server.__init__ (self, addr, cert_path, key_path, nextProtos=['h2', 'http/1.1'], settings=settings) + + def create_connection (self, conn, addr): + if conn.next_proto == b'h2/3': + return h2_connection (self, conn, addr) + else: + return connection (self, conn, addr) + +class h2_openssl_server (openssl_server): + + protocol = 'h2' + + def create_connection (self, conn, addr): + # ensure that negotiation finishes... + selected = conn.ssl.get_alpn_selected() + LOG ('h2_openssl_server', 'selected', selected) + if selected == b'h2': + return h2_connection (self, conn, addr) + else: + return connection (self, conn, addr) + +class h2_s2n_server (s2n_server): + + protocol = 'h2' + + def create_connection (self, conn, addr): + + def unproto (n): + from coro.ssl.s2n import PROTOCOL + return PROTOCOL.reverse_map.get (n, "unknown") + + LOG ('h2_s2n_server', repr(conn)) + conn._check_negotiated() + s2n = conn.s2n_conn + LOG ('ALPN', s2n.get_application_protocol()) + if conn.s2n_conn.get_application_protocol() == b'h2': + return h2_connection (self, conn, addr) + else: + return connection (self, conn, addr) + +# -------------------------------------------------------------------------------- +# h2 client +# -------------------------------------------------------------------------------- + +from coro.http import client as http_client + +class h2_client_request (http_client.request): + + _has_body = False + + def wake (self): + if self.rfile and self.force: + self.content = self.rfile.read() + self.latch.wake_all() + if self.rfile and not self.force: + self.rfile.wait() + + def wait (self): + pass + + def has_body (self): + return self._has_body + +class h2_client (h2_protocol, http_client.client): + + is_server = False + + def __init__ (self, host, port=443, conn=None, inflight=100): + self.counter = 1 + self.send_mutex = coro.mutex() + http_client.client.__init__ (self, host, port, conn, inflight) + # replace the fifo with a dictionary (h2 is not serialized) + self.pending = {} + + def send_thread (self): + self.conn.send (preface) + h2_protocol.send_thread (self) + + def read_thread (self): + try: + self.read_frames() + except coro.ClosedError as err: + for sid, req in self.pending.iteritems(): + req.set_error (err) + + def close (self): + self.conn.close() + + def send_frame (self, frame): + with self.send_mutex: + return self.conn.send (frame) + + def push_data_frame (self, stream_id, data, last): + if last: + flags = FLAG_FIN + else: + flags = 0 + self.send_frame (self.pack_data_frame (stream_id, flags, data)) + + def send_request (self, method, uri, headers, content=None, force=False): + try: + self.inflight.acquire (1) + req = h2_client_request (method.upper(), uri, headers, content, force) + sid = self._send_request (method, uri, headers, content) + self.pending[sid] = req + LOG ('send_request', repr(req)) + return req + finally: + self.inflight.release (1) + + def _send_request (self, method, uri, headers, content): + if not headers.has_key ('host'): + headers['host'] = self.host + if content: + has_data = True + else: + has_data = False + headers.set_one (':method', method) + headers.set_one (':scheme', 'https') + headers.set_one (':path', uri) + #headers.set_one (':version', 'HTTP/1.1') + sid = self.push_syn_stream (headers, has_data) + if content: + # tricky, hold one block back + last = None + for block in content: + if last: + self.push_data_frame (sid, block, False) + last = block + self.push_data_frame (sid, last, True) + return sid diff --git a/coro/http/hpack.py b/coro/http/hpack.py new file mode 100644 index 0000000..de55b96 --- /dev/null +++ b/coro/http/hpack.py @@ -0,0 +1,370 @@ +# -*- Mode: Python -*- + +static_table = [ + (None, None), + (':authority', None), + (':method', 'GET'), + (':method', 'POST'), + (':path', '/'), + (':path', '/index.html'), + (':scheme', 'http'), + (':scheme', 'https'), + (':status', '200'), + (':status', '204'), + (':status', '206'), + (':status', '304'), + (':status', '400'), + (':status', '404'), + (':status', '500'), + ('accept-charset', None), + ('accept-encoding', 'gzip, deflate'), + ('accept-language', None), + ('accept-ranges', None), + ('accept', None), + ('access-control-allow-origin', None), + ('age', None), + ('allow', None), + ('authorization', None), + ('cache-control', None), + ('content-disposition', None), + ('content-encoding', None), + ('content-language', None), + ('content-length', None), + ('content-location', None), + ('content-range', None), + ('content-type', None), + ('cookie', None), + ('date', None), + ('etag', None), + ('expect', None), + ('expires', None), + ('from', None), + ('host', None), + ('if-match', None), + ('if-modified-since', None), + ('if-none-match', None), + ('if-range', None), + ('if-unmodified-since', None), + ('last-modified', None), + ('link', None), + ('location', None), + ('max-forwards', None), + ('proxy-authenticate', None), + ('proxy-authorization', None), + ('range', None), + ('referer', None), + ('refresh', None), + ('retry-after', None), + ('server', None), + ('set-cookie', None), + ('strict-transport-security', None), + ('transfer-encoding', None), + ('user-agent', None), + ('vary', None), + ('via', None), + ('www-authenticate', None), +] + +nstatic = len(static_table) + +static_map = {} +for i, key in enumerate (static_table): + static_map[key] = i + +# convert from ascii to a binary tree. +# each leaf is a character. +def from_ascii (s, pos=0): + if s[pos] == '.': + pos += 1 + l, pos = from_ascii (s, pos) + r, pos = from_ascii (s, pos) + return [l, r], pos + elif s[pos] == 'Z': + return 256, pos + 1 + else: + return int (s[pos:pos+2], 16), pos + 2 + +# source: see huffman.py +huffman_table, _ = from_ascii ( + '.....3031.3261..6365.696f...7374..2025.2d2e...2f33.3435..3637.3839.....3d41.5f62' + '..6466.6768...6c6d.6e70..7275..3a42.4344.....4546.4748..494a.4b4c...4d4e.4f50..' + '5152.5354....5556.5759..6a6b.7176...7778.797a...262a.2c3b..585a...2122.2829..3f.' + '272b..7c.233e...0024.405b..5d7e..5e7d..3c60.7b....5cc3.d0.8082...83a2.b8c2..e0e2' + '..99a1.a7ac.....b0b1.b3d1..d8d9.e3e5...e6.8184..8586.8892...9a9c.a0a3..a4a9.aaad' + '.....b2b5.b9ba..bbbd.bec4...c6e4.e8e9...0187.898a..8b8c.8d8f.....9395.9697..989b' + '.9d9e...a5a6.a8ae..afb4.b6b7....bcbf.c5e7..ef.098e..9091.949f....abce.d7e1..eced' + '..c7cf.eaeb.....c0c1.c8c9..cacd.d2d5...dadb.eef0..f2f3.ff.cbcc.....d3d4.d6dd..de' + 'df.f1f4...f5f6.f7f8..fafb.fcfd....fe.0203..0405.0607...080b.0c0e..0f10.1112....1' + '314.1517..1819.1a1b...1c1d.1e1f..7fdc.f9..0a0d.16Z' +) + +# build a map from byte -> (num, bits) +# e.g. huffman_map[ord('A')] = (33, 6) +def make_huffman_map (t): + m = {} + def loop (t, n, bits): + if isinstance (t, int): + m[t] = (n, bits) + else: + loop (t[0], n << 1 | 0, bits + 1) + loop (t[1], n << 1 | 1, bits + 1) + loop (t, 0, 0) + return m + +huffman_map = make_huffman_map (huffman_table) + +masks = {i : (1<> shift & masks[slice] + bits -= slice + self.left -= slice + if self.left == 0: + self.emit_byte() + + def encode (self, s): + for ch in s: + code, bits = huffman_map[ord(ch)] + self.emit (code, bits) + + def done (self): + self.emit (0xffffffff, self.left) + return ''.join (self.data) + +def huffman_encode (s): + h = HuffmanEncoder() + h.encode (s) + return h.done() + +class DynamicTable: + + def __init__ (self, max_size=1024): + self.table = [] + self.size = 0 + self.max_size = max_size + + def __getitem__ (self, index): + if index < nstatic: + return static_table[index] + else: + return self.table[index-nstatic] + + def entry_size (self, name, val): + return len(name) + len(val) + 32 + + def evict_one (self): + (k, v) = self.table.pop() + self.size -= self.entry_size (k, v) + print 'evicted', k, v, self.size + + def __setitem__ (self, name, val): + es = self.entry_size (name, val) + while self.size + es > self.max_size: + self.evict_one() + self.table.insert (0, (name, val)) + self.size += es + + def set_size (self, size): + self.max_size = size + while self.size > self.max_size: + self.evict_one() + +class Decoder: + + def __init__ (self, table=None): + self.data = '' + self.pos = 0 + # used when pulling off huffman-encoded bits. + self.bpos = 0 + if table is None: + table = DynamicTable() + self.dyn = table + + def feed (self, data): + self.data = data + self.pos = 0 + + @property + def done (self): + return self.pos >= len(self.data) + + @property + def byte (self): + return ord(self.data[self.pos]) + + def next_byte (self): + self.pos += 1 + self.bpos = 7 + + def get_bytes (self, n): + result = self.data[self.pos:self.pos+n] + self.pos += n + assert (len(result) == n) + return result + + def get_integer (self, nbits): + # fetch an integer from the lower of + # the current byte. + mask = masks[nbits] + r = self.byte & mask + if r == mask: + # more octets + r = 0 + while 1: + self.next_byte() + r <<= 7 + r |= self.byte & 0x7f + if not self.byte & 0x80: + break + self.next_byte() + return r + mask + else: + self.next_byte() + return r + + def get_bit (self): + r = (self.byte & (1 << self.bpos)) != 0 + self.bpos -= 1 + if self.bpos < 0: + self.next_byte() + return r + + def get_pair0 (self, index): + if index == 0: + name = self.get_literal() + else: + name = self.dyn[index][0] + val = self.get_literal() + return name, val + + def get_header (self): + if self.byte >> 7 == 0x1: + # index name and value + index = self.get_integer (7) + assert index != 0 + return self.dyn[index] + elif self.byte >> 6 == 0b01: + # literal with incremental indexing. + index = self.get_integer (6) + name, val = self.get_pair0 (index) + self.dyn[name] = val + return name, val + elif self.byte >> 4 in (0b0000, 0b0001): + never = self.byte >> 4 & 0b0001 + # literal without indexing + index = self.get_integer (4) + name, val = self.get_pair0 (index) + return name, val + elif self.byte >> 5 == 0b001: + self.dyn.set_size (self.get_integer (5)) + + def get_literal (self): + is_huffman = self.byte & 0b10000000 + lit_len = self.get_integer (7) + if is_huffman: + return self.get_huffman (lit_len) + else: + return self.get_bytes (lit_len) + + def get_huffman (self, nbytes): + r = [] + stop = self.pos + nbytes + while 1: + t = huffman_table + while 1: + b = self.get_bit() + t = t[b] + if isinstance (t, int): + r.append (chr (t)) + break + if self.pos == stop: + return ''.join (r) + return ''.join (r) + +class Encoder: + + def __init__ (self, table=None): + # not used yet. + if table is None: + table = DynamicTable() + self.table = table + self.data = [] + + def emit (self, b): + self.data.append (chr(b)) + + def emit_integer (self, n0, n1_bits, n1): + mask = masks[n1_bits] + if n1 < mask: + self.emit ((n0 << n1_bits) | n1) + else: + self.emit ((n0 << n1_bits) | mask) + n1 -= mask + # encode remaining bits 7 at a time... + while n1 >= 0b10000000: + self.emit ((n1 & 0b01111111) | 0b10000000) + n1 >>= 7 + # and any leftover (or 0). + self.emit (n1) + + def emit_literal (self, s): + s0 = huffman_encode (s) + if len(s0) > len(s): + # oops, not huffman-friendly + self.emit_integer (0b0, 7, len(s)) + self.data.append (s) + else: + self.emit_integer (0b1, 7, len(s0)) + self.data.append (s0) + + def emit_header (self, name, val): + index = static_map.get ((name, val), None) + if index is not None: + # index name and value + self.emit_integer (1, 7, index) + else: + index = static_map.get ((name, None), None) + # XXX no dyntable yet + index = None + if index is not None: + # index name, literal value + self.emit_integer (0b0000, 4, index) + self.emit_literal (val) + else: + # literal name, literal value + self.emit_integer (0b0000, 4, 0) + self.emit_literal (name) + self.emit_literal (val) + + def __call__ (self, hset): + self.data = [] + # rfc7540 8.1.2.1 Pseudo-Header Fields requires that + # pseudo-headers precede normal headers. + pseudo = [] + normal = [] + for name, vals in hset: + if name.startswith (':'): + pseudo.append ((name, vals)) + else: + normal.append ((name, vals)) + items = pseudo + normal + for name, vals in items: + for val in vals: + self.emit_header (name, val) + return ''.join (self.data) From 8d56a3421c2892f6b67dcd05f9039d18edb6168d Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 13:29:20 -0700 Subject: [PATCH 27/46] no wait_for_xxx(), this is handled by cys2n. close, shutdown: handle closed sockets gracefully. --- coro/ssl/s2n/__init__.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/coro/ssl/s2n/__init__.py b/coro/ssl/s2n/__init__.py index 692924d..afd21e1 100644 --- a/coro/ssl/s2n/__init__.py +++ b/coro/ssl/s2n/__init__.py @@ -1,7 +1,7 @@ # -*- Mode: Python -*- import coro -from cys2n import MODE, PROTOCOL, Config +from cys2n import MODE, PROTOCOL, Config, Error as S2NError from ._s2n import * from coro.log import Facility @@ -58,7 +58,6 @@ def recv (self, block_size): break else: left -= len(b) - self.wait_for_read() return ''.join (r) read = recv @@ -85,8 +84,6 @@ def send (self, data): pos += n if not more: break - else: - self.wait_for_write() left -= n return pos @@ -105,14 +102,20 @@ def readv (self, _ignore): raise NotImplementedError def shutdown (self, how=None): - more = 1 - while more: - more = self.s2n_conn.shutdown() - - def close (self): try: - coro.with_timeout (1, self.shutdown) - except coro.TimeoutError: + self.s2n_conn.shutdown() + except S2NError: pass - finally: - coro.sock.close (self) + + def close (self): + if self.fd != -1: + # another thread closed us already. + return + else: + try: + coro.with_timeout (1, self.shutdown) + except coro.TimeoutError: + pass + finally: + LOG ('coro.sock.close', self.orig_fd) + coro.sock.close (self) From 5dabae03e61259fbd36b365e01a2d95d44103223 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 13:29:46 -0700 Subject: [PATCH 28/46] set_info_callback: useful for debugging tls sessions. --- coro/ssl/openssl.pxi | 14 ++++++++++++++ coro/ssl/openssl.pyx | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/coro/ssl/openssl.pxi b/coro/ssl/openssl.pxi index 17f273c..e9acebb 100644 --- a/coro/ssl/openssl.pxi +++ b/coro/ssl/openssl.pxi @@ -268,6 +268,9 @@ cdef extern from "openssl/ssl.h": ctypedef struct SSL ctypedef struct SSL_CTX ctypedef struct SSL_METHOD + int SSL_CB_ALERT + int SSL_CB_LOOP + int SSL_CB_EXIT void SSL_CTX_free (SSL_CTX *) SSL_CTX * SSL_CTX_new (SSL_METHOD *) SSL * SSL_new (SSL_CTX *) @@ -287,6 +290,17 @@ cdef extern from "openssl/ssl.h": int SSL_CTX_check_private_key (SSL_CTX *) int SSL_check_private_key (SSL *) int SSL_CTX_load_verify_locations (SSL_CTX *, char *, char *) + const char *SSL_state_string_long(const SSL *s) + const char *SSL_alert_type_string_long(int value); + const char *SSL_alert_desc_string_long(int value); + void SSL_set_info_callback ( + SSL *ssl, + void (*cb) (const SSL *ssl, int type, int val) + ) + void SSL_CTX_set_info_callback ( + SSL_CTX *ctx, + void (*cb) (const SSL *ssl, int type, int val) + ) IF NPN: # next protocol ('NPN') support diff --git a/coro/ssl/openssl.pyx b/coro/ssl/openssl.pyx index 1c7998a..db9f5a8 100644 --- a/coro/ssl/openssl.pyx +++ b/coro/ssl/openssl.pyx @@ -864,6 +864,18 @@ cdef class dh_param: # ================================================================================ +from coro.log import Facility + +LOG = Facility ('openssl') + +cdef void info_callback (const SSL * s, int where, int ret): + if where & SSL_CB_LOOP: + LOG ('loop', SSL_state_string_long (s)) + elif where & SSL_CB_ALERT: + LOG ('alert', SSL_alert_type_string_long (ret), SSL_alert_desc_string_long(ret)) + elif where & SSL_CB_EXIT: + LOG ('exit', ret, SSL_state_string_long (s)) + cdef class ssl_ctx cdef class ssl: @@ -1165,6 +1177,9 @@ cdef class ssl_ctx: def set_options (self, long options): return SSL_CTX_set_options (self.ctx, options) + def set_info_callback (self): + SSL_CTX_set_info_callback (self.ctx, &info_callback) + IF NPN: def set_next_protos (self, list protos): r = [] From e76a5faa2f81fee15ab5864ccc8dc747211fdd10 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 13:32:23 -0700 Subject: [PATCH 29/46] remove some spdy cruft. place constants into namespace classes. read_frames: catch OSError. send_thread: catch OSError. send_goaway: new fun. frame_priority: new. --- coro/http/h2.py | 255 +++++++++++++++++++++++++++++------------------- 1 file changed, 154 insertions(+), 101 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index c48e65a..2311568 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -37,9 +37,6 @@ def _gen_h2 (self): else: yield block -FLAG_FIN = 0x01 -FLAG_UNIDIRECTIONAL = 0x02 - class h2_server_request (http_request): def __init__ (self, flags, stream_id, client, headers): @@ -66,7 +63,7 @@ def can_deflate (self): return True def has_body (self): - return not (self.flags & FLAG_FIN) + return not (self.flags & FLAGS.END_STREAM) def make_content_file (self): # XXX probably untested... @@ -114,13 +111,36 @@ def unpack_frame_header (head): def pack_frame_header (length, ftype, flags, stream_id): lentype = (length << 8) | (ftype & 0xff) - return struct.pack ('>LBL', lentype, flags, stream_id) - -# this is a mixin class used for both server and client. - -class h2_protocol: - - frame_types = { + return struct.pack ('>LBl', lentype, flags, stream_id) + +class ERROR: + NO_ERROR = 0x0 + PROTOCOL_ERROR = 0x1 + INTERNAL_ERROR = 0x2 + FLOW_CONTROL_ERROR = 0x3 + SETTINGS_TIMEOUT = 0x4 + STREAM_CLOSED = 0x5 + FRAME_SIZE_ERROR = 0x6 + REFUSED_STREAM = 0x7 + CANCEL = 0x8 + COMPRESSION_ERROR = 0x9 + CONNECT_ERROR = 0xa + ENHANCE_YOUR_CALM = 0xb + INADEQUATE_SECURITY = 0xc + HTTP_1_1_REQUIRED = 0xd + +class FRAME: + DATA = 0 + HEADERS = 1 + PRIORITY = 2 + RST_STREAM = 3 + SETTINGS = 4 + PUSH_PROMISE = 5 + PING = 6 + GOAWAY = 7 + WINDOW_UPDATE = 8 + CONTINUATION = 9 + types = { 0: 'data', 1: 'headers', 2: 'priority', @@ -133,6 +153,26 @@ class h2_protocol: 9: 'continuation', } +class SETTINGS: + HEADER_TABLE_SIZE = 0x01 + ENABLE_PUSH = 0x02 + MAX_CONCURRENT_STREAMS = 0x03 + INITIAL_WINDOW_SIZE = 0x04 + MAX_FRAME_SIZE = 0x05 + MAX_HEADER_LIST_SIZE = 0x06 + +class FLAGS: + END_STREAM = 0x01 + END_HEADERS = 0x04 + PADDED = 0x08 + PRIORITY = 0x20 + PING_ACK = 0x01 + SETTINGS_ACK = 0x01 + +# this is a mixin class used for both server and client. + +class h2_protocol: + protocol = 'h2' preface = 'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' is_server = True @@ -162,23 +202,28 @@ def read_frames (self): else: LOG ('preface', preface) assert (preface == self.preface) - while 1: - head = self.read_exact (9) - if not head: - self.close() - return - flen, ftype, flags, stream_id = unpack_frame_header (head) - if flen: - payload = self.read_exact (flen) - else: - payload = None - method_name = 'frame_%s' % (self.frame_types.get (ftype, '')) - if method_name == 'frame_': - self.log ('unknown h2 frame type: %d' % (ftype,)) - else: - LOG ('frame', method_name, flags, stream_id, payload) - method = getattr (self, method_name) - method (flags, stream_id, payload) + try: + while 1: + head = self.read_exact (9) + if not head: + self.close() + return + flen, ftype, flags, stream_id = unpack_frame_header (head) + LOG ('frame header', flen, ftype, flags, stream_id) + if flen: + payload = self.read_exact (flen) + else: + payload = b'' + method_name = 'frame_%s' % (FRAME.types.get (ftype, '')) + if method_name == 'frame_': + self.log ('unknown h2 frame type: %d' % (ftype,)) + else: + LOG ('frame', method_name, flags, stream_id, payload) + method = getattr (self, method_name) + method (flags, stream_id, payload) + except OSError: + LOG ('OSError') + self.close() def unpack_http_header (self, data): hs = header_set() @@ -205,6 +250,7 @@ class h2_connection (h2_protocol, connection): def run (self): self.streams = {} + self.priorities = {} self.encoder = Encoder() self.decoder = Decoder() self.ofifo = coro.fifo() @@ -212,6 +258,8 @@ def run (self): coro.spawn (self.send_thread) try: self.read_frames() + except coro.oserrors.ECONNRESET: + LOG ('connection reset') finally: self.ofifo.push (None) @@ -220,14 +268,19 @@ def close (self): self.conn.close() def send_thread (self): - while 1: - block = self.ofifo.pop() - if block is None: - break - else: - LOG ('send', block) - self.conn.send (block) - self.obuf.release (len(block)) + try: + while 1: + block = self.ofifo.pop() + if block is None: + break + else: + LOG ('send', len(block)) + self.conn.send (block) + self.obuf.release (len(block)) + except OSError: + LOG ('OSError') + finally: + self.close() def push_frame (self, frame): self.obuf.acquire (len(frame)) @@ -237,16 +290,16 @@ def push_frame (self, frame): def send_frame (self, ftype, flags, stream_id, data): dlen = len(data) head = pack_frame_header (dlen, ftype, flags, stream_id) + LOG ('send_frame', FRAME.types[ftype], flags, stream_id) return self.push_frame (head + data) def push_headers (self, req, has_data): LOG ('push_headers', req, not not has_data) - flags = self.FLAGS_END_HEADERS + flags = FLAGS.END_HEADERS if not has_data: - flags |= self.FLAGS_END_STREAM + flags |= FLAGS.END_STREAM hdata = self.pack_http_header (req.reply_headers) - LOG ('reply_headers', req.reply_headers) - LOG ('push_headers', hdata) + LOG ('push_headers', req.stream_id) req.output.sent += self.send_frame (0x01, flags, req.stream_id, hdata) def push_ping (self, flags=0, data=None): @@ -258,7 +311,7 @@ def push_ping (self, flags=0, data=None): def push_data (self, req, data, last): if last: - flags = self.FLAGS_END_STREAM + flags = FLAGS.END_STREAM else: flags = 0 LOG ('push_data', len(data)) @@ -267,55 +320,63 @@ def push_data (self, req, data, last): def frame_settings (self, flags, stream_id, payload): plen = len(payload) n, check = divmod (plen, 6) - assert check == 0 - self.h2_settings = {} - for i in range (0, 6, plen): - ident, value = struct.unpack ('>HL', payload[i:i+6]) - self.h2_settings[ident] = value - LOG ('settings', self.h2_settings) - # ack it. - self.send_frame (0x04, 0x01, 0, '') - self.push_ping() - - FLAGS_PING_ACK = 0x01 + if flags & FLAGS.SETTINGS_ACK: + LOG ('settings', 'ack') + else: + assert check == 0 + # XXX store these into ivars + self.h2_settings = {} + for i in range (0, 6, plen): + ident, value = struct.unpack ('>HL', payload[i:i+6]) + self.h2_settings[ident] = value + LOG ('settings', self.h2_settings) + # ack it. + self.send_frame (0x04, FLAGS.SETTINGS_ACK, 0, '') + self.push_settings() + + initial_window_size = 16 * 1024 * 1024 + def push_settings (self): + # let's just set the initial window size + payload = struct.pack ('>HL', SETTINGS.INITIAL_WINDOW_SIZE, self.initial_window_size) + self.send_frame (FRAME.SETTINGS, 0, 0, payload) + def frame_ping (self, flags, stream_id, payload): assert len(payload) == 8 - if flags & self.FLAGS_PING_ACK: + LOG ('ping', flags, stream_id, payload) + if flags & FLAGS.PING_ACK: assert payload == self.last_ping else: assert len(payload) == 8 - self.push_ping (0x01, payload) + self.send_frame (0x06, FLAGS.PING_ACK, 0, payload) def frame_window_update (self, flags, stream_id, payload): increment, = struct.unpack ('>l', payload) assert increment >= 0 LOG ('window_update', increment) - FLAGS_END_STREAM = 0x01 - FLAGS_END_HEADERS = 0x04 - FLAGS_PADDED = 0x08 - FLAGS_PRIORITY = 0x20 - def frame_headers (self, flags, stream_id, payload): pos = 0 pad_len = 0 stream_dep = 0 weight = 0 assert stream_id > 0 - if flags & self.FLAGS_PADDED: + if flags & FLAGS.PADDED: pad_len, = struct.unpack ('>B', payload[pos:pos+1]) pos += 1 - LOG ('headers', 'padded', pad_len) - if flags & self.FLAGS_PRIORITY: + #LOG ('headers', 'padded', pad_len) + if flags & FLAGS.PRIORITY: stream_dep, weight = struct.unpack ('>lB', payload[:5]) pos += 5 - LOG ('headers', 'priority', stream_dep, weight) - if flags & self.FLAGS_END_STREAM: - LOG ('headers', 'end_stream') - if flags & self.FLAGS_END_HEADERS: - LOG ('headers', 'end_headers') + #LOG ('headers', 'priority', stream_dep, weight) + if flags & FLAGS.END_STREAM: + #LOG ('headers', 'end_stream') + pass + if flags & FLAGS.END_HEADERS: + #LOG ('headers', 'end_headers') + pass else: raise NotImplementedError + LOG ('headers', flags, stream_id, pad_len, stream_dep, weight) if pad_len: header_block = payload[pos:-pad_len] else: @@ -337,18 +398,38 @@ def unpack_http_header (self, header_block): def pack_http_header (self, hset): return self.encoder (hset) + def send_goaway (self, last_stream_id, error_code, debug_data): + payload = struct.pack ('>lL', last_stream_id, error_code) + debug_data + self.send_frame (0x07, 0x00, 0x00, payload) + def frame_rst_stream (self, flags, stream_id, payload): LOG ('frame_rst_stream', stream_id) - del self.streams[stream_id] + try: + del self.streams[stream_id] + except KeyError: + LOG ('bad rst_stream', stream_id) + try: + del self.priorities[stream_id] + except KeyError: + pass - def frame_data (self, flags, stream_id, payload): - import pdb; pdb.set_trace() def frame_priority (self, flags, stream_id, payload): + stream_dep, weight = struct.unpack ('LL', data) - # LOG ('reset: %x status=%d %s' % (stream_id, status_code, self.status_codes.get (status_code, 'unknown'))) - # del self.streams[stream_id] - - # def frame_goaway (self, flags, data): - # last_stream_id, = struct.unpack ('>L', data) - # LOG ('goaway last_stream_id=%d' % (last_stream_id,)) - # # XXX arrange for the connection to close - # self.close() - - # def frame_ping (self, flags, data): - # ping_id, = struct.unpack ('>L', data) - # LOG ('ping', flags, ping_id) - # self.send_frame (self.pack_control_frame (6, 0, data)) - - # def frame_settings (self, flags, data): - # self.log ('H2 settings frame received [ignored]') - # pass - - # def frame_headers (self, flags, data): - # self.log ('H2 headers frame received [ignored]') - # pass - - # def frame_window_update (self, flags, data): - # stream_id, delta_window_size = struct.unpack ('>LL', data) - # self.log ('h2 window update', stream_id, delta_window_size) - class h2_tlslite_server (tlslite_server): protocol = 'h2' @@ -402,7 +455,7 @@ def __init__ (self, addr, cert_path, key_path, settings=None): tlslite_server.__init__ (self, addr, cert_path, key_path, nextProtos=['h2', 'http/1.1'], settings=settings) def create_connection (self, conn, addr): - if conn.next_proto == b'h2/3': + if conn.next_proto == b'h2': return h2_connection (self, conn, addr) else: return connection (self, conn, addr) From 4f37916cc11d20b203796d0019031ec0ccbe6255 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 13:32:54 -0700 Subject: [PATCH 30/46] note about the firefox magic setting to work with openssl. --- coro/http/demo/h2_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/coro/http/demo/h2_server.py b/coro/http/demo/h2_server.py index 281eddc..b3f73cd 100644 --- a/coro/http/demo/h2_server.py +++ b/coro/http/demo/h2_server.py @@ -3,13 +3,19 @@ import coro.http.spdy import coro.backdoor +# note: firefox will not connect to openssl h2 server unless you set +# 'network.http.spdy.enforce-tls-profile' to false. [see about:config] +# chrome works, though. +# according to https://nghttp2.org/blog/2014/09/15/host-lucid-erlang-http-slash-2-server/ +# this is because AEAD is missing. + # god how I love openssl. cipher_suite = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK" ctx = coro.ssl.new_ctx ( cert=coro.ssl.x509 (open ('cert/server.crt').read()), key=coro.ssl.pkey (open ('cert/server.key').read(), '', True), - alpn_protos=['h2'], #, 'http/1.1'], + alpn_protos=['h2', 'http/1.1'], proto='tlsv2', ciphers = cipher_suite, dhparam=coro.ssl.dh_param (open ('cert/dhparam.pem').read()), From d3c0c7ba830475cf2061343fa5933165c62c4a16 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 16:24:37 -0700 Subject: [PATCH 31/46] make safari ios happy. frame_settings: do you even iterate, bro? push_settings: allow override of initial settings via class var. run: push_settings at connect time (safari insists). --- coro/http/h2.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 2311568..a2002d3 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -256,6 +256,7 @@ def run (self): self.ofifo = coro.fifo() self.obuf = coro.semaphore (self.output_buffer_size) coro.spawn (self.send_thread) + self.push_settings() try: self.read_frames() except coro.oserrors.ECONNRESET: @@ -326,19 +327,22 @@ def frame_settings (self, flags, stream_id, payload): assert check == 0 # XXX store these into ivars self.h2_settings = {} - for i in range (0, 6, plen): + for i in range (0, plen, 6): ident, value = struct.unpack ('>HL', payload[i:i+6]) self.h2_settings[ident] = value LOG ('settings', self.h2_settings) # ack it. - self.send_frame (0x04, FLAGS.SETTINGS_ACK, 0, '') - self.push_settings() + self.send_frame (FRAME.SETTINGS, FLAGS.SETTINGS_ACK, 0, '') - initial_window_size = 16 * 1024 * 1024 + initial_window_size = 65535 + initial_settings = [ + (SETTINGS.INITIAL_WINDOW_SIZE, initial_window_size), + ] def push_settings (self): - # let's just set the initial window size - payload = struct.pack ('>HL', SETTINGS.INITIAL_WINDOW_SIZE, self.initial_window_size) - self.send_frame (FRAME.SETTINGS, 0, 0, payload) + payload = [] + for key, val in self.initial_settings: + payload.append (struct.pack ('>HL', key, val)) + self.send_frame (FRAME.SETTINGS, 0, 0, b''.join (payload)) def frame_ping (self, flags, stream_id, payload): assert len(payload) == 8 From 1824442650e6bf1096fe4ae98222c546af65bfc3 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 17:51:35 -0700 Subject: [PATCH 32/46] frame_data: a start on handling requests with data. --- coro/http/h2.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index a2002d3..51a86c3 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -8,8 +8,9 @@ # Note: not trying to share code with spdy, since the plan is to deprecate it completely. from coro.http import connection, tlslite_server, openssl_server, s2n_server, http_request -from coro.http.protocol import header_set, http_file +from coro.http.protocol import header_set, http_file, latch from coro.http.hpack import Encoder, Decoder +from coro import read_stream from coro.log import Facility LOG = Facility ('h2') @@ -23,7 +24,12 @@ class h2_file (http_file): # override http_file's content generator (which is a 'pull' generator) # with this coro.fifo-based 'push' generator. - def get_content_gen (self, headers): + def __init__ (self, headers, stream): + self.streami = stream + self.streamo = read_stream.buffered_stream (self.get_content_gen().next) + self.done_cv = latch() + + def get_content_gen (self): self.content_fifo = coro.fifo() return self._gen_h2() @@ -58,11 +64,14 @@ def __init__ (self, flags, stream_id, client, headers): request = '%s %s %s' % (method, url, version) # XXX consider removing method/url/version? http_request.__init__ (self, client, request, headers) + if self.has_body(): + self.make_content_file() def can_deflate (self): return True def has_body (self): + LOG ('has_body', not (self.flags & FLAGS.END_STREAM)) return not (self.flags & FLAGS.END_STREAM) def make_content_file (self): @@ -431,7 +440,15 @@ def frame_goaway (self, flags, stream_id, payload): self.close() def frame_data (self, flags, stream_id, payload): - import pdb; pdb.set_trace() + probe = self.streams.get (stream_id, None) + if probe is not None: + probe.file.content_fifo.push (payload) + if flags & FLAGS.END_STREAM: + probe.file.content_fifo.push (None) + del self.streams[stream_id] + else: + self.log ('orphaned data frame [%d bytes] for stream %d\n' % (len(payload), stream_id)) + def frame_push_promise (self, flags, stream_id, payload): import pdb; pdb.set_trace() def frame_continuation (self, flags, stream_id, payload): From 4ba0fd1270192955cda79759ef9a64038485da07 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Tue, 22 Sep 2015 18:42:55 -0700 Subject: [PATCH 33/46] HuffmanEncoder.done: don't pad when 8 bits remain. --- coro/http/hpack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coro/http/hpack.py b/coro/http/hpack.py index de55b96..de628d2 100644 --- a/coro/http/hpack.py +++ b/coro/http/hpack.py @@ -144,7 +144,8 @@ def encode (self, s): self.emit (code, bits) def done (self): - self.emit (0xffffffff, self.left) + if self.left < 8: + self.emit (0xffffffff, self.left) return ''.join (self.data) def huffman_encode (s): From 53e4a2d5dc89a14b54678e46ba346fb2f8b77cbd Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 23 Sep 2015 17:38:01 -0700 Subject: [PATCH 34/46] sync with samrushing/hpack:d8a55594335fd799c1a9744e53111634bb9bcbb3 --- coro/http/hpack.py | 94 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/coro/http/hpack.py b/coro/http/hpack.py index de628d2..2fd5c75 100644 --- a/coro/http/hpack.py +++ b/coro/http/hpack.py @@ -113,7 +113,7 @@ def loop (t, n, bits): huffman_map = make_huffman_map (huffman_table) -masks = {i : (1<= len(self.data) @@ -286,26 +330,24 @@ def get_literal (self): def get_huffman (self, nbytes): r = [] stop = self.pos + nbytes - while 1: + while self.pos < stop: t = huffman_table - while 1: + while self.pos < stop: b = self.get_bit() t = t[b] if isinstance (t, int): r.append (chr (t)) break - if self.pos == stop: - return ''.join (r) return ''.join (r) class Encoder: def __init__ (self, table=None): - # not used yet. if table is None: table = DynamicTable() self.table = table self.data = [] + self.never = set() def emit (self, b): self.data.append (chr(b)) @@ -335,24 +377,34 @@ def emit_literal (self, s): self.data.append (s0) def emit_header (self, name, val): - index = static_map.get ((name, val), None) - if index is not None: - # index name and value - self.emit_integer (1, 7, index) + if name in self.never: + # literal name, literal value, never index + self.emit_integer (0b0001, 4, 0) + self.emit_literal (name) + self.emit_literal (val) else: - index = static_map.get ((name, None), None) - # XXX no dyntable yet - index = None + index, both = self.table.get_index (name, val) if index is not None: - # index name, literal value - self.emit_integer (0b0000, 4, index) - self.emit_literal (val) + if both: + # index name and value + self.emit_integer (0b1, 7, index) + else: + # index name, literal/new value + self.table[name] = val + self.emit_integer (0b01, 6, index) + self.emit_literal (val) else: - # literal name, literal value - self.emit_integer (0b0000, 4, 0) + # literal name, literal value, new name + self.table[name] = val + self.emit_integer (0b01, 6, 0) self.emit_literal (name) self.emit_literal (val) + def flush (self): + r = b''.join (self.data) + self.data = [] + return r + def __call__ (self, hset): self.data = [] # rfc7540 8.1.2.1 Pseudo-Header Fields requires that @@ -368,4 +420,4 @@ def __call__ (self, hset): for name, vals in items: for val in vals: self.emit_header (name, val) - return ''.join (self.data) + return self.flush() From 449a87f1112c87f7477e05debe7b06d4fe01178d Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 23 Sep 2015 17:38:30 -0700 Subject: [PATCH 35/46] listdir_handler. --- coro/http/handlers.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/coro/http/handlers.py b/coro/http/handlers.py index 2419f55..24e4056 100644 --- a/coro/http/handlers.py +++ b/coro/http/handlers.py @@ -297,3 +297,46 @@ def handle_unauthorized (self, request): 'nonce="%s"' % (nonce,), ]) request.error (401) + +class listdir_handler (file_handler): + + css = ( + 'body { font-family: monospace; }' + 'tr:nth-child(odd) {' + ' background-color:#f0f0f0;' + '}' + 'tr:nth-child(even) {' + ' background-color:#e0e0e0;' + '}' + '.ellipsis {' + ' text-overflow: ellipsis;' + ' overflow: hidden;' + ' width:20em;' + ' display:block;' + '}' + ) + + time_format = '%Y-%m-%d %H:%M:%S' + + def handle_directory_listing (self, request, path): + try: + subs = os.listdir (path) + except OSError: + request.error (403) + else: + subs.sort() + stats = [os.stat(os.path.join(path, x)) for x in subs] + request.push ('' % (self.css,)) + request.push ('') + for i, sub in enumerate (subs): + st = stats[i] + if stat.S_ISDIR(st.st_mode): + sub += '/' + timestamp = time.strftime (self.time_format, time.localtime (st.st_mtime)) + request.push ( + '' % ( + timestamp, st.st_size, sub, sub + ) + ) + request.push ('
datesizename
%s%d%s
') + request.done() From 16cd214b8d5fa12f4362759bd8aa2a3ddd4fae78 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 23 Sep 2015 17:38:56 -0700 Subject: [PATCH 36/46] use listdir_handler to serve the shrapnel source tree. --- coro/http/demo/h2_s2n_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coro/http/demo/h2_s2n_server.py b/coro/http/demo/h2_s2n_server.py index b0c3371..157f166 100644 --- a/coro/http/demo/h2_s2n_server.py +++ b/coro/http/demo/h2_s2n_server.py @@ -16,7 +16,7 @@ server = coro.http.h2.h2_s2n_server (cfg) server.push_handler (coro.http.handlers.favicon_handler()) server.push_handler (coro.http.handlers.coro_status_handler()) -server.push_handler (coro.http.handlers.file_handler ('.')) +server.push_handler (coro.http.handlers.listdir_handler ('../../..')) coro.spawn (server.start, ('0.0.0.0', 8443)) coro.spawn (coro.backdoor.serve, unix_path='/tmp/h2s.bd') coro.event_loop (30.0) From a4d254c293d3e660b84b91f545e21746167072e8 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 23 Sep 2015 17:39:44 -0700 Subject: [PATCH 37/46] send_thread: use pop_all and writev. remove more spdy-related cruft. --- coro/http/h2.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 51a86c3..2cd4ecd 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -15,9 +15,7 @@ from coro.log import Facility LOG = Facility ('h2') -# When a reply is large (say >1MB) we still get a form of head-blocking behavior -# unless we chop it up into bits. Think about an architecture that would -# automatically do that. [i.e., a configurable max size for data frames] +# XXX address MAX_FRAME_SIZE here. class h2_file (http_file): @@ -234,14 +232,6 @@ def read_frames (self): LOG ('OSError') self.close() - def unpack_http_header (self, data): - hs = header_set() - hs.headers = unpack_http_header (self.inflate (data)) - return hs - - def pack_http_header (self, hset): - return self.deflate (pack_http_header (hset.headers)) - # -------------------------------------------------------------------------------- # h2 server # -------------------------------------------------------------------------------- @@ -280,13 +270,14 @@ def close (self): def send_thread (self): try: while 1: - block = self.ofifo.pop() - if block is None: + blocks = self.ofifo.pop_all() + if not blocks: break else: - LOG ('send', len(block)) - self.conn.send (block) - self.obuf.release (len(block)) + total_size = sum ([len(x) for x in blocks]) + LOG ('send', total_size) + self.conn.writev (blocks) + self.obuf.release (total_size) except OSError: LOG ('OSError') finally: From 5b3d23536575f5a9a43be015d33b4e2882752045 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Wed, 23 Sep 2015 17:50:22 -0700 Subject: [PATCH 38/46] send_thread: handle None sentinel properly. --- coro/http/h2.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 2cd4ecd..9562ba3 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -269,11 +269,13 @@ def close (self): def send_thread (self): try: - while 1: + done = False + while not done: blocks = self.ofifo.pop_all() - if not blocks: - break - else: + if None in blocks: + done = True + blocks = [x for x in blocks if x is not None] + if blocks: total_size = sum ([len(x) for x in blocks]) LOG ('send', total_size) self.conn.writev (blocks) From b3856b45141d153a4c405195ad8af2282f054f3a Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 24 Sep 2015 14:46:37 -0700 Subject: [PATCH 39/46] disabled/removed lots of debug logging. --- coro/http/h2.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 9562ba3..4c6915a 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -69,7 +69,7 @@ def can_deflate (self): return True def has_body (self): - LOG ('has_body', not (self.flags & FLAGS.END_STREAM)) + #LOG ('has_body', not (self.flags & FLAGS.END_STREAM)) return not (self.flags & FLAGS.END_STREAM) def make_content_file (self): @@ -207,7 +207,6 @@ def read_frames (self): self.close() return else: - LOG ('preface', preface) assert (preface == self.preface) try: while 1: @@ -216,7 +215,6 @@ def read_frames (self): self.close() return flen, ftype, flags, stream_id = unpack_frame_header (head) - LOG ('frame header', flen, ftype, flags, stream_id) if flen: payload = self.read_exact (flen) else: @@ -225,7 +223,7 @@ def read_frames (self): if method_name == 'frame_': self.log ('unknown h2 frame type: %d' % (ftype,)) else: - LOG ('frame', method_name, flags, stream_id, payload) + #LOG ('frame', method_name, flags, stream_id, payload) method = getattr (self, method_name) method (flags, stream_id, payload) except OSError: @@ -277,7 +275,7 @@ def send_thread (self): blocks = [x for x in blocks if x is not None] if blocks: total_size = sum ([len(x) for x in blocks]) - LOG ('send', total_size) + #LOG ('send', total_size) self.conn.writev (blocks) self.obuf.release (total_size) except OSError: @@ -293,16 +291,16 @@ def push_frame (self, frame): def send_frame (self, ftype, flags, stream_id, data): dlen = len(data) head = pack_frame_header (dlen, ftype, flags, stream_id) - LOG ('send_frame', FRAME.types[ftype], flags, stream_id) + #LOG ('send_frame', FRAME.types[ftype], flags, stream_id) return self.push_frame (head + data) def push_headers (self, req, has_data): - LOG ('push_headers', req, not not has_data) + #LOG ('push_headers', req, not not has_data) flags = FLAGS.END_HEADERS if not has_data: flags |= FLAGS.END_STREAM hdata = self.pack_http_header (req.reply_headers) - LOG ('push_headers', req.stream_id) + #LOG ('push_headers', req.stream_id) req.output.sent += self.send_frame (0x01, flags, req.stream_id, hdata) def push_ping (self, flags=0, data=None): @@ -317,14 +315,15 @@ def push_data (self, req, data, last): flags = FLAGS.END_STREAM else: flags = 0 - LOG ('push_data', len(data)) + #LOG ('push_data', len(data)) req.output.sent += self.send_frame (0x00, flags, req.stream_id, data) def frame_settings (self, flags, stream_id, payload): plen = len(payload) n, check = divmod (plen, 6) if flags & FLAGS.SETTINGS_ACK: - LOG ('settings', 'ack') + #LOG ('settings', 'ack') + pass else: assert check == 0 # XXX store these into ivars @@ -332,7 +331,7 @@ def frame_settings (self, flags, stream_id, payload): for i in range (0, plen, 6): ident, value = struct.unpack ('>HL', payload[i:i+6]) self.h2_settings[ident] = value - LOG ('settings', self.h2_settings) + #LOG ('settings', self.h2_settings) # ack it. self.send_frame (FRAME.SETTINGS, FLAGS.SETTINGS_ACK, 0, '') @@ -348,7 +347,7 @@ def push_settings (self): def frame_ping (self, flags, stream_id, payload): assert len(payload) == 8 - LOG ('ping', flags, stream_id, payload) + #LOG ('ping', flags, stream_id, payload) if flags & FLAGS.PING_ACK: assert payload == self.last_ping else: @@ -358,7 +357,7 @@ def frame_ping (self, flags, stream_id, payload): def frame_window_update (self, flags, stream_id, payload): increment, = struct.unpack ('>l', payload) assert increment >= 0 - LOG ('window_update', increment) + #LOG ('window_update', increment) def frame_headers (self, flags, stream_id, payload): pos = 0 @@ -382,7 +381,7 @@ def frame_headers (self, flags, stream_id, payload): pass else: raise NotImplementedError - LOG ('headers', flags, stream_id, pad_len, stream_dep, weight) + #LOG ('headers', flags, stream_id, pad_len, stream_dep, weight) if pad_len: header_block = payload[pos:-pad_len] else: @@ -398,7 +397,6 @@ def unpack_http_header (self, header_block): while not self.decoder.done: hname, hval = self.decoder.get_header() hs[hname] = hval - LOG ('headers', hs.headers) return hs def pack_http_header (self, hset): @@ -409,7 +407,7 @@ def send_goaway (self, last_stream_id, error_code, debug_data): self.send_frame (0x07, 0x00, 0x00, payload) def frame_rst_stream (self, flags, stream_id, payload): - LOG ('frame_rst_stream', stream_id) + #LOG ('frame_rst_stream', stream_id) try: del self.streams[stream_id] except KeyError: @@ -421,7 +419,7 @@ def frame_rst_stream (self, flags, stream_id, payload): def frame_priority (self, flags, stream_id, payload): stream_dep, weight = struct.unpack (' Date: Thu, 24 Sep 2015 17:14:39 -0700 Subject: [PATCH 40/46] demo pygmentizing handler. --- coro/http/demo/pygments_handler.py | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 coro/http/demo/pygments_handler.py diff --git a/coro/http/demo/pygments_handler.py b/coro/http/demo/pygments_handler.py new file mode 100644 index 0000000..beec26c --- /dev/null +++ b/coro/http/demo/pygments_handler.py @@ -0,0 +1,60 @@ +# -*- Mode: Python -*- + +import coro +from coro.http.handlers import file_handler + +import pygments +import pygments.lexer +import pygments.lexers +import pygments.formatters +import pygments.util + +from coro.log import Facility + +LOG = Facility ('pygments_handler') + +# unfortunately there doesn't seem to be any way to +# stream data to the lexer, it needs the entire file +# in a string. + +class pygments_request_wrapper: + + def __init__ (self, req, lex): + self._req = req + self._lex = lex + self._data = [] + + def __getattr__ (self, name): + return getattr (self._req, name) + + def push (self, data): + self._data.append (data) + + def done (self): + form = pygments.formatters.get_formatter_by_name ('html', full=True) + self._req.reply_headers.set_one ('content-type', 'text/html', override=True) + code = b''.join (self._data) + form.format (pygments.lex (code, self._lex), self) + self._req.done() + + def write (self, data): + if type(data) is unicode: + self._req.push (data.encode ('utf8')) + else: + self._req.push (data) + +class pygments_handler: + + def __init__ (self, inner_handler): + self.inner_handler = inner_handler + + def match (self, req): + return self.inner_handler.match (req) + + def handle_request (self, req): + try: + lex = pygments.lexers.get_lexer_for_filename (req.path) + req = pygments_request_wrapper (req, lex) + except pygments.util.ClassNotFound: + pass + self.inner_handler.handle_request (req) From 0b3fe7c9d611c2d20527348bd46c5cef78b3500b Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 24 Sep 2015 17:15:41 -0700 Subject: [PATCH 41/46] use pygments_handler wrapped around a listdir_handler. To show off shrapnel's source code tree. --- coro/http/demo/h2_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coro/http/demo/h2_server.py b/coro/http/demo/h2_server.py index b3f73cd..760636a 100644 --- a/coro/http/demo/h2_server.py +++ b/coro/http/demo/h2_server.py @@ -2,6 +2,7 @@ import coro.ssl import coro.http.spdy import coro.backdoor +from pygments_handler import pygments_handler # note: firefox will not connect to openssl h2 server unless you set # 'network.http.spdy.enforce-tls-profile' to false. [see about:config] @@ -24,7 +25,9 @@ server = coro.http.h2.h2_openssl_server (ctx) server.push_handler (coro.http.handlers.favicon_handler()) server.push_handler (coro.http.handlers.coro_status_handler()) -server.push_handler (coro.http.handlers.file_handler ('.')) +lh = coro.http.handlers.listdir_handler ('../../..') +ph = pygments_handler (lh) +server.push_handler (ph) coro.spawn (server.start, ('0.0.0.0', 8443)) coro.spawn (coro.backdoor.serve, unix_path='/tmp/h2s.bd') coro.event_loop (30.0) From 52c89d901ce9669f0a3f9931bc84bc9be47be57d Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 24 Sep 2015 17:16:15 -0700 Subject: [PATCH 42/46] set_one: 'override' option. --- coro/http/protocol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coro/http/protocol.py b/coro/http/protocol.py index 0bc015b..30037ef 100644 --- a/coro/http/protocol.py +++ b/coro/http/protocol.py @@ -161,12 +161,12 @@ def get_one (self, key): else: return r[0] - def set_one (self, key, val): + def set_one (self, key, val, override=False): """Set the value of a header expected to have at most one value. If a value is already present, raise ValueError.""" r = self.headers.get (key, None) - if r is None: - self.headers[key] = val + if override or r is None: + self.headers[key] = [val] else: raise ValueError ("header %r already has a value: %r" % (key, r)) From ba08ddf3fcd37b3e3ae20a6510c26ad335790575 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 8 Oct 2015 14:24:29 -0700 Subject: [PATCH 43/46] pygments_request_wrapper: call _req.set_deflate(). --- coro/http/demo/pygments_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coro/http/demo/pygments_handler.py b/coro/http/demo/pygments_handler.py index beec26c..6d779b9 100644 --- a/coro/http/demo/pygments_handler.py +++ b/coro/http/demo/pygments_handler.py @@ -33,6 +33,7 @@ def push (self, data): def done (self): form = pygments.formatters.get_formatter_by_name ('html', full=True) self._req.reply_headers.set_one ('content-type', 'text/html', override=True) + self._req.set_deflate() code = b''.join (self._data) form.format (pygments.lex (code, self._lex), self) self._req.done() From a6e2c48b81547e039487072ec6b42165ed3c1258 Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 8 Oct 2015 14:25:22 -0700 Subject: [PATCH 44/46] synchronize with https://github.com/samrushing/hpack/. --- coro/http/hpack.py | 91 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/coro/http/hpack.py b/coro/http/hpack.py index 2fd5c75..d8cde43 100644 --- a/coro/http/hpack.py +++ b/coro/http/hpack.py @@ -153,6 +153,14 @@ def huffman_encode (s): h.encode (s) return h.done() +# The DynamicTable is used for both encoding and decoding. +# In encoding mode, both the linear 'table' and reverse 'map' are +# used. The map records the position in the table of an entry at a +# particular time. The 'clock' is used to avoid having to adjust the +# indices in the map every time a new entry is added. For example, +# if an entry for 'xxx' was added at clock=10, then its position at +# clock=15 will be 5. + class DynamicTable: def __init__ (self, max_size=4096): @@ -168,26 +176,53 @@ def __getitem__ (self, index): else: return self.table[index-nstatic] + def get_static_index (self, name, val): + probe0 = static_map.get ((name, val), None) + if probe0 is not None: + return probe0, True + else: + probe1 = static_map.get ((name, None), None) + if probe1 is not None: + return probe1, False + else: + return None, False + + def get_dynamic_index (self, name, val): + vals = self.map.get (name, None) + if vals is None: + return None, False + else: + probe0 = vals.get (val, None) + if probe0 is None: + index, both = vals.values()[0], False + else: + index, both = probe0, True + return nstatic + (self.clock - index) - 1, both + def get_index (self, name, val): - probe = static_map.get ((name, val), None) - if probe is not None: - return probe, True + # returns , + # index: integer or None + # both: bool to indicate if name *and* value were present. + probe0, both0 = self.get_static_index (name, val) + if both0: + return probe0, both0 else: - probe = static_map.get ((name, None), None) - if probe is not None: - return probe, False + probe1, both1 = self.get_dynamic_index (name, val) + if both1: + return probe1, both1 + elif probe1 is None: + # nothing in either table + return None, False + elif probe0 is not None: + # prefer the lower index of the static map. + return probe0, False else: - probe = self.map.get (name, None) - if probe is not None: - if not probe.has_key (val): - index = probe.values()[0] - both = False - else: - index = probe[val] - both = True - return nstatic + (self.clock - index) - 1, both - else: - return None, False + return probe1, both1 + + def check (self): + for k, v in self.table: + assert (self.map.has_key (k)) + assert v in self.map[k] def entry_size (self, name, val): return len(name) + len(val) + 32 @@ -209,6 +244,13 @@ def add_map (self, name, val): else: self.map[name] = { val : self.clock } + def nvals (self, name): + probe = self.map.get (name, None) + if probe is None: + return 0 + else: + return len(probe) + def __setitem__ (self, name, val): es = self.entry_size (name, val) while self.size + es > self.max_size: @@ -217,6 +259,7 @@ def __setitem__ (self, name, val): self.size += es self.add_map (name, val) self.clock += 1 + self.check() def set_size (self, size): self.max_size = size @@ -342,10 +385,11 @@ def get_huffman (self, nbytes): class Encoder: - def __init__ (self, table=None): + def __init__ (self, table=None, max_vals=5): if table is None: table = DynamicTable() self.table = table + self.max_vals = max_vals self.data = [] self.never = set() @@ -390,9 +434,14 @@ def emit_header (self, name, val): self.emit_integer (0b1, 7, index) else: # index name, literal/new value - self.table[name] = val - self.emit_integer (0b01, 6, index) - self.emit_literal (val) + if self.table.nvals (name) >= self.max_vals: + # a header like 'date', which varies. + self.emit_integer (0b0000, 4, index) + self.emit_literal (val) + else: + self.table[name] = val + self.emit_integer (0b01, 6, index) + self.emit_literal (val) else: # literal name, literal value, new name self.table[name] = val From a7ba8a088ac7b50be7d9af6e078d3c571736c88f Mon Sep 17 00:00:00 2001 From: Sam Rushing Date: Thu, 8 Oct 2015 14:36:39 -0700 Subject: [PATCH 45/46] refactored to accommodate h2 client. ERROR,FRAME,SETTINGS,FLAGS: use namespaces for protocol constants. LOG/DEBUG: commented-out debug stuff reformatted. h2_file: simplified, removed some cruft. h2_server_request: push headers opportunistically. set server and date headers. h2_protocol: moved code from h2_connection here. h2_connection: handle_headers. h2_client_request: don't bother deriving from http_client.request. use two latches: http design can't work here because the entire connection is not devoted to sending reply content. add :authority header. frame_data: overridden to retire inflight sem. --- coro/http/h2.py | 446 +++++++++++++++++++++++++++--------------------- 1 file changed, 251 insertions(+), 195 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 4c6915a..2f27b87 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -5,16 +5,86 @@ import sys import os +__version__ = '0.1' + # Note: not trying to share code with spdy, since the plan is to deprecate it completely. from coro.http import connection, tlslite_server, openssl_server, s2n_server, http_request from coro.http.protocol import header_set, http_file, latch from coro.http.hpack import Encoder, Decoder +from coro.http.http_date import build_http_date from coro import read_stream from coro.log import Facility LOG = Facility ('h2') +DEBUG = LOG + +def unpack_frame_header (head): + lentype, flags, stream_id = struct.unpack ('>LBl', head) + assert stream_id >= 0 + return lentype >> 8, lentype & 0xff, flags, stream_id + +def pack_frame_header (length, ftype, flags, stream_id): + lentype = (length << 8) | (ftype & 0xff) + return struct.pack ('>LBl', lentype, flags, stream_id) + +class ERROR: + NO_ERROR = 0x0 + PROTOCOL_ERROR = 0x1 + INTERNAL_ERROR = 0x2 + FLOW_CONTROL_ERROR = 0x3 + SETTINGS_TIMEOUT = 0x4 + STREAM_CLOSED = 0x5 + FRAME_SIZE_ERROR = 0x6 + REFUSED_STREAM = 0x7 + CANCEL = 0x8 + COMPRESSION_ERROR = 0x9 + CONNECT_ERROR = 0xa + ENHANCE_YOUR_CALM = 0xb + INADEQUATE_SECURITY = 0xc + HTTP_1_1_REQUIRED = 0xd + +class FRAME: + DATA = 0 + HEADERS = 1 + PRIORITY = 2 + RST_STREAM = 3 + SETTINGS = 4 + PUSH_PROMISE = 5 + PING = 6 + GOAWAY = 7 + WINDOW_UPDATE = 8 + CONTINUATION = 9 + types = { + 0: 'data', + 1: 'headers', + 2: 'priority', + 3: 'rst_stream', + 4: 'settings', + 5: 'push_promise', + 6: 'ping', + 7: 'goaway', + 8: 'window_update', + 9: 'continuation', + } + +class SETTINGS: + HEADER_TABLE_SIZE = 0x01 + ENABLE_PUSH = 0x02 + MAX_CONCURRENT_STREAMS = 0x03 + INITIAL_WINDOW_SIZE = 0x04 + MAX_FRAME_SIZE = 0x05 + MAX_HEADER_LIST_SIZE = 0x06 + +class FLAGS: + END_STREAM = 0x01 + END_HEADERS = 0x04 + PADDED = 0x08 + PRIORITY = 0x20 + PING_ACK = 0x01 + SETTINGS_ACK = 0x01 + # XXX address MAX_FRAME_SIZE here. class h2_file (http_file): @@ -22,21 +92,19 @@ class h2_file (http_file): # override http_file's content generator (which is a 'pull' generator) # with this coro.fifo-based 'push' generator. - def __init__ (self, headers, stream): - self.streami = stream - self.streamo = read_stream.buffered_stream (self.get_content_gen().next) - self.done_cv = latch() - - def get_content_gen (self): + def __init__ (self, headers): self.content_fifo = coro.fifo() - return self._gen_h2() + self.streamo = read_stream.buffered_stream (self._gen_h2().next) + + def push (self, data): + ##DEBUG ('h2_file', 'push', data) + self.content_fifo.push (data) def _gen_h2 (self): while 1: block = self.content_fifo.pop() if block is None: - # LOG ('gen_h2: end of content') - self.done_cv.wake_all() + ##DEBUG ('gen_h2: end of content') break else: yield block @@ -65,23 +133,20 @@ def __init__ (self, flags, stream_id, client, headers): if self.has_body(): self.make_content_file() - def can_deflate (self): - return True - def has_body (self): #LOG ('has_body', not (self.flags & FLAGS.END_STREAM)) return not (self.flags & FLAGS.END_STREAM) def make_content_file (self): # XXX probably untested... - self.file = h2_file (self.request_headers, self.client.stream) + self.file = h2_file (self.request_headers) def push_headers (self, has_data=False): reason = self.responses[self.reply_code] - #self.reply_headers[':status'] = '%d %s' % (self.reply_code, reason) self.reply_headers[':status'] = '%d' % (self.reply_code,) - #self.reply_headers[':version'] = 'HTTP/1.1' - self.client.push_headers (self, has_data) + self.reply_headers['server'] = 'shrapnel h2/%s' % __version__ + self.reply_headers['date'] = build_http_date (coro.now_usec / coro.microseconds) + self.output.sent += self.client.push_headers (self, self.reply_headers, self.stream_id, has_data) self.sent_headers = True def push_data (self, data, last=False): @@ -104,78 +169,13 @@ def push (self, data, flush=False): def done (self): if not self.sent_headers: - self.push_syn_reply (has_data=False) + self.push_headers (has_data=False) else: if self.deflate: self.push_data (self.deflate.flush()) self.push_data (None, last=True) http_request.done (self) -def unpack_frame_header (head): - lentype, flags, stream_id = struct.unpack ('>LBl', head) - assert stream_id >= 0 - return lentype >> 8, lentype & 0xff, flags, stream_id - -def pack_frame_header (length, ftype, flags, stream_id): - lentype = (length << 8) | (ftype & 0xff) - return struct.pack ('>LBl', lentype, flags, stream_id) - -class ERROR: - NO_ERROR = 0x0 - PROTOCOL_ERROR = 0x1 - INTERNAL_ERROR = 0x2 - FLOW_CONTROL_ERROR = 0x3 - SETTINGS_TIMEOUT = 0x4 - STREAM_CLOSED = 0x5 - FRAME_SIZE_ERROR = 0x6 - REFUSED_STREAM = 0x7 - CANCEL = 0x8 - COMPRESSION_ERROR = 0x9 - CONNECT_ERROR = 0xa - ENHANCE_YOUR_CALM = 0xb - INADEQUATE_SECURITY = 0xc - HTTP_1_1_REQUIRED = 0xd - -class FRAME: - DATA = 0 - HEADERS = 1 - PRIORITY = 2 - RST_STREAM = 3 - SETTINGS = 4 - PUSH_PROMISE = 5 - PING = 6 - GOAWAY = 7 - WINDOW_UPDATE = 8 - CONTINUATION = 9 - types = { - 0: 'data', - 1: 'headers', - 2: 'priority', - 3: 'rst_stream', - 4: 'settings', - 5: 'push_promise', - 6: 'ping', - 7: 'goaway', - 8: 'window_update', - 9: 'continuation', - } - -class SETTINGS: - HEADER_TABLE_SIZE = 0x01 - ENABLE_PUSH = 0x02 - MAX_CONCURRENT_STREAMS = 0x03 - INITIAL_WINDOW_SIZE = 0x04 - MAX_FRAME_SIZE = 0x05 - MAX_HEADER_LIST_SIZE = 0x06 - -class FLAGS: - END_STREAM = 0x01 - END_HEADERS = 0x04 - PADDED = 0x08 - PRIORITY = 0x20 - PING_ACK = 0x01 - SETTINGS_ACK = 0x01 - # this is a mixin class used for both server and client. class h2_protocol: @@ -183,6 +183,19 @@ class h2_protocol: protocol = 'h2' preface = 'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' is_server = True + protocol = 'h2' + # default to 400K buffered output + output_buffer_size = 400 * 1024 + h2_settings = None + last_ping = None + + def __init__ (self): + self.streams = {} + self.priorities = {} + self.encoder = Encoder() + self.decoder = Decoder() + self.ofifo = coro.fifo() + self.obuf = coro.semaphore (self.output_buffer_size) # for those socket types not implementing read_exact. def read_exact (self, size): @@ -221,50 +234,15 @@ def read_frames (self): payload = b'' method_name = 'frame_%s' % (FRAME.types.get (ftype, '')) if method_name == 'frame_': - self.log ('unknown h2 frame type: %d' % (ftype,)) + LOG ('unknown h2 frame type: %d' % (ftype,)) else: - #LOG ('frame', method_name, flags, stream_id, payload) + ##DEBUG ('frame', method_name, flags, stream_id, payload) method = getattr (self, method_name) method (flags, stream_id, payload) except OSError: LOG ('OSError') self.close() -# -------------------------------------------------------------------------------- -# h2 server -# -------------------------------------------------------------------------------- - -# XXX not a fan of multiple inheritance, but this seems to be the cleanest way to share -# XXX the code between server and client... - -class h2_connection (h2_protocol, connection): - - protocol = 'h2' - # default to 400K buffered output - output_buffer_size = 400 * 1024 - h2_settings = None - last_ping = None - - def run (self): - self.streams = {} - self.priorities = {} - self.encoder = Encoder() - self.decoder = Decoder() - self.ofifo = coro.fifo() - self.obuf = coro.semaphore (self.output_buffer_size) - coro.spawn (self.send_thread) - self.push_settings() - try: - self.read_frames() - except coro.oserrors.ECONNRESET: - LOG ('connection reset') - finally: - self.ofifo.push (None) - - def close (self): - self.ofifo.push (None) - self.conn.close() - def send_thread (self): try: done = False @@ -275,7 +253,7 @@ def send_thread (self): blocks = [x for x in blocks if x is not None] if blocks: total_size = sum ([len(x) for x in blocks]) - #LOG ('send', total_size) + ##DEBUG ('send', total_size) self.conn.writev (blocks) self.obuf.release (total_size) except OSError: @@ -291,38 +269,38 @@ def push_frame (self, frame): def send_frame (self, ftype, flags, stream_id, data): dlen = len(data) head = pack_frame_header (dlen, ftype, flags, stream_id) - #LOG ('send_frame', FRAME.types[ftype], flags, stream_id) + ##DEBUG ('send_frame', FRAME.types[ftype], flags, stream_id) return self.push_frame (head + data) - def push_headers (self, req, has_data): - #LOG ('push_headers', req, not not has_data) + def push_headers (self, req, headers, stream_id, has_data): + ##DEBUG ('push_headers', req, not not has_data) flags = FLAGS.END_HEADERS if not has_data: flags |= FLAGS.END_STREAM - hdata = self.pack_http_header (req.reply_headers) - #LOG ('push_headers', req.stream_id) - req.output.sent += self.send_frame (0x01, flags, req.stream_id, hdata) + hdata = self.pack_http_header (headers) + ##DEBUG ('push_headers', stream_id, headers.headers) + return self.send_frame (FRAME.HEADERS, flags, stream_id, hdata) def push_ping (self, flags=0, data=None): if data is None: data = os.urandom (8) assert len(data) == 8 self.last_ping = data - self.send_frame (0x06, flags, 0, data) + self.send_frame (FRAME.PING, flags, 0, data) def push_data (self, req, data, last): if last: flags = FLAGS.END_STREAM else: flags = 0 - #LOG ('push_data', len(data)) - req.output.sent += self.send_frame (0x00, flags, req.stream_id, data) + ##DEBUG ('push_data', len(data)) + req.output.sent += self.send_frame (FRAME.DATA, flags, req.stream_id, data) def frame_settings (self, flags, stream_id, payload): plen = len(payload) n, check = divmod (plen, 6) if flags & FLAGS.SETTINGS_ACK: - #LOG ('settings', 'ack') + ##DEBUG ('settings', 'ack') pass else: assert check == 0 @@ -331,7 +309,7 @@ def frame_settings (self, flags, stream_id, payload): for i in range (0, plen, 6): ident, value = struct.unpack ('>HL', payload[i:i+6]) self.h2_settings[ident] = value - #LOG ('settings', self.h2_settings) + ##DEBUG ('settings', self.h2_settings) # ack it. self.send_frame (FRAME.SETTINGS, FLAGS.SETTINGS_ACK, 0, '') @@ -347,17 +325,17 @@ def push_settings (self): def frame_ping (self, flags, stream_id, payload): assert len(payload) == 8 - #LOG ('ping', flags, stream_id, payload) + ##DEBUG ('ping', flags, stream_id, payload) if flags & FLAGS.PING_ACK: assert payload == self.last_ping else: assert len(payload) == 8 - self.send_frame (0x06, FLAGS.PING_ACK, 0, payload) + self.send_frame (FRAME.PING, FLAGS.PING_ACK, 0, payload) def frame_window_update (self, flags, stream_id, payload): increment, = struct.unpack ('>l', payload) assert increment >= 0 - #LOG ('window_update', increment) + ##DEBUG ('window_update', increment) def frame_headers (self, flags, stream_id, payload): pos = 0 @@ -368,28 +346,22 @@ def frame_headers (self, flags, stream_id, payload): if flags & FLAGS.PADDED: pad_len, = struct.unpack ('>B', payload[pos:pos+1]) pos += 1 - #LOG ('headers', 'padded', pad_len) if flags & FLAGS.PRIORITY: stream_dep, weight = struct.unpack ('>lB', payload[:5]) pos += 5 - #LOG ('headers', 'priority', stream_dep, weight) if flags & FLAGS.END_STREAM: - #LOG ('headers', 'end_stream') pass if flags & FLAGS.END_HEADERS: - #LOG ('headers', 'end_headers') pass else: raise NotImplementedError - #LOG ('headers', flags, stream_id, pad_len, stream_dep, weight) + ##DEBUG ('headers', flags, stream_id, pad_len, stream_dep, weight) if pad_len: header_block = payload[pos:-pad_len] else: header_block = payload[pos:] headers = self.unpack_http_header (header_block) - req = h2_server_request (flags, stream_id, self, headers) - self.streams[stream_id] = req - coro.spawn (self.handle_request, req) + self.handle_headers (flags, stream_id, headers) def unpack_http_header (self, header_block): self.decoder.feed (header_block) @@ -404,10 +376,10 @@ def pack_http_header (self, hset): def send_goaway (self, last_stream_id, error_code, debug_data): payload = struct.pack ('>lL', last_stream_id, error_code) + debug_data - self.send_frame (0x07, 0x00, 0x00, payload) + self.send_frame (FRAME.GOAWAY, 0x00, 0x00, payload) def frame_rst_stream (self, flags, stream_id, payload): - #LOG ('frame_rst_stream', stream_id) + ##DEBUG ('frame_rst_stream', stream_id) try: del self.streams[stream_id] except KeyError: @@ -419,7 +391,7 @@ def frame_rst_stream (self, flags, stream_id, payload): def frame_priority (self, flags, stream_id, payload): stream_dep, weight = struct.unpack (' Date: Thu, 8 Oct 2015 14:51:05 -0700 Subject: [PATCH 46/46] h2_protocol: raise NotImplementedError on PUSH_PROMISE or CONTINUATION. --- coro/http/h2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coro/http/h2.py b/coro/http/h2.py index 2f27b87..de21056 100644 --- a/coro/http/h2.py +++ b/coro/http/h2.py @@ -415,9 +415,9 @@ def frame_data (self, flags, stream_id, payload): LOG ('orphaned data frame [%d bytes] for stream %d\n' % (len(payload), stream_id)) def frame_push_promise (self, flags, stream_id, payload): - import pdb; pdb.set_trace() + raise NotImplementedError def frame_continuation (self, flags, stream_id, payload): - import pdb; pdb.set_trace() + raise NotImplementedError # -------------------------------------------------------------------------------- # h2 server