Merge pull request #452 from h2o/kazuho/pacer

pacing

CMakeLists.txt

@@ -72,6 +72,7 @@ SET(UNITTEST_SOURCE_FILES
     t/loss.c
     t/lossy.c
     t/maxsender.c
+    t/pacer.c
     t/ranges.c
     t/rate.c
     t/remote_cid.c
@@ -127,7 +128,7 @@ ADD_EXECUTABLE(udpfw t/udpfw.c)
 
 ADD_CUSTOM_TARGET(check env BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} WITH_DTRACE=${WITH_DTRACE} prove --exec "sh -c" -v ${CMAKE_CURRENT_BINARY_DIR}/*.t t/*.t
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    DEPENDS cli test.t)
+    DEPENDS cli udpfw test.t)
 
 ADD_CUSTOM_TARGET(format clang-format -i `git ls-files include lib src t | egrep '\\.[ch]$$'`)
 

include/quicly.h

@@ -334,6 +334,14 @@ struct st_quicly_context_t {
      * whether to use ECN on the send side; ECN is always on on the receive side
      */
     unsigned enable_ecn : 1;
+    /**
+     * if pacing should be used
+     */
+    unsigned use_pacing : 1;
+    /**
+     * if CC should take app-limited into consideration
+     */
+    unsigned respect_app_limited : 1;
     /**
      *
      */

include/quicly/cc.h

@@ -156,7 +156,7 @@ struct st_quicly_cc_type_t {
      * Called when a packet is newly acknowledged.
      */
     void (*cc_on_acked)(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t largest_acked, uint32_t inflight,
-                        uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size);
+                        int cc_limited, uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size);
     /**
      * Called when a packet is detected as lost.
      * @param bytes    bytes declared lost, or zero iff ECN_CE is observed

include/quicly/pacer.h

@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2021 Fastly, Kazuho Oku
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef quicly_pacer_h
+#define quicly_pacer_h
+
+#include <assert.h>
+#include <stddef.h>
+#include <inttypes.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Simple pacer. The design guarantees that the formula below is met for any given pacer-restricted period:
+ *
+ *   flow_rate * duration + 8 * mtu <= bytes_sent < flow_rate * duration + 10 * mtu
+ */
+typedef struct st_quicly_pacer_t {
+    /**
+     * clock
+     */
+    int64_t at;
+    /**
+     * amount of credit being spent at `at`
+     */
+    size_t bytes_sent;
+} quicly_pacer_t;
+
+#define QUICLY_PACER_BURST_LOW 8   /* lower bound in packets */
+#define QUICLY_PACER_BURST_HIGH 10 /* high bound in packets */
+
+/**
+ * resets the pacer
+ */
+static void quicly_pacer_reset(quicly_pacer_t *pacer);
+/**
+ * returns when the next chunk of data can be sent
+ */
+static int64_t quicly_pacer_can_send_at(quicly_pacer_t *pacer, uint32_t bytes_per_msec, uint16_t mtu);
+/**
+ * returns the number of bytes that can be sent at this moment
+ */
+static uint64_t quicly_pacer_get_window(quicly_pacer_t *pacer, int64_t now, uint32_t bytes_per_msec, uint16_t mtu);
+/**
+ * updates the window size available at current time
+ */
+static void quicly_pacer_consume_window(quicly_pacer_t *pacer, size_t delta);
+/**
+ * Calculates the flow rate as `bytes_per_msec`. The returned value is no less than 1.
+ */
+static uint32_t quicly_pacer_calc_send_rate(uint32_t multiplier, uint32_t cwnd, uint32_t rtt);
+
+/* inline definitions */
+
+inline void quicly_pacer_reset(quicly_pacer_t *pacer)
+{
+    pacer->at = INT64_MIN;
+    pacer->bytes_sent = 0;
+}
+
+inline int64_t quicly_pacer_can_send_at(quicly_pacer_t *pacer, uint32_t bytes_per_msec, uint16_t mtu)
+{
+    /* return "now" if we have room in current msec */
+    size_t burst_size = QUICLY_PACER_BURST_LOW * mtu + 1;
+    size_t burst_credit = burst_size > bytes_per_msec ? burst_size - bytes_per_msec : 0;
+    if (pacer->bytes_sent < bytes_per_msec + burst_credit)
+        return 0;
+
+    /* calculate delay; the value is rounded down, as it is better for a pacer to be a bit aggressive than not */
+    int64_t delay = (pacer->bytes_sent - burst_credit) / bytes_per_msec;
+    assert(delay > 0);
+    return pacer->at + delay;
+}
+
+inline uint64_t quicly_pacer_get_window(quicly_pacer_t *pacer, int64_t now, uint32_t bytes_per_msec, uint16_t mtu)
+{
+    assert(pacer->at <= now);
+
+    /* Determine when it is possible to sent one packet. Return if that is a moment in future. */
+    int64_t can_send_at = quicly_pacer_can_send_at(pacer, bytes_per_msec, mtu);
+    if (now < can_send_at)
+        return 0;
+
+    /* Calculate the upper bound of burst window (the size is later rounded up) */
+    size_t burst_window = (QUICLY_PACER_BURST_HIGH - 1) * mtu + 1;
+    if (burst_window < bytes_per_msec)
+        burst_window = bytes_per_msec;
+
+    /* Additional amount of data that we can send in `now - restricted_at` milliseconds is that difference multiplied by
+     * `bytes_per_msec`. Adjust `bytes_sent` by that amount before setting `restricted_at` to `now`. `uint64_t` is used to store
+     * window and delta so that the multiplication would not overflow assuming that the quiescence period is shorter than 2**32
+     * milliseconds. */
+    uint64_t window, delta = (now - pacer->at) * bytes_per_msec;
+    if (pacer->bytes_sent > delta) {
+        pacer->bytes_sent -= delta;
+        if (burst_window > pacer->bytes_sent) {
+            window = (burst_window - pacer->bytes_sent + mtu - 1) / mtu;
+            if (window < 2)
+                window = 2;
+        } else {
+            window = 2;
+        }
+    } else {
+        pacer->bytes_sent = 0;
+        window = (burst_window + mtu - 1) / mtu;
+    }
+    window *= mtu;
+
+    pacer->at = now;
+
+    return window;
+}
+
+inline void quicly_pacer_consume_window(quicly_pacer_t *pacer, size_t delta)
+{
+    pacer->bytes_sent += delta;
+}
+
+inline uint32_t quicly_pacer_calc_send_rate(uint32_t multiplier, uint32_t cwnd, uint32_t rtt)
+{
+    return (cwnd * multiplier + rtt - 1) / rtt;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/quicly/rate.h

@@ -91,13 +91,17 @@ typedef struct st_quicly_rate_t {
  */
 void quicly_ratemeter_init(quicly_ratemeter_t *meter);
 /**
- * Notifies the estimator that the flow is CWND-limited at the point of sending packets *starting* from packet number `pn`.
+ * returns if currently marked as CC-limited
  */
-void quicly_ratemeter_in_cwnd_limited(quicly_ratemeter_t *meter, uint64_t pn);
+static int quicly_ratemeter_is_cc_limited(quicly_ratemeter_t *meter);
 /**
- * Notifies that the estimator that the flow is not CWND-limited when the packet number of the next packet will be `pn`.
+ * Notifies the estimator that the flow is becoming CC-limited at the point of sending packets *starting* from packet number `pn`.
  */
-void quicly_ratemeter_not_cwnd_limited(quicly_ratemeter_t *meter, uint64_t pn);
+void quicly_ratemeter_enter_cc_limited(quicly_ratemeter_t *meter, uint64_t pn);
+/**
+ * Notifies that the estimator that the flow is exiting from CC-limited when the packet number of the next packet will be `pn`.
+ */
+void quicly_ratemeter_exit_cc_limited(quicly_ratemeter_t *meter, uint64_t pn);
 /**
  * Given three values, update the estimation.
  * @param bytes_acked  total number of bytes being acked from the beginning of the connection; i.e.,
@@ -109,6 +113,13 @@ void quicly_ratemeter_on_ack(quicly_ratemeter_t *meter, int64_t now, uint64_t by
  */
 void quicly_ratemeter_report(quicly_ratemeter_t *meter, quicly_rate_t *rate);
 
+/* inline definitions */
+
+inline int quicly_ratemeter_is_cc_limited(quicly_ratemeter_t *meter)
+{
+    return meter->pn_cwnd_limited.start != UINT64_MAX && meter->pn_cwnd_limited.end == UINT64_MAX;
+}
+
 #ifdef __cplusplus
 }
 #endif

lib/cc-cubic.c

@@ -23,6 +23,7 @@
 #include <math.h>
 #include "quicly/cc.h"
 #include "quicly.h"
+#include "quicly/pacer.h"
 
 #define QUICLY_MIN_CWND 2
 
@@ -61,13 +62,15 @@ static uint32_t calc_w_est(const quicly_cc_t *cc, cubic_float_t t_sec, cubic_flo
 
 /* TODO: Avoid increase if sender was application limited. */
 static void cubic_on_acked(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t largest_acked, uint32_t inflight,
-                           uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
+                           int cc_limited, uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
 {
     assert(inflight >= bytes);
     /* Do not increase congestion window while in recovery. */
     if (largest_acked < cc->recovery_end)
         return;
 
+    /* TODO: respect cc_limited */
+
     /* Slow start. */
     if (cc->cwnd < cc->ssthresh) {
         cc->cwnd += bytes;

lib/cc-pico.c

@@ -20,6 +20,7 @@
  * IN THE SOFTWARE.
  */
 #include <math.h>
+#include "quicly/pacer.h"
 #include "quicly/cc.h"
 #include "quicly.h"
 
@@ -61,14 +62,17 @@ static uint32_t calc_bytes_per_mtu_increase(uint32_t cwnd, uint32_t rtt, uint32_
 
 /* TODO: Avoid increase if sender was application limited. */
 static void pico_on_acked(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t largest_acked, uint32_t inflight,
-                          uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
+                          int cc_limited, uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
 {
     assert(inflight >= bytes);
 
     /* Do not increase congestion window while in recovery. */
     if (largest_acked < cc->recovery_end)
         return;
 
+    if (!cc_limited)
+        return;
+
     cc->state.pico.stash += bytes;
 
     /* Calculate the amount of bytes required to be acked for incrementing CWND by one MTU. */

lib/cc-reno.c

@@ -21,10 +21,11 @@
  */
 #include "quicly/cc.h"
 #include "quicly.h"
+#include "quicly/pacer.h"
 
 /* TODO: Avoid increase if sender was application limited. */
 static void reno_on_acked(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t largest_acked, uint32_t inflight,
-                          uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
+                          int cc_limited, uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size)
 {
     assert(inflight >= bytes);
     /* Do not increase congestion window while in recovery. */
@@ -33,12 +34,16 @@ static void reno_on_acked(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t b
 
     /* Slow start. */
     if (cc->cwnd < cc->ssthresh) {
-        cc->cwnd += bytes;
-        if (cc->cwnd_maximum < cc->cwnd)
-            cc->cwnd_maximum = cc->cwnd;
+        if (cc_limited) {
+            cc->cwnd += bytes;
+            if (cc->cwnd_maximum < cc->cwnd)
+                cc->cwnd_maximum = cc->cwnd;
+        }
         return;
     }
     /* Congestion avoidance. */
+    if (!cc_limited)
+        return;
     cc->state.reno.stash += bytes;
     if (cc->state.reno.stash < cc->cwnd)
         return;

lib/defaults.c

@@ -51,6 +51,8 @@ const quicly_context_t quicly_spec_context = {NULL,
                                               DEFAULT_MAX_INITIAL_HANDSHAKE_PACKETS,
                                               0, /* enlarge_client_hello */
                                               1, /* enable_ecn */
+                                              0, /* use_pacing */
+                                              1, /* cc_recognize_app_limited */
                                               NULL,
                                               NULL, /* on_stream_open */
                                               &quicly_default_stream_scheduler,
@@ -82,6 +84,8 @@ const quicly_context_t quicly_performant_context = {NULL,
                                                     DEFAULT_MAX_INITIAL_HANDSHAKE_PACKETS,
                                                     0, /* enlarge_client_hello */
                                                     1, /* enable_ecn */
+                                                    0, /* use_pacing */
+                                                    1, /* cc_recognize_app_limited */
                                                     NULL,
                                                     NULL, /* on_stream_open */
                                                     &quicly_default_stream_scheduler,