update version of upstream envoy (#13)

* update version of upstream envoy
* udpate build container
* add envoy_api_dependencies
* update do_ci with upstream envoy changes
* udpate coverage build
* envoy_gloo_cc_test for visibility
* c_str -> getStringView
* compile fix
* sendlocalreply
* fix test
* use TestUtility::loadFromYaml
* compile fix
* compile fix
* compile fix
* compiler fix
* fix query string

Author: yuval-k

WORKSPACE

@@ -4,6 +4,9 @@ load("//bazel:repositories.bzl", "envoy_gloo_dependencies")
 
 envoy_gloo_dependencies()
 
+load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies")
+envoy_api_dependencies()
+
 load("@envoy//bazel:repositories.bzl", "GO_VERSION", "envoy_dependencies")
 load("@envoy//bazel:cc_configure.bzl", "cc_configure")
 

bazel/envoy_test.bzl

@@ -0,0 +1,83 @@
+
+# Copied from @envoy/bazel/envoy_test.bzl and uses public visibility for tests
+# So we can target them to use our coverage.
+
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_cc_test_library",
+)
+
+load(
+    "@envoy//bazel:envoy_internal.bzl",
+    "envoy_copts",
+    "envoy_select_force_libcpp",
+    "envoy_linkstatic",
+    "tcmalloc_external_dep",
+)
+
+# Compute the test linkopts based on various options.
+def _envoy_test_linkopts():
+    return select({
+        "@envoy//bazel:apple": [
+            # See note here: https://luajit.org/install.html
+            "-pagezero_size 10000",
+            "-image_base 100000000",
+        ],
+        "@envoy//bazel:windows_x86_64": [
+            "-DEFAULTLIB:advapi32.lib",
+            "-DEFAULTLIB:ws2_32.lib",
+            "-WX",
+        ],
+
+        # TODO(mattklein123): It's not great that we universally link against the following libs.
+        # In particular, -latomic and -lrt are not needed on all platforms. Make this more granular.
+        "//conditions:default": ["-pthread", "-lrt", "-ldl"],
+    }) + envoy_select_force_libcpp(["-lc++fs"], ["-lstdc++fs", "-latomic"])
+
+# Envoy C++ test targets should be specified with this function.
+def envoy_gloo_cc_test(
+        name,
+        srcs = [],
+        data = [],
+        # List of pairs (Bazel shell script target, shell script args)
+        repository = "",
+        external_deps = [],
+        deps = [],
+        tags = [],
+        args = [],
+        copts = [],
+        shard_count = None,
+        coverage = True,
+        local = False,
+        size = "medium"):
+    test_lib_tags = []
+    if coverage:
+        test_lib_tags.append("coverage_test_lib")
+    envoy_cc_test_library(
+        name = name + "_lib_internal_only",
+        srcs = srcs,
+        data = data,
+        external_deps = external_deps,
+        deps = deps,
+        repository = repository,
+        tags = test_lib_tags,
+        copts = copts,
+    )
+    native.cc_test(
+        name = name,
+        copts = envoy_copts(repository, test = True) + copts,
+        linkopts = _envoy_test_linkopts(),
+        linkstatic = envoy_linkstatic(),
+        malloc = tcmalloc_external_dep(repository),
+        deps = [
+            ":" + name + "_lib_internal_only",
+            repository + "//test:main",
+        ],
+        # from https://github.com/google/googletest/blob/6e1970e2376c14bf658eb88f655a054030353f9f/googlemock/src/gmock.cc#L51
+        # 2 - by default, mocks act as StrictMocks.
+        args = args + ["--gmock_default_mock_behavior=2"],
+        tags = tags + ["coverage_test"],
+        local = local,
+        shard_count = shard_count,
+        size = size,
+    )

bazel/repository_locations.bzl

@@ -1,6 +1,6 @@
 REPOSITORY_LOCATIONS = dict(
     envoy = dict(
-        commit = "7ed6d2187df94c4cb96f7dccb8643bf764af2ccb",
+        commit = "8d1ad35aa724962f64f7535531e408c9a93d364c",
         remote = "https://github.com/envoyproxy/envoy",
     ),
     inja = dict(

ci/do_ci.sh

@@ -48,10 +48,10 @@ function setup_gcc_toolchain() {
 }
 
 function setup_clang_toolchain() {
-  export PATH=/usr/lib/llvm-7/bin:$PATH
+  export PATH=/usr/lib/llvm-8/bin:$PATH
   export CC=clang
   export CXX=clang++
-  export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-7/bin/llvm-symbolizer
+  export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer
   echo "$CC/$CXX toolchain configured"
 }
 
@@ -65,20 +65,6 @@ function cleanup() {
 cleanup
 trap cleanup EXIT
 
-
-# link prebuilt stuff
-
-ln -sf /thirdparty "${ENVOY_CIDIR}"/prebuilt
-ln -sf /thirdparty_build "${ENVOY_CIDIR}"/prebuilt
-
-# Replace the existing Bazel output cache with a copy of the image's prebuilt deps.
-if [[ -d /bazel-prebuilt-output && ! -d "${TEST_TMPDIR}/_bazel_${USER}" ]]; then
-  BAZEL_OUTPUT_BASE="$(bazel info output_base)"
-  mkdir -p "${TEST_TMPDIR}/_bazel_${USER}/install"
-  rsync -a /bazel-prebuilt-root/install/* "${TEST_TMPDIR}/_bazel_${USER}/install/"
-  rsync -a /bazel-prebuilt-output "${BAZEL_OUTPUT_BASE}"
-fi
-
 set -e
 
 # try compiling with clang as google cloud doesn't seem to like gcc.

cloudbuild.yaml

@@ -1,5 +1,5 @@
 steps:
-- name: 'envoyproxy/envoy-build:698009170e362f9ca0594f2b1927fbbee199bf98'
+- name: 'envoyproxy/envoy-build:cfc514546bc0284536893cca5fa43d7128edcd35'
   args: ['ci/do_ci.sh', 'test']
   volumes:
   - name: 'vol-build'
@@ -8,7 +8,7 @@ steps:
   - 'COMMIT_SHA=$COMMIT_SHA'
   timeout: 900s
 
-- name: 'envoyproxy/envoy-build:698009170e362f9ca0594f2b1927fbbee199bf98'
+- name: 'envoyproxy/envoy-build:cfc514546bc0284536893cca5fa43d7128edcd35'
   args: ['ci/do_ci.sh', 'build']
   volumes:
   - name: 'vol-build'

source/extensions/filters/http/aws_lambda/aws_authenticator.cc

@@ -91,7 +91,7 @@ AwsAuthenticator::prepareHeaders(const HeaderList &headers_to_sign) {
 
     canonical_headers_stream << ':';
     if (headerEntry != nullptr) {
-      canonical_headers_stream << headerEntry->value().c_str();
+      canonical_headers_stream << headerEntry->value().getStringView();
       // TODO: add warning if null
     }
     canonical_headers_stream << '\n';
@@ -119,25 +119,13 @@ std::string AwsAuthenticator::getBodyHexSha() {
 
 void AwsAuthenticator::fetchUrl() {
   const Http::HeaderString &canonical_url = request_headers_->Path()->value();
-  url_len_ = canonical_url.size();
-  url_start_ = canonical_url.c_str();
-  const char *query_string_start =
+  url_base_ = canonical_url.getStringView();
+  query_string_ =
       Http::Utility::findQueryStringStart(canonical_url);
-  // bug in: findQueryStringStart - query_string_start will never be null as
-  // implied in config_impl.cc, but rather it is the end iterator of the string.
-  if (query_string_start != nullptr) {
-    url_len_ = query_string_start - url_start_;
-    if (url_len_ < canonical_url.size()) {
-      // we now know that query_string_start != std::end
-
-      // +1 to skip the question mark
-      // These should be sorted alphabetically, but I will leave that to the
-      // caller (which is internal, hence it's ok)
-      query_string_len_ = canonical_url.size() - url_len_ - 1;
-      query_string_start_ = query_string_start + 1;
-    } else {
-      query_string_start = nullptr;
-    }
+  if (query_string_.length() != 0) {
+    url_base_.remove_suffix(query_string_.length());
+    // remove the question mark
+    query_string_.remove_prefix(1);
   }
 }
 
@@ -150,10 +138,10 @@ std::string AwsAuthenticator::computeCanonicalRequestHash(
 
   canonicalRequestHash.update(request_method);
   canonicalRequestHash.update('\n');
-  canonicalRequestHash.update(url_start_, url_len_);
+  canonicalRequestHash.update(url_base_);
   canonicalRequestHash.update('\n');
-  if (query_string_start_ != nullptr) {
-    canonicalRequestHash.update(query_string_start_, query_string_len_);
+  if (query_string_.length() != 0) {
+    canonicalRequestHash.update(query_string_);
   }
   canonicalRequestHash.update('\n');
   canonicalRequestHash.update(canonical_headers);
@@ -279,6 +267,10 @@ void AwsAuthenticator::Sha256::update(const std::string &data) {
   update(data.c_str(), data.size());
 }
 
+void AwsAuthenticator::Sha256::update(const absl::string_view& data) {
+  update(data.data(), data.size());
+}
+
 void AwsAuthenticator::Sha256::update(const uint8_t *bytes, size_t size) {
   SHA256_Update(&context_, bytes, size);
 }

source/extensions/filters/http/aws_lambda/aws_authenticator.h

@@ -77,6 +77,8 @@ class AwsAuthenticator {
     Sha256();
     void update(const Buffer::Instance &data);
     void update(const std::string &data);
+    void update(const absl::string_view& data);
+
     void update(char c);
     void update(const uint8_t *bytes, size_t size);
     void update(const char *chars, size_t size);
@@ -119,10 +121,8 @@ class AwsAuthenticator {
   std::string first_key_;
   const std::string *service_{};
   const std::string *method_{};
-  const char *query_string_start_{};
-  size_t query_string_len_{};
-  const char *url_start_{};
-  size_t url_len_{};
+  absl::string_view query_string_{};
+  absl::string_view url_base_{};
 
   Http::HeaderMap *request_headers_{};
 };

source/extensions/filters/http/aws_lambda/aws_lambda_filter.cc

@@ -23,6 +23,12 @@ namespace Extensions {
 namespace HttpFilters {
 namespace AwsLambda {
 
+struct RcDetailsValues {
+  // The jwt_authn filter rejected the request
+  const std::string FunctionNotFound = "aws_lambda_function_not_fou8nd";
+};
+typedef ConstSingleton<RcDetailsValues> RcDetails;
+
 class AWSLambdaHeaderValues {
 public:
   const Http::LowerCaseString InvocationType{"x-amz-invocation-type"};
@@ -78,7 +84,7 @@ AWSLambdaFilter::decodeHeaders(Http::HeaderMap &headers, bool end_stream) {
   if (!function_on_route_) {
     decoder_callbacks_->sendLocalReply(Http::Code::NotFound,
                                        "no function present for AWS upstream",
-                                       nullptr, absl::nullopt);
+                                       nullptr, absl::nullopt, RcDetails::get().FunctionNotFound);
     return Http::FilterHeadersStatus::StopIteration;
   }
 

source/extensions/filters/http/nats/streaming/nats_streaming_filter.cc

@@ -21,6 +21,13 @@ namespace HttpFilters {
 namespace Nats {
 namespace Streaming {
 
+struct RcDetailsValues {
+  // The jwt_authn filter rejected the request
+  const std::string PayloadTooLarge = "nats_payload_too_big";
+  const std::string Completion = "nats_completion";
+};
+typedef ConstSingleton<RcDetailsValues> RcDetails;
+
 NatsStreamingFilter::NatsStreamingFilter(
     NatsStreamingFilterConfigSharedPtr config,
     Envoy::Nats::Streaming::ClientPtr nats_streaming_client)
@@ -52,10 +59,10 @@ NatsStreamingFilter::decodeHeaders(Envoy::Http::HeaderMap &headers,
   auto mutable_headers = payload_.mutable_headers();
   headers.iterate(
       [](const Envoy::Http::HeaderEntry &e, void *ctx) {
-        Envoy::Protobuf::Map<Envoy::ProtobufTypes::String,
-                             Envoy::ProtobufTypes::String> *mutable_headers =
-            static_cast<Envoy::Protobuf::Map<Envoy::ProtobufTypes::String,
-                                             Envoy::ProtobufTypes::String> *>(
+        Envoy::Protobuf::Map<std::string,
+                             std::string> *mutable_headers =
+            static_cast<Envoy::Protobuf::Map<std::string,
+                                             std::string> *>(
                 ctx);
         (*mutable_headers)[std::string(e.key().getStringView())] =
             std::string(e.value().getStringView());
@@ -84,7 +91,7 @@ NatsStreamingFilter::decodeData(Envoy::Buffer::Instance &data,
 
     decoder_callbacks_->sendLocalReply(Http::Code::PayloadTooLarge,
                                        "nats streaming paylaod too large",
-                                       nullptr, absl::nullopt);
+                                       nullptr, absl::nullopt, RcDetails::get().PayloadTooLarge);
     return Http::FilterDataStatus::StopIterationNoBuffer;
   }
 
@@ -171,7 +178,7 @@ void NatsStreamingFilter::onCompletion(Http::Code response_code,
   in_flight_request_ = nullptr;
 
   decoder_callbacks_->sendLocalReply(response_code, body_text, nullptr,
-                                     absl::nullopt);
+                                     absl::nullopt, RcDetails::get().Completion);
 }
 
 void NatsStreamingFilter::onCompletion(Http::Code response_code,

source/extensions/filters/http/transformation/body_header_transformer.cc

@@ -23,7 +23,7 @@ void BodyHeaderTransformer::transform(Http::HeaderMap &header_map,
          void *context) -> Http::HeaderMap::Iterate {
         json *headers_ptr = static_cast<json *>(context);
         json &headers = *headers_ptr;
-        headers[header.key().c_str()] = header.value().c_str();
+        headers[std::string(header.key().getStringView())] = std::string(header.value().getStringView());
         return Http::HeaderMap::Iterate::Continue;
       },
       &headers);