akka-http and play appsec support

.github/workflows/ci.yml

@@ -128,6 +128,8 @@ jobs:
           weblog: spring-boot-payara
         - library: java
           weblog: akka-http
+        - library: java
+          weblog: play
         - library: nodejs
           weblog: express4
         - library: nodejs

tests/appsec/test_blocking_addresses.py

@@ -65,6 +65,9 @@ def setup_path_params(self):
         context.library < "[email protected]", reason="When supported, path parameter detection happens on subsequent WAF run"
     )
     @missing_feature(library="nodejs", reason="Not supported yet")
+    @missing_feature(
+        context.library == "java" and context.weblog_variant == "akka-http", reason="path parameters not supported"
+    )
     @irrelevant(context.library == "ruby" and context.weblog_variant == "rack")
     @irrelevant(context.library == "golang" and context.weblog_variant == "net-http")
     def test_path_params(self):
@@ -142,7 +145,15 @@ def setup_response_status(self):
     @missing_feature(
         context.library == "java"
         and context.weblog_variant
-        not in ("spring-boot", "uds-spring-boot", "spring-boot-jetty", "spring-boot-undertow", "spring-boot-wildfly")
+        not in (
+            "akka-http",
+            "play",
+            "spring-boot",
+            "uds-spring-boot",
+            "spring-boot-jetty",
+            "spring-boot-undertow",
+            "spring-boot-wildfly",
+        )
     )
     @missing_feature(context.library == "golang", reason="No blocking on server.response.*")
     @missing_feature(context.library < "[email protected]")
@@ -156,7 +167,10 @@ def test_response_status(self):
     def setup_not_found(self):
         self.rnf_req = weblog.get(path="/finger_print")
 
-    @missing_feature(context.library == "java", reason="Happens on a subsequent WAF run")
+    @missing_feature(
+        context.library == "java" and context.weblog_variant not in ("akka-http", "play"),
+        reason="Happens on a subsequent WAF run",
+    )
     @missing_feature(context.library == "ruby", reason="Not working")
     @missing_feature(library="nodejs", reason="Not supported yet")
     @missing_feature(context.library == "golang", reason="No blocking on server.response.*")
@@ -170,7 +184,10 @@ def setup_response_header(self):
         self.rsh_req = weblog.get(path="/headers")
 
     @missing_feature(context.library < "[email protected]")
-    @missing_feature(context.library == "java", reason="Happens on a subsequent WAF run")
+    @missing_feature(
+        context.library == "java" and context.weblog_variant not in ("akka-http", "play"),
+        reason="Happens on a subsequent WAF run",
+    )
     @missing_feature(context.library == "ruby")
     @missing_feature(context.library == "php", reason="Headers already sent at this stage")
     @missing_feature(library="nodejs", reason="Not supported yet")
@@ -502,7 +519,7 @@ def setup_non_blocking_plain_text(self):
         )
 
     @irrelevant(
-        context.weblog_variant in ("jersey-grizzly2", "resteasy-netty3"),
+        context.weblog_variant in ("akka-http", "play", "jersey-grizzly2", "resteasy-netty3"),
         reason="Blocks on text/plain if parsed to a String",
     )
     def test_non_blocking_plain_text(self):

tests/appsec/waf/test_blocking.py

@@ -128,7 +128,7 @@ def setup_accept_partial_html(self):
     def test_accept_partial_html(self):
         """Blocking with Accept: text/*"""
         assert self.r_aph.status_code == 403
-        assert self.r_aph.headers.get("content-type", "") in HTML_CONTENT_TYPES
+        assert self.r_aph.headers.get("content-type", "").lower() in HTML_CONTENT_TYPES
         assert self.r_aph.text in BLOCK_TEMPLATE_HTML_ANY
 
     def setup_accept_full_json(self):
@@ -145,7 +145,7 @@ def setup_accept_full_json(self):
     def test_accept_full_json(self):
         """Blocking with Accept: application/json"""
         assert self.r_afj.status_code == 403
-        assert self.r_afj.headers.get("content-type", "") in JSON_CONTENT_TYPES
+        assert self.r_afj.headers.get("content-type", "").lower() in JSON_CONTENT_TYPES
         assert self.r_afj.text in BLOCK_TEMPLATE_JSON_ANY
 
     def setup_accept_full_html(self):
@@ -165,7 +165,7 @@ def setup_accept_full_html(self):
     def test_accept_full_html(self):
         """Blocking with Accept: text/html"""
         assert self.r_afh.status_code == 403
-        assert self.r_afh.headers.get("content-type", "") in HTML_CONTENT_TYPES
+        assert self.r_afh.headers.get("content-type", "").lower() in HTML_CONTENT_TYPES
         assert self.r_afh.text in BLOCK_TEMPLATE_HTML_ANY
 
     def setup_json_template_v1(self):
@@ -181,7 +181,7 @@ def setup_json_template_v1(self):
     def test_json_template_v1(self):
         """HTML block template is v1 minified"""
         assert self.r_json_v1.status_code == 403
-        assert self.r_json_v1.headers.get("content-type", "") in JSON_CONTENT_TYPES
+        assert self.r_json_v1.headers.get("content-type", "").lower() in JSON_CONTENT_TYPES
         assert self.r_json_v1.text.rstrip() == BLOCK_TEMPLATE_JSON_MIN_V1.rstrip()
 
     def setup_html_template_v2(self):
@@ -197,7 +197,7 @@ def setup_html_template_v2(self):
     def test_html_template_v2(self):
         """HTML block template is v2 minified"""
         assert self.r_html_v2.status_code == 403
-        assert self.r_html_v2.headers.get("content-type", "") in HTML_CONTENT_TYPES
+        assert self.r_html_v2.headers.get("content-type", "").lower() in HTML_CONTENT_TYPES
         assert self.r_html_v2.text == BLOCK_TEMPLATE_HTML_MIN_V2
 
 
@@ -226,7 +226,10 @@ def test_custom_redirect(self):
     def setup_custom_redirect_wrong_status_code(self):
         self.r_cr = weblog.get("/waf/", headers={"User-Agent": "Canary/v3"}, allow_redirects=False)
 
-    @bug(context.library == "java", reason="Do not check the configured redirect status code")
+    @bug(
+        context.library == "java" and context.weblog_variant not in ("akka-http", "play"),
+        reason="Do not check the configured redirect status code",
+    )
     @bug(context.library == "golang", reason="Do not check the configured redirect status code")
     def test_custom_redirect_wrong_status_code(self):
         """Block with an HTTP redirection but default to 303 status code, because the configured status code is not a valid redirect status code"""

tests/appsec/waf/test_rules.py

@@ -79,6 +79,9 @@ def setup_lfi_in_path(self):
     @bug(context.weblog_variant == "uwsgi-poc" and context.library == "python")
     @irrelevant(library="python", weblog_variant="django-poc")
     @irrelevant(library="dotnet", reason="lfi patterns are always filtered by the host web-server")
+    @irrelevant(
+        context.weblog_variant in ("akka-http", "play") and context.library == "java", reason="path is normalized to /"
+    )
     def test_lfi_in_path(self):
         """ AppSec catches LFI attacks in URL path like /.."""
         interfaces.library.assert_waf_attack(self.r_5, rules.lfi.crs_930_110)

utils/build/docker/java/akka-http/pom.xml

@@ -10,7 +10,7 @@
     <version>1.0.0</version>
 
     <properties>
-        <scala.version>2.13.5</scala.version>
+        <scala.version>2.13.10</scala.version>
         <akka.version>2.8.0</akka.version>
         <akka.http.version>10.5.0</akka.http.version>
     </properties>
@@ -26,6 +26,16 @@
             <artifactId>akka-http-jackson_2.13</artifactId>
             <version>${akka.http.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-scala_2.13</artifactId>
+            <version>2.13.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.typesafe.akka</groupId>
+            <artifactId>akka-http-xml_2.13</artifactId>
+            <version>${akka.http.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.typesafe.akka</groupId>
             <artifactId>akka-actor_2.13</artifactId>
@@ -140,6 +150,7 @@
                                 <filter>
                                     <artifact>*:*</artifact>
                                     <excludes>
+                                        <exclude>META-INF/*.MF</exclude>
                                         <exclude>META-INF/*.SF</exclude>
                                         <exclude>META-INF/*.DSA</exclude>
                                         <exclude>META-INF/*.RSA</exclude>

utils/build/docker/java/akka-http/src/main/scala/com/datadoghq/akka_http/AppSecRoutes.scala

@@ -0,0 +1,181 @@
+package com.datadoghq.akka_http
+
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.marshalling.Marshaller
+import akka.http.scaladsl.model.Uri.Path
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.headers._
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.unmarshalling._
+
+import java.util
+import scala.concurrent.Future
+import scala.xml.{Elem, XML}
+
+object AppSecRoutes {
+  val route: Route =
+    path("") {
+      get {
+        val span = tracer.buildSpan("test-span").start
+        span.setTag("test-tag", "my value")
+        withSpan(span) {
+          complete("Hello world!")
+        }
+      }
+    } ~
+      path("headers") {
+        get {
+          val entity = HttpEntity(ContentTypes.`text/plain(UTF-8)`, "012345678901234567890123456789012345678901")
+          respondWithHeaders(RawHeader("Content-Language", "en-US")) {
+            complete(entity)
+          }
+        }
+      } ~
+      path("tag_value" / Segment / """\d{3}""".r) { (value, code) =>
+        get {
+          parameter("content-language".?) { clo =>
+            setRootSpanTag("appsec.events.system_tests_appsec_event.value", value)
+
+            val resp = complete(
+              HttpResponse(
+                status = StatusCodes.custom(code.toInt, "some reason"),
+                entity = HttpEntity(ContentTypes.`text/plain(UTF-8)`, "Value tagged")
+              )
+            )
+
+            clo match {
+              case Some(cl) => respondWithHeaders(RawHeader("Content-Language", cl)) { resp }
+              case None => resp
+            }
+          }
+        } ~
+          post {
+            formFieldMap { _ =>
+              setRootSpanTag("appsec.events.system_tests_appsec_event.value", value)
+              complete(
+                HttpResponse(
+                  status = StatusCodes.custom(code.toInt, "some reason"),
+                  entity = HttpEntity(ContentTypes.`text/plain(UTF-8)`, "Value tagged")
+                )
+              )
+            }
+          }
+      } ~
+      path("params" / Segments) { segments: Seq[String] =>
+        get {
+          complete(segments.toString())
+        }
+      } ~
+      path("waf") {
+        get {
+          complete("Hello world!")
+        } ~
+          post {
+            formFieldMultiMap { fields: Map[String, List[String]] =>
+              complete(fields.toString)
+            } ~
+              entity(Unmarshaller.messageUnmarshallerFromEntityUnmarshaller(generalizedJsonUnmarshaller)) { value =>
+                complete(value.toString)
+              } ~ entity(as[XmlObject]) { xmlObj =>
+              complete(xmlObj.toString)
+            } ~
+              entity(Unmarshaller.messageUnmarshallerFromEntityUnmarshaller(
+                Unmarshaller.byteArrayUnmarshaller.forContentTypes(
+                  MediaTypes.`application/octet-stream`))) { arr: Array[Byte] =>
+                complete(s"Hello world (${arr.length})")
+              } ~
+              entity(as[String]) { s: String =>
+                // interpret as string as fallback, regardless of content-type
+                complete(s)
+              }
+          }
+      } ~
+      path("waf" / RemainingPath) { remaining: Path =>
+        get {
+          complete(remaining.toString())
+        }
+      } ~
+      path("make_distant_call") {
+        get {
+          parameter("url") { url =>
+            complete(StatusCodes.OK, makeDistantCall(url))(Marshaller.futureMarshaller(jsonMarshaller))
+          }
+        }
+      } ~
+      path("status") {
+        get {
+          parameter("code".as[Int]) { code =>
+            complete(StatusCodes.custom(code, "whatever reason"))
+          }
+        }
+      } ~
+      path("user_login_success_event") {
+        get {
+          parameter("event_user_id".?("system_tests_user")) { userId =>
+            eventTracker.trackLoginSuccessEvent(userId, metadata)
+            complete("ok")
+          }
+        }
+      } ~
+      path("user_login_failure_event") {
+        get {
+          parameters("event_user_id".?("system_tests_user"),
+            "event_user_exists".as[Boolean].?(true)) { (userId, userExists) =>
+            eventTracker.trackLoginFailureEvent(userId, userExists, metadata)
+            complete("ok")
+          }
+        }
+      } ~
+      path("custom_event") {
+        get {
+          parameter("event_name".?("system_tests_event")) { eventName =>
+            eventTracker.trackCustomEvent(eventName, metadata)
+            complete("ok")
+          }
+        }
+      }
+
+  case class XmlObject(value: String, attack: String)
+
+  implicit val xmlObjectUnmarshaller: FromEntityUnmarshaller[XmlObject] =
+    Unmarshaller.stringUnmarshaller.forContentTypes(MediaTypes.`text/xml`, MediaTypes.`application/xml`).map { string =>
+      val xmlData: Elem = XML.loadString(string)
+      val value = (xmlData \ "value").text
+      val attack = (xmlData \ "attack").text
+      XmlObject(value, attack)
+    }
+
+  case class DistantCallResponse(
+                                  url: String,
+                                  status_code: Int,
+                                  request_headers: Map[String, String],
+                                  response_headers: Map[String, String]
+                                )
+
+  private def makeDistantCall(url: String): Future[DistantCallResponse] = {
+    val request = HttpRequest(uri = url)
+    val requestHeaders = request.headers.map(h => (h.name(), h.value())).toMap
+
+    Http().singleRequest(request).map { response =>
+      val statusCode = response.status.intValue()
+      val responseHeaders = response.headers.map(h => (h.name(), h.value())).toMap
+
+      response.discardEntityBytes()
+
+      DistantCallResponse(
+        url = url,
+        status_code = statusCode,
+        request_headers = requestHeaders,
+        response_headers = responseHeaders,
+      )
+    }
+  }
+
+  private val metadata: util.Map[String, String] =  {
+    val h = new util.HashMap[String, String]
+    h.put("metadata0", "value0")
+    h.put("metadata1", "value1")
+    h
+  }
+}

utils/build/docker/java/akka-http/src/main/scala/com/datadoghq/akka_http/IastRoutes.scala

@@ -2,10 +2,9 @@ package com.datadoghq.akka_http
 
 import akka.http.javadsl.marshallers.jackson.Jackson
 import akka.http.scaladsl.marshalling.Marshaller
-import akka.http.scaladsl.model.{HttpEntity, RequestEntity, StatusCodes}
+import akka.http.scaladsl.model.{RequestEntity, StatusCodes}
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
-import akka.http.scaladsl.unmarshalling.Unmarshaller
 import com.datadoghq.system_tests.iast.infra.{LdapServer, SqlServer}
 import com.datadoghq.system_tests.iast.utils._
 
@@ -203,12 +202,6 @@ object IastRoutes {
       }
   }
 
-  private val jsonMarshaller : Marshaller[Object, RequestEntity] =
-    Jackson.marshaller().asScala.map(_.asInstanceOf[RequestEntity] /* just downcast */)
-
-  implicit val mapJsonUnmarshaller : Unmarshaller[HttpEntity, java.util.Map[String, Object]] =
-    Jackson.unmarshaller(classOf[java.util.Map[String, Object]]).asScala
-
   private def paramOrFormField(p: String) = {
     parameter(p) | formField(p)
   }