diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4666319..4084c99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Run tests run: sbt +test - name: Check Scala formatting @@ -52,6 +54,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Get current version id: ver run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}" @@ -80,6 +84,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Docker login run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD env: diff --git a/CHANGELOG b/CHANGELOG index 12c4ba3..f876028 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,16 @@ +Version 0.14.0 (2024-12-20) +--------------------------- +Install sbt in GitHub workflows +Check if JSON entity exceeds max depth during conversion from String to Circe JSON +Add max JSON depth check to /keygen endpoint +Do not allow requests with too large payload +Fix graceful shutdown to be more selective about when it sleeps +Make maxJsonDepth in SelfSyntaxChecker.validateSchema configurable +Add auth to validation endpoint +Bump sbt-snowplow-release to 0.3.2 +Bump jackson to 2.15.0 +Change SLULA license from 1.0 to 1.1 + Version 0.12.1 (2024-03-29) --------------------------- Bump postgresql to 42.5.5 (#154) diff --git a/LICENSE.md b/LICENSE.md index 6abbe69..0a7d3cc 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,18 +1,20 @@ # Snowplow Limited Use License Agreement -_Version 1.0, January 2024_ +_Version 1.1, November, 2024_ -This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. +This Snowplow Limited Use License Agreement, Version 1.1 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. -## LICENSE GRANT AND CONDITIONS +## 1. LICENSE GRANT AND CONDITIONS -**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Excluded Purpose. +**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Competing Use, and Licensee may exercise the License only for Non-Production Use or Non-Commercial Use. -**1.2** For purposes of this Agreement, an “Excluded Purpose” is any use that is either a Competing Use or a Highly-Available Production Use, or both of them. +**1.2 Definitions.** For purposes of this Agreement: -* **1.2.1** A “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. +* **1.2.1** “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. -* **1.2.2** Highly-Available Production Use is any highly-available use, including without limitation any use where multiple instances of any Software component run concurrently to avoid a single point of failure, in a production environment, where production means use on live data. +* **1.2.2** “Non-Production Use” means any use of the Software to process test or synthetic data to evaluate the sufficiency of the Software for use by Licensee. + +* **1.2.3** “Non-Commercial Use” is only: (a) personal use for research, experiment, personal study, or hobby projects, without any anticipated commercial application, or (b) use for teaching purposes by lecturers of a school or university. **1.3 Conditions.** In consideration of the License, Licensee’s use of the Software is subject to the following conditions: @@ -22,8 +24,8 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s ``` This software is made available by Snowplow Analytics, Ltd., - under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - located at https://docs.snowplow.io/limited-use-license-1.0 + under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + located at https://docs.snowplow.io/limited-use-license-1.1 BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. ``` @@ -32,23 +34,23 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s **1.5 No Sublicensing.** The License does not include the right to sublicense the Software, however, each recipient to which Licensee provides the Software may exercise the Licenses so long as such recipient agrees to the terms and conditions of this Agreement. -## TERM AND TERMINATION +## 2. TERM AND TERMINATION This Agreement will continue unless and until earlier terminated as set forth herein. If Licensee breaches any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the License will terminate automatically and permanently. -## INTELLECTUAL PROPERTY +## 3. INTELLECTUAL PROPERTY As between the parties, Snowplow will retain all right, title, and interest in the Software, and all intellectual property rights therein. Snowplow hereby reserves all rights not expressly granted to Licensee in this Agreement. Snowplow hereby reserves all rights in its trademarks and service marks, and no licenses therein are granted in this Agreement. -## DISCLAIMER +## 4. DISCLAIMER SNOWPLOW HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. -## LIMITATION OF LIABILITY +## 5. LIMITATION OF LIABILITY SNOWPLOW WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW. -## GENERAL +## 6. GENERAL **6.1 Governing Law.** This Agreement will be governed by and interpreted in accordance with the laws of the state of Delaware, without reference to its conflict of laws principles. If Licensee is located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Delaware, USA. If Licensee is located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The tribunal will consist of one arbitrator. The place of arbitration will be in the State of Delaware, USA. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. diff --git a/README.md b/README.md index bda98bd..9a26820 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,6 @@ Licensed under the [Snowplow Limited Use License Agreement][license]. _(If you a [roadmap]: https://github.com/snowplow/snowplow/projects/7 [contributing]: https://docs.snowplow.io/docs/contributing/ -[license]: https://docs.snowplow.io/limited-use-license-1.0 +[license]: https://docs.snowplow.io/limited-use-license-1.1 [license-image]: https://img.shields.io/badge/license-Snowplow--Limited--Use-blue.svg?style=flat [faq]: https://docs.snowplow.io/docs/contributing/limited-use-license-faq/ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ac8c60a..6ddd735 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ services: POSTGRES_PASSWORD: ${IGLU_DB_PASSWORD:-iglusecret} iglu-server: container_name: iglu-server - image: snowplow/iglu-server:${IGLU_VERSION:-0.11.0} + image: snowplow/iglu-server:${IGLU_VERSION:-0.13.1} command: --config /snowplow/config/config.hocon environment: IGLU_SUPER_API_KEY: ${IGLU_SUPER_API_KEY:-5fb4713d-73ad-4163-93a9-2b82f0177c5b} diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index c311ec6..e7bf7dd 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -37,13 +37,13 @@ object BuildSettings { ) lazy val licenseSettings = Seq( - licenses += ("Snowplow Limited Use License Agreement", url("https://docs.snowplow.io/limited-use-license-1.0")), + licenses += ("Snowplow Limited Use License Agreement", url("https://docs.snowplow.io/limited-use-license-1.1")), headerLicense := Some(HeaderLicense.Custom( """|Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. | |This software is made available by Snowplow Analytics, Ltd., - |under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - |located at https://docs.snowplow.io/limited-use-license-1.0 + |under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + |located at https://docs.snowplow.io/limited-use-license-1.1 |BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION |OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. |""".stripMargin diff --git a/project/Dependencies.scala b/project/Dependencies.scala index af4935e..43f514f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,7 +18,7 @@ object Dependencies { object V { val IgluCore = "1.1.2" - val SchemaDdl = "0.18.0" + val SchemaDdl = "0.24.0" val IgluClient = "1.1.1" val Http4s = "0.21.34" @@ -34,7 +34,7 @@ object Dependencies { val Slf4j = "1.7.36" val ScalaCache = "0.28.0" val Postgresql = "42.5.5" - val Jackson = "2.14.1" + val Jackson = "2.15.0" val Snakeyaml = "2.0" val Guava = "32.0.0-jre" diff --git a/project/plugins.sbt b/project/plugins.sbt index 99fd934..e9e3241 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -5,5 +5,5 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.3") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") -addSbtPlugin("com.snowplowanalytics" % "sbt-snowplow-release" % "0.3.1") +addSbtPlugin("com.snowplowanalytics" % "sbt-snowplow-release" % "0.3.2") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 8ef932d..c7d0c25 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,8 +1,8 @@ # Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. @@ -24,6 +24,7 @@ "enable": false "maxAge": "365 days" } + "maxPayloadSize": 100000 } "database" { @@ -59,6 +60,8 @@ "schemaPublished": [] } + "maxJsonDepth": 40 + preTerminationPeriod: "1 seconds" preTerminationUnhealthy: false diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/Config.scala b/src/main/scala/com/snowplowanalytics/iglu/server/Config.scala index c8a66d4..5568bcb 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/Config.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/Config.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -51,7 +51,8 @@ case class Config( superApiKey: Option[UUID], preTerminationPeriod: FiniteDuration, preTerminationUnhealthy: Boolean, - license: Config.License + license: Config.License, + maxJsonDepth: Int ) object Config { @@ -220,7 +221,8 @@ object Config { idleTimeout: Option[FiniteDuration], maxConnections: Option[Int], threadPool: ThreadPool, - hsts: Config.Hsts + hsts: Config.Hsts, + maxPayloadSize: Long ) implicit val httpConfigCirceEncoder: Encoder[Http] = diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/Main.scala b/src/main/scala/com/snowplowanalytics/iglu/server/Main.scala index 879dad0..f9895fc 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/Main.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/Main.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/Server.scala b/src/main/scala/com/snowplowanalytics/iglu/server/Server.scala index 5930818..1a7db02 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/Server.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/Server.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -15,18 +15,19 @@ import java.util.UUID import scala.concurrent.duration._ import scala.concurrent.ExecutionContext import cats.data.Kleisli -import cats.effect.{Blocker, ContextShift, ExitCase, ExitCode, IO, Resource, Sync, Timer} +import cats.syntax.all._ +import cats.effect.{Blocker, ContextShift, ExitCode, IO, Resource, Sync, Timer} import cats.effect.concurrent.Ref import io.circe.syntax._ import org.typelevel.log4cats.slf4j.Slf4jLogger import fs2.Stream -import fs2.concurrent.SignallingRef +import fs2.concurrent.{Signal, SignallingRef} import org.http4s.{Headers, HttpApp, HttpRoutes, MediaType, Method, Request, Response, Status} import org.http4s.headers.{`Content-Type`, `Strict-Transport-Security`} import org.http4s.client.blaze.BlazeClientBuilder import org.http4s.server.Router import org.http4s.server.blaze.BlazeServerBuilder -import org.http4s.server.middleware.{AutoSlash, CORS, HSTS, Logger} +import org.http4s.server.middleware.{AutoSlash, CORS, EntityLimiter, HSTS, Logger} import org.http4s.syntax.string._ import org.http4s.server.{defaults => Http4sDefaults} import org.http4s.util.{CaseInsensitiveString => CIString} @@ -37,6 +38,7 @@ import org.http4s.rho.swagger.models.{ApiKeyAuthDefinition, In, Info, SecurityRe import org.http4s.rho.swagger.SwaggerMetadata import doobie.implicits._ import doobie.util.transactor.Transactor +import sun.misc.{Signal => JvmSignal, SignalHandler} import com.snowplowanalytics.iglu.server.migrations.{Bootstrap, MigrateFrom} import com.snowplowanalytics.iglu.server.codecs.Swagger import com.snowplowanalytics.iglu.server.middleware.{BadRequestHandler, CachingMiddleware} @@ -65,6 +67,12 @@ object Server { .withBodyStream(Utils.toBytes(IgluResponse.EndpointNotFound: IgluResponse)) .withContentType(`Content-Type`(MediaType.application.json)) + val PayloadTooLarge: Response[IO] = + Response[IO]() + .withStatus(Status.PayloadTooLarge) + .withBodyStream(Utils.toBytes(IgluResponse.PayloadTooLarge: IgluResponse)) + .withContentType(`Content-Type`(MediaType.application.json)) + def addSwagger(storage: Storage[IO], superKey: Option[UUID], config: Config.Swagger)( service: (String, RoutesConstructor) ) = { @@ -93,12 +101,25 @@ object Server { swaggerConfig: Config.Swagger, blocker: Blocker, isHealthy: IO[Boolean], - hsts: Config.Hsts + hsts: Config.Hsts, + maxPayloadSize: Long, + maxJsonDepth: Int )(implicit cs: ContextShift[IO]): HttpApp[IO] = { val serverRoutes = - httpRoutes(storage, superKey, debug, patchesAllowed, webhook, cache, swaggerConfig, blocker, isHealthy) + httpRoutes( + storage, + superKey, + debug, + patchesAllowed, + webhook, + cache, + swaggerConfig, + blocker, + isHealthy, + maxJsonDepth + ) val server = Kleisli[IO, Request[IO], Response[IO]](req => Router(serverRoutes: _*).run(req).getOrElse(NotFound)) - hstsMiddleware(hsts)(server) + entityLimiter(maxPayloadSize)(hstsMiddleware(hsts)(server)) } def hstsMiddleware(hsts: Config.Hsts): HttpApp[IO] => HttpApp[IO] = @@ -106,6 +127,11 @@ object Server { HSTS(_, `Strict-Transport-Security`.unsafeFromDuration(hsts.maxAge)) else identity + def entityLimiter(maxPayloadSize: Long): HttpApp[IO] => HttpApp[IO] = + EntityLimiter(_, maxPayloadSize).recover { + case _: EntityLimiter.EntityTooLarge => PayloadTooLarge + } + def httpRoutes( storage: Storage[IO], superKey: Option[UUID], @@ -115,14 +141,15 @@ object Server { cache: CachingMiddleware.ResponseCache[IO], swaggerConfig: Config.Swagger, blocker: Blocker, - isHealthy: IO[Boolean] + isHealthy: IO[Boolean], + maxJsonDepth: Int )(implicit cs: ContextShift[IO]): List[(String, HttpRoutes[IO])] = { val services: List[(String, RoutesConstructor)] = List( "/api/meta" -> MetaService.asRoutes(debug, patchesAllowed, isHealthy), - "/api/schemas" -> SchemaService.asRoutes(patchesAllowed, webhook), - "/api/auth" -> AuthService.asRoutes, - "/api/validation" -> ValidationService.asRoutes, - "/api/drafts" -> DraftService.asRoutes + "/api/schemas" -> SchemaService.asRoutes(patchesAllowed, webhook, maxJsonDepth), + "/api/auth" -> AuthService.asRoutes(maxJsonDepth), + "/api/validation" -> ValidationService.asRoutes(maxJsonDepth), + "/api/drafts" -> DraftService.asRoutes(maxJsonDepth) ) val debugRoute = "/api/debug" -> DebugService.asRoutes(storage, ioSwagger.createRhoMiddleware()) @@ -185,38 +212,63 @@ object Server { config.swagger, blocker, isHealthy, - config.repoServer.hsts + config.repoServer.hsts, + config.repoServer.maxPayloadSize, + config.maxJsonDepth ) ) .withIdleTimeout(config.repoServer.idleTimeout.getOrElse(Http4sDefaults.IdleTimeout)) .withMaxConnections(config.repoServer.maxConnections.getOrElse(Http4sDefaults.MaxConnections)) def run(config: Config)(implicit cs: ContextShift[IO], timer: Timer[IO]): IO[ExitCode] = + runStream(config).compile.lastOrError + + def runStream(config: Config)(implicit cs: ContextShift[IO], timer: Timer[IO]): Stream[IO, ExitCode] = for { - signal <- SignallingRef[IO, Boolean](false) - ref <- Ref[IO].of(ExitCode.Success) - isHealthy <- Ref[IO].of(true) - stream = Stream.resource(buildServer(config, isHealthy.get)).flatMap(_.serveWhile(signal, ref)) - exitCode <- stream.compile.lastOrError.start.bracketCase(_.join) { - case (_, ExitCase.Completed) => IO.unit - case (_, ExitCase.Error(e)) => IO.raiseError(e) - case (fiber, ExitCase.Canceled) => - // We received a SIGINT - for { - _ <- logger.warn("Received shutdown signal") - _ <- if (config.preTerminationUnhealthy) { - logger.warn(s"Setting health endpoint to unhealthy") *> isHealthy.set(false) - } else IO.unit - _ <- logger.warn(s"Sleeping for ${config.preTerminationPeriod}") - _ <- IO.sleep(config.preTerminationPeriod) - _ <- logger.warn("Terminating the server") - _ <- signal.set(true) - _ <- fiber.join - _ <- logger.warn("Server terminated") - } yield () - } + sigToExit <- Stream.eval(SignallingRef[IO, Boolean](false)) + sigToPause <- Stream.eval(SignallingRef[IO, Boolean](false)) + refExitCode <- Stream.eval(Ref[IO].of(ExitCode.Success)) + _ <- Stream.eval(addShutdownHook(sigToPause)) + builder <- Stream.resource(buildServer(config, getIsHealthy(config, sigToPause))) + exitCode <- builder.serveWhile(sigToExit, refExitCode).concurrently(handleSigTerm(config, sigToPause, sigToExit)) } yield exitCode + def handleSigTerm(config: Config, sigToPause: Signal[IO, Boolean], sigToExit: Ref[IO, Boolean])( + implicit timer: Timer[IO] + ): Stream[IO, Unit] = + sigToPause.discrete.evalMap { + case true => + // We got a SIGTERM + for { + _ <- logger.warn( + s"Initiating server shutdown. Sleeping for ${config.preTerminationPeriod} as part of graceful shutdown." + ) + _ <- IO.sleep(config.preTerminationPeriod) + _ <- logger.warn("Terminating the server") + _ <- sigToExit.set(true) + } yield () + case false => + IO.unit + } + + def getIsHealthy(config: Config, sigToPause: Ref[IO, Boolean]): IO[Boolean] = + if (config.preTerminationUnhealthy) + sigToPause.get.map { isTerminating => + // healthy if not terminating + !isTerminating + } + else IO.pure(true) + + def addShutdownHook(received: Ref[IO, Boolean]): IO[Unit] = + IO.delay { + val handler = new SignalHandler { + override def handle(signal: JvmSignal): Unit = + received.set(true).unsafeRunSync() + } + JvmSignal.handle(new JvmSignal("TERM"), handler) + () + } + def setup(config: Config, migrate: Option[MigrateFrom])(implicit cs: ContextShift[IO]): IO[ExitCode] = config.database match { case pg: Config.StorageConfig.Postgres => diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/Utils.scala b/src/main/scala/com/snowplowanalytics/iglu/server/Utils.scala index 2b20384..1fa1f8f 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/Utils.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/Utils.scala @@ -2,19 +2,25 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ package com.snowplowanalytics.iglu.server -import io.circe.Encoder +import io.circe.{Decoder, Encoder, Json} import io.circe.syntax._ +import cats.implicits._ +import cats.effect.Sync + import fs2.{Stream, text} +import org.http4s.{DecodeResult, EntityDecoder, InvalidMessageBodyFailure} +import org.http4s.circe.jsonDecoder + import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaMap, SchemaVer} object Utils { @@ -25,4 +31,37 @@ object Utils { def toBytes[F[_], A: Encoder](a: A): Stream[F, Byte] = Stream.emit(a.asJson.noSpaces).through(text.utf8Encode) + def jsonOfWithDepthCheck[F[_]: Sync, A: Decoder](maxJsonDepth: Int): EntityDecoder[F, A] = + jsonDecoderWithDepthCheck(maxJsonDepth).flatMapR { json => + json + .as[A] + .fold( + failure => + DecodeResult.failureT[F, A]( + InvalidMessageBodyFailure(s"Could not decode JSON body", Some(failure)) + ), + DecodeResult.successT[F, A](_) + ) + } + + def jsonDecoderWithDepthCheck[F[_]: Sync](maxJsonDepth: Int): EntityDecoder[F, Json] = + jsonDecoder[F].transform( + _.flatMap { json => + if (checkIfExceedMaxDepth(json, maxJsonDepth)) + InvalidMessageBodyFailure("Maximum allowed JSON depth exceeded").asLeft + else json.asRight + } + ) + + private def checkIfExceedMaxDepth(json: Json, maxJsonDepth: Int): Boolean = + if (maxJsonDepth <= 0) true + else + json.fold( + jsonNull = false, + jsonBoolean = _ => false, + jsonNumber = _ => false, + jsonString = _ => false, + jsonArray = _.exists(checkIfExceedMaxDepth(_, maxJsonDepth - 1)), + jsonObject = _.toList.exists { case (_, j) => checkIfExceedMaxDepth(j, maxJsonDepth - 1) } + ) } diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/Webhook.scala b/src/main/scala/com/snowplowanalytics/iglu/server/Webhook.scala index e91856f..695faaa 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/Webhook.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/Webhook.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/JsonCodecs.scala b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/JsonCodecs.scala index b077b68..84c507a 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/JsonCodecs.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/JsonCodecs.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/Swagger.scala b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/Swagger.scala index 9371f2e..9874900 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/Swagger.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/Swagger.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/UriParsers.scala b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/UriParsers.scala index e6db3dd..9de56d6 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/UriParsers.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/UriParsers.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/package.scala b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/package.scala index d0cdf65..1d8975b 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/codecs/package.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/codecs/package.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/BadRequestHandler.scala b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/BadRequestHandler.scala index b79d3f0..6b6b699 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/BadRequestHandler.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/BadRequestHandler.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/CachingMiddleware.scala b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/CachingMiddleware.scala index c017ca3..3359a50 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/CachingMiddleware.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/CachingMiddleware.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/PermissionMiddleware.scala b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/PermissionMiddleware.scala index fba2ab6..9331b97 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/middleware/PermissionMiddleware.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/middleware/PermissionMiddleware.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Bootstrap.scala b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Bootstrap.scala index 5ede1b4..96613e0 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Bootstrap.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Bootstrap.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Fifth.scala b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Fifth.scala index ea39f28..48f7e8c 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Fifth.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/Fifth.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/MigrateFrom.scala b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/MigrateFrom.scala index 19f27e0..ffc7c77 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/migrations/MigrateFrom.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/migrations/MigrateFrom.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/IgluResponse.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/IgluResponse.scala index 87b09cc..dd82120 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/IgluResponse.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/IgluResponse.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -38,11 +38,13 @@ object IgluResponse { val DataInvalidationMessage = "The data for a field instance is invalid against its schema" val NotFoundEndpoint = "The endpoint does not exist" val NonSequentialSchemaVersion = "The schema version is not sequential" + val PayloadTooLargeError = "The payload is too large" case object SchemaNotFound extends IgluResponse case object SchemaNonSequential extends IgluResponse case object EndpointNotFound extends IgluResponse case object InvalidSchema extends IgluResponse + case object PayloadTooLarge extends IgluResponse case class SchemaMismatch(uriSchemaKey: SchemaKey, payloadSchemaKey: SchemaKey) extends IgluResponse case class SchemaUploaded(updated: Boolean, location: SchemaKey) extends IgluResponse @@ -94,6 +96,8 @@ object IgluResponse { ) case InvalidSchema => Json.fromFields(List("message" -> Json.fromString(DecodeError))) + case PayloadTooLarge => + Json.fromFields(List("message" -> Json.fromString(PayloadTooLargeError))) case SchemaNonSequential => Json.fromFields(List("message" -> Json.fromString(NonSequentialSchemaVersion))) case SchemaValidationReport(report) => diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/Permission.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/Permission.scala index 3146132..01cd852 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/Permission.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/Permission.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -58,6 +58,10 @@ case class Permission( /** Check if user has enough rights to create particular schema */ def canCreatePermission(requestedVendor: String): Boolean = key.contains(Permission.KeyAction.Create) && vendor.check(requestedVendor) + + /** It is enough to have any valid apikey to perform validation */ + def canValidate: Boolean = + schema.nonEmpty || key.nonEmpty } object Permission { diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/Schema.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/Schema.scala index 6e85d5e..ec8de68 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/Schema.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/Schema.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/SchemaDraft.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/SchemaDraft.scala index e8f637e..3c29579 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/SchemaDraft.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/SchemaDraft.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/VersionCursor.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/VersionCursor.scala index 00bef40..098a174 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/VersionCursor.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/VersionCursor.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/model/package.scala b/src/main/scala/com/snowplowanalytics/iglu/server/model/package.scala index 558b4e1..27f3d30 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/model/package.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/model/package.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/AuthService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/AuthService.scala index aca6e38..5ab7233 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/AuthService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/AuthService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -33,18 +33,24 @@ import com.snowplowanalytics.iglu.server.model.Permission import com.snowplowanalytics.iglu.server.model.IgluResponse import com.snowplowanalytics.iglu.server.storage.Storage -class AuthService[F[+_]: Sync](swagger: SwaggerSyntax[F], ctx: AuthedContext[F, Permission], db: Storage[F]) - extends RhoRoutes[F] { +class AuthService[F[+_]: Sync]( + swagger: SwaggerSyntax[F], + ctx: AuthedContext[F, Permission], + db: Storage[F], + maxJsonDepth: Int +) extends RhoRoutes[F] { import swagger._ import AuthService._ val apikey = paramD[UUID]("key", "UUID apikey to delete") + val jsonGenerateKey = Utils.jsonOfWithDepthCheck[F, GenerateKey](maxJsonDepth) + "Route to delete api key" ** DELETE / "keygen" +? apikey >>> ctx.auth |>> deleteKey _ "Route to generate new keys" ** - POST / "keygen" >>> ctx.auth ^ jsonOf[F, GenerateKey] |>> { (authInfo: Permission, gk: GenerateKey) => + POST / "keygen" >>> ctx.auth ^ jsonGenerateKey |>> { (authInfo: Permission, gk: GenerateKey) => if (authInfo.key.contains(Permission.KeyAction.Create)) { val vendorPrefix = Permission.Vendor.parse(gk.vendorPrefix) if (authInfo.canCreatePermission(vendorPrefix.asString)) { @@ -71,13 +77,13 @@ object AuthService { implicit val schemaGenerateReq: Decoder[GenerateKey] = deriveDecoder[GenerateKey] - def asRoutes( + def asRoutes(maxJsonDepth: Int)( db: Storage[IO], superKey: Option[UUID], ctx: AuthedContext[IO, Permission], rhoMiddleware: RhoMiddleware[IO] ): HttpRoutes[IO] = { - val service = new AuthService(swaggerSyntax, ctx, db).toRoutes(rhoMiddleware) + val service = new AuthService(swaggerSyntax, ctx, db, maxJsonDepth).toRoutes(rhoMiddleware) PermissionMiddleware.wrapService(db, superKey, ctx, service) } } diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/DebugService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/DebugService.scala index 6a57e80..1f8a3c8 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/DebugService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/DebugService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/DraftService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/DraftService.scala index f180845..5c4e156 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/DraftService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/DraftService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -18,26 +18,30 @@ import cats.implicits._ import io.circe._ import org.http4s.HttpRoutes -import org.http4s.circe._ import org.http4s.rho.{AuthedContext, RhoMiddleware, RhoRoutes} import org.http4s.rho.swagger.SwaggerSyntax import org.http4s.rho.swagger.syntax.io +import com.snowplowanalytics.iglu.server.Utils import com.snowplowanalytics.iglu.server.storage.Storage import com.snowplowanalytics.iglu.server.middleware.PermissionMiddleware import com.snowplowanalytics.iglu.server.model.{DraftVersion, IgluResponse, Permission, SchemaDraft} import com.snowplowanalytics.iglu.server.codecs.UriParsers._ import com.snowplowanalytics.iglu.server.codecs.JsonCodecs._ -class DraftService[F[+_]: Sync](swagger: SwaggerSyntax[F], db: Storage[F], ctx: AuthedContext[F, Permission]) - extends RhoRoutes[F] { +class DraftService[F[+_]: Sync]( + swagger: SwaggerSyntax[F], + db: Storage[F], + ctx: AuthedContext[F, Permission], + maxJsonDepth: Int +) extends RhoRoutes[F] { import swagger._ import DraftService._ implicit val C: Clock[F] = Clock.create[F] val version = pathVar[DraftVersion]("version", "Draft version") val isPublic = paramD[Boolean]("isPublic", false, "Should schema be created as public") - val schemaBody = jsonOf[F, Json] + val schemaBody = Utils.jsonDecoderWithDepthCheck(maxJsonDepth) "Get a particular draft by its URI" ** GET / 'vendor / 'name / 'format / version >>> ctx.auth |>> getDraft _ @@ -80,13 +84,13 @@ class DraftService[F[+_]: Sync](swagger: SwaggerSyntax[F], db: Storage[F], ctx: object DraftService { - def asRoutes( + def asRoutes(maxJsonDepth: Int)( db: Storage[IO], superKey: Option[UUID], ctx: AuthedContext[IO, Permission], rhoMiddleware: RhoMiddleware[IO] ): HttpRoutes[IO] = { - val service = new DraftService(io, db, ctx).toRoutes(rhoMiddleware) + val service = new DraftService(io, db, ctx, maxJsonDepth).toRoutes(rhoMiddleware) PermissionMiddleware.wrapService(db, superKey, ctx, service) } diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/MetaService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/MetaService.scala index b50dd28..1a7b674 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/MetaService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/MetaService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/SchemaService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/SchemaService.scala index cd02ac2..8859f47 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/SchemaService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/SchemaService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -20,7 +20,6 @@ import cats.implicits._ import io.circe.Json import org.http4s.HttpRoutes -import org.http4s.circe._ import org.http4s.rho.{AuthedContext, RhoMiddleware, RhoRoutes} import org.http4s.rho.swagger.SwaggerSyntax import org.http4s.rho.swagger.syntax.{io => swaggerSyntax} @@ -42,7 +41,8 @@ class SchemaService[F[+_]: Sync]( ctx: AuthedContext[F, Permission], db: Storage[F], patchesAllowed: Boolean, - webhooks: Webhook.WebhookClient[F] + webhooks: Webhook.WebhookClient[F], + maxJsonDepth: Int ) extends RhoRoutes[F] { import swagger._ @@ -56,9 +56,11 @@ class SchemaService[F[+_]: Sync]( val version = pathVar[SchemaVer.Full]("version", "SchemaVer") val isPublic = paramD[Boolean]("isPublic", false, "Should schema be created as public") - val schemaOrJson = jsonOf[F, SchemaBody] + val schemaOrJson = Utils.jsonOfWithDepthCheck[F, SchemaBody](maxJsonDepth) - private val validationService = new ValidationService[F](swagger, ctx, db) + val jsonBody = Utils.jsonDecoderWithDepthCheck(maxJsonDepth) + + private val validationService = new ValidationService[F](swagger, ctx, db, maxJsonDepth) "Get a particular schema by its Iglu URI" ** GET / 'vendor / 'name / 'format / version +? reprCanonical >>> ctx.auth |>> getSchema _ @@ -85,8 +87,9 @@ class SchemaService[F[+_]: Sync]( POST +? isPublic >>> ctx.auth ^ schemaOrJson |>> postSchema _ "Schema validation endpoint (deprecated)" ** - POST / "validate" / 'vendor / 'name / "jsonschema" / 'version ^ jsonDecoder[F] |>> { - (_: String, _: String, _: String, json: Json) => validationService.validateSchema(Schema.Format.Jsonschema, json) + POST / "validate" / 'vendor / 'name / "jsonschema" / 'version >>> ctx.auth ^ jsonBody |>> { + (_: String, _: String, _: String, authInfo: Permission, json: Json) => + validationService.validateSchema(Schema.Format.Jsonschema, authInfo, json) } def getSchema( @@ -172,7 +175,7 @@ class SchemaService[F[+_]: Sync]( supersedingInfo: SupersedingInfo ) = if (permission.canCreateSchema(schema.self.schemaKey.vendor)) { - ValidationService.validateJsonSchema(schema.normalize) match { + ValidationService.validateJsonSchema(schema.normalize, maxJsonDepth) match { case Validated.Invalid(report) if report.exists(_.level == Linter.Level.Error) => BadRequest(IgluResponse.SchemaValidationReport(report): IgluResponse) case _ => addSchema(schema, isPublic, supersedingInfo) @@ -228,14 +231,16 @@ object SchemaService { def asRoutes( patchesAllowed: Boolean, - webhook: Webhook.WebhookClient[IO] + webhook: Webhook.WebhookClient[IO], + maxJsonDepth: Int )( db: Storage[IO], superKey: Option[UUID], ctx: AuthedContext[IO, Permission], rhoMiddleware: RhoMiddleware[IO] ): HttpRoutes[IO] = { - val service = new SchemaService(swaggerSyntax, ctx, db, patchesAllowed, webhook).toRoutes(rhoMiddleware) + val service = + new SchemaService(swaggerSyntax, ctx, db, patchesAllowed, webhook, maxJsonDepth).toRoutes(rhoMiddleware) PermissionMiddleware.wrapService(db, superKey, ctx, service) } diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/StaticService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/StaticService.scala index d445558..535fbb0 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/StaticService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/StaticService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/service/ValidationService.scala b/src/main/scala/com/snowplowanalytics/iglu/server/service/ValidationService.scala index 827b49a..2ae7301 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/service/ValidationService.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/service/ValidationService.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -15,7 +15,6 @@ import java.util.UUID import io.circe.Json import org.http4s.HttpRoutes -import org.http4s.circe._ import org.http4s.rho.{AuthedContext, RhoMiddleware, RhoRoutes} import org.http4s.rho.swagger.SwaggerSyntax import org.http4s.rho.swagger.syntax.io @@ -34,72 +33,88 @@ import com.snowplowanalytics.iglu.schemaddl.jsonschema.{Pointer, SelfSyntaxCheck import com.snowplowanalytics.iglu.schemaddl.jsonschema.SanityLinter.lint import com.snowplowanalytics.iglu.schemaddl.jsonschema.Linter.{Level, Message, allLintersMap} +import com.snowplowanalytics.iglu.server.Utils import com.snowplowanalytics.iglu.server.storage.Storage import com.snowplowanalytics.iglu.server.middleware.PermissionMiddleware import com.snowplowanalytics.iglu.server.model.{IgluResponse, Permission, Schema} import com.snowplowanalytics.iglu.server.codecs.JsonCodecs._ import com.snowplowanalytics.iglu.server.codecs.UriParsers._ -class ValidationService[F[+_]: Sync](swagger: SwaggerSyntax[F], ctx: AuthedContext[F, Permission], db: Storage[F]) - extends RhoRoutes[F] { +class ValidationService[F[+_]: Sync]( + swagger: SwaggerSyntax[F], + ctx: AuthedContext[F, Permission], + db: Storage[F], + maxJsonDepth: Int +) extends RhoRoutes[F] { + import swagger._ import ValidationService._ + val jsonBody = Utils.jsonDecoderWithDepthCheck(maxJsonDepth) + val schemaFormat = pathVar[Schema.Format]("format", "Schema format, e.g. jsonschema") "This route allows you to validate schemas" ** - POST / "validate" / "schema" / schemaFormat ^ jsonDecoder[F] |>> validateSchema _ + POST / "validate" / "schema" / schemaFormat >>> ctx.auth ^ jsonBody |>> validateSchema _ "This route allows you to validate self-describing instances" ** - POST / "validate" / "instance" >>> ctx.auth ^ jsonDecoder[F] |>> validateData _ - - def validateSchema(format: Schema.Format, schema: Json) = - format match { - case Schema.Format.Jsonschema => - validateJsonSchema(schema) match { - case Validated.Valid(sd) => - val message = s"The schema provided is a valid self-describing ${sd.self.schemaKey.toSchemaUri} schema" - Ok(IgluResponse.Message(message): IgluResponse) - case Validated.Invalid(report) => - Ok(IgluResponse.SchemaValidationReport(report): IgluResponse) - } + POST / "validate" / "instance" >>> ctx.auth ^ jsonBody |>> validateData _ + + def validateSchema(format: Schema.Format, authInfo: Permission, schema: Json) = + if (!authInfo.canValidate) { + Forbidden("") + } else { + format match { + case Schema.Format.Jsonschema => + validateJsonSchema(schema, maxJsonDepth) match { + case Validated.Valid(sd) => + val message = s"The schema provided is a valid self-describing ${sd.self.schemaKey.toSchemaUri} schema" + Ok(IgluResponse.Message(message): IgluResponse) + case Validated.Invalid(report) => + Ok(IgluResponse.SchemaValidationReport(report): IgluResponse) + } + } } def validateData(authInfo: Permission, instance: Json) = - SelfDescribingData.parse(instance) match { - case Right(SelfDescribingData(key, data)) => - for { - schema <- db.getSchema(SchemaMap(key)) - response <- schema match { - case Some(Schema(_, meta, schemaBody, _)) if meta.isPublic || authInfo.canRead(key.vendor) => - CirceValidator.validate(data, schemaBody) match { - case Left(ValidatorError.InvalidData(report)) => - Ok(IgluResponse.InstanceValidationReport(report): IgluResponse) - case Left(ValidatorError.InvalidSchema(_)) => - val message = s"Schema ${key.toSchemaUri} fetched from DB is invalid" - InternalServerError(IgluResponse.Message(message): IgluResponse) - case Right(_) => - Ok(IgluResponse.Message(s"Instance is valid ${key.toSchemaUri}"): IgluResponse) - } - case _ => - NotFound(IgluResponse.SchemaNotFound: IgluResponse) - } - } yield response - case Left(error) => - BadRequest(IgluResponse.Message(s"JSON payload is not self-describing, ${error.code}"): IgluResponse) + if (!authInfo.canValidate) { + NotFound(IgluResponse.SchemaNotFound: IgluResponse) + } else { + SelfDescribingData.parse(instance) match { + case Right(SelfDescribingData(key, data)) => + for { + schema <- db.getSchema(SchemaMap(key)) + response <- schema match { + case Some(Schema(_, meta, schemaBody, _)) if meta.isPublic || authInfo.canRead(key.vendor) => + CirceValidator.validate(data, schemaBody) match { + case Left(ValidatorError.InvalidData(report)) => + Ok(IgluResponse.InstanceValidationReport(report): IgluResponse) + case Left(ValidatorError.InvalidSchema(_)) => + val message = s"Schema ${key.toSchemaUri} fetched from DB is invalid" + InternalServerError(IgluResponse.Message(message): IgluResponse) + case Right(_) => + Ok(IgluResponse.Message(s"Instance is valid ${key.toSchemaUri}"): IgluResponse) + } + case _ => + NotFound(IgluResponse.SchemaNotFound: IgluResponse) + } + } yield response + case Left(error) => + BadRequest(IgluResponse.Message(s"JSON payload is not self-describing, ${error.code}"): IgluResponse) + } } } object ValidationService { - def asRoutes( + def asRoutes(maxJsonDepth: Int)( db: Storage[IO], superKey: Option[UUID], ctx: AuthedContext[IO, Permission], rhoMiddleware: RhoMiddleware[IO] ): HttpRoutes[IO] = { - val service = new ValidationService[IO](io, ctx, db).toRoutes(rhoMiddleware) + val service = new ValidationService[IO](io, ctx, db, maxJsonDepth).toRoutes(rhoMiddleware) PermissionMiddleware.wrapService(db, superKey, ctx, service) } @@ -108,9 +123,9 @@ object ValidationService { val NotSelfDescribing = Message(Pointer.Root, "JSON Schema is not self-describing", Level.Error) val NotSchema = Message(Pointer.Root, "Cannot extract JSON Schema", Level.Error) - def validateJsonSchema(schema: Json): LintReport[SelfDescribingSchema[Json]] = { + def validateJsonSchema(schema: Json, maxJsonDepth: Int): LintReport[SelfDescribingSchema[Json]] = { val generalCheck = - SelfSyntaxChecker.validateSchema(schema) + SelfSyntaxChecker.validateSchema(schema, maxJsonDepth) val selfDescribingCheck = SelfDescribingSchema .parse(schema) diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/storage/InMemory.scala b/src/main/scala/com/snowplowanalytics/iglu/server/storage/InMemory.scala index 48bac32..0fb703d 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/storage/InMemory.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/storage/InMemory.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/storage/Postgres.scala b/src/main/scala/com/snowplowanalytics/iglu/server/storage/Postgres.scala index fd898f4..0895a6a 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/storage/Postgres.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/storage/Postgres.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/main/scala/com/snowplowanalytics/iglu/server/storage/Storage.scala b/src/main/scala/com/snowplowanalytics/iglu/server/storage/Storage.scala index 716a4b5..9ddd51b 100644 --- a/src/main/scala/com/snowplowanalytics/iglu/server/storage/Storage.scala +++ b/src/main/scala/com/snowplowanalytics/iglu/server/storage/Storage.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/resources/valid-dummy-config.conf b/src/test/resources/valid-dummy-config.conf index 3101607..d546230 100644 --- a/src/test/resources/valid-dummy-config.conf +++ b/src/test/resources/valid-dummy-config.conf @@ -1,8 +1,8 @@ # Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. @@ -24,6 +24,7 @@ repoServer { enable = true maxAge = "365 days" } + maxPayloadSize = 5000 } # 'postgres' contains configuration options for the postgre instance the server is using @@ -34,3 +35,5 @@ database { # Enable additional debug endpoint to output all internal state debug = true + +maxJsonDepth = 50 diff --git a/src/test/resources/valid-pg-config.conf b/src/test/resources/valid-pg-config.conf index 22ffc52..1916d2d 100644 --- a/src/test/resources/valid-pg-config.conf +++ b/src/test/resources/valid-pg-config.conf @@ -1,8 +1,8 @@ # Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/src/test/resources/valid-pg-minimal.conf b/src/test/resources/valid-pg-minimal.conf index 9b73152..df95ec1 100644 --- a/src/test/resources/valid-pg-minimal.conf +++ b/src/test/resources/valid-pg-minimal.conf @@ -1,8 +1,8 @@ # Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/ConfigSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/ConfigSpec.scala index b7104bc..288405f 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/ConfigSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/ConfigSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -69,7 +69,7 @@ class ConfigSpec extends org.specs2.Specification { pool, false ), - Config.Http("0.0.0.0", 8080, Some(10.seconds), None, Config.ThreadPool.Global, noHsts), + Config.Http("0.0.0.0", 8080, Some(10.seconds), None, Config.ThreadPool.Global, noHsts, 100000), true, true, List( @@ -85,7 +85,8 @@ class ConfigSpec extends org.specs2.Specification { None, 42.seconds, true, - Config.License(false) + Config.License(false), + 40 ) val result = Config.serverCommand.parse(input.split(" ").toList).leftMap(_.toString).flatMap(_.read) result must beRight(expected) @@ -98,7 +99,7 @@ class ConfigSpec extends org.specs2.Specification { val expected = Config( Config.StorageConfig.Dummy, - Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Fixed(2), Config.Hsts(true, 365.days)), + Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Fixed(2), Config.Hsts(true, 365.days), 5000), true, false, Nil, @@ -106,7 +107,8 @@ class ConfigSpec extends org.specs2.Specification { None, 1.seconds, false, - Config.License(true) + Config.License(true), + 50 ) val result = Config.serverCommand.parse(input.split(" ").toList).leftMap(_.toString).flatMap(_.read) result must beRight(expected) @@ -127,7 +129,7 @@ class ConfigSpec extends org.specs2.Specification { Config.StorageConfig.ConnectionPool.NoPool(Config.ThreadPool.Fixed(2)), true ), - Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Global, noHsts), + Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Global, noHsts, 3000), true, true, List( @@ -142,7 +144,8 @@ class ConfigSpec extends org.specs2.Specification { Some(UUID.fromString("a71aa7d9-6cde-40f7-84b1-046d65dedf9e")), 10.seconds, true, - Config.License(true) + Config.License(true), + 100 ) val expected = json"""{ @@ -172,7 +175,8 @@ class ConfigSpec extends org.specs2.Specification { "hsts": { "enable": false, "maxAge": "365 days" - } + }, + "maxPayloadSize": 3000 }, "debug" : true, "patchesAllowed" : true, @@ -205,7 +209,8 @@ class ConfigSpec extends org.specs2.Specification { "preTerminationUnhealthy": true, "license": { "accept": true - } + }, + "maxJsonDepth": 100 }""" input.asJson must beEqualTo(expected) @@ -230,7 +235,7 @@ class ConfigSpec extends org.specs2.Specification { pool, true ), - Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Fixed(4), noHsts), + Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Fixed(4), noHsts, 100000), false, false, Nil, @@ -238,7 +243,8 @@ class ConfigSpec extends org.specs2.Specification { None, 1.seconds, false, - Config.License(false) + Config.License(false), + 40 ) val result = Config.serverCommand.parse(input.split(" ").toList).leftMap(_.toString).flatMap(_.read) result must beRight(expected) diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/ServerSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/ServerSpec.scala index b81149d..1ff4ac5 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/ServerSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/ServerSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -14,6 +14,7 @@ import cats.implicits._ import cats.effect.{ContextShift, IO, Resource, Timer} import io.circe.Json +import io.circe.parser.{parse => parseJson} import io.circe.literal._ import org.http4s._ @@ -44,6 +45,8 @@ class ServerSpec extends Specification { Create a new private schema via PUT, return it with proper apikey, hide for no apikey $e3 Create a new public schema via POST, get it from /schemas, delete it $e4 Return an HSTS header when configured to do so $e5 + Return 413 when size of the schema that is sent to /schemas exceeds the max payload size $e6 + Return 413 when size of the schema that is sent to /validation exceeds the max payload size $e7 ${action(System.clearProperty("org.slf4j.simpleLogger.defaultLogLevel"))} """ import ServerSpec._ @@ -178,6 +181,52 @@ class ServerSpec extends Specification { on.and(off) } + + def e6 = { + val schema = SelfDescribingSchema[Json]( + SchemaMap("com.acme", "first", "jsonschema", SchemaVer.Full(1, 0, 0)), + parseJson(s"""{"properties": {"${"a" * 10000}": {"type": "string"}}}""").toOption.get + ).normalize + + val reqs = List( + Request[IO](Method.POST, uri"/".withQueryParam("isPublic", "true")) + .withEntity(schema) + .withHeaders(Header("apikey", InMemory.DummySuperKey.toString)) + ) + + val expected = List( + TestResponse(413, json"""{"message":"The payload is too large"}""") + ) + + val action = for { + responses <- ServerSpec.executeRequests(reqs) + results <- responses.traverse(res => TestResponse.build[Json](res)) + } yield results + + execute(action) must beEqualTo(expected) + } + + def e7 = { + val schema = SelfDescribingSchema[Json]( + SchemaMap("com.acme", "first", "jsonschema", SchemaVer.Full(1, 0, 0)), + parseJson(s"""{"properties": {"${"a" * 10000}": {"type": "string"}}}""").toOption.get + ).normalize + + val reqs = List( + Request[IO](Method.POST, uri"http://localhost:8080/api/validation/validate/schema/jsonschema").withEntity(schema) + ) + + val expected = List( + TestResponse(413, json"""{"message":"The payload is too large"}""") + ) + + val action = for { + responses <- ServerSpec.executeRequests(reqs) + results <- responses.traverse(res => TestResponse.build[Json](res)) + } yield results + + execute(action) must beEqualTo(expected) + } } object ServerSpec { @@ -189,7 +238,7 @@ object ServerSpec { .StorageConfig .ConnectionPool .Hikari(None, None, None, None, Config.ThreadPool.Cached, Config.ThreadPool.Cached) - def httpConfig(hsts: Config.Hsts) = Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Cached, hsts) + def httpConfig(hsts: Config.Hsts) = Config.Http("0.0.0.0", 8080, None, None, Config.ThreadPool.Cached, hsts, 10000) val storageConfig = Config .StorageConfig @@ -215,7 +264,8 @@ object ServerSpec { None, 10.seconds, false, - Config.License(true) + Config.License(true), + 20 ) private def runServer(hsts: Config.Hsts) = Server.buildServer(config(hsts), IO.pure(true)).flatMap(_.resource) diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/SpecHelpers.scala b/src/test/scala/com/snowplowanalytics/iglu/server/SpecHelpers.scala index 7465667..ed177e9 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/SpecHelpers.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/SpecHelpers.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -30,10 +30,12 @@ object SpecHelpers { val ctx = new AuthedContext[IO, Permission] - val now = Instant.ofEpochMilli(1537621061000L) - val superKey = UUID.fromString("4ed2d87a-6da5-48e8-a23b-36a26e61f974") - val readKey = UUID.fromString("1eaad173-1da5-eef8-a2cb-3fa26e61f975") - val readKeyAcme = UUID.fromString("2abad125-0ba1-faf2-b2cc-4fa26e61f971") + val now = Instant.ofEpochMilli(1537621061000L) + val superKey = UUID.fromString("4ed2d87a-6da5-48e8-a23b-36a26e61f974") + val readKey = UUID.fromString("1eaad173-1da5-eef8-a2cb-3fa26e61f975") + val readKeyAcme = UUID.fromString("2abad125-0ba1-faf2-b2cc-4fa26e61f971") + val readKeyAcme2 = UUID.fromString("930fecd3-dc01-4e85-a718-7bf53ba8e4cd") + val nonExistentKey = UUID.fromString("e8f1161f-5fbc-46df-b0ea-063f025e20c9") val schemaZero = json"""{"type": "object", "properties": {"one": {}}}""" val selfSchemaZero = @@ -73,17 +75,68 @@ object SpecHelpers { schemas, Map.empty, Map( - superKey -> Permission.Super, - readKey -> Permission.ReadOnlyAny, - readKeyAcme -> Permission(Vendor(List("com", "acme"), false), Some(SchemaAction.Read), Set.empty) + superKey -> Permission.Super, + readKey -> Permission.ReadOnlyAny, + readKeyAcme -> Permission(Vendor(List("com", "acme"), false), Some(SchemaAction.Read), Set.empty), + readKeyAcme2 -> Permission(Vendor(List("com", "acme2"), false), Some(SchemaAction.Read), Set.empty) ), drafts ) - def toBytes(entity: Json) = - Stream.emits(entity.noSpaces.stripMargin.getBytes).covary[IO] + def toBytes(entity: Json): Stream[IO, Byte] = + toBytes(entity.noSpaces.stripMargin) + + def toBytes(string: String): Stream[IO, Byte] = + Stream.emits(string.getBytes).covary[IO] implicit class SchemaKeyUri(schemaKey: SchemaKey) { def uri: Uri = Uri.unsafeFromString(schemaKey.toPath) } + + def createDeepJsonSchema(depth: Int): String = { + val s = createDeepJson( + depth, + """{"l": { "properties": """, + """, "type": "object", "description": "l"}}""", + s"""{ "stop_$depth": { "type": "string", "maxLength": 100, "description": "stop"}}""" + ) + s""" + { + "$$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "type": "object", + "description": "Deep schema for testing", + "self": { + "vendor": "com.acme", + "format": "jsonschema", + "version": "1-0-0", + "name": "deep" + }, + "properties": $s + } + """ + } + + def createDeepSDJ(depth: Int): String = { + val s = createDeepJson(depth, """{"1":""", "}", s""""depth-$depth"""") + s""" + { + "schema":"iglu:com.acme/deep/jsonschema/1-0-0", + "data": $s + } + """ + } + + def createDeepJsonArray(depth: Int): String = + createDeepJson(depth, "[", "]", s""""depth-$depth"""") + + def createDeepJson( + depth: Int, + p: String, + s: String, + middle: String + ): String = { + val prefix = (1 to depth).map(_ => p).mkString + val suffix = (1 to depth).map(_ => s).mkString + s"""$prefix$middle$suffix""" + } } diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/WebhookSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/WebhookSpec.scala index 7e9ab74..43d7028 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/WebhookSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/WebhookSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/model/PermissionSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/model/PermissionSpec.scala index d87e4a7..81f277b 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/model/PermissionSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/model/PermissionSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/model/SchemaSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/model/SchemaSpec.scala index 83782ee..c992761 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/model/SchemaSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/model/SchemaSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/model/VersionCursorSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/model/VersionCursorSpec.scala index 416cd5f..b8904ad 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/model/VersionCursorSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/model/VersionCursorSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/service/AuthServiceSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/service/AuthServiceSpec.scala index a6b3eb5..6da91b5 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/service/AuthServiceSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/service/AuthServiceSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -18,11 +18,16 @@ import org.http4s.rho.swagger.syntax.io.createRhoMiddleware import java.util.UUID import com.snowplowanalytics.iglu.server.storage.InMemory +import com.snowplowanalytics.iglu.server.SpecHelpers._ class AuthServiceSpec extends org.specs2.Specification with StorageAgnosticSpec with InMemoryStorageSpec { - def getState(request: Request[IO], superApiKey: Option[UUID] = None): IO[(List[Response[IO]], InMemory.State)] = + def getState( + request: Request[IO], + superApiKey: Option[UUID] = None, + maxJsonDepth: Int = 20 + ): IO[(List[Response[IO]], InMemory.State)] = sendRequestsGetState[InMemory.State](storage => - AuthService.asRoutes(storage, superApiKey, SpecHelpers.ctx, createRhoMiddleware()) + AuthService.asRoutes(maxJsonDepth)(storage, superApiKey, SpecHelpers.ctx, createRhoMiddleware()) )(storage => storage.asInstanceOf[InMemory[IO]].ref.get)(List(request)) def is = s2""" @@ -32,6 +37,7 @@ class AuthServiceSpec extends org.specs2.Specification with StorageAgnosticSpec /keygen doesn't authorize without apikey header $e4 /keygen doesn't authorize with unkown apikey in header $e5 /keygen deletes key $e6 + /keygen rejects JSON body that exceeds maximum allowed JSON depth $e7 """ def e1 = { @@ -150,4 +156,31 @@ class AuthServiceSpec extends org.specs2.Specification with StorageAgnosticSpec nokey.and(deletedResponse) } + + def e7 = { + val deepJsonSchema = createDeepJsonSchema(100000) + val deepJsonArray = createDeepJsonArray(1000000) + val wrongApikey = "c99ce0f9-cb5b-4b6f-88f3-2baed041be9b" + + def executeTest(body: String, apikey: String) = { + val req = Request( + Method.POST, + Uri.uri("/keygen"), + headers = Headers.of(Header("apikey", apikey)), + body = toBytes(body) + ) + + val expected = List((422, "The request body was invalid.")) + + val (resp, _) = getState(req).unsafeRunSync() + val result = resp.map(r => (r.status.code, r.bodyText.compile.foldMonoid.unsafeRunSync())) + + result must beEqualTo(expected) + } + + executeTest(deepJsonSchema, SpecHelpers.superKey.toString) + executeTest(deepJsonArray, wrongApikey) + executeTest(deepJsonSchema, SpecHelpers.superKey.toString) + executeTest(deepJsonArray, wrongApikey) + } } diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/service/SchemaServiceSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/service/SchemaServiceSpec.scala index 9fc546c..e565bad 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/service/SchemaServiceSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/service/SchemaServiceSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -24,7 +24,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaMap, SchemaVer, SelfDes import com.snowplowanalytics.iglu.server.codecs.JsonCodecs._ import com.snowplowanalytics.iglu.server.model.{IgluResponse, Schema} import com.snowplowanalytics.iglu.server.model.SchemaSpec.testSchema -import com.snowplowanalytics.iglu.server.SpecHelpers.SchemaKeyUri +import com.snowplowanalytics.iglu.server.SpecHelpers._ import org.http4s.rho.swagger.syntax.io.createRhoMiddleware import java.time.Instant @@ -32,7 +32,7 @@ import java.time.Instant trait SchemaServiceSpecBase extends org.specs2.Specification with StorageAgnosticSpec { def sendRequests(requests: List[Request[IO]], patchesAllowed: Boolean): IO[Response[IO]] = sendRequestsGetResponse(storage => - SchemaService.asRoutes(patchesAllowed, Webhook.WebhookClient(List(), client))( + SchemaService.asRoutes(patchesAllowed, Webhook.WebhookClient(List(), client), 20)( storage, None, SpecHelpers.ctx, @@ -42,7 +42,7 @@ trait SchemaServiceSpecBase extends org.specs2.Specification with StorageAgnosti def getState(requests: List[Request[IO]], patchesAllowed: Boolean): IO[(List[Response[IO]], List[Schema])] = sendRequestsGetState(storage => - SchemaService.asRoutes(patchesAllowed, Webhook.WebhookClient(List(), client))( + SchemaService.asRoutes(patchesAllowed, Webhook.WebhookClient(List(), client), 20)( storage, None, SpecHelpers.ctx, @@ -73,6 +73,7 @@ trait SchemaServiceSpecBase extends org.specs2.Specification with StorageAgnosti PUT request with 'supersededBy' field adds schema with superseding info when patches aren't allowed $e18 PUT request with 'supersedes' field adds schema with superseding info $e19 Prohibits adding schema superseded by version smaller than itself $e20 + Doesn't accept JSON body that exceeds maximum allowed JSON depth $e21 """ def e1 = { @@ -729,6 +730,32 @@ trait SchemaServiceSpecBase extends org.specs2.Specification with StorageAgnosti matchStatement.unsafeRunSync() } + + def e21 = { + val deepJsonSchema = createDeepJsonSchema(100000) + val deepJsonArray = createDeepJsonArray(1000000) + val testSchemaURI = uri"/com.acme/deep/jsonschema/1-0-0" + + val expected = (422, "The request body was invalid.") + + def executeTest(httpMethod: Method, uri: Uri, body: String) = { + val request = Request[IO](httpMethod, uri) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.superKey.toString))) + .withContentType(headers.`Content-Type`(MediaType.application.json)) + .withBodyStream(toBytes(body)) + val response = sendRequests(List(request), false) + .map(r => (r.status.code, r.bodyText.compile.foldMonoid.unsafeRunSync())) + .unsafeRunSync() + response must beEqualTo(expected) + } + + executeTest(Method.PUT, testSchemaURI, deepJsonSchema) + .and(executeTest(Method.PUT, testSchemaURI, deepJsonArray)) + .and(executeTest(Method.POST, uri"", deepJsonSchema)) + .and(executeTest(Method.POST, uri"", deepJsonArray)) + .and(executeTest(Method.POST, uri"/validate/com.acme/deep/jsonschema/1-0-0", deepJsonSchema)) + .and(executeTest(Method.POST, uri"/validate/com.acme/deep/jsonschema/1-0-0", deepJsonArray)) + } } class SchemaServiceSpec extends SchemaServiceSpecBase with InMemoryStorageSpec diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/service/StorageAgnosticSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/service/StorageAgnosticSpec.scala index 45399ba..b2c42cb 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/service/StorageAgnosticSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/service/StorageAgnosticSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/service/SupersedingLogicSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/service/SupersedingLogicSpec.scala index 5b261ac..4c089ff 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/service/SupersedingLogicSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/service/SupersedingLogicSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -26,7 +26,7 @@ import org.http4s.{Header, Headers, Method, Request, Response} trait SupersedingLogicSpecBase extends org.specs2.Specification with StorageAgnosticSpec { private def sendRequests(requests: List[Request[IO]]): IO[(List[Response[IO]], Unit)] = sendRequestsGetState(storage => - SchemaService.asRoutes(patchesAllowed = true, Webhook.WebhookClient(List(), client))( + SchemaService.asRoutes(patchesAllowed = true, Webhook.WebhookClient(List(), client), 20)( storage, None, SpecHelpers.ctx, diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/service/ValidationServiceSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/service/ValidationServiceSpec.scala index a1e4d85..1b2b72a 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/service/ValidationServiceSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/service/ValidationServiceSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -20,18 +20,18 @@ import org.http4s.{Service => _, _} import org.http4s.implicits._ import org.http4s.circe._ import org.http4s.rho.swagger.syntax.io.createRhoMiddleware -import SpecHelpers.toBytes +import SpecHelpers._ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosticSpec with InMemoryStorageSpec { - def sendRequests(requests: List[Request[IO]]) = - sendRequestsGetState(storage => ValidationService.asRoutes(storage, None, SpecHelpers.ctx, createRhoMiddleware()))( - _ => IO.unit - )( + def sendRequests(requests: List[Request[IO]], maxJsonDepth: Int = 20) = + sendRequestsGetState(storage => + ValidationService.asRoutes(maxJsonDepth)(storage, None, SpecHelpers.ctx, createRhoMiddleware()) + )(_ => IO.unit)( requests ) - def sendRequest(req: Request[IO]) = - sendRequests(List(req)).flatMap { case (responses, _) => responses.last.as[Json] }.unsafeRunSync() + def sendRequest(req: Request[IO], maxJsonDepth: Int = 20) = + sendRequests(List(req), maxJsonDepth).flatMap { case (responses, _) => responses.last.as[Json] }.unsafeRunSync() def sendRequestGetText(req: Request[IO]) = sendRequests(List(req)) @@ -46,12 +46,17 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti POST /validate/schema/jsonschema reports malformed request for non-json body $e5 POST /validate/schema/jsonschema reports malformed JSON Schema on unknown properties $e11 POST /validate/schema/jsonschema reports about invalid schema name $e12 + POST /validate/schema/jsonschema returns error when given apikey doesn't exist $e14 + POST /validate/schema/jsonschema performs as expected with any existing key $e15 POST /validate/instance reports invalid instance for the root of an instance $e6 POST /validate/instance reports valid instance $e7 POST /validate/instance returns 404 Schema not found if schema does not exist $e8 POST /validate/instance validates an instance with private schema if apikey is appropriate $e9 POST /validate/instance pretends a private schema does not exist if apikey is inappropriate $e10 + POST /validate/instance pretends a private schema does not exist if apikey doesn't exist $e16 + + POST /validate endpoints don't accept JSON body that exceeds maximum allowed JSON depth $e17 """ def e1 = { @@ -76,6 +81,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti }""" val request = Request[IO](Method.POST, Uri.uri("/validate/schema/jsonschema")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -102,6 +108,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti json"""{"message" : "The schema provided is a valid self-describing iglu:com.acme/nonexistent/jsonschema/1-0-0 schema"}""" val request = Request[IO](Method.POST, uri"/validate/schema/jsonschema") + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -132,6 +139,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti }""" val request = Request[IO](Method.POST, Uri.uri("/validate/schema/jsonschema")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -152,6 +160,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti }""" val request = Request[IO](Method.POST, Uri.uri("/validate/schema/jsonschema")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -163,7 +172,9 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti def e5 = { val expected = "The request body was malformed." val request = - Request[IO](Method.POST, Uri.uri("/validate/schema/jsonschema")).withBodyStream(Stream.emits("non-json".getBytes)) + Request[IO](Method.POST, Uri.uri("/validate/schema/jsonschema")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) + .withBodyStream(Stream.emits("non-json".getBytes)) val response = sendRequestGetText(request) @@ -185,7 +196,9 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti ] }""" - val request = Request[IO](Method.POST, Uri.uri("/validate/instance")).withBodyStream(toBytes(instance)) + val request = Request[IO](Method.POST, Uri.uri("/validate/instance")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) + .withBodyStream(toBytes(instance)) val response = sendRequest(request) response must beEqualTo(expected) @@ -196,7 +209,9 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti json"""{"schema" : "iglu:com.acme/event/jsonschema/1-0-0", "data" : {"one": null} } """ val expected = json"""{"message" : "Instance is valid iglu:com.acme/event/jsonschema/1-0-0"}""" - val request = Request[IO](Method.POST, Uri.uri("/validate/instance")).withBodyStream(toBytes(instance)) + val request = Request[IO](Method.POST, Uri.uri("/validate/instance")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) + .withBodyStream(toBytes(instance)) val response = sendRequest(request) response must beEqualTo(expected) @@ -208,7 +223,9 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti val expected = json"""{"message" : "The schema is not found"}""" - val request = Request[IO](Method.POST, Uri.uri("/validate/instance")).withBodyStream(toBytes(instance)) + val request = Request[IO](Method.POST, Uri.uri("/validate/instance")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) + .withBodyStream(toBytes(instance)) val (responses, _) = sendRequests(List(request)).unsafeRunSync() val response = responses.last @@ -235,7 +252,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti val expected = json"""{"message" : "The schema is not found"}""" val request = Request[IO](Method.POST, Uri.uri("/validate/instance")) - .withHeaders(Headers.of(Header("apikey", "00000000-1111-eeee-0000-eeeeeeeeffff"))) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKeyAcme2.toString))) .withBodyStream(toBytes(instance)) val (responses, _) = sendRequests(List(request)).unsafeRunSync() @@ -278,6 +295,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti }""" val request = Request[IO](Method.POST, uri"/validate/schema/jsonschema") + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -313,6 +331,7 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti }""" val request = Request[IO](Method.POST, uri"/validate/schema/jsonschema") + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) .withContentType(headers.`Content-Type`(MediaType.application.json)) .withBodyStream(toBytes(selfDescribingSchema)) @@ -321,4 +340,108 @@ class ValidationServiceSpec extends org.specs2.Specification with StorageAgnosti response must beEqualTo(expected) } + + def e14 = { + val selfDescribingSchema = json""" + { + "$$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "self": { + "vendor": "com.acme", + "name": "nonexistent", + "format": "jsonschema", + "version": "1-0-0" + }, + "type": "object", + "description": "schema with no issues", + "properties": { } + }""" + + val request = Request[IO](Method.POST, uri"/validate/schema/jsonschema") + .withContentType(headers.`Content-Type`(MediaType.application.json)) + .withBodyStream(toBytes(selfDescribingSchema)) + + val requests = List( + request, + request.withHeaders(Headers.of(Header("apikey", SpecHelpers.nonExistentKey.toString))) + ) + + val responses = sendRequests(requests).map { case (responses, _) => responses.map(_.status.code) }.unsafeRunSync() + + responses must beEqualTo(List(403, 403)) + } + + def e15 = { + val selfDescribingSchema = json""" + { + "$$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "self": { + "vendor": "com.acme", + "name": "nonexistent", + "format": "jsonschema", + "version": "1-0-0" + }, + "type": "object", + "description": "schema with no issues", + "properties": { } + }""" + + val request = Request[IO](Method.POST, uri"/validate/schema/jsonschema") + .withHeaders(Headers.of(Header("apikey", SpecHelpers.superKey.toString))) + .withContentType(headers.`Content-Type`(MediaType.application.json)) + .withBodyStream(toBytes(selfDescribingSchema)) + + val requests = List( + request.withHeaders(Headers.of(Header("apikey", SpecHelpers.superKey.toString))), + request.withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))), + request.withHeaders(Headers.of(Header("apikey", SpecHelpers.readKeyAcme.toString))), + request.withHeaders(Headers.of(Header("apikey", SpecHelpers.readKeyAcme2.toString))) + ) + + val responses = sendRequests(requests).map { case (responses, _) => responses.map(_.status.code) }.unsafeRunSync() + + responses must beEqualTo(List(200, 200, 200, 200)) + } + + def e16 = { + val instance = + json"""{"schema" : "iglu:com.acme/secret/jsonschema/1-0-0", "data" : {} } """ + val expected = + json"""{"message" : "The schema is not found"}""" + val request = Request[IO](Method.POST, Uri.uri("/validate/instance")) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.nonExistentKey.toString))) + .withBodyStream(toBytes(instance)) + + val (responses, _) = sendRequests(List(request)).unsafeRunSync() + val response = responses.last + + val bodyExpectation = response.as[Json].unsafeRunSync() must beEqualTo(expected) + val statusExpectation = response.status.code must beEqualTo(404) + bodyExpectation.and(statusExpectation) + } + + def e17 = { + val deepJsonSchema = createDeepJsonSchema(100000) + val deepJsonArray = createDeepJsonArray(1000000) + val deepSDJ = createDeepSDJ(100000) + val expected = List((422, "The request body was invalid.")) + + def executeTest(uri: Uri, body: String) = { + val request = Request[IO](Method.POST, uri) + .withHeaders(Headers.of(Header("apikey", SpecHelpers.readKey.toString))) + .withContentType(headers.`Content-Type`(MediaType.application.json)) + .withBodyStream(toBytes(body)) + val response = sendRequests(List(request), 40) + .map { + case (responses, _) => + responses.map(r => (r.status.code, r.bodyText.compile.foldMonoid.unsafeRunSync())) + } + .unsafeRunSync() + response must beEqualTo(expected) + } + + executeTest(uri"/validate/schema/jsonschema", deepJsonSchema) + .and(executeTest(uri"/validate/schema/jsonschema", deepJsonArray)) + .and(executeTest(uri"/validate/instance", deepSDJ)) + .and(executeTest(uri"/validate/instance", deepJsonArray)) + } } diff --git a/src/test/scala/com/snowplowanalytics/iglu/server/storage/PostgresSpec.scala b/src/test/scala/com/snowplowanalytics/iglu/server/storage/PostgresSpec.scala index 83feffb..7c4b536 100644 --- a/src/test/scala/com/snowplowanalytics/iglu/server/storage/PostgresSpec.scala +++ b/src/test/scala/com/snowplowanalytics/iglu/server/storage/PostgresSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2014-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */