Merge pull request #604 from uptane/trx-master

Trx master

keyserver/src/main/scala/com/advancedtelematic/tuf/keyserver/http/RootRoleResource.scala

@@ -2,41 +2,41 @@ package com.advancedtelematic.tuf.keyserver.http
 
 import akka.http.scaladsl.marshalling.ToResponseMarshallable
 import akka.http.scaladsl.model.StatusCodes
-import akka.stream.Materializer
 import cats.data.Validated.{Invalid, Valid}
 import com.advancedtelematic.libats.data.ErrorRepresentation
-import com.advancedtelematic.libats.http.UUIDKeyAkka._
-import com.advancedtelematic.libtuf.data.ClientCodecs._
+import com.advancedtelematic.libats.http.UUIDKeyAkka.*
+import com.advancedtelematic.libtuf.data.ClientCodecs.*
 import com.advancedtelematic.libtuf.data.ErrorCodes
-import com.advancedtelematic.libtuf.data.TufCodecs._
-import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, RoleType, _}
-import com.advancedtelematic.libtuf_server.data.Marshalling._
+import com.advancedtelematic.libtuf.data.TufCodecs.*
+import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, RoleType, *}
+import com.advancedtelematic.libtuf_server.data.Marshalling.*
 import com.advancedtelematic.tuf.keyserver.Settings
 import com.advancedtelematic.tuf.keyserver.daemon.DefaultKeyGenerationOp
 import com.advancedtelematic.tuf.keyserver.data.KeyServerDataType.KeyGenRequestStatus
 import com.advancedtelematic.tuf.keyserver.db.SignedRootRoleRepository.MissingSignedRole
 import com.advancedtelematic.tuf.keyserver.db.{KeyGenRequestSupport, SignedRootRoleRepository}
-import com.advancedtelematic.tuf.keyserver.roles._
-import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
-import io.circe._
-import io.circe.syntax._
-import slick.jdbc.MySQLProfile.api._
+import com.advancedtelematic.tuf.keyserver.roles.*
+import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.*
+import io.circe.*
+import io.circe.syntax.*
+import slick.jdbc.MySQLProfile.api.*
 
+import java.time.Instant
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 
 class RootRoleResource()
                       (implicit val db: Database, val ec: ExecutionContext)
   extends KeyGenRequestSupport with Settings {
-  import ClientRootGenRequest._
-  import akka.http.scaladsl.server.Directives._
+  import ClientRootGenRequest.*
+  import akka.http.scaladsl.server.Directives.*
 
   val keyGenerationRequests = new KeyGenerationRequests()
   val signedRootRoles = new SignedRootRoles()
   val rootRoleKeyEdit = new RootRoleKeyEdit()
   val roleSigning = new RoleSigning()
 
-  private def createRootNow(repoId: RepoId, genRequest: ClientRootGenRequest) = {
+    private def createRootNow(repoId: RepoId, genRequest: ClientRootGenRequest) = {
     require(genRequest.threshold > 0, "threshold must be greater than 0")
 
     val keyGenerationOp = DefaultKeyGenerationOp()
@@ -73,8 +73,8 @@ class RootRoleResource()
           case genRequest =>
             createRootLater(repoId, genRequest)
         } ~
-        get {
-          val f = signedRootRoles.findFreshAndPersist(repoId)
+        (get & optionalHeaderValueByName("x-trx-expire-not-before").map(_.map(Instant.parse))) { expireNotBefore =>
+          val f = signedRootRoles.findFreshAndPersist(repoId, expireNotBefore)
           complete(f)
         }
       } ~

keyserver/src/main/scala/com/advancedtelematic/tuf/keyserver/roles/KeyGenerationRequests.scala

@@ -41,13 +41,14 @@ extends KeyRepositorySupport with SignedRootRoleSupport {
   def findLatest(repoId: RepoId): Future[SignedPayload[RootRole]] =
     find(repoId).map(_.content)
 
-  def findFreshAndPersist(repoId: RepoId): Future[SignedPayload[RootRole]] = async {
+  def findFreshAndPersist(repoId: RepoId, expireNotBefore: Option[Instant] = None): Future[SignedPayload[RootRole]] = async {
     val signedRole = await(findAndPersist(repoId))
 
-    if (signedRole.expiresAt.isBefore(Instant.now.plus(1, ChronoUnit.HOURS))) {
+    if (signedRole.expiresAt.isBefore(Instant.now.plus(1, ChronoUnit.HOURS)) || // existing role expires within the hour
+      expireNotBefore.exists(signedRole.expiresAt.isBefore)) { // existing role expiration is earlier than expireNotBefore
       val versionedRole = signedRole.content.signed
       val nextVersion = versionedRole.version + 1
-      val nextExpires = Instant.now.plus(defaultRoleExpire)
+      val nextExpires = List(Option(Instant.now.plus(defaultRoleExpire)), expireNotBefore).flatten.max
       val newRole = versionedRole.copy(expires = nextExpires, version = nextVersion)
 
       val f = signRootRole(newRole)

keyserver/src/test/scala/com/advancedtelematic/tuf/keyserver/http/RootRoleResourceSpec.scala

@@ -2,34 +2,34 @@ package com.advancedtelematic.tuf.keyserver.http
 
 import java.time.{Duration, Instant}
 import java.time.temporal.ChronoUnit
-
 import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.model.headers.RawHeader
 import akka.http.scaladsl.testkit.RouteTest
 import com.advancedtelematic.tuf.util.{KeyTypeSpecSupport, ResourceSpec, RootGenerationSpecSupport, TufKeyserverSpec}
-import io.circe.generic.auto._
-import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
-import cats.syntax.show._
+import io.circe.generic.auto.*
+import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.*
+import cats.syntax.show.*
 import com.advancedtelematic.libats.data.ErrorRepresentation
 import com.advancedtelematic.libtuf.crypt.TufCrypto
-import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TufPrivateKey, _}
+import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TufPrivateKey, *}
 import com.advancedtelematic.tuf.keyserver.data.KeyServerDataType.{Key, KeyGenId, KeyGenRequestStatus, SignedRootRole}
 import io.circe.{Encoder, Json}
 import org.scalatest.Inspectors
 import org.scalatest.concurrent.PatienceConfiguration
-import io.circe.syntax._
-import com.advancedtelematic.libtuf.data.ClientCodecs._
+import io.circe.syntax.*
+import com.advancedtelematic.libtuf.data.ClientCodecs.*
 import com.advancedtelematic.libtuf.data.ClientDataType.{RoleKeys, RootRole}
 import com.advancedtelematic.libtuf.data.ErrorCodes
-import com.advancedtelematic.libtuf.data.TufCodecs._
+import com.advancedtelematic.libtuf.data.TufCodecs.*
 import com.advancedtelematic.tuf.keyserver.db.{KeyGenRequestSupport, KeyRepository, KeyRepositorySupport, SignedRootRoleSupport}
 import eu.timepit.refined.api.Refined
 import org.scalatest.time.{Millis, Seconds, Span}
-import com.advancedtelematic.libtuf.data.RootManipulationOps._
-import cats.syntax.either._
+import com.advancedtelematic.libtuf.data.RootManipulationOps.*
+import cats.syntax.either.*
 import com.advancedtelematic.tuf.keyserver.roles.SignedRootRoles
 
 import scala.concurrent.{ExecutionContext, Future}
-import org.scalatest.OptionValues._
+import org.scalatest.OptionValues.*
 
 class RootRoleResourceSpec extends TufKeyserverSpec
   with ResourceSpec
@@ -753,6 +753,29 @@ class RootRoleResourceSpec extends TufKeyserverSpec
     }
   }
 
+  keyTypeTest("GET returns renewed root if old root would expire before `expires-not-before`") { keyType =>
+    val repoId = RepoId.generate()
+
+    Post(apiUri(s"root/${repoId.show}"), ClientRootGenRequest(1, keyType, forceSync = Some(false))) ~> routes ~> check {
+      status shouldBe StatusCodes.Accepted
+    }
+
+    processKeyGenerationRequest(repoId).futureValue
+
+    val signedRootRoles = new SignedRootRoles()
+
+    signedRootRoles.findFreshAndPersist(repoId).futureValue
+
+    val expiresNotBefore = "2222-01-01T00:00:00Z"
+
+    Get(apiUri(s"root/${repoId.show}")).addHeader(RawHeader("x-trx-expire-not-before", expiresNotBefore)) ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+      val signed = responseAs[SignedPayload[RootRole]].signed
+      signed.version shouldBe 2
+      signed.expires shouldBe Instant.parse(expiresNotBefore)
+    }
+  }
+
   test("GET on versioned root.json returns renewed root if old expired") {
     val repoId = RepoId.generate()
 

libtuf-server/src/main/scala/com/advancedtelematic/libtuf_server/keyserver/KeyserverClient.scala

@@ -3,20 +3,22 @@ package com.advancedtelematic.libtuf_server.keyserver
 import akka.actor.ActorSystem
 import akka.http.scaladsl.model.Uri.Path
 import akka.http.scaladsl.model.Uri.Path.{Empty, Slash}
-import akka.http.scaladsl.model.{StatusCodes, _}
-import cats.syntax.show._
+import akka.http.scaladsl.model.headers.RawHeader
+import akka.http.scaladsl.model.{StatusCodes, *}
+import cats.syntax.show.*
 import com.advancedtelematic.libats.data.ErrorCode
 import com.advancedtelematic.libats.http.Errors.{RawError, RemoteServiceError}
 import com.advancedtelematic.libats.http.ServiceHttpClientSupport
 import com.advancedtelematic.libats.http.tracing.Tracing.ServerRequestTracing
 import com.advancedtelematic.libats.http.tracing.TracingHttpClient
-import com.advancedtelematic.libtuf.data.ClientCodecs._
+import com.advancedtelematic.libtuf.data.ClientCodecs.*
 import com.advancedtelematic.libtuf.data.ClientDataType.{RootRole, TufRole}
-import com.advancedtelematic.libtuf.data.TufCodecs._
-import com.advancedtelematic.libtuf.data.TufDataType.RoleType._
+import com.advancedtelematic.libtuf.data.TufCodecs.*
+import com.advancedtelematic.libtuf.data.TufDataType.RoleType.*
 import com.advancedtelematic.libtuf.data.TufDataType.{KeyId, KeyType, RepoId, SignedPayload, TufKeyPair}
 import io.circe.{Codec, Json}
 
+import java.time.Instant
 import scala.concurrent.Future
 
 object KeyserverClient {
@@ -32,7 +34,7 @@ trait KeyserverClient {
 
   def sign[T : Codec : TufRole](repoId: RepoId, payload: T): Future[SignedPayload[T]]
 
-  def fetchRootRole(repoId: RepoId): Future[SignedPayload[RootRole]]
+  def fetchRootRole(repoId: RepoId, expiresNotBefore: Option[Instant] = None): Future[SignedPayload[RootRole]]
 
   def fetchRootRole(repoId: RepoId, version: Int): Future[SignedPayload[RootRole]]
 
@@ -92,10 +94,15 @@ class KeyserverHttpClient(uri: Uri, httpClient: HttpRequest => Future[HttpRespon
     }
   }
 
-  override def fetchRootRole(repoId: RepoId): Future[SignedPayload[RootRole]] = {
+  override def fetchRootRole(repoId: RepoId, expireNotBefore: Option[Instant]): Future[SignedPayload[RootRole]] = {
     val req = HttpRequest(HttpMethods.GET, uri = apiUri(Path("root") / repoId.show))
 
-    execHttpUnmarshalled[SignedPayload[RootRole]](req).handleErrors {
+    val reqWithParams = expireNotBefore match {
+      case Some(e) => req.withHeaders(RawHeader("x-trx-expire-not-before", e.toString))
+      case None => req
+    }
+
+    execHttpUnmarshalled[SignedPayload[RootRole]](reqWithParams).handleErrors {
       case RemoteServiceError(_, StatusCodes.NotFound, _, _, _, _) =>
         Future.failed(RootRoleNotFound)
       case RemoteServiceError(_, StatusCodes.Locked, _, _, _, _) =>

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/Errors.scala

@@ -18,7 +18,6 @@ object ErrorCodes {
   val RoleChecksumMismatch = ErrorCode("role_checksum_mismatch")
   val NoRepoForNamespace = ErrorCode("no_repo_for_namespace")
   val NoUriForUnmanagedTarget = ErrorCode("no_uri_for_unmanaged_target")
-  val TooManyReposForNamespace = ErrorCode("too_many_repos_for_namespace")
   val DelegationNotFound = ErrorCode("delegations_not_found")
   val DelegationNotDefined = ErrorCode("delegations_not_defined")
   val InvalidTrustedDelegations = ErrorCode("trusted_delegations_invalid")
@@ -31,6 +30,7 @@ object ErrorCodes {
   val InvalidDelegatedTarget = ErrorCode("invalid_delegated_target")
   val MissingRemoteDelegationUri = ErrorCode("missing_remote_delegation_uri")
   val ImmutableFields = ErrorCode("immutable_fields_specified")
+  val SetRootExpireError = ErrorCode("set_root_expire_failed")
 }
 
 object Errors {
@@ -41,7 +41,6 @@ object Errors {
   val NoUriForUnamanagedTarget = RawError(ErrorCodes.NoUriForUnmanagedTarget, StatusCodes.ExpectationFailed, "Cannot redirect to unmanaged resource, no known URI for this resource")
   val RoleChecksumNotProvided = RawError(ErrorCodes.RoleChecksumNotProvided, StatusCodes.PreconditionRequired, "A targets role already exists, but no previous checksum was sent")
   val RoleChecksumMismatch = RawError(ErrorCodes.RoleChecksumMismatch, StatusCodes.PreconditionFailed, "Provided checksum of previous role does not match current checksum")
-  val TooManyReposForNamespace = RawError(ErrorCodes.TooManyReposForNamespace, StatusCodes.BadRequest, "Too many repos found for this namespace. Use the /repo/:repo_id API instead")
 
   def MissingRemoteDelegationUri(repoId: RepoId, delegationName: DelegatedRoleName) =
     RawError(ErrorCodes.MissingRemoteDelegationUri, StatusCodes.PreconditionFailed, s"Role $repoId/$delegationName does not have a remote uri and therefore cannot be refreshed")
@@ -87,4 +86,7 @@ object Errors {
 
   case class RequestCanceledByUpstream(ex: Throwable)
     extends com.advancedtelematic.libats.http.Errors.Error(ErrorCodes.RequestCanceledByUpstream, StatusCodes.BadRequest, ex.getMessage, Some(ex))
+
+  case class SetRootExpire(ex: Throwable)
+    extends com.advancedtelematic.libats.http.Errors.Error(ErrorCodes.SetRootExpireError, StatusCodes.BadGateway, "expire-not-before was set on reposerver roles but not on root.json. Check the attached cause and try again", cause = Option(ex))
 }

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoResource.scala

@@ -252,8 +252,9 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp
 
   def deleteTargetItem(namespace: Namespace, repoId: RepoId, filename: TargetFilename): Route = complete {
     for {
-      _ <- targetStore.delete(repoId, filename)
+      targetItem <- targetStore.targetItemRepo.findByFilename(repoId, filename)
       _ <- targetRoleEdit.deleteTargetItem(repoId, filename)
+      _ <- targetStore.delete(targetItem)
       _ <- tufTargetsPublisher.targetsMetaModified(namespace)
     } yield StatusCodes.NoContent
   }
@@ -465,8 +466,16 @@ class RepoResource(keyserverClient: KeyserverClient, namespaceValidation: Namesp
       pathPrefix("targets") {
         path("expire" / "not-before") {
           (put & entity(as[ExpireNotBeforeRequest])) { req =>
-            val f = repoNamespaceRepo.setExpiresNotBefore(repoId, Option(req.expireAt))
-            complete(f.map(_ => StatusCodes.NoContent))
+            onComplete(repoNamespaceRepo.setExpiresNotBefore(repoId, Option(req.expireAt))) { _ =>
+              val f = keyserverClient.fetchRootRole(repoId, Option(req.expireAt)).map { _ =>
+                StatusCodes.NoContent
+              }.recoverWith {
+                case err =>
+                  FastFuture.failed(Errors.SetRootExpire(err))
+              }
+
+              complete(f)
+            }
           }
         } ~
         path(TargetFilenamePath) { filename =>

reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoResourceSpec.scala

@@ -1314,6 +1314,25 @@ class RepoResourceSpec extends TufReposerverSpec with RepoResourceSpecUtil
     }
   }
 
+  test("expire-not-before field is used to get upstream root.json") {
+    withRandomNamepace { implicit ns =>
+      createRepo()
+
+      val notBefore = "2222-01-01T00:00:00Z"
+      val notBeforeIs = Instant.parse(notBefore)
+
+      Put(apiUri(s"user_repo/targets/expire/not-before"), ExpireNotBeforeRequest(notBeforeIs)).namespaced ~> routes ~> check {
+        status shouldBe StatusCodes.NoContent
+      }
+
+      Get(apiUri(s"user_repo/root.json")).namespaced ~> routes ~> check {
+        status shouldBe StatusCodes.OK
+        val targetsRole = responseAs[SignedPayload[RootRole]].signed
+        targetsRole.expires shouldBe notBeforeIs
+      }
+    }
+  }
+
   test("targets.json expire date is set according to expires-not-before when adding a target") {
     withRandomNamepace { implicit ns =>
       createRepo()
@@ -1402,7 +1421,6 @@ class RepoResourceSpec extends TufReposerverSpec with RepoResourceSpecUtil
       Get(apiUri(s"user_repo/target_items")).namespaced ~> routes ~> check {
         status shouldBe StatusCodes.OK
         val paged = responseAs[PaginationResult[ClientTargetItem]]
-        println(paged)
         paged.total shouldBe 100
         paged.offset shouldBe 0 // default
         paged.limit shouldBe 50 // default

reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/util/ResourceSpec.scala

@@ -129,7 +129,15 @@ class FakeKeyserverClient extends KeyserverClient {
     }
   }
 
-  override def fetchRootRole(repoId: RepoId): Future[SignedPayload[RootRole]] =
+private def refreshAndSaveRoot(repoId: RepoId, role: SignedPayload[RootRole], expireNotBefore: Instant): Future[SignedPayload[RootRole]] = {
+    val newRole = role.signed.copy(expires = expireNotBefore)
+    sign(repoId, newRole).map { signedPayload =>
+      rootRoles.put(repoId, signedPayload)
+      signedPayload
+    }
+  }
+
+  override def fetchRootRole(repoId: RepoId, expireNotBefore: Option[Instant] = None): Future[SignedPayload[RootRole]] =
     Future.fromTry {
       Try {
         if(pendingRequests.asScala.getOrElse(repoId, false))
@@ -139,6 +147,11 @@ class FakeKeyserverClient extends KeyserverClient {
       }.recover {
         case _: NoSuchElementException => throw RootRoleNotFound
       }
+    }.flatMap { existing =>
+      expireNotBefore match {
+        case Some(e) => refreshAndSaveRoot(repoId, existing, e)
+        case None => FastFuture.successful(existing)
+      }
     }
 
   override def fetchUnsignedRoot(repoId: RepoId): Future[RootRole] =