Merge pull request #33 from ing-bank/feature/get-user-groups

replace userAssumedGroup with Set(UserGroup)

.travis.yml

@@ -11,7 +11,7 @@ services:
 language: scala
 
 scala:
-- 2.12.7
+- 2.12.8
 
 env:
   global:

build.sbt

@@ -3,9 +3,9 @@ import com.typesafe.sbt.packager.docker.ExecCmd
 import scalariform.formatter.preferences._
 
 name := "airlock"
-version := "0.0.8"
+version := "0.1.0"
 
-scalaVersion := "2.12.7"
+scalaVersion := "2.12.8"
 
 scalacOptions += "-unchecked"
 scalacOptions += "-deprecation"
@@ -19,7 +19,7 @@ scalacOptions += "-Xfatal-warnings"
 updateOptions := updateOptions.value.withCachedResolution(cachedResoluton = true)
 
 val akkaVersion       = "10.1.5"
-val akkaStreamVersion = "2.5.17"
+val akkaStreamVersion = "2.5.19"
 
 libraryDependencies ++= Seq(
     "com.typesafe.scala-logging"   %% "scala-logging"          % "3.9.0",

docker-compose.yml

@@ -31,7 +31,7 @@ services:
     - "6080:6080"
 
   airlock-sts:
-    image: wbaa/airlock-sts:0.0.10
+    image: wbaa/airlock-sts:0.1.0
     environment:
     - STS_HOST=0.0.0.0
     - STS_PORT=12345
@@ -47,7 +47,7 @@ services:
     - "mariadb"
 
   keycloak:
-    image: wbaa/airlock-dev-keycloak:0.0.2
+    image: wbaa/airlock-dev-keycloak:0.0.3
     environment:
     - KEYCLOAK_USER=admin
     - KEYCLOAK_PASSWORD=admin
@@ -60,7 +60,7 @@ services:
     - "21000:21000"
 
   mariadb:
-    image: wbaa/airlock-dev-mariadb:0.0.1
+    image: wbaa/airlock-dev-mariadb:0.0.2
     environment:
     - MYSQL_ROOT_PASSWORD=admin
     ports:

src/it/scala/com/ing/wbaa/airlock/proxy/AirlockS3ProxyItTest.scala

@@ -6,7 +6,7 @@ import akka.http.scaladsl.model.Uri.{Authority, Host}
 import akka.stream.ActorMaterializer
 import com.amazonaws.auth.BasicSessionCredentials
 import com.amazonaws.services.securitytoken.AWSSecurityTokenService
-import com.amazonaws.services.securitytoken.model.{AssumeRoleWithWebIdentityRequest, GetSessionTokenRequest}
+import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest
 import com.ing.wbaa.airlock.proxy.config.{AtlasSettings, HttpSettings, StorageS3Settings, StsSettings}
 import com.ing.wbaa.airlock.proxy.data._
 import com.ing.wbaa.airlock.proxy.handler.RequestHandlerS3
@@ -89,27 +89,5 @@ class AirlockS3ProxyItTest extends AsyncWordSpec with DiagrammedAssertions
         }
       }
     }
-
-    "connect to ceph with credentials from STS (AssumeRole)" in withSdkToMockProxy { (stsSdk, s3ProxyAuthority) =>
-      retrieveKeycloackToken(validKeycloakCredentials).map { keycloakToken =>
-        val cred = stsSdk.assumeRoleWithWebIdentity(new AssumeRoleWithWebIdentityRequest()
-          .withRoleArn("arn:aws:iam::123456789012:role/user")
-          .withProviderId("provider")
-          .withRoleSessionName("sessionName")
-          .withWebIdentityToken(keycloakToken.access_token))
-          .getCredentials
-
-        val sessionCredentials = new BasicSessionCredentials(
-          cred.getAccessKeyId,
-          cred.getSecretAccessKey,
-          cred.getSessionToken
-        )
-
-        val s3Sdk = getAmazonS3("S3SignerType", s3ProxyAuthority, sessionCredentials)
-        withBucket(s3Sdk) { testBucket =>
-          assert(s3Sdk.listBuckets().asScala.toList.map(_.getName).contains(testBucket))
-        }
-      }
-    }
   }
 }

src/it/scala/com/ing/wbaa/airlock/proxy/handler/RadosGatewayHandlerItTest.scala

@@ -23,7 +23,7 @@ class RadosGatewayHandlerItTest extends WordSpec with DiagrammedAssertions with
 
   def testUser: User = User(UserRawJson(
     Random.alphanumeric.take(32).mkString,
-    None,
+    Set.empty[String],
     Random.alphanumeric.take(32).mkString,
     Random.alphanumeric.take(32).mkString
   ))
@@ -97,7 +97,7 @@ class RadosGatewayHandlerItTest extends WordSpec with DiagrammedAssertions with
     }
 
     "list all buckets" in {
-      assert(listAllBuckets.head == "home")
+      assert(listAllBuckets.contains("demobucket"))
     }
   }
 }

src/it/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3ItTest.scala

@@ -44,7 +44,7 @@ class RequestHandlerS3ItTest extends AsyncWordSpec with DiagrammedAssertions wit
       override val atlasSettings: AtlasSettings = new AtlasSettings(system.settings.config)
 
       override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] =
-        Future(Some(User(UserRawJson("userId", Some("group"), "accesskey", "secretkey"))))
+        Future(Some(User(UserRawJson("userId", Set("group"), "accesskey", "secretkey"))))
 
       def createLineageFromRequest(httpRequest: HttpRequest, userSTS: User, clientIPAddress: RemoteAddress): Future[LineagePostGuidResponse] = Future.failed(LineageProviderAtlasException("Create lineage failed"))
     }

src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthenticationProviderSTSItTest.scala

@@ -2,7 +2,7 @@ package com.ing.wbaa.airlock.proxy.provider
 
 import akka.actor.ActorSystem
 import akka.stream.ActorMaterializer
-import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest
+import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest
 import com.ing.wbaa.airlock.proxy.config.StsSettings
 import com.ing.wbaa.airlock.proxy.data._
 import com.ing.wbaa.testkit.awssdk.StsSdkHelpers
@@ -24,19 +24,16 @@ class AuthenticationProviderSTSItTest extends AsyncWordSpec with DiagrammedAsser
 
   private val validKeycloakCredentials = Map(
     "grant_type" -> "password",
-    "username" -> "userone",
+    "username" -> "testuser",
     "password" -> "password",
     "client_id" -> "sts-airlock"
   )
 
   def withAwsCredentialsValidInSTS(testCode: AwsRequestCredential => Future[Assertion]): Future[Assertion] = {
     val stsSdk = getAmazonSTSSdk(StsSettings(testSystem).stsBaseUri)
     retrieveKeycloackToken(validKeycloakCredentials).flatMap { keycloakToken =>
-      val cred = stsSdk.assumeRoleWithWebIdentity(new AssumeRoleWithWebIdentityRequest()
-        .withRoleArn("arn:aws:iam::123456789012:role/user")
-        .withProviderId("provider")
-        .withRoleSessionName("sessionName")
-        .withWebIdentityToken(keycloakToken.access_token))
+      val cred = stsSdk.getSessionToken(new GetSessionTokenRequest()
+        .withTokenCode(keycloakToken.access_token))
         .getCredentials
 
       testCode(AwsRequestCredential(AwsAccessKey(cred.getAccessKeyId), Some(AwsSessionToken(cred.getSessionToken))))
@@ -48,8 +45,10 @@ class AuthenticationProviderSTSItTest extends AsyncWordSpec with DiagrammedAsser
       "succeeds for valid credentials" in {
         withAwsCredentialsValidInSTS { awsCredential =>
           areCredentialsActive(awsCredential).map { userResult =>
-            assert(userResult.map(_.userName).contains(UserName("userone")))
-            assert(userResult.flatMap(_.userAssumedGroup).contains(UserAssumedGroup("user")))
+            assert(userResult.map(_.userName).contains(UserName("testuser")))
+            assert(userResult.map(_.userGroups).head.contains(UserGroup("testgroup")))
+            assert(userResult.map(_.userGroups).head.contains(UserGroup("group3")))
+            assert(userResult.map(_.userGroups).head.size == 2)
             assert(userResult.exists(_.accessKey.value.length == 32))
             assert(userResult.exists(_.secretKey.value.length == 32))
           }

src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRangerItTest.scala

@@ -22,7 +22,7 @@ class AuthorizationProviderRangerItTest extends AsyncWordSpec with DiagrammedAss
 
   val user = User(
     UserName("testuser"),
-    Some(UserAssumedGroup("testgroup")),
+    Set(UserGroup("testgroup")),
     AwsAccessKey("accesskey"),
     AwsSecretKey("secretkey")
   )
@@ -70,15 +70,15 @@ class AuthorizationProviderRangerItTest extends AsyncWordSpec with DiagrammedAss
 
       "doesn't authorize for unauthorized user and group" in withAuthorizationProviderRanger() { apr =>
         assert(!apr.isUserAuthorizedForRequest(s3Request, user.copy(
-          userName = UserName("unauthorized"), userAssumedGroup = Some(UserAssumedGroup("unauthorized"))), clientIPAddress, headerIPs))
+          userName = UserName("unauthorized"), userGroups = Set(UserGroup("unauthorized"))), clientIPAddress, headerIPs))
       }
 
       "does authorize for unauthorized user but authorized group" in withAuthorizationProviderRanger() { apr =>
         assert(apr.isUserAuthorizedForRequest(s3Request, user.copy(userName = UserName("unauthorized")), clientIPAddress, headerIPs))
       }
 
       "does authorize for authorized user but unauthorized group" in withAuthorizationProviderRanger() { apr =>
-        assert(apr.isUserAuthorizedForRequest(s3Request, user.copy(userAssumedGroup = Some(UserAssumedGroup("unauthorized"))), clientIPAddress, headerIPs))
+        assert(apr.isUserAuthorizedForRequest(s3Request, user.copy(userGroups = Set(UserGroup("unauthorized"))), clientIPAddress, headerIPs))
       }
 
       "authorize allow-list-buckets with default settings" in withAuthorizationProviderRanger() { apr =>

src/it/scala/com/ing/wbaa/airlock/proxy/provider/LineageProviderAtlasItTest.scala

@@ -30,7 +30,7 @@ class LineageProviderAtlasItTest extends AsyncWordSpec with DiagrammedAssertions
   }
   val remoteClientIP = RemoteAddress(InetAddress.getByName("127.0.0.1"))
 
-  val userSTS = User(UserName("fakeUser"), None, AwsAccessKey("a"), AwsSecretKey("k"))
+  val userSTS = User(UserName("fakeUser"), Set.empty[UserGroup], AwsAccessKey("a"), AwsSecretKey("k"))
 
   def withLineageProviderAtlas(atlasTestSettings: AtlasSettings = AtlasSettings(testSystem))(testCode: LineageProviderAtlas => Future[Assertion]) =
     testCode(new LineageProviderAtlas {

src/main/scala/com/ing/wbaa/airlock/proxy/data/User.scala

@@ -1,25 +1,25 @@
 package com.ing.wbaa.airlock.proxy.data
 
 case class UserName(value: String) extends AnyVal
-case class UserAssumedGroup(value: String) extends AnyVal
+case class UserGroup(value: String) extends AnyVal
 
 case class UserRawJson(
     userName: String,
-    userAssumedGroup: Option[String],
+    userGroups: Set[String],
     accessKey: String,
     secretKey: String)
 
 case class User(
     userName: UserName,
-    userAssumedGroup: Option[UserAssumedGroup],
+    userGroups: Set[UserGroup],
     accessKey: AwsAccessKey,
     secretKey: AwsSecretKey)
 
 object User {
   def apply(userRawJson: UserRawJson): User =
     User(
       UserName(userRawJson.userName),
-      userRawJson.userAssumedGroup.map(UserAssumedGroup),
+      userRawJson.userGroups.map(UserGroup),
       AwsAccessKey(userRawJson.accessKey),
       AwsSecretKey(userRawJson.secretKey)
     )

src/main/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRanger.scala

@@ -55,7 +55,7 @@ trait AuthorizationProviderRanger extends LazyLogging {
         rangerResource,
         request.accessType.rangerName,
         user.userName.value + rangerSettings.userDomainPostfix,
-        user.userAssumedGroup.map(_.value).toSet[String].asJava
+        user.userGroups.map(_.value).asJava
       )
       // We're using the original client's IP address for verification in Ranger. Ranger seems to use the
       // RemoteIPAddress variable for this. For the header IPs we use the ForwardedAddresses: this is not

src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceSpec.scala

@@ -27,7 +27,7 @@ class ProxyServiceSpec extends FlatSpec with DiagrammedAssertions with Scalatest
       Future(HttpResponse(status = StatusCodes.OK))
 
     override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] = Future(
-      Some(User(UserName("okUser"), Some(UserAssumedGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey")))
+      Some(User(UserName("okUser"), Set(UserGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey")))
     )
 
     override def isUserAuthorizedForRequest(request: S3Request, user: User, clientIPAddress: RemoteAddress, headerIPs: HeaderIPs): Boolean = true

src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceWithListAllBucketsSpec.scala

@@ -27,7 +27,7 @@ class ProxyServiceWithListAllBucketsSpec extends FlatSpec with DiagrammedAsserti
       Future(HttpResponse(status = StatusCodes.OK))
 
     override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] = Future(
-      Some(User(UserName("okUser"), Some(UserAssumedGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey")))
+      Some(User(UserName("okUser"), Set(UserGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey")))
     )
 
     override def isUserAuthorizedForRequest(request: S3Request, user: User, clientIPAddress: RemoteAddress, headerIPs: HeaderIPs): Boolean = true

src/test/scala/com/ing/wbaa/airlock/proxy/data/JsonProtocolsSpec.scala

@@ -12,23 +12,24 @@ class JsonProtocolsSpec extends WordSpec with DiagrammedAssertions with JsonProt
         val jsonString =
           """{
             | "userName": "user",
-            | "userAssumedGroup": "group1",
+            | "userGroups": ["group1"],
             | "accessKey": "accesskey",
             | "secretKey": "secretkey"
             |}""".stripMargin
         val result = jsonString.parseJson.convertTo[UserRawJson]
-        assert(result == UserRawJson("user", Some("group1"), "accesskey", "secretkey"))
+        assert(result == UserRawJson("user", Set("group1"), "accesskey", "secretkey"))
       }
 
       "does not have a group" in {
         val jsonString =
           """{
             | "userName": "user",
+            | "userGroups": [],
             | "accessKey": "accesskey",
             | "secretKey": "secretkey"
             |}""".stripMargin
         val result = jsonString.parseJson.convertTo[UserRawJson]
-        assert(result == UserRawJson("user", None, "accesskey", "secretkey"))
+        assert(result == UserRawJson("user", Set.empty[String], "accesskey", "secretkey"))
       }
 
       "fail when fields are missing" in {

src/test/scala/com/ing/wbaa/airlock/proxy/data/UserSpec.scala

@@ -6,8 +6,8 @@ class UserSpec extends WordSpec with DiagrammedAssertions {
   "UserRawJson" should {
     "convert to User in apply of UserRawJson" in {
       assert(
-        User(UserRawJson("u", Some("g"), "a", "s")) ==
-          User(UserName("u"), Some(UserAssumedGroup("g")), AwsAccessKey("a"), AwsSecretKey("s"))
+        User(UserRawJson("u", Set("g"), "a", "s")) ==
+          User(UserName("u"), Set(UserGroup("g")), AwsAccessKey("a"), AwsSecretKey("s"))
       )
     }
   }

src/test/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3Spec.scala

@@ -34,7 +34,7 @@ class RequestHandlerS3Spec extends AsyncWordSpec with DiagrammedAssertions with
         val initialNumFiredRequests = numFiredRequests
         executeRequest(
           HttpRequest(),
-          User(UserRawJson("u", None, "a", "s"))
+          User(UserRawJson("u", Set.empty[String], "a", "s"))
         ).map(_ => assert(numFiredRequests - initialNumFiredRequests == 2))
       }
     }