From 1d7e74e402baee8698d57cc9b468ba3820c91d51 Mon Sep 17 00:00:00 2001 From: Krzysztof Zmij Date: Wed, 2 Jan 2019 13:14:08 +0100 Subject: [PATCH 1/2] change userAssumedGroup to userGroups --- docker-compose.yml | 6 ++--- .../airlock/proxy/AirlockS3ProxyItTest.scala | 24 +------------------ .../handler/RadosGatewayHandlerItTest.scala | 4 ++-- .../handler/RequestHandlerS3ItTest.scala | 2 +- .../AuthenticationProviderSTSItTest.scala | 17 +++++++------ .../AuthorizationProviderRangerItTest.scala | 6 ++--- .../provider/LineageProviderAtlasItTest.scala | 2 +- .../ing/wbaa/airlock/proxy/data/User.scala | 8 +++---- .../AuthorizationProviderRanger.scala | 2 +- .../airlock/proxy/api/ProxyServiceSpec.scala | 2 +- .../ProxyServiceWithListAllBucketsSpec.scala | 2 +- .../proxy/data/JsonProtocolsSpec.scala | 7 +++--- .../wbaa/airlock/proxy/data/UserSpec.scala | 4 ++-- .../proxy/handler/RequestHandlerS3Spec.scala | 2 +- 14 files changed, 33 insertions(+), 55 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f0515c8..d06967db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: - "6080:6080" airlock-sts: - image: wbaa/airlock-sts:0.0.10 + image: wbaa/airlock-sts:feature_get-user-groups environment: - STS_HOST=0.0.0.0 - STS_PORT=12345 @@ -47,7 +47,7 @@ services: - "mariadb" keycloak: - image: wbaa/airlock-dev-keycloak:0.0.2 + image: wbaa/airlock-dev-keycloak:0.0.3 environment: - KEYCLOAK_USER=admin - KEYCLOAK_PASSWORD=admin @@ -60,7 +60,7 @@ services: - "21000:21000" mariadb: - image: wbaa/airlock-dev-mariadb:0.0.1 + image: wbaa/airlock-dev-mariadb:0.0.2 environment: - MYSQL_ROOT_PASSWORD=admin ports: diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/AirlockS3ProxyItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/AirlockS3ProxyItTest.scala index 6f7a968f..22a034ef 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/AirlockS3ProxyItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/AirlockS3ProxyItTest.scala @@ -6,7 +6,7 @@ import akka.http.scaladsl.model.Uri.{Authority, Host} import akka.stream.ActorMaterializer import com.amazonaws.auth.BasicSessionCredentials import com.amazonaws.services.securitytoken.AWSSecurityTokenService -import com.amazonaws.services.securitytoken.model.{AssumeRoleWithWebIdentityRequest, GetSessionTokenRequest} +import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest import com.ing.wbaa.airlock.proxy.config.{AtlasSettings, HttpSettings, StorageS3Settings, StsSettings} import com.ing.wbaa.airlock.proxy.data._ import com.ing.wbaa.airlock.proxy.handler.RequestHandlerS3 @@ -89,27 +89,5 @@ class AirlockS3ProxyItTest extends AsyncWordSpec with DiagrammedAssertions } } } - - "connect to ceph with credentials from STS (AssumeRole)" in withSdkToMockProxy { (stsSdk, s3ProxyAuthority) => - retrieveKeycloackToken(validKeycloakCredentials).map { keycloakToken => - val cred = stsSdk.assumeRoleWithWebIdentity(new AssumeRoleWithWebIdentityRequest() - .withRoleArn("arn:aws:iam::123456789012:role/user") - .withProviderId("provider") - .withRoleSessionName("sessionName") - .withWebIdentityToken(keycloakToken.access_token)) - .getCredentials - - val sessionCredentials = new BasicSessionCredentials( - cred.getAccessKeyId, - cred.getSecretAccessKey, - cred.getSessionToken - ) - - val s3Sdk = getAmazonS3("S3SignerType", s3ProxyAuthority, sessionCredentials) - withBucket(s3Sdk) { testBucket => - assert(s3Sdk.listBuckets().asScala.toList.map(_.getName).contains(testBucket)) - } - } - } } } diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RadosGatewayHandlerItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RadosGatewayHandlerItTest.scala index 4c711b8c..280305df 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RadosGatewayHandlerItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RadosGatewayHandlerItTest.scala @@ -23,7 +23,7 @@ class RadosGatewayHandlerItTest extends WordSpec with DiagrammedAssertions with def testUser: User = User(UserRawJson( Random.alphanumeric.take(32).mkString, - None, + Set.empty[String], Random.alphanumeric.take(32).mkString, Random.alphanumeric.take(32).mkString )) @@ -97,7 +97,7 @@ class RadosGatewayHandlerItTest extends WordSpec with DiagrammedAssertions with } "list all buckets" in { - assert(listAllBuckets.head == "home") + assert(listAllBuckets.contains("demobucket")) } } } diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3ItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3ItTest.scala index 571ac8b0..efd58419 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3ItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3ItTest.scala @@ -44,7 +44,7 @@ class RequestHandlerS3ItTest extends AsyncWordSpec with DiagrammedAssertions wit override val atlasSettings: AtlasSettings = new AtlasSettings(system.settings.config) override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] = - Future(Some(User(UserRawJson("userId", Some("group"), "accesskey", "secretkey")))) + Future(Some(User(UserRawJson("userId", Set("group"), "accesskey", "secretkey")))) def createLineageFromRequest(httpRequest: HttpRequest, userSTS: User, clientIPAddress: RemoteAddress): Future[LineagePostGuidResponse] = Future.failed(LineageProviderAtlasException("Create lineage failed")) } diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthenticationProviderSTSItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthenticationProviderSTSItTest.scala index 301ceb5d..d272c6bb 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthenticationProviderSTSItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthenticationProviderSTSItTest.scala @@ -2,7 +2,7 @@ package com.ing.wbaa.airlock.proxy.provider import akka.actor.ActorSystem import akka.stream.ActorMaterializer -import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest +import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest import com.ing.wbaa.airlock.proxy.config.StsSettings import com.ing.wbaa.airlock.proxy.data._ import com.ing.wbaa.testkit.awssdk.StsSdkHelpers @@ -24,7 +24,7 @@ class AuthenticationProviderSTSItTest extends AsyncWordSpec with DiagrammedAsser private val validKeycloakCredentials = Map( "grant_type" -> "password", - "username" -> "userone", + "username" -> "testuser", "password" -> "password", "client_id" -> "sts-airlock" ) @@ -32,11 +32,8 @@ class AuthenticationProviderSTSItTest extends AsyncWordSpec with DiagrammedAsser def withAwsCredentialsValidInSTS(testCode: AwsRequestCredential => Future[Assertion]): Future[Assertion] = { val stsSdk = getAmazonSTSSdk(StsSettings(testSystem).stsBaseUri) retrieveKeycloackToken(validKeycloakCredentials).flatMap { keycloakToken => - val cred = stsSdk.assumeRoleWithWebIdentity(new AssumeRoleWithWebIdentityRequest() - .withRoleArn("arn:aws:iam::123456789012:role/user") - .withProviderId("provider") - .withRoleSessionName("sessionName") - .withWebIdentityToken(keycloakToken.access_token)) + val cred = stsSdk.getSessionToken(new GetSessionTokenRequest() + .withTokenCode(keycloakToken.access_token)) .getCredentials testCode(AwsRequestCredential(AwsAccessKey(cred.getAccessKeyId), Some(AwsSessionToken(cred.getSessionToken)))) @@ -48,8 +45,10 @@ class AuthenticationProviderSTSItTest extends AsyncWordSpec with DiagrammedAsser "succeeds for valid credentials" in { withAwsCredentialsValidInSTS { awsCredential => areCredentialsActive(awsCredential).map { userResult => - assert(userResult.map(_.userName).contains(UserName("userone"))) - assert(userResult.flatMap(_.userAssumedGroup).contains(UserAssumedGroup("user"))) + assert(userResult.map(_.userName).contains(UserName("testuser"))) + assert(userResult.map(_.userGroups).head.contains(UserGroup("testgroup"))) + assert(userResult.map(_.userGroups).head.contains(UserGroup("group3"))) + assert(userResult.map(_.userGroups).head.size == 2) assert(userResult.exists(_.accessKey.value.length == 32)) assert(userResult.exists(_.secretKey.value.length == 32)) } diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRangerItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRangerItTest.scala index 4e5da73c..cb020a93 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRangerItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRangerItTest.scala @@ -22,7 +22,7 @@ class AuthorizationProviderRangerItTest extends AsyncWordSpec with DiagrammedAss val user = User( UserName("testuser"), - Some(UserAssumedGroup("testgroup")), + Set(UserGroup("testgroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey") ) @@ -70,7 +70,7 @@ class AuthorizationProviderRangerItTest extends AsyncWordSpec with DiagrammedAss "doesn't authorize for unauthorized user and group" in withAuthorizationProviderRanger() { apr => assert(!apr.isUserAuthorizedForRequest(s3Request, user.copy( - userName = UserName("unauthorized"), userAssumedGroup = Some(UserAssumedGroup("unauthorized"))), clientIPAddress, headerIPs)) + userName = UserName("unauthorized"), userGroups = Set(UserGroup("unauthorized"))), clientIPAddress, headerIPs)) } "does authorize for unauthorized user but authorized group" in withAuthorizationProviderRanger() { apr => @@ -78,7 +78,7 @@ class AuthorizationProviderRangerItTest extends AsyncWordSpec with DiagrammedAss } "does authorize for authorized user but unauthorized group" in withAuthorizationProviderRanger() { apr => - assert(apr.isUserAuthorizedForRequest(s3Request, user.copy(userAssumedGroup = Some(UserAssumedGroup("unauthorized"))), clientIPAddress, headerIPs)) + assert(apr.isUserAuthorizedForRequest(s3Request, user.copy(userGroups = Set(UserGroup("unauthorized"))), clientIPAddress, headerIPs)) } "authorize allow-list-buckets with default settings" in withAuthorizationProviderRanger() { apr => diff --git a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/LineageProviderAtlasItTest.scala b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/LineageProviderAtlasItTest.scala index dd95a443..ec803407 100644 --- a/src/it/scala/com/ing/wbaa/airlock/proxy/provider/LineageProviderAtlasItTest.scala +++ b/src/it/scala/com/ing/wbaa/airlock/proxy/provider/LineageProviderAtlasItTest.scala @@ -30,7 +30,7 @@ class LineageProviderAtlasItTest extends AsyncWordSpec with DiagrammedAssertions } val remoteClientIP = RemoteAddress(InetAddress.getByName("127.0.0.1")) - val userSTS = User(UserName("fakeUser"), None, AwsAccessKey("a"), AwsSecretKey("k")) + val userSTS = User(UserName("fakeUser"), Set.empty[UserGroup], AwsAccessKey("a"), AwsSecretKey("k")) def withLineageProviderAtlas(atlasTestSettings: AtlasSettings = AtlasSettings(testSystem))(testCode: LineageProviderAtlas => Future[Assertion]) = testCode(new LineageProviderAtlas { diff --git a/src/main/scala/com/ing/wbaa/airlock/proxy/data/User.scala b/src/main/scala/com/ing/wbaa/airlock/proxy/data/User.scala index 6ab15c66..f3f5d4fe 100644 --- a/src/main/scala/com/ing/wbaa/airlock/proxy/data/User.scala +++ b/src/main/scala/com/ing/wbaa/airlock/proxy/data/User.scala @@ -1,17 +1,17 @@ package com.ing.wbaa.airlock.proxy.data case class UserName(value: String) extends AnyVal -case class UserAssumedGroup(value: String) extends AnyVal +case class UserGroup(value: String) extends AnyVal case class UserRawJson( userName: String, - userAssumedGroup: Option[String], + userGroups: Set[String], accessKey: String, secretKey: String) case class User( userName: UserName, - userAssumedGroup: Option[UserAssumedGroup], + userGroups: Set[UserGroup], accessKey: AwsAccessKey, secretKey: AwsSecretKey) @@ -19,7 +19,7 @@ object User { def apply(userRawJson: UserRawJson): User = User( UserName(userRawJson.userName), - userRawJson.userAssumedGroup.map(UserAssumedGroup), + userRawJson.userGroups.map(UserGroup), AwsAccessKey(userRawJson.accessKey), AwsSecretKey(userRawJson.secretKey) ) diff --git a/src/main/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRanger.scala b/src/main/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRanger.scala index bba6e441..dcbd8a18 100644 --- a/src/main/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRanger.scala +++ b/src/main/scala/com/ing/wbaa/airlock/proxy/provider/AuthorizationProviderRanger.scala @@ -55,7 +55,7 @@ trait AuthorizationProviderRanger extends LazyLogging { rangerResource, request.accessType.rangerName, user.userName.value + rangerSettings.userDomainPostfix, - user.userAssumedGroup.map(_.value).toSet[String].asJava + user.userGroups.map(_.value).asJava ) // We're using the original client's IP address for verification in Ranger. Ranger seems to use the // RemoteIPAddress variable for this. For the header IPs we use the ForwardedAddresses: this is not diff --git a/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceSpec.scala b/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceSpec.scala index de482ce3..1280ae26 100644 --- a/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceSpec.scala +++ b/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceSpec.scala @@ -27,7 +27,7 @@ class ProxyServiceSpec extends FlatSpec with DiagrammedAssertions with Scalatest Future(HttpResponse(status = StatusCodes.OK)) override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] = Future( - Some(User(UserName("okUser"), Some(UserAssumedGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey"))) + Some(User(UserName("okUser"), Set(UserGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey"))) ) override def isUserAuthorizedForRequest(request: S3Request, user: User, clientIPAddress: RemoteAddress, headerIPs: HeaderIPs): Boolean = true diff --git a/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceWithListAllBucketsSpec.scala b/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceWithListAllBucketsSpec.scala index f20f2f5c..561c49ce 100644 --- a/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceWithListAllBucketsSpec.scala +++ b/src/test/scala/com/ing/wbaa/airlock/proxy/api/ProxyServiceWithListAllBucketsSpec.scala @@ -27,7 +27,7 @@ class ProxyServiceWithListAllBucketsSpec extends FlatSpec with DiagrammedAsserti Future(HttpResponse(status = StatusCodes.OK)) override def areCredentialsActive(awsRequestCredential: AwsRequestCredential): Future[Option[User]] = Future( - Some(User(UserName("okUser"), Some(UserAssumedGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey"))) + Some(User(UserName("okUser"), Set(UserGroup("okGroup")), AwsAccessKey("accesskey"), AwsSecretKey("secretkey"))) ) override def isUserAuthorizedForRequest(request: S3Request, user: User, clientIPAddress: RemoteAddress, headerIPs: HeaderIPs): Boolean = true diff --git a/src/test/scala/com/ing/wbaa/airlock/proxy/data/JsonProtocolsSpec.scala b/src/test/scala/com/ing/wbaa/airlock/proxy/data/JsonProtocolsSpec.scala index cec3e341..1c0c61de 100644 --- a/src/test/scala/com/ing/wbaa/airlock/proxy/data/JsonProtocolsSpec.scala +++ b/src/test/scala/com/ing/wbaa/airlock/proxy/data/JsonProtocolsSpec.scala @@ -12,23 +12,24 @@ class JsonProtocolsSpec extends WordSpec with DiagrammedAssertions with JsonProt val jsonString = """{ | "userName": "user", - | "userAssumedGroup": "group1", + | "userGroups": ["group1"], | "accessKey": "accesskey", | "secretKey": "secretkey" |}""".stripMargin val result = jsonString.parseJson.convertTo[UserRawJson] - assert(result == UserRawJson("user", Some("group1"), "accesskey", "secretkey")) + assert(result == UserRawJson("user", Set("group1"), "accesskey", "secretkey")) } "does not have a group" in { val jsonString = """{ | "userName": "user", + | "userGroups": [], | "accessKey": "accesskey", | "secretKey": "secretkey" |}""".stripMargin val result = jsonString.parseJson.convertTo[UserRawJson] - assert(result == UserRawJson("user", None, "accesskey", "secretkey")) + assert(result == UserRawJson("user", Set.empty[String], "accesskey", "secretkey")) } "fail when fields are missing" in { diff --git a/src/test/scala/com/ing/wbaa/airlock/proxy/data/UserSpec.scala b/src/test/scala/com/ing/wbaa/airlock/proxy/data/UserSpec.scala index 62c11954..de04ff47 100644 --- a/src/test/scala/com/ing/wbaa/airlock/proxy/data/UserSpec.scala +++ b/src/test/scala/com/ing/wbaa/airlock/proxy/data/UserSpec.scala @@ -6,8 +6,8 @@ class UserSpec extends WordSpec with DiagrammedAssertions { "UserRawJson" should { "convert to User in apply of UserRawJson" in { assert( - User(UserRawJson("u", Some("g"), "a", "s")) == - User(UserName("u"), Some(UserAssumedGroup("g")), AwsAccessKey("a"), AwsSecretKey("s")) + User(UserRawJson("u", Set("g"), "a", "s")) == + User(UserName("u"), Set(UserGroup("g")), AwsAccessKey("a"), AwsSecretKey("s")) ) } } diff --git a/src/test/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3Spec.scala b/src/test/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3Spec.scala index 6b89458c..af42b3ab 100644 --- a/src/test/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3Spec.scala +++ b/src/test/scala/com/ing/wbaa/airlock/proxy/handler/RequestHandlerS3Spec.scala @@ -34,7 +34,7 @@ class RequestHandlerS3Spec extends AsyncWordSpec with DiagrammedAssertions with val initialNumFiredRequests = numFiredRequests executeRequest( HttpRequest(), - User(UserRawJson("u", None, "a", "s")) + User(UserRawJson("u", Set.empty[String], "a", "s")) ).map(_ => assert(numFiredRequests - initialNumFiredRequests == 2)) } } From dbb15d359c6f8c51db7b73b80d27aadbf4260bfb Mon Sep 17 00:00:00 2001 From: Krzysztof Zmij Date: Wed, 2 Jan 2019 13:17:06 +0100 Subject: [PATCH 2/2] update scala and akka versions --- .travis.yml | 2 +- build.sbt | 6 +++--- docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e31a4d0..003fe897 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ services: language: scala scala: -- 2.12.7 +- 2.12.8 env: global: diff --git a/build.sbt b/build.sbt index d7cf9a23..dc7e9fa5 100644 --- a/build.sbt +++ b/build.sbt @@ -3,9 +3,9 @@ import com.typesafe.sbt.packager.docker.ExecCmd import scalariform.formatter.preferences._ name := "airlock" -version := "0.0.8" +version := "0.1.0" -scalaVersion := "2.12.7" +scalaVersion := "2.12.8" scalacOptions += "-unchecked" scalacOptions += "-deprecation" @@ -19,7 +19,7 @@ scalacOptions += "-Xfatal-warnings" updateOptions := updateOptions.value.withCachedResolution(cachedResoluton = true) val akkaVersion = "10.1.5" -val akkaStreamVersion = "2.5.17" +val akkaStreamVersion = "2.5.19" libraryDependencies ++= Seq( "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0", diff --git a/docker-compose.yml b/docker-compose.yml index d06967db..53dbd0d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: - "6080:6080" airlock-sts: - image: wbaa/airlock-sts:feature_get-user-groups + image: wbaa/airlock-sts:0.1.0 environment: - STS_HOST=0.0.0.0 - STS_PORT=12345