Merge pull request #27 from Ferlab-Ste-Justine/ferload2

feat: CQDG-369 new apis for using ferload by resources ids

.github/workflows/build.yml

@@ -1,24 +1,18 @@
 name: Test and build
 on:
-  pull_request:
-    branches:
-      - main
+  push:
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly

.github/workflows/publish_with_latest.yml

@@ -10,20 +10,16 @@ jobs:
     name: Publish Image using latest tag
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.github/workflows/publish_with_semver_tag.yml

@@ -10,20 +10,15 @@ jobs:
     name: Publish Image using tags
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.github/workflows/publish_with_sha.yml

@@ -10,20 +10,15 @@ jobs:
     name: Publish Image using commit sha and timestamp
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.gitignore

@@ -1,9 +1,11 @@
-logs
+# common IDEs to be ignored
+.idea/
+.metals/
+.vscode/
+
+# build tool specific entries
+.bloop
+.bsp
 target
-/.idea
-/.g8
-/.idea_modules
-/.classpath
-/.project
-/.settings
-/RUNNING_PID
+metals.sbt
+project/project

.scalafmt.conf

@@ -0,0 +1,3 @@
+version = 3.7.14
+maxColumn = 140
+runner.dialect = scala3

Dockerfile

@@ -1,9 +1,23 @@
-FROM adoptopenjdk/openjdk11:alpine-jre
+FROM amazoncorretto:17-alpine as build-jre
+WORKDIR /tmp/jre
+# required for strip-debug to work
+RUN apk add --no-cache binutils
+# Build small JRE image
+RUN jlink \
+         --add-modules ALL-MODULE-PATH \
+         --strip-debug \
+         --no-man-pages \
+         --no-header-files \
+         --compress=2 \
+         --output slim
 
-RUN apk update && apk add bash ca-certificates openssl
-
-COPY target/universal/ferload.tgz .
-
-RUN tar xvf ferload.tgz
-
-ENTRYPOINT ["/ferload/bin/ferload"]
+FROM alpine:latest
+WORKDIR /app
+ENV JAVA_HOME=/jre
+ENV JAVA_OPTS="-XX:+UseG1GC -XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XshowSettings:vm -XX:+PrintCommandLineFlags"
+ENV PATH="$PATH:$JAVA_HOME/bin"
+RUN apk update && apk add ca-certificates openssl
+COPY --from=build-jre /tmp/jre/slim $JAVA_HOME
+COPY target/scala-3.3.1/ferload.jar .
+EXPOSE 9090
+ENTRYPOINT ["java", "-jar", "ferload.jar"]

README.md

@@ -1,6 +1,99 @@
-# ferload
+# Ferload
 
-This service is used to dowload files stored in an object store s3 compliant. It will :
-- Verify if user has access to the object (validating jwt token)
-- Generate a presigned url for this object
+    [![Docker Pulls](https://img.shields.io/docker/pulls/ferlabcrsj/ferload)](https://hub.docker.com/r/ferlab/ferload)
+    [![Docker Image Size (latest SEMVER)](https://img.shields.io/docker/image-size/ferlabcrsj/ferload?sort=semver)](https://hub.docker.com/r/ferlab/ferload)
+    [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ferlabcrsj/ferload?sort=semver)](https://hub.docker.com/r/ferlab/ferload)
 
+Ferload is an api that allows to control access to files stored in any object store S3-compliant, and generate presigned url if user is granted.
+Ferload is compliant with [GA4GH Data Repository Service](https://ga4gh.github.io/data-repository-service-schemas/). For now, only Bearers tokens are supported (not Passport and Visas).
+
+
+## Technologies
+
+Ferload is developed in Scala3 and is based on [tapir](https://tapir.softwaremill.com/en/latest/) and [cats-effect](https://typelevel.org/cats-effect/).
+
+## Quick start
+
+If you don't have [sbt](https://www.scala-sbt.org) installed already, you can use the provided wrapper script:
+
+```shell
+./sbtx -h # shows an usage of a wrapper script
+./sbtx compile # build the project
+./sbtx test # run the tests
+./sbtx run # run the application (Main)
+```
+
+For more details check the [sbtx usage](https://github.com/dwijnand/sbt-extras#sbt--h) page.
+
+Otherwise, if sbt is already installed, you can use the standard commands:
+
+```shell
+sbt compile # build the project
+sbt test # run the tests
+sbt run # run the application (Main)
+```
+
+## Build
+
+```shell
+sbt asembly
+docker build -t ferload .
+```
+
+## Environment variables
+
+Keyckloak Authentication server information :
+
+- `AUTH_URL` : Keycloak URL
+- `AUTH_REALM` : Keycloak Realm
+- `AUTH_CLIENT_ID` : Id of the client that contains resource definition and permissions
+- `AUTH_CLIENT_SECRET` : Secret of the client that contains resource definition and permissions
+- `AUTH_RESOURCES_POLICY_GLOBAL_NAME` : Name of the resource a user should have access to be able to download all files.
+  Works only with endpoints that fetch files by urls. Can be empty.
+
+Ferload Client: This section is used to configure ferload clients taht can be installed to download files by requesting ferload endpoints.
+- `FERLOAD_CLIENT_METHOD` : 2 possible values : `token`or `password`. Default `token`.
+- `FERLOAD_CLIENT_CLIENT_ID` : client id to use to authenticate user in case of `password` method.
+- `FERLOAD_CLIENT_TOKEN_LINK` : url to use to fetch new token in case of `token` method.
+- `FERLOAD_CLIENT_TOKEN_HELPER` : text to display in ferload client to explain how to get a new token. Used only if `FERLOAD_CLIENT_METHOD` is `token`. 
+
+  AWS S3 information :
+
+- `AWS_ACCESS_KEY` : Access key of the AWS account
+- `AWS_SECRET_KEY` : Secret key of the AWS account
+- `AWS_BUCKET` : Default bucket to use if objects are fetched by urls. Can be empty.
+- `AWS_ENDPOINT`: Endpoint to S3 service. Can be empty.
+- `AWS_PATH_ACCESS_STYLE` : Path access style to S3 service (true for minio, false for AWS). Default false.
+- `AWS_PRESIGNED_URL_EXPIRATION_IN_SECONDS` : Expiration time of presigned urls. Default 3600.
+- `AWS_REGION` : Region of the AWS account. Can be empty.
+
+DRS Information : 
+- `DRS_ID` : DRS Server ID.
+- `DRS_NAME` : DRS Name.
+- `DRS_ORGANIZATION_NAME` : DRS Organization.
+- `DRS_ORGANIZATION_URL` : DRS Organization url. 
+- `DRS_SELF_HOST` : DRS self host, used in `self_uri` property of `DrsObject`. 
+- `DRS_VERSION` : DRS Version. Default 1.3.0
+- `DRS_CONTACT_URL` : DRS contact url. Can be empty.
+- `DRS_DESCRIPTION` : DRS description. Can be empty.
+- `DRS_DOCUMENTATION_URL` : DRS documentation url. Can be empty.
+- `DRS_ENVIRONMENT` :  DRS environment. Can be empty.
+
+HTTP Server information :
+- `HTTP_HOST` : Address HTTP server should listen to. Default 0.0.0.0 (all interfaces)
+- `HTTP_PORT`: Port HTTP server should listen to. Default 9090
+
+Log configuration :
+- `LOG_LEVEL` : Log level. Default WARN.
+
+## Endpoints :
+
+- Swagger : /docs
+- Status : /status
+- Prometheus : /metrics
+
+## Links:
+
+* [tapir documentation](https://tapir.softwaremill.com/en/latest/)
+* [tapir github](https://github.com/softwaremill/tapir)
+* [sbtx wrapper](https://github.com/dwijnand/sbt-extras#installation)

app/services/keycloak/PermsService.scala

@@ -2,11 +2,12 @@ package services.keycloak
 
 import auth.UserRequest
 import org.keycloak.authorization.client.util.HttpResponseException
-import org.keycloak.representations.idm.authorization.UmaPermissionRepresentation
+import org.keycloak.representations.idm.authorization.{ResourceRepresentation, ScopeRepresentation, UmaPermissionRepresentation}
 import play.api.Configuration
 import play.api.mvc.AnyContent
 
 import java.util
+import java.util.{Collections, OptionalInt}
 import javax.inject.{Inject, Singleton}
 import scala.jdk.CollectionConverters._
 
@@ -41,6 +42,24 @@ class PermsService @Inject()(config: Configuration, permsClient: PermsClient) {
     }
   }
 
+  def createResources(token: String, resources: Set[CreateResource]) = {
+    val protection = permsClient.authzClient.protection(token) // insure token has resource creation rights
+    val resource = protection.resource()
+    resources.map {
+      r =>
+        val scopes: java.util.Set[ScopeRepresentation] = r.scopes.map(s => new ScopeRepresentation(s)).asJava
+        val rr = new ResourceRepresentation(r.name.getOrElse(r.id), scopes, r.uri.orNull, r.`type`.orNull)
+        rr.setId(r.id)
+        val attributes = new util.HashMap[String, util.List[String]]()
+        attributes.put("location", Collections.singletonList(r.location))
+        attributes.put("checksum", r.checksums.toList.asJava)
+        rr.setAttributes(attributes)
+        rr
+    }.map { r =>
+      resource.create(r)
+    }
+  }
+
   def createPermissions(token: String, userName: String, files: Set[String]): (Set[String], Set[String]) = {
     val protection = permsClient.authzClient.protection(token) // insure token has resource creation rights
     val resources = protection.resource()
@@ -75,3 +94,5 @@ class PermsService @Inject()(config: Configuration, permsClient: PermsClient) {
     (createdPerms, files.diff(createdPerms))
   }
 }
+
+case class CreateResource(id: String, name: Option[String], `type`: Option[String], uri: Option[String], scopes: Set[String], checksums: Set[String], location: String, size: Option[Long])