Migrate SandboxStorage to S3 (#1827)

saveourtool · Feb 6, 2023 · c2fd444 · c2fd444
1 parent b3a39a3
commit c2fd444
Show file tree

Hide file tree

Showing 13 changed files with 72 additions and 36 deletions.
diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt
@@ -4,7 +4,7 @@ import com.saveourtool.save.backend.configs.ConfigProperties
 import com.saveourtool.save.s3.S3Operations
 import com.saveourtool.save.storage.AbstractS3Storage
 import com.saveourtool.save.storage.concatS3Key
-import com.saveourtool.save.storage.s3KeyToParts
+import com.saveourtool.save.storage.s3KeyToPartsTill
 import com.saveourtool.save.utils.AvatarType
 import com.saveourtool.save.utils.orNotFound
 import org.springframework.stereotype.Service
@@ -22,7 +22,7 @@ class AvatarStorage(
     concatS3Key(configProperties.s3Storage.prefix, "images", "avatars")
 ) {
     override fun buildKey(s3KeySuffix: String): AvatarKey {
-        val (typeStr, objectName) = s3KeySuffix.s3KeyToParts()
+        val (typeStr, objectName) = s3KeySuffix.s3KeyToPartsTill(prefix)
         return AvatarKey(
             type = AvatarType.findByUrlPath(typeStr)
                 .orNotFound {

diff --git a/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml b/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml
@@ -36,6 +36,8 @@ spec:
                   fieldPath: status.hostIP
             - name: DATABASE_SECRETS_PATH
               value: {{ .Values.mysql.dbPasswordFile }}
+            - name: S3_SECRETS_PATH
+              value: {{ .Values.s3.secretFile }}
             - name: JAVA_TOOL_OPTIONS
               value: -XX:ReservedCodeCacheSize=48M
             - name: POD_NAMESPACE
@@ -56,6 +58,8 @@ spec:
               mountPath: /home/cnb/files
             - name: database-secret
               mountPath: {{ .Values.mysql.dbPasswordFile }}
+            - name: s3-secrets
+              mountPath: {{ .Values.s3.secretFile }}
           {{- include "spring-boot.management" .Values.sandbox | nindent 10 }}
           resources:
             limits:
@@ -126,5 +130,8 @@ spec:
         - name: database-secret
           secret:
             secretName: db-secrets
+        - name: s3-secrets
+          secret:
+            secretName: s3-secrets
         - name: migrations-data
           emptyDir: { }
diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt
@@ -22,7 +22,11 @@ abstract class AbstractS3Storage<K>(
     prefix: String,
 ) : Storage<K> {
     private val log: Logger = getLogger(this::class)
-    private val prefix = prefix.removeSuffix(PATH_DELIMITER) + PATH_DELIMITER
+
+    /**
+     * A common prefix endings with [PATH_DELIMITER] for all s3 keys in this storage
+     */
+    protected val prefix: String = prefix.removeSuffix(PATH_DELIMITER) + PATH_DELIMITER
 
     override fun list(): Flux<K> = s3Operations.listObjectsV2(prefix)
         .flatMapIterable { response ->

diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt
@@ -11,9 +11,10 @@ const val PATH_DELIMITER = "/"
 
 /**
  * @receiver key in S3 as [String]
+ * @param prefix a common prefix for all keys in storage
  * @return parts [this] split by [PATH_DELIMITER]
  */
-fun String.s3KeyToParts(): List<String> = removePrefix(PATH_DELIMITER).removeSuffix(PATH_DELIMITER).split(PATH_DELIMITER)
+fun String.s3KeyToPartsTill(prefix: String): List<String> = removePrefix(prefix).removePrefix(PATH_DELIMITER).removeSuffix(PATH_DELIMITER).split(PATH_DELIMITER)
 
 /**
  * @param parts should not end or start with [PATH_DELIMITER] -- will be deleted

diff --git a/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt b/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt
@@ -34,15 +34,17 @@ class S3StorageUtilsKtTest {
 
     @Test
     fun testS3KeyToParts() {
-        doTestConcatS3Key("only-prefix/", listOf("only-prefix"))
-        doTestConcatS3Key("prefix/suffix", listOf("prefix", "suffix"))
-        doTestConcatS3Key("prefix/middle/suffix", listOf("prefix", "middle", "suffix"))
+        doTestConcatS3Key("", "only-prefix/", listOf("only-prefix"))
+        doTestConcatS3Key("", "prefix/suffix", listOf("prefix", "suffix"))
+        doTestConcatS3Key("", "prefix/middle/suffix", listOf("prefix", "middle", "suffix"))
+        doTestConcatS3Key("prefix", "prefix/middle/suffix", listOf("middle", "suffix"))
+        doTestConcatS3Key("prefix/", "prefix/middle/suffix", listOf("middle", "suffix"))
     }
 
-    private fun doTestConcatS3Key(s3Key: String, expectedValue: List<String>) {
+    private fun doTestConcatS3Key(prefix: String, s3Key: String, expectedValue: List<String>) {
         Assertions.assertEquals(
             expectedValue,
-            s3Key.s3KeyToParts()
+            s3Key.s3KeyToPartsTill(prefix)
         )
     }
 }
diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt
@@ -1,28 +1,22 @@
 package com.saveourtool.save.sandbox.config
 
+import com.saveourtool.save.s3.S3OperationsProperties
 import com.saveourtool.save.service.LokiConfig
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.boot.context.properties.ConstructorBinding
 
 /**
- * @property fileStorage configuration of file storage
+ * @property s3Storage configuration of S3 storage
  * @property agentSettings properties for save-agents
  * @property lokiConfig config of loki service for logging
  */
 @ConstructorBinding
 @ConfigurationProperties(prefix = "sandbox")
 data class ConfigProperties(
-    val fileStorage: FileStorageConfig,
+    val s3Storage: S3OperationsProperties,
     val agentSettings: AgentSettings,
     val lokiConfig: LokiConfig? = null,
 ) {
-    /**
-     * @property location location of file storage
-     */
-    data class FileStorageConfig(
-        val location: String,
-    )
-
     /**
      * @property sandboxUrl the URL of save-sandbox that will be reported to save-agents.
      */

diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt
@@ -0,0 +1,20 @@
+package com.saveourtool.save.sandbox.config
+
+import com.saveourtool.save.s3.DefaultS3Operations
+import com.saveourtool.save.s3.S3Operations
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+
+/**
+ * Configuration for S3
+ */
+@Configuration
+class S3Configuration(
+    private val configProperties: ConfigProperties,
+) {
+    /**
+     * @return [S3Operations] as a Spring's bean
+     */
+    @Bean
+    fun s3Operations(): S3Operations = DefaultS3Operations(configProperties.s3Storage)
+}
diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt
@@ -1,32 +1,36 @@
 package com.saveourtool.save.sandbox.storage
 
+import com.saveourtool.save.s3.S3Operations
 import com.saveourtool.save.sandbox.config.ConfigProperties
-import com.saveourtool.save.storage.AbstractFileBasedStorage
-import com.saveourtool.save.utils.pathNamesTill
+import com.saveourtool.save.storage.AbstractS3Storage
+import com.saveourtool.save.storage.concatS3Key
+import com.saveourtool.save.storage.s3KeyToPartsTill
 import org.springframework.stereotype.Component
 import reactor.core.publisher.Flux
-import java.nio.file.Path
-import kotlin.io.path.div
 
 /**
  * Storage implementation for sandbox
  */
 @Component
 class SandboxStorage(
     configProperties: ConfigProperties,
-) : AbstractFileBasedStorage<SandboxStorageKey>(Path.of(configProperties.fileStorage.location) / "sandbox", PATH_PARTS_COUNT) {
+    s3Operations: S3Operations,
+) : AbstractS3Storage<SandboxStorageKey>(
+    s3Operations,
+    concatS3Key(configProperties.s3Storage.prefix, "sandbox"),
+) {
     @Suppress("DestructuringDeclarationWithTooManyEntries")
-    override fun buildKey(rootDir: Path, pathToContent: Path): SandboxStorageKey {
-        val (filename, typeName, userId) = pathToContent.pathNamesTill(rootDir)
+    override fun buildKey(s3KeySuffix: String): SandboxStorageKey {
+        val (filename, typeName, userId) = s3KeySuffix.s3KeyToPartsTill(prefix)
         return SandboxStorageKey(
             userId.toLong(),
             SandboxStorageKeyType.valueOf(typeName),
             filename,
         )
     }
 
-    override fun buildPathToContent(rootDir: Path, key: SandboxStorageKey): Path =
-            rootDir / key.userId.toString() / key.type.name / key.fileName
+    override fun buildS3KeySuffix(key: SandboxStorageKey): String =
+            concatS3Key(key.userId.toString(), key.type.name, key.fileName)
 
     /**
      * @param userId
@@ -39,8 +43,4 @@ class SandboxStorage(
     ): Flux<SandboxStorageKey> = list().filter {
         it.userId == userId && it.type in types
     }
-
-    companion object {
-        private const val PATH_PARTS_COUNT = 3  // userId + key.type + fileName
-    }
 }
diff --git a/save-sandbox/src/main/resources/application-dev.properties b/save-sandbox/src/main/resources/application-dev.properties
@@ -1,3 +1,7 @@
 spring.datasource.url=jdbc:mysql://localhost:3306/save_sandbox
 spring.datasource.username=root
-spring.datasource.password=123
+spring.datasource.password=123
+sandbox.s3-storage.endpoint=http://localhost:9000
+sandbox.s3-storage.bucketName=cnb
+sandbox.s3-storage.credentials.accessKeyId=admin
+sandbox.s3-storage.credentials.secretAccessKey=12345678
diff --git a/save-sandbox/src/main/resources/application-mac.properties b/save-sandbox/src/main/resources/application-mac.properties
diff --git a/save-sandbox/src/main/resources/application-win.properties b/save-sandbox/src/main/resources/application-win.properties
diff --git a/save-sandbox/src/main/resources/application.properties b/save-sandbox/src/main/resources/application.properties
@@ -1,6 +1,5 @@
 server.port=5400
 spring.liquibase.enabled=false
-sandbox.file-storage.location=/home/cnb/files
 # suppress inspection "HttpUrlsUsage"
 sandbox.agent-settings.sandbox-url=http://host.docker.internal:${server.port}
 orchestrator.container-name-prefix=sandbox-execution-
@@ -13,3 +12,8 @@ spring.jpa.properties.hibernate.jdbc.batch_size=100
 spring.jpa.properties.hibernate.order_inserts=true
 spring.jpa.properties.hibernate.order_updates=true
 logging.level.org.hibernate.engine.internal.StatisticalLoggingSessionEventListener=WARN
+sandbox.s3-storage.endpoint=${s3-storage.endpoint}
+sandbox.s3-storage.bucketName=${s3-storage.bucketName}
+sandbox.s3-storage.prefix=cnb/sandbox
+sandbox.s3-storage.credentials.accessKeyId=${s3-storage.credentials.accessKeyId}
+sandbox.s3-storage.credentials.secretAccessKey=${s3-storage.credentials.secretAccessKey}
diff --git a/save-sandbox/src/main/resources/bootstrap.yml b/save-sandbox/src/main/resources/bootstrap.yml
@@ -22,7 +22,9 @@ spring:
         paths: /home/cnb/config/application.properties
       secrets:
         enabled: true
-        paths: ${DATABASE_SECRETS_PATH}
+        paths:
+          - ${DATABASE_SECRETS_PATH}
+          - ${S3_SECRETS_PATH}
 kubernetes:
   informer:
     enabled: true