From c2fd44462d7c9aed9221b950666a75fe67a9260e Mon Sep 17 00:00:00 2001 From: Nariman Abdullin Date: Mon, 6 Feb 2023 16:09:26 +0300 Subject: [PATCH] Migrate SandboxStorage to S3 (#1827) --- .../save/backend/storage/AvatarStorage.kt | 4 +-- .../templates/sandbox-deployment.yaml | 7 +++++ .../save/storage/AbstractS3Storage.kt | 6 ++++- .../save/storage/S3StorageUtils.kt | 3 ++- .../save/storage/S3StorageUtilsKtTest.kt | 12 +++++---- .../save/sandbox/config/ConfigProperties.kt | 12 +++------ .../save/sandbox/config/S3Configuration.kt | 20 ++++++++++++++ .../save/sandbox/storage/SandboxStorage.kt | 26 +++++++++---------- .../main/resources/application-dev.properties | 6 ++++- .../main/resources/application-mac.properties | 1 - .../main/resources/application-win.properties | 1 - .../src/main/resources/application.properties | 6 ++++- save-sandbox/src/main/resources/bootstrap.yml | 4 ++- 13 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt delete mode 100644 save-sandbox/src/main/resources/application-mac.properties delete mode 100644 save-sandbox/src/main/resources/application-win.properties diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt index 4acc81181e..3a41b04e18 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/AvatarStorage.kt @@ -4,7 +4,7 @@ import com.saveourtool.save.backend.configs.ConfigProperties import com.saveourtool.save.s3.S3Operations import com.saveourtool.save.storage.AbstractS3Storage import com.saveourtool.save.storage.concatS3Key -import com.saveourtool.save.storage.s3KeyToParts +import com.saveourtool.save.storage.s3KeyToPartsTill import com.saveourtool.save.utils.AvatarType import com.saveourtool.save.utils.orNotFound import org.springframework.stereotype.Service @@ -22,7 +22,7 @@ class AvatarStorage( concatS3Key(configProperties.s3Storage.prefix, "images", "avatars") ) { override fun buildKey(s3KeySuffix: String): AvatarKey { - val (typeStr, objectName) = s3KeySuffix.s3KeyToParts() + val (typeStr, objectName) = s3KeySuffix.s3KeyToPartsTill(prefix) return AvatarKey( type = AvatarType.findByUrlPath(typeStr) .orNotFound { diff --git a/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml b/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml index f87cc92ca1..5eca3cfc30 100644 --- a/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml +++ b/save-cloud-charts/save-cloud/templates/sandbox-deployment.yaml @@ -36,6 +36,8 @@ spec: fieldPath: status.hostIP - name: DATABASE_SECRETS_PATH value: {{ .Values.mysql.dbPasswordFile }} + - name: S3_SECRETS_PATH + value: {{ .Values.s3.secretFile }} - name: JAVA_TOOL_OPTIONS value: -XX:ReservedCodeCacheSize=48M - name: POD_NAMESPACE @@ -56,6 +58,8 @@ spec: mountPath: /home/cnb/files - name: database-secret mountPath: {{ .Values.mysql.dbPasswordFile }} + - name: s3-secrets + mountPath: {{ .Values.s3.secretFile }} {{- include "spring-boot.management" .Values.sandbox | nindent 10 }} resources: limits: @@ -126,5 +130,8 @@ spec: - name: database-secret secret: secretName: db-secrets + - name: s3-secrets + secret: + secretName: s3-secrets - name: migrations-data emptyDir: { } diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt index 8df728c44c..876acc1539 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/AbstractS3Storage.kt @@ -22,7 +22,11 @@ abstract class AbstractS3Storage( prefix: String, ) : Storage { private val log: Logger = getLogger(this::class) - private val prefix = prefix.removeSuffix(PATH_DELIMITER) + PATH_DELIMITER + + /** + * A common prefix endings with [PATH_DELIMITER] for all s3 keys in this storage + */ + protected val prefix: String = prefix.removeSuffix(PATH_DELIMITER) + PATH_DELIMITER override fun list(): Flux = s3Operations.listObjectsV2(prefix) .flatMapIterable { response -> diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt index 9ee50ef6e0..efb4891d02 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/storage/S3StorageUtils.kt @@ -11,9 +11,10 @@ const val PATH_DELIMITER = "/" /** * @receiver key in S3 as [String] + * @param prefix a common prefix for all keys in storage * @return parts [this] split by [PATH_DELIMITER] */ -fun String.s3KeyToParts(): List = removePrefix(PATH_DELIMITER).removeSuffix(PATH_DELIMITER).split(PATH_DELIMITER) +fun String.s3KeyToPartsTill(prefix: String): List = removePrefix(prefix).removePrefix(PATH_DELIMITER).removeSuffix(PATH_DELIMITER).split(PATH_DELIMITER) /** * @param parts should not end or start with [PATH_DELIMITER] -- will be deleted diff --git a/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt b/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt index 1950a24701..1c1e151069 100644 --- a/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt +++ b/save-demo/src/test/kotlin/com/saveourtool/save/storage/S3StorageUtilsKtTest.kt @@ -34,15 +34,17 @@ class S3StorageUtilsKtTest { @Test fun testS3KeyToParts() { - doTestConcatS3Key("only-prefix/", listOf("only-prefix")) - doTestConcatS3Key("prefix/suffix", listOf("prefix", "suffix")) - doTestConcatS3Key("prefix/middle/suffix", listOf("prefix", "middle", "suffix")) + doTestConcatS3Key("", "only-prefix/", listOf("only-prefix")) + doTestConcatS3Key("", "prefix/suffix", listOf("prefix", "suffix")) + doTestConcatS3Key("", "prefix/middle/suffix", listOf("prefix", "middle", "suffix")) + doTestConcatS3Key("prefix", "prefix/middle/suffix", listOf("middle", "suffix")) + doTestConcatS3Key("prefix/", "prefix/middle/suffix", listOf("middle", "suffix")) } - private fun doTestConcatS3Key(s3Key: String, expectedValue: List) { + private fun doTestConcatS3Key(prefix: String, s3Key: String, expectedValue: List) { Assertions.assertEquals( expectedValue, - s3Key.s3KeyToParts() + s3Key.s3KeyToPartsTill(prefix) ) } } diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt index 1de79e007f..b738469f4d 100644 --- a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt +++ b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/ConfigProperties.kt @@ -1,28 +1,22 @@ package com.saveourtool.save.sandbox.config +import com.saveourtool.save.s3.S3OperationsProperties import com.saveourtool.save.service.LokiConfig import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.ConstructorBinding /** - * @property fileStorage configuration of file storage + * @property s3Storage configuration of S3 storage * @property agentSettings properties for save-agents * @property lokiConfig config of loki service for logging */ @ConstructorBinding @ConfigurationProperties(prefix = "sandbox") data class ConfigProperties( - val fileStorage: FileStorageConfig, + val s3Storage: S3OperationsProperties, val agentSettings: AgentSettings, val lokiConfig: LokiConfig? = null, ) { - /** - * @property location location of file storage - */ - data class FileStorageConfig( - val location: String, - ) - /** * @property sandboxUrl the URL of save-sandbox that will be reported to save-agents. */ diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt new file mode 100644 index 0000000000..a6d044991c --- /dev/null +++ b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/config/S3Configuration.kt @@ -0,0 +1,20 @@ +package com.saveourtool.save.sandbox.config + +import com.saveourtool.save.s3.DefaultS3Operations +import com.saveourtool.save.s3.S3Operations +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +/** + * Configuration for S3 + */ +@Configuration +class S3Configuration( + private val configProperties: ConfigProperties, +) { + /** + * @return [S3Operations] as a Spring's bean + */ + @Bean + fun s3Operations(): S3Operations = DefaultS3Operations(configProperties.s3Storage) +} diff --git a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt index 249065281e..d7bdf535ef 100644 --- a/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt +++ b/save-sandbox/src/main/kotlin/com/saveourtool/save/sandbox/storage/SandboxStorage.kt @@ -1,12 +1,12 @@ package com.saveourtool.save.sandbox.storage +import com.saveourtool.save.s3.S3Operations import com.saveourtool.save.sandbox.config.ConfigProperties -import com.saveourtool.save.storage.AbstractFileBasedStorage -import com.saveourtool.save.utils.pathNamesTill +import com.saveourtool.save.storage.AbstractS3Storage +import com.saveourtool.save.storage.concatS3Key +import com.saveourtool.save.storage.s3KeyToPartsTill import org.springframework.stereotype.Component import reactor.core.publisher.Flux -import java.nio.file.Path -import kotlin.io.path.div /** * Storage implementation for sandbox @@ -14,10 +14,14 @@ import kotlin.io.path.div @Component class SandboxStorage( configProperties: ConfigProperties, -) : AbstractFileBasedStorage(Path.of(configProperties.fileStorage.location) / "sandbox", PATH_PARTS_COUNT) { + s3Operations: S3Operations, +) : AbstractS3Storage( + s3Operations, + concatS3Key(configProperties.s3Storage.prefix, "sandbox"), +) { @Suppress("DestructuringDeclarationWithTooManyEntries") - override fun buildKey(rootDir: Path, pathToContent: Path): SandboxStorageKey { - val (filename, typeName, userId) = pathToContent.pathNamesTill(rootDir) + override fun buildKey(s3KeySuffix: String): SandboxStorageKey { + val (filename, typeName, userId) = s3KeySuffix.s3KeyToPartsTill(prefix) return SandboxStorageKey( userId.toLong(), SandboxStorageKeyType.valueOf(typeName), @@ -25,8 +29,8 @@ class SandboxStorage( ) } - override fun buildPathToContent(rootDir: Path, key: SandboxStorageKey): Path = - rootDir / key.userId.toString() / key.type.name / key.fileName + override fun buildS3KeySuffix(key: SandboxStorageKey): String = + concatS3Key(key.userId.toString(), key.type.name, key.fileName) /** * @param userId @@ -39,8 +43,4 @@ class SandboxStorage( ): Flux = list().filter { it.userId == userId && it.type in types } - - companion object { - private const val PATH_PARTS_COUNT = 3 // userId + key.type + fileName - } } diff --git a/save-sandbox/src/main/resources/application-dev.properties b/save-sandbox/src/main/resources/application-dev.properties index 371563ea00..e159f45e18 100644 --- a/save-sandbox/src/main/resources/application-dev.properties +++ b/save-sandbox/src/main/resources/application-dev.properties @@ -1,3 +1,7 @@ spring.datasource.url=jdbc:mysql://localhost:3306/save_sandbox spring.datasource.username=root -spring.datasource.password=123 \ No newline at end of file +spring.datasource.password=123 +sandbox.s3-storage.endpoint=http://localhost:9000 +sandbox.s3-storage.bucketName=cnb +sandbox.s3-storage.credentials.accessKeyId=admin +sandbox.s3-storage.credentials.secretAccessKey=12345678 \ No newline at end of file diff --git a/save-sandbox/src/main/resources/application-mac.properties b/save-sandbox/src/main/resources/application-mac.properties deleted file mode 100644 index a67460d707..0000000000 --- a/save-sandbox/src/main/resources/application-mac.properties +++ /dev/null @@ -1 +0,0 @@ -sandbox.file-storage.location=/Users/Shared/.save-cloud/cnb/files diff --git a/save-sandbox/src/main/resources/application-win.properties b/save-sandbox/src/main/resources/application-win.properties deleted file mode 100644 index ec1ad1a72a..0000000000 --- a/save-sandbox/src/main/resources/application-win.properties +++ /dev/null @@ -1 +0,0 @@ -sandbox.file-storage.location=${user.home}/.save-cloud/cnb/files diff --git a/save-sandbox/src/main/resources/application.properties b/save-sandbox/src/main/resources/application.properties index 42fbd78dcc..e58f754fdf 100644 --- a/save-sandbox/src/main/resources/application.properties +++ b/save-sandbox/src/main/resources/application.properties @@ -1,6 +1,5 @@ server.port=5400 spring.liquibase.enabled=false -sandbox.file-storage.location=/home/cnb/files # suppress inspection "HttpUrlsUsage" sandbox.agent-settings.sandbox-url=http://host.docker.internal:${server.port} orchestrator.container-name-prefix=sandbox-execution- @@ -13,3 +12,8 @@ spring.jpa.properties.hibernate.jdbc.batch_size=100 spring.jpa.properties.hibernate.order_inserts=true spring.jpa.properties.hibernate.order_updates=true logging.level.org.hibernate.engine.internal.StatisticalLoggingSessionEventListener=WARN +sandbox.s3-storage.endpoint=${s3-storage.endpoint} +sandbox.s3-storage.bucketName=${s3-storage.bucketName} +sandbox.s3-storage.prefix=cnb/sandbox +sandbox.s3-storage.credentials.accessKeyId=${s3-storage.credentials.accessKeyId} +sandbox.s3-storage.credentials.secretAccessKey=${s3-storage.credentials.secretAccessKey} diff --git a/save-sandbox/src/main/resources/bootstrap.yml b/save-sandbox/src/main/resources/bootstrap.yml index cce5b29b24..e0567bc53a 100644 --- a/save-sandbox/src/main/resources/bootstrap.yml +++ b/save-sandbox/src/main/resources/bootstrap.yml @@ -22,7 +22,9 @@ spring: paths: /home/cnb/config/application.properties secrets: enabled: true - paths: ${DATABASE_SECRETS_PATH} + paths: + - ${DATABASE_SECRETS_PATH} + - ${S3_SECRETS_PATH} kubernetes: informer: enabled: true