feat: 支持上传和下载同步进行 #2520

TencentBlueKing · Aug 29, 2024 · 22f7fb7 · 22f7fb7
1 parent 7695c60
commit 22f7fb7
Show file tree

Hide file tree

Showing 16 changed files with 240 additions and 22 deletions.
diff --git a/...backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt b/...backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt
@@ -27,6 +27,7 @@
 
 package com.tencent.bkrepo.fs.server.api
 
+import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType
 import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest
 import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest
 import com.tencent.bkrepo.auth.pojo.user.CreateUserToProjectRequest
@@ -73,4 +74,11 @@ interface RAuthClient {
     fun createUserToProject(
         @RequestBody request: CreateUserToProjectRequest
     ): Mono<Response<Boolean>>
+
+    @PostMapping("/account/credential")
+    fun checkAccountCredential(
+        @RequestParam accesskey: String,
+        @RequestParam secretkey: String,
+        @RequestParam authorizationGrantType: AuthorizationGrantType? = null
+    ): Mono<Response<String?>>
 }
diff --git a/...d/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt b/...d/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt
@@ -78,9 +78,12 @@ interface RRepositoryClient {
     @PostMapping("/node/rename")
     fun renameNode(@RequestBody nodeRenameRequest: NodeRenameRequest): Mono<Response<Void>>
 
-    @PostMapping("/node/fs/create")
+    @PostMapping("/node/create")
     fun createNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono<Response<NodeDetail>>
 
+    @PostMapping("/node/fs/create")
+    fun createFsNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono<Response<NodeDetail>>
+
     @PutMapping("/node/fs/length")
     fun setLength(@RequestBody nodeSetLengthRequest: NodeSetLengthRequest): Mono<Response<Void>>
 

diff --git a/.../boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt b/.../boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt
@@ -111,6 +111,13 @@ class RouteConfiguration(
             addMetrics(serverMetrics.uploadingCount)
         }
 
+        "/stream".nest{
+            filter(artifactFileCleanupFilterFunction::filter)
+            PUT(DEFAULT_MAPPING_URI, fileOperationsHandler::stream)
+            addMetrics(serverMetrics.uploadingCount)
+        }
+
+
         "/client".nest {
             POST("/create/{projectId}/{repoName}", clientHandler::createClient)
             DELETE("/delete/{projectId}/{repoName}/{clientId}", clientHandler::removeClient)

diff --git a/...server/config/propteries/IoaProperties.kt → ...server/config/properties/IoaProperties.kt b/...server/config/propteries/IoaProperties.kt → ...server/config/properties/IoaProperties.kt
@@ -25,7 +25,7 @@
  *  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.fs.server.config.propteries
+package com.tencent.bkrepo.fs.server.config.properties
 
 import org.springframework.boot.context.properties.ConfigurationProperties
 

diff --git a/...src/main/kotlin/com/tencent/bkrepo/fs/server/config/properties/PropertiesConfiguration.kt b/...src/main/kotlin/com/tencent/bkrepo/fs/server/config/properties/PropertiesConfiguration.kt
@@ -0,0 +1,11 @@
+package com.tencent.bkrepo.fs.server.config.properties
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+@EnableConfigurationProperties(
+    IoaProperties::class,
+    StreamProperties::class,
+)
+class PropertiesConfiguration
diff --git a/...server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/properties/StreamProperties.kt b/...server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/properties/StreamProperties.kt
@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.fs.server.config.properties
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.util.unit.DataSize
+
+@ConfigurationProperties("stream")
+data class StreamProperties(
+    var blockSize: DataSize = DataSize.ofMegabytes(8)
+)
diff --git a/...s-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt b/...s-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt
@@ -28,9 +28,13 @@
 package com.tencent.bkrepo.fs.server.filter
 
 import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER
+import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID
+import com.tencent.bkrepo.common.api.constant.PLATFORM_KEY
 import com.tencent.bkrepo.common.api.constant.USER_KEY
 import com.tencent.bkrepo.common.security.exception.AuthenticationException
+import com.tencent.bkrepo.fs.server.service.PermissionService
 import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.bearerToken
+import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.platformCredentials
 import com.tencent.bkrepo.fs.server.utils.SecurityManager
 import io.jsonwebtoken.ExpiredJwtException
 import io.jsonwebtoken.JwtException
@@ -42,7 +46,8 @@ import org.springframework.web.reactive.function.server.ServerResponse
  * 认证过滤器，处理服务器认证
  * */
 class AuthHandlerFilterFunction(
-    private val securityManager: SecurityManager
+    private val securityManager: SecurityManager,
+    private val permissionService: PermissionService
 ) : CoHandlerFilterFunction {
 
     override suspend fun filter(
@@ -53,6 +58,18 @@ class AuthHandlerFilterFunction(
             return next(request)
         }
         var user = ANONYMOUS_USER
+
+        val platformAuthCredentials = request.platformCredentials()
+        if (platformAuthCredentials != null) {
+            request.exchange().attributes[PLATFORM_KEY] = permissionService.checkPlatformAccount(
+                accessKey = platformAuthCredentials.accessKey,
+                secretKey = platformAuthCredentials.secretKey
+            )
+            request.exchange().attributes[USER_KEY] =
+                request.headers().header(AUTH_HEADER_UID).firstOrNull() ?: ANONYMOUS_USER
+            return next(request)
+        }
+
         val token = if (request.path().startsWith("/service")) {
             request.headers().header("X-BKREPO-MS-UID").firstOrNull()?.let {
                 user = it

diff --git a/...fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt b/...fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt
@@ -28,10 +28,13 @@
 package com.tencent.bkrepo.fs.server.filter
 
 import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
+import com.tencent.bkrepo.common.api.constant.PLATFORM_KEY
+import com.tencent.bkrepo.common.api.constant.USER_KEY
 import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
 import com.tencent.bkrepo.common.artifact.constant.REPO_NAME
 import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_PERMIT
 import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_REPOSITORY
+import com.tencent.bkrepo.fs.server.service.PermissionService
 import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.bearerToken
 import com.tencent.bkrepo.fs.server.utils.SecurityManager
 import org.springframework.http.HttpStatus
@@ -40,7 +43,10 @@ import org.springframework.web.reactive.function.server.ServerRequest
 import org.springframework.web.reactive.function.server.ServerResponse
 import org.springframework.web.reactive.function.server.buildAndAwait
 
-class PermissionFilterFunction(private val securityManager: SecurityManager) : CoHandlerFilterFunction {
+class PermissionFilterFunction(
+    private val securityManager: SecurityManager,
+    private val permissionService: PermissionService
+) : CoHandlerFilterFunction {
     private val matcher = AntPathMatcher()
     override suspend fun filter(
         request: ServerRequest,
@@ -49,22 +55,42 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
         if (uncheckedUrlPrefixList.any { request.path().startsWith(it) }) {
             return next(request)
         }
+        val projectId = request.pathVariable(PROJECT_ID)
+        val repoName = request.pathVariable(REPO_NAME)
         val action = request.getAction()
+        val platformKey = request.exchange().attributes[PLATFORM_KEY]
+        if (platformKey != null) {
+            val userId = request.exchange().attributes[USER_KEY].toString()
+            return if (permissionService.checkPermission(projectId, repoName, action, userId)) {
+                next(request)
+            } else {
+                ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
+            }
+        }
+
         val token = request.bearerToken()
-        return if (token == null) {
+        return if (checkToken(token, request, action)) {
+            next(request)
+        } else {
             ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
+        }
+    }
+
+    private fun checkToken(
+        token: String?,
+        request: ServerRequest,
+        action: PermissionAction,
+    ): Boolean {
+        return if (token == null) {
+            false
         } else {
             val jws = securityManager.validateToken(token)
             val repo = jws.body[JWT_CLAIMS_REPOSITORY]
             val permit = jws.body[JWT_CLAIMS_PERMIT].toString()
             val projectId = request.pathVariable(PROJECT_ID)
             val repoName = request.pathVariable(REPO_NAME)
             val requestRepo = "$projectId/$repoName"
-            if (requestRepo == repo && checkAction(permit, action)) {
-                next(request)
-            } else {
-                ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
-            }
+            return requestRepo == repo && checkAction(permit, action)
         }
     }
 
@@ -93,6 +119,7 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
             "/node/mkdir/**",
             "/node/set-length/**",
             "/block/**",
+            "/stream/**"
         )
         private val uncheckedUrlPrefixList = listOf(
             "/login", "/devx/login", "/service", "/token", "/ioa", "/client/metrics/push"

diff --git a/...t-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt b/...t-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt
@@ -41,6 +41,7 @@ import com.tencent.bkrepo.fs.server.io.RegionInputStreamResource
 import com.tencent.bkrepo.fs.server.request.BlockRequest
 import com.tencent.bkrepo.fs.server.request.FlushRequest
 import com.tencent.bkrepo.fs.server.request.NodeRequest
+import com.tencent.bkrepo.fs.server.request.StreamRequest
 import com.tencent.bkrepo.fs.server.resolveRange
 import com.tencent.bkrepo.fs.server.service.FileOperationService
 import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder
@@ -60,6 +61,7 @@ import org.springframework.web.reactive.function.server.ServerResponse.temporary
 import org.springframework.web.reactive.function.server.buildAndAwait
 import java.net.URI
 
+
 /**
  * 文件操作相关处理器
  *
@@ -151,6 +153,13 @@ class FileOperationsHandler(
         return ReactiveResponseBuilder.success(blockNode)
     }
 
+    suspend fun stream(request: ServerRequest): ServerResponse {
+        val user = ReactiveSecurityUtils.getUser()
+        val streamRequest = StreamRequest(request)
+        val nodeDetail = fileOperationService.stream(streamRequest, user)
+        return ReactiveResponseBuilder.success(nodeDetail)
+    }
+
     companion object {
         private val logger = LoggerFactory.getLogger(FileOperationsHandler::class.java)
     }

diff --git a/...t-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/...t-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
@@ -33,7 +33,6 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode
 import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5
 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256
 import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils
-import com.tencent.bkrepo.fs.server.api.RGenericClient
 import com.tencent.bkrepo.fs.server.api.RRepositoryClient
 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY
 import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder
@@ -76,7 +75,6 @@ import java.time.Duration
  * 处理节点操作的请求
  * */
 class NodeOperationsHandler(
-    rGenericClient: RGenericClient,
     private val rRepositoryClient: RRepositoryClient,
     private val fileNodeService: FileNodeService
 ) {
@@ -298,7 +296,7 @@ class NodeOperationsHandler(
                 nodeMetadata = listOf(fsAttr),
                 operator = user
             )
-            return rRepositoryClient.createNode(nodeCreateRequest).awaitSingle().data!!
+            return rRepositoryClient.createFsNode(nodeCreateRequest).awaitSingle().data!!
         }
     }
 

diff --git a/...d/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt b/...d/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/request/StreamRequest.kt
@@ -0,0 +1,12 @@
+package com.tencent.bkrepo.fs.server.request
+
+import com.tencent.bkrepo.common.api.exception.ParameterInvalidException
+import org.springframework.web.reactive.function.server.ServerRequest
+import org.springframework.web.reactive.function.server.queryParamOrNull
+
+class StreamRequest(val request: ServerRequest) : NodeRequest(request) {
+    val size = request.queryParamOrNull("size")?.toLong()
+        ?: throw ParameterInvalidException("required size parameter.")
+    val overwrite = request.headers().header("X-BKREPO-OVERWRITE").firstOrNull()?.toBoolean() ?: false
+    val expires = request.headers().header("X-BKREPO-EXPIRES").firstOrNull()?.toLong() ?: 0
+}