Skip to content

Commit

Permalink
feat: 支持上传和下载同步进行 #2520
Browse files Browse the repository at this point in the history
  • Loading branch information
yaoxuwan committed Aug 29, 2024
1 parent 7695c60 commit 22f7fb7
Show file tree
Hide file tree
Showing 16 changed files with 240 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

package com.tencent.bkrepo.fs.server.api

import com.tencent.bkrepo.auth.pojo.oauth.AuthorizationGrantType
import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest
import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest
import com.tencent.bkrepo.auth.pojo.user.CreateUserToProjectRequest
Expand Down Expand Up @@ -73,4 +74,11 @@ interface RAuthClient {
fun createUserToProject(
@RequestBody request: CreateUserToProjectRequest
): Mono<Response<Boolean>>

@PostMapping("/account/credential")
fun checkAccountCredential(
@RequestParam accesskey: String,
@RequestParam secretkey: String,
@RequestParam authorizationGrantType: AuthorizationGrantType? = null
): Mono<Response<String?>>
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,12 @@ interface RRepositoryClient {
@PostMapping("/node/rename")
fun renameNode(@RequestBody nodeRenameRequest: NodeRenameRequest): Mono<Response<Void>>

@PostMapping("/node/fs/create")
@PostMapping("/node/create")
fun createNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono<Response<NodeDetail>>

@PostMapping("/node/fs/create")
fun createFsNode(@RequestBody nodeCreateRequest: NodeCreateRequest): Mono<Response<NodeDetail>>

@PutMapping("/node/fs/length")
fun setLength(@RequestBody nodeSetLengthRequest: NodeSetLengthRequest): Mono<Response<Void>>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,13 @@ class RouteConfiguration(
addMetrics(serverMetrics.uploadingCount)
}

"/stream".nest{
filter(artifactFileCleanupFilterFunction::filter)
PUT(DEFAULT_MAPPING_URI, fileOperationsHandler::stream)
addMetrics(serverMetrics.uploadingCount)
}


"/client".nest {
POST("/create/{projectId}/{repoName}", clientHandler::createClient)
DELETE("/delete/{projectId}/{repoName}/{clientId}", clientHandler::removeClient)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.fs.server.config.propteries
package com.tencent.bkrepo.fs.server.config.properties

import org.springframework.boot.context.properties.ConfigurationProperties

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package com.tencent.bkrepo.fs.server.config.properties

import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration

@Configuration
@EnableConfigurationProperties(
IoaProperties::class,
StreamProperties::class,
)
class PropertiesConfiguration
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package com.tencent.bkrepo.fs.server.config.properties

import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.util.unit.DataSize

@ConfigurationProperties("stream")
data class StreamProperties(
var blockSize: DataSize = DataSize.ofMegabytes(8)
)
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,13 @@
package com.tencent.bkrepo.fs.server.filter

import com.tencent.bkrepo.common.api.constant.ANONYMOUS_USER
import com.tencent.bkrepo.common.api.constant.AUTH_HEADER_UID
import com.tencent.bkrepo.common.api.constant.PLATFORM_KEY
import com.tencent.bkrepo.common.api.constant.USER_KEY
import com.tencent.bkrepo.common.security.exception.AuthenticationException
import com.tencent.bkrepo.fs.server.service.PermissionService
import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.bearerToken
import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.platformCredentials
import com.tencent.bkrepo.fs.server.utils.SecurityManager
import io.jsonwebtoken.ExpiredJwtException
import io.jsonwebtoken.JwtException
Expand All @@ -42,7 +46,8 @@ import org.springframework.web.reactive.function.server.ServerResponse
* 认证过滤器,处理服务器认证
* */
class AuthHandlerFilterFunction(
private val securityManager: SecurityManager
private val securityManager: SecurityManager,
private val permissionService: PermissionService
) : CoHandlerFilterFunction {

override suspend fun filter(
Expand All @@ -53,6 +58,18 @@ class AuthHandlerFilterFunction(
return next(request)
}
var user = ANONYMOUS_USER

val platformAuthCredentials = request.platformCredentials()
if (platformAuthCredentials != null) {
request.exchange().attributes[PLATFORM_KEY] = permissionService.checkPlatformAccount(
accessKey = platformAuthCredentials.accessKey,
secretKey = platformAuthCredentials.secretKey
)
request.exchange().attributes[USER_KEY] =
request.headers().header(AUTH_HEADER_UID).firstOrNull() ?: ANONYMOUS_USER
return next(request)
}

val token = if (request.path().startsWith("/service")) {
request.headers().header("X-BKREPO-MS-UID").firstOrNull()?.let {
user = it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,13 @@
package com.tencent.bkrepo.fs.server.filter

import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
import com.tencent.bkrepo.common.api.constant.PLATFORM_KEY
import com.tencent.bkrepo.common.api.constant.USER_KEY
import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
import com.tencent.bkrepo.common.artifact.constant.REPO_NAME
import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_PERMIT
import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_REPOSITORY
import com.tencent.bkrepo.fs.server.service.PermissionService
import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils.bearerToken
import com.tencent.bkrepo.fs.server.utils.SecurityManager
import org.springframework.http.HttpStatus
Expand All @@ -40,7 +43,10 @@ import org.springframework.web.reactive.function.server.ServerRequest
import org.springframework.web.reactive.function.server.ServerResponse
import org.springframework.web.reactive.function.server.buildAndAwait

class PermissionFilterFunction(private val securityManager: SecurityManager) : CoHandlerFilterFunction {
class PermissionFilterFunction(
private val securityManager: SecurityManager,
private val permissionService: PermissionService
) : CoHandlerFilterFunction {
private val matcher = AntPathMatcher()
override suspend fun filter(
request: ServerRequest,
Expand All @@ -49,22 +55,42 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
if (uncheckedUrlPrefixList.any { request.path().startsWith(it) }) {
return next(request)
}
val projectId = request.pathVariable(PROJECT_ID)
val repoName = request.pathVariable(REPO_NAME)
val action = request.getAction()
val platformKey = request.exchange().attributes[PLATFORM_KEY]
if (platformKey != null) {
val userId = request.exchange().attributes[USER_KEY].toString()
return if (permissionService.checkPermission(projectId, repoName, action, userId)) {
next(request)
} else {
ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
}
}

val token = request.bearerToken()
return if (token == null) {
return if (checkToken(token, request, action)) {
next(request)
} else {
ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
}
}

private fun checkToken(
token: String?,
request: ServerRequest,
action: PermissionAction,
): Boolean {
return if (token == null) {
false
} else {
val jws = securityManager.validateToken(token)
val repo = jws.body[JWT_CLAIMS_REPOSITORY]
val permit = jws.body[JWT_CLAIMS_PERMIT].toString()
val projectId = request.pathVariable(PROJECT_ID)
val repoName = request.pathVariable(REPO_NAME)
val requestRepo = "$projectId/$repoName"
if (requestRepo == repo && checkAction(permit, action)) {
next(request)
} else {
ServerResponse.status(HttpStatus.FORBIDDEN).buildAndAwait()
}
return requestRepo == repo && checkAction(permit, action)
}
}

Expand Down Expand Up @@ -93,6 +119,7 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
"/node/mkdir/**",
"/node/set-length/**",
"/block/**",
"/stream/**"
)
private val uncheckedUrlPrefixList = listOf(
"/login", "/devx/login", "/service", "/token", "/ioa", "/client/metrics/push"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import com.tencent.bkrepo.fs.server.io.RegionInputStreamResource
import com.tencent.bkrepo.fs.server.request.BlockRequest
import com.tencent.bkrepo.fs.server.request.FlushRequest
import com.tencent.bkrepo.fs.server.request.NodeRequest
import com.tencent.bkrepo.fs.server.request.StreamRequest
import com.tencent.bkrepo.fs.server.resolveRange
import com.tencent.bkrepo.fs.server.service.FileOperationService
import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder
Expand All @@ -60,6 +61,7 @@ import org.springframework.web.reactive.function.server.ServerResponse.temporary
import org.springframework.web.reactive.function.server.buildAndAwait
import java.net.URI


/**
* 文件操作相关处理器
*
Expand Down Expand Up @@ -151,6 +153,13 @@ class FileOperationsHandler(
return ReactiveResponseBuilder.success(blockNode)
}

suspend fun stream(request: ServerRequest): ServerResponse {
val user = ReactiveSecurityUtils.getUser()
val streamRequest = StreamRequest(request)
val nodeDetail = fileOperationService.stream(streamRequest, user)
return ReactiveResponseBuilder.success(nodeDetail)
}

companion object {
private val logger = LoggerFactory.getLogger(FileOperationsHandler::class.java)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5
import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256
import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils
import com.tencent.bkrepo.fs.server.api.RGenericClient
import com.tencent.bkrepo.fs.server.api.RRepositoryClient
import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY
import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder
Expand Down Expand Up @@ -76,7 +75,6 @@ import java.time.Duration
* 处理节点操作的请求
* */
class NodeOperationsHandler(
rGenericClient: RGenericClient,
private val rRepositoryClient: RRepositoryClient,
private val fileNodeService: FileNodeService
) {
Expand Down Expand Up @@ -298,7 +296,7 @@ class NodeOperationsHandler(
nodeMetadata = listOf(fsAttr),
operator = user
)
return rRepositoryClient.createNode(nodeCreateRequest).awaitSingle().data!!
return rRepositoryClient.createFsNode(nodeCreateRequest).awaitSingle().data!!
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package com.tencent.bkrepo.fs.server.request

import com.tencent.bkrepo.common.api.exception.ParameterInvalidException
import org.springframework.web.reactive.function.server.ServerRequest
import org.springframework.web.reactive.function.server.queryParamOrNull

class StreamRequest(val request: ServerRequest) : NodeRequest(request) {
val size = request.queryParamOrNull("size")?.toLong()
?: throw ParameterInvalidException("required size parameter.")
val overwrite = request.headers().header("X-BKREPO-OVERWRITE").firstOrNull()?.toBoolean() ?: false
val expires = request.headers().header("X-BKREPO-EXPIRES").firstOrNull()?.toLong() ?: 0
}
Loading

0 comments on commit 22f7fb7

Please sign in to comment.