feat: 上传/下载记录信息标识来源 #2233

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt

@@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.artifact.metrics.export.ArtifactMetricsExporter
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
 import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
 import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream
+import com.tencent.bkrepo.common.artifact.util.TransferUserAgentUtil
 import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.common.service.actuator.CommonTagProvider
@@ -100,6 +101,12 @@ class ArtifactTransferListener(
                 repoName = repositoryDetail?.name ?: UNKNOWN,
                 clientIp = clientIp,
                 fullPath = ArtifactContextHolder.getArtifactInfo()?.getArtifactFullPath() ?: UNKNOWN,
+                agent = TransferUserAgentUtil.getUserAgent(
+                    webPlatformId = artifactMetricsProperties.webPlatformId,
+                    host = artifactMetricsProperties.host,
+                    builderAgentList = artifactMetricsProperties.builderAgentList,
+                    clientAgentList = artifactMetricsProperties.clientAgentList
+                ).name
             )
             if (SecurityUtils.getUserId() != SYSTEM_USER) {
                 projectUsageStatisticsService.inc(projectId = projectId, receivedBytes = throughput.bytes)
@@ -139,6 +146,12 @@ class ArtifactTransferListener(
                 repoName = repositoryDetail?.name ?: UNKNOWN,
                 clientIp = clientIp,
                 fullPath = getFullPath(),
+                agent = TransferUserAgentUtil.getUserAgent(
+                    webPlatformId = artifactMetricsProperties.webPlatformId,
+                    host = artifactMetricsProperties.host,
+                    builderAgentList = artifactMetricsProperties.builderAgentList,
+                    clientAgentList = artifactMetricsProperties.clientAgentList
+                ).name
             )
             if (SecurityUtils.getUserId() != SYSTEM_USER) {
                 projectUsageStatisticsService.inc(projectId = projectId, responseBytes = throughput.bytes)
@@ -188,7 +201,13 @@ class ArtifactTransferListener(
                 project = projectId,
                 repoName = repoName,
                 clientIp = HttpContextHolder.getClientAddress(),
-                fullPath = fullPath
+                fullPath = fullPath,
+                agent = TransferUserAgentUtil.getUserAgent(
+                    webPlatformId = artifactMetricsProperties.webPlatformId,
+                    host = artifactMetricsProperties.host,
+                    builderAgentList = artifactMetricsProperties.builderAgentList,
+                    clientAgentList = artifactMetricsProperties.clientAgentList
+                ).name
             )
             if (artifactMetricsProperties.collectByLog) {
                 logger.info(

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/interceptor/DownloadInterceptorFactory.kt

@@ -57,9 +57,9 @@ class DownloadInterceptorFactory(
     companion object {
         private val logger = LoggerFactory.getLogger(DownloadInterceptorFactory::class.java)
         private lateinit var properties: DownloadInterceptorProperties
-        private const val ANDROID_APP_USER_AGENT = "BKCI_APP"
-        private const val ANDROID_APP_USER_AGENT_NEW = "BK_CI APP"
-        private const val IOS_APP_USER_AGENT = "com.apple.appstored"
+        const val ANDROID_APP_USER_AGENT = "BKCI_APP"
+        const val ANDROID_APP_USER_AGENT_NEW = "BK_CI APP"
+        const val IOS_APP_USER_AGENT = "com.apple.appstored"
         private const val INTERCEPTORS = "interceptors"
         private const val TYPE = "type"
         private val forbidRule = mapOf(

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt

@@ -57,4 +57,20 @@ data class ArtifactMetricsProperties(
      * 是否使用influxdb存储指标数据
      */
     var useInfluxDb: Boolean = true,
+    /**
+     * 页面host
+     */
+    var host: String = "",
+    /**
+     * 构建机agent列表
+     */
+    var builderAgentList: List<String> = emptyList(),
+    /**
+     * 客户端agent列表
+     */
+    var clientAgentList: List<String> = emptyList(),
+    /**
+     * web 平台账号id
+     */
+    var webPlatformId: String = ""
 )

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt

@@ -60,7 +60,9 @@ data class ArtifactTransferRecord(
     @Column(name = "repoName")
     val repoName: String,
     @Column(name = "fullPath")
-    val fullPath: String
+    val fullPath: String,
+    @Column(name = "agent")
+    val agent: String,
 ) {
     companion object {
         const val RECEIVE = "RECEIVE"

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt

@@ -44,6 +44,8 @@ class ArtifactTransferRecordLog(
     val clientIp: String = record.clientIp
     val project: String = record.project
     val repoName: String = record.repoName
+    val agent: String = record.agent
+    val fullPath: String = record.fullPath
     val service: String? = commonTag["service"]
     val instance: String? = commonTag["instance"]
     val host: String? = commonTag["host"]

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/TransferUserAgent.kt

@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.metrics
+
+enum class TransferUserAgent {
+    // 构建机
+    BUILDER,
+    // 制品库/蓝盾页面
+    BK_WEB,
+    // api调用
+    OPENAPI,
+    // 制品库客户端或者蓝盾客户端
+    BK_CLIENT;
+}

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt

@@ -73,6 +73,7 @@ class ArtifactMetricsExporter(
         labels[PROJECT_ID] = record.project
         labels[ArtifactTransferRecord::elapsed.name] = record.elapsed.toString()
         labels[ArtifactTransferRecord::type.name] = record.type
+        labels[ArtifactTransferRecord::agent.name] = record.agent
         return labels
     }
 

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/TransferUserAgentUtil.kt

@@ -0,0 +1,82 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.util
+
+import com.tencent.bkrepo.common.artifact.interceptor.DownloadInterceptorFactory
+import com.tencent.bkrepo.common.artifact.metrics.TransferUserAgent
+import com.tencent.bkrepo.common.security.util.SecurityUtils
+import com.tencent.bkrepo.common.service.util.HttpContextHolder
+
+object TransferUserAgentUtil {
+
+    fun getUserAgent(
+        webPlatformId: String,
+        host: String,
+        clientAgentList: List<String>,
+        builderAgentList: List<String>,
+    ): TransferUserAgent {
+        val userAgent = HttpContextHolder.getUserAgent()
+        val platformId = SecurityUtils.getPlatformId()
+        val referer = HttpContextHolder.getReferer()
+        return when {
+            isClient(userAgent, platformId, referer, webPlatformId, clientAgentList) -> TransferUserAgent.BK_CLIENT
+            isBuilder(userAgent, builderAgentList) -> TransferUserAgent.BUILDER
+            isWeb(platformId, referer, webPlatformId, host) -> TransferUserAgent.BK_WEB
+            else -> TransferUserAgent.OPENAPI
+        }
+    }
+
+    private fun isClient(
+        userAgent: String,
+        platformId: String?,
+        referer: String?,
+        webPlatformId: String,
+        clientAgentList: List<String>,
+        ): Boolean {
+        val android = userAgent.contains(DownloadInterceptorFactory.ANDROID_APP_USER_AGENT)
+            || userAgent.contains(DownloadInterceptorFactory.ANDROID_APP_USER_AGENT_NEW)
+        val ios = userAgent.contains(DownloadInterceptorFactory.IOS_APP_USER_AGENT)
+        val otherClient = clientAgentList.firstOrNull { userAgent.startsWith(it) } != null
+        val desktopClient = (platformId == webPlatformId) && referer.isNullOrBlank()
+        return android || ios || otherClient || desktopClient
+    }
+
+    private fun isBuilder(userAgent: String, builderAgentList: List<String>): Boolean {
+        return builderAgentList.firstOrNull { userAgent.startsWith(it) } != null
+    }
+
+    private fun isWeb(
+        platformId: String?,
+        referer: String?,
+        webPlatformId: String,
+        host: String,
+    ): Boolean {
+        if (referer.isNullOrBlank()) return false
+        return platformId == webPlatformId && referer.contains(host)
+    }
+}

src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/util/HttpContextHolder.kt

@@ -111,4 +111,20 @@ object HttpContextHolder {
             return request.getHeader(HttpHeaders.X_FORWARDED_FOR)
         } else StringPool.UNKNOWN
     }
+
+    fun getUserAgent(): String {
+        val requestAttributes = RequestContextHolder.getRequestAttributes()
+        return if (requestAttributes is ServletRequestAttributes) {
+            val request = requestAttributes.request
+            return request.getHeader(HttpHeaders.USER_AGENT) ?: StringPool.UNKNOWN
+        } else StringPool.UNKNOWN
+    }
+
+    fun getReferer(): String? {
+        val requestAttributes = RequestContextHolder.getRequestAttributes()
+        return if (requestAttributes is ServletRequestAttributes) {
+            val request = requestAttributes.request
+            return request.getHeader(HttpHeaders.REFERER) ?: null
+        } else null
+    }
 }