From 7728d69fc8d2cf8cf4b710c0b1b10cdf951158a3 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Tue, 3 Feb 2026 16:33:44 +0800 Subject: [PATCH 1/6] feat(file): split file storage info from agentKnowledge --- .../src/services/fileUpload.ts | 43 +- data-agent-frontend/src/views/AgentCreate.vue | 781 +++++++++--------- data-agent-frontend/src/views/AgentDetail.vue | 474 +++++------ .../controller/AgentKnowledgeController.java | 57 +- .../controller/FileUploadController.java | 29 +- .../converter/AgentKnowledgeConverter.java | 13 +- .../agentknowledge/CreateKnowledgeDTO.java | 4 +- .../ai/dataagent/entity/AgentKnowledge.java | 5 +- .../ai/dataagent/entity/FileStorage.java | 65 ++ .../ai/dataagent/event/FileDeletionEvent.java | 32 + .../ai/dataagent/event/FileEventListener.java | 74 ++ .../mapper/AgentKnowledgeMapper.java | 12 +- .../dataagent/mapper/FileStorageMapper.java | 81 ++ .../properties/FileStorageProperties.java | 4 +- .../service/agent/AgentServiceImpl.java | 11 +- .../service/file/FileResourceCleanerTask.java | 87 ++ .../service/file/FileStorageProvider.java | 52 ++ ...Enum.java => FileStorageProviderEnum.java} | 2 +- ...y.java => FileStorageProviderFactory.java} | 16 +- .../service/file/FileStorageService.java | 23 +- .../file/impls/FileStorageServiceImpl.java | 145 ++++ ...java => LocalFileStorageProviderImpl.java} | 77 +- ...l.java => OssFileStorageProviderImpl.java} | 105 +-- .../AgentKnowledgeResourceManager.java | 25 +- .../knowledge/AgentKnowledgeServiceImpl.java | 22 +- .../cloud/ai/dataagent/vo/FileStorageVo.java | 40 + .../src/main/resources/sql/h2/schema-h2.sql | 102 ++- .../src/main/resources/sql/schema.sql | 147 ++-- 28 files changed, 1526 insertions(+), 1002 deletions(-) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/FileStorage.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileDeletionEvent.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/FileStorageMapper.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/{FileStorageServiceEnum.java => FileStorageProviderEnum.java} (94%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/{FileStorageServiceFactory.java => FileStorageProviderFactory.java} (76%) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/{LocalFileStorageServiceImpl.java => LocalFileStorageProviderImpl.java} (54%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/{OssFileStorageServiceImpl.java => OssFileStorageProviderImpl.java} (59%) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java diff --git a/data-agent-frontend/src/services/fileUpload.ts b/data-agent-frontend/src/services/fileUpload.ts index 34900543d..26ad920b3 100644 --- a/data-agent-frontend/src/services/fileUpload.ts +++ b/data-agent-frontend/src/services/fileUpload.ts @@ -13,40 +13,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import axios from 'axios'; +import {ApiResponse} from "@/services/common.ts"; /** * 业务API服务 * 封装所有业务相关的API调用 */ -interface UploadResponse { - success: boolean; - message?: string; - url?: string; +interface FileStorage { + id: number; + filePath?: string; + url: string; + filename?: string; } +export type FileUploadResult = ApiResponse; + // 文件上传API export const fileUploadApi = { // 上传头像 - uploadAvatar(file: File): Promise { + async uploadAvatar(file: File): Promise { const formData = new FormData(); formData.append('file', file); - const url = '/api/upload/avatar'; - return fetch(url, { - method: 'POST', - body: formData, - }).then(async response => { - if (!response.ok) { - const text = await response.text().catch(() => ''); - throw new Error(`Upload failed: ${response.status} ${text}`); + try { + const response = await axios.post(url, formData, { + headers: { + 'Content-Type': 'multipart/form-data' + } + }); + if (response.data.success) { + return response.data.data ?? null; } - const ct = response.headers.get('content-type') || ''; - if (ct.includes('application/json')) { - return await response.json(); + throw new Error(response.data.message); + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + return null; } - const text = await response.text(); - return { success: true, message: 'ok', url: text }; - }); + throw error; + } }, }; diff --git a/data-agent-frontend/src/views/AgentCreate.vue b/data-agent-frontend/src/views/AgentCreate.vue index 23ce4f3f2..1ee608f0d 100644 --- a/data-agent-frontend/src/views/AgentCreate.vue +++ b/data-agent-frontend/src/views/AgentCreate.vue @@ -32,29 +32,35 @@
智能体头像
- + + + 重新生成 - - + + + + + + {{ uploading ? '上传中...' : '上传图片' }}
@@ -66,16 +72,16 @@
- +
@@ -86,11 +92,11 @@
@@ -101,11 +107,11 @@
@@ -116,9 +122,9 @@
@@ -126,14 +132,14 @@
- - - + + +
@@ -147,12 +153,12 @@
取消 {{ loading ? '创建中...' : '创建智能体' }} @@ -165,377 +171,374 @@ diff --git a/data-agent-frontend/src/views/AgentDetail.vue b/data-agent-frontend/src/views/AgentDetail.vue index 389a82c93..2c1bcb2a1 100644 --- a/data-agent-frontend/src/views/AgentDetail.vue +++ b/data-agent-frontend/src/views/AgentDetail.vue @@ -22,100 +22,118 @@
{{ agent.name }}
{{ headerUploading ? '上传中...' : '替换头像' }}

{{ agent.name }}

 - + - + + + 基本信息 - + + + 数据源配置 - + + + 自定义 PROMPT 配置 - + + + 智能体知识配置 - + + + 业务知识配置 - + + + 语义模型配置 - + + + 预设问题管理 - + + + 前往运行页面 - + + + 访问 API @@ -125,33 +143,33 @@ @@ -161,12 +179,47 @@ diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java index 4e9ae3454..ad9409b70 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java @@ -15,8 +15,6 @@ */ package com.alibaba.cloud.ai.dataagent.controller; -import com.alibaba.cloud.ai.dataagent.service.file.ByteArrayMultipartFile; -import com.alibaba.cloud.ai.dataagent.vo.PageResult; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.AgentKnowledgeQueryDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.CreateKnowledgeDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.UpdateKnowledgeDTO; @@ -24,19 +22,27 @@ import com.alibaba.cloud.ai.dataagent.vo.AgentKnowledgeVO; import com.alibaba.cloud.ai.dataagent.vo.ApiResponse; import com.alibaba.cloud.ai.dataagent.vo.PageResponse; +import com.alibaba.cloud.ai.dataagent.vo.PageResult; import jakarta.validation.Valid; +import java.util.List; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.http.MediaType; import org.springframework.http.codec.multipart.FilePart; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import java.util.List; - /** * Agent Knowledge Management Controller */ @@ -79,39 +85,18 @@ public Mono> createKnowledge(@RequestPart("agentId @RequestPart(value = "content", required = false) String content, @RequestPart(value = "file", required = false) FilePart filePart, @RequestPart(value = "splitterType", required = false) String splitterType) { + + return Mono.fromCallable(() -> { + CreateKnowledgeDTO dto = buildCreateKnowledgeDTO(agentId, title, type, question, content, filePart, + splitterType); + AgentKnowledgeVO knowledge = agentKnowledgeService.createKnowledge(dto); + return ApiResponse.success("创建知识成功，后台向量存储开始更新，请耐心等待...", knowledge); + }).subscribeOn(Schedulers.boundedElastic()); - // 如果没有文件，直接同步处理 - if (filePart == null) { - return Mono.fromCallable(() -> { - CreateKnowledgeDTO dto = buildCreateKnowledgeDTO(agentId, title, type, question, content, null, - splitterType); - AgentKnowledgeVO knowledge = agentKnowledgeService.createKnowledge(dto); - return ApiResponse.success("创建知识成功，后台向量存储开始更新，请耐心等待...", knowledge); - }).subscribeOn(Schedulers.boundedElastic()); - } - - // 有文件时，先读取文件内容再处理 - String filename = filePart.filename(); - String fileContentType = filePart.headers().getContentType() != null - ? filePart.headers().getContentType().toString() : "application/octet-stream"; - - return DataBufferUtils.join(filePart.content()).flatMap(dataBuffer -> { - byte[] bytes = new byte[dataBuffer.readableByteCount()]; - dataBuffer.read(bytes); - DataBufferUtils.release(dataBuffer); - - return Mono.fromCallable(() -> { - MultipartFile multipartFile = new ByteArrayMultipartFile(bytes, filename, fileContentType); - CreateKnowledgeDTO dto = buildCreateKnowledgeDTO(agentId, title, type, question, content, multipartFile, - splitterType); - AgentKnowledgeVO knowledge = agentKnowledgeService.createKnowledge(dto); - return ApiResponse.success("创建知识成功，后台向量存储开始更新，请耐心等待...", knowledge); - }).subscribeOn(Schedulers.boundedElastic()); - }); } private CreateKnowledgeDTO buildCreateKnowledgeDTO(String agentId, String title, String type, String question, - String content, MultipartFile file, String splitterType) { + String content, FilePart file, String splitterType) { CreateKnowledgeDTO dto = new CreateKnowledgeDTO(); dto.setAgentId(Integer.parseInt(agentId)); dto.setTitle(title); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/FileUploadController.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/FileUploadController.java index 59f23f980..e1b4f1628 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/FileUploadController.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/FileUploadController.java @@ -17,8 +17,8 @@ import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; -import com.alibaba.cloud.ai.dataagent.vo.UploadResponse; -import org.springframework.http.server.reactive.ServerHttpRequest; +import com.alibaba.cloud.ai.dataagent.vo.ApiResponse; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -27,13 +27,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.http.codec.multipart.FilePart; +import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; -import org.springframework.http.codec.multipart.FilePart; import reactor.core.publisher.Mono; /** @@ -57,29 +58,25 @@ public class FileUploadController { * 上传头像图片 */ @PostMapping(value = "/avatar", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public Mono> uploadAvatar(@RequestPart("file") FilePart file) { + public Mono> uploadAvatar(@RequestPart("file") FilePart file) { // 验证文件类型 String contentType = file.headers().getContentType() != null ? file.headers().getContentType().toString() : null; if (contentType == null || !contentType.startsWith("image/")) { - return Mono.just(ResponseEntity.badRequest().body(UploadResponse.error("只支持图片文件"))); + return Mono.just(ApiResponse.error("只支持图片文件")); } if (file.headers().getContentLength() > fileStorageProperties.getImageSize()) { - return Mono.just(ResponseEntity.badRequest().body(UploadResponse.error("图片大小超过最大限制"))); + return Mono.just(ApiResponse.error("图片大小超过最大限制")); } // 使用文件存储服务存储文件 - return fileStorageService.storeFile(file, "avatars").map(filePath -> { - String fileUrl = fileStorageService.getFileUrl(filePath); - // 提取文件名 - String filename = filePath.substring(filePath.lastIndexOf("/") + 1); - return ResponseEntity.ok(UploadResponse.ok("上传成功", fileUrl, filename)); - }).onErrorResume(e -> { - log.error("头像上传失败", e); - return Mono - .just(ResponseEntity.internalServerError().body(UploadResponse.error("上传失败: " + e.getMessage()))); - }); + return fileStorageService.storeFile(file, "avatars") + .map(fileStorageVo -> ApiResponse.success("上传成功", fileStorageVo)) + .onErrorResume(e -> { + log.error("头像上传失败", e); + return Mono.just(ApiResponse.error("上传失败: " + e.getMessage())); + }); } /** diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/converter/AgentKnowledgeConverter.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/converter/AgentKnowledgeConverter.java index 6df895a78..fea29a7c9 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/converter/AgentKnowledgeConverter.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/converter/AgentKnowledgeConverter.java @@ -20,9 +20,9 @@ import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.vo.AgentKnowledgeVO; -import org.springframework.stereotype.Component; - +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; import java.time.LocalDateTime; +import org.springframework.stereotype.Component; @Component public class AgentKnowledgeConverter { @@ -45,7 +45,7 @@ public AgentKnowledgeVO toVo(AgentKnowledge po) { return vo; } - public AgentKnowledge toEntityForCreate(CreateKnowledgeDTO createKnowledgeDto, String storagePath) { + public AgentKnowledge toEntityForCreate(CreateKnowledgeDTO createKnowledgeDto, FileStorageVo fileStorageVo) { // 创建AgentKnowledge对象 AgentKnowledge knowledge = new AgentKnowledge(); knowledge.setAgentId(createKnowledgeDto.getAgentId()); @@ -64,11 +64,8 @@ public AgentKnowledge toEntityForCreate(CreateKnowledgeDTO createKnowledgeDto, S knowledge.setUpdatedTime(now); // 如果是文档类型，设置文件相关信息 - if (createKnowledgeDto.getFile() != null && !createKnowledgeDto.getFile().isEmpty()) { - knowledge.setSourceFilename(createKnowledgeDto.getFile().getOriginalFilename()); - knowledge.setFilePath(storagePath); - knowledge.setFileSize(createKnowledgeDto.getFile().getSize()); - knowledge.setFileType(createKnowledgeDto.getFile().getContentType()); + if (fileStorageVo != null) { + knowledge.setFileId(fileStorageVo.getId()); } // 设置分块策略类型，默认值为token diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/dto/knowledge/agentknowledge/CreateKnowledgeDTO.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/dto/knowledge/agentknowledge/CreateKnowledgeDTO.java index 57742f07d..f55e0cccd 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/dto/knowledge/agentknowledge/CreateKnowledgeDTO.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/dto/knowledge/agentknowledge/CreateKnowledgeDTO.java @@ -20,7 +20,7 @@ import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import lombok.Data; -import org.springframework.web.multipart.MultipartFile; +import org.springframework.http.codec.multipart.FilePart; /** * 创建知识DTO @@ -60,7 +60,7 @@ public class CreateKnowledgeDTO { /** * 上传的文件（当type=DOCUMENT时必填） */ - private MultipartFile file; + private FilePart file; /** * 分块策略类型：token, recursive 默认值是 token diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/AgentKnowledge.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/AgentKnowledge.java index 2535db44e..6daf0e22b 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/AgentKnowledge.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/AgentKnowledge.java @@ -18,12 +18,11 @@ import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.fasterxml.jackson.annotation.JsonFormat; +import java.time.LocalDateTime; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.time.LocalDateTime; - /** * Agent Knowledge Entity Class */ @@ -56,6 +55,8 @@ public class AgentKnowledge { // 操作失败的错误信息 private String errorMsg; + private Long fileId; + private String sourceFilename; // 文件路径 diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/FileStorage.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/FileStorage.java new file mode 100644 index 000000000..92c15e136 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/entity/FileStorage.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.entity; + +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; +import com.fasterxml.jackson.annotation.JsonFormat; +import java.time.LocalDateTime; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Data +public class FileStorage { + + private Long id; + + // 文件名 + private String filename; + + // 文件路径 + private String filePath; + + // 文件大小（字节） + private Long fileSize; + + // 文件o类型 + private String fileType; + + // 文件后缀 + private String fileExtension; + + // 存储类型 + private FileStorageProviderEnum storageType; + + // 0=未删除, 1=已删除 + private Integer isDeleted; + + // 0=物理资源未清理, 1=物理资源已清理 + // 默认值是 0 + private Integer isCleaned; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private LocalDateTime createdTime; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private LocalDateTime updatedTime; + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileDeletionEvent.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileDeletionEvent.java new file mode 100644 index 000000000..29469ed41 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileDeletionEvent.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.event; + +import java.time.Clock; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class FileDeletionEvent extends ApplicationEvent { + + private final Long fileId; + + public FileDeletionEvent(Object source, Long fileId) { + super(source, Clock.systemDefaultZone()); + this.fileId = fileId; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java new file mode 100644 index 000000000..62addb3fd --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.event; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import java.time.LocalDateTime; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +@Component +@Slf4j +@RequiredArgsConstructor +public class FileEventListener { + + private final FileStorageMapper fileStorageMapper; + + private final FileStorageService fileStorageService; + + @Async("dbOperationExecutor") + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void handleDeletionEvent(FileDeletionEvent event) { + Long id = event.getFileId(); + log.info("Starting async resource cleanup for fileId: {}", id); + + // 1. 重新查询 + FileStorage fileStorage = fileStorageMapper.findById(id); + if (fileStorage == null) { + log.warn("File record physically missing, skipping cleanup. ID: {}", id); + return; + } + + try { + // 3. 删除文件 + boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage.getFilePath()); + + // 4. 更新清理状态 + if (fileDeleted) { + // 只有都成功了，才标记为资源已清理 + fileStorage.setIsCleaned(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.update(fileStorage); + log.info("Resources cleaned up successfully. FileId: {}", id); + } + else { + log.error("Cleanup incomplete. FileId: {}, FileDeleted: {}", id, fileDeleted); + // isResourceCleaned=0，有定时任务兜底清理。 + } + + } + catch (Exception e) { + log.error("Exception during async cleanup for agentKnowledgeId: {}", id, e); + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/AgentKnowledgeMapper.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/AgentKnowledgeMapper.java index a2a122b65..e8a1a4079 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/AgentKnowledgeMapper.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/AgentKnowledgeMapper.java @@ -17,10 +17,14 @@ import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.AgentKnowledgeQueryDTO; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; -import org.apache.ibatis.annotations.*; - import java.time.LocalDateTime; import java.util.List; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Options; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; @Mapper public interface AgentKnowledgeMapper { @@ -37,8 +41,8 @@ public interface AgentKnowledgeMapper { @Insert(""" - INSERT INTO agent_knowledge (agent_id, title, content, type, question, is_recall, embedding_status, source_filename, file_path, file_size, file_type, splitter_type, is_deleted, is_resource_cleaned, created_time, updated_time) - VALUES (#{agentId}, #{title}, #{content}, #{type}, #{question}, #{isRecall}, #{embeddingStatus}, #{sourceFilename}, #{filePath}, #{fileSize}, #{fileType}, #{splitterType}, #{isDeleted}, #{isResourceCleaned}, #{createdTime}, #{updatedTime}) + INSERT INTO agent_knowledge (agent_id, title, content, type, question, file_id, is_recall, embedding_status, splitter_type, is_deleted, is_resource_cleaned, created_time, updated_time) + VALUES (#{agentId}, #{title}, #{content}, #{type}, #{question}, #{fileId}, #{isRecall}, #{embeddingStatus}, #{splitterType}, #{isDeleted}, #{isResourceCleaned}, #{createdTime}, #{updatedTime}) """) @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/FileStorageMapper.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/FileStorageMapper.java new file mode 100644 index 000000000..cf6587c6e --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/mapper/FileStorageMapper.java @@ -0,0 +1,81 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.mapper; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import java.time.LocalDateTime; +import java.util.List; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Options; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +@Mapper +public interface FileStorageMapper { + + @Select(""" + SELECT * FROM file_storage ORDER BY created_time DESC + """) + List findAll(); + + @Select(""" + SELECT * FROM file_storage WHERE id = #{id} + """) + FileStorage findById(Long id); + + @Select(""" + SELECT * FROM file_storage WHERE file_path = #{filePath} + """) + FileStorage findByFilePath(String filePath); + + @Insert(""" + INSERT INTO file_storage (filename, file_path, file_size, file_type, file_extension, storage_type, is_deleted, is_cleaned, created_time, updated_time) + VALUES (#{filename}, #{filePath}, #{fileSize}, #{fileType}, #{fileExtension}, #{storageType}, #{isDeleted}, #{isCleaned}, #{createdTime}, #{updatedTime}) + """) + @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") + int insert(FileStorage fileStorage); + + @Update(""" + + """) + int update(FileStorage fileStorage); + + @Delete(""" + DELETE FROM file_storage WHERE id = #{id} + """) + int deleteById(Long id); + + @Select(""" + SELECT * FROM file_storage + WHERE is_deleted = 1 + AND is_cleaned = 0 + AND updated_time < #{beforeTime} + LIMIT #{limit} + """) + List selectDirtyRecords(LocalDateTime beforeTime, int limit); + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java index c10101083..c75922e37 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java @@ -16,7 +16,7 @@ package com.alibaba.cloud.ai.dataagent.properties; import com.alibaba.cloud.ai.dataagent.constant.Constant; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceEnum; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; import java.nio.file.Path; import lombok.Getter; import lombok.Setter; @@ -35,7 +35,7 @@ public class FileStorageProperties { /** * 存储类型：local（本地存储）、oss（阿里云OSS） */ - private FileStorageServiceEnum type = FileStorageServiceEnum.LOCAL; + private FileStorageProviderEnum type = FileStorageProviderEnum.LOCAL; /** * 对象存储路径前缀（通用配置，对OSS和本地存储都适用） diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java index 51e889c67..812589145 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java @@ -17,16 +17,15 @@ import com.alibaba.cloud.ai.dataagent.entity.Agent; import com.alibaba.cloud.ai.dataagent.mapper.AgentMapper; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.alibaba.cloud.ai.dataagent.service.vectorstore.AgentVectorStoreService; import com.alibaba.cloud.ai.dataagent.util.ApiKeyUtil; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - import java.time.LocalDateTime; import java.util.HashMap; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; /** * Agent Service Class @@ -40,7 +39,7 @@ public class AgentServiceImpl implements AgentService { private final AgentVectorStoreService agentVectorStoreService; - private final FileStorageService fileStorageService; + private final FileStorageProvider fileStorageService; @Override public List findAll() { diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java new file mode 100644 index 000000000..d5a506de1 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java @@ -0,0 +1,87 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; +import java.time.LocalDateTime; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +@RequiredArgsConstructor +public class FileResourceCleanerTask { + + private final FileStorageMapper mapper; + + private final FileStorageService fileStorageService; + + /** + * 每隔 1 小时执行一次兜底清理 cron = "0 0 * * * ?" (整点执行) + */ + @Scheduled(cron = "0 0 * * * ?") + public void cleanupZombieResources() { + log.info("Starting zombie resources cleanup task..."); + + // 1. 定义时间缓冲：只处理 30 分钟前删除的数据 + // 这样不会跟用户刚刚操作的异步任务冲突 + LocalDateTime timeBuffer = LocalDateTime.now().minusMinutes(30); + int batchSize = 100; + + // 2. 查询脏数据 + List dirtyRecords = mapper.selectDirtyRecords(timeBuffer, batchSize); + + if (dirtyRecords.isEmpty()) { + log.info("No zombie resources found. Task finished."); + return; + } + + log.info("Found {} zombie records to clean.", dirtyRecords.size()); + + // 3. 逐条清理 + for (FileStorage fileStorage : dirtyRecords) { + try { + cleanupSingleRecord(fileStorage); + } + catch (Exception e) { + // 单条失败不影响其他记录，只记录日志，等下个周期再试 + log.error("Failed to clean resources for ID: {}", fileStorage.getId(), e); + } + } + } + + private void cleanupSingleRecord(FileStorage fileStorage) { + Long id = fileStorage.getId(); + // B. 删除文件 + boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage.getFilePath()); + + // C. 如果都清理干净了，更新数据库状态 + if (fileDeleted) { + fileStorage.setIsCleaned(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + mapper.update(fileStorage); + log.info("Zombie resource cleaned: ID={}", id); + } + else { + log.warn("Partial cleanup for ID={}, FileDel={}", id, fileDeleted); + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java new file mode 100644 index 000000000..53efb7f39 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import org.springframework.core.io.Resource; +import org.springframework.http.codec.multipart.FilePart; + +public interface FileStorageProvider { + + /** + * 存储文件 + * @param file 上传的文件 + * @param fileStorage 文件存储信息 + */ + void storeFile(FilePart file, FileStorage fileStorage); + + /** + * 删除文件 + * @param filePath 文件路径 + * @return 是否删除成功 + */ + boolean deleteFile(String filePath); + + /** + * 获取文件访问URL + * @param filePath 文件路径 + * @return 访问URL + */ + String getFileUrl(String filePath); + + /** + * 获取文件资源对象 + * @param filePath 文件路径 + * @return 文件资源对象 + */ + Resource getFileResource(String filePath); + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java similarity index 94% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java index ed5bb2377..e59eaa280 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java @@ -15,7 +15,7 @@ */ package com.alibaba.cloud.ai.dataagent.service.file; -public enum FileStorageServiceEnum { +public enum FileStorageProviderEnum { LOCAL, OSS diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java similarity index 76% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java index b042bdfef..df92a72e9 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java @@ -17,33 +17,33 @@ import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.impls.LocalFileStorageServiceImpl; -import com.alibaba.cloud.ai.dataagent.service.file.impls.OssFileStorageServiceImpl; +import com.alibaba.cloud.ai.dataagent.service.file.impls.LocalFileStorageProviderImpl; +import com.alibaba.cloud.ai.dataagent.service.file.impls.OssFileStorageProviderImpl; import lombok.AllArgsConstructor; import org.springframework.beans.factory.FactoryBean; import org.springframework.stereotype.Component; @Component @AllArgsConstructor -public class FileStorageServiceFactory implements FactoryBean { +public class FileStorageProviderFactory implements FactoryBean { private final FileStorageProperties properties; private final OssStorageProperties ossProperties; @Override - public FileStorageService getObject() { - if (FileStorageServiceEnum.OSS.equals(properties.getType())) { - return new OssFileStorageServiceImpl(properties, ossProperties); + public FileStorageProvider getObject() { + if (FileStorageProviderEnum.OSS.equals(properties.getType())) { + return new OssFileStorageProviderImpl(properties, ossProperties); } else { - return new LocalFileStorageServiceImpl(properties); + return new LocalFileStorageProviderImpl(properties); } } @Override public Class getObjectType() { - return FileStorageService.class; + return FileStorageProvider.class; } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java index b20fbb587..ca76c650e 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java @@ -15,9 +15,10 @@ */ package com.alibaba.cloud.ai.dataagent.service.file; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; -import org.springframework.web.multipart.MultipartFile; import reactor.core.publisher.Mono; public interface FileStorageService { @@ -28,22 +29,28 @@ public interface FileStorageService { * @param subPath 子路径（如 "avatars"） * @return 存储后的文件路径 */ - Mono storeFile(FilePart filePart, String subPath); + Mono storeFile(FilePart filePart, String subPath); /** - * 存储文件（同步版本，用于传统同步代码） - * @param file 上传的文件 - * @param subPath 子路径（如 "avatars"） - * @return 存储后的文件路径 + * 通过文件Id获取文件存储信息 + * @param id 文件ID + * @return 文件存储信息 */ - String storeFile(MultipartFile file, String subPath); + FileStorage getFileById(Long id); /** * 删除文件 * @param filePath 文件路径 * @return 是否删除成功 */ - boolean deleteFile(String filePath); + boolean deleteFileResource(String filePath); + + /** + * 删除文件 + * @param id 文件Id + * @return 是否删除成功 + */ + boolean deleteFileById(Long id); /** * 获取文件访问URL diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java new file mode 100644 index 000000000..a6cb54335 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java @@ -0,0 +1,145 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file.impls; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.event.FileDeletionEvent; +import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; +import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; +import java.time.LocalDateTime; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.codec.multipart.FilePart; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; + +@Service +@RequiredArgsConstructor +@Slf4j +public class FileStorageServiceImpl implements FileStorageService { + + private final FileStorageProvider fileStorageProvider; + + private final FileStorageProperties fileStorageProperties; + + private final FileStorageMapper fileStorageMapper; + + private final ApplicationEventPublisher eventPublisher; + + public Mono storeFile(FilePart file, String subPath) { + + if (file == null || file.headers().getContentLength() == 0 || !StringUtils.hasText(file.filename())) { + log.warn("文件为空，无法上传"); + throw new IllegalArgumentException("文件为空，无法上传"); + } + + String originalFilename = file.filename(); + String extension = ""; + if (originalFilename.contains(".")) { + extension = originalFilename.substring(originalFilename.lastIndexOf(".")); + } + String filename = UUID.randomUUID() + extension; + + String filePath = buildFilePath(subPath, filename); + + // 获取 Content-Type + MediaType contentType = file.headers().getContentType(); + String contentTypeStr = contentType != null ? contentType.toString() : "application/octet-stream"; + + FileStorage storage = FileStorage.builder() + .filename(originalFilename) + .filePath(filePath) + .fileSize(file.headers().getContentLength()) + .fileType(contentTypeStr) + .fileExtension(extension) + .storageType(fileStorageProperties.getType()) + .isDeleted(0) + .isCleaned(0) + .build(); + + fileStorageProvider.storeFile(file, storage); + + storage.setCreatedTime(LocalDateTime.now()); + storage.setUpdatedTime(LocalDateTime.now()); + + fileStorageMapper.insert(storage); + + return Mono.just(FileStorageVo.builder() + .id(storage.getId()) + .filePath(storage.getFilePath()) + .url(getFileUrl(filePath)) + .filename(storage.getFilename()) + .build()); + } + + @Override + public FileStorage getFileById(Long id) { + return fileStorageMapper.findById(id); + } + + public boolean deleteFileResource(String filePath) { + return fileStorageProvider.deleteFile(filePath); + } + + public boolean deleteFileById(Long id) { + FileStorage fileStorage = fileStorageMapper.findById(id); + return deleteFile(fileStorage); + } + + private boolean deleteFile(FileStorage fileStorage) { + fileStorage.setIsDeleted(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.update(fileStorage); + eventPublisher.publishEvent(new FileDeletionEvent(this, fileStorage.getId())); + return true; + } + + public String getFileUrl(String filePath) { + return fileStorageProvider.getFileUrl(filePath); + } + + public Resource getFileResource(String filePath) { + return fileStorageProvider.getFileResource(filePath); + } + + /** + * 构建存储路径 + */ + private String buildFilePath(String subPath, String filename) { + StringBuilder pathBuilder = new StringBuilder(); + + if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { + pathBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); + } + + if (StringUtils.hasText(subPath)) { + pathBuilder.append(subPath).append("/"); + } + + pathBuilder.append(filename); + + return pathBuilder.toString(); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java similarity index 54% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java index 5bce82a1b..6d97a5ad8 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java @@ -15,79 +15,39 @@ */ package com.alibaba.cloud.ai.dataagent.service.file.impls; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.UUID; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; -import org.springframework.util.StringUtils; -import org.springframework.web.multipart.MultipartFile; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; @Slf4j @AllArgsConstructor -public class LocalFileStorageServiceImpl implements FileStorageService { +public class LocalFileStorageProviderImpl implements FileStorageProvider { private final FileStorageProperties fileStorageProperties; @Override - public Mono storeFile(FilePart filePart, String subPath) { - String originalFilename = filePart.filename(); - String extension = ""; - if (originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - - String storagePath = buildStoragePath(subPath, filename); - - Path filePath = fileStorageProperties.getLocalBasePath().resolve(storagePath); - - checkPathSecurity(filePath); - - return Mono.fromCallable(() -> { - Path uploadDir = filePath.getParent(); - if (!Files.exists(uploadDir)) { - Files.createDirectories(uploadDir); - } - return filePath; - }).subscribeOn(Schedulers.boundedElastic()).flatMap(filePart::transferTo).then(Mono.fromCallable(() -> { - log.info("文件存储成功: {}", storagePath); - return storagePath; - })); - } - - @Override - public String storeFile(MultipartFile file, String subPath) { + public void storeFile(FilePart file, FileStorage fileStorage) { try { - String originalFilename = file.getOriginalFilename(); - String extension = ""; - if (originalFilename != null && originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - String storagePath = buildStoragePath(subPath, filename); + Path storagePath = fileStorageProperties.getLocalBasePath().resolve(fileStorage.getFilePath()); - Path filePath = fileStorageProperties.getLocalBasePath().resolve(storagePath); + checkPathSecurity(storagePath); - checkPathSecurity(filePath); - - Path uploadDir = filePath.getParent(); + Path uploadDir = storagePath.getParent(); if (!Files.exists(uploadDir)) { Files.createDirectories(uploadDir); } - Files.copy(file.getInputStream(), filePath); + file.transferTo(storagePath).block(); - log.info("文件存储成功: {}", storagePath); - return storagePath; + log.info("文件存储成功: {}", fileStorage); } catch (IOException e) { @@ -146,23 +106,4 @@ private void checkPathSecurity(Path filePath) { } } - /** - * 构建本地存储路径 - */ - private String buildStoragePath(String subPath, String filename) { - StringBuilder pathBuilder = new StringBuilder(); - - if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { - pathBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); - } - - if (StringUtils.hasText(subPath)) { - pathBuilder.append(subPath).append("/"); - } - - pathBuilder.append(filename); - - return pathBuilder.toString(); - } - } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java similarity index 59% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java index 66d74b0db..00b3a13f9 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java @@ -15,27 +15,25 @@ */ package com.alibaba.cloud.ai.dataagent.service.file.impls; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; import com.aliyun.oss.model.OSSObject; import com.aliyun.oss.model.ObjectMetadata; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.UUID; +import java.nio.file.Files; +import java.nio.file.Path; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; -import org.springframework.core.io.buffer.DataBufferUtils; -import org.springframework.http.MediaType; import org.springframework.http.codec.multipart.FilePart; import org.springframework.util.StringUtils; -import org.springframework.web.multipart.MultipartFile; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -43,7 +41,7 @@ * 阿里云OSS文件存储服务实现 */ @Slf4j -public class OssFileStorageServiceImpl implements FileStorageService { +public class OssFileStorageProviderImpl implements FileStorageProvider { private final FileStorageProperties fileStorageProperties; @@ -51,7 +49,7 @@ public class OssFileStorageServiceImpl implements FileStorageService { private OSS ossClient; - public OssFileStorageServiceImpl(FileStorageProperties fileStorageProperties, OssStorageProperties ossProperties) { + public OssFileStorageProviderImpl(FileStorageProperties fileStorageProperties, OssStorageProperties ossProperties) { this.fileStorageProperties = fileStorageProperties; this.ossProperties = ossProperties; } @@ -72,79 +70,32 @@ public void destroy() { } @Override - public Mono storeFile(FilePart file, String subPath) { - if (file == null || !StringUtils.hasText(file.filename())) { - log.warn("文件为空，无法上传到OSS"); - return Mono.error(new IllegalArgumentException("文件为空，无法上传到OSS")); - } - - String originalFilename = file.filename(); - String extension = ""; - if (originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - String objectKey = buildObjectKey(subPath, filename); - - // 获取 Content-Type - MediaType contentType = file.headers().getContentType(); - String contentTypeStr = contentType != null ? contentType.toString() : "application/octet-stream"; - - // 使用 DataBufferUtils 收集文件内容，然后在 boundedElastic 线程池上执行 OSS 上传 - return DataBufferUtils.join(file.content()).flatMap(dataBuffer -> { - byte[] bytes = new byte[dataBuffer.readableByteCount()]; - dataBuffer.read(bytes); - DataBufferUtils.release(dataBuffer); - - return Mono.fromCallable(() -> { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(bytes.length); - metadata.setContentType(contentTypeStr); - metadata.setCacheControl("no-cache"); - - try (InputStream inputStream = new ByteArrayInputStream(bytes)) { - ossClient.putObject(ossProperties.getBucketName(), objectKey, inputStream, metadata); - log.info("文件上传成功: {}", objectKey); - return objectKey; - } - }).subscribeOn(Schedulers.boundedElastic()); - }).onErrorMap(e -> { - log.error("文件存储失败，上传OSS失败", e); - return new RuntimeException("文件存储失败: " + e.getMessage(), e); - }); - } - - @Override - public String storeFile(MultipartFile file, String subPath) { + public void storeFile(FilePart file, FileStorage fileStorage) { try { - if (file == null || file.isEmpty()) { - log.warn("文件为空，无法上传到OSS"); - throw new IllegalArgumentException("文件为空，无法上传到OSS"); - } - - String originalFilename = file.getOriginalFilename(); - String extension = ""; - if (originalFilename != null && originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - - String objectKey = buildObjectKey(subPath, filename); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(file.getSize()); - metadata.setContentType(file.getContentType()); + metadata.setContentLength(fileStorage.getFileSize()); + metadata.setContentType(fileStorage.getFileType()); metadata.setCacheControl("no-cache"); - try (InputStream inputStream = file.getInputStream()) { - ossClient.putObject(ossProperties.getBucketName(), objectKey, inputStream, metadata); - log.info("文件上传成功: {}", objectKey); - return objectKey; - } - } - catch (IOException e) { - log.error("文件存储失败，获取输入流错误", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); + Path tempFile = Path.of("/tmp/uploads/" + fileStorage.getFilePath()); + + file.transferTo(tempFile).then(Mono.fromCallable(() -> { + // 在阻塞线程池中处理文件 + try (InputStream is = Files.newInputStream(tempFile)) { + ossClient.putObject(ossProperties.getBucketName(), fileStorage.getFilePath(), is, metadata); + log.info("文件上传成功: {}", fileStorage); + return "处理成功"; + } + }).subscribeOn(Schedulers.boundedElastic())).publishOn(Schedulers.boundedElastic()).doFinally(signal -> { + // 清理临时文件 + try { + Files.deleteIfExists(tempFile); + } + catch (IOException e) { + // 记录日志 + } + }).block(); + } catch (Exception e) { log.error("文件存储失败，上传OSS失败", e); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java index 218fa1fd4..e1e6c5613 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java @@ -17,11 +17,14 @@ import com.alibaba.cloud.ai.dataagent.constant.Constant; import com.alibaba.cloud.ai.dataagent.constant.DocumentMetadataConstant; -import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; -import com.alibaba.cloud.ai.dataagent.util.DocumentConverterUtil; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.service.vectorstore.AgentVectorStoreService; +import com.alibaba.cloud.ai.dataagent.util.DocumentConverterUtil; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; @@ -30,10 +33,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - // 智能体知识的向量资源和文件资源管理 @Slf4j @Component @@ -75,11 +74,15 @@ private void processQaKnowledge(AgentKnowledge knowledge) { private void processDocumentKnowledge(AgentKnowledge knowledge) { + String filePath = knowledge.getFilePath(); + if (knowledge.getFileId() != null) { + filePath = fileStorageService.getFileById(knowledge.getFileId()).getFilePath(); + } + // 处理文档 - List documents = getAndSplitDocument(knowledge.getFilePath(), knowledge.getSplitterType()); + List documents = getAndSplitDocument(filePath, knowledge.getSplitterType()); if (documents == null || documents.isEmpty()) { - log.error("No documents extracted from file: knowledgeId={}, filePath={}", knowledge.getId(), - knowledge.getFilePath()); + log.error("No documents extracted from file: knowledgeId={}, filePath={}", knowledge.getId(), filePath); throw new RuntimeException("No documents extracted from file"); } @@ -90,7 +93,7 @@ private void processDocumentKnowledge(AgentKnowledge knowledge) { // 添加到向量存储 agentVectorStoreService.addDocuments(knowledge.getAgentId().toString(), documentsWithMetadata); log.info("Successfully vectorized DOCUMENT knowledge: id={}, filePath={}, documentCount={}, splitterType={}", - knowledge.getId(), knowledge.getFilePath(), documentsWithMetadata.size(), knowledge.getSplitterType()); + knowledge.getId(), filePath, documentsWithMetadata.size(), knowledge.getSplitterType()); } @@ -155,7 +158,7 @@ public boolean deleteKnowledgeFile(AgentKnowledge knowledge) { } try { - boolean fileDeleted = fileStorageService.deleteFile(knowledge.getFilePath()); + boolean fileDeleted = fileStorageService.deleteFileResource(knowledge.getFilePath()); if (fileDeleted) { log.info("Successfully deleted knowledge file, filePath: {}", knowledge.getFilePath()); return true; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java index 2ed251384..50cb18f10 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java @@ -15,19 +15,22 @@ */ package com.alibaba.cloud.ai.dataagent.service.knowledge; -import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; -import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.converter.AgentKnowledgeConverter; -import com.alibaba.cloud.ai.dataagent.vo.PageResult; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.AgentKnowledgeQueryDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.CreateKnowledgeDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.UpdateKnowledgeDTO; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; +import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.event.AgentKnowledgeDeletionEvent; import com.alibaba.cloud.ai.dataagent.event.AgentKnowledgeEmbeddingEvent; import com.alibaba.cloud.ai.dataagent.mapper.AgentKnowledgeMapper; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.vo.AgentKnowledgeVO; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; +import com.alibaba.cloud.ai.dataagent.vo.PageResult; +import java.time.LocalDateTime; +import java.util.List; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; @@ -35,9 +38,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; -import java.time.LocalDateTime; -import java.util.List; - @Slf4j @Service @AllArgsConstructor @@ -62,13 +62,14 @@ public AgentKnowledgeVO getKnowledgeById(Integer id) { @Override @Transactional public AgentKnowledgeVO createKnowledge(CreateKnowledgeDTO createKnowledgeDto) { - String storagePath = null; + FileStorageVo fileStorage = null; checkCreateKnowledgeDto(createKnowledgeDto); if (createKnowledgeDto.getType().equals(KnowledgeType.DOCUMENT.getCode())) { // 将文件保存到磁盘 try { - storagePath = fileStorageService.storeFile(createKnowledgeDto.getFile(), AGENT_KNOWLEDGE_FILE_PATH); + fileStorage = fileStorageService.storeFile(createKnowledgeDto.getFile(), AGENT_KNOWLEDGE_FILE_PATH) + .block(); } catch (Exception e) { log.error("Failed to store file, agentId:{} title:{} type:{} ", createKnowledgeDto.getAgentId(), @@ -77,7 +78,7 @@ public AgentKnowledgeVO createKnowledge(CreateKnowledgeDTO createKnowledgeDto) { } } - AgentKnowledge knowledge = agentKnowledgeConverter.toEntityForCreate(createKnowledgeDto, storagePath); + AgentKnowledge knowledge = agentKnowledgeConverter.toEntityForCreate(createKnowledgeDto, fileStorage); if (agentKnowledgeMapper.insert(knowledge) <= 0) { log.error("Failed to create knowledge, agentId:{} title:{} type:{} ", knowledge.getAgentId(), @@ -153,6 +154,9 @@ public boolean deleteKnowledge(Integer id) { knowledge.setUpdatedTime(LocalDateTime.now()); if (agentKnowledgeMapper.update(knowledge) > 0) { + if (knowledge.getFileId() != null) { + fileStorageService.deleteFileById(knowledge.getFileId()); + } eventPublisher.publishEvent(new AgentKnowledgeDeletionEvent(this, id)); return true; } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java new file mode 100644 index 000000000..ed688943a --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.vo; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 通用上传响应实体。 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class FileStorageVo { + + private Long id; + + private String url; + + private String filePath; + + private String filename; + +} diff --git a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql index 5eadd0907..013463d42 100644 --- a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql +++ b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql @@ -2,25 +2,25 @@ -- 智能体表 CREATE TABLE IF NOT EXISTS agent ( - id INT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL COMMENT '智能体名称', - description TEXT COMMENT '智能体描述', - avatar TEXT COMMENT '头像URL', - status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', - api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', - api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', - prompt TEXT COMMENT '自定义Prompt配置', - category VARCHAR(100) COMMENT '分类', - admin_id BIGINT COMMENT '管理员ID', - tags TEXT COMMENT '标签，逗号分隔', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (id), - INDEX idx_agent_name (name), - INDEX idx_agent_status (status), - INDEX idx_agent_category (category), - INDEX idx_agent_admin_id (admin_id) - ) ENGINE = InnoDB COMMENT = '智能体表'; + id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL COMMENT '智能体名称', + description TEXT COMMENT '智能体描述', + avatar TEXT COMMENT '头像URL', + status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', + api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', + api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', + prompt TEXT COMMENT '自定义Prompt配置', + category VARCHAR(100) COMMENT '分类', + admin_id BIGINT COMMENT '管理员ID', + tags TEXT COMMENT '标签，逗号分隔', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (id), + INDEX idx_agent_name (name), + INDEX idx_agent_status (status), + INDEX idx_agent_category (category), + INDEX idx_agent_admin_id (admin_id) +) ENGINE = InnoDB COMMENT = '智能体表'; -- 业务知识表 CREATE TABLE IF NOT EXISTS business_knowledge ( @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS business_knowledge ( INDEX idx_business_knowledge_is_recall (is_recall), INDEX idx_business_knowledge_embedding_status (embedding_status), INDEX idx_business_knowledge_is_deleted (is_deleted), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '业务知识表'; -- 语义模型表 @@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS semantic_model ( INDEX idx_semantic_model_agent_id (agent_id), INDEX idx_semantic_model_business_name (business_name), INDEX idx_semantic_model_status (status), - CONSTRAINT fk_semantic_model_agent FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + CONSTRAINT fk_semantic_model_agent FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '语义模型表'; @@ -133,7 +133,7 @@ CREATE TABLE IF NOT EXISTS logical_relation ( PRIMARY KEY (id), INDEX idx_logical_relation_datasource_id (datasource_id), INDEX idx_logical_relation_source_table (datasource_id, source_table_name), - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '逻辑外键配置表'; -- 智能体数据源关联表 @@ -149,8 +149,8 @@ CREATE TABLE IF NOT EXISTS agent_datasource ( INDEX idx_agent_datasource_agent_id (agent_id), INDEX idx_agent_datasource_datasource_id (datasource_id), INDEX idx_agent_datasource_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE, - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE, + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体数据源关联表'; -- 智能体预设问题表 @@ -166,7 +166,7 @@ CREATE TABLE IF NOT EXISTS agent_preset_question ( INDEX idx_agent_preset_question_agent_id (agent_id), INDEX idx_agent_preset_question_sort_order (sort_order), INDEX idx_agent_preset_question_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体预设问题表'; -- 会话表 @@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS chat_session ( INDEX idx_chat_session_status (status), INDEX idx_chat_session_is_pinned (is_pinned), INDEX idx_chat_session_create_time (create_time), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天会话表'; -- 消息表 @@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS chat_message ( INDEX idx_chat_message_role (role), INDEX idx_chat_message_message_type (message_type), INDEX idx_chat_message_create_time (create_time), - FOREIGN KEY (session_id) REFERENCES chat_session(id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES chat_session (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天消息表'; -- 用户Prompt配置表 @@ -228,19 +228,18 @@ CREATE TABLE IF NOT EXISTS user_prompt_config ( INDEX idx_user_prompt_config_display_order (display_order ASC) ) ENGINE = InnoDB COMMENT = '用户Prompt配置表'; -CREATE TABLE IF NOT EXISTS agent_datasource_tables -( - id INT AUTO_INCREMENT PRIMARY KEY, - agent_datasource_id INT NOT NULL COMMENT '智能体数据源ID', - table_name VARCHAR(255) NOT NULL COMMENT '数据表名', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '更新时间', - CONSTRAINT uk_agent_datasource_tables_agent_datasource_id_table_name - UNIQUE (agent_datasource_id, table_name), - CONSTRAINT fk_agent_datasource_tables_agent_datasource_id - FOREIGN KEY (agent_datasource_id) REFERENCES agent_datasource (id) - ON UPDATE CASCADE ON DELETE CASCADE - ) ENGINE = InnoDB COMMENT = '某个智能体某个数据源所选中的数据表'; +CREATE TABLE IF NOT EXISTS agent_datasource_tables ( + id INT AUTO_INCREMENT PRIMARY KEY, + agent_datasource_id INT NOT NULL COMMENT '智能体数据源ID', + table_name VARCHAR(255) NOT NULL COMMENT '数据表名', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '更新时间', + CONSTRAINT uk_agent_datasource_tables_agent_datasource_id_table_name + UNIQUE (agent_datasource_id, table_name), + CONSTRAINT fk_agent_datasource_tables_agent_datasource_id + FOREIGN KEY (agent_datasource_id) REFERENCES agent_datasource (id) + ON UPDATE CASCADE ON DELETE CASCADE +) ENGINE = InnoDB COMMENT = '某个智能体某个数据源所选中的数据表'; -- 模型配置表 @@ -250,7 +249,7 @@ CREATE TABLE IF NOT EXISTS `model_config` ( `base_url` varchar(255) NOT NULL COMMENT '关键配置', `api_key` varchar(255) NOT NULL COMMENT 'API密钥', `model_name` varchar(255) NOT NULL COMMENT '模型名称', - `temperature` decimal(10,2) DEFAULT '0.00' COMMENT '温度参数', + `temperature` decimal(10, 2) DEFAULT '0.00' COMMENT '温度参数', `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', @@ -266,4 +265,23 @@ CREATE TABLE IF NOT EXISTS `model_config` ( `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', PRIMARY KEY (`id`) -) ENGINE=InnoDB; +) ENGINE = InnoDB; + +-- 文件存储表 +CREATE TABLE `file_storage` ( + `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', + `filename` VARCHAR(255) NOT NULL COMMENT '文件名', + `file_path` VARCHAR(1024) NOT NULL COMMENT '文件路径', + `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', + `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', + `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', + `storage_type` VARCHAR(50) NOT NULL COMMENT '存储类型：LOCAL/S3/OSS', + `is_deleted` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '删除状态：0=未删除, 1=已删除', + `is_cleaned` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '清理状态：0=物理资源未清理, 1=物理资源已清理', + `created_time` DATETIME DEFAULT NULL COMMENT '创建时间', + `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_filename` (`filename`), + KEY `idx_file_path` (`file_path`), + KEY `idx_updated_time` (`updated_time`) +) ENGINE = InnoDB COMMENT ='文件存储表'; diff --git a/data-agent-management/src/main/resources/sql/schema.sql b/data-agent-management/src/main/resources/sql/schema.sql index 160bcedf0..28174fff9 100644 --- a/data-agent-management/src/main/resources/sql/schema.sql +++ b/data-agent-management/src/main/resources/sql/schema.sql @@ -2,25 +2,25 @@ -- 智能体表 CREATE TABLE IF NOT EXISTS agent ( - id INT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL COMMENT '智能体名称', - description TEXT COMMENT '智能体描述', - avatar TEXT COMMENT '头像URL', - status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', - api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', - api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', - prompt TEXT COMMENT '自定义Prompt配置', - category VARCHAR(100) COMMENT '分类', - admin_id BIGINT COMMENT '管理员ID', - tags TEXT COMMENT '标签，逗号分隔', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (id), - INDEX idx_name (name), - INDEX idx_status (status), - INDEX idx_category (category), - INDEX idx_admin_id (admin_id) - ) ENGINE = InnoDB COMMENT = '智能体表'; + id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL COMMENT '智能体名称', + description TEXT COMMENT '智能体描述', + avatar TEXT COMMENT '头像URL', + status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', + api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', + api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', + prompt TEXT COMMENT '自定义Prompt配置', + category VARCHAR(100) COMMENT '分类', + admin_id BIGINT COMMENT '管理员ID', + tags TEXT COMMENT '标签，逗号分隔', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (id), + INDEX idx_name (name), + INDEX idx_status (status), + INDEX idx_category (category), + INDEX idx_admin_id (admin_id) +) ENGINE = InnoDB COMMENT = '智能体表'; -- 业务知识表 CREATE TABLE IF NOT EXISTS business_knowledge ( @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS business_knowledge ( INDEX idx_is_recall (is_recall), INDEX idx_embedding_status (embedding_status), INDEX idx_is_deleted (is_deleted), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '业务知识表'; -- 语义模型表 @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS `semantic_model` ( KEY `idx_field_name` (`business_name`) USING BTREE, KEY `idx_status` (`status`) USING BTREE, CONSTRAINT `fk_semantic_model_agent` FOREIGN KEY (`agent_id`) REFERENCES `agent` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='语义模型表'; +) ENGINE = InnoDB AUTO_INCREMENT = 4 DEFAULT CHARSET = utf8mb4 COMMENT ='语义模型表'; -- 智能体知识表 @@ -78,6 +78,7 @@ CREATE TABLE IF NOT EXISTS `agent_knowledge` ( `is_recall` int(11) DEFAULT 1 COMMENT '业务状态: 1=召回, 0=非召回', `embedding_status` varchar(20) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '向量化状态：PENDING待处理，PROCESSING处理中，COMPLETED已完成，FAILED失败', `error_msg` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '操作失败的错误信息', + `file_id` int COMMENT '文件ID', `source_filename` varchar(500) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '上传时的原始文件名', `file_path` varchar(500) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '文件在服务器上的物理存储路径', `file_size` bigint(20) DEFAULT NULL COMMENT '文件大小 (字节)', @@ -88,10 +89,10 @@ CREATE TABLE IF NOT EXISTS `agent_knowledge` ( `is_deleted` int(11) DEFAULT 0 COMMENT '逻辑删除字段，0=未删除, 1=已删除', `is_resource_cleaned` int(11) DEFAULT 0 COMMENT '0=物理资源（文件和向量）未清理, 1=物理资源已清理', PRIMARY KEY (`id`) USING BTREE, - KEY `idx_agent_id_status` (`agent_id`,`is_recall`) USING BTREE, + KEY `idx_agent_id_status` (`agent_id`, `is_recall`) USING BTREE, KEY `idx_embedding_status` (`embedding_status`) USING BTREE, KEY `idx_is_deleted` (`is_deleted`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=18 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='智能体知识源管理表 (支持文档、QA、FAQ)'; +) ENGINE = InnoDB AUTO_INCREMENT = 18 DEFAULT CHARSET = utf8mb4 COMMENT ='智能体知识源管理表 (支持文档、QA、FAQ)'; -- 数据源表 CREATE TABLE IF NOT EXISTS datasource ( @@ -133,7 +134,7 @@ CREATE TABLE IF NOT EXISTS logical_relation ( PRIMARY KEY (id), INDEX idx_datasource_id (datasource_id) COMMENT '加速根据数据源查找关系的查询', INDEX idx_source_table (datasource_id, source_table_name) COMMENT '加速根据表名查找关系的查询', - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT = '逻辑外键配置表'; -- 智能体数据源关联表 @@ -149,8 +150,8 @@ CREATE TABLE IF NOT EXISTS agent_datasource ( INDEX idx_agent_id (agent_id), INDEX idx_datasource_id (datasource_id), INDEX idx_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE, - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE, + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体数据源关联表'; -- 智能体预设问题表 @@ -166,7 +167,7 @@ CREATE TABLE IF NOT EXISTS agent_preset_question ( INDEX idx_agent_id (agent_id), INDEX idx_sort_order (sort_order), INDEX idx_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体预设问题表'; -- 会话表 @@ -185,7 +186,7 @@ CREATE TABLE IF NOT EXISTS chat_session ( INDEX idx_status (status), INDEX idx_is_pinned (is_pinned), INDEX idx_create_time (create_time), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天会话表'; -- 消息表 @@ -202,7 +203,7 @@ CREATE TABLE IF NOT EXISTS chat_message ( INDEX idx_role (role), INDEX idx_message_type (message_type), INDEX idx_create_time (create_time), - FOREIGN KEY (session_id) REFERENCES chat_session(id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES chat_session (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天消息表'; -- 用户Prompt配置表 @@ -228,43 +229,61 @@ CREATE TABLE IF NOT EXISTS user_prompt_config ( INDEX idx_display_order (display_order ASC) ) ENGINE = InnoDB COMMENT = '用户Prompt配置表'; -create table if not exists agent_datasource_tables -( - id int auto_increment primary key, - agent_datasource_id int not null comment '智能体数据源ID', - table_name varchar(255) not null comment '数据表名', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null comment '更新时间', - constraint agent_datasource_tables_agent_datasource_id_table_name_uindex - unique (agent_datasource_id, table_name), - constraint agent_datasource_tables_agent_datasource_id_fk - foreign key (agent_datasource_id) references agent_datasource (id) - on update cascade on delete cascade +create table if not exists agent_datasource_tables ( + id int auto_increment primary key, + agent_datasource_id int not null comment '智能体数据源ID', + table_name varchar(255) not null comment '数据表名', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null comment '更新时间', + constraint agent_datasource_tables_agent_datasource_id_table_name_uindex + unique (agent_datasource_id, table_name), + constraint agent_datasource_tables_agent_datasource_id_fk + foreign key (agent_datasource_id) references agent_datasource (id) + on update cascade on delete cascade ) - comment '某个智能体某个数据源所选中的数据表'; + comment '某个智能体某个数据源所选中的数据表'; -- 模型配置表 CREATE TABLE IF NOT EXISTS `model_config` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `provider` varchar(255) NOT NULL COMMENT '厂商标识 (方便前端展示回显，实际调用主要靠 baseUrl)', - `base_url` varchar(255) NOT NULL COMMENT '关键配置', - `api_key` varchar(255) NOT NULL COMMENT 'API密钥', - `model_name` varchar(255) NOT NULL COMMENT '模型名称', - `temperature` decimal(10,2) unsigned DEFAULT '0.00' COMMENT '温度参数', - `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', - `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', - `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', - `completions_path` varchar(255) DEFAULT NULL COMMENT 'Chat模型专用。附加到 Base URL 的路径。例如OpenAi的/v1/chat/completions', - `embeddings_path` varchar(255) DEFAULT NULL COMMENT '嵌入模型专用。附加到 Base URL 的路径。', - `created_time` datetime DEFAULT NULL COMMENT '创建时间', - `updated_time` datetime DEFAULT NULL COMMENT '更新时间', - `is_deleted` int(11) DEFAULT '0' COMMENT '0=未删除, 1=已删除', - -- 新增 AI 代理配置字段（默认关闭以确保零侵入性） - `proxy_enabled` tinyint(1) DEFAULT '0' COMMENT '是否启用代理：0-禁用，1-启用', - `proxy_host` varchar(255) DEFAULT NULL COMMENT '代理主机地址', - `proxy_port` int(11) DEFAULT NULL COMMENT '代理端口', - `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', - `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', - PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `id` int(11) NOT NULL AUTO_INCREMENT, + `provider` varchar(255) NOT NULL COMMENT '厂商标识 (方便前端展示回显，实际调用主要靠 baseUrl)', + `base_url` varchar(255) NOT NULL COMMENT '关键配置', + `api_key` varchar(255) NOT NULL COMMENT 'API密钥', + `model_name` varchar(255) NOT NULL COMMENT '模型名称', + `temperature` decimal(10, 2) unsigned DEFAULT '0.00' COMMENT '温度参数', + `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', + `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', + `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', + `completions_path` varchar(255) DEFAULT NULL COMMENT 'Chat模型专用。附加到 Base URL 的路径。例如OpenAi的/v1/chat/completions', + `embeddings_path` varchar(255) DEFAULT NULL COMMENT '嵌入模型专用。附加到 Base URL 的路径。', + `created_time` datetime DEFAULT NULL COMMENT '创建时间', + `updated_time` datetime DEFAULT NULL COMMENT '更新时间', + `is_deleted` int(11) DEFAULT '0' COMMENT '0=未删除, 1=已删除', + -- 新增 AI 代理配置字段（默认关闭以确保零侵入性） + `proxy_enabled` tinyint(1) DEFAULT '0' COMMENT '是否启用代理：0-禁用，1-启用', + `proxy_host` varchar(255) DEFAULT NULL COMMENT '代理主机地址', + `proxy_port` int(11) DEFAULT NULL COMMENT '代理端口', + `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', + `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', + PRIMARY KEY (`id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + +-- 文件存储表 +CREATE TABLE `file_storage` ( + `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', + `filename` VARCHAR(255) NOT NULL COMMENT '文件名', + `file_path` VARCHAR(1024) NOT NULL COMMENT '文件路径', + `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', + `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', + `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', + `storage_type` VARCHAR(50) NOT NULL COMMENT '存储类型：LOCAL/S3/OSS', + `is_deleted` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '删除状态：0=未删除, 1=已删除', + `is_cleaned` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '清理状态：0=物理资源未清理, 1=物理资源已清理', + `created_time` DATETIME DEFAULT NULL COMMENT '创建时间', + `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_filename` (`filename`), + KEY `idx_file_path` (`file_path`), + KEY `idx_updated_time` (`updated_time`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT ='文件存储表'; From b132bfb758d6b2d831ffb744435d5af7e88398f7 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Tue, 3 Feb 2026 17:48:52 +0800 Subject: [PATCH 2/6] refactor: refactor del file resource logic --- .../src/services/fileUpload.ts | 6 +- data-agent-frontend/src/views/AgentCreate.vue | 771 +++++++++--------- data-agent-frontend/src/views/AgentDetail.vue | 463 ++++++----- .../ai/dataagent/event/FileEventListener.java | 13 +- .../service/file/FileResourceCleanerTask.java | 8 +- .../service/file/FileStorageService.java | 7 + .../file/impls/FileStorageServiceImpl.java | 13 + 7 files changed, 644 insertions(+), 637 deletions(-) diff --git a/data-agent-frontend/src/services/fileUpload.ts b/data-agent-frontend/src/services/fileUpload.ts index 26ad920b3..eb5dacfa1 100644 --- a/data-agent-frontend/src/services/fileUpload.ts +++ b/data-agent-frontend/src/services/fileUpload.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import axios from 'axios'; -import {ApiResponse} from "@/services/common.ts"; +import { ApiResponse } from '@/services/common.ts'; /** * 业务API服务 @@ -40,8 +40,8 @@ export const fileUploadApi = { try { const response = await axios.post(url, formData, { headers: { - 'Content-Type': 'multipart/form-data' - } + 'Content-Type': 'multipart/form-data', + }, }); if (response.data.success) { return response.data.data ?? null; diff --git a/data-agent-frontend/src/views/AgentCreate.vue b/data-agent-frontend/src/views/AgentCreate.vue index 1ee608f0d..f65ae9acb 100644 --- a/data-agent-frontend/src/views/AgentCreate.vue +++ b/data-agent-frontend/src/views/AgentCreate.vue @@ -32,35 +32,35 @@
智能体头像
- + 重新生成 - + - + {{ uploading ? '上传中...' : '上传图片' }}
@@ -72,16 +72,16 @@
- +
@@ -92,11 +92,11 @@
@@ -107,11 +107,11 @@
@@ -122,9 +122,9 @@
@@ -132,14 +132,14 @@
- - - + + +
@@ -153,12 +153,12 @@
取消 {{ loading ? '创建中...' : '创建智能体' }} @@ -171,374 +171,373 @@ diff --git a/data-agent-frontend/src/views/AgentDetail.vue b/data-agent-frontend/src/views/AgentDetail.vue index 2c1bcb2a1..c1d377cc6 100644 --- a/data-agent-frontend/src/views/AgentDetail.vue +++ b/data-agent-frontend/src/views/AgentDetail.vue @@ -22,59 +22,59 @@
{{ agent.name }}
{{ headerUploading ? '上传中...' : '替换头像' }}

{{ agent.name }}

 - + - + 基本信息 @@ -82,7 +82,7 @@ - + 数据源配置 @@ -90,7 +90,7 @@ - + 自定义 PROMPT 配置 @@ -98,19 +98,19 @@ - + 智能体知识配置 - + 业务知识配置 - + 语义模型配置 @@ -118,7 +118,7 @@ - + 预设问题管理 @@ -126,13 +126,13 @@ - + 前往运行页面 - + 访问 API @@ -143,33 +143,33 @@ @@ -179,221 +179,220 @@ diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java index 62addb3fd..c7190f405 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/FileEventListener.java @@ -18,7 +18,6 @@ import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; -import java.time.LocalDateTime; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Async; @@ -49,20 +48,14 @@ public void handleDeletionEvent(FileDeletionEvent event) { } try { - // 3. 删除文件 - boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage.getFilePath()); - - // 4. 更新清理状态 + // 2. 删除文件 + boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage); if (fileDeleted) { - // 只有都成功了，才标记为资源已清理 - fileStorage.setIsCleaned(1); - fileStorage.setUpdatedTime(LocalDateTime.now()); - fileStorageMapper.update(fileStorage); log.info("Resources cleaned up successfully. FileId: {}", id); } else { log.error("Cleanup incomplete. FileId: {}, FileDeleted: {}", id, fileDeleted); - // isResourceCleaned=0，有定时任务兜底清理。 + // isCleaned=0，有定时任务兜底清理。 } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java index d5a506de1..7d86087f3 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java @@ -68,15 +68,11 @@ public void cleanupZombieResources() { } private void cleanupSingleRecord(FileStorage fileStorage) { + Long id = fileStorage.getId(); - // B. 删除文件 - boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage.getFilePath()); + boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage); - // C. 如果都清理干净了，更新数据库状态 if (fileDeleted) { - fileStorage.setIsCleaned(1); - fileStorage.setUpdatedTime(LocalDateTime.now()); - mapper.update(fileStorage); log.info("Zombie resource cleaned: ID={}", id); } else { diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java index ca76c650e..4c6fa2778 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java @@ -45,6 +45,13 @@ public interface FileStorageService { */ boolean deleteFileResource(String filePath); + /** + * 删除文件 + * @param fileStorage 文件信息 + * @return 是否删除成功 + */ + boolean deleteFileResource(FileStorage fileStorage); + /** * 删除文件 * @param id 文件Id diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java index a6cb54335..09e5202f7 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java @@ -102,6 +102,19 @@ public boolean deleteFileResource(String filePath) { return fileStorageProvider.deleteFile(filePath); } + public boolean deleteFileResource(FileStorage fileStorage) { + // 1. 删除文件 + boolean fileDeleted = deleteFileResource(fileStorage.getFilePath()); + + // 2. 更新清理状态 + if (fileDeleted) { + fileStorage.setIsCleaned(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.update(fileStorage); + } + return fileDeleted; + } + public boolean deleteFileById(Long id) { FileStorage fileStorage = fileStorageMapper.findById(id); return deleteFile(fileStorage); From 82de172ff35119f545a37f727179cf93db0e43ec Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Feb 2026 17:53:51 +0800 Subject: [PATCH 3/6] feat(file): make fileStorage can switch, get file resource by storage type --- .../config/DataAgentConfiguration.java | 122 +++++++++++++++--- .../config/FileStorageConfiguration.java | 52 ++++++++ .../controller/AgentKnowledgeController.java | 2 +- .../event/AgentKnowledgeEventListener.java | 7 +- .../properties/OssStorageProperties.java | 2 + .../file/FileStorageProviderFactory.java | 49 ------- .../service/file/FileStorageService.java | 8 +- .../file/impls/FileStorageServiceImpl.java | 35 +++-- .../impls/OssFileStorageProviderImpl.java | 21 +-- .../AgentKnowledgeResourceManager.java | 23 ++-- .../src/main/resources/application-h2.yml | 1 + .../src/main/resources/application.yml | 1 + 12 files changed, 208 insertions(+), 115 deletions(-) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java index 3aab99704..0888f6d0e 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java @@ -15,26 +15,120 @@ */ package com.alibaba.cloud.ai.dataagent.config; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.AGENT_ID; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.COLUMN_DOCUMENTS__FOR_SCHEMA_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.DB_DIALECT_TYPE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.EVIDENCE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.EVIDENCE_RECALL_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.FEASIBILITY_ASSESSMENT_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.FEASIBILITY_ASSESSMENT_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.GENEGRATED_SEMANTIC_MODEL_PROMPT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.HUMAN_FEEDBACK_DATA; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.HUMAN_FEEDBACK_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.HUMAN_REVIEW_ENABLED; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.INPUT_KEY; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.INTENT_RECOGNITION_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.INTENT_RECOGNITION_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.IS_ONLY_NL2SQL; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.MULTI_TURN_CONTEXT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.NL2SQL_GRAPH_NAME; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLANNER_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLANNER_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_CURRENT_STEP; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_EXECUTOR_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_NEXT_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_REPAIR_COUNT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_VALIDATION_ERROR; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PLAN_VALIDATION_STATUS; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_ANALYSIS_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_ANALYZE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_EXECUTE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_EXECUTE_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_FALLBACK_MODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_GENERATE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_GENERATE_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_IS_SUCCESS; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.PYTHON_TRIES_COUNT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.QUERY_ENHANCE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.QUERY_ENHANCE_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.REPORT_GENERATOR_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.RESULT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SCHEMA_RECALL_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SEMANTIC_CONSISTENCY_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SEMANTIC_CONSISTENCY_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_EXECUTE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_EXECUTE_NODE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_GENERATE_COUNT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_GENERATE_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_GENERATE_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_GENERATE_SCHEMA_MISSING_ADVICE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_REGENERATE_REASON; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.SQL_RESULT_LIST_MEMORY; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_DOCUMENTS_FOR_SCHEMA_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_EXCEPTION_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_NODE; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_OUTPUT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_RETRY_COUNT; +import static com.alibaba.cloud.ai.graph.StateGraph.END; +import static com.alibaba.cloud.ai.graph.StateGraph.START; +import static com.alibaba.cloud.ai.graph.action.AsyncEdgeAction.edge_async; + import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.properties.DataAgentProperties; -import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; +import com.alibaba.cloud.ai.dataagent.service.aimodelconfig.AiModelRegistry; import com.alibaba.cloud.ai.dataagent.service.vectorstore.SimpleVectorStoreInitialization; -import com.alibaba.cloud.ai.dataagent.splitter.SentenceSplitter; -import com.alibaba.cloud.ai.transformer.splitter.RecursiveCharacterTextSplitter; -import com.alibaba.cloud.ai.dataagent.splitter.SemanticTextSplitter; import com.alibaba.cloud.ai.dataagent.splitter.ParagraphTextSplitter; +import com.alibaba.cloud.ai.dataagent.splitter.SemanticTextSplitter; +import com.alibaba.cloud.ai.dataagent.splitter.SentenceSplitter; +import com.alibaba.cloud.ai.dataagent.strategy.EnhancedTokenCountBatchingStrategy; import com.alibaba.cloud.ai.dataagent.util.McpServerToolUtil; import com.alibaba.cloud.ai.dataagent.util.NodeBeanUtil; -import com.alibaba.cloud.ai.dataagent.service.aimodelconfig.AiModelRegistry; -import com.alibaba.cloud.ai.dataagent.strategy.EnhancedTokenCountBatchingStrategy; -import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.*; -import com.alibaba.cloud.ai.dataagent.workflow.node.*; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.FeasibilityAssessmentDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.HumanFeedbackDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.IntentRecognitionDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.PlanExecutorDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.PythonExecutorDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.QueryEnhanceDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.SQLExecutorDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.SchemaRecallDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.SemanticConsistenceDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.SqlGenerateDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.dispatcher.TableRelationDispatcher; +import com.alibaba.cloud.ai.dataagent.workflow.node.EvidenceRecallNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.FeasibilityAssessmentNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.HumanFeedbackNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.IntentRecognitionNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.PlanExecutorNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.PlannerNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.PythonAnalyzeNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.PythonExecuteNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.PythonGenerateNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.QueryEnhanceNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.ReportGeneratorNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.SchemaRecallNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.SemanticConsistencyNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.SqlExecuteNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.SqlGenerateNode; +import com.alibaba.cloud.ai.dataagent.workflow.node.TableRelationNode; import com.alibaba.cloud.ai.graph.GraphRepresentation; import com.alibaba.cloud.ai.graph.KeyStrategy; import com.alibaba.cloud.ai.graph.KeyStrategyFactory; import com.alibaba.cloud.ai.graph.StateGraph; import com.alibaba.cloud.ai.graph.exception.GraphStateException; +import com.alibaba.cloud.ai.transformer.splitter.RecursiveCharacterTextSplitter; import com.knuddels.jtokkit.api.EncodingType; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.embedding.BatchingStrategy; import org.springframework.ai.embedding.EmbeddingModel; @@ -67,16 +161,6 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.alibaba.cloud.ai.dataagent.constant.Constant.*; -import static com.alibaba.cloud.ai.graph.StateGraph.END; -import static com.alibaba.cloud.ai.graph.StateGraph.START; -import static com.alibaba.cloud.ai.graph.action.AsyncEdgeAction.edge_async; - /** * DataAgent的自动配置类 * @@ -86,7 +170,7 @@ @Slf4j @Configuration @EnableAsync -@EnableConfigurationProperties({ CodeExecutorProperties.class, DataAgentProperties.class, FileStorageProperties.class }) +@EnableConfigurationProperties({ CodeExecutorProperties.class, DataAgentProperties.class }) public class DataAgentConfiguration implements DisposableBean { /** diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java new file mode 100644 index 000000000..b9cc0a354 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.config; + +import com.alibaba.cloud.ai.dataagent.constant.Constant; +import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; +import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; +import com.alibaba.cloud.ai.dataagent.service.file.impls.LocalFileStorageProviderImpl; +import com.alibaba.cloud.ai.dataagent.service.file.impls.OssFileStorageProviderImpl; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration +@EnableConfigurationProperties({ FileStorageProperties.class, OssStorageProperties.class }) +@RequiredArgsConstructor +public class FileStorageConfiguration { + + private final FileStorageProperties fileStorageProperties; + + private final OssStorageProperties ossStorageProperties; + + @Bean("local") + public FileStorageProvider localFileStorageProvider() { + return new LocalFileStorageProviderImpl(fileStorageProperties); + } + + @Bean("oss") + @ConditionalOnProperty(name = Constant.PROJECT_PROPERTIES_PREFIX + ".file.oss.enabled", havingValue = "true") + public FileStorageProvider ossFileStorageProvider() { + return new OssFileStorageProviderImpl(fileStorageProperties, ossStorageProperties); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java index ad9409b70..f1fc071ac 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/controller/AgentKnowledgeController.java @@ -85,7 +85,7 @@ public Mono> createKnowledge(@RequestPart("agentId @RequestPart(value = "content", required = false) String content, @RequestPart(value = "file", required = false) FilePart filePart, @RequestPart(value = "splitterType", required = false) String splitterType) { - + return Mono.fromCallable(() -> { CreateKnowledgeDTO dto = buildCreateKnowledgeDTO(agentId, title, type, question, content, filePart, splitterType); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/AgentKnowledgeEventListener.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/AgentKnowledgeEventListener.java index def2d4aa1..dded88c56 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/AgentKnowledgeEventListener.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/event/AgentKnowledgeEventListener.java @@ -15,10 +15,11 @@ */ package com.alibaba.cloud.ai.dataagent.event; -import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; import com.alibaba.cloud.ai.dataagent.mapper.AgentKnowledgeMapper; import com.alibaba.cloud.ai.dataagent.service.knowledge.AgentKnowledgeResourceManager; +import java.time.LocalDateTime; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Async; @@ -26,8 +27,6 @@ import org.springframework.transaction.event.TransactionPhase; import org.springframework.transaction.event.TransactionalEventListener; -import java.time.LocalDateTime; - @Component @Slf4j @RequiredArgsConstructor @@ -58,7 +57,7 @@ public void handleEmbeddingEvent(AgentKnowledgeEmbeddingEvent event) { updateStatus(knowledge, EmbeddingStatus.PROCESSING, null); // 3. 执行核心向量化逻辑 - agentKnowledgeResourceManager.doEmbedingToVectorStore(knowledge); + agentKnowledgeResourceManager.doEmbeddingToVectorStore(knowledge); // 4. 更新状态为 COMPLETED updateStatus(knowledge, EmbeddingStatus.COMPLETED, null); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java index 4c6530018..bcba8210f 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java @@ -28,6 +28,8 @@ @ConfigurationProperties(prefix = Constant.PROJECT_PROPERTIES_PREFIX + ".file.oss") public class OssStorageProperties { + private boolean enabled = false; + /** * OSS访问密钥ID */ diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java deleted file mode 100644 index df92a72e9..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.file; - -import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; -import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.impls.LocalFileStorageProviderImpl; -import com.alibaba.cloud.ai.dataagent.service.file.impls.OssFileStorageProviderImpl; -import lombok.AllArgsConstructor; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.stereotype.Component; - -@Component -@AllArgsConstructor -public class FileStorageProviderFactory implements FactoryBean { - - private final FileStorageProperties properties; - - private final OssStorageProperties ossProperties; - - @Override - public FileStorageProvider getObject() { - if (FileStorageProviderEnum.OSS.equals(properties.getType())) { - return new OssFileStorageProviderImpl(properties, ossProperties); - } - else { - return new LocalFileStorageProviderImpl(properties); - } - } - - @Override - public Class getObjectType() { - return FileStorageProvider.class; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java index 4c6fa2778..f7e3dc866 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java @@ -61,16 +61,16 @@ public interface FileStorageService { /** * 获取文件访问URL - * @param filePath 文件路径 + * @param fileStorage 文件存储信息 * @return 访问URL */ - String getFileUrl(String filePath); + String getFileUrl(FileStorage fileStorage); /** * 获取文件资源对象 - * @param filePath 文件路径 + * @param fileStorage 文件路径 * @return 文件资源对象 */ - Resource getFileResource(String filePath); + Resource getFileResource(FileStorage fileStorage); } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java index 09e5202f7..3901d28b4 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java @@ -20,9 +20,12 @@ import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; import java.time.LocalDateTime; +import java.util.Locale; +import java.util.Map; import java.util.UUID; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -39,7 +42,7 @@ @Slf4j public class FileStorageServiceImpl implements FileStorageService { - private final FileStorageProvider fileStorageProvider; + private final Map fileStorageProviders; private final FileStorageProperties fileStorageProperties; @@ -78,33 +81,43 @@ public Mono storeFile(FilePart file, String subPath) { .isCleaned(0) .build(); - fileStorageProvider.storeFile(file, storage); + getStorageProvider(fileStorageProperties.getType()).storeFile(file, storage); storage.setCreatedTime(LocalDateTime.now()); storage.setUpdatedTime(LocalDateTime.now()); fileStorageMapper.insert(storage); - + return Mono.just(FileStorageVo.builder() .id(storage.getId()) .filePath(storage.getFilePath()) - .url(getFileUrl(filePath)) + .url(getFileUrl(storage)) .filename(storage.getFilename()) .build()); } + private FileStorageProvider getStorageProvider(FileStorageProviderEnum storageProviderEnum) { + if (storageProviderEnum == null) { + storageProviderEnum = fileStorageProperties.getType(); + } + if (!fileStorageProviders.containsKey(storageProviderEnum.name().toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("Invalid storage provider: " + storageProviderEnum); + } + return fileStorageProviders.get(storageProviderEnum.name().toLowerCase(Locale.ROOT)); + } + @Override public FileStorage getFileById(Long id) { return fileStorageMapper.findById(id); } public boolean deleteFileResource(String filePath) { - return fileStorageProvider.deleteFile(filePath); + return getStorageProvider(fileStorageProperties.getType()).deleteFile(filePath); } public boolean deleteFileResource(FileStorage fileStorage) { // 1. 删除文件 - boolean fileDeleted = deleteFileResource(fileStorage.getFilePath()); + boolean fileDeleted = getStorageProvider(fileStorage.getStorageType()).deleteFile(fileStorage.getFilePath()); // 2. 更新清理状态 if (fileDeleted) { @@ -128,12 +141,14 @@ private boolean deleteFile(FileStorage fileStorage) { return true; } - public String getFileUrl(String filePath) { - return fileStorageProvider.getFileUrl(filePath); + @Override + public String getFileUrl(FileStorage fileStorage) { + return getStorageProvider(fileStorage.getStorageType()).getFileUrl(fileStorage.getFilePath()); } - public Resource getFileResource(String filePath) { - return fileStorageProvider.getFileResource(filePath); + @Override + public Resource getFileResource(FileStorage fileStorage) { + return getStorageProvider(fileStorage.getStorageType()).getFileResource(fileStorage.getFilePath()); } /** diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java index 00b3a13f9..2078b3150 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java @@ -77,7 +77,7 @@ public void storeFile(FilePart file, FileStorage fileStorage) { metadata.setContentType(fileStorage.getFileType()); metadata.setCacheControl("no-cache"); - Path tempFile = Path.of("/tmp/uploads/" + fileStorage.getFilePath()); + Path tempFile = Path.of("/tmp", fileStorage.getFilePath()); file.transferTo(tempFile).then(Mono.fromCallable(() -> { // 在阻塞线程池中处理文件 @@ -166,23 +166,4 @@ public long contentLength() { }; } - /** - * 构建OSS对象键 - */ - private String buildObjectKey(String subPath, String filename) { - StringBuilder keyBuilder = new StringBuilder(); - - if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { - keyBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); - } - - if (StringUtils.hasText(subPath)) { - keyBuilder.append(subPath).append("/"); - } - - keyBuilder.append(filename); - - return keyBuilder.toString(); - } - } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java index e1e6c5613..68dc06e83 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java @@ -18,6 +18,7 @@ import com.alibaba.cloud.ai.dataagent.constant.Constant; import com.alibaba.cloud.ai.dataagent.constant.DocumentMetadataConstant; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.service.vectorstore.AgentVectorStoreService; @@ -51,7 +52,7 @@ public AgentKnowledgeResourceManager(TextSplitterFactory textSplitterFactory, Fi this.agentVectorStoreService = agentVectorStoreService; } - public void doEmbedingToVectorStore(AgentKnowledge agentKnowledge) throws Exception { + public void doEmbeddingToVectorStore(AgentKnowledge agentKnowledge) throws Exception { // delete old data this.deleteFromVectorStore(agentKnowledge.getAgentId(), agentKnowledge.getId()); @@ -74,15 +75,20 @@ private void processQaKnowledge(AgentKnowledge knowledge) { private void processDocumentKnowledge(AgentKnowledge knowledge) { - String filePath = knowledge.getFilePath(); + FileStorage fileStorage; if (knowledge.getFileId() != null) { - filePath = fileStorageService.getFileById(knowledge.getFileId()).getFilePath(); + fileStorage = fileStorageService.getFileById(knowledge.getFileId()); + } + else { + fileStorage = new FileStorage(); + fileStorage.setFilePath(knowledge.getFilePath()); } // 处理文档 - List documents = getAndSplitDocument(filePath, knowledge.getSplitterType()); + List documents = getAndSplitDocument(fileStorage, knowledge.getSplitterType()); if (documents == null || documents.isEmpty()) { - log.error("No documents extracted from file: knowledgeId={}, filePath={}", knowledge.getId(), filePath); + log.error("No documents extracted from file: knowledgeId={}, filePath={}", knowledge.getId(), + fileStorage.getFilePath()); throw new RuntimeException("No documents extracted from file"); } @@ -93,13 +99,14 @@ private void processDocumentKnowledge(AgentKnowledge knowledge) { // 添加到向量存储 agentVectorStoreService.addDocuments(knowledge.getAgentId().toString(), documentsWithMetadata); log.info("Successfully vectorized DOCUMENT knowledge: id={}, filePath={}, documentCount={}, splitterType={}", - knowledge.getId(), filePath, documentsWithMetadata.size(), knowledge.getSplitterType()); + knowledge.getId(), fileStorage.getFilePath(), documentsWithMetadata.size(), + knowledge.getSplitterType()); } - private List getAndSplitDocument(String filePath, String splitterType) { + private List getAndSplitDocument(FileStorage fileStorage, String splitterType) { // 使用FileStorageService获取文件资源对象 - Resource resource = fileStorageService.getFileResource(filePath); + Resource resource = fileStorageService.getFileResource(fileStorage); // 使用TikaDocumentReader读取文件 TikaDocumentReader tikaDocumentReader = new TikaDocumentReader(resource); diff --git a/data-agent-management/src/main/resources/application-h2.yml b/data-agent-management/src/main/resources/application-h2.yml index ac6c78d46..114ee0dda 100644 --- a/data-agent-management/src/main/resources/application-h2.yml +++ b/data-agent-management/src/main/resources/application-h2.yml @@ -50,6 +50,7 @@ spring: url-prefix: /uploads imageSize: 2097152 oss: + enabled: false access-key-id: ${OSS_ACCESS_KEY_ID:} access-key-secret: ${OSS_ACCESS_KEY_SECRET:} endpoint: ${OSS_ENDPOINT:} diff --git a/data-agent-management/src/main/resources/application.yml b/data-agent-management/src/main/resources/application.yml index 30f606553..92c24515b 100644 --- a/data-agent-management/src/main/resources/application.yml +++ b/data-agent-management/src/main/resources/application.yml @@ -47,6 +47,7 @@ spring: url-prefix: /uploads imageSize: 2097152 oss: + enabled: false access-key-id: ${OSS_ACCESS_KEY_ID:} access-key-secret: ${OSS_ACCESS_KEY_SECRET:} endpoint: ${OSS_ENDPOINT:} From aaa157533db18108fc3f06c28fceec97ac1ddd9e Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Mon, 9 Feb 2026 15:32:26 +0800 Subject: [PATCH 4/6] refactor: add delete temp file failed msg --- .../cloud/ai/dataagent/config/DataAgentConfiguration.java | 1 + .../ai/dataagent/config/FileStorageConfiguration.java | 2 +- .../service/file/impls/OssFileStorageProviderImpl.java | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java index 0888f6d0e..7764a8999 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java @@ -69,6 +69,7 @@ import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_NODE; import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_OUTPUT; import static com.alibaba.cloud.ai.dataagent.constant.Constant.TABLE_RELATION_RETRY_COUNT; +import static com.alibaba.cloud.ai.dataagent.constant.Constant.TRACE_THREAD_ID; import static com.alibaba.cloud.ai.graph.StateGraph.END; import static com.alibaba.cloud.ai.graph.StateGraph.START; import static com.alibaba.cloud.ai.graph.action.AsyncEdgeAction.edge_async; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java index b9cc0a354..015b207b9 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/FileStorageConfiguration.java @@ -46,7 +46,7 @@ public FileStorageProvider localFileStorageProvider() { @Bean("oss") @ConditionalOnProperty(name = Constant.PROJECT_PROPERTIES_PREFIX + ".file.oss.enabled", havingValue = "true") public FileStorageProvider ossFileStorageProvider() { - return new OssFileStorageProviderImpl(fileStorageProperties, ossStorageProperties); + return new OssFileStorageProviderImpl(ossStorageProperties); } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java index 2078b3150..2dce4816d 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java @@ -16,7 +16,6 @@ package com.alibaba.cloud.ai.dataagent.service.file.impls; import com.alibaba.cloud.ai.dataagent.entity.FileStorage; -import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.aliyun.oss.OSS; @@ -43,14 +42,11 @@ @Slf4j public class OssFileStorageProviderImpl implements FileStorageProvider { - private final FileStorageProperties fileStorageProperties; - private final OssStorageProperties ossProperties; private OSS ossClient; - public OssFileStorageProviderImpl(FileStorageProperties fileStorageProperties, OssStorageProperties ossProperties) { - this.fileStorageProperties = fileStorageProperties; + public OssFileStorageProviderImpl(OssStorageProperties ossProperties) { this.ossProperties = ossProperties; } @@ -92,7 +88,7 @@ public void storeFile(FilePart file, FileStorage fileStorage) { Files.deleteIfExists(tempFile); } catch (IOException e) { - // 记录日志 + log.warn("无法删除临时文件: {}", tempFile, e); } }).block(); From a4048f05d9d912ab2d899659c274234e287fb0c4 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Mon, 2 Mar 2026 09:26:03 +0800 Subject: [PATCH 5/6] refactor: modify the file_path to prefix index --- data-agent-management/src/main/resources/sql/h2/schema-h2.sql | 4 ++-- data-agent-management/src/main/resources/sql/schema.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql index 013463d42..cfa7c1c35 100644 --- a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql +++ b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql @@ -271,7 +271,7 @@ CREATE TABLE IF NOT EXISTS `model_config` ( CREATE TABLE `file_storage` ( `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', `filename` VARCHAR(255) NOT NULL COMMENT '文件名', - `file_path` VARCHAR(1024) NOT NULL COMMENT '文件路径', + `file_path` VARCHAR(512) NOT NULL COMMENT '文件路径', `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', @@ -282,6 +282,6 @@ CREATE TABLE `file_storage` ( `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', PRIMARY KEY (`id`), KEY `idx_filename` (`filename`), - KEY `idx_file_path` (`file_path`), + KEY `idx_file_path` (`file_path`(50)), KEY `idx_updated_time` (`updated_time`) ) ENGINE = InnoDB COMMENT ='文件存储表'; diff --git a/data-agent-management/src/main/resources/sql/schema.sql b/data-agent-management/src/main/resources/sql/schema.sql index 28174fff9..82d43603a 100644 --- a/data-agent-management/src/main/resources/sql/schema.sql +++ b/data-agent-management/src/main/resources/sql/schema.sql @@ -273,7 +273,7 @@ CREATE TABLE IF NOT EXISTS `model_config` ( CREATE TABLE `file_storage` ( `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', `filename` VARCHAR(255) NOT NULL COMMENT '文件名', - `file_path` VARCHAR(1024) NOT NULL COMMENT '文件路径', + `file_path` VARCHAR(512) NOT NULL COMMENT '文件路径', `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', @@ -284,6 +284,6 @@ CREATE TABLE `file_storage` ( `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', PRIMARY KEY (`id`), KEY `idx_filename` (`filename`), - KEY `idx_file_path` (`file_path`), + KEY `idx_file_path` (`file_path`(50)), KEY `idx_updated_time` (`updated_time`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT ='文件存储表'; From 826f668ec945f464e53f2680f90905b7e246d89b Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Mon, 2 Mar 2026 14:41:00 +0800 Subject: [PATCH 6/6] fix: fix epoll-2 block bug --- .../exception/InternalServerException.java | 4 + .../service/file/FileStorageProvider.java | 3 +- .../file/impls/FileStorageServiceImpl.java | 56 +++++++----- .../impls/LocalFileStorageProviderImpl.java | 29 ++++--- .../impls/OssFileStorageProviderImpl.java | 86 ++++++++++++------- 5 files changed, 114 insertions(+), 64 deletions(-) diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/exception/InternalServerException.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/exception/InternalServerException.java index 801096012..492b17cba 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/exception/InternalServerException.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/exception/InternalServerException.java @@ -21,4 +21,8 @@ public InternalServerException(String message) { super(message); } + public InternalServerException(String message, Throwable cause) { + super(message, cause); + } + } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java index 53efb7f39..20dabe90a 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java @@ -18,6 +18,7 @@ import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; +import reactor.core.publisher.Mono; public interface FileStorageProvider { @@ -26,7 +27,7 @@ public interface FileStorageProvider { * @param file 上传的文件 * @param fileStorage 文件存储信息 */ - void storeFile(FilePart file, FileStorage fileStorage); + Mono storeFile(FilePart file, FileStorage fileStorage); /** * 删除文件 diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java index 3901d28b4..804bf935d 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java @@ -17,12 +17,14 @@ import com.alibaba.cloud.ai.dataagent.entity.FileStorage; import com.alibaba.cloud.ai.dataagent.event.FileDeletionEvent; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; +import java.time.Duration; import java.time.LocalDateTime; import java.util.Locale; import java.util.Map; @@ -31,11 +33,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; import org.springframework.core.io.Resource; +import org.springframework.dao.DataAccessException; import org.springframework.http.MediaType; import org.springframework.http.codec.multipart.FilePart; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; @Service @RequiredArgsConstructor @@ -52,24 +56,23 @@ public class FileStorageServiceImpl implements FileStorageService { public Mono storeFile(FilePart file, String subPath) { + // 1. 前置校验（轻量同步操作，可立即执行） if (file == null || file.headers().getContentLength() == 0 || !StringUtils.hasText(file.filename())) { log.warn("文件为空，无法上传"); - throw new IllegalArgumentException("文件为空，无法上传"); + return Mono.error(new IllegalArgumentException("文件为空，无法上传")); // ✅ 响应式返回错误 } + // 2. 提取文件元数据 String originalFilename = file.filename(); - String extension = ""; - if (originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } + String extension = originalFilename.contains(".") + ? originalFilename.substring(originalFilename.lastIndexOf(".")) : ""; String filename = UUID.randomUUID() + extension; - String filePath = buildFilePath(subPath, filename); - // 获取 Content-Type MediaType contentType = file.headers().getContentType(); String contentTypeStr = contentType != null ? contentType.toString() : "application/octet-stream"; + // 3. 构建 FileStorage 实体 FileStorage storage = FileStorage.builder() .filename(originalFilename) .filePath(filePath) @@ -81,19 +84,32 @@ public Mono storeFile(FilePart file, String subPath) { .isCleaned(0) .build(); - getStorageProvider(fileStorageProperties.getType()).storeFile(file, storage); - - storage.setCreatedTime(LocalDateTime.now()); - storage.setUpdatedTime(LocalDateTime.now()); - - fileStorageMapper.insert(storage); - - return Mono.just(FileStorageVo.builder() - .id(storage.getId()) - .filePath(storage.getFilePath()) - .url(getFileUrl(storage)) - .filename(storage.getFilename()) - .build()); + // 4. 核心响应式链：文件存储 → 数据库插入 → 结果转换 + return getStorageProvider(storage.getStorageType()).storeFile(file, storage) + // 5. 文件存储成功后，执行数据库插入（阻塞 JDBC，需切换线程） + .flatMap(storedStorage -> Mono.fromCallable(() -> { + // ✅ 阻塞的 JDBC 操作包裹在 fromCallable 中 + storedStorage.setCreatedTime(LocalDateTime.now()); + storedStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.insert(storedStorage); + return storedStorage; + }).subscribeOn(Schedulers.boundedElastic())) + // 6. 数据库插入成功后，转换为 VO 返回给前端 + .map(storedStorage -> FileStorageVo.builder() + .id(storedStorage.getId()) + .filePath(storedStorage.getFilePath()) + .url(getFileUrl(storedStorage)) + .filename(storedStorage.getFilename()) + .build()) + // 7. 可观测性：日志埋点 + .doOnSubscribe(sub -> log.debug("开始处理文件上传: {}, subPath: {}", originalFilename, subPath)) + .doOnSuccess(vo -> log.info("文件上传并入库成功: fileId={}, url={}", vo.getId(), vo.getUrl())) + .doOnError(e -> log.error("文件上传流程失败: filename={}", originalFilename, e)) + // 8. 响应式错误处理：统一转换异常类型 + .onErrorMap(IllegalArgumentException.class, e -> e) // 参数错误直接抛出 + .onErrorMap(DataAccessException.class, e -> new InternalServerException("数据库操作失败: " + e.getMessage(), e)) + // 9. 可选：添加超时保护 + .timeout(Duration.ofSeconds(60)); } private FileStorageProvider getStorageProvider(FileStorageProviderEnum storageProviderEnum) { diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java index 6d97a5ad8..2faf9cf25 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java @@ -16,6 +16,7 @@ package com.alibaba.cloud.ai.dataagent.service.file.impls; import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import java.io.IOException; @@ -26,6 +27,8 @@ import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; @Slf4j @AllArgsConstructor @@ -34,9 +37,9 @@ public class LocalFileStorageProviderImpl implements FileStorageProvider { private final FileStorageProperties fileStorageProperties; @Override - public void storeFile(FilePart file, FileStorage fileStorage) { - try { - + public Mono storeFile(FilePart file, FileStorage fileStorage) { + return Mono.fromCallable(() -> { + // 1. 执行所有同步/阻塞的 IO 操作 Path storagePath = fileStorageProperties.getLocalBasePath().resolve(fileStorage.getFilePath()); checkPathSecurity(storagePath); @@ -45,15 +48,17 @@ public void storeFile(FilePart file, FileStorage fileStorage) { if (!Files.exists(uploadDir)) { Files.createDirectories(uploadDir); } - file.transferTo(storagePath).block(); - - log.info("文件存储成功: {}", fileStorage); - - } - catch (IOException e) { - log.error("文件存储失败", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); - } + return storagePath; // 返回计算结果给下一步 + }) + // 2. 关键：切换到 boundedElastic 线程池，避免阻塞 I/O 线程 + .subscribeOn(Schedulers.boundedElastic()) + // 3. 执行响应式文件传输（file.transferTo 返回 Mono） + .flatMap(storagePath -> file.transferTo(storagePath).thenReturn(fileStorage)) + // 4. 成功日志（可选） + .doOnSuccess(stored -> log.info("文件存储成功: {}", stored)) + // 5. 响应式错误处理 + .doOnError(e -> log.error("文件存储失败", e)) + .onErrorMap(IOException.class, e -> new InternalServerException("文件存储失败: " + e.getMessage(), e)); } @Override diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java index 2dce4816d..1a392da1a 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java @@ -16,6 +16,7 @@ package com.alibaba.cloud.ai.dataagent.service.file.impls; import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.aliyun.oss.OSS; @@ -28,6 +29,7 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.Resource; @@ -66,37 +68,59 @@ public void destroy() { } @Override - public void storeFile(FilePart file, FileStorage fileStorage) { - try { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(fileStorage.getFileSize()); - metadata.setContentType(fileStorage.getFileType()); - metadata.setCacheControl("no-cache"); - - Path tempFile = Path.of("/tmp", fileStorage.getFilePath()); - - file.transferTo(tempFile).then(Mono.fromCallable(() -> { - // 在阻塞线程池中处理文件 - try (InputStream is = Files.newInputStream(tempFile)) { - ossClient.putObject(ossProperties.getBucketName(), fileStorage.getFilePath(), is, metadata); - log.info("文件上传成功: {}", fileStorage); - return "处理成功"; - } - }).subscribeOn(Schedulers.boundedElastic())).publishOn(Schedulers.boundedElastic()).doFinally(signal -> { - // 清理临时文件 - try { - Files.deleteIfExists(tempFile); - } - catch (IOException e) { - log.warn("无法删除临时文件: {}", tempFile, e); - } - }).block(); - - } - catch (Exception e) { - log.error("文件存储失败，上传OSS失败", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); - } + public Mono storeFile(FilePart file, FileStorage fileStorage) { + + // 1. 准备 OSS 元数据（纯内存操作，可立即执行） + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(fileStorage.getFileSize()); + metadata.setContentType(fileStorage.getFileType()); + metadata.setCacheControl("no-cache"); + + // 2. 定义临时文件路径 + Path tempFile = Path.of("/tmp", fileStorage.getFilePath()); + + return Mono.defer(() -> { + // 确保每次订阅时重新执行（避免临时文件路径冲突） + + // 3. 第一步：将上传文件保存到本地临时文件（响应式 API） + return file.transferTo(tempFile) + + // 4. 第二步：上传到 OSS（阻塞操作，需切换线程） + .then(Mono.fromCallable(() -> { + // 确保父目录存在 + Files.createDirectories(tempFile.getParent()); + + // 阻塞 IO：读取本地文件 + 上传 OSS + try (InputStream is = Files.newInputStream(tempFile)) { + ossClient.putObject(ossProperties.getBucketName(), fileStorage.getFilePath(), is, metadata); + log.info("文件上传 OSS 成功: {}", fileStorage); + return fileStorage; // 返回业务对象 + } + }).subscribeOn(Schedulers.boundedElastic())) + + .publishOn(Schedulers.boundedElastic()) + + // 5. 第三步：无论成功失败，清理临时文件 + .doFinally(signal -> { + try { + Files.deleteIfExists(tempFile); + log.debug("临时文件已清理: {}", tempFile); + } + catch (IOException e) { + log.warn("清理临时文件失败: {}", tempFile, e); + // 注意：doFinally 中抛异常会影响主流程，建议只记录日志 + } + }) + + // 6. 响应式错误处理：转换异常类型 + .onErrorMap(IOException.class, e -> new InternalServerException("文件处理失败: " + e.getMessage(), e)) + // 7. 可选：添加超时保护，防止大文件卡死 + .timeout(Duration.ofSeconds(60)); + }) + // 8. 日志埋点（可观测性） + .doOnSubscribe(sub -> log.debug("开始处理文件上传: {}", fileStorage.getFilename())) + .doOnSuccess(stored -> log.info("文件上传流程完成: {}", stored)) + .doOnError(e -> log.error("文件上传流程异常: {}", fileStorage.getFilename(), e)); } @Override