diff --git a/data-agent-frontend/src/services/fileUpload.ts b/data-agent-frontend/src/services/fileUpload.ts index 34900543d..eb5dacfa1 100644 --- a/data-agent-frontend/src/services/fileUpload.ts +++ b/data-agent-frontend/src/services/fileUpload.ts @@ -13,40 +13,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import axios from 'axios'; +import { ApiResponse } from '@/services/common.ts'; /** * 业务API服务 * 封装所有业务相关的API调用 */ -interface UploadResponse { - success: boolean; - message?: string; - url?: string; +interface FileStorage { + id: number; + filePath?: string; + url: string; + filename?: string; } +export type FileUploadResult = ApiResponse; + // 文件上传API export const fileUploadApi = { // 上传头像 - uploadAvatar(file: File): Promise { + async uploadAvatar(file: File): Promise { const formData = new FormData(); formData.append('file', file); - const url = '/api/upload/avatar'; - return fetch(url, { - method: 'POST', - body: formData, - }).then(async response => { - if (!response.ok) { - const text = await response.text().catch(() => ''); - throw new Error(`Upload failed: ${response.status} ${text}`); + try { + const response = await axios.post(url, formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }); + if (response.data.success) { + return response.data.data ?? null; } - const ct = response.headers.get('content-type') || ''; - if (ct.includes('application/json')) { - return await response.json(); + throw new Error(response.data.message); + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + return null; } - const text = await response.text(); - return { success: true, message: 'ok', url: text }; - }); + throw error; + } }, }; diff --git a/data-agent-frontend/src/views/AgentCreate.vue b/data-agent-frontend/src/views/AgentCreate.vue index 23ce4f3f2..f65ae9acb 100644 --- a/data-agent-frontend/src/views/AgentCreate.vue +++ b/data-agent-frontend/src/views/AgentCreate.vue @@ -41,12 +41,18 @@
- + + + 重新生成 - - + + + + + + {{ uploading ? '上传中...' : '上传图片' }} + """) + int update(FileStorage fileStorage); + + @Delete(""" + DELETE FROM file_storage WHERE id = #{id} + """) + int deleteById(Long id); + + @Select(""" + SELECT * FROM file_storage + WHERE is_deleted = 1 + AND is_cleaned = 0 + AND updated_time < #{beforeTime} + LIMIT #{limit} + """) + List selectDirtyRecords(LocalDateTime beforeTime, int limit); + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java index c10101083..c75922e37 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/FileStorageProperties.java @@ -16,7 +16,7 @@ package com.alibaba.cloud.ai.dataagent.properties; import com.alibaba.cloud.ai.dataagent.constant.Constant; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceEnum; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; import java.nio.file.Path; import lombok.Getter; import lombok.Setter; @@ -35,7 +35,7 @@ public class FileStorageProperties { /** * 存储类型：local（本地存储）、oss（阿里云OSS） */ - private FileStorageServiceEnum type = FileStorageServiceEnum.LOCAL; + private FileStorageProviderEnum type = FileStorageProviderEnum.LOCAL; /** * 对象存储路径前缀（通用配置，对OSS和本地存储都适用） diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java index 4c6530018..bcba8210f 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/OssStorageProperties.java @@ -28,6 +28,8 @@ @ConfigurationProperties(prefix = Constant.PROJECT_PROPERTIES_PREFIX + ".file.oss") public class OssStorageProperties { + private boolean enabled = false; + /** * OSS访问密钥ID */ diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java index 51e889c67..812589145 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/agent/AgentServiceImpl.java @@ -17,16 +17,15 @@ import com.alibaba.cloud.ai.dataagent.entity.Agent; import com.alibaba.cloud.ai.dataagent.mapper.AgentMapper; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import com.alibaba.cloud.ai.dataagent.service.vectorstore.AgentVectorStoreService; import com.alibaba.cloud.ai.dataagent.util.ApiKeyUtil; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - import java.time.LocalDateTime; import java.util.HashMap; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; /** * Agent Service Class @@ -40,7 +39,7 @@ public class AgentServiceImpl implements AgentService { private final AgentVectorStoreService agentVectorStoreService; - private final FileStorageService fileStorageService; + private final FileStorageProvider fileStorageService; @Override public List findAll() { diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java new file mode 100644 index 000000000..7d86087f3 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileResourceCleanerTask.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; +import java.time.LocalDateTime; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +@RequiredArgsConstructor +public class FileResourceCleanerTask { + + private final FileStorageMapper mapper; + + private final FileStorageService fileStorageService; + + /** + * 每隔 1 小时执行一次兜底清理 cron = "0 0 * * * ?" (整点执行) + */ + @Scheduled(cron = "0 0 * * * ?") + public void cleanupZombieResources() { + log.info("Starting zombie resources cleanup task..."); + + // 1. 定义时间缓冲：只处理 30 分钟前删除的数据 + // 这样不会跟用户刚刚操作的异步任务冲突 + LocalDateTime timeBuffer = LocalDateTime.now().minusMinutes(30); + int batchSize = 100; + + // 2. 查询脏数据 + List dirtyRecords = mapper.selectDirtyRecords(timeBuffer, batchSize); + + if (dirtyRecords.isEmpty()) { + log.info("No zombie resources found. Task finished."); + return; + } + + log.info("Found {} zombie records to clean.", dirtyRecords.size()); + + // 3. 逐条清理 + for (FileStorage fileStorage : dirtyRecords) { + try { + cleanupSingleRecord(fileStorage); + } + catch (Exception e) { + // 单条失败不影响其他记录，只记录日志，等下个周期再试 + log.error("Failed to clean resources for ID: {}", fileStorage.getId(), e); + } + } + } + + private void cleanupSingleRecord(FileStorage fileStorage) { + + Long id = fileStorage.getId(); + boolean fileDeleted = fileStorageService.deleteFileResource(fileStorage); + + if (fileDeleted) { + log.info("Zombie resource cleaned: ID={}", id); + } + else { + log.warn("Partial cleanup for ID={}, FileDel={}", id, fileDeleted); + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java new file mode 100644 index 000000000..20dabe90a --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProvider.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import org.springframework.core.io.Resource; +import org.springframework.http.codec.multipart.FilePart; +import reactor.core.publisher.Mono; + +public interface FileStorageProvider { + + /** + * 存储文件 + * @param file 上传的文件 + * @param fileStorage 文件存储信息 + */ + Mono storeFile(FilePart file, FileStorage fileStorage); + + /** + * 删除文件 + * @param filePath 文件路径 + * @return 是否删除成功 + */ + boolean deleteFile(String filePath); + + /** + * 获取文件访问URL + * @param filePath 文件路径 + * @return 访问URL + */ + String getFileUrl(String filePath); + + /** + * 获取文件资源对象 + * @param filePath 文件路径 + * @return 文件资源对象 + */ + Resource getFileResource(String filePath); + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java similarity index 94% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java index ed5bb2377..e59eaa280 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceEnum.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageProviderEnum.java @@ -15,7 +15,7 @@ */ package com.alibaba.cloud.ai.dataagent.service.file; -public enum FileStorageServiceEnum { +public enum FileStorageProviderEnum { LOCAL, OSS diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java index b20fbb587..f7e3dc866 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageService.java @@ -15,9 +15,10 @@ */ package com.alibaba.cloud.ai.dataagent.service.file; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; -import org.springframework.web.multipart.MultipartFile; import reactor.core.publisher.Mono; public interface FileStorageService { @@ -28,35 +29,48 @@ public interface FileStorageService { * @param subPath 子路径（如 "avatars"） * @return 存储后的文件路径 */ - Mono storeFile(FilePart filePart, String subPath); + Mono storeFile(FilePart filePart, String subPath); /** - * 存储文件（同步版本，用于传统同步代码） - * @param file 上传的文件 - * @param subPath 子路径（如 "avatars"） - * @return 存储后的文件路径 + * 通过文件Id获取文件存储信息 + * @param id 文件ID + * @return 文件存储信息 */ - String storeFile(MultipartFile file, String subPath); + FileStorage getFileById(Long id); /** * 删除文件 * @param filePath 文件路径 * @return 是否删除成功 */ - boolean deleteFile(String filePath); + boolean deleteFileResource(String filePath); + + /** + * 删除文件 + * @param fileStorage 文件信息 + * @return 是否删除成功 + */ + boolean deleteFileResource(FileStorage fileStorage); + + /** + * 删除文件 + * @param id 文件Id + * @return 是否删除成功 + */ + boolean deleteFileById(Long id); /** * 获取文件访问URL - * @param filePath 文件路径 + * @param fileStorage 文件存储信息 * @return 访问URL */ - String getFileUrl(String filePath); + String getFileUrl(FileStorage fileStorage); /** * 获取文件资源对象 - * @param filePath 文件路径 + * @param fileStorage 文件路径 * @return 文件资源对象 */ - Resource getFileResource(String filePath); + Resource getFileResource(FileStorage fileStorage); } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java deleted file mode 100644 index b042bdfef..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/FileStorageServiceFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.file; - -import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; -import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.impls.LocalFileStorageServiceImpl; -import com.alibaba.cloud.ai.dataagent.service.file.impls.OssFileStorageServiceImpl; -import lombok.AllArgsConstructor; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.stereotype.Component; - -@Component -@AllArgsConstructor -public class FileStorageServiceFactory implements FactoryBean { - - private final FileStorageProperties properties; - - private final OssStorageProperties ossProperties; - - @Override - public FileStorageService getObject() { - if (FileStorageServiceEnum.OSS.equals(properties.getType())) { - return new OssFileStorageServiceImpl(properties, ossProperties); - } - else { - return new LocalFileStorageServiceImpl(properties); - } - } - - @Override - public Class getObjectType() { - return FileStorageService.class; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java new file mode 100644 index 000000000..804bf935d --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/FileStorageServiceImpl.java @@ -0,0 +1,189 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file.impls; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.event.FileDeletionEvent; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; +import com.alibaba.cloud.ai.dataagent.mapper.FileStorageMapper; +import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProviderEnum; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.core.io.Resource; +import org.springframework.dao.DataAccessException; +import org.springframework.http.MediaType; +import org.springframework.http.codec.multipart.FilePart; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +@Service +@RequiredArgsConstructor +@Slf4j +public class FileStorageServiceImpl implements FileStorageService { + + private final Map fileStorageProviders; + + private final FileStorageProperties fileStorageProperties; + + private final FileStorageMapper fileStorageMapper; + + private final ApplicationEventPublisher eventPublisher; + + public Mono storeFile(FilePart file, String subPath) { + + // 1. 前置校验（轻量同步操作，可立即执行） + if (file == null || file.headers().getContentLength() == 0 || !StringUtils.hasText(file.filename())) { + log.warn("文件为空，无法上传"); + return Mono.error(new IllegalArgumentException("文件为空，无法上传")); // ✅ 响应式返回错误 + } + + // 2. 提取文件元数据 + String originalFilename = file.filename(); + String extension = originalFilename.contains(".") + ? originalFilename.substring(originalFilename.lastIndexOf(".")) : ""; + String filename = UUID.randomUUID() + extension; + String filePath = buildFilePath(subPath, filename); + + MediaType contentType = file.headers().getContentType(); + String contentTypeStr = contentType != null ? contentType.toString() : "application/octet-stream"; + + // 3. 构建 FileStorage 实体 + FileStorage storage = FileStorage.builder() + .filename(originalFilename) + .filePath(filePath) + .fileSize(file.headers().getContentLength()) + .fileType(contentTypeStr) + .fileExtension(extension) + .storageType(fileStorageProperties.getType()) + .isDeleted(0) + .isCleaned(0) + .build(); + + // 4. 核心响应式链：文件存储 → 数据库插入 → 结果转换 + return getStorageProvider(storage.getStorageType()).storeFile(file, storage) + // 5. 文件存储成功后，执行数据库插入（阻塞 JDBC，需切换线程） + .flatMap(storedStorage -> Mono.fromCallable(() -> { + // ✅ 阻塞的 JDBC 操作包裹在 fromCallable 中 + storedStorage.setCreatedTime(LocalDateTime.now()); + storedStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.insert(storedStorage); + return storedStorage; + }).subscribeOn(Schedulers.boundedElastic())) + // 6. 数据库插入成功后，转换为 VO 返回给前端 + .map(storedStorage -> FileStorageVo.builder() + .id(storedStorage.getId()) + .filePath(storedStorage.getFilePath()) + .url(getFileUrl(storedStorage)) + .filename(storedStorage.getFilename()) + .build()) + // 7. 可观测性：日志埋点 + .doOnSubscribe(sub -> log.debug("开始处理文件上传: {}, subPath: {}", originalFilename, subPath)) + .doOnSuccess(vo -> log.info("文件上传并入库成功: fileId={}, url={}", vo.getId(), vo.getUrl())) + .doOnError(e -> log.error("文件上传流程失败: filename={}", originalFilename, e)) + // 8. 响应式错误处理：统一转换异常类型 + .onErrorMap(IllegalArgumentException.class, e -> e) // 参数错误直接抛出 + .onErrorMap(DataAccessException.class, e -> new InternalServerException("数据库操作失败: " + e.getMessage(), e)) + // 9. 可选：添加超时保护 + .timeout(Duration.ofSeconds(60)); + } + + private FileStorageProvider getStorageProvider(FileStorageProviderEnum storageProviderEnum) { + if (storageProviderEnum == null) { + storageProviderEnum = fileStorageProperties.getType(); + } + if (!fileStorageProviders.containsKey(storageProviderEnum.name().toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("Invalid storage provider: " + storageProviderEnum); + } + return fileStorageProviders.get(storageProviderEnum.name().toLowerCase(Locale.ROOT)); + } + + @Override + public FileStorage getFileById(Long id) { + return fileStorageMapper.findById(id); + } + + public boolean deleteFileResource(String filePath) { + return getStorageProvider(fileStorageProperties.getType()).deleteFile(filePath); + } + + public boolean deleteFileResource(FileStorage fileStorage) { + // 1. 删除文件 + boolean fileDeleted = getStorageProvider(fileStorage.getStorageType()).deleteFile(fileStorage.getFilePath()); + + // 2. 更新清理状态 + if (fileDeleted) { + fileStorage.setIsCleaned(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.update(fileStorage); + } + return fileDeleted; + } + + public boolean deleteFileById(Long id) { + FileStorage fileStorage = fileStorageMapper.findById(id); + return deleteFile(fileStorage); + } + + private boolean deleteFile(FileStorage fileStorage) { + fileStorage.setIsDeleted(1); + fileStorage.setUpdatedTime(LocalDateTime.now()); + fileStorageMapper.update(fileStorage); + eventPublisher.publishEvent(new FileDeletionEvent(this, fileStorage.getId())); + return true; + } + + @Override + public String getFileUrl(FileStorage fileStorage) { + return getStorageProvider(fileStorage.getStorageType()).getFileUrl(fileStorage.getFilePath()); + } + + @Override + public Resource getFileResource(FileStorage fileStorage) { + return getStorageProvider(fileStorage.getStorageType()).getFileResource(fileStorage.getFilePath()); + } + + /** + * 构建存储路径 + */ + private String buildFilePath(String subPath, String filename) { + StringBuilder pathBuilder = new StringBuilder(); + + if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { + pathBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); + } + + if (StringUtils.hasText(subPath)) { + pathBuilder.append(subPath).append("/"); + } + + pathBuilder.append(filename); + + return pathBuilder.toString(); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java similarity index 54% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java index 5bce82a1b..2faf9cf25 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/LocalFileStorageProviderImpl.java @@ -15,85 +15,50 @@ */ package com.alibaba.cloud.ai.dataagent.service.file.impls; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.UUID; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.codec.multipart.FilePart; -import org.springframework.util.StringUtils; -import org.springframework.web.multipart.MultipartFile; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @Slf4j @AllArgsConstructor -public class LocalFileStorageServiceImpl implements FileStorageService { +public class LocalFileStorageProviderImpl implements FileStorageProvider { private final FileStorageProperties fileStorageProperties; @Override - public Mono storeFile(FilePart filePart, String subPath) { - String originalFilename = filePart.filename(); - String extension = ""; - if (originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - - String storagePath = buildStoragePath(subPath, filename); - - Path filePath = fileStorageProperties.getLocalBasePath().resolve(storagePath); - - checkPathSecurity(filePath); - + public Mono storeFile(FilePart file, FileStorage fileStorage) { return Mono.fromCallable(() -> { - Path uploadDir = filePath.getParent(); - if (!Files.exists(uploadDir)) { - Files.createDirectories(uploadDir); - } - return filePath; - }).subscribeOn(Schedulers.boundedElastic()).flatMap(filePart::transferTo).then(Mono.fromCallable(() -> { - log.info("文件存储成功: {}", storagePath); - return storagePath; - })); - } - - @Override - public String storeFile(MultipartFile file, String subPath) { - try { - String originalFilename = file.getOriginalFilename(); - String extension = ""; - if (originalFilename != null && originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - - String storagePath = buildStoragePath(subPath, filename); - - Path filePath = fileStorageProperties.getLocalBasePath().resolve(storagePath); + // 1. 执行所有同步/阻塞的 IO 操作 + Path storagePath = fileStorageProperties.getLocalBasePath().resolve(fileStorage.getFilePath()); - checkPathSecurity(filePath); + checkPathSecurity(storagePath); - Path uploadDir = filePath.getParent(); + Path uploadDir = storagePath.getParent(); if (!Files.exists(uploadDir)) { Files.createDirectories(uploadDir); } - Files.copy(file.getInputStream(), filePath); - - log.info("文件存储成功: {}", storagePath); - return storagePath; - - } - catch (IOException e) { - log.error("文件存储失败", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); - } + return storagePath; // 返回计算结果给下一步 + }) + // 2. 关键：切换到 boundedElastic 线程池，避免阻塞 I/O 线程 + .subscribeOn(Schedulers.boundedElastic()) + // 3. 执行响应式文件传输（file.transferTo 返回 Mono） + .flatMap(storagePath -> file.transferTo(storagePath).thenReturn(fileStorage)) + // 4. 成功日志（可选） + .doOnSuccess(stored -> log.info("文件存储成功: {}", stored)) + // 5. 响应式错误处理 + .doOnError(e -> log.error("文件存储失败", e)) + .onErrorMap(IOException.class, e -> new InternalServerException("文件存储失败: " + e.getMessage(), e)); } @Override @@ -146,23 +111,4 @@ private void checkPathSecurity(Path filePath) { } } - /** - * 构建本地存储路径 - */ - private String buildStoragePath(String subPath, String filename) { - StringBuilder pathBuilder = new StringBuilder(); - - if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { - pathBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); - } - - if (StringUtils.hasText(subPath)) { - pathBuilder.append(subPath).append("/"); - } - - pathBuilder.append(filename); - - return pathBuilder.toString(); - } - } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java new file mode 100644 index 000000000..1a392da1a --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageProviderImpl.java @@ -0,0 +1,189 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.file.impls; + +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.exception.InternalServerException; +import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; +import com.alibaba.cloud.ai.dataagent.service.file.FileStorageProvider; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.codec.multipart.FilePart; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * 阿里云OSS文件存储服务实现 + */ +@Slf4j +public class OssFileStorageProviderImpl implements FileStorageProvider { + + private final OssStorageProperties ossProperties; + + private OSS ossClient; + + public OssFileStorageProviderImpl(OssStorageProperties ossProperties) { + this.ossProperties = ossProperties; + } + + @PostConstruct + public void init() { + this.ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(), + ossProperties.getAccessKeySecret()); + log.info("OSS客户端初始化完成，endpoint: {}, bucket: {}", ossProperties.getEndpoint(), ossProperties.getBucketName()); + } + + @PreDestroy + public void destroy() { + if (ossClient != null) { + ossClient.shutdown(); + log.info("OSS客户端已关闭"); + } + } + + @Override + public Mono storeFile(FilePart file, FileStorage fileStorage) { + + // 1. 准备 OSS 元数据（纯内存操作，可立即执行） + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(fileStorage.getFileSize()); + metadata.setContentType(fileStorage.getFileType()); + metadata.setCacheControl("no-cache"); + + // 2. 定义临时文件路径 + Path tempFile = Path.of("/tmp", fileStorage.getFilePath()); + + return Mono.defer(() -> { + // 确保每次订阅时重新执行（避免临时文件路径冲突） + + // 3. 第一步：将上传文件保存到本地临时文件（响应式 API） + return file.transferTo(tempFile) + + // 4. 第二步：上传到 OSS（阻塞操作，需切换线程） + .then(Mono.fromCallable(() -> { + // 确保父目录存在 + Files.createDirectories(tempFile.getParent()); + + // 阻塞 IO：读取本地文件 + 上传 OSS + try (InputStream is = Files.newInputStream(tempFile)) { + ossClient.putObject(ossProperties.getBucketName(), fileStorage.getFilePath(), is, metadata); + log.info("文件上传 OSS 成功: {}", fileStorage); + return fileStorage; // 返回业务对象 + } + }).subscribeOn(Schedulers.boundedElastic())) + + .publishOn(Schedulers.boundedElastic()) + + // 5. 第三步：无论成功失败，清理临时文件 + .doFinally(signal -> { + try { + Files.deleteIfExists(tempFile); + log.debug("临时文件已清理: {}", tempFile); + } + catch (IOException e) { + log.warn("清理临时文件失败: {}", tempFile, e); + // 注意：doFinally 中抛异常会影响主流程，建议只记录日志 + } + }) + + // 6. 响应式错误处理：转换异常类型 + .onErrorMap(IOException.class, e -> new InternalServerException("文件处理失败: " + e.getMessage(), e)) + // 7. 可选：添加超时保护，防止大文件卡死 + .timeout(Duration.ofSeconds(60)); + }) + // 8. 日志埋点（可观测性） + .doOnSubscribe(sub -> log.debug("开始处理文件上传: {}", fileStorage.getFilename())) + .doOnSuccess(stored -> log.info("文件上传流程完成: {}", stored)) + .doOnError(e -> log.error("文件上传流程异常: {}", fileStorage.getFilename(), e)); + } + + @Override + public boolean deleteFile(String filePath) { + if (!StringUtils.hasText(filePath)) { + log.info("删除文件失败，路径为空"); + return false; + } + try { + if (ossClient.doesObjectExist(ossProperties.getBucketName(), filePath)) { + ossClient.deleteObject(ossProperties.getBucketName(), filePath); + log.info("成功从OSS删除文件: {}", filePath); + } + else { + // 删除是个等幂的操作，不存在也是当做被删除了 + log.info("OSS中文件不存在，跳过删除，视为成功: {}", filePath); + } + return true; + } + catch (Exception e) { + log.error("从OSS删除文件失败: {}", filePath, e); + return false; + } + } + + @Override + public String getFileUrl(String filePath) { + try { + if (StringUtils.hasText(ossProperties.getCustomDomain())) { + return ossProperties.getCustomDomain() + "/" + filePath; + } + + String bucketDomain = String.format("https://%s.%s", ossProperties.getBucketName(), + ossProperties.getEndpoint().replace("https://", "").replace("http://", "")); + return bucketDomain + "/" + filePath; + + } + catch (Exception e) { + log.error("生成OSS文件URL失败: {}", filePath, e); + return filePath; + } + } + + /** + * This implementation throws IllegalStateException if attempting to read the + * underlying stream multiple times. + */ + @Override + public Resource getFileResource(String filePath) { + + if (!StringUtils.hasText(filePath)) { + log.info("获取文件失败，路径为空"); + return null; + } + + OSSObject result = ossClient.getObject(ossProperties.getBucketName(), filePath); + // todo: 临时处理,只能读取一次,不能重复读 + return new InputStreamResource(result.getObjectContent()) { + @Override + public long contentLength() { + return result.getObjectMetadata().getContentLength(); + } + }; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java deleted file mode 100644 index 66d74b0db..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/file/impls/OssFileStorageServiceImpl.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.file.impls; - -import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; -import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.model.OSSObject; -import com.aliyun.oss.model.ObjectMetadata; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.UUID; -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.InputStreamResource; -import org.springframework.core.io.Resource; -import org.springframework.core.io.buffer.DataBufferUtils; -import org.springframework.http.MediaType; -import org.springframework.http.codec.multipart.FilePart; -import org.springframework.util.StringUtils; -import org.springframework.web.multipart.MultipartFile; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -/** - * 阿里云OSS文件存储服务实现 - */ -@Slf4j -public class OssFileStorageServiceImpl implements FileStorageService { - - private final FileStorageProperties fileStorageProperties; - - private final OssStorageProperties ossProperties; - - private OSS ossClient; - - public OssFileStorageServiceImpl(FileStorageProperties fileStorageProperties, OssStorageProperties ossProperties) { - this.fileStorageProperties = fileStorageProperties; - this.ossProperties = ossProperties; - } - - @PostConstruct - public void init() { - this.ossClient = new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(), - ossProperties.getAccessKeySecret()); - log.info("OSS客户端初始化完成，endpoint: {}, bucket: {}", ossProperties.getEndpoint(), ossProperties.getBucketName()); - } - - @PreDestroy - public void destroy() { - if (ossClient != null) { - ossClient.shutdown(); - log.info("OSS客户端已关闭"); - } - } - - @Override - public Mono storeFile(FilePart file, String subPath) { - if (file == null || !StringUtils.hasText(file.filename())) { - log.warn("文件为空，无法上传到OSS"); - return Mono.error(new IllegalArgumentException("文件为空，无法上传到OSS")); - } - - String originalFilename = file.filename(); - String extension = ""; - if (originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - String objectKey = buildObjectKey(subPath, filename); - - // 获取 Content-Type - MediaType contentType = file.headers().getContentType(); - String contentTypeStr = contentType != null ? contentType.toString() : "application/octet-stream"; - - // 使用 DataBufferUtils 收集文件内容，然后在 boundedElastic 线程池上执行 OSS 上传 - return DataBufferUtils.join(file.content()).flatMap(dataBuffer -> { - byte[] bytes = new byte[dataBuffer.readableByteCount()]; - dataBuffer.read(bytes); - DataBufferUtils.release(dataBuffer); - - return Mono.fromCallable(() -> { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(bytes.length); - metadata.setContentType(contentTypeStr); - metadata.setCacheControl("no-cache"); - - try (InputStream inputStream = new ByteArrayInputStream(bytes)) { - ossClient.putObject(ossProperties.getBucketName(), objectKey, inputStream, metadata); - log.info("文件上传成功: {}", objectKey); - return objectKey; - } - }).subscribeOn(Schedulers.boundedElastic()); - }).onErrorMap(e -> { - log.error("文件存储失败，上传OSS失败", e); - return new RuntimeException("文件存储失败: " + e.getMessage(), e); - }); - } - - @Override - public String storeFile(MultipartFile file, String subPath) { - try { - if (file == null || file.isEmpty()) { - log.warn("文件为空，无法上传到OSS"); - throw new IllegalArgumentException("文件为空，无法上传到OSS"); - } - - String originalFilename = file.getOriginalFilename(); - String extension = ""; - if (originalFilename != null && originalFilename.contains(".")) { - extension = originalFilename.substring(originalFilename.lastIndexOf(".")); - } - String filename = UUID.randomUUID() + extension; - - String objectKey = buildObjectKey(subPath, filename); - - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(file.getSize()); - metadata.setContentType(file.getContentType()); - metadata.setCacheControl("no-cache"); - - try (InputStream inputStream = file.getInputStream()) { - ossClient.putObject(ossProperties.getBucketName(), objectKey, inputStream, metadata); - log.info("文件上传成功: {}", objectKey); - return objectKey; - } - } - catch (IOException e) { - log.error("文件存储失败，获取输入流错误", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); - } - catch (Exception e) { - log.error("文件存储失败，上传OSS失败", e); - throw new RuntimeException("文件存储失败: " + e.getMessage(), e); - } - } - - @Override - public boolean deleteFile(String filePath) { - if (!StringUtils.hasText(filePath)) { - log.info("删除文件失败，路径为空"); - return false; - } - try { - if (ossClient.doesObjectExist(ossProperties.getBucketName(), filePath)) { - ossClient.deleteObject(ossProperties.getBucketName(), filePath); - log.info("成功从OSS删除文件: {}", filePath); - } - else { - // 删除是个等幂的操作，不存在也是当做被删除了 - log.info("OSS中文件不存在，跳过删除，视为成功: {}", filePath); - } - return true; - } - catch (Exception e) { - log.error("从OSS删除文件失败: {}", filePath, e); - return false; - } - } - - @Override - public String getFileUrl(String filePath) { - try { - if (StringUtils.hasText(ossProperties.getCustomDomain())) { - return ossProperties.getCustomDomain() + "/" + filePath; - } - - String bucketDomain = String.format("https://%s.%s", ossProperties.getBucketName(), - ossProperties.getEndpoint().replace("https://", "").replace("http://", "")); - return bucketDomain + "/" + filePath; - - } - catch (Exception e) { - log.error("生成OSS文件URL失败: {}", filePath, e); - return filePath; - } - } - - /** - * This implementation throws IllegalStateException if attempting to read the - * underlying stream multiple times. - */ - @Override - public Resource getFileResource(String filePath) { - - if (!StringUtils.hasText(filePath)) { - log.info("获取文件失败，路径为空"); - return null; - } - - OSSObject result = ossClient.getObject(ossProperties.getBucketName(), filePath); - // todo: 临时处理,只能读取一次,不能重复读 - return new InputStreamResource(result.getObjectContent()) { - @Override - public long contentLength() { - return result.getObjectMetadata().getContentLength(); - } - }; - } - - /** - * 构建OSS对象键 - */ - private String buildObjectKey(String subPath, String filename) { - StringBuilder keyBuilder = new StringBuilder(); - - if (StringUtils.hasText(fileStorageProperties.getPathPrefix())) { - keyBuilder.append(fileStorageProperties.getPathPrefix()).append("/"); - } - - if (StringUtils.hasText(subPath)) { - keyBuilder.append(subPath).append("/"); - } - - keyBuilder.append(filename); - - return keyBuilder.toString(); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java index 218fa1fd4..68dc06e83 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeResourceManager.java @@ -17,11 +17,15 @@ import com.alibaba.cloud.ai.dataagent.constant.Constant; import com.alibaba.cloud.ai.dataagent.constant.DocumentMetadataConstant; -import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; -import com.alibaba.cloud.ai.dataagent.util.DocumentConverterUtil; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.entity.FileStorage; +import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.service.vectorstore.AgentVectorStoreService; +import com.alibaba.cloud.ai.dataagent.util.DocumentConverterUtil; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; import org.springframework.ai.reader.tika.TikaDocumentReader; @@ -30,10 +34,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - // 智能体知识的向量资源和文件资源管理 @Slf4j @Component @@ -52,7 +52,7 @@ public AgentKnowledgeResourceManager(TextSplitterFactory textSplitterFactory, Fi this.agentVectorStoreService = agentVectorStoreService; } - public void doEmbedingToVectorStore(AgentKnowledge agentKnowledge) throws Exception { + public void doEmbeddingToVectorStore(AgentKnowledge agentKnowledge) throws Exception { // delete old data this.deleteFromVectorStore(agentKnowledge.getAgentId(), agentKnowledge.getId()); @@ -75,11 +75,20 @@ private void processQaKnowledge(AgentKnowledge knowledge) { private void processDocumentKnowledge(AgentKnowledge knowledge) { + FileStorage fileStorage; + if (knowledge.getFileId() != null) { + fileStorage = fileStorageService.getFileById(knowledge.getFileId()); + } + else { + fileStorage = new FileStorage(); + fileStorage.setFilePath(knowledge.getFilePath()); + } + // 处理文档 - List documents = getAndSplitDocument(knowledge.getFilePath(), knowledge.getSplitterType()); + List documents = getAndSplitDocument(fileStorage, knowledge.getSplitterType()); if (documents == null || documents.isEmpty()) { log.error("No documents extracted from file: knowledgeId={}, filePath={}", knowledge.getId(), - knowledge.getFilePath()); + fileStorage.getFilePath()); throw new RuntimeException("No documents extracted from file"); } @@ -90,13 +99,14 @@ private void processDocumentKnowledge(AgentKnowledge knowledge) { // 添加到向量存储 agentVectorStoreService.addDocuments(knowledge.getAgentId().toString(), documentsWithMetadata); log.info("Successfully vectorized DOCUMENT knowledge: id={}, filePath={}, documentCount={}, splitterType={}", - knowledge.getId(), knowledge.getFilePath(), documentsWithMetadata.size(), knowledge.getSplitterType()); + knowledge.getId(), fileStorage.getFilePath(), documentsWithMetadata.size(), + knowledge.getSplitterType()); } - private List getAndSplitDocument(String filePath, String splitterType) { + private List getAndSplitDocument(FileStorage fileStorage, String splitterType) { // 使用FileStorageService获取文件资源对象 - Resource resource = fileStorageService.getFileResource(filePath); + Resource resource = fileStorageService.getFileResource(fileStorage); // 使用TikaDocumentReader读取文件 TikaDocumentReader tikaDocumentReader = new TikaDocumentReader(resource); @@ -155,7 +165,7 @@ public boolean deleteKnowledgeFile(AgentKnowledge knowledge) { } try { - boolean fileDeleted = fileStorageService.deleteFile(knowledge.getFilePath()); + boolean fileDeleted = fileStorageService.deleteFileResource(knowledge.getFilePath()); if (fileDeleted) { log.info("Successfully deleted knowledge file, filePath: {}", knowledge.getFilePath()); return true; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java index 2ed251384..50cb18f10 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/knowledge/AgentKnowledgeServiceImpl.java @@ -15,19 +15,22 @@ */ package com.alibaba.cloud.ai.dataagent.service.knowledge; -import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; -import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.converter.AgentKnowledgeConverter; -import com.alibaba.cloud.ai.dataagent.vo.PageResult; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.AgentKnowledgeQueryDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.CreateKnowledgeDTO; import com.alibaba.cloud.ai.dataagent.dto.knowledge.agentknowledge.UpdateKnowledgeDTO; import com.alibaba.cloud.ai.dataagent.entity.AgentKnowledge; +import com.alibaba.cloud.ai.dataagent.enums.EmbeddingStatus; +import com.alibaba.cloud.ai.dataagent.enums.KnowledgeType; import com.alibaba.cloud.ai.dataagent.event.AgentKnowledgeDeletionEvent; import com.alibaba.cloud.ai.dataagent.event.AgentKnowledgeEmbeddingEvent; import com.alibaba.cloud.ai.dataagent.mapper.AgentKnowledgeMapper; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.vo.AgentKnowledgeVO; +import com.alibaba.cloud.ai.dataagent.vo.FileStorageVo; +import com.alibaba.cloud.ai.dataagent.vo.PageResult; +import java.time.LocalDateTime; +import java.util.List; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; @@ -35,9 +38,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; -import java.time.LocalDateTime; -import java.util.List; - @Slf4j @Service @AllArgsConstructor @@ -62,13 +62,14 @@ public AgentKnowledgeVO getKnowledgeById(Integer id) { @Override @Transactional public AgentKnowledgeVO createKnowledge(CreateKnowledgeDTO createKnowledgeDto) { - String storagePath = null; + FileStorageVo fileStorage = null; checkCreateKnowledgeDto(createKnowledgeDto); if (createKnowledgeDto.getType().equals(KnowledgeType.DOCUMENT.getCode())) { // 将文件保存到磁盘 try { - storagePath = fileStorageService.storeFile(createKnowledgeDto.getFile(), AGENT_KNOWLEDGE_FILE_PATH); + fileStorage = fileStorageService.storeFile(createKnowledgeDto.getFile(), AGENT_KNOWLEDGE_FILE_PATH) + .block(); } catch (Exception e) { log.error("Failed to store file, agentId:{} title:{} type:{} ", createKnowledgeDto.getAgentId(), @@ -77,7 +78,7 @@ public AgentKnowledgeVO createKnowledge(CreateKnowledgeDTO createKnowledgeDto) { } } - AgentKnowledge knowledge = agentKnowledgeConverter.toEntityForCreate(createKnowledgeDto, storagePath); + AgentKnowledge knowledge = agentKnowledgeConverter.toEntityForCreate(createKnowledgeDto, fileStorage); if (agentKnowledgeMapper.insert(knowledge) <= 0) { log.error("Failed to create knowledge, agentId:{} title:{} type:{} ", knowledge.getAgentId(), @@ -153,6 +154,9 @@ public boolean deleteKnowledge(Integer id) { knowledge.setUpdatedTime(LocalDateTime.now()); if (agentKnowledgeMapper.update(knowledge) > 0) { + if (knowledge.getFileId() != null) { + fileStorageService.deleteFileById(knowledge.getFileId()); + } eventPublisher.publishEvent(new AgentKnowledgeDeletionEvent(this, id)); return true; } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java new file mode 100644 index 000000000..ed688943a --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/vo/FileStorageVo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.vo; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 通用上传响应实体。 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class FileStorageVo { + + private Long id; + + private String url; + + private String filePath; + + private String filename; + +} diff --git a/data-agent-management/src/main/resources/application-h2.yml b/data-agent-management/src/main/resources/application-h2.yml index ac6c78d46..114ee0dda 100644 --- a/data-agent-management/src/main/resources/application-h2.yml +++ b/data-agent-management/src/main/resources/application-h2.yml @@ -50,6 +50,7 @@ spring: url-prefix: /uploads imageSize: 2097152 oss: + enabled: false access-key-id: ${OSS_ACCESS_KEY_ID:} access-key-secret: ${OSS_ACCESS_KEY_SECRET:} endpoint: ${OSS_ENDPOINT:} diff --git a/data-agent-management/src/main/resources/application.yml b/data-agent-management/src/main/resources/application.yml index 30f606553..92c24515b 100644 --- a/data-agent-management/src/main/resources/application.yml +++ b/data-agent-management/src/main/resources/application.yml @@ -47,6 +47,7 @@ spring: url-prefix: /uploads imageSize: 2097152 oss: + enabled: false access-key-id: ${OSS_ACCESS_KEY_ID:} access-key-secret: ${OSS_ACCESS_KEY_SECRET:} endpoint: ${OSS_ENDPOINT:} diff --git a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql index 5eadd0907..cfa7c1c35 100644 --- a/data-agent-management/src/main/resources/sql/h2/schema-h2.sql +++ b/data-agent-management/src/main/resources/sql/h2/schema-h2.sql @@ -2,25 +2,25 @@ -- 智能体表 CREATE TABLE IF NOT EXISTS agent ( - id INT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL COMMENT '智能体名称', - description TEXT COMMENT '智能体描述', - avatar TEXT COMMENT '头像URL', - status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', - api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', - api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', - prompt TEXT COMMENT '自定义Prompt配置', - category VARCHAR(100) COMMENT '分类', - admin_id BIGINT COMMENT '管理员ID', - tags TEXT COMMENT '标签，逗号分隔', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (id), - INDEX idx_agent_name (name), - INDEX idx_agent_status (status), - INDEX idx_agent_category (category), - INDEX idx_agent_admin_id (admin_id) - ) ENGINE = InnoDB COMMENT = '智能体表'; + id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL COMMENT '智能体名称', + description TEXT COMMENT '智能体描述', + avatar TEXT COMMENT '头像URL', + status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', + api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', + api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', + prompt TEXT COMMENT '自定义Prompt配置', + category VARCHAR(100) COMMENT '分类', + admin_id BIGINT COMMENT '管理员ID', + tags TEXT COMMENT '标签，逗号分隔', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (id), + INDEX idx_agent_name (name), + INDEX idx_agent_status (status), + INDEX idx_agent_category (category), + INDEX idx_agent_admin_id (admin_id) +) ENGINE = InnoDB COMMENT = '智能体表'; -- 业务知识表 CREATE TABLE IF NOT EXISTS business_knowledge ( @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS business_knowledge ( INDEX idx_business_knowledge_is_recall (is_recall), INDEX idx_business_knowledge_embedding_status (embedding_status), INDEX idx_business_knowledge_is_deleted (is_deleted), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '业务知识表'; -- 语义模型表 @@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS semantic_model ( INDEX idx_semantic_model_agent_id (agent_id), INDEX idx_semantic_model_business_name (business_name), INDEX idx_semantic_model_status (status), - CONSTRAINT fk_semantic_model_agent FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + CONSTRAINT fk_semantic_model_agent FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '语义模型表'; @@ -133,7 +133,7 @@ CREATE TABLE IF NOT EXISTS logical_relation ( PRIMARY KEY (id), INDEX idx_logical_relation_datasource_id (datasource_id), INDEX idx_logical_relation_source_table (datasource_id, source_table_name), - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '逻辑外键配置表'; -- 智能体数据源关联表 @@ -149,8 +149,8 @@ CREATE TABLE IF NOT EXISTS agent_datasource ( INDEX idx_agent_datasource_agent_id (agent_id), INDEX idx_agent_datasource_datasource_id (datasource_id), INDEX idx_agent_datasource_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE, - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE, + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体数据源关联表'; -- 智能体预设问题表 @@ -166,7 +166,7 @@ CREATE TABLE IF NOT EXISTS agent_preset_question ( INDEX idx_agent_preset_question_agent_id (agent_id), INDEX idx_agent_preset_question_sort_order (sort_order), INDEX idx_agent_preset_question_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体预设问题表'; -- 会话表 @@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS chat_session ( INDEX idx_chat_session_status (status), INDEX idx_chat_session_is_pinned (is_pinned), INDEX idx_chat_session_create_time (create_time), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天会话表'; -- 消息表 @@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS chat_message ( INDEX idx_chat_message_role (role), INDEX idx_chat_message_message_type (message_type), INDEX idx_chat_message_create_time (create_time), - FOREIGN KEY (session_id) REFERENCES chat_session(id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES chat_session (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天消息表'; -- 用户Prompt配置表 @@ -228,19 +228,18 @@ CREATE TABLE IF NOT EXISTS user_prompt_config ( INDEX idx_user_prompt_config_display_order (display_order ASC) ) ENGINE = InnoDB COMMENT = '用户Prompt配置表'; -CREATE TABLE IF NOT EXISTS agent_datasource_tables -( - id INT AUTO_INCREMENT PRIMARY KEY, - agent_datasource_id INT NOT NULL COMMENT '智能体数据源ID', - table_name VARCHAR(255) NOT NULL COMMENT '数据表名', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '更新时间', - CONSTRAINT uk_agent_datasource_tables_agent_datasource_id_table_name - UNIQUE (agent_datasource_id, table_name), - CONSTRAINT fk_agent_datasource_tables_agent_datasource_id - FOREIGN KEY (agent_datasource_id) REFERENCES agent_datasource (id) - ON UPDATE CASCADE ON DELETE CASCADE - ) ENGINE = InnoDB COMMENT = '某个智能体某个数据源所选中的数据表'; +CREATE TABLE IF NOT EXISTS agent_datasource_tables ( + id INT AUTO_INCREMENT PRIMARY KEY, + agent_datasource_id INT NOT NULL COMMENT '智能体数据源ID', + table_name VARCHAR(255) NOT NULL COMMENT '数据表名', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL COMMENT '更新时间', + CONSTRAINT uk_agent_datasource_tables_agent_datasource_id_table_name + UNIQUE (agent_datasource_id, table_name), + CONSTRAINT fk_agent_datasource_tables_agent_datasource_id + FOREIGN KEY (agent_datasource_id) REFERENCES agent_datasource (id) + ON UPDATE CASCADE ON DELETE CASCADE +) ENGINE = InnoDB COMMENT = '某个智能体某个数据源所选中的数据表'; -- 模型配置表 @@ -250,7 +249,7 @@ CREATE TABLE IF NOT EXISTS `model_config` ( `base_url` varchar(255) NOT NULL COMMENT '关键配置', `api_key` varchar(255) NOT NULL COMMENT 'API密钥', `model_name` varchar(255) NOT NULL COMMENT '模型名称', - `temperature` decimal(10,2) DEFAULT '0.00' COMMENT '温度参数', + `temperature` decimal(10, 2) DEFAULT '0.00' COMMENT '温度参数', `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', @@ -266,4 +265,23 @@ CREATE TABLE IF NOT EXISTS `model_config` ( `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', PRIMARY KEY (`id`) -) ENGINE=InnoDB; +) ENGINE = InnoDB; + +-- 文件存储表 +CREATE TABLE `file_storage` ( + `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', + `filename` VARCHAR(255) NOT NULL COMMENT '文件名', + `file_path` VARCHAR(512) NOT NULL COMMENT '文件路径', + `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', + `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', + `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', + `storage_type` VARCHAR(50) NOT NULL COMMENT '存储类型：LOCAL/S3/OSS', + `is_deleted` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '删除状态：0=未删除, 1=已删除', + `is_cleaned` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '清理状态：0=物理资源未清理, 1=物理资源已清理', + `created_time` DATETIME DEFAULT NULL COMMENT '创建时间', + `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_filename` (`filename`), + KEY `idx_file_path` (`file_path`(50)), + KEY `idx_updated_time` (`updated_time`) +) ENGINE = InnoDB COMMENT ='文件存储表'; diff --git a/data-agent-management/src/main/resources/sql/schema.sql b/data-agent-management/src/main/resources/sql/schema.sql index 160bcedf0..82d43603a 100644 --- a/data-agent-management/src/main/resources/sql/schema.sql +++ b/data-agent-management/src/main/resources/sql/schema.sql @@ -2,25 +2,25 @@ -- 智能体表 CREATE TABLE IF NOT EXISTS agent ( - id INT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) NOT NULL COMMENT '智能体名称', - description TEXT COMMENT '智能体描述', - avatar TEXT COMMENT '头像URL', - status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', - api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', - api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', - prompt TEXT COMMENT '自定义Prompt配置', - category VARCHAR(100) COMMENT '分类', - admin_id BIGINT COMMENT '管理员ID', - tags TEXT COMMENT '标签，逗号分隔', - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', - PRIMARY KEY (id), - INDEX idx_name (name), - INDEX idx_status (status), - INDEX idx_category (category), - INDEX idx_admin_id (admin_id) - ) ENGINE = InnoDB COMMENT = '智能体表'; + id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL COMMENT '智能体名称', + description TEXT COMMENT '智能体描述', + avatar TEXT COMMENT '头像URL', + status VARCHAR(50) DEFAULT 'draft' COMMENT '状态：draft-待发布，published-已发布，offline-已下线', + api_key VARCHAR(255) DEFAULT NULL COMMENT '访问 API Key，格式 sk-xxx', + api_key_enabled TINYINT DEFAULT 0 COMMENT 'API Key 是否启用：0-禁用，1-启用', + prompt TEXT COMMENT '自定义Prompt配置', + category VARCHAR(100) COMMENT '分类', + admin_id BIGINT COMMENT '管理员ID', + tags TEXT COMMENT '标签，逗号分隔', + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (id), + INDEX idx_name (name), + INDEX idx_status (status), + INDEX idx_category (category), + INDEX idx_admin_id (admin_id) +) ENGINE = InnoDB COMMENT = '智能体表'; -- 业务知识表 CREATE TABLE IF NOT EXISTS business_knowledge ( @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS business_knowledge ( INDEX idx_is_recall (is_recall), INDEX idx_embedding_status (embedding_status), INDEX idx_is_deleted (is_deleted), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '业务知识表'; -- 语义模型表 @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS `semantic_model` ( KEY `idx_field_name` (`business_name`) USING BTREE, KEY `idx_status` (`status`) USING BTREE, CONSTRAINT `fk_semantic_model_agent` FOREIGN KEY (`agent_id`) REFERENCES `agent` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='语义模型表'; +) ENGINE = InnoDB AUTO_INCREMENT = 4 DEFAULT CHARSET = utf8mb4 COMMENT ='语义模型表'; -- 智能体知识表 @@ -78,6 +78,7 @@ CREATE TABLE IF NOT EXISTS `agent_knowledge` ( `is_recall` int(11) DEFAULT 1 COMMENT '业务状态: 1=召回, 0=非召回', `embedding_status` varchar(20) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '向量化状态：PENDING待处理，PROCESSING处理中，COMPLETED已完成，FAILED失败', `error_msg` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '操作失败的错误信息', + `file_id` int COMMENT '文件ID', `source_filename` varchar(500) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '上传时的原始文件名', `file_path` varchar(500) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '文件在服务器上的物理存储路径', `file_size` bigint(20) DEFAULT NULL COMMENT '文件大小 (字节)', @@ -88,10 +89,10 @@ CREATE TABLE IF NOT EXISTS `agent_knowledge` ( `is_deleted` int(11) DEFAULT 0 COMMENT '逻辑删除字段，0=未删除, 1=已删除', `is_resource_cleaned` int(11) DEFAULT 0 COMMENT '0=物理资源（文件和向量）未清理, 1=物理资源已清理', PRIMARY KEY (`id`) USING BTREE, - KEY `idx_agent_id_status` (`agent_id`,`is_recall`) USING BTREE, + KEY `idx_agent_id_status` (`agent_id`, `is_recall`) USING BTREE, KEY `idx_embedding_status` (`embedding_status`) USING BTREE, KEY `idx_is_deleted` (`is_deleted`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=18 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='智能体知识源管理表 (支持文档、QA、FAQ)'; +) ENGINE = InnoDB AUTO_INCREMENT = 18 DEFAULT CHARSET = utf8mb4 COMMENT ='智能体知识源管理表 (支持文档、QA、FAQ)'; -- 数据源表 CREATE TABLE IF NOT EXISTS datasource ( @@ -133,7 +134,7 @@ CREATE TABLE IF NOT EXISTS logical_relation ( PRIMARY KEY (id), INDEX idx_datasource_id (datasource_id) COMMENT '加速根据数据源查找关系的查询', INDEX idx_source_table (datasource_id, source_table_name) COMMENT '加速根据表名查找关系的查询', - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT = '逻辑外键配置表'; -- 智能体数据源关联表 @@ -149,8 +150,8 @@ CREATE TABLE IF NOT EXISTS agent_datasource ( INDEX idx_agent_id (agent_id), INDEX idx_datasource_id (datasource_id), INDEX idx_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE, - FOREIGN KEY (datasource_id) REFERENCES datasource(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE, + FOREIGN KEY (datasource_id) REFERENCES datasource (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体数据源关联表'; -- 智能体预设问题表 @@ -166,7 +167,7 @@ CREATE TABLE IF NOT EXISTS agent_preset_question ( INDEX idx_agent_id (agent_id), INDEX idx_sort_order (sort_order), INDEX idx_is_active (is_active), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '智能体预设问题表'; -- 会话表 @@ -185,7 +186,7 @@ CREATE TABLE IF NOT EXISTS chat_session ( INDEX idx_status (status), INDEX idx_is_pinned (is_pinned), INDEX idx_create_time (create_time), - FOREIGN KEY (agent_id) REFERENCES agent(id) ON DELETE CASCADE + FOREIGN KEY (agent_id) REFERENCES agent (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天会话表'; -- 消息表 @@ -202,7 +203,7 @@ CREATE TABLE IF NOT EXISTS chat_message ( INDEX idx_role (role), INDEX idx_message_type (message_type), INDEX idx_create_time (create_time), - FOREIGN KEY (session_id) REFERENCES chat_session(id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES chat_session (id) ON DELETE CASCADE ) ENGINE = InnoDB COMMENT = '聊天消息表'; -- 用户Prompt配置表 @@ -228,43 +229,61 @@ CREATE TABLE IF NOT EXISTS user_prompt_config ( INDEX idx_display_order (display_order ASC) ) ENGINE = InnoDB COMMENT = '用户Prompt配置表'; -create table if not exists agent_datasource_tables -( - id int auto_increment primary key, - agent_datasource_id int not null comment '智能体数据源ID', - table_name varchar(255) not null comment '数据表名', - create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', - update_time timestamp default CURRENT_TIMESTAMP null comment '更新时间', - constraint agent_datasource_tables_agent_datasource_id_table_name_uindex - unique (agent_datasource_id, table_name), - constraint agent_datasource_tables_agent_datasource_id_fk - foreign key (agent_datasource_id) references agent_datasource (id) - on update cascade on delete cascade +create table if not exists agent_datasource_tables ( + id int auto_increment primary key, + agent_datasource_id int not null comment '智能体数据源ID', + table_name varchar(255) not null comment '数据表名', + create_time timestamp default CURRENT_TIMESTAMP null comment '创建时间', + update_time timestamp default CURRENT_TIMESTAMP null comment '更新时间', + constraint agent_datasource_tables_agent_datasource_id_table_name_uindex + unique (agent_datasource_id, table_name), + constraint agent_datasource_tables_agent_datasource_id_fk + foreign key (agent_datasource_id) references agent_datasource (id) + on update cascade on delete cascade ) - comment '某个智能体某个数据源所选中的数据表'; + comment '某个智能体某个数据源所选中的数据表'; -- 模型配置表 CREATE TABLE IF NOT EXISTS `model_config` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `provider` varchar(255) NOT NULL COMMENT '厂商标识 (方便前端展示回显，实际调用主要靠 baseUrl)', - `base_url` varchar(255) NOT NULL COMMENT '关键配置', - `api_key` varchar(255) NOT NULL COMMENT 'API密钥', - `model_name` varchar(255) NOT NULL COMMENT '模型名称', - `temperature` decimal(10,2) unsigned DEFAULT '0.00' COMMENT '温度参数', - `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', - `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', - `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', - `completions_path` varchar(255) DEFAULT NULL COMMENT 'Chat模型专用。附加到 Base URL 的路径。例如OpenAi的/v1/chat/completions', - `embeddings_path` varchar(255) DEFAULT NULL COMMENT '嵌入模型专用。附加到 Base URL 的路径。', - `created_time` datetime DEFAULT NULL COMMENT '创建时间', - `updated_time` datetime DEFAULT NULL COMMENT '更新时间', - `is_deleted` int(11) DEFAULT '0' COMMENT '0=未删除, 1=已删除', - -- 新增 AI 代理配置字段（默认关闭以确保零侵入性） - `proxy_enabled` tinyint(1) DEFAULT '0' COMMENT '是否启用代理：0-禁用，1-启用', - `proxy_host` varchar(255) DEFAULT NULL COMMENT '代理主机地址', - `proxy_port` int(11) DEFAULT NULL COMMENT '代理端口', - `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', - `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', - PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `id` int(11) NOT NULL AUTO_INCREMENT, + `provider` varchar(255) NOT NULL COMMENT '厂商标识 (方便前端展示回显，实际调用主要靠 baseUrl)', + `base_url` varchar(255) NOT NULL COMMENT '关键配置', + `api_key` varchar(255) NOT NULL COMMENT 'API密钥', + `model_name` varchar(255) NOT NULL COMMENT '模型名称', + `temperature` decimal(10, 2) unsigned DEFAULT '0.00' COMMENT '温度参数', + `is_active` tinyint(1) DEFAULT '0' COMMENT '是否激活', + `max_tokens` int(11) DEFAULT '2000' COMMENT '输出响应最大令牌数', + `model_type` varchar(20) NOT NULL DEFAULT 'CHAT' COMMENT '模型类型 (CHAT/EMBEDDING)', + `completions_path` varchar(255) DEFAULT NULL COMMENT 'Chat模型专用。附加到 Base URL 的路径。例如OpenAi的/v1/chat/completions', + `embeddings_path` varchar(255) DEFAULT NULL COMMENT '嵌入模型专用。附加到 Base URL 的路径。', + `created_time` datetime DEFAULT NULL COMMENT '创建时间', + `updated_time` datetime DEFAULT NULL COMMENT '更新时间', + `is_deleted` int(11) DEFAULT '0' COMMENT '0=未删除, 1=已删除', + -- 新增 AI 代理配置字段（默认关闭以确保零侵入性） + `proxy_enabled` tinyint(1) DEFAULT '0' COMMENT '是否启用代理：0-禁用，1-启用', + `proxy_host` varchar(255) DEFAULT NULL COMMENT '代理主机地址', + `proxy_port` int(11) DEFAULT NULL COMMENT '代理端口', + `proxy_username` varchar(255) DEFAULT NULL COMMENT '代理用户名（可选）', + `proxy_password` varchar(255) DEFAULT NULL COMMENT '代理密码（可选）', + PRIMARY KEY (`id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + +-- 文件存储表 +CREATE TABLE `file_storage` ( + `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键', + `filename` VARCHAR(255) NOT NULL COMMENT '文件名', + `file_path` VARCHAR(512) NOT NULL COMMENT '文件路径', + `file_size` INT DEFAULT NULL COMMENT '文件大小（字节）', + `file_type` VARCHAR(255) DEFAULT NULL COMMENT '文件类型', + `file_extension` VARCHAR(20) DEFAULT NULL COMMENT '文件后缀，如：.jpg/.pdf/.docx', + `storage_type` VARCHAR(50) NOT NULL COMMENT '存储类型：LOCAL/S3/OSS', + `is_deleted` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '删除状态：0=未删除, 1=已删除', + `is_cleaned` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '清理状态：0=物理资源未清理, 1=物理资源已清理', + `created_time` DATETIME DEFAULT NULL COMMENT '创建时间', + `updated_time` DATETIME DEFAULT NULL COMMENT '更新时间', + PRIMARY KEY (`id`), + KEY `idx_filename` (`filename`), + KEY `idx_file_path` (`file_path`(50)), + KEY `idx_updated_time` (`updated_time`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT ='文件存储表';