From b1f6110406abdbb0ae0532438d77acd98681a122 Mon Sep 17 00:00:00 2001 From: YunaiV Date: Sun, 1 Sep 2024 09:23:00 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E4=BB=A3=E7=A0=81=E8=AF=84=E5=AE=A1?= =?UTF-8?q?=E3=80=91AI=20=E5=A4=A7=E6=A8=A1=E5=9E=8B=EF=BC=9A=E7=9F=A5?= =?UTF-8?q?=E8=AF=86=E5=BA=93=E7=9A=84=E9=80=BB=E8=BE=91=20=E3=80=90?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96=E3=80=91AI=20=E5=A4=A7?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=EF=BC=9Atransformer=20=E7=9A=84=20onnx?= =?UTF-8?q?=E3=80=81tokenizer=20=E8=B5=B0=20CDN=EF=BC=8C=E9=81=BF=E5=85=8D?= =?UTF-8?q?=20github=20=E3=80=90=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E3=80=91AI=20=E5=A4=A7=E6=A8=A1=E5=9E=8B=EF=BC=9A=E5=B0=86=20s?= =?UTF-8?q?pring-ai=20=E8=B0=83=E6=95=B4=E6=88=90=20group.springframework.?= =?UTF-8?q?ai=EF=BC=8C=E8=A7=A3=E5=86=B3=20spring-ai=20=E6=9A=82=E6=97=B6?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E4=BD=BF=E7=94=A8=E9=98=BF=E9=87=8C=E4=BA=91?= =?UTF-8?q?=20maven=20=E5=8A=A0=E9=80=9F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yudao/module/ai/enums/AiChatRoleEnum.java | 21 +-- .../module/ai/enums/ErrorCodeConstants.java | 7 +- .../AiKnowledgeDocumentStatusEnum.java | 39 +++++ .../knowledge/AiKnowledgeController.java | 51 +++++++ .../AiKnowledgeDocumentController.java | 51 +++++++ .../AiKnowledgeSegmentController.java | 51 +++++++ .../AiKnowledgeDocumentPageReqVO.java | 14 ++ .../document/AiKnowledgeDocumentRespVO.java | 38 +++++ .../AiKnowledgeDocumentUpdateReqVO.java | 26 ++++ .../knowledge/AiKnowledgeCreateMyReqVO.java | 28 ++++ .../AiKnowledgeDocumentCreateReqVO.java | 26 ++++ .../vo/knowledge/AiKnowledgeRespVO.java | 26 ++++ .../knowledge/AiKnowledgeUpdateMyReqVO.java | 32 ++++ .../segment/AiKnowledgeSegmentPageReqVO.java | 20 +++ .../vo/segment/AiKnowledgeSegmentRespVO.java | 34 +++++ .../AiKnowledgeSegmentUpdateReqVO.java | 17 +++ .../AiKnowledgeSegmentUpdateStatusReqVO.java | 22 +++ .../ai/dal/dataobject/image/AiImageDO.java | 3 +- .../dataobject/knowledge/AiKnowledgeDO.java | 61 ++++++++ .../knowledge/AiKnowledgeDocumentDO.java | 64 ++++++++ .../knowledge/AiKnowledgeSegmentDO.java | 60 ++++++++ .../dal/dataobject/mindmap/AiMindMapDO.java | 3 +- .../ai/dal/dataobject/music/AiMusicDO.java | 3 +- .../ai/dal/dataobject/write/AiWriteDO.java | 3 +- .../knowledge/AiKnowledgeDocumentMapper.java | 24 +++ .../mysql/knowledge/AiKnowledgeMapper.java | 25 ++++ .../knowledge/AiKnowledgeSegmentMapper.java | 25 ++++ .../knowledge/AiKnowledgeDocumentService.java | 39 +++++ .../AiKnowledgeDocumentServiceImpl.java | 141 ++++++++++++++++++ .../knowledge/AiKnowledgeSegmentService.java | 38 +++++ .../AiKnowledgeSegmentServiceImpl.java | 42 ++++++ .../service/knowledge/AiKnowledgeService.java | 50 +++++++ .../knowledge/AiKnowledgeServiceImpl.java | 78 ++++++++++ .../ai/service/knowledge/DocService.java | 15 -- .../ai/service/knowledge/DocServiceImpl.java | 42 ------ .../ai/service/model/AiApiKeyService.java | 18 +++ .../ai/service/model/AiApiKeyServiceImpl.java | 20 +++ .../src/main/resources/application.yaml | 6 + .../yudao-spring-boot-starter-ai/pom.xml | 30 ++-- .../ai/config/YudaoAiAutoConfiguration.java | 64 ++++---- .../ai/core/factory/AiModelFactory.java | 13 ++ .../ai/core/factory/AiModelFactoryImpl.java | 33 +++- .../ai/core/factory/AiVectorStoreFactory.java | 28 ++++ .../factory/AiVectorStoreFactoryImpl.java | 52 +++++++ .../RedisVectorStoreAutoConfiguration.java | 4 +- 45 files changed, 1355 insertions(+), 132 deletions(-) create mode 100644 yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentPageReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentRespVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeRespVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateMyReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentRespVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateStatusReqVO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java delete mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java create mode 100644 yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java create mode 100644 yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/AiChatRoleEnum.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/AiChatRoleEnum.java index 811189efed..029961bf3f 100644 --- a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/AiChatRoleEnum.java +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/AiChatRoleEnum.java @@ -1,11 +1,8 @@ package cn.iocoder.yudao.module.ai.enums; -import cn.iocoder.yudao.framework.common.core.IntArrayValuable; import lombok.AllArgsConstructor; import lombok.Getter; -import java.util.Arrays; - /** * AI 内置聊天角色的枚举 * @@ -13,16 +10,16 @@ */ @AllArgsConstructor @Getter -public enum AiChatRoleEnum implements IntArrayValuable { +public enum AiChatRoleEnum { - AI_WRITE_ROLE(1, "写作助手", """ + AI_WRITE_ROLE("写作助手", """ 你是一位出色的写作助手，能够帮助用户生成创意和灵感，并在用户提供场景和提示词时生成对应的回复。你的任务包括： 1. 撰写建议：根据用户提供的主题或问题，提供详细的写作建议、情节发展方向、角色设定以及背景描写，确保内容结构清晰、有逻辑。 2. 回复生成：根据用户提供的场景和提示词，生成合适的对话或文字回复，确保语气和风格符合场景需求。 除此之外不需要除了正文内容外的其他回复，如标题、开头、任何解释性语句或道歉。 """), - AI_MIND_MAP_ROLE(2, "导图助手", """ + AI_MIND_MAP_ROLE("导图助手", """ 你是一位非常优秀的思维导图助手，你会把用户的所有提问都总结成思维导图，然后以 Markdown 格式输出。markdown 只需要输出一级标题，二级标题，三级标题，四级标题，最多输出四级，除此之外不要输出任何其他 markdown 标记。下面是一个合格的例子： # Geek-AI 助手 ## 完整的开源系统 @@ -39,11 +36,6 @@ public enum AiChatRoleEnum implements IntArrayValuable { 除此之外不要任何解释性语句。 """); - // TODO @xin：这个 role 是不是删除掉好点哈。= = 目前主要是没做角色枚举。这里多了 role 反倒容易误解哈 - /** - * 角色 - */ - private final Integer role; /** * 角色名 */ @@ -54,11 +46,4 @@ public enum AiChatRoleEnum implements IntArrayValuable { */ private final String systemMessage; - public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiChatRoleEnum::getRole).toArray(); - - @Override - public int[] array() { - return ARRAYS; - } - } diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java index 50934e7a0a..e1dd1a9568 100644 --- a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/ErrorCodeConstants.java @@ -4,7 +4,7 @@ /** * AI 错误码枚举类 - * + *

* ai 系统，使用 1-040-000-000 段 */ public interface ErrorCodeConstants { @@ -52,4 +52,9 @@ public interface ErrorCodeConstants { // ========== API 思维导图 1-040-008-000 ========== ErrorCode MIND_MAP_NOT_EXISTS = new ErrorCode(1_040_008_000, "思维导图不存在!"); + // ========== API 知识库 1-022-008-000 ========== + ErrorCode KNOWLEDGE_NOT_EXISTS = new ErrorCode(1_022_008_000, "知识库不存在!"); + ErrorCode KNOWLEDGE_DOCUMENT_NOT_EXISTS = new ErrorCode(1_022_008_001, "文档不存在!"); + ErrorCode KNOWLEDGE_SEGMENT_NOT_EXISTS = new ErrorCode(1_022_008_002, "段落不存在!"); + } diff --git a/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java new file mode 100644 index 0000000000..a37fa86435 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-api/src/main/java/cn/iocoder/yudao/module/ai/enums/knowledge/AiKnowledgeDocumentStatusEnum.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.enums.knowledge; + +import cn.iocoder.yudao.framework.common.core.IntArrayValuable; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * AI 知识库-文档状态的枚举 + * + * @author xiaoxin + */ +@AllArgsConstructor +@Getter +public enum AiKnowledgeDocumentStatusEnum implements IntArrayValuable { + + IN_PROGRESS(10, "索引中"), + SUCCESS(20, "可用"), + FAIL(30, "失败"); + + /** + * 状态 + */ + private final Integer status; + + /** + * 状态名 + */ + private final String name; + + public static final int[] ARRAYS = Arrays.stream(values()).mapToInt(AiKnowledgeDocumentStatusEnum::getStatus).toArray(); + + @Override + public int[] array() { + return ARRAYS; + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java new file mode 100644 index 0000000000..dc2c8e3aec --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeController.java @@ -0,0 +1,51 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeRespVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; +import static cn.iocoder.yudao.framework.security.core.util.SecurityFrameworkUtils.getLoginUserId; + +@Tag(name = "管理后台 - AI 知识库") +@RestController +@RequestMapping("/ai/knowledge") +@Validated +public class AiKnowledgeController { + + @Resource + private AiKnowledgeService knowledgeService; + + @GetMapping("/my-page") + @Operation(summary = "获取【我的】知识库分页") + public CommonResult> getKnowledgePageMy(@Validated PageParam pageReqVO) { + PageResult pageResult = knowledgeService.getKnowledgePageMy(getLoginUserId(), pageReqVO); + return success(BeanUtils.toBean(pageResult, AiKnowledgeRespVO.class)); + } + + @PostMapping("/create-my") + @Operation(summary = "创建【我的】知识库") + public CommonResult createKnowledgeMy(@RequestBody @Valid AiKnowledgeCreateMyReqVO createReqVO) { + return success(knowledgeService.createKnowledgeMy(createReqVO, getLoginUserId())); + } + + @PutMapping("/update-my") + @Operation(summary = "更新【我的】知识库") + public CommonResult updateKnowledgeMy(@RequestBody @Valid AiKnowledgeUpdateMyReqVO updateReqVO) { + knowledgeService.updateKnowledgeMy(updateReqVO, getLoginUserId()); + return success(true); + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java new file mode 100644 index 0000000000..d86210556e --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeDocumentController.java @@ -0,0 +1,51 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentRespVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeDocumentService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; + +@Tag(name = "管理后台 - AI 知识库文档") +@RestController +@RequestMapping("/ai/knowledge/document") +@Validated +public class AiKnowledgeDocumentController { + + @Resource + private AiKnowledgeDocumentService documentService; + + @PostMapping("/create") + @Operation(summary = "新建文档") + public CommonResult createKnowledgeDocument(@Valid AiKnowledgeDocumentCreateReqVO reqVO) { + Long knowledgeDocumentId = documentService.createKnowledgeDocument(reqVO); + return success(knowledgeDocumentId); + } + + @GetMapping("/page") + @Operation(summary = "获取文档分页") + public CommonResult> getKnowledgeDocumentPageMy(@Valid AiKnowledgeDocumentPageReqVO pageReqVO) { + PageResult pageResult = documentService.getKnowledgeDocumentPage(pageReqVO); + return success(BeanUtils.toBean(pageResult, AiKnowledgeDocumentRespVO.class)); + } + + @PutMapping("/update") + @Operation(summary = "更新文档") + public CommonResult updateKnowledgeDocument(@Valid @RequestBody AiKnowledgeDocumentUpdateReqVO reqVO) { + documentService.updateKnowledgeDocument(reqVO); + return success(true); + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java new file mode 100644 index 0000000000..a0d0952a83 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java @@ -0,0 +1,51 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.CommonResult; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentRespVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeSegmentService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success; + +@Tag(name = "管理后台 - AI 知识库段落") +@RestController +@RequestMapping("/ai/knowledge/segment") +@Validated +public class AiKnowledgeSegmentController { + + @Resource + private AiKnowledgeSegmentService segmentService; + + @GetMapping("/page") + @Operation(summary = "获取段落分页") + public CommonResult> getKnowledgeSegmentPageMy(@Valid AiKnowledgeSegmentPageReqVO pageReqVO) { + PageResult pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO); + return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class)); + } + + @PutMapping("/update") + @Operation(summary = "更新段落内容") + public CommonResult updateKnowledgeSegment(@Valid @RequestBody AiKnowledgeSegmentUpdateReqVO reqVO) { + segmentService.updateKnowledgeSegment(reqVO); + return success(true); + } + + @PutMapping("/update-status") + @Operation(summary = "启禁用段落内容") + public CommonResult updateKnowledgeSegmentStatus(@Valid @RequestBody AiKnowledgeSegmentUpdateStatusReqVO reqVO) { + segmentService.updateKnowledgeSegmentStatus(reqVO); + return success(true); + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentPageReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentPageReqVO.java new file mode 100644 index 0000000000..76c001bd35 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentPageReqVO.java @@ -0,0 +1,14 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document; + +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Schema(description = "管理后台 - AI 知识库文档的分页 Request VO") +@Data +public class AiKnowledgeDocumentPageReqVO extends PageParam { + + @Schema(description = "文档名称", example = "Java 开发手册") + private String name; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentRespVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentRespVO.java new file mode 100644 index 0000000000..96ca61b3d2 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentRespVO.java @@ -0,0 +1,38 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document; + +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Schema(description = "管理后台 - AI 知识库-文档 Response VO") +@Data +public class AiKnowledgeDocumentRespVO extends PageParam { + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long id; + + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long knowledgeId; + + @Schema(description = "名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册") + private String name; + + @Schema(description = "内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 是一门面向对象的语言.....") + private String content; + + @Schema(description = "文档 url", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn") + private String url; + + @Schema(description = "token 数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "1024") + private Integer tokens; + + @Schema(description = "字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "1008") + private Integer wordCount; + + @Schema(description = "切片状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + private Integer sliceStatus; + + @Schema(description = "文档状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java new file mode 100644 index 0000000000..2cc6a32f3c --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/document/AiKnowledgeDocumentUpdateReqVO.java @@ -0,0 +1,26 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.validation.InEnum; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + + +@Schema(description = "管理后台 - AI 更新 知识库-文档 Request VO") +@Data +public class AiKnowledgeDocumentUpdateReqVO { + + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "15583") + @NotNull(message = "编号不能为空") + private Long id; + + @Schema(description = "是否启用", example = "1") + @InEnum(CommonStatusEnum.class) + private Integer status; + + @Schema(description = "名称", example = "Java 开发手册") + private String name; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java new file mode 100644 index 0000000000..44a5e87eec --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeCreateMyReqVO.java @@ -0,0 +1,28 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Schema(description = "管理后台 - AI 知识库创建【我的】 Request VO") +@Data +public class AiKnowledgeCreateMyReqVO { + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "存储 ruoyi-vue-pro 操作文档") + private String description; + + @Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java new file mode 100644 index 0000000000..9cc5290ab3 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeDocumentCreateReqVO.java @@ -0,0 +1,26 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.hibernate.validator.constraints.URL; + + +@Schema(description = "管理后台 - AI 知识库文档的创建 Request VO") +@Data +public class AiKnowledgeDocumentCreateReqVO { + + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long knowledgeId; + + @Schema(description = "文档名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "三方登陆") + @NotBlank(message = "文档名称不能为空") + private String name; + + @Schema(description = "文档 URL", requiredMode = Schema.RequiredMode.REQUIRED, example = "https://doc.iocoder.cn") + @URL(message = "文档 URL 格式不正确") + private String url; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeRespVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeRespVO.java new file mode 100644 index 0000000000..3ff8a1c757 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeRespVO.java @@ -0,0 +1,26 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + + +@Schema(description = "管理后台 - AI 知识库 Response VO") +@Data +public class AiKnowledgeRespVO { + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long id; + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "ruoyi-vue-pro 用户指南") + private String name; + + @Schema(description = "知识库描述", example = "帮助你快速构建系统") + private String description; + + @Schema(description = "模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "14") + private Long modelId; + + @Schema(description = "模型标识", requiredMode = Schema.RequiredMode.REQUIRED, example = "qwen-72b-chat") + private String model; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateMyReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateMyReqVO.java new file mode 100644 index 0000000000..987c9bf4ac --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/knowledge/AiKnowledgeUpdateMyReqVO.java @@ -0,0 +1,32 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Schema(description = "管理后台 - AI 知识库更新【我的】 Request VO") +@Data +public class AiKnowledgeUpdateMyReqVO { + + @Schema(description = "对话编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1204") + @NotNull(message = "知识库编号不能为空") + private Long id; + + @Schema(description = "知识库名称", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + @NotBlank(message = "知识库名称不能为空") + private String name; + + @Schema(description = "知识库描述", requiredMode = Schema.RequiredMode.REQUIRED, example = "") + private String description; + + @Schema(description = "可见权限，只能选择哪些人可见", requiredMode = Schema.RequiredMode.REQUIRED, example = "[1]") + private List visibilityPermissions; + + @Schema(description = "嵌入模型编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "嵌入模型不能为空") + private Long modelId; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java new file mode 100644 index 0000000000..8be3db501b --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentPageReqVO.java @@ -0,0 +1,20 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment; + +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Schema(description = "管理后台 - AI 知识库分段的分页 Request VO") +@Data +public class AiKnowledgeSegmentPageReqVO extends PageParam { + + @Schema(description = "分段状态", example = "1") + private Integer status; + + @Schema(description = "文档编号", example = "1") + private Integer documentId; + + @Schema(description = "分段内容关键字", example = "Java 开发") + private String keyword; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentRespVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentRespVO.java new file mode 100644 index 0000000000..5e3f2d8cbb --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentRespVO.java @@ -0,0 +1,34 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Schema(description = "管理后台 - AI 知识库-文档 Response VO") +@Data +public class AiKnowledgeSegmentRespVO { + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long id; + + @Schema(description = "文档编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long documentId; + + @Schema(description = "知识库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long knowledgeId; + + @Schema(description = "向量库编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1858496a-1dde-4edf-a43e-0aed08f37f8c") + private String vectorId; + + @Schema(description = "切片内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册") + private String content; + + @Schema(description = "token 数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "1024") + private Integer tokens; + + @Schema(description = "字符数", requiredMode = Schema.RequiredMode.REQUIRED, example = "1008") + private Integer wordCount; + + @Schema(description = "文档状态", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java new file mode 100644 index 0000000000..23b1461e2d --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java @@ -0,0 +1,17 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + + +@Schema(description = "管理后台 - AI 更新 知识库-段落 request VO") +@Data +public class AiKnowledgeSegmentUpdateReqVO { + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long id; + + @Schema(description = "切片内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册") + private String content; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateStatusReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateStatusReqVO.java new file mode 100644 index 0000000000..2516c7dfb2 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateStatusReqVO.java @@ -0,0 +1,22 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.validation.InEnum; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + + +@Schema(description = "管理后台 - AI 知识库段落的更新状态 Request VO") +@Data +public class AiKnowledgeSegmentUpdateStatusReqVO { + + @Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790") + private Long id; + + @Schema(description = "是否启用", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + @NotNull(message = "是否启用不能为空") + @InEnum(CommonStatusEnum.class) + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java index a894caee66..09c8cf6657 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java @@ -5,7 +5,6 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; import cn.iocoder.yudao.module.ai.enums.image.AiImageStatusEnum; import cn.iocoder.yudao.module.system.api.user.dto.AdminUserRespDTO; -import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -30,7 +29,7 @@ public class AiImageDO extends BaseDO { /** * 编号 */ - @TableId(type = IdType.AUTO) + @TableId private Long id; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java new file mode 100644 index 0000000000..756d8cdb3e --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDO.java @@ -0,0 +1,61 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler; +import lombok.Data; + +import java.util.List; + +/** + * AI 知识库 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge", autoResultMap = true) +@Data +public class AiKnowledgeDO extends BaseDO { + + /** + * 编号 + */ + @TableId + private Long id; + /** + * 用户编号 + *

+ * 关联 AdminUserDO 的 userId 字段 + */ + private Long userId; + /** + * 知识库名称 + */ + private String name; + /** + * 知识库描述 + */ + private String description; + // TODO @新：如果全部可见，需要怎么设置？ + /** + * 可见权限,只能选择哪些人可见 + */ + @TableField(typeHandler = JacksonTypeHandler.class) + private List visibilityPermissions; + /** + * 嵌入模型编号 + */ + private Long modelId; + /** + * 模型标识 + */ + private String model; + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java new file mode 100644 index 0000000000..c5e526cce1 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeDocumentDO.java @@ -0,0 +1,64 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_document") +@Data +public class AiKnowledgeDocumentDO extends BaseDO { + + /** + * 编号 + */ + @TableId + private Long id; + /** + * 知识库编号 + * + * 关联 {@link AiKnowledgeDO#getId()} + */ + private Long knowledgeId; + /** + * 文件名称 + */ + private String name; + /** + * 内容 + */ + private String content; + /** + * 文件 URL + */ + private String url; + /** + * token 数量 + */ + private Integer tokens; + /** + * 字符数 + */ + private Integer wordCount; + /** + * 切片状态 + *

+ * 枚举 {@link AiKnowledgeDocumentStatusEnum} + */ + private Integer sliceStatus; + + /** + * 状态 + *

+ * 枚举 {@link CommonStatusEnum} + */ + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java new file mode 100644 index 0000000000..84f7de6549 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/knowledge/AiKnowledgeSegmentDO.java @@ -0,0 +1,60 @@ +package cn.iocoder.yudao.module.ai.dal.dataobject.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * AI 知识库-文档分段 DO + * + * @author xiaoxin + */ +@TableName(value = "ai_knowledge_segment") +@Data +public class AiKnowledgeSegmentDO extends BaseDO { + + public static final String FIELD_KNOWLEDGE_ID = "knowledgeId"; + + /** + * 编号 + */ + @TableId + private Long id; + /** + * 向量库的编号 + */ + private String vectorId; + /** + * 知识库编号 + * + * 关联 {@link AiKnowledgeDO#getId()} + */ + private Long knowledgeId; + /** + * 文档编号 + * + * 关联 {@link AiKnowledgeDocumentDO#getId()} + */ + private Long documentId; + /** + * 切片内容 + */ + private String content; + /** + * 字符数 + */ + private Integer wordCount; + /** + * token 数量 + */ + private Integer tokens; + /** + * 状态 + * + * 枚举 {@link CommonStatusEnum} + */ + private Integer status; + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java index 0442a52d75..334f494708 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/mindmap/AiMindMapDO.java @@ -2,7 +2,6 @@ import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; -import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -19,7 +18,7 @@ public class AiMindMapDO extends BaseDO { /** * 编号 */ - @TableId(type = IdType.AUTO) + @TableId private Long id; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java index 8a6cbe8288..b7d90239bc 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/music/AiMusicDO.java @@ -4,7 +4,6 @@ import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; import cn.iocoder.yudao.module.ai.enums.music.AiMusicGenerateModeEnum; import cn.iocoder.yudao.module.ai.enums.music.AiMusicStatusEnum; -import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -25,7 +24,7 @@ public class AiMusicDO extends BaseDO { /** * 编号 */ - @TableId(type = IdType.AUTO) + @TableId private Long id; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java index 752876f2a6..01af53462d 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/write/AiWriteDO.java @@ -3,7 +3,6 @@ import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO; import cn.iocoder.yudao.module.ai.enums.write.AiWriteTypeEnum; -import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -20,7 +19,7 @@ public class AiWriteDO extends BaseDO { /** * 编号 */ - @TableId(type = IdType.AUTO) + @TableId private Long id; /** diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java new file mode 100644 index 0000000000..7692d1cede --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeDocumentMapper.java @@ -0,0 +1,24 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库-文档 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeDocumentMapper extends BaseMapperX { + + default PageResult selectPage(AiKnowledgeDocumentPageReqVO reqVO) { + return selectPage(reqVO, new LambdaQueryWrapperX() + .likeIfPresent(AiKnowledgeDocumentDO::getName, reqVO.getName()) + .orderByDesc(AiKnowledgeDocumentDO::getId)); + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java new file mode 100644 index 0000000000..2bf23411a6 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeMapper.java @@ -0,0 +1,25 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库基础信息 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeMapper extends BaseMapperX { + + default PageResult selectPageByMy(Long userId, PageParam pageReqVO) { + return selectPage(pageReqVO, new LambdaQueryWrapperX() + .eq(AiKnowledgeDO::getUserId, userId) + .eq(AiKnowledgeDO::getStatus, CommonStatusEnum.ENABLE.getStatus()) + .orderByDesc(AiKnowledgeDO::getId)); + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java new file mode 100644 index 0000000000..912d18cbc6 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java @@ -0,0 +1,25 @@ +package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; +import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * AI 知识库-分片 Mapper + * + * @author xiaoxin + */ +@Mapper +public interface AiKnowledgeSegmentMapper extends BaseMapperX { + + default PageResult selectPage(AiKnowledgeSegmentPageReqVO reqVO) { + return selectPage(reqVO, new LambdaQueryWrapperX() + .eq(AiKnowledgeSegmentDO::getDocumentId, reqVO.getDocumentId()) + .eqIfPresent(AiKnowledgeSegmentDO::getStatus, reqVO.getStatus()) + .likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword()) + .orderByDesc(AiKnowledgeSegmentDO::getId)); + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java new file mode 100644 index 0000000000..3de0ac01de --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentService.java @@ -0,0 +1,39 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; + +/** + * AI 知识库-文档 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeDocumentService { + + /** + * 创建文档 + * + * @param createReqVO 文档创建 Request VO + * @return 文档编号 + */ + Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO); + + + /** + * 获取文档分页 + * + * @param pageReqVO 分页参数 + * @return 文档分页 + */ + PageResult getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO); + + /** + * 更新文档 + * + * @param reqVO 更新信息 + */ + void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java new file mode 100644 index 0000000000..99f0621c81 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -0,0 +1,141 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.http.HttpUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.document.AiKnowledgeDocumentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeDocumentCreateReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeDocumentMapper; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; +import cn.iocoder.yudao.module.ai.enums.knowledge.AiKnowledgeDocumentStatusEnum; +import cn.iocoder.yudao.module.ai.service.model.AiApiKeyService; +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.document.Document; +import org.springframework.ai.reader.tika.TikaDocumentReader; +import org.springframework.ai.tokenizer.TokenCountEstimator; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_DOCUMENT_NOT_EXISTS; + +/** + * AI 知识库文档 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentService { + + @Resource + private AiKnowledgeDocumentMapper documentMapper; + @Resource + private AiKnowledgeSegmentMapper segmentMapper; + + @Resource + private TokenTextSplitter tokenTextSplitter; + @Resource + private TokenCountEstimator tokenCountEstimator; + + @Resource + private AiApiKeyService apiKeyService; + @Resource + private AiKnowledgeService knowledgeService; + @Resource + private AiChatModelService chatModelService; + + @Override + @Transactional(rollbackFor = Exception.class) + public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) { + // 0. 校验 + AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId()); + AiChatModelDO model = chatModelService.validateChatModel(knowledge.getModelId()); + + // 1.1 下载文档 + TikaDocumentReader loader = new TikaDocumentReader(downloadFile(createReqVO.getUrl())); + List documents = loader.get(); + Document document = CollUtil.getFirst(documents); + // 1.2 文档记录入库 + String content = document.getContent(); + AiKnowledgeDocumentDO documentDO = BeanUtils.toBean(createReqVO, AiKnowledgeDocumentDO.class) + .setTokens(tokenCountEstimator.estimate(content)).setWordCount(content.length()) + .setStatus(CommonStatusEnum.ENABLE.getStatus()).setSliceStatus(AiKnowledgeDocumentStatusEnum.SUCCESS.getStatus()); + documentMapper.insert(documentDO); + Long documentId = documentDO.getId(); + if (CollUtil.isEmpty(documents)) { + return documentId; + } + + // 2.1 文档分段 + List segments = tokenTextSplitter.apply(documents); + // 2.2 分段内容入库 + List segmentDOList = CollectionUtils.convertList(segments, + segment -> new AiKnowledgeSegmentDO().setContent(segment.getContent()).setDocumentId(documentId) + .setKnowledgeId(createReqVO.getKnowledgeId()).setVectorId(segment.getId()) + .setTokens(tokenCountEstimator.estimate(segment.getContent())).setWordCount(segment.getContent().length()) + .setStatus(CommonStatusEnum.ENABLE.getStatus())); + segmentMapper.insertBatch(segmentDOList); + + // 3.1 获取向量存储实例 + VectorStore vectorStore = apiKeyService.getOrCreateVectorStore(model.getKeyId()); + // 3.2 向量化并存储 + segments.forEach(segment -> segment.getMetadata().put(AiKnowledgeSegmentDO.FIELD_KNOWLEDGE_ID, createReqVO.getKnowledgeId())); + vectorStore.add(segments); + return documentId; + } + + @Override + public PageResult getKnowledgeDocumentPage(AiKnowledgeDocumentPageReqVO pageReqVO) { + return documentMapper.selectPage(pageReqVO); + } + + @Override + public void updateKnowledgeDocument(AiKnowledgeDocumentUpdateReqVO reqVO) { + // 1. 校验文档是否存在 + validateKnowledgeDocumentExists(reqVO.getId()); + // 2. 更新文档 + AiKnowledgeDocumentDO document = BeanUtils.toBean(reqVO, AiKnowledgeDocumentDO.class); + documentMapper.updateById(document); + } + + /** + * 校验文档是否存在 + * + * @param id 文档编号 + * @return 文档信息 + */ + private AiKnowledgeDocumentDO validateKnowledgeDocumentExists(Long id) { + AiKnowledgeDocumentDO knowledgeDocument = documentMapper.selectById(id); + if (knowledgeDocument == null) { + throw exception(KNOWLEDGE_DOCUMENT_NOT_EXISTS); + } + return knowledgeDocument; + } + + private org.springframework.core.io.Resource downloadFile(String url) { + try { + byte[] bytes = HttpUtil.downloadBytes(url); + return new ByteArrayResource(bytes); + } catch (Exception e) { + log.error("[downloadFile][url({}) 下载失败]", url, e); + throw new RuntimeException(e); + } + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java new file mode 100644 index 0000000000..8ecb2d24ae --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java @@ -0,0 +1,38 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; + +/** + * AI 知识库段落 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeSegmentService { + + /** + * 获取段落分页 + * + * @param pageReqVO 分页查询 + * @return 文档分页 + */ + PageResult getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO); + + /** + * 更新段落的内容 + * + * @param reqVO 更新内容 + */ + void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO); + + /** + * 更新段落的状态 + * + * @param reqVO 更新内容 + */ + void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO); + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java new file mode 100644 index 0000000000..7f751b1761 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java @@ -0,0 +1,42 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeSegmentMapper; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +/** + * AI 知识库分片 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService { + + @Resource + private AiKnowledgeSegmentMapper segmentMapper; + + @Override + public PageResult getKnowledgeSegmentPage(AiKnowledgeSegmentPageReqVO pageReqVO) { + return segmentMapper.selectPage(pageReqVO); + } + + @Override + public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) { + segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class)); + // TODO @xin 重新向量化 + } + + @Override + public void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO) { + segmentMapper.updateById(BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class)); + // TODO @xin 1.禁用删除向量 2.启用重新向量化 + } +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java new file mode 100644 index 0000000000..9f43c53283 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeService.java @@ -0,0 +1,50 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; + +/** + * AI 知识库-基础信息 Service 接口 + * + * @author xiaoxin + */ +public interface AiKnowledgeService { + + /** + * 创建【我的】知识库 + * + * @param createReqVO 创建信息 + * @param userId 用户编号 + * @return 编号 + */ + Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId); + + + /** + * 创建【我的】知识库 + * + * @param updateReqVO 更新信息 + * @param userId 用户编号 + */ + void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId); + + + /** + * 校验知识库是否存在 + * + * @param id 记录编号 + */ + AiKnowledgeDO validateKnowledgeExists(Long id); + + /** + * 获得【我的】知识库分页 + * + * @param userId 用户编号 + * @param pageReqVO 分页查询 + * @return 知识库分页 + */ + PageResult getKnowledgePageMy(Long userId, PageParam pageReqVO); +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java new file mode 100644 index 0000000000..1948bb00e6 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java @@ -0,0 +1,78 @@ +package cn.iocoder.yudao.module.ai.service.knowledge; + +import cn.hutool.core.util.ObjUtil; +import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; +import cn.iocoder.yudao.framework.common.pojo.PageParam; +import cn.iocoder.yudao.framework.common.pojo.PageResult; +import cn.iocoder.yudao.framework.common.util.object.BeanUtils; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeCreateMyReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeUpdateMyReqVO; +import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; +import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO; +import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper; +import cn.iocoder.yudao.module.ai.service.model.AiChatModelService; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception; +import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_NOT_EXISTS; + +/** + * AI 知识库-基础信息 Service 实现类 + * + * @author xiaoxin + */ +@Service +@Slf4j +public class AiKnowledgeServiceImpl implements AiKnowledgeService { + + @Resource + private AiChatModelService chatModalService; + + @Resource + private AiKnowledgeMapper knowledgeMapper; + + @Override + public Long createKnowledgeMy(AiKnowledgeCreateMyReqVO createReqVO, Long userId) { + // 1. 校验模型配置 + AiChatModelDO model = chatModalService.validateChatModel(createReqVO.getModelId()); + + // 2. 插入知识库 + AiKnowledgeDO knowledgeBase = BeanUtils.toBean(createReqVO, AiKnowledgeDO.class) + .setModel(model.getModel()).setUserId(userId).setStatus(CommonStatusEnum.ENABLE.getStatus()); + knowledgeMapper.insert(knowledgeBase); + return knowledgeBase.getId(); + } + + @Override + public void updateKnowledgeMy(AiKnowledgeUpdateMyReqVO updateReqVO, Long userId) { + // 1.1 校验知识库存在 + AiKnowledgeDO knowledgeBaseDO = validateKnowledgeExists(updateReqVO.getId()); + if (ObjUtil.notEqual(knowledgeBaseDO.getUserId(), userId)) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + // 1.2 校验模型配置 + AiChatModelDO model = chatModalService.validateChatModel(updateReqVO.getModelId()); + + // 2. 更新知识库 + AiKnowledgeDO updateDO = BeanUtils.toBean(updateReqVO, AiKnowledgeDO.class); + updateDO.setModel(model.getModel()); + knowledgeMapper.updateById(updateDO); + } + + @Override + public AiKnowledgeDO validateKnowledgeExists(Long id) { + AiKnowledgeDO knowledgeBase = knowledgeMapper.selectById(id); + if (knowledgeBase == null) { + throw exception(KNOWLEDGE_NOT_EXISTS); + } + return knowledgeBase; + } + + @Override + public PageResult getKnowledgePageMy(Long userId, PageParam pageReqVO) { + return knowledgeMapper.selectPageByMy(userId, pageReqVO); + } + +} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java deleted file mode 100644 index 47905d4b15..0000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java +++ /dev/null @@ -1,15 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -/** - * AI 知识库 Service 接口 - * - * @author xiaoxin - */ -public interface DocService { - - /** - * 向量化文档 - */ - void embeddingDoc(); - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java deleted file mode 100644 index 7ba5018bd5..0000000000 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -package cn.iocoder.yudao.module.ai.service.knowledge; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.ai.document.Document; -import org.springframework.ai.reader.tika.TikaDocumentReader; -import org.springframework.ai.transformer.splitter.TokenTextSplitter; -import org.springframework.ai.vectorstore.RedisVectorStore; -import org.springframework.beans.factory.annotation.Value; - -import java.util.List; - -/** - * AI 知识库 Service 实现类 - * - * @author xiaoxin - */ -//@Service // TODO 芋艿：临时注释，避免无法启动 -@Slf4j -public class DocServiceImpl implements DocService { - - @Resource - private RedisVectorStore vectorStore; - @Resource - private TokenTextSplitter tokenTextSplitter; - - // TODO @xin 临时测试用，后续删 - @Value("classpath:/webapp/test/Fel.pdf") - private org.springframework.core.io.Resource data; - - @Override - public void embeddingDoc() { - // 读取文件 - TikaDocumentReader loader = new TikaDocumentReader(data); - List documents = loader.get(); - // 文档分段 - List segments = tokenTextSplitter.apply(documents); - // 向量化并存储 - vectorStore.add(segments); - } - -} diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyService.java index fe8fdd194e..f5f8813492 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyService.java @@ -9,7 +9,9 @@ import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiApiKeyDO; import jakarta.validation.Valid; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.image.ImageModel; +import org.springframework.ai.vectorstore.VectorStore; import java.util.List; @@ -111,4 +113,20 @@ public interface AiApiKeyService { */ SunoApi getSunoApi(); + /** + * 获得 EmbeddingModel 对象 + * + * @param id 编号 + * @return EmbeddingModel 对象 + */ + EmbeddingModel getEmbeddingModel(Long id); + + /** + * 获得 VectorStore 对象 + * + * @param id 编号 + * @return VectorStore 对象 + */ + VectorStore getOrCreateVectorStore(Long id); + } \ No newline at end of file diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java index 590b10a4c8..bf11ec2184 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiApiKeyServiceImpl.java @@ -2,6 +2,7 @@ import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory; +import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory; import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; @@ -13,7 +14,9 @@ import cn.iocoder.yudao.module.ai.dal.mysql.model.AiApiKeyMapper; import jakarta.annotation.Resource; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.image.ImageModel; +import org.springframework.ai.vectorstore.VectorStore; import org.springframework.stereotype.Service; import org.springframework.validation.annotation.Validated; @@ -36,6 +39,8 @@ public class AiApiKeyServiceImpl implements AiApiKeyService { @Resource private AiModelFactory modelFactory; + @Resource + private AiVectorStoreFactory vectorFactory; @Override public Long createApiKey(AiApiKeySaveReqVO createReqVO) { @@ -132,4 +137,19 @@ public SunoApi getSunoApi() { } return modelFactory.getOrCreateSunoApi(apiKey.getApiKey(), apiKey.getUrl()); } + + @Override + public EmbeddingModel getEmbeddingModel(Long id) { + AiApiKeyDO apiKey = validateApiKey(id); + AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); + return modelFactory.getOrCreateEmbeddingModel(platform, apiKey.getApiKey(), apiKey.getUrl()); + } + + @Override + public VectorStore getOrCreateVectorStore(Long id) { + AiApiKeyDO apiKey = validateApiKey(id); + AiPlatformEnum platform = AiPlatformEnum.validatePlatform(apiKey.getPlatform()); + return vectorFactory.getOrCreateVectorStore(getEmbeddingModel(id), platform, apiKey.getApiKey(), apiKey.getUrl()); + } + } \ No newline at end of file diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/resources/application.yaml b/yudao-module-ai/yudao-module-ai-biz/src/main/resources/application.yaml index f365fb3806..b4f650acfa 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/resources/application.yaml +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/resources/application.yaml @@ -108,6 +108,12 @@ spring: redis: index: default-index prefix: "default:" + embedding: + transformer: + onnx: + model-uri: http://test.yudao.iocoder.cn/model.onnx + tokenizer: + uri: http://test.yudao.iocoder.cn/tokenizer.json qianfan: # 文心一言 api-key: x0cuLZ7XsaTCU08vuJWO87Lg secret-key: R9mYF9dl9KASgi5RUq0FQt3wRisSnOcK diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml index 62fe50330f..916e0b416e 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml @@ -14,58 +14,57 @@ ${project.artifactId} AI 大模型拓展，接入国内外大模型 - 1.0.0-M1 + group.springframework.ai + 1.1.0 - org.springframework.ai + ${spring-ai.groupId} spring-ai-zhipuai-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-openai-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-azure-openai-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-ollama-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-stability-ai-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-transformers-spring-boot-starter ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-tika-document-reader ${spring-ai.version} - org.springframework.ai + ${spring-ai.groupId} spring-ai-redis-store ${spring-ai.version} - - org.springframework.data - spring-data-redis - true + cn.iocoder.cloud + yudao-spring-boot-starter-redis @@ -73,11 +72,10 @@ yudao-common - - group.springframework.ai + ${spring-ai.groupId} spring-ai-qianfan-spring-boot-starter - 1.1.0 + ${spring-ai.version} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java index 543444fddd..79a1f345b4 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java @@ -2,6 +2,8 @@ import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactory; import cn.iocoder.yudao.framework.ai.core.factory.AiModelFactoryImpl; +import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactory; +import cn.iocoder.yudao.framework.ai.core.factory.AiVectorStoreFactoryImpl; import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatModel; import cn.iocoder.yudao.framework.ai.core.model.deepseek.DeepSeekChatOptions; import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; @@ -10,20 +12,15 @@ import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatOptions; import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration; import lombok.extern.slf4j.Slf4j; -import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties; -import org.springframework.ai.document.MetadataMode; -import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.tokenizer.JTokkitTokenCountEstimator; +import org.springframework.ai.tokenizer.TokenCountEstimator; import org.springframework.ai.transformer.splitter.TokenTextSplitter; -import org.springframework.ai.transformers.TransformersEmbeddingModel; -import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.autoconfigure.data.redis.RedisProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Lazy; -import redis.clients.jedis.JedisPooled; /** * 芋道 AI 自动配置 @@ -41,6 +38,12 @@ public AiModelFactory aiModelFactory() { return new AiModelFactoryImpl(); } + @Bean + public AiVectorStoreFactory aiVectorFactory() { + return new AiVectorStoreFactoryImpl(); + } + + // ========== 各种 AI Client 创建 ========== @Bean @@ -83,35 +86,42 @@ public SunoApi sunoApi(YudaoAiProperties yudaoAiProperties) { } // ========== rag 相关 ========== - @Bean - @Lazy // TODO 芋艿：临时注释，避免无法启动 - public EmbeddingModel transformersEmbeddingClient() { - return new TransformersEmbeddingModel(MetadataMode.EMBED); - } + // TODO @xin 免费版本 +// @Bean +// @Lazy // TODO 芋艿：临时注释，避免无法启动」 +// public EmbeddingModel transformersEmbeddingClient() { +// return new TransformersEmbeddingModel(MetadataMode.EMBED); +// } /** - * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑 + * TODO @xin 默认版本先不弄，目前都先取对应的 EmbeddingModel */ +// @Bean +// @Lazy // TODO 芋艿：临时注释，避免无法启动 +// public RedisVectorStore vectorStore(TongYiTextEmbeddingModel tongYiTextEmbeddingModel, RedisVectorStoreProperties properties, +// RedisProperties redisProperties) { +// var config = RedisVectorStore.RedisVectorStoreConfig.builder() +// .withIndexName(properties.getIndex()) +// .withPrefix(properties.getPrefix()) +// .build(); +// +// RedisVectorStore redisVectorStore = new RedisVectorStore(config, tongYiTextEmbeddingModel, +// new JedisPooled(redisProperties.getHost(), redisProperties.getPort()), +// properties.isInitializeSchema()); +// redisVectorStore.afterPropertiesSet(); +// return redisVectorStore; +// } + @Bean @Lazy // TODO 芋艿：临时注释，避免无法启动 - public RedisVectorStore vectorStore(TransformersEmbeddingModel transformersEmbeddingModel, RedisVectorStoreProperties properties, - RedisProperties redisProperties) { - var config = RedisVectorStore.RedisVectorStoreConfig.builder() - .withIndexName(properties.getIndex()) - .withPrefix(properties.getPrefix()) - .build(); - - RedisVectorStore redisVectorStore = new RedisVectorStore(config, transformersEmbeddingModel, - new JedisPooled(redisProperties.getHost(), redisProperties.getPort()), - properties.isInitializeSchema()); - redisVectorStore.afterPropertiesSet(); - return redisVectorStore; + public TokenTextSplitter tokenTextSplitter() { + return new TokenTextSplitter(500, 100, 5, 10000, true); } @Bean @Lazy // TODO 芋艿：临时注释，避免无法启动 - public TokenTextSplitter tokenTextSplitter() { - return new TokenTextSplitter(500, 100, 5, 10000, true); + public TokenCountEstimator tokenCountEstimator() { + return new JTokkitTokenCountEstimator(); } } \ No newline at end of file diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java index b6d7b3dd08..7e84653759 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactory.java @@ -4,6 +4,7 @@ import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi; import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.image.ImageModel; /** @@ -79,4 +80,16 @@ public interface AiModelFactory { */ SunoApi getOrCreateSunoApi(String apiKey, String url); + /** + * 基于指定配置，获得 EmbeddingModel 对象 + * + * 如果不存在，则进行创建 + * + * @param platform 平台 + * @param apiKey API KEY + * @param url API URL + * @return ChatModel 对象 + */ + EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url); + } diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java index c9b04dc1ef..aa46c45f22 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java @@ -21,6 +21,7 @@ import com.alibaba.cloud.ai.tongyi.image.TongYiImagesProperties; import com.alibaba.dashscope.aigc.generation.Generation; import com.alibaba.dashscope.aigc.imagesynthesis.ImageSynthesis; +import com.alibaba.dashscope.embeddings.TextEmbedding; import com.azure.ai.openai.OpenAIClient; import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiAutoConfiguration; import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiChatProperties; @@ -37,6 +38,7 @@ import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiImageProperties; import org.springframework.ai.azure.openai.AzureOpenAiChatModel; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.image.ImageModel; import org.springframework.ai.model.function.FunctionCallbackContext; import org.springframework.ai.ollama.OllamaChatModel; @@ -175,6 +177,20 @@ public SunoApi getOrCreateSunoApi(String apiKey, String url) { return Singleton.get(cacheKey, (Func0) () -> new SunoApi(url)); } + @Override + public EmbeddingModel getOrCreateEmbeddingModel(AiPlatformEnum platform, String apiKey, String url) { + String cacheKey = buildClientCacheKey(EmbeddingModel.class, platform, apiKey, url); + return Singleton.get(cacheKey, (Func0) () -> { + // TODO @xin 先测试一个 + switch (platform) { + case TONG_YI: + return buildTongYiEmbeddingModel(apiKey); + default: + throw new IllegalArgumentException(StrUtil.format("未知平台({})", platform)); + } + }); + } + private static String buildClientCacheKey(Class clazz, Object... params) { if (ArrayUtil.isEmpty(params)) { return clazz.getName(); @@ -238,8 +254,7 @@ private static DeepSeekChatModel buildDeepSeekChatModel(String apiKey) { } /** - * 可参考 {@link ZhiPuAiAutoConfiguration#zhiPuAiChatModel( - * ZhiPuAiConnectionProperties, ZhiPuAiChatProperties, RestClient.Builder, List, FunctionCallbackContext, RetryTemplate, ResponseErrorHandler)} + * 可参考 {@link ZhiPuAiAutoConfiguration#zhiPuAiChatModel(ZhiPuAiConnectionProperties, ZhiPuAiChatProperties, RestClient.Builder, List, FunctionCallbackContext, RetryTemplate, ResponseErrorHandler)} */ private ZhiPuAiChatModel buildZhiPuChatModel(String apiKey, String url) { url = StrUtil.blankToDefault(url, ZhiPuAiConnectionProperties.DEFAULT_BASE_URL); @@ -248,8 +263,7 @@ private ZhiPuAiChatModel buildZhiPuChatModel(String apiKey, String url) { } /** - * 可参考 {@link ZhiPuAiAutoConfiguration#zhiPuAiImageModel( - * ZhiPuAiConnectionProperties, ZhiPuAiImageProperties, RestClient.Builder, RetryTemplate, ResponseErrorHandler)} + * 可参考 {@link ZhiPuAiAutoConfiguration#zhiPuAiImageModel(ZhiPuAiConnectionProperties, ZhiPuAiImageProperties, RestClient.Builder, RetryTemplate, ResponseErrorHandler)} */ private ZhiPuAiImageModel buildZhiPuAiImageModel(String apiKey, String url) { url = StrUtil.blankToDefault(url, ZhiPuAiConnectionProperties.DEFAULT_BASE_URL); @@ -315,4 +329,15 @@ private StabilityAiImageModel buildStabilityAiImageModel(String apiKey, String u return new StabilityAiImageModel(stabilityAiApi); } + // ========== 各种创建 EmbeddingModel 的方法 ========== + + /** + * 可参考 {@link TongYiAutoConfiguration#tongYiTextEmbeddingClient(TextEmbedding, TongYiConnectionProperties)} + */ + private EmbeddingModel buildTongYiEmbeddingModel(String apiKey) { + TongYiConnectionProperties connectionProperties = new TongYiConnectionProperties(); + connectionProperties.setApiKey(apiKey); + return new TongYiAutoConfiguration().tongYiTextEmbeddingClient(SpringUtil.getBean(TextEmbedding.class), connectionProperties); + } + } diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java new file mode 100644 index 0000000000..dad58a2c00 --- /dev/null +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactory.java @@ -0,0 +1,28 @@ +package cn.iocoder.yudao.framework.ai.core.factory; + +import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.VectorStore; + +// TODO @xin：也放到 AiModelFactory 里面好了，后续改成 AiFactory +/** + * AI Vector 模型工厂的接口类 + * + * @author xiaoxin + */ +public interface AiVectorStoreFactory { + + /** + * 基于指定配置，获得 VectorStore 对象 + *

+ * 如果不存在，则进行创建 + * + * @param embeddingModel 嵌入模型 + * @param platform 平台 + * @param apiKey API KEY + * @param url API URL + * @return VectorStore 对象 + */ + VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url); + +} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java new file mode 100644 index 0000000000..ec04c5e888 --- /dev/null +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiVectorStoreFactoryImpl.java @@ -0,0 +1,52 @@ +package cn.iocoder.yudao.framework.ai.core.factory; + +import cn.hutool.core.lang.Singleton; +import cn.hutool.core.lang.func.Func0; +import cn.hutool.core.util.ArrayUtil; +import cn.hutool.core.util.StrUtil; +import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum; +import cn.iocoder.yudao.framework.common.util.spring.SpringUtils; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.RedisVectorStore; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import redis.clients.jedis.JedisPooled; + +/** + * AI Vector 模型工厂的实现类 + * 使用 redisVectorStore 实现 VectorStore + * + * @author xiaoxin + */ +public class AiVectorStoreFactoryImpl implements AiVectorStoreFactory { + + @Override + public VectorStore getOrCreateVectorStore(EmbeddingModel embeddingModel, AiPlatformEnum platform, String apiKey, String url) { + String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey, url); + return Singleton.get(cacheKey, (Func0) () -> { + // TODO 芋艿 @xin 这两个配置取哪好呢 + // TODO 不同模型的向量维度可能会不一样，目前看貌似是以 index 来做区分的，维度不一样存不到一个 index 上 + // TODO 回复：好的哈 + String index = "default-index"; + String prefix = "default:"; + var config = RedisVectorStore.RedisVectorStoreConfig.builder() + .withIndexName(index) + .withPrefix(prefix) + .build(); + RedisProperties redisProperties = SpringUtils.getBean(RedisProperties.class); + RedisVectorStore redisVectorStore = new RedisVectorStore(config, embeddingModel, + new JedisPooled(redisProperties.getHost(), redisProperties.getPort()), + true); + redisVectorStore.afterPropertiesSet(); + return redisVectorStore; + }); + } + + private static String buildClientCacheKey(Class clazz, Object... params) { + if (ArrayUtil.isEmpty(params)) { + return clazz.getName(); + } + return StrUtil.format("{}#{}", clazz.getName(), ArrayUtil.join(params, "_")); + } + +} diff --git a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java index 61c38dd1d3..a72d50c4a8 100644 --- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java +++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java @@ -19,6 +19,7 @@ import org.springframework.ai.vectorstore.RedisVectorStore; import org.springframework.ai.vectorstore.RedisVectorStore.RedisVectorStoreConfig; import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; @@ -31,13 +32,14 @@ * TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突 * * TODO 这个官方，有说啥时候 fix 哇？ + * TODO 看着是列在1.0.0-M2版本 * * @author Christian Tzolov * @author Eddú Meléndez */ @AutoConfiguration(after = RedisAutoConfiguration.class) @ConditionalOnClass({JedisPooled.class, JedisConnectionFactory.class, RedisVectorStore.class, EmbeddingModel.class}) -//@ConditionalOnBean(JedisConnectionFactory.class) +@ConditionalOnBean(JedisConnectionFactory.class) @EnableConfigurationProperties(RedisVectorStoreProperties.class) public class RedisVectorStoreAutoConfiguration {