feat: ✨ 添加基于 ES 的向量数据库的实现

StephenQiu30 · StephenQiu30 · commit ae4f8467385f · 2026-03-24T18:36:09.000+08:00
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/DocumentProcessingProperties.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/DocumentProcessingProperties.java
@@ -0,0 +1,19 @@
+package com.stephen.cloud.ai.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Data
+@Component
+@ConfigurationProperties(prefix = "document.processing")
+public class DocumentProcessingProperties {
+
+    private int chunkSize = 800;
+
+    private int overlapSize = 100;
+
+    private long maxFileSize = 10485760L;
+
+    private String uploadPath = "uploads/knowledge";
+}
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/RagRetrievalProperties.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/RagRetrievalProperties.java
@@ -0,0 +1,15 @@
+package com.stephen.cloud.ai.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Data
+@Component
+@ConfigurationProperties(prefix = "rag.retrieval")
+public class RagRetrievalProperties {
+
+    private int topK = 5;
+
+    private Double similarityThreshold = 0.7D;
+}
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/VectorStoreConfig.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/config/VectorStoreConfig.java
@@ -3,12 +3,22 @@
 import org.springframework.ai.transformer.splitter.TokenTextSplitter;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import jakarta.annotation.Resource;
 
 @Configuration
 public class VectorStoreConfig {
 
+    @Resource
+    private DocumentProcessingProperties documentProcessingProperties;
+
     @Bean
     public TokenTextSplitter tokenTextSplitter() {
-        return new TokenTextSplitter(800, 100, 5, 10000, true);
+        return new TokenTextSplitter(
+                documentProcessingProperties.getChunkSize(),
+                documentProcessingProperties.getOverlapSize(),
+                5,
+                10000,
+                true
+        );
     }
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/controller/DocumentController.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/controller/DocumentController.java
@@ -40,12 +40,11 @@ public BaseResponse<Boolean> deleteDocument(@RequestBody DeleteRequest deleteReq
         if (deleteRequest == null || deleteRequest.getId() <= 0) {
             throw new BusinessException(ErrorCode.PARAMS_ERROR);
         }
-        Long id = deleteRequest.getId();
-        Document oldDocument = documentService.getById(id);
-        ThrowUtils.throwIf(oldDocument == null, ErrorCode.NOT_FOUND_ERROR);
-        Long userId = SecurityUtils.getLoginUserId();
-        ThrowUtils.throwIf(!oldDocument.getUserId().equals(userId) && !SecurityUtils.isAdmin(), ErrorCode.NO_AUTH_ERROR);
-        return ResultUtils.success(documentService.removeById(id));
+        return ResultUtils.success(documentService.deleteDocumentById(
+                deleteRequest.getId(),
+                SecurityUtils.getLoginUserId(),
+                SecurityUtils.isAdmin()
+        ));
     }
 
     @GetMapping("/get/vo")
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/controller/KnowledgeBaseController.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/controller/KnowledgeBaseController.java
@@ -1,13 +1,10 @@
 package com.stephen.cloud.ai.controller;
 
 import cn.dev33.satoken.annotation.SaCheckRole;
-import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.stephen.cloud.ai.convert.KnowledgeBaseConvert;
 import com.stephen.cloud.ai.model.entity.KnowledgeBase;
-import com.stephen.cloud.ai.service.DocumentService;
 import com.stephen.cloud.ai.service.KnowledgeBaseService;
-import com.stephen.cloud.api.ai.model.dto.document.DocumentQueryRequest;
 import com.stephen.cloud.api.ai.model.dto.knowledgebase.KnowledgeBaseAddRequest;
 import com.stephen.cloud.api.ai.model.dto.knowledgebase.KnowledgeBaseEditRequest;
 import com.stephen.cloud.api.ai.model.dto.knowledgebase.KnowledgeBaseQueryRequest;
@@ -32,9 +29,6 @@ public class KnowledgeBaseController {
     @Resource
     private KnowledgeBaseService knowledgeBaseService;
 
-    @Resource
-    private DocumentService documentService;
-
     @PostMapping("/add")
     @Operation(summary = "创建知识库")
     @OperationLog(module = "知识库管理", action = "创建知识库")
@@ -56,15 +50,11 @@ public BaseResponse<Boolean> deleteKnowledgeBase(@RequestBody DeleteRequest dele
         if (deleteRequest == null || deleteRequest.getId() <= 0) {
             throw new BusinessException(ErrorCode.PARAMS_ERROR);
         }
-        Long id = deleteRequest.getId();
-        KnowledgeBase oldKnowledgeBase = knowledgeBaseService.getById(id);
-        ThrowUtils.throwIf(oldKnowledgeBase == null, ErrorCode.NOT_FOUND_ERROR);
-        Long userId = SecurityUtils.getLoginUserId();
-        ThrowUtils.throwIf(!oldKnowledgeBase.getUserId().equals(userId) && !SecurityUtils.isAdmin(), ErrorCode.NO_AUTH_ERROR);
-        documentService.remove(new LambdaQueryWrapper<com.stephen.cloud.ai.model.entity.Document>()
-                .eq(com.stephen.cloud.ai.model.entity.Document::getKnowledgeBaseId, id));
-        boolean result = knowledgeBaseService.removeById(id);
-        return ResultUtils.success(result);
+        return ResultUtils.success(knowledgeBaseService.deleteKnowledgeBaseById(
+                deleteRequest.getId(),
+                SecurityUtils.getLoginUserId(),
+                SecurityUtils.isAdmin()
+        ));
     }
 
     @PostMapping("/update")
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/knowledge/etl/DocumentETLPipeline.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/knowledge/etl/DocumentETLPipeline.java
@@ -11,6 +11,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 @Component
 public class DocumentETLPipeline {
@@ -35,6 +36,12 @@ public int process(String filePath, String fileExtension, Map<String, Object> me
             document.getMetadata().putAll(metadata);
         }
         List<Document> chunks = tokenTextSplitter.apply(documents);
+        for (int i = 0; i < chunks.size(); i++) {
+            Document chunk = chunks.get(i);
+            chunk.getMetadata().put("chunkIndex", i);
+            String chunkId = metadata.get("documentId") + "_" + i + "_" + UUID.randomUUID();
+            chunk.getMetadata().put("chunkId", chunkId);
+        }
         vectorStore.add(chunks);
         return chunks.size();
     }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/DocumentService.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/DocumentService.java
@@ -22,4 +22,6 @@ public interface DocumentService extends IService<Document> {
     Page<DocumentVO> getDocumentVOPage(Page<Document> page, HttpServletRequest request);
 
     void sendDocumentProcessMessage(Long documentId);
+
+    boolean deleteDocumentById(Long id, Long loginUserId, boolean isAdmin);
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/KnowledgeBaseService.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/KnowledgeBaseService.java
@@ -19,4 +19,6 @@ public interface KnowledgeBaseService extends IService<KnowledgeBase> {
     Page<KnowledgeBaseVO> getKnowledgeBaseVOPage(Page<KnowledgeBase> page, HttpServletRequest request);
 
     boolean isNameUnique(String name, Long excludeId);
+
+    boolean deleteKnowledgeBaseById(Long id, Long loginUserId, boolean isAdmin);
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/VectorStoreService.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/VectorStoreService.java
@@ -0,0 +1,14 @@
+package com.stephen.cloud.ai.service;
+
+import org.springframework.ai.document.Document;
+
+import java.util.List;
+
+public interface VectorStoreService {
+
+    List<Document> similaritySearch(String query, Long knowledgeBaseId, Integer topK, Double similarityThreshold);
+
+    void deleteByKnowledgeBaseId(Long knowledgeBaseId);
+
+    void deleteByDocumentId(Long documentId);
+}
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/DocumentServiceImpl.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/DocumentServiceImpl.java
@@ -4,12 +4,14 @@
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.stephen.cloud.ai.config.DocumentProcessingProperties;
 import com.stephen.cloud.ai.convert.DocumentConvert;
 import com.stephen.cloud.ai.mapper.DocumentMapper;
 import com.stephen.cloud.ai.model.entity.Document;
 import com.stephen.cloud.ai.mq.DocumentProcessProducer;
 import com.stephen.cloud.ai.mq.model.DocumentProcessMessage;
 import com.stephen.cloud.ai.service.DocumentService;
+import com.stephen.cloud.ai.service.VectorStoreService;
 import com.stephen.cloud.api.ai.model.dto.document.DocumentQueryRequest;
 import com.stephen.cloud.api.ai.model.vo.DocumentVO;
 import com.stephen.cloud.api.user.client.UserFeignClient;
@@ -22,8 +24,8 @@
 import jakarta.servlet.http.HttpServletRequest;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.File;
@@ -38,29 +40,29 @@ public class DocumentServiceImpl extends ServiceImpl<DocumentMapper, Document> i
 
     private static final Set<String> SUPPORTED_EXTENSIONS = Set.of("pdf", "doc", "docx", "md", "txt", "ppt", "pptx", "html");
 
-    @Value("${document.processing.max-file-size:10485760}")
-    private long maxFileSize;
-
-    @Value("${document.processing.upload-path:uploads/knowledge}")
-    private String uploadPath;
+    @Resource
+    private DocumentProcessingProperties documentProcessingProperties;
 
     @Resource
     private DocumentProcessProducer documentProcessProducer;
 
     @Resource
     private UserFeignClient userFeignClient;
 
+    @Resource
+    private VectorStoreService vectorStoreService;
+
     @Override
     public Long uploadDocument(MultipartFile file, Long knowledgeBaseId, Long userId) {
         ThrowUtils.throwIf(file == null || file.isEmpty(), ErrorCode.PARAMS_ERROR, "文件不能为空");
         ThrowUtils.throwIf(knowledgeBaseId == null || knowledgeBaseId <= 0, ErrorCode.PARAMS_ERROR);
-        ThrowUtils.throwIf(file.getSize() > maxFileSize, ErrorCode.PARAMS_ERROR, "文件大小超过限制");
+        ThrowUtils.throwIf(file.getSize() > documentProcessingProperties.getMaxFileSize(), ErrorCode.PARAMS_ERROR, "文件大小超过限制");
         String originalFilename = file.getOriginalFilename();
         String extension = StringUtils.lowerCase(FilenameUtils.getExtension(originalFilename));
         ThrowUtils.throwIf(StringUtils.isBlank(extension) || !SUPPORTED_EXTENSIONS.contains(extension),
                 ErrorCode.PARAMS_ERROR, "不支持的文件格式");
         try {
-            Path saveDir = Path.of(uploadPath, String.valueOf(knowledgeBaseId));
+            Path saveDir = Path.of(documentProcessingProperties.getUploadPath(), String.valueOf(knowledgeBaseId));
             Files.createDirectories(saveDir);
             String saveName = System.currentTimeMillis() + "_" + UUID.randomUUID() + "." + extension;
             Path savePath = saveDir.resolve(saveName);
@@ -171,4 +173,15 @@ public void sendDocumentProcessMessage(Long documentId) {
         message.setFileExtension(document.getFileExtension());
         documentProcessProducer.sendMessage(message);
     }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public boolean deleteDocumentById(Long id, Long loginUserId, boolean isAdmin) {
+        ThrowUtils.throwIf(id == null || id <= 0, ErrorCode.PARAMS_ERROR);
+        Document oldDocument = this.getById(id);
+        ThrowUtils.throwIf(oldDocument == null, ErrorCode.NOT_FOUND_ERROR);
+        ThrowUtils.throwIf(!Objects.equals(oldDocument.getUserId(), loginUserId) && !isAdmin, ErrorCode.NO_AUTH_ERROR);
+        vectorStoreService.deleteByDocumentId(id);
+        return this.removeById(id);
+    }
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/KnowledgeBaseServiceImpl.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/KnowledgeBaseServiceImpl.java
@@ -6,8 +6,11 @@
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.stephen.cloud.ai.convert.KnowledgeBaseConvert;
 import com.stephen.cloud.ai.mapper.KnowledgeBaseMapper;
+import com.stephen.cloud.ai.model.entity.Document;
 import com.stephen.cloud.ai.model.entity.KnowledgeBase;
+import com.stephen.cloud.ai.service.DocumentService;
 import com.stephen.cloud.ai.service.KnowledgeBaseService;
+import com.stephen.cloud.ai.service.VectorStoreService;
 import com.stephen.cloud.api.ai.model.dto.knowledgebase.KnowledgeBaseQueryRequest;
 import com.stephen.cloud.api.ai.model.vo.KnowledgeBaseVO;
 import com.stephen.cloud.api.user.client.UserFeignClient;
@@ -20,6 +23,7 @@
 import jakarta.servlet.http.HttpServletRequest;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.*;
 import java.util.stream.Collectors;
@@ -30,6 +34,12 @@ public class KnowledgeBaseServiceImpl extends ServiceImpl<KnowledgeBaseMapper, K
     @Resource
     private UserFeignClient userFeignClient;
 
+    @Resource
+    private DocumentService documentService;
+
+    @Resource
+    private VectorStoreService vectorStoreService;
+
     @Override
     public void validKnowledgeBase(KnowledgeBase knowledgeBase, boolean add) {
         if (knowledgeBase == null) {
@@ -120,4 +130,16 @@ public boolean isNameUnique(String name, Long excludeId) {
         }
         return this.count(queryWrapper) == 0;
     }
+
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public boolean deleteKnowledgeBaseById(Long id, Long loginUserId, boolean isAdmin) {
+        ThrowUtils.throwIf(id == null || id <= 0, ErrorCode.PARAMS_ERROR);
+        KnowledgeBase oldKnowledgeBase = this.getById(id);
+        ThrowUtils.throwIf(oldKnowledgeBase == null, ErrorCode.NOT_FOUND_ERROR);
+        ThrowUtils.throwIf(!Objects.equals(oldKnowledgeBase.getUserId(), loginUserId) && !isAdmin, ErrorCode.NO_AUTH_ERROR);
+        vectorStoreService.deleteByKnowledgeBaseId(id);
+        documentService.remove(new LambdaQueryWrapper<Document>().eq(Document::getKnowledgeBaseId, id));
+        return this.removeById(id);
+    }
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/RAGServiceImpl.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/RAGServiceImpl.java
@@ -8,14 +8,14 @@
 import com.stephen.cloud.ai.mapper.RAGHistoryMapper;
 import com.stephen.cloud.ai.model.entity.RAGHistory;
 import com.stephen.cloud.ai.service.RAGService;
+import com.stephen.cloud.ai.service.VectorStoreService;
 import com.stephen.cloud.api.ai.model.vo.RAGAnswerVO;
 import com.stephen.cloud.api.ai.model.vo.RAGHistoryVO;
 import com.stephen.cloud.api.ai.model.vo.SourceVO;
 import jakarta.annotation.Resource;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.ai.chat.client.ChatClient;
 import org.springframework.ai.document.Document;
-import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.stereotype.Service;
 import reactor.core.publisher.Flux;
 
@@ -25,19 +25,19 @@
 @Service
 public class RAGServiceImpl implements RAGService {
 
-    @Resource
-    private VectorStore vectorStore;
-
     @Resource
     private ChatClient chatClient;
 
     @Resource
     private RAGHistoryMapper ragHistoryMapper;
 
+    @Resource
+    private VectorStoreService vectorStoreService;
+
     @Override
     public RAGAnswerVO ask(String question, Long knowledgeBaseId, Long userId, Integer topK) {
         long start = System.currentTimeMillis();
-        List<Document> docs = filterByKnowledgeBase(vectorStore.similaritySearch(question), knowledgeBaseId, topK);
+        List<Document> docs = vectorStoreService.similaritySearch(question, knowledgeBaseId, topK, null);
         String context = buildContext(docs);
         String prompt = buildPrompt(question, context);
         String answer = chatClient.prompt().user(prompt).call().content();
@@ -53,7 +53,7 @@ public RAGAnswerVO ask(String question, Long knowledgeBaseId, Long userId, Integ
 
     @Override
     public Flux<String> askStream(String question, Long knowledgeBaseId, Long userId, Integer topK) {
-        List<Document> docs = filterByKnowledgeBase(vectorStore.similaritySearch(question), knowledgeBaseId, topK);
+        List<Document> docs = vectorStoreService.similaritySearch(question, knowledgeBaseId, topK, null);
         String context = buildContext(docs);
         String prompt = buildPrompt(question, context);
         return chatClient.prompt().user(prompt).stream().content();
@@ -139,17 +139,4 @@ private List<SourceVO> buildSources(List<Document> docs) {
         return sourceList;
     }
 
-    private List<Document> filterByKnowledgeBase(List<Document> docs, Long knowledgeBaseId, Integer topK) {
-        if (CollUtil.isEmpty(docs)) {
-            return List.of();
-        }
-        int limit = topK == null || topK <= 0 ? 5 : topK;
-        return docs.stream().filter(doc -> {
-            if (knowledgeBaseId == null || knowledgeBaseId <= 0) {
-                return true;
-            }
-            Object kbIdObj = doc.getMetadata().get("knowledgeBaseId");
-            return kbIdObj != null && String.valueOf(knowledgeBaseId).equals(String.valueOf(kbIdObj));
-        }).limit(limit).toList();
-    }
 }
diff --git a/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/VectorStoreServiceImpl.java b/algorithm-service/algorithm-ai-service/src/main/java/com/stephen/cloud/ai/service/impl/VectorStoreServiceImpl.java
diff --git a/algorithm-service/algorithm-ai-service/src/main/resources/application.yml b/algorithm-service/algorithm-ai-service/src/main/resources/application.yml

Original file line number	Diff line number	Diff line change
`@@ -22,4 +22,6 @@ public interface DocumentService extends IService<Document> {`
`22`	`22`	`Page<DocumentVO> getDocumentVOPage(Page<Document> page, HttpServletRequest request);`
`23`	`23`
`24`	`24`	`void sendDocumentProcessMessage(Long documentId);`
	`25`	`+`
	`26`	`+ boolean deleteDocumentById(Long id, Long loginUserId, boolean isAdmin);`
`25`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,4 +19,6 @@ public interface KnowledgeBaseService extends IService<KnowledgeBase> {`
`19`	`19`	`Page<KnowledgeBaseVO> getKnowledgeBaseVOPage(Page<KnowledgeBase> page, HttpServletRequest request);`
`20`	`20`
`21`	`21`	`boolean isNameUnique(String name, Long excludeId);`
	`22`	`+`
	`23`	`+ boolean deleteKnowledgeBaseById(Long id, Long loginUserId, boolean isAdmin);`
`22`	`24`	`}`