lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持配置多个工作区进行回答

Browse Source
This commit is contained in:
Luke.Ye 2025-07-11 17:49:54 +08:00
parent 4669f71d26
commit 3ae551a18e
8 changed files with 182 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Readme.md
View File

@ -14,6 +14,7 @@
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口（ob仓库监听发生删除事件，则同步删除ES） |
| 1.1.2 | 2025-07-07 | Luke.Ye | 增加llm后端接口 |
| 1.1.3 | 2025-07-12 | Luke.Ye | 支持配置多个工作区进行回答 |

147
src/main/java/com/knowledge/base/application/service/LLMAppServiceImpl.java
View File

@ -4,7 +4,6 @@ import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.DynamicConfig;
@ -13,6 +12,7 @@ import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
import com.knowledge.base.infrastructure.south.llm.AnythingLLMService;
import com.knowledge.base.infrastructure.south.llm.LLMServiceFactory;
import com.knowledge.base.infrastructure.south.llm.WorkspaceSelector;
import com.knowledge.base.infrastructure.util.RateLimiterManager;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import com.knowledge.base.infrastructure.util.http.FilteredSseOutputAdapter;
@ -34,21 +34,11 @@ import java.util.stream.Collectors;
public class LLMAppServiceImpl implements LLMAppService {
private final LLMServiceFactory llmServiceFactory;
private final RateLimiterManager rateLimiterManager;
private final AnythingLLMService anythingLLMService;
private final FileElasticsearchGateway esGateway;
private final DynamicConfig dynamicConfig;
/**
* pin和unpin的并发控制
*/
private static boolean PROCESSING = false;
private static String PROCESSING_QUESTION = StrUtil.EMPTY;
private final WorkspaceSelector workspaceSelector;
@Override
public String getToken(String password) throws Exception {
@ -57,15 +47,50 @@ public class LLMAppServiceImpl implements LLMAppService {
@Override
public SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception {
SseEmitter emitter = new SseEmitter(300 * 1000L); // 超时时间设为5分钟
SseEmitter emitter = new SseEmitter(300 * 1000L); // 5分钟超时
boolean autoPin = BooleanUtil.toBoolean(dynamicConfig.getEnableAutoPin());
List<WorkspaceAttachment> pinnedDocs = Collections.emptyList();
String slugId = StrUtil.EMPTY;
List<WorkspaceAttachment> finalPinnedDocs = getWorkspaceAttachments(llmToken, question);
// 只有自动pin时，才进行pin相关逻辑
if (autoPin) {
List<String> keywords = extractKeywords(question);
log.info("正在回答问题：question={}, keywords={}", question, JSONUtil.toJsonStr(keywords));
// 查询ES
List<FileEsModel> fileEsModels = esGateway.searchByKeywords(keywords);
List<String> esFilepaths = fileEsModels.stream()
.map(FileEsModel::getFilepath)
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());
if (CollectionUtil.isNotEmpty(esFilepaths)) {
// 只在查到ES文档时，才占用slug
slugId = workspaceSelector.acquire(question);
if (StrUtil.isNotEmpty(slugId)) {
// pin逻辑
pinnedDocs = pinDocsByKeywords(llmToken, slugId, esFilepaths, question);
} else {
log.info("无可用的LLM工作区(slug)，降级为无pin模式: question={}, 正在回答的问题： ｛｝", question, JSONUtil.toJsonStr(workspaceSelector.getAllInUseSlugQuestions()));
}
} else {
log.info("[ask] ES未查到可pin文档，跳过pin和占用slug, question={}", question);
}
}
String usedSlug = slugId;
final List<WorkspaceAttachment> finalPinnedDocs = pinnedDocs;
ThreadPoolUtil.execute(() -> {
try {
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_LLM_ASK).acquire();
WriterAdapter adapter = new FilteredSseOutputAdapter(emitter);
if(StrUtil.isBlank((String)params.get(ConstantConfig.LLM_SLUG_KEY))) {
params.put(ConstantConfig.LLM_SLUG_KEY, usedSlug);
}
llmServiceFactory.current().streamAnswer(llmToken, question, params, adapter);
emitter.complete();
} catch (Exception e) {
log.error("LLM调用异常", e);
@ -76,78 +101,35 @@ public class LLMAppServiceImpl implements LLMAppService {
log.warn("SSE发送错误信息失败", ioException);
}
} finally {
unpinLlmAttachments(llmToken, question, finalPinnedDocs);
// 只有有pin逻辑才unpin和释放slug
if (StrUtil.isNotBlank(usedSlug)) {
unpinLlmAttachments(llmToken, usedSlug, question, finalPinnedDocs);
workspaceSelector.release(usedSlug);
}
}
}, ThreadPoolConfig.SSE_POOL);
return emitter;
}
private void unpinLlmAttachments(String llmToken, String question, List<WorkspaceAttachment> finalPinnedDocs) {
if(BooleanUtil.toBoolean(dynamicConfig.getEnableAutoPin())) {
// 问题结束后，需要unpin掉
log.info("问题已回答完成： question: {}, pinnedDocs: {}", question, JSONUtil.toJsonStr(finalPinnedDocs));
finalPinnedDocs.forEach(doc -> {
try {
anythingLLMService.updatePin(llmToken, ConstantConfig.DEFAULT_SLUG_ID, doc.getDocpath(), false);
} catch (Exception e) {
log.warn("unpin失败: {}", doc, e);
}
});
PROCESSING = false;
}else {
log.info("[unpinLlmAttachments] 自动pin文档功能未开启. {}", dynamicConfig.getEnableAutoPin());
}
}
private List<WorkspaceAttachment> getWorkspaceAttachments(String llmToken, String question) {
List<WorkspaceAttachment> finalPinnedDocs;
if(BooleanUtil.toBoolean(dynamicConfig.getEnableAutoPin())) {
List<String> keywords = extractKeywords(question);
log.info("正在回答问题： question: {}, keywords: {}", question, JSONUtil.toJsonStr(keywords));
List<WorkspaceAttachment> pinnedDocs = pinDocsByKeywords(llmToken, ConstantConfig.DEFAULT_SLUG_ID, keywords, question);
finalPinnedDocs = pinnedDocs;
} else {
finalPinnedDocs = Collections.emptyList();
log.info("[getWorkspaceAttachments] 自动pin文档功能未开启. {}", dynamicConfig.getEnableAutoPin());
}
return finalPinnedDocs;
}
/**
* pin逻辑：带slugId、提前查好esFilepaths
*/
@Override
public List<WorkspaceAttachment> pinDocsByKeywords(String llmToken, String workspaceSlug, List<String> keywords, String question) {
if(CollectionUtil.isEmpty(keywords)) {
public List<WorkspaceAttachment> pinDocsByKeywords(String llmToken, String workspaceSlug, List<String> esFilepaths, String question) {
if (CollectionUtil.isEmpty(esFilepaths)) {
return Collections.emptyList();
}
if(PROCESSING) {
log.info("有问题正在处理，请稍等，question: ｛｝", PROCESSING_QUESTION);
return Collections.emptyList();
}
List<FileEsModel> fileEsModels = esGateway.searchByKeywords(keywords);
Set<String> esFilepaths = fileEsModels.stream()
.map(FileEsModel::getFilepath)
.filter(Objects::nonNull)
.collect(Collectors.toSet());
if (esFilepaths.isEmpty()) {
log.info("[pinDocsByKeywords] 未查到匹配ES文档，跳过pin操作。");
return Lists.newArrayList();
}
log.info("[pinDocsByKeywords] 已查询到关键词关联的文档。 {}", JSONUtil.toJsonStr(esFilepaths));
// 将当前问题标注为处理中
markQuestionProcessing(question);
// 2. 获取当前工作区所有附件（docPath -> url）
// 获取当前工作区所有附件
List<WorkspaceAttachment> attachments = anythingLLMService.fetchAttachments(llmToken, workspaceSlug);
if (attachments == null || attachments.isEmpty()) {
log.info("[pinDocsByKeywords] 当前工作区无已嵌入附件。");
return Collections.emptyList();
}
// 3. 找到需要pin的docPath集合
// 找到需要pin的docPath集合
List<WorkspaceAttachment> toPinDocs = attachments.stream().filter(att -> {
String url = att.getUrl();
if (url == null) return false;
@ -166,7 +148,7 @@ public class LLMAppServiceImpl implements LLMAppService {
});
}).collect(Collectors.toList());
// 4. 只unpin之前已pin的文档
// 只unpin之前已pin的文档
attachments.stream()
.filter(WorkspaceAttachment::isPinned)
.forEach(att -> {
@ -178,7 +160,7 @@ public class LLMAppServiceImpl implements LLMAppService {
}
});
// 5. pin目标文档
// pin目标文档
toPinDocs.forEach(doc -> {
try {
anythingLLMService.updatePin(llmToken, workspaceSlug, doc.getDocpath(), true);
@ -193,6 +175,22 @@ public class LLMAppServiceImpl implements LLMAppService {
return toPinDocs;
}
/**
* 回答结束后，unpin之前pin的文档
*/
private void unpinLlmAttachments(String llmToken, String slugId, String question, List<WorkspaceAttachment> finalPinnedDocs) {
if (CollectionUtil.isNotEmpty(finalPinnedDocs)) {
log.info("问题已回答完成: question={}, slug={}, pinnedDocs={}", question, slugId, JSONUtil.toJsonStr(finalPinnedDocs));
finalPinnedDocs.forEach(doc -> {
try {
anythingLLMService.updatePin(llmToken, slugId, doc.getDocpath(), false);
} catch (Exception e) {
log.warn("unpin失败: {}", doc, e);
}
});
}
}
/**
* 提取所有被LEFT & RIGHT 包裹的子串，返回数组
*/
@ -207,11 +205,4 @@ public class LLMAppServiceImpl implements LLMAppService {
}
return keywords;
}
public static void markQuestionProcessing(String question) {
PROCESSING = true;
PROCESSING_QUESTION = question;
}
}

3
src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java
View File

@ -33,7 +33,8 @@ public class ConstantConfig {
/**
* 以下是AnythingLLM相关
*/
public static final String DEFAULT_SLUG_ID = "87e14982-a821-48d8-9c6b-3557d0bb2f96";
public static final String LLM_SLUG_KEY = "slug";
public static final String LLM_WS_NAME_KEY = "wsName";
public static final String KEYWORD_PATTERN_LEFT = "#";
public static final String KEYWORD_PATTERN_RIGHT = "#";

30
src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java
View File

@ -1,10 +1,15 @@
package com.knowledge.base.infrastructure.config;
import cn.hutool.core.util.StrUtil;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
/**
* @author Luke.ye
* @date 2025/5/8 09:59
@ -27,21 +32,38 @@ public class DynamicConfig {
@Value("${token.expire.time:86400000}")
private long tokenExpireTime;
@Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx,ppt,pptx,txt}")
@Value("${os.supported.searchable.file.suffix:pdf,doc,docx,xls,xlsx,ppt,pptx,txt}")
private String supportedSearchFileSuffix;
@Value("${file.import.rate.limit: 10}")
@Value("${file.import.rate.limit:10}")
private String fileImportRateLimit;
@Value("${llm.sse.rate.limit: 3}")
@Value("${llm.sse.rate.limit:3}")
private String llmSseRateLimit;
@Value("${llm.enable.auto.pin: false}")
@Value("${llm.enable.auto.pin:false}")
private String enableAutoPin;
@Value("{llm.default.workspace.name:部门知识库}")
private String llmDefaultWsName;
@Value("{llm.default.slug.id:87e14982-a821-48d8-9c6b-3557d0bb2f96}")
private String llmDefaultSlugId;
@Value("${llm.active.slug.ids: 87e14982-a821-48d8-9c6b-3557d0bb2f96,bcd9ba38-36a6-4e1d-a0a4-933a96fce665}")
private String llmActiveSlugIds;
@Value("${markdown.path}")
private String mdDirectoryPath;
@Value("${exclude.file.path.prefix}")
private String mdExcludePrefix;
public Set<String> getLlmActiveSlugs() {
// 支持逗号、分号和换行分隔
return Arrays.stream(llmActiveSlugIds.split("[,;\\n]"))
.map(String::trim)
.filter(StrUtil::isNotBlank)
.collect(Collectors.toSet());
}
}

20
src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java
View File

@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
import com.knowledge.base.infrastructure.util.HttpHelper;
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
@ -38,6 +39,7 @@ public class AnythingLLMServiceImpl implements AnythingLLMService {
private final HttpHelper httpHelper;
private final UserCacheService userCacheService;
private final DynamicConfig dynamicConfig;
@Override
public boolean supports(String type) {
@ -53,9 +55,9 @@ public class AnythingLLMServiceImpl implements AnythingLLMService {
@Override
public void streamAnswer(String llmToken, String question, Map<String, Object> params, WriterAdapter writer) throws Exception {
String slug = (String) params.get("slug");
String slug = (String) params.get(ConstantConfig.LLM_SLUG_KEY);
if (StrUtil.isBlank(slug)) {
slug = fetchSlugByWsName(llmToken, (String) params.getOrDefault("wsName", "部门知识库"));
slug = fetchSlugByWsName(llmToken, (String) params.getOrDefault(ConstantConfig.LLM_WS_NAME_KEY, dynamicConfig.getLlmDefaultWsName()));
}
String url = baseUrl + "/api/workspace/" + slug + "/stream-chat";
@ -85,17 +87,19 @@ public class AnythingLLMServiceImpl implements AnythingLLMService {
Map<String, Object> resMap = httpHelper.get(url, llmToken);
List<HashMap> workspaces = (List<HashMap>)resMap.get("workspaces");
if(CollectionUtil.isEmpty(workspaces)) {
return ConstantConfig.DEFAULT_SLUG_ID;
return dynamicConfig.getLlmDefaultSlugId();
}
Optional<HashMap> targetWorkspace = workspaces.stream().filter(workspace -> workspace.get("name").equals(wsName)).findFirst();
if(targetWorkspace.isEmpty()) {
return ConstantConfig.DEFAULT_SLUG_ID;
return dynamicConfig.getLlmDefaultSlugId();
}
String slug = (String) targetWorkspace.get().get("slug");
// slug数据进缓存
userCacheService.cacheAnythingLLMSlugId(wsName, slug);
return StrUtil.isBlank(slug) ? ConstantConfig.DEFAULT_SLUG_ID : slug;
String slug = (String) targetWorkspace.get().get(ConstantConfig.LLM_SLUG_KEY);
if(StrUtil.isNotBlank(slug)) {
// slug数据进缓存
userCacheService.cacheAnythingLLMSlugId(wsName, slug);
}
return StrUtil.isBlank(slug) ? dynamicConfig.getLlmDefaultSlugId() : slug;
}

61
src/main/java/com/knowledge/base/infrastructure/south/llm/WorkspaceSelector.java Normal file
View File

@ -0,0 +1,61 @@
package com.knowledge.base.infrastructure.south.llm;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@Component
public class WorkspaceSelector {
private final Set<String> availableSlugs;
private final Map<String, Boolean> slugInUse = new ConcurrentHashMap<>();
// 仅为日志打印保存 slug->question，释放时清理即可
private final Map<String, String> slugQuestion = new ConcurrentHashMap<>();
@Autowired
public WorkspaceSelector(DynamicConfig dynamicConfig) {
this.availableSlugs = dynamicConfig.getLlmActiveSlugs();
availableSlugs.forEach(slug -> slugInUse.put(slug, false));
}
/**
* 获取一个空闲的slug，返回StrUtil.EMPTY则无可用
* 并记录question，方便日志打印
*/
public synchronized String acquire(String question) {
for (String slug : availableSlugs) {
if (!slugInUse.getOrDefault(slug, false)) {
slugInUse.put(slug, true);
slugQuestion.put(slug, question);
return slug;
}
}
return StrUtil.EMPTY;
}
/** 释放slug，同时清理question */
public synchronized void release(String slug) {
if (availableSlugs.contains(slug)) {
slugInUse.put(slug, false);
slugQuestion.remove(slug);
}
}
/** 获取当前slug对应的question，主要用于日志打印，无占用返回null */
public String getQuestion(String slug) {
return slugQuestion.get(slug);
}
/**
* 获取当前所有已占用的slug及对应的question
*/
public Map<String, String> getAllInUseSlugQuestions() {
// 返回不可变拷贝，避免外部修改内部map
return new ConcurrentHashMap<>(slugQuestion);
}
}

9
src/test/java/com/knowledge/base/application/service/LLMAppServiceImplTest.java
View File

@ -3,9 +3,10 @@ package com.knowledge.base.application.service;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
import com.knowledge.base.infrastructure.south.llm.AnythingLLMServiceImpl;
import com.knowledge.base.infrastructure.south.llm.AnythingLLMServiceTest;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -31,8 +32,8 @@ public class LLMAppServiceImplTest {
private Map<String, Object> llmRepo = Maps.newHashMap();
public static String getSlug() {
return ConstantConfig.DEFAULT_SLUG_ID;
public String getSlug() {
return AnythingLLMServiceTest.DEFAULT_TEST_SLUG_ID;
}
@BeforeEach
@ -52,7 +53,7 @@ public class LLMAppServiceImplTest {
result.forEach(doc -> {
try {
llmService.updatePin(token, ConstantConfig.DEFAULT_SLUG_ID, doc.getDocpath(), false);
llmService.updatePin(token, getSlug(), doc.getDocpath(), false);
} catch (Exception e) {
log.warn("unpin失败: {}", doc, e);
}

11
src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java
View File

@ -5,7 +5,6 @@ import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
import com.knowledge.base.infrastructure.util.BeanTool;
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
@ -37,6 +36,8 @@ public class AnythingLLMServiceTest {
private String token;
public static final String DEFAULT_TEST_SLUG_ID = "e6fa5a5a-1220-4eb3-979f-ee37b3711ba2";
private Map<String, Object> llmRepo = Maps.newHashMap();
@BeforeEach
@ -48,8 +49,8 @@ public class AnythingLLMServiceTest {
}
}
public static String getSlug() {
return ConstantConfig.DEFAULT_SLUG_ID;
public String getSlug() {
return DEFAULT_TEST_SLUG_ID;
}
@Test
@ -86,7 +87,7 @@ public class AnythingLLMServiceTest {
public void testGetLocalFileItems() {
try {
String slug = getSlug();
Map<String, Object> result = llmService.getLocalFileItems(token, slug, "test");
Map<String, Object> result = llmService.getLocalFileItems(token, slug, "custom-documents");
llmRepo.put("local-files", result);
log.info("本地文件项：{}", JSONUtil.toJsonStr(result));
} catch (Exception e) {
@ -148,7 +149,7 @@ public class AnythingLLMServiceTest {
WriterAdapter writer = line -> log.info("回答流：{}", line);
Map<String, Object> params = new HashMap<>();
params.put("wsName", "部门知识库");
params.put("slug", getSlug());
// llmService.streamAnswer(token, "介绍一下黄金圈法则", params, writer);
} catch (Exception e) {
log.warn("测试 streamAnswer 异常", e);