Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 925f222b87 | |||
| 958f5253df | |||
| 147654f2e5 | |||
| 499530575d | |||
| 9e33822bdd | |||
| 3bc1e4ca43 | |||
| 705d403186 | |||
| 8e5821a2d6 | |||
| f85bf1ce3b | |||
| b0c5fa0f75 | |||
| 49e175c0a1 | |||
| 2e7ab01ed6 | |||
| fec5ce0ca2 | |||
| 3ae551a18e | |||
| 4669f71d26 | |||
| a489d26f8a | |||
| b3f988dc17 | |||
| 3cf63bb70c | |||
| 333c30862f | |||
| 9a7e855d65 | |||
| 133dd724ba | |||
| e3d46789e0 | |||
| 566ed1d880 | |||
| 69a3a1bb49 | |||
| d4e0d06ded | |||
| f9507b5d92 | |||
| 068bf0419c | |||
| e4d8214eac | |||
| e6d2567de8 | |||
| d3fb2d829a | |||
| 54a9241dbb | |||
| da85233841 | |||
| 53467e8545 |
1
.gitignore
vendored
1
.gitignore
vendored
@ -37,3 +37,4 @@ build/
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
/knowledge-base-server-log/
|
/knowledge-base-server-log/
|
||||||
knowledge-base-server.iml
|
knowledge-base-server.iml
|
||||||
|
project_code**
|
||||||
@ -13,9 +13,16 @@
|
|||||||
| 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |
|
| 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |
|
||||||
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
|
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
|
||||||
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口(ob仓库监听发生删除事件,则同步删除ES) |
|
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口(ob仓库监听发生删除事件,则同步删除ES) |
|
||||||
|
| 1.1.2 | 2025-07-07 | Luke.Ye | 增加llm后端接口 |
|
||||||
|
| 1.1.3 | 2025-07-12 | Luke.Ye | 支持配置多个工作区进行回答 |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 开发顺序
|
## 开发顺序
|
||||||
|
PO ---> Mapper ---> iface ---> persistence
|
||||||
|
|
||||||
PO ---> Mapper ---> iface ---> persistence
|
|
||||||
|
## llm接口调用顺序
|
||||||
|
- importObsidianToLLM(导入本地文档) ---> updateEmbeddings(将本地文档嵌入工作区) ---> updatePin(是否pin住) ---> streamAnswer(开始问答)
|
||||||
|
- removeFolderFromLLM(移除文档)
|
||||||
|
- fetchAttachments(获取当前工作区下已经嵌入的附件)
|
||||||
77
boudle_code.py
Normal file
77
boudle_code.py
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
import os
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
# ================= 配置区域 =================
|
||||||
|
# 输出文件名包含版本号(时间戳)
|
||||||
|
TIMESTAMP = datetime.datetime.now().strftime("%Y%m%d_%H%M")
|
||||||
|
OUTPUT_FILE = f"project_code_v{TIMESTAMP}.txt"
|
||||||
|
|
||||||
|
# 需要包含的文件后缀 (根据你的截图,主要是 Java, XML, Yaml, Dockerfile, Shell)
|
||||||
|
INCLUDE_EXTENSIONS = {'.java', '.xml', '.yml', '.yaml', '.properties', '.sql', '.sh', 'Dockerfile', '.md'}
|
||||||
|
|
||||||
|
# 需要完全匹配的文件名 (无后缀的文件,如 Dockerfile)
|
||||||
|
INCLUDE_FILENAMES = {'Dockerfile', 'Makefile', 'Jenkinsfile'}
|
||||||
|
|
||||||
|
# 需要忽略的目录
|
||||||
|
IGNORE_DIRS = {
|
||||||
|
'target', '.idea', '.git', '.mvn', 'wrapper', 'build',
|
||||||
|
'arthas-output', 'knowledge-base-server-log', 'test' # 如果不需要测试代码,可以加上 'test'
|
||||||
|
}
|
||||||
|
# ===========================================
|
||||||
|
|
||||||
|
def generate_tree(startpath):
|
||||||
|
"""生成项目目录树结构字符串"""
|
||||||
|
tree_str = "【项目目录结构】:\n"
|
||||||
|
for root, dirs, files in os.walk(startpath):
|
||||||
|
dirs[:] = [d for d in dirs if d not in IGNORE_DIRS]
|
||||||
|
level = root.replace(startpath, '').count(os.sep)
|
||||||
|
indent = ' ' * 4 * (level)
|
||||||
|
tree_str += '{}{}/\n'.format(indent, os.path.basename(root))
|
||||||
|
subindent = ' ' * 4 * (level + 1)
|
||||||
|
for f in files:
|
||||||
|
# 简单过滤显示在树中的文件
|
||||||
|
if any(f.endswith(ext) for ext in INCLUDE_EXTENSIONS) or f in INCLUDE_FILENAMES:
|
||||||
|
tree_str += '{}{}\n'.format(subindent, f)
|
||||||
|
return tree_str + "\n" + "="*50 + "\n\n"
|
||||||
|
|
||||||
|
def is_target_file(filename):
|
||||||
|
"""判断是否为目标文件"""
|
||||||
|
if filename in INCLUDE_FILENAMES:
|
||||||
|
return True
|
||||||
|
_, ext = os.path.splitext(filename)
|
||||||
|
return ext in INCLUDE_EXTENSIONS
|
||||||
|
|
||||||
|
def merge_files():
|
||||||
|
root_dir = os.getcwd() # 获取当前脚本所在目录
|
||||||
|
|
||||||
|
with open(OUTPUT_FILE, 'w', encoding='utf-8') as outfile:
|
||||||
|
# 1. 写入目录树
|
||||||
|
print("正在生成目录结构...")
|
||||||
|
outfile.write(generate_tree(root_dir))
|
||||||
|
|
||||||
|
# 2. 遍历并写入文件内容
|
||||||
|
print("正在合并代码文件...")
|
||||||
|
for root, dirs, files in os.walk(root_dir):
|
||||||
|
# 移除忽略的目录
|
||||||
|
dirs[:] = [d for d in dirs if d not in IGNORE_DIRS]
|
||||||
|
|
||||||
|
for file in files:
|
||||||
|
if is_target_file(file):
|
||||||
|
file_path = os.path.join(root, file)
|
||||||
|
rel_path = os.path.relpath(file_path, root_dir)
|
||||||
|
|
||||||
|
# 写入文件头分割线
|
||||||
|
header = f"\n\n{'='*20}\nFile Path: {rel_path}\n{'='*20}\n"
|
||||||
|
outfile.write(header)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(file_path, 'r', encoding='utf-8') as infile:
|
||||||
|
outfile.write(infile.read())
|
||||||
|
except Exception as e:
|
||||||
|
outfile.write(f"\n[Error reading file: {e}]\n")
|
||||||
|
print(f"Error reading {rel_path}: {e}")
|
||||||
|
|
||||||
|
print(f"✅ 完成!文件已保存为: {OUTPUT_FILE}")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
merge_files()
|
||||||
@ -1,4 +1,7 @@
|
|||||||
redis-cli --raw
|
redis-cli --raw
|
||||||
|
|
||||||
# 获取redis中的key-value
|
# 获取redis中的key-value
|
||||||
redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done
|
redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done
|
||||||
|
|
||||||
|
# 清空redis数据库
|
||||||
|
flushdb
|
||||||
22
scripts/mv_kb_jar.sh
Normal file
22
scripts/mv_kb_jar.sh
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# 源文件路径
|
||||||
|
SRC="/mnt/d/00-projects/00-micro-sass/knowledge-base-server/target/knowledge-base-server-0.0.1-SNAPSHOT.jar"
|
||||||
|
# 目标文件夹
|
||||||
|
DST_DIR="/mnt/d/00-projects/00-micro-sass/docker-knowledge-base-server"
|
||||||
|
# 目标完整路径
|
||||||
|
DST="$DST_DIR/knowledge-base-server-0.0.1-SNAPSHOT.jar"
|
||||||
|
|
||||||
|
# 创建目标目录(如果不存在)
|
||||||
|
mkdir -p "$DST_DIR"
|
||||||
|
|
||||||
|
# 移动文件(如目标已存在则直接覆盖)
|
||||||
|
mv -f "$SRC" "$DST"
|
||||||
|
|
||||||
|
# 输出操作结果
|
||||||
|
if [[ $? -eq 0 ]]; then
|
||||||
|
echo "文件已成功移动到 $DST"
|
||||||
|
else
|
||||||
|
echo "文件移动失败!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@ -1,7 +1,9 @@
|
|||||||
package com.knowledge.base.application.service;
|
package com.knowledge.base.application.service;
|
||||||
|
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
public interface LLMAppService {
|
public interface LLMAppService {
|
||||||
@ -22,6 +24,17 @@ public interface LLMAppService {
|
|||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception;
|
SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按关键词pin文档(其它文档取消pin)
|
||||||
|
*
|
||||||
|
* @param llmToken LLM鉴权token
|
||||||
|
* @param workspaceSlug 当前工作区slug
|
||||||
|
* @param keywords 搜索关键词
|
||||||
|
* @param question 当前的问题
|
||||||
|
* @return 本次被pin的docPath集合
|
||||||
|
*/
|
||||||
|
List<WorkspaceAttachment> pinDocsByKeywords(String llmToken, String workspaceSlug, List<String> keywords, String question);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,18 @@
|
|||||||
package com.knowledge.base.application.service;
|
package com.knowledge.base.application.service;
|
||||||
|
|
||||||
|
import cn.hutool.core.collection.CollectionUtil;
|
||||||
|
import cn.hutool.core.util.BooleanUtil;
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import com.knowledge.base.domain.doc.model.FileEsModel;
|
||||||
|
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
|
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
|
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
|
||||||
|
import com.knowledge.base.infrastructure.south.llm.AnythingLLMService;
|
||||||
import com.knowledge.base.infrastructure.south.llm.LLMServiceFactory;
|
import com.knowledge.base.infrastructure.south.llm.LLMServiceFactory;
|
||||||
|
import com.knowledge.base.infrastructure.south.llm.WorkspaceSelector;
|
||||||
import com.knowledge.base.infrastructure.util.RateLimiterManager;
|
import com.knowledge.base.infrastructure.util.RateLimiterManager;
|
||||||
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
|
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
|
||||||
import com.knowledge.base.infrastructure.util.http.FilteredSseOutputAdapter;
|
import com.knowledge.base.infrastructure.util.http.FilteredSseOutputAdapter;
|
||||||
@ -12,7 +23,10 @@ import org.springframework.stereotype.Service;
|
|||||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Map;
|
import java.util.*;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@ -20,8 +34,11 @@ import java.util.Map;
|
|||||||
public class LLMAppServiceImpl implements LLMAppService {
|
public class LLMAppServiceImpl implements LLMAppService {
|
||||||
|
|
||||||
private final LLMServiceFactory llmServiceFactory;
|
private final LLMServiceFactory llmServiceFactory;
|
||||||
|
|
||||||
private final RateLimiterManager rateLimiterManager;
|
private final RateLimiterManager rateLimiterManager;
|
||||||
|
private final AnythingLLMService anythingLLMService;
|
||||||
|
private final FileElasticsearchGateway esGateway;
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
private final WorkspaceSelector workspaceSelector;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getToken(String password) throws Exception {
|
public String getToken(String password) throws Exception {
|
||||||
@ -30,13 +47,51 @@ public class LLMAppServiceImpl implements LLMAppService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception {
|
public SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception {
|
||||||
SseEmitter emitter = new SseEmitter(300 * 1000L); // 超时时间设为5分钟
|
SseEmitter emitter = new SseEmitter(300 * 1000L); // 5分钟超时
|
||||||
|
boolean autoPin = BooleanUtil.toBoolean(dynamicConfig.getEnableAutoPin());
|
||||||
|
List<WorkspaceAttachment> pinnedDocs = Collections.emptyList();
|
||||||
|
String slugId = StrUtil.EMPTY;
|
||||||
|
|
||||||
|
// 只有自动pin时,才进行pin相关逻辑
|
||||||
|
if (autoPin) {
|
||||||
|
List<String> keywords = extractKeywords(question);
|
||||||
|
log.info("正在回答问题:question={}, keywords={}", question, JSONUtil.toJsonStr(keywords));
|
||||||
|
|
||||||
|
// 查询ES
|
||||||
|
List<FileEsModel> fileEsModels = esGateway.searchByKeywords(keywords);
|
||||||
|
List<String> esFilepaths = fileEsModels.stream()
|
||||||
|
.map(FileEsModel::getFilepath)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.distinct()
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
if (CollectionUtil.isNotEmpty(esFilepaths)) {
|
||||||
|
// 只在查到ES文档时,才占用slug
|
||||||
|
slugId = workspaceSelector.acquire(question);
|
||||||
|
if (StrUtil.isNotEmpty(slugId)) {
|
||||||
|
// pin逻辑
|
||||||
|
pinnedDocs = pinDocsByKeywords(llmToken, slugId, esFilepaths, question);
|
||||||
|
log.info("question={} , 已占用 slugId={}, pinnedDocs={}", question, slugId, JSONUtil.toJsonStr(pinnedDocs));
|
||||||
|
} else {
|
||||||
|
log.info("无可用的LLM工作区(slug),降级为无pin模式: question={}, 正在回答的问题: {}", question, JSONUtil.toJsonStr(workspaceSelector.getAllInUseSlugQuestions()));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[ask] ES未查到可pin文档,跳过pin和占用slug, question={}", question);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String usedSlug = slugId;
|
||||||
|
final List<WorkspaceAttachment> finalPinnedDocs = pinnedDocs;
|
||||||
ThreadPoolUtil.execute(() -> {
|
ThreadPoolUtil.execute(() -> {
|
||||||
try {
|
try {
|
||||||
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_LLM_ASK).acquire();
|
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_LLM_ASK).acquire();
|
||||||
WriterAdapter adapter = new FilteredSseOutputAdapter(emitter);
|
WriterAdapter adapter = new FilteredSseOutputAdapter(emitter);
|
||||||
llmServiceFactory.current().streamAnswer(llmToken, question, params, adapter);
|
|
||||||
|
if(StrUtil.isBlank((String)params.get(ConstantConfig.LLM_SLUG_KEY))) {
|
||||||
|
params.put(ConstantConfig.LLM_SLUG_KEY, usedSlug);
|
||||||
|
}
|
||||||
|
llmServiceFactory.current().streamAnswer(llmToken, question.replaceAll(ConstantConfig.KEYWORD_PATTERN_LEFT, StrUtil.EMPTY), params, adapter);
|
||||||
|
|
||||||
emitter.complete();
|
emitter.complete();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("LLM调用异常", e);
|
log.error("LLM调用异常", e);
|
||||||
@ -46,9 +101,109 @@ public class LLMAppServiceImpl implements LLMAppService {
|
|||||||
} catch (IOException ioException) {
|
} catch (IOException ioException) {
|
||||||
log.warn("SSE发送错误信息失败", ioException);
|
log.warn("SSE发送错误信息失败", ioException);
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
// 只有有pin逻辑才unpin和释放slug
|
||||||
|
if (StrUtil.isNotBlank(usedSlug)) {
|
||||||
|
unpinLlmAttachments(llmToken, usedSlug, question, finalPinnedDocs);
|
||||||
|
workspaceSelector.release(usedSlug);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}, ThreadPoolConfig.SSE_POOL);
|
}, ThreadPoolConfig.SSE_POOL);
|
||||||
|
|
||||||
return emitter;
|
return emitter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* pin逻辑:带slugId、提前查好esFilepaths
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public List<WorkspaceAttachment> pinDocsByKeywords(String llmToken, String workspaceSlug, List<String> esFilepaths, String question) {
|
||||||
|
if (CollectionUtil.isEmpty(esFilepaths)) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取当前工作区所有附件
|
||||||
|
List<WorkspaceAttachment> attachments = anythingLLMService.fetchAttachments(llmToken, workspaceSlug);
|
||||||
|
if (attachments == null || attachments.isEmpty()) {
|
||||||
|
log.info("[pinDocsByKeywords] 当前工作区无已嵌入附件。");
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 找到需要pin的docPath集合
|
||||||
|
List<WorkspaceAttachment> toPinDocs = attachments.stream().filter(att -> {
|
||||||
|
String url = att.getUrl();
|
||||||
|
if (url == null) return false;
|
||||||
|
// .md 文件
|
||||||
|
if (url.endsWith(".md")) {
|
||||||
|
return esFilepaths.stream().anyMatch(f -> url.endsWith(f));
|
||||||
|
}
|
||||||
|
// 其它文件(如 .pptx、.pdf、.docx等)
|
||||||
|
int lastSlash = url.lastIndexOf('/');
|
||||||
|
String fileName = lastSlash != -1 ? url.substring(lastSlash + 1) : url;
|
||||||
|
// esFilepaths 里也提取末尾文件名
|
||||||
|
return esFilepaths.stream().anyMatch(f -> {
|
||||||
|
int esLastSlash = f.lastIndexOf('/');
|
||||||
|
String esName = esLastSlash != -1 ? f.substring(esLastSlash + 1) : f;
|
||||||
|
return fileName.equals(esName);
|
||||||
|
});
|
||||||
|
}).collect(Collectors.toList());
|
||||||
|
|
||||||
|
// 只unpin之前已pin的文档
|
||||||
|
attachments.stream()
|
||||||
|
.filter(WorkspaceAttachment::isPinned)
|
||||||
|
.forEach(att -> {
|
||||||
|
try {
|
||||||
|
log.info("unpin doc {}", att.getUrl());
|
||||||
|
anythingLLMService.updatePin(llmToken, workspaceSlug, att.getDocpath(), false);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("取消pin失败: {}", att.getDocpath(), e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// pin目标文档
|
||||||
|
toPinDocs.forEach(doc -> {
|
||||||
|
try {
|
||||||
|
anythingLLMService.updatePin(llmToken, workspaceSlug, doc.getDocpath(), true);
|
||||||
|
doc.setPinned(true);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("pin失败: {}", doc, e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
log.info("[pinDocsByKeywords] 共pin住文档{}条: {}", toPinDocs.size(), JSONUtil.toJsonStr(toPinDocs));
|
||||||
|
|
||||||
|
return toPinDocs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 回答结束后,unpin之前pin的文档
|
||||||
|
*/
|
||||||
|
private void unpinLlmAttachments(String llmToken, String slugId, String question, List<WorkspaceAttachment> finalPinnedDocs) {
|
||||||
|
if (CollectionUtil.isNotEmpty(finalPinnedDocs)) {
|
||||||
|
log.info("问题已回答完成: question={}, slug={}, pinnedDocs={}", question, slugId, JSONUtil.toJsonStr(finalPinnedDocs));
|
||||||
|
finalPinnedDocs.forEach(doc -> {
|
||||||
|
try {
|
||||||
|
anythingLLMService.updatePin(llmToken, slugId, doc.getDocpath(), false);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("unpin失败: {}", doc, e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提取所有被LEFT & RIGHT 包裹的子串,返回数组
|
||||||
|
*/
|
||||||
|
public static List<String> extractKeywords(String text) {
|
||||||
|
List<String> keywords = new ArrayList<>();
|
||||||
|
String left = ConstantConfig.KEYWORD_PATTERN_LEFT;
|
||||||
|
String right = ConstantConfig.KEYWORD_PATTERN_RIGHT;
|
||||||
|
Pattern pattern = Pattern.compile(Pattern.quote(left) + "(.*?)" + Pattern.quote(right));
|
||||||
|
Matcher matcher = pattern.matcher(text);
|
||||||
|
while (matcher.find()) {
|
||||||
|
keywords.add(matcher.group(1));
|
||||||
|
}
|
||||||
|
return keywords;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,7 +9,7 @@ import com.knowledge.base.domain.doc.repository.po.File;
|
|||||||
import com.knowledge.base.domain.doc.repository.po.OSRecord;
|
import com.knowledge.base.domain.doc.repository.po.OSRecord;
|
||||||
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
|
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import com.knowledge.base.infrastructure.util.BeanConvertUtil;
|
import com.knowledge.base.infrastructure.util.BeanTool;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -57,13 +57,13 @@ public class FileDomainServiceImpl implements FileDomainService {
|
|||||||
public PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader) {
|
public PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader) {
|
||||||
PageResult<OSRecord> osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader);
|
PageResult<OSRecord> osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader);
|
||||||
|
|
||||||
return new PageResult<>(BeanConvertUtil.convertList(osRecordPageResult.getList(), OSRecordDO.class),
|
return new PageResult<>(BeanTool.convertList(osRecordPageResult.getList(), OSRecordDO.class),
|
||||||
osRecordPageResult.getTotal());
|
osRecordPageResult.getTotal());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void save(OSRecordDO osRecordDO) {
|
public void save(OSRecordDO osRecordDO) {
|
||||||
osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class));
|
osRecordRepository.save(BeanTool.convert(osRecordDO, OSRecord.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -72,7 +72,7 @@ public class FileDomainServiceImpl implements FileDomainService {
|
|||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
|
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
|
||||||
return osRecordOpt.map(osRecord -> BeanConvertUtil.convert(osRecord, OSRecordDO.class));
|
return osRecordOpt.map(osRecord -> BeanTool.convert(osRecord, OSRecordDO.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -87,7 +87,7 @@ public class FileDomainServiceImpl implements FileDomainService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<OSRecordDO> batchQueryOSRecord(List<Long> recordIds) {
|
public List<OSRecordDO> batchQueryOSRecord(List<Long> recordIds) {
|
||||||
return BeanConvertUtil.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class);
|
return BeanTool.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@ -23,17 +23,19 @@ public class ConstantConfig {
|
|||||||
|
|
||||||
public static final long USER_CACHE_EXPIRED_MINUTES = Duration.ofDays(10).toMinutes();
|
public static final long USER_CACHE_EXPIRED_MINUTES = Duration.ofDays(10).toMinutes();
|
||||||
|
|
||||||
public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public";
|
public static final String SHARE_BASE_URL = "https://share.wisdompulse.cn/public";
|
||||||
|
|
||||||
public static final String DEFAULT_UPLOADER = "ADMIN";
|
public static final String DEFAULT_UPLOADER = "ADMIN";
|
||||||
|
|
||||||
public static final String UNKNOWN_LOCAL_RELA_PATH = "UNKNOWN";
|
public static final String UNKNOWN_LOCAL_RELA_PATH = "UNKNOWN";
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 以下是AnythingLLM相关
|
* 以下是AnythingLLM相关
|
||||||
*/
|
*/
|
||||||
public static final String DEFAULT_SLUG_ID = "87e14982-a821-48d8-9c6b-3557d0bb2f96";
|
public static final String LLM_SLUG_KEY = "slug";
|
||||||
|
public static final String LLM_WS_NAME_KEY = "wsName";
|
||||||
|
public static final String KEYWORD_PATTERN_LEFT = "#";
|
||||||
|
public static final String KEYWORD_PATTERN_RIGHT = "#";
|
||||||
|
|
||||||
|
public static final String OS_HOST = "s3.wisdompulse.cn";
|
||||||
}
|
}
|
||||||
@ -12,12 +12,16 @@ public class CorsConfig implements WebMvcConfigurer {
|
|||||||
@Override
|
@Override
|
||||||
public void addCorsMappings(CorsRegistry registry) {
|
public void addCorsMappings(CorsRegistry registry) {
|
||||||
registry.addMapping("/**")
|
registry.addMapping("/**")
|
||||||
.allowedOriginPatterns("*") // 或指定域名如 https://admin.wisdompulse.cn
|
.allowedOriginPatterns(
|
||||||
|
"chrome-extension://*",
|
||||||
|
"http://app.wisdompulse.cn",
|
||||||
|
"https://share.wisdompulse.cn",
|
||||||
|
"http://share.wisdompulse.cn",
|
||||||
|
"http://app-test.wisdompulse.cn"
|
||||||
|
)
|
||||||
.allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS")
|
.allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS")
|
||||||
.allowedHeaders("*")
|
.allowedHeaders("*")
|
||||||
.allowCredentials(true)
|
.allowCredentials(true)
|
||||||
.maxAge(3600);
|
.maxAge(3600);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1,44 +1,103 @@
|
|||||||
package com.knowledge.base.infrastructure.config;
|
package com.knowledge.base.infrastructure.config;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import com.knowledge.base.infrastructure.monitor.ThreadPoolMonitorStarter;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.cloud.context.config.annotation.RefreshScope;
|
import org.springframework.cloud.context.config.annotation.RefreshScope;
|
||||||
|
import org.springframework.cloud.context.scope.refresh.RefreshScopeRefreshedEvent;
|
||||||
|
import org.springframework.context.event.EventListener;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import javax.annotation.PostConstruct;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author Luke.ye
|
* 动态配置类,支持 Nacos 热更新
|
||||||
* @date 2025/5/8 09:59
|
*
|
||||||
|
* @author Luke
|
||||||
|
* @date 2025/5/8
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Getter
|
||||||
@Component
|
@Component
|
||||||
@RefreshScope
|
@RefreshScope
|
||||||
@Getter
|
|
||||||
public class DynamicConfig {
|
public class DynamicConfig {
|
||||||
//是否记录请求和响应信息 Y-记录 N-不记录 默认记录
|
|
||||||
@Value("${micro.saas.doc.parser.recordMsgBody:Y}")
|
@Value("${micro.saas.doc.parser.recordMsgBody:Y}")
|
||||||
private String recordMsgBody;
|
private String recordMsgBody;
|
||||||
|
|
||||||
@Value("${cookie.domain.name:wisdompulse.cn}")
|
@Value("${cookie.domain.name:.wisdompulse.cn}")
|
||||||
private String cookieDomainName;
|
private String cookieDomainName;
|
||||||
|
|
||||||
@Value("${import.schedule.cron:* 0/30 * * * ?}")
|
@Value("${import.schedule.cron:* 0/30 * * * ?}")
|
||||||
private String importScheduleCron;
|
private String importScheduleCron;
|
||||||
|
|
||||||
// token失效时间,默认24小时
|
|
||||||
@Value("${token.expire.time:86400000}")
|
@Value("${token.expire.time:86400000}")
|
||||||
private long tokenExpireTime;
|
private long tokenExpireTime;
|
||||||
|
|
||||||
@Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx,ppt,pptx,txt}")
|
@Value("${os.supported.searchable.file.suffix:pdf,doc,docx,xls,xlsx,ppt,pptx,txt}")
|
||||||
private String supportedSearchFileSuffix;
|
private String supportedSearchFileSuffix;
|
||||||
|
|
||||||
@Value("${file.import.rate.limit: 10}")
|
@Value("${file.import.rate.limit:10}")
|
||||||
private String fileImportRateLimit;
|
private String fileImportRateLimit;
|
||||||
|
|
||||||
@Value("${llm.sse.rate.limit: 3}")
|
@Value("${llm.sse.rate.limit:3}")
|
||||||
private String llmSseRateLimit;
|
private String llmSseRateLimit;
|
||||||
|
|
||||||
|
@Value("${llm.enable.auto.pin:false}")
|
||||||
|
private String enableAutoPin;
|
||||||
|
|
||||||
|
@Value("${llm.default.workspace.name:部门知识库}")
|
||||||
|
private String llmDefaultWsName;
|
||||||
|
|
||||||
|
@Value("${llm.default.slug.id:87e14982-a821-48d8-9c6b-3557d0bb2f96}")
|
||||||
|
private String llmDefaultSlugId;
|
||||||
|
|
||||||
|
@Value("${llm.active.slug.ids:0fbab149-0bdb-4499-93c7-936dec40812d,bcd9ba38-36a6-4e1d-a0a4-933a96fce665}")
|
||||||
|
private String llmSharedActiveSlugIds;
|
||||||
|
|
||||||
@Value("${markdown.path}")
|
@Value("${markdown.path}")
|
||||||
private String mdDirectoryPath;
|
private String mdDirectoryPath;
|
||||||
|
|
||||||
@Value("${exclude.file.path.prefix}")
|
@Value("${exclude.file.path.prefix}")
|
||||||
private String mdExcludePrefix;
|
private String mdExcludePrefix;
|
||||||
|
|
||||||
|
@Value("${thread.pool.monitor.interval.seconds:60}")
|
||||||
|
private long threadPoolMonitorIntervalSeconds;
|
||||||
|
|
||||||
|
@Value("${thread.pool.monitor.enabled:true}")
|
||||||
|
private boolean enableThreadPoolMonitor;
|
||||||
|
|
||||||
|
@Value("${es.show.context.length:150}")
|
||||||
|
private String esShowContextLen;
|
||||||
|
|
||||||
|
// 监听配置刷新事件,主动刷新线程池监控
|
||||||
|
@EventListener
|
||||||
|
public void onRefresh(RefreshScopeRefreshedEvent event) {
|
||||||
|
log.info("[config-refresh-event] 捕获配置刷新事件,重新判断是否需要刷新线程池监控,当前开关值为:{}", enableThreadPoolMonitor);
|
||||||
|
if (enableThreadPoolMonitor) {
|
||||||
|
ThreadPoolMonitorStarter.getInstance().refreshAll();
|
||||||
|
} else {
|
||||||
|
ThreadPoolMonitorStarter.getInstance().clearAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<String> getLlmActiveSlugs() {
|
||||||
|
return Arrays.stream(llmSharedActiveSlugIds.split("[,;\\n]"))
|
||||||
|
.map(String::trim)
|
||||||
|
.filter(StrUtil::isNotBlank)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void init() {
|
||||||
|
log.info("[config-init] DynamicConfig 初始化完成,线程池监控开关 enableThreadPoolMonitor={}", enableThreadPoolMonitor);
|
||||||
|
if (enableThreadPoolMonitor) {
|
||||||
|
ThreadPoolMonitorStarter.getInstance().refreshAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,87 @@
|
|||||||
|
package com.knowledge.base.infrastructure.monitor;
|
||||||
|
|
||||||
|
import cn.hutool.extra.spring.SpringUtil;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import javax.annotation.PreDestroy;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 线程池监控调度器(支持动态配置控制、自动刷新、清理)
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class ThreadPoolMonitorStarter {
|
||||||
|
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
|
||||||
|
private final ScheduledExecutorService monitorScheduler = Executors.newSingleThreadScheduledExecutor(
|
||||||
|
r -> new Thread(r, "thread-monitor-scheduler"));
|
||||||
|
|
||||||
|
private final Map<String, ScheduledFuture<?>> monitorTasks = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
private final Map<String, ExecutorService> registeredPools = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
public void register(String poolName, ExecutorService pool) {
|
||||||
|
registeredPools.put(poolName, pool);
|
||||||
|
refresh(poolName, pool);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void refreshAll() {
|
||||||
|
clearAll();
|
||||||
|
for (Map.Entry<String, ExecutorService> entry : registeredPools.entrySet()) {
|
||||||
|
refresh(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void refresh(String poolName, ExecutorService pool) {
|
||||||
|
if (!"Y".equalsIgnoreCase(dynamicConfig.getRecordMsgBody())) {
|
||||||
|
log.info("[thread-monitor] 未开启线程池监控配置,跳过 {}", poolName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(pool instanceof ThreadPoolExecutor)) {
|
||||||
|
log.info("[thread-monitor] {} 非 ThreadPoolExecutor,无法监控", poolName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ThreadPoolExecutor executor = (ThreadPoolExecutor) pool;
|
||||||
|
|
||||||
|
ScheduledFuture<?> future = monitorScheduler.scheduleAtFixedRate(() -> {
|
||||||
|
log.info("[thread-monitor] {} - 活跃线程数: {}, 最大线程数: {}, 核心线程数: {}, 排队任务数: {}, 总任务数: {}, 已完成任务数: {}",
|
||||||
|
poolName,
|
||||||
|
executor.getActiveCount(),
|
||||||
|
executor.getMaximumPoolSize(),
|
||||||
|
executor.getCorePoolSize(),
|
||||||
|
executor.getQueue().size(),
|
||||||
|
executor.getTaskCount(),
|
||||||
|
executor.getCompletedTaskCount());
|
||||||
|
}, 0, 1, TimeUnit.MINUTES);
|
||||||
|
|
||||||
|
monitorTasks.put(poolName, future);
|
||||||
|
log.info("[thread-monitor] 线程池 {} 监控任务已启动", poolName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void clearAll() {
|
||||||
|
monitorTasks.forEach((name, future) -> {
|
||||||
|
future.cancel(true);
|
||||||
|
log.info("[thread-monitor] 已取消线程池 {} 的监控任务", name);
|
||||||
|
});
|
||||||
|
monitorTasks.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void destroy() {
|
||||||
|
clearAll();
|
||||||
|
monitorScheduler.shutdownNow();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ThreadPoolMonitorStarter getInstance() {
|
||||||
|
return SpringUtil.getBean(ThreadPoolMonitorStarter.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -11,6 +11,7 @@ import com.knowledge.base.domain.common.model.PageResult;
|
|||||||
import com.knowledge.base.domain.doc.model.FileEsField;
|
import com.knowledge.base.domain.doc.model.FileEsField;
|
||||||
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
|
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
|
||||||
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
import com.knowledge.base.infrastructure.north.dto.SearchReq;
|
import com.knowledge.base.infrastructure.north.dto.SearchReq;
|
||||||
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
|
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
|
||||||
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
|
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
|
||||||
@ -44,6 +45,8 @@ public class FileQueryController {
|
|||||||
|
|
||||||
private final FileElasticsearchGateway esGateway;
|
private final FileElasticsearchGateway esGateway;
|
||||||
|
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private FileCacheService fileCacheService;
|
private FileCacheService fileCacheService;
|
||||||
|
|
||||||
@ -78,13 +81,10 @@ public class FileQueryController {
|
|||||||
LOGGER.error("SearchController#resolveFilePath Error. Invalid Input: " + JSONUtil.toJsonStr(req));
|
LOGGER.error("SearchController#resolveFilePath Error. Invalid Input: " + JSONUtil.toJsonStr(req));
|
||||||
return Maps.newHashMap();
|
return Maps.newHashMap();
|
||||||
}
|
}
|
||||||
List<String> namesWithRelativePath = (List)req.get("fileNames");
|
List<String> fileNames = (List<String>) req.get("fileNames");
|
||||||
Map<String, Object> filePaths = Maps.newHashMap();
|
|
||||||
namesWithRelativePath.forEach(pathFile -> {
|
Map<String, String> filePaths = esGateway.getUrlsByFileNames(fileNames);
|
||||||
if(fileCacheService.getMeta(pathFile).isPresent()) {
|
|
||||||
filePaths.put(pathFile, CacheUtil.loadFileMetaProp(pathFile, DocMetaPropEnum.ACCESS_URL.code, String.class));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Map<String, Object> result = Maps.newHashMap();
|
Map<String, Object> result = Maps.newHashMap();
|
||||||
result.put("filePaths", filePaths);
|
result.put("filePaths", filePaths);
|
||||||
return result;
|
return result;
|
||||||
@ -174,7 +174,7 @@ public class FileQueryController {
|
|||||||
result.put("url", source.get(FileEsField.URL));
|
result.put("url", source.get(FileEsField.URL));
|
||||||
|
|
||||||
String content = (String) source.get(FileEsField.CONTENT);
|
String content = (String) source.get(FileEsField.CONTENT);
|
||||||
String summary = extractMultiSnippet(content, keywords, 50);
|
String summary = extractMultiSnippet(content, keywords, Integer.parseInt(dynamicConfig.getEsShowContextLen().trim()));
|
||||||
result.put("summary", summary);
|
result.put("summary", summary);
|
||||||
|
|
||||||
results.add(result);
|
results.add(result);
|
||||||
@ -208,7 +208,7 @@ public class FileQueryController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (matchedSnippets.isEmpty()) {
|
if (matchedSnippets.isEmpty()) {
|
||||||
return content.length() <= 100 ? content : content.substring(0, 100) + "...";
|
return content.length() <= contextLength ? content : content.substring(0, contextLength) + "...";
|
||||||
}
|
}
|
||||||
|
|
||||||
return String.join(" ... ", matchedSnippets);
|
return String.join(" ... ", matchedSnippets);
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package com.knowledge.base.infrastructure.north.controller;
|
|||||||
import cn.hutool.core.io.FileUtil;
|
import cn.hutool.core.io.FileUtil;
|
||||||
import cn.hutool.core.lang.UUID;
|
import cn.hutool.core.lang.UUID;
|
||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
import com.knowledge.base.application.service.DocAppService;
|
import com.knowledge.base.application.service.DocAppService;
|
||||||
import com.knowledge.base.application.service.UserAppService;
|
import com.knowledge.base.application.service.UserAppService;
|
||||||
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
|
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
|
||||||
@ -143,8 +144,10 @@ public class FileWriteController {
|
|||||||
// file字段在主线程停止后,会自动删除,因此需要先转出来
|
// file字段在主线程停止后,会自动删除,因此需要先转出来
|
||||||
byte[] fileBytes = file.getBytes();
|
byte[] fileBytes = file.getBytes();
|
||||||
|
|
||||||
ThreadPoolUtil.execute(() ->
|
logger.info("[uploadFolderToOS] 已上传至对象存储: {}", originalName);
|
||||||
handleAsyncRecord(finalToken, bucket, originFileNameWithSuffix, suffix, s3FullPath, finalUrl, fileBytes, searchable)
|
ThreadPoolUtil.execute(() -> {
|
||||||
|
handleAsyncRecord(finalToken, bucket, originFileNameWithSuffix, suffix, s3FullPath, finalUrl, fileBytes, searchable);
|
||||||
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -153,7 +156,7 @@ public class FileWriteController {
|
|||||||
"count", filePathMap.size(),
|
"count", filePathMap.size(),
|
||||||
"files", filePathMap
|
"files", filePathMap
|
||||||
));
|
));
|
||||||
} catch (Exception e) {
|
} catch (Throwable e) {
|
||||||
logger.error("上传失败", e);
|
logger.error("上传失败", e);
|
||||||
return ResponseEntity.status(500).body("上传失败: " + e.getMessage());
|
return ResponseEntity.status(500).body("上传失败: " + e.getMessage());
|
||||||
}
|
}
|
||||||
@ -217,6 +220,7 @@ public class FileWriteController {
|
|||||||
dto.setUploadTime(now.toString());
|
dto.setUploadTime(now.toString());
|
||||||
dto.setExpireTime(expireTime.toString());
|
dto.setExpireTime(expireTime.toString());
|
||||||
docAppService.saveOSUplodRecord(dto);
|
docAppService.saveOSUplodRecord(dto);
|
||||||
|
logger.info("[OS异步写入] 初始化上传记录成功. OSRecordDTO = {}", JSONUtil.toJsonStr(dto));
|
||||||
|
|
||||||
// 允许保存到本地
|
// 允许保存到本地
|
||||||
if (allowSaveToLocal) {
|
if (allowSaveToLocal) {
|
||||||
@ -243,7 +247,7 @@ public class FileWriteController {
|
|||||||
|
|
||||||
logger.info("本地文件信息已保存并导入ES: localRelaPath = {}, ESImportRes = {}", localRelaFilePath, res);
|
logger.info("本地文件信息已保存并导入ES: localRelaPath = {}, ESImportRes = {}", localRelaFilePath, res);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Throwable e) {
|
||||||
logger.error("异步写入失败: {}", originFileNameWithSuffix, e);
|
logger.error("异步写入失败: {}", originFileNameWithSuffix, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import com.google.common.collect.Lists;
|
|||||||
import com.knowledge.base.application.service.UserAppService;
|
import com.knowledge.base.application.service.UserAppService;
|
||||||
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
|
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
|
||||||
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
|
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
|
||||||
import com.knowledge.base.infrastructure.north.dto.role.UserRoleDTO;
|
import com.knowledge.base.infrastructure.north.dto.role.UserRoleDTO;
|
||||||
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
|
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
|
||||||
@ -17,9 +18,8 @@ import org.springframework.web.bind.annotation.*;
|
|||||||
|
|
||||||
import java.net.URLDecoder;
|
import java.net.URLDecoder;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.List;
|
import java.util.*;
|
||||||
import java.util.Map;
|
import java.util.stream.Collectors;
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/v1/user")
|
@RequestMapping("/api/v1/user")
|
||||||
@ -32,31 +32,57 @@ public class UserQueryController {
|
|||||||
|
|
||||||
private final FileCacheService fileCacheService;
|
private final FileCacheService fileCacheService;
|
||||||
|
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
|
||||||
@GetMapping("/check")
|
@GetMapping("/check")
|
||||||
public ResponseEntity<?> check(
|
public ResponseEntity<?> check(
|
||||||
|
@RequestHeader(value = "X-Auth-Token", required = false) String forwardedToken,
|
||||||
@CookieValue(value = ConstantConfig.COOKIE_KEY, required = false) String cookieToken,
|
@CookieValue(value = ConstantConfig.COOKIE_KEY, required = false) String cookieToken,
|
||||||
@RequestHeader(value = "Authorization", required = false) String headerToken,
|
@RequestHeader(value = "Authorization", required = false) String headerToken,
|
||||||
@RequestHeader(value = "X-Original-URI", required = false) String originalUri
|
@RequestHeader(value = "Cookie", required = false) String rawCookie,
|
||||||
|
@RequestHeader(value = "X-Original-URI", required = false) String originalUri,
|
||||||
|
@RequestHeader(value = "X-Original-HOST", required = false, defaultValue = StrUtil.EMPTY) String host
|
||||||
) {
|
) {
|
||||||
String token = headerToken != null ? headerToken : cookieToken;
|
originalUri = URLDecoder.decode(
|
||||||
originalUri = URLDecoder.decode(originalUri, StandardCharsets.UTF_8);
|
originalUri == null ? StrUtil.EMPTY : originalUri,
|
||||||
|
StandardCharsets.UTF_8
|
||||||
|
);
|
||||||
|
|
||||||
LOGGER.info("[权限校验] token={}, originalUri={}", headerToken != null ? headerToken : cookieToken, originalUri);
|
String token = firstNonBlank(
|
||||||
|
forwardedToken,
|
||||||
|
normalizeAuthorization(headerToken),
|
||||||
|
cookieToken,
|
||||||
|
extractCookie(rawCookie, ConstantConfig.COOKIE_KEY)
|
||||||
|
);
|
||||||
|
|
||||||
if (token == null || token.isBlank()) {
|
LOGGER.info("[权限校验] token={}, host={}, originalUri={}, rawCookieExists={}",
|
||||||
|
maskToken(token), host, originalUri, StrUtil.isNotBlank(rawCookie));
|
||||||
|
|
||||||
|
if (StrUtil.isBlank(token)) {
|
||||||
LOGGER.warn("[权限校验] 未提供token,拒绝访问");
|
LOGGER.warn("[权限校验] 未提供token,拒绝访问");
|
||||||
return ResponseEntity.status(401).body(Map.of("code", 401, "msg", "未授权"));
|
return ResponseEntity.status(401).body(Map.of("code", 401, "msg", "未授权"));
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<UserTokenDTO> userTokenOpt = userAppService.findToken(token);
|
Optional<UserTokenDTO> userTokenOpt = null;
|
||||||
if (userTokenOpt.isEmpty()) {
|
try {
|
||||||
LOGGER.warn("[权限校验] token无效: {}", token);
|
userTokenOpt = userAppService.findToken(token);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOGGER.error("userAppService.findToken(token) error", e);
|
||||||
|
}
|
||||||
|
if (Objects.isNull(userTokenOpt) || userTokenOpt.isEmpty()) {
|
||||||
|
LOGGER.warn("[权限校验] token无效: {}", maskToken(token));
|
||||||
return ResponseEntity.status(401).body(Map.of("code", 401, "msg", "无效token"));
|
return ResponseEntity.status(401).body(Map.of("code", 401, "msg", "无效token"));
|
||||||
}
|
}
|
||||||
|
|
||||||
Long userId = userTokenOpt.get().getUserId();
|
Long userId = userTokenOpt.get().getUserId();
|
||||||
LOGGER.info("[权限校验] 解析到userId={}", userId);
|
LOGGER.info("[权限校验] 解析到userId={}", userId);
|
||||||
|
|
||||||
|
// os的权限校验很简单,token有效即可
|
||||||
|
if (ConstantConfig.OS_HOST.equals(host)) {
|
||||||
|
LOGGER.info("[权限校验] 访问通过: userId={}, host={}, uri={}", userId, host, originalUri);
|
||||||
|
return ResponseEntity.ok(Map.of("code", 0, "msg", "权限校验通过"));
|
||||||
|
}
|
||||||
|
|
||||||
// 查询用户角色
|
// 查询用户角色
|
||||||
List<UserRoleDTO> userRoles = userAppService.listUserRoles(userId);
|
List<UserRoleDTO> userRoles = userAppService.listUserRoles(userId);
|
||||||
if (userRoles == null || userRoles.isEmpty()) {
|
if (userRoles == null || userRoles.isEmpty()) {
|
||||||
@ -75,7 +101,9 @@ public class UserQueryController {
|
|||||||
for (Long roleId : roleIds) {
|
for (Long roleId : roleIds) {
|
||||||
List<RoleFileRuleDTO> rules = userAppService.listRoleFileRules(roleId);
|
List<RoleFileRuleDTO> rules = userAppService.listRoleFileRules(roleId);
|
||||||
LOGGER.info("[权限校验] 角色[{}]规则数={}", roleId, rules != null ? rules.size() : 0);
|
LOGGER.info("[权限校验] 角色[{}]规则数={}", roleId, rules != null ? rules.size() : 0);
|
||||||
if (rules != null) allRules.addAll(rules);
|
if (rules != null) {
|
||||||
|
allRules.addAll(rules);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<UserDTO> userDTO = userAppService.findById(userId);
|
Optional<UserDTO> userDTO = userAppService.findById(userId);
|
||||||
@ -84,12 +112,14 @@ public class UserQueryController {
|
|||||||
boolean hasPermission = false;
|
boolean hasPermission = false;
|
||||||
for (RoleFileRuleDTO rule : allRules) {
|
for (RoleFileRuleDTO rule : allRules) {
|
||||||
String pattern = rule.getFilePattern();
|
String pattern = rule.getFilePattern();
|
||||||
if (pattern == null) continue;
|
if (StrUtil.isBlank(pattern)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
LOGGER.info("[权限校验] 开始判断用户[{}]的角色规则[{}]是否允许访问: {}", userId, pattern, originalUri);
|
LOGGER.info("[权限校验] 开始判断用户[{}]的角色规则[{}]是否允许访问: {}", userId, pattern, originalUri);
|
||||||
|
|
||||||
// 动态替换占位符
|
// 动态替换占位符
|
||||||
if (pattern.contains("${username}") && userName != null) {
|
if (pattern.contains("${username}") && StrUtil.isNotBlank(userName)) {
|
||||||
pattern = pattern.replace("${username}", userName);
|
pattern = pattern.replace("${username}", userName);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -101,7 +131,18 @@ public class UserQueryController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 先前缀匹配,再正则匹配
|
// 先前缀匹配,再正则匹配
|
||||||
if (originalUri.startsWith(pattern) || originalUri.matches(pattern)) {
|
boolean matched = false;
|
||||||
|
if (originalUri.startsWith(pattern)) {
|
||||||
|
matched = true;
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
matched = originalUri.matches(pattern);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOGGER.warn("[权限校验] 规则正则非法,跳过。pattern={}, err={}", pattern, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (matched) {
|
||||||
LOGGER.info("[权限校验] 用户[{}]的角色规则[{}]允许访问: {}", userId, pattern, originalUri);
|
LOGGER.info("[权限校验] 用户[{}]的角色规则[{}]允许访问: {}", userId, pattern, originalUri);
|
||||||
hasPermission = true;
|
hasPermission = true;
|
||||||
break;
|
break;
|
||||||
@ -117,6 +158,66 @@ public class UserQueryController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String... values) {
|
||||||
|
if (values == null || values.length == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (String value : values) {
|
||||||
|
if (StrUtil.isNotBlank(value)) {
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeAuthorization(String authorization) {
|
||||||
|
if (StrUtil.isBlank(authorization)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String value = authorization.trim();
|
||||||
|
if (value.regionMatches(true, 0, "Bearer ", 0, 7)) {
|
||||||
|
return value.substring(7).trim();
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractCookie(String rawCookie, String cookieName) {
|
||||||
|
if (StrUtil.isBlank(rawCookie) || StrUtil.isBlank(cookieName)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String[] pairs = rawCookie.split(";");
|
||||||
|
for (String pair : pairs) {
|
||||||
|
if (StrUtil.isBlank(pair)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
String item = pair.trim();
|
||||||
|
int idx = item.indexOf('=');
|
||||||
|
if (idx <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
String name = item.substring(0, idx).trim();
|
||||||
|
String value = item.substring(idx + 1).trim();
|
||||||
|
if (cookieName.equals(name)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String maskToken(String token) {
|
||||||
|
if (StrUtil.isBlank(token)) {
|
||||||
|
return StrUtil.EMPTY;
|
||||||
|
}
|
||||||
|
if (token.length() <= 8) {
|
||||||
|
return "****";
|
||||||
|
}
|
||||||
|
return token.substring(0, 4) + "****" + token.substring(token.length() - 4);
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping("/username")
|
@GetMapping("/username")
|
||||||
public ResponseEntity<?> getUsernameByToken(
|
public ResponseEntity<?> getUsernameByToken(
|
||||||
@CookieValue(value = ConstantConfig.COOKIE_KEY, required = false) String cookieToken,
|
@CookieValue(value = ConstantConfig.COOKIE_KEY, required = false) String cookieToken,
|
||||||
|
|||||||
@ -9,10 +9,11 @@ import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.http.HttpHeaders;
|
||||||
|
import org.springframework.http.ResponseCookie;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
|
||||||
import javax.servlet.http.Cookie;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@ -50,7 +51,7 @@ public class UserWriteController {
|
|||||||
|
|
||||||
@DeleteMapping
|
@DeleteMapping
|
||||||
public ResponseEntity<?> deleteUser(@RequestParam Long id) {
|
public ResponseEntity<?> deleteUser(@RequestParam Long id) {
|
||||||
if(Objects.isNull(id)) {
|
if (Objects.isNull(id)) {
|
||||||
return ResponseEntity.badRequest().body(Map.of("msg", "userId 不能为空"));
|
return ResponseEntity.badRequest().body(Map.of("msg", "userId 不能为空"));
|
||||||
}
|
}
|
||||||
return ResponseEntity.ok(userAppService.deleteUser(id));
|
return ResponseEntity.ok(userAppService.deleteUser(id));
|
||||||
@ -58,7 +59,7 @@ public class UserWriteController {
|
|||||||
|
|
||||||
@PutMapping("/{userId}")
|
@PutMapping("/{userId}")
|
||||||
public ResponseEntity<?> updateUser(@PathVariable Long userId, @RequestBody UserDTO userDTO) {
|
public ResponseEntity<?> updateUser(@PathVariable Long userId, @RequestBody UserDTO userDTO) {
|
||||||
if(Objects.isNull(userId) || Objects.isNull(userDTO)) {
|
if (Objects.isNull(userId) || Objects.isNull(userDTO)) {
|
||||||
return ResponseEntity.badRequest().body(Map.of("msg", "参数错误:id或用户信息为空!"));
|
return ResponseEntity.badRequest().body(Map.of("msg", "参数错误:id或用户信息为空!"));
|
||||||
}
|
}
|
||||||
return ResponseEntity.ok(userAppService.updateUser(userId, userDTO));
|
return ResponseEntity.ok(userAppService.updateUser(userId, userDTO));
|
||||||
@ -66,13 +67,19 @@ public class UserWriteController {
|
|||||||
|
|
||||||
@PostMapping("/change-pwd")
|
@PostMapping("/change-pwd")
|
||||||
public ResponseEntity<?> changeUserPassword(@RequestBody Map<String, String> body) {
|
public ResponseEntity<?> changeUserPassword(@RequestBody Map<String, String> body) {
|
||||||
if(MapUtil.isEmpty(body)) {
|
if (MapUtil.isEmpty(body)) {
|
||||||
return ResponseEntity.badRequest().body(Map.of("msg", "参数错误,body为空"));
|
return ResponseEntity.badRequest().body(Map.of("msg", "参数错误,body为空"));
|
||||||
}
|
}
|
||||||
body.getOrDefault("userId", "");
|
body.getOrDefault("userId", "");
|
||||||
body.getOrDefault("oldPassword", "");
|
body.getOrDefault("oldPassword", "");
|
||||||
body.getOrDefault("newPassword", "");
|
body.getOrDefault("newPassword", "");
|
||||||
return ResponseEntity.ok(userAppService.changePassword(Long.parseLong(body.get("userId")), body.get("oldPassword"), body.get("newPassword")));
|
return ResponseEntity.ok(
|
||||||
|
userAppService.changePassword(
|
||||||
|
Long.parseLong(body.get("userId")),
|
||||||
|
body.get("oldPassword"),
|
||||||
|
body.get("newPassword")
|
||||||
|
)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/login")
|
@PostMapping("/login")
|
||||||
@ -87,14 +94,14 @@ public class UserWriteController {
|
|||||||
|
|
||||||
UserDTO user = userOpt.get();
|
UserDTO user = userOpt.get();
|
||||||
long expireMs = dynamicConfig.getTokenExpireTime();
|
long expireMs = dynamicConfig.getTokenExpireTime();
|
||||||
|
long maxAgeSeconds = expireMs / 1000;
|
||||||
UserTokenDTO token = userAppService.createToken(user.getId(), expireMs);
|
UserTokenDTO token = userAppService.createToken(user.getId(), expireMs);
|
||||||
|
|
||||||
Cookie cookie = new Cookie(ConstantConfig.COOKIE_KEY, token.getToken());
|
ResponseCookie authCookie = buildAuthCookie(token.getToken(), maxAgeSeconds);
|
||||||
cookie.setPath("/");
|
response.addHeader(HttpHeaders.SET_COOKIE, authCookie.toString());
|
||||||
cookie.setDomain(dynamicConfig.getCookieDomainName());
|
|
||||||
cookie.setHttpOnly(true);
|
LOGGER.info("[登录] 用户[{}]登录成功,已写入认证cookie,domain={}, maxAge={}s",
|
||||||
cookie.setMaxAge((int) (expireMs / 1000));
|
user.getUsername(), dynamicConfig.getCookieDomainName(), maxAgeSeconds);
|
||||||
response.addCookie(cookie);
|
|
||||||
|
|
||||||
return ResponseEntity.ok(Map.of(
|
return ResponseEntity.ok(Map.of(
|
||||||
"token", token.getToken(),
|
"token", token.getToken(),
|
||||||
@ -109,21 +116,18 @@ public class UserWriteController {
|
|||||||
HttpServletResponse response
|
HttpServletResponse response
|
||||||
) {
|
) {
|
||||||
String token = headerToken != null ? headerToken : cookieToken;
|
String token = headerToken != null ? headerToken : cookieToken;
|
||||||
if (token != null) {
|
if (token != null && !token.isBlank()) {
|
||||||
userAppService.removeToken(token);
|
userAppService.removeToken(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
Cookie cookie = new Cookie(ConstantConfig.COOKIE_KEY, "");
|
ResponseCookie clearCookie = buildAuthCookie("", 0);
|
||||||
cookie.setPath("/");
|
response.addHeader(HttpHeaders.SET_COOKIE, clearCookie.toString());
|
||||||
cookie.setDomain(dynamicConfig.getCookieDomainName());
|
|
||||||
cookie.setHttpOnly(true);
|
LOGGER.info("[登出] 已清理认证cookie,domain={}", dynamicConfig.getCookieDomainName());
|
||||||
cookie.setMaxAge(0);
|
|
||||||
response.addCookie(cookie);
|
|
||||||
|
|
||||||
return ResponseEntity.ok(Map.of("msg", "已登出"));
|
return ResponseEntity.ok(Map.of("msg", "已登出"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@PostMapping("/add-user-role")
|
@PostMapping("/add-user-role")
|
||||||
public ResponseEntity<?> addUserRole(@RequestParam Long userId, @RequestParam Long roleId) {
|
public ResponseEntity<?> addUserRole(@RequestParam Long userId, @RequestParam Long roleId) {
|
||||||
return ResponseEntity.ok(userAppService.addUserRole(userId, roleId));
|
return ResponseEntity.ok(userAppService.addUserRole(userId, roleId));
|
||||||
@ -133,4 +137,20 @@ public class UserWriteController {
|
|||||||
public ResponseEntity<?> removeUserRole(@RequestParam Long userId, @RequestParam Long roleId) {
|
public ResponseEntity<?> removeUserRole(@RequestParam Long userId, @RequestParam Long roleId) {
|
||||||
return ResponseEntity.ok(userAppService.removeUserRole(userId, roleId));
|
return ResponseEntity.ok(userAppService.removeUserRole(userId, roleId));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
private ResponseCookie buildAuthCookie(String token, long maxAgeSeconds) {
|
||||||
|
ResponseCookie.ResponseCookieBuilder builder = ResponseCookie
|
||||||
|
.from(ConstantConfig.COOKIE_KEY, token == null ? "" : token)
|
||||||
|
.path("/")
|
||||||
|
.httpOnly(true)
|
||||||
|
.sameSite("Lax")
|
||||||
|
.maxAge(maxAgeSeconds);
|
||||||
|
|
||||||
|
String cookieDomainName = dynamicConfig.getCookieDomainName();
|
||||||
|
if (cookieDomainName != null && !cookieDomainName.trim().isEmpty()) {
|
||||||
|
builder.domain(cookieDomainName.trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
package com.knowledge.base.infrastructure.north.dto.llm;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class WorkspaceAttachment {
|
||||||
|
private String docpath;
|
||||||
|
private String url;
|
||||||
|
private boolean pinned;
|
||||||
|
}
|
||||||
@ -1,6 +1,9 @@
|
|||||||
package com.knowledge.base.infrastructure.south.es;
|
package com.knowledge.base.infrastructure.south.es;
|
||||||
|
|
||||||
import cn.hutool.core.bean.BeanUtil;
|
import cn.hutool.core.bean.BeanUtil;
|
||||||
|
import cn.hutool.core.bean.copier.CopyOptions;
|
||||||
|
import cn.hutool.core.collection.CollectionUtil;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
import com.knowledge.base.domain.doc.model.FileEsField;
|
import com.knowledge.base.domain.doc.model.FileEsField;
|
||||||
import com.knowledge.base.domain.doc.model.FileEsModel;
|
import com.knowledge.base.domain.doc.model.FileEsModel;
|
||||||
import com.knowledge.base.infrastructure.util.SafeIdUtil;
|
import com.knowledge.base.infrastructure.util.SafeIdUtil;
|
||||||
@ -30,8 +33,11 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author Luke.ye
|
* @author Luke.ye
|
||||||
@ -46,6 +52,73 @@ public class FileElasticsearchGateway {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private RestHighLevelClient esClient;
|
private RestHighLevelClient esClient;
|
||||||
|
|
||||||
|
|
||||||
|
public List<FileEsModel> searchByKeywords(List<String> keywords) {
|
||||||
|
if (CollectionUtil.isEmpty(keywords)) {
|
||||||
|
return Lists.newArrayList();
|
||||||
|
}
|
||||||
|
List<List<String>> keywordGroups = keywords.stream()
|
||||||
|
.map(Collections::singletonList)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
QueryBuilder query = buildAllMustSearchQuery(keywordGroups);
|
||||||
|
|
||||||
|
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||||
|
.query(query)
|
||||||
|
.from(0)
|
||||||
|
.size(3)
|
||||||
|
.sort("_score", SortOrder.DESC);
|
||||||
|
|
||||||
|
SearchRequest request = new SearchRequest(INDEX_NAME).source(builder);
|
||||||
|
SearchResponse response;
|
||||||
|
try {
|
||||||
|
response = esClient.search(request, RequestOptions.DEFAULT);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("[searchByKeywords] error.", e);
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<FileEsModel> result = Lists.newArrayList();
|
||||||
|
for (SearchHit hit : response.getHits().getHits()) {
|
||||||
|
FileEsModel fileEsModel = BeanUtil.mapToBean(hit.getSourceAsMap(), FileEsModel.class, true, CopyOptions.create());
|
||||||
|
result.add(fileEsModel);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueryBuilder buildAllMustSearchQuery(List<List<String>> keywordGroups) {
|
||||||
|
BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();
|
||||||
|
for (List<String> group : keywordGroups) {
|
||||||
|
BoolQueryBuilder groupQuery = QueryBuilders.boolQuery();
|
||||||
|
|
||||||
|
for (String keyword : group) {
|
||||||
|
BoolQueryBuilder fieldQuery = QueryBuilders.boolQuery();
|
||||||
|
|
||||||
|
// 标题(filename)权重高
|
||||||
|
fieldQuery.should(QueryBuilders.matchPhraseQuery(FileEsField.FILENAME, keyword).boost(10.0f));
|
||||||
|
fieldQuery.should(QueryBuilders.matchPhraseQuery(FileEsField.CONTENT, keyword).boost(1.0f));
|
||||||
|
|
||||||
|
// 英文关键词额外加wildcard匹配,filename.keyword也加权
|
||||||
|
if (isEnglishLike(keyword)) {
|
||||||
|
fieldQuery.should(QueryBuilders.wildcardQuery("filename.keyword", "*" + keyword + "*").boost(10.0f));
|
||||||
|
}
|
||||||
|
|
||||||
|
groupQuery.must(fieldQuery);
|
||||||
|
}
|
||||||
|
// 必须命中所有关键词分组
|
||||||
|
outerQuery.must(groupQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 过滤过期文档
|
||||||
|
outerQuery.filter(QueryBuilders.rangeQuery(FileEsField.EXPIRE_TIME)
|
||||||
|
.gte(System.currentTimeMillis())
|
||||||
|
.timeZone("+08:00"));
|
||||||
|
|
||||||
|
return outerQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public SearchResponse searchFileDocuments(List<List<String>> keywordGroups, int page, int size) {
|
public SearchResponse searchFileDocuments(List<List<String>> keywordGroups, int page, int size) {
|
||||||
try {
|
try {
|
||||||
int from = (page - 1) * size;
|
int from = (page - 1) * size;
|
||||||
@ -230,5 +303,27 @@ public class FileElasticsearchGateway {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据 fileName批量查出 url
|
||||||
|
*/
|
||||||
|
public Map<String, String> getUrlsByFileNames(List<String> fileNames) {
|
||||||
|
Map<String, String> result = new HashMap<>();
|
||||||
|
if (CollectionUtil.isEmpty(fileNames)) return result;
|
||||||
|
List<List<String>> keywordGroups = fileNames.stream().map(Collections::singletonList).collect(Collectors.toList());
|
||||||
|
SearchResponse response = searchFileDocuments(keywordGroups, 1, fileNames.size());
|
||||||
|
for (SearchHit hit : response.getHits().getHits()) {
|
||||||
|
Map<String, Object> src = hit.getSourceAsMap();
|
||||||
|
String filename = String.valueOf(src.get(FileEsField.FILENAME));
|
||||||
|
String url = String.valueOf(src.get("url"));
|
||||||
|
// 严格精确匹配
|
||||||
|
if (fileNames.contains(filename)) {
|
||||||
|
result.put(filename, url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,66 @@
|
|||||||
|
package com.knowledge.base.infrastructure.south.llm;
|
||||||
|
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 具体的使用顺序可参考 Readme.md
|
||||||
|
*/
|
||||||
|
public interface AnythingLLMService extends LLMService{
|
||||||
|
/**
|
||||||
|
* 将OB仓库中的文档导入LLM本地文件
|
||||||
|
* @param llmToken
|
||||||
|
* @param vaultDir
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
Map<String, Object> importObsidianToLLM(String llmToken, File vaultDir);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 从本地文件中移除文件夹
|
||||||
|
* @param llmToken
|
||||||
|
* @param folderName
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
String removeFolderFromLLM(String llmToken, String folderName);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取本地文件列表
|
||||||
|
* @param llmToken
|
||||||
|
* @param workspaceSlug
|
||||||
|
* @param keyword
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
Map<String, Object> getLocalFileItems(String llmToken, String workspaceSlug, String keyword);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将本地文件嵌入工作区
|
||||||
|
* @param llmToken
|
||||||
|
* @param workspaceSlug
|
||||||
|
* @param adds
|
||||||
|
* @param deletes
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
Map<String, Object> updateEmbeddings(String llmToken, String workspaceSlug, List<String> adds, List<String> deletes);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取工作区下所有的附件
|
||||||
|
* @param llmToken
|
||||||
|
* @param workspaceSlug
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
List<WorkspaceAttachment> fetchAttachments(String llmToken, String workspaceSlug);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 更新工作区中文件的pin状态
|
||||||
|
* @param llmToken
|
||||||
|
* @param workspaceSlug
|
||||||
|
* @param docPath
|
||||||
|
* @param pinStatus
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
String updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus);
|
||||||
|
}
|
||||||
@ -2,8 +2,14 @@ package com.knowledge.base.infrastructure.south.llm;
|
|||||||
|
|
||||||
import cn.hutool.core.collection.CollectionUtil;
|
import cn.hutool.core.collection.CollectionUtil;
|
||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
|
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
|
||||||
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
import com.knowledge.base.infrastructure.config.ConstantConfig;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
import com.knowledge.base.infrastructure.util.HttpHelper;
|
import com.knowledge.base.infrastructure.util.HttpHelper;
|
||||||
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
|
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@ -14,23 +20,26 @@ import org.springframework.beans.factory.annotation.Value;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.HashMap;
|
import java.nio.file.Files;
|
||||||
import java.util.List;
|
import java.nio.file.Path;
|
||||||
import java.util.Map;
|
import java.util.*;
|
||||||
import java.util.Optional;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class AnythingLLMServiceImpl implements LLMService {
|
public class AnythingLLMServiceImpl implements AnythingLLMService {
|
||||||
|
|
||||||
@Value("${llm.remote.base-url}")
|
@Value("${llm.remote.base-url}")
|
||||||
private String baseUrl;
|
private String baseUrl;
|
||||||
|
|
||||||
private final HttpHelper httpHelper;
|
private final HttpHelper httpHelper;
|
||||||
private final UserCacheService userCacheService;
|
private final UserCacheService userCacheService;
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean supports(String type) {
|
public boolean supports(String type) {
|
||||||
@ -46,11 +55,12 @@ public class AnythingLLMServiceImpl implements LLMService {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void streamAnswer(String llmToken, String question, Map<String, Object> params, WriterAdapter writer) throws Exception {
|
public void streamAnswer(String llmToken, String question, Map<String, Object> params, WriterAdapter writer) throws Exception {
|
||||||
String slug = (String) params.get("slug");
|
String slug = (String) params.get(ConstantConfig.LLM_SLUG_KEY);
|
||||||
if (StrUtil.isBlank(slug)) {
|
if (StrUtil.isBlank(slug)) {
|
||||||
slug = fetchSlugByWsName(llmToken, (String) params.getOrDefault("wsName", "部门知识库"));
|
slug = fetchSlugByWsName(llmToken, (String) params.getOrDefault(ConstantConfig.LLM_WS_NAME_KEY, dynamicConfig.getLlmDefaultWsName()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.info("[准备开始回答问题]: question={}, slug={}", question, slug);
|
||||||
String url = baseUrl + "/api/workspace/" + slug + "/stream-chat";
|
String url = baseUrl + "/api/workspace/" + slug + "/stream-chat";
|
||||||
|
|
||||||
Map<String, Object> body = Map.of(
|
Map<String, Object> body = Map.of(
|
||||||
@ -78,16 +88,233 @@ public class AnythingLLMServiceImpl implements LLMService {
|
|||||||
Map<String, Object> resMap = httpHelper.get(url, llmToken);
|
Map<String, Object> resMap = httpHelper.get(url, llmToken);
|
||||||
List<HashMap> workspaces = (List<HashMap>)resMap.get("workspaces");
|
List<HashMap> workspaces = (List<HashMap>)resMap.get("workspaces");
|
||||||
if(CollectionUtil.isEmpty(workspaces)) {
|
if(CollectionUtil.isEmpty(workspaces)) {
|
||||||
return ConstantConfig.DEFAULT_SLUG_ID;
|
return dynamicConfig.getLlmDefaultSlugId();
|
||||||
}
|
}
|
||||||
Optional<HashMap> targetWorkspace = workspaces.stream().filter(workspace -> workspace.get("name").equals(wsName)).findFirst();
|
Optional<HashMap> targetWorkspace = workspaces.stream().filter(workspace -> workspace.get("name").equals(wsName)).findFirst();
|
||||||
if(targetWorkspace.isEmpty()) {
|
if(targetWorkspace.isEmpty()) {
|
||||||
return ConstantConfig.DEFAULT_SLUG_ID;
|
return dynamicConfig.getLlmDefaultSlugId();
|
||||||
}
|
}
|
||||||
|
|
||||||
String slug = (String) targetWorkspace.get().get("slug");
|
String slug = (String) targetWorkspace.get().get(ConstantConfig.LLM_SLUG_KEY);
|
||||||
// slug数据进缓存
|
if(StrUtil.isNotBlank(slug)) {
|
||||||
userCacheService.cacheAnythingLLMSlugId(wsName, slug);
|
// slug数据进缓存
|
||||||
return StrUtil.isBlank(slug) ? ConstantConfig.DEFAULT_SLUG_ID : slug;
|
userCacheService.cacheAnythingLLMSlugId(wsName, slug);
|
||||||
|
}
|
||||||
|
return StrUtil.isBlank(slug) ? dynamicConfig.getLlmDefaultSlugId() : slug;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> importObsidianToLLM(String llmToken, File vaultDir) {
|
||||||
|
if (!vaultDir.exists() || !vaultDir.isDirectory()) {
|
||||||
|
throw new IllegalArgumentException("无效的 Obsidian vault 目录: " + vaultDir.getAbsolutePath());
|
||||||
|
}
|
||||||
|
|
||||||
|
String url = baseUrl + "/api/ext/obsidian/vault";
|
||||||
|
List<Map<String, Object>> fileList = new ArrayList<>();
|
||||||
|
String vaultName = vaultDir.getName();
|
||||||
|
|
||||||
|
try (Stream<Path> pathStream = Files.walk(vaultDir.toPath())) {
|
||||||
|
pathStream.filter(Files::isRegularFile)
|
||||||
|
.filter(p -> p.toString().endsWith(".md"))
|
||||||
|
.forEach(path -> {
|
||||||
|
try {
|
||||||
|
String content = Files.readString(path, StandardCharsets.UTF_8);
|
||||||
|
Path relativePath = vaultDir.toPath().relativize(path);
|
||||||
|
String normalizedPath = vaultName + "/" + relativePath.toString().replace(File.separator, "/");
|
||||||
|
|
||||||
|
Map<String, Object> fileMap = new LinkedHashMap<>();
|
||||||
|
fileMap.put("name", path.getFileName().toString());
|
||||||
|
fileMap.put("path", normalizedPath);
|
||||||
|
fileMap.put("content", content);
|
||||||
|
|
||||||
|
fileList.add(fileMap);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.warn("读取文件失败: {}", path.toAbsolutePath(), e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("遍历 Obsidian vault 目录失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构建 JSON 请求体(纯字符串)
|
||||||
|
Map<String, Object> requestMap = Map.of("files", fileList);
|
||||||
|
String requestBodyStr = JSONUtil.toJsonStr(requestMap); // 或 objectMapper.writeValueAsString(...)
|
||||||
|
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
headers.put("Content-Type", "text/plain;charset=UTF-8");
|
||||||
|
|
||||||
|
try {
|
||||||
|
return httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"POST",
|
||||||
|
headers,
|
||||||
|
requestBodyStr,
|
||||||
|
new TypeReference<Map<String, Object>>() {}
|
||||||
|
);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("上传 Obsidian 文件失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String removeFolderFromLLM(String llmToken, String folderName) {
|
||||||
|
String url = baseUrl + "/api/system/remove-folder";
|
||||||
|
|
||||||
|
Map<String, Object> body = Map.of("name", folderName);
|
||||||
|
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
headers.put("Content-Type", "application/json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
return httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"DELETE",
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
new TypeReference<String>() {}
|
||||||
|
);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("删除文件夹失败: " + folderName, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<WorkspaceAttachment> fetchAttachments(String llmToken, String workspaceSlug) {
|
||||||
|
String url = baseUrl + "/api/workspace/" + workspaceSlug;
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
|
||||||
|
try {
|
||||||
|
Map<String, Object> response = httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"GET",
|
||||||
|
headers,
|
||||||
|
null,
|
||||||
|
new TypeReference<Map<String, Object>>() {}
|
||||||
|
);
|
||||||
|
|
||||||
|
List<WorkspaceAttachment> result = new ArrayList<>();
|
||||||
|
if (response.containsKey("workspace")) {
|
||||||
|
Map workspace = (Map) response.get("workspace");
|
||||||
|
Object documents = workspace.get("documents");
|
||||||
|
if (documents instanceof Iterable) {
|
||||||
|
for (Object docObj : (Iterable) documents) {
|
||||||
|
if (docObj instanceof Map) {
|
||||||
|
Map doc = (Map) docObj;
|
||||||
|
String docpath = (String) doc.get("docpath");
|
||||||
|
String metadataStr = (String) doc.get("metadata");
|
||||||
|
boolean pinned = Boolean.TRUE.equals(doc.get("pinned"));
|
||||||
|
String urlVal = "";
|
||||||
|
if (docpath != null && metadataStr != null) {
|
||||||
|
try {
|
||||||
|
Map<String, Object> metadata = new ObjectMapper().readValue(metadataStr, new TypeReference<Map<String, Object>>() {});
|
||||||
|
urlVal = (String) metadata.getOrDefault("url", "");
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("解析 metadata 失败: {}", metadataStr, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
WorkspaceAttachment attachment = new WorkspaceAttachment();
|
||||||
|
attachment.setDocpath(docpath);
|
||||||
|
attachment.setUrl(urlVal);
|
||||||
|
attachment.setPinned(pinned);
|
||||||
|
result.add(attachment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("获取工作区附件失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus) {
|
||||||
|
String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-pin";
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
|
||||||
|
Map<String, Object> body = new HashMap<>();
|
||||||
|
body.put("docPath", docPath);
|
||||||
|
body.put("pinStatus", pinStatus);
|
||||||
|
|
||||||
|
try {
|
||||||
|
return httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"POST",
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
new TypeReference<String>() {}
|
||||||
|
);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("设置 pin 状态失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> getLocalFileItems(String llmToken, String workspaceSlug, String keyword) {
|
||||||
|
String url = baseUrl + "/api/system/local-files";
|
||||||
|
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
|
||||||
|
try {
|
||||||
|
Map<String, Object> response = httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"GET",
|
||||||
|
headers,
|
||||||
|
null,
|
||||||
|
new TypeReference<Map<String, Object>>() {}
|
||||||
|
);
|
||||||
|
|
||||||
|
// 从 localFiles -> items 遍历,查找 name 包含关键词的一级目录
|
||||||
|
if (response.containsKey("localFiles")) {
|
||||||
|
Map localFiles = (Map) response.get("localFiles");
|
||||||
|
Object items = localFiles.get("items");
|
||||||
|
if (items instanceof List) {
|
||||||
|
for (Object folderObj : (List) items) {
|
||||||
|
if (folderObj instanceof Map) {
|
||||||
|
Map folder = (Map) folderObj;
|
||||||
|
String name = (String) folder.get("name");
|
||||||
|
if (name != null && name.contains(keyword)) {
|
||||||
|
return folder;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Maps.newHashMap();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("获取本地文件失败:" + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> updateEmbeddings(String llmToken, String workspaceSlug, List<String> adds, List<String> deletes) {
|
||||||
|
String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-embeddings";
|
||||||
|
|
||||||
|
Map<String, String> headers = new HashMap<>();
|
||||||
|
headers.put("Authorization", "Bearer " + llmToken);
|
||||||
|
|
||||||
|
Map<String, Object> body = new HashMap<>();
|
||||||
|
body.put("adds", adds);
|
||||||
|
body.put("deletes", deletes);
|
||||||
|
|
||||||
|
try {
|
||||||
|
return httpHelper.exchange(
|
||||||
|
url,
|
||||||
|
"POST",
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
new TypeReference<Map<String, Object>>() {} // 可忽略返回,仅校验状态
|
||||||
|
);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException("调用 update-embeddings 失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,79 @@
|
|||||||
|
package com.knowledge.base.infrastructure.south.llm;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@Slf4j
|
||||||
|
public class WorkspaceSelector {
|
||||||
|
|
||||||
|
private final DynamicConfig dynamicConfig;
|
||||||
|
// slugInUse 状态只保留当前 Nacos 配置的 slug
|
||||||
|
private final Map<String, Boolean> slugInUse = new ConcurrentHashMap<>();
|
||||||
|
private final Map<String, String> slugQuestion = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
public WorkspaceSelector(DynamicConfig dynamicConfig) {
|
||||||
|
this.dynamicConfig = dynamicConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前最新的slug集合
|
||||||
|
*/
|
||||||
|
private Set<String> getCurrentSlugs() {
|
||||||
|
Set<String> slugs = dynamicConfig.getLlmActiveSlugs();
|
||||||
|
// 同步slugInUse和slugQuestion,仅保留现有slug
|
||||||
|
slugInUse.keySet().removeIf(slug -> !slugs.contains(slug));
|
||||||
|
slugQuestion.keySet().removeIf(slug -> !slugs.contains(slug));
|
||||||
|
// 为新slug初始化状态
|
||||||
|
for (String slug : slugs) {
|
||||||
|
slugInUse.putIfAbsent(slug, false);
|
||||||
|
}
|
||||||
|
return slugs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取一个空闲的slug,返回StrUtil.EMPTY则无可用
|
||||||
|
*/
|
||||||
|
public synchronized String acquire(String question) {
|
||||||
|
Set<String> availableSlugs = getCurrentSlugs();
|
||||||
|
for (String slug : availableSlugs) {
|
||||||
|
if (!slugInUse.getOrDefault(slug, false)) {
|
||||||
|
slugInUse.put(slug, true);
|
||||||
|
slugQuestion.put(slug, question);
|
||||||
|
return slug;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StrUtil.EMPTY;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 释放slug,同时清理question */
|
||||||
|
public synchronized void release(String slug) {
|
||||||
|
Set<String> availableSlugs = getCurrentSlugs();
|
||||||
|
if (availableSlugs.contains(slug)) {
|
||||||
|
log.info("slug is released: {}, question: {}", slug, slugQuestion.get(slug));
|
||||||
|
slugInUse.put(slug, false);
|
||||||
|
slugQuestion.remove(slug);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 获取当前slug对应的question,主要用于日志打印,无占用返回null */
|
||||||
|
public String getQuestion(String slug) {
|
||||||
|
return slugQuestion.get(slug);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前所有已占用的slug及对应的question
|
||||||
|
*/
|
||||||
|
public Map<String, String> getAllInUseSlugQuestions() {
|
||||||
|
// 返回当前有效slug的快照
|
||||||
|
getCurrentSlugs(); // 同步最新
|
||||||
|
return new ConcurrentHashMap<>(slugQuestion);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -52,11 +52,11 @@ public class MinioOSGatewayImpl implements ObjectStorageGateway {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void ensureBucketExists(String bucket) throws Exception {
|
public void ensureBucketExists(String bucket) throws Exception {
|
||||||
boolean exists = minioClient.bucketExists(
|
boolean exists = localMinioClient.bucketExists(
|
||||||
BucketExistsArgs.builder().bucket(bucket).build()
|
BucketExistsArgs.builder().bucket(bucket).build()
|
||||||
);
|
);
|
||||||
if (!exists) {
|
if (!exists) {
|
||||||
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucket).build());
|
localMinioClient.makeBucket(MakeBucketArgs.builder().bucket(bucket).build());
|
||||||
log.info("已创建 bucket: {}", bucket);
|
log.info("已创建 bucket: {}", bucket);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -103,7 +103,7 @@ public class MinioOSGatewayImpl implements ObjectStorageGateway {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Iterable<Result<DeleteError>> results = minioClient.removeObjects(
|
Iterable<Result<DeleteError>> results = localMinioClient.removeObjects(
|
||||||
RemoveObjectsArgs.builder()
|
RemoveObjectsArgs.builder()
|
||||||
.bucket(bucket)
|
.bucket(bucket)
|
||||||
.objects(objectPaths.stream()
|
.objects(objectPaths.stream()
|
||||||
|
|||||||
@ -1,35 +0,0 @@
|
|||||||
package com.knowledge.base.infrastructure.util;
|
|
||||||
|
|
||||||
import cn.hutool.core.collection.CollectionUtil;
|
|
||||||
import org.springframework.beans.BeanUtils;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
public class BeanConvertUtil {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将 source 转换为指定类型的目标对象
|
|
||||||
*/
|
|
||||||
public static <S, T> T convert(S source, Class<T> targetClass) {
|
|
||||||
if (source == null) return null;
|
|
||||||
try {
|
|
||||||
T target = targetClass.getDeclaredConstructor().newInstance();
|
|
||||||
BeanUtils.copyProperties(source, target);
|
|
||||||
return target;
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new RuntimeException("BeanConvertUtil.convert error", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 批量转换:List<S> → List<T>
|
|
||||||
*/
|
|
||||||
public static <S, T> List<T> convertList(List<S> sourceList, Class<T> targetClass) {
|
|
||||||
if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList();
|
|
||||||
return sourceList.stream()
|
|
||||||
.map(source -> convert(source, targetClass))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -0,0 +1,90 @@
|
|||||||
|
package com.knowledge.base.infrastructure.util;
|
||||||
|
|
||||||
|
import cn.hutool.core.bean.BeanUtil;
|
||||||
|
import cn.hutool.core.collection.CollectionUtil;
|
||||||
|
import cn.hutool.core.lang.TypeReference;
|
||||||
|
import cn.hutool.core.util.ObjectUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import org.springframework.beans.BeanUtils;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
public class BeanTool {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将 source 转换为指定类型的目标对象
|
||||||
|
*/
|
||||||
|
public static <S, T> T convert(S source, Class<T> targetClass) {
|
||||||
|
if (source == null) return null;
|
||||||
|
try {
|
||||||
|
T target = targetClass.getDeclaredConstructor().newInstance();
|
||||||
|
BeanUtils.copyProperties(source, target);
|
||||||
|
return target;
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("BeanConvertUtil.convert error", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 批量转换:List<S> → List<T>
|
||||||
|
*/
|
||||||
|
public static <S, T> List<T> convertList(List<S> sourceList, Class<T> targetClass) {
|
||||||
|
if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList();
|
||||||
|
return sourceList.stream()
|
||||||
|
.map(source -> convert(source, targetClass))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取嵌套字段的值(路径支持.分隔 & 数组索引)
|
||||||
|
* @param map 原始 Map
|
||||||
|
* @param key 路径,如 data.destination 或 data.results[0].file
|
||||||
|
* @param <T> 返回类型
|
||||||
|
* @return 值(可为 null)
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public static <T> T getMapV(String key, Object map) {
|
||||||
|
Object value = BeanUtil.getProperty(map, key);
|
||||||
|
return (T) value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取嵌套字段值并转换为指定类型(通过 JSON 编解码转换)
|
||||||
|
* @param key 路径
|
||||||
|
* @param map 原始 Map
|
||||||
|
* @param clazz 目标类型
|
||||||
|
* @param <T> 类型参数
|
||||||
|
* @return 指定类型的值,转换失败或字段不存在时返回 null
|
||||||
|
*/
|
||||||
|
public static <T> T getMapV(String key, Object map, Class<T> clazz) {
|
||||||
|
Object value = BeanUtil.getProperty(map, key);
|
||||||
|
if (ObjectUtil.isEmpty(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return JSONUtil.toBean(JSONUtil.toJsonStr(value), clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取嵌套字段并反序列化为指定泛型类型
|
||||||
|
*
|
||||||
|
* @param key 字段路径(支持嵌套,如 "data.items")
|
||||||
|
* @param map 原始 Map 对象
|
||||||
|
* @param typeReference 泛型类型引用(如 new TypeReference<List<Map<String, Object>>>(){})
|
||||||
|
* @return 类型化对象或 null
|
||||||
|
*/
|
||||||
|
public static <T> T getMapV(String key, Object map, TypeReference<T> typeReference) {
|
||||||
|
Object value = BeanUtil.getProperty(map, key);
|
||||||
|
if (ObjectUtil.isEmpty(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// value 已经是对象,如果是 Map/List 类型,可直接转换;否则先转 JSON 字符串再解析
|
||||||
|
if (value instanceof CharSequence) {
|
||||||
|
return JSONUtil.toBean((String) value, typeReference, false);
|
||||||
|
} else {
|
||||||
|
return JSONUtil.toBean(JSONUtil.toJsonStr(value), typeReference, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@ -2,6 +2,7 @@ package com.knowledge.base.infrastructure.util;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.core.type.TypeReference;
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import okhttp3.*;
|
import okhttp3.*;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@ -101,6 +102,81 @@ public class HttpHelper {
|
|||||||
return builder.build();
|
return builder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 通用 HTTP 请求方法,支持任意 method、headers、请求体,自动反序列化返回
|
||||||
|
*
|
||||||
|
* @param url 请求地址
|
||||||
|
* @param method 请求方法,如 GET、POST、PUT、DELETE
|
||||||
|
* @param headers 请求头(可选,可为 null)
|
||||||
|
* @param body 请求体对象(可选,可为 null)
|
||||||
|
* @param responseType 返回类型(例如 Map.class)
|
||||||
|
* @param <T> 泛型类型
|
||||||
|
* @return 反序列化后的对象
|
||||||
|
* @throws IOException 请求失败或 JSON 解析异常
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* 通用 HTTP 请求方法,支持任意 method、headers、请求体,支持泛型 TypeReference<T> 返回(Jackson)
|
||||||
|
*/
|
||||||
|
public <T> T exchange(String url, String method, Map<String, String> headers, Object body, TypeReference<T> typeRef) throws IOException {
|
||||||
|
if (headers == null) {
|
||||||
|
headers = Maps.newHashMap();
|
||||||
|
}
|
||||||
|
|
||||||
|
RequestBody requestBody = null;
|
||||||
|
boolean hasBody = body != null && !"GET".equalsIgnoreCase(method);
|
||||||
|
|
||||||
|
String contentType = headers.getOrDefault("Content-Type", "application/json; charset=utf-8");
|
||||||
|
|
||||||
|
if (hasBody) {
|
||||||
|
String text;
|
||||||
|
if (body instanceof String) {
|
||||||
|
text = (String) body;
|
||||||
|
} else {
|
||||||
|
text = objectMapper.writeValueAsString(body);
|
||||||
|
}
|
||||||
|
requestBody = RequestBody.create(text, MediaType.get(contentType));
|
||||||
|
}
|
||||||
|
|
||||||
|
Request.Builder builder = new Request.Builder().url(url);
|
||||||
|
headers.forEach(builder::addHeader);
|
||||||
|
|
||||||
|
switch (method.toUpperCase()) {
|
||||||
|
case "POST":
|
||||||
|
builder.post(requestBody != null ? requestBody : RequestBody.create(new byte[0]));
|
||||||
|
break;
|
||||||
|
case "PUT":
|
||||||
|
builder.put(requestBody != null ? requestBody : RequestBody.create(new byte[0]));
|
||||||
|
break;
|
||||||
|
case "DELETE":
|
||||||
|
builder.delete(requestBody != null ? requestBody : null);
|
||||||
|
break;
|
||||||
|
case "GET":
|
||||||
|
default:
|
||||||
|
builder.get();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
Request request = builder.build();
|
||||||
|
|
||||||
|
try (Response response = client.newCall(request).execute()) {
|
||||||
|
if (!response.isSuccessful()) {
|
||||||
|
throw new IOException("请求失败: " + response.code() + " - " + response.message());
|
||||||
|
}
|
||||||
|
|
||||||
|
String responseBody = response.body().string();
|
||||||
|
|
||||||
|
// 如果期望的是 String 类型,直接返回
|
||||||
|
if (typeRef.getType().getTypeName().equals(String.class.getTypeName())) {
|
||||||
|
return (T) responseBody;
|
||||||
|
}
|
||||||
|
|
||||||
|
return objectMapper.readValue(responseBody, typeRef);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行请求并解析为 Map
|
* 执行请求并解析为 Map
|
||||||
*/
|
*/
|
||||||
|
|||||||
@ -2,8 +2,10 @@ package com.knowledge.base.infrastructure.util;
|
|||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.*;
|
import java.nio.file.*;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
|
||||||
@ -71,5 +73,24 @@ public class LocalFileUtil {
|
|||||||
|
|
||||||
return targetFilePath;
|
return targetFilePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static List<File> listMarkdownFiles(File root) {
|
||||||
|
List<File> result = new ArrayList<>();
|
||||||
|
if (root == null || !root.exists()) return result;
|
||||||
|
|
||||||
|
File[] files = root.listFiles();
|
||||||
|
if (files == null) return result;
|
||||||
|
|
||||||
|
for (File file : files) {
|
||||||
|
if (file.isDirectory()) {
|
||||||
|
result.addAll(listMarkdownFiles(file));
|
||||||
|
} else if (file.getName().endsWith(".md")) {
|
||||||
|
result.add(file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,22 +1,23 @@
|
|||||||
package com.knowledge.base.infrastructure.util;
|
package com.knowledge.base.infrastructure.util;
|
||||||
|
|
||||||
import cn.hutool.core.thread.ThreadFactoryBuilder;
|
import cn.hutool.core.thread.ThreadFactoryBuilder;
|
||||||
|
import com.knowledge.base.infrastructure.monitor.ThreadPoolMonitorStarter;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
import java.util.concurrent.*;
|
import java.util.concurrent.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 通用线程池工具类
|
* 通用线程池工具类
|
||||||
* 支持外部自定义线程池传入,未传入时使用默认线程池
|
* 支持外部自定义线程池执行任务,并可选是否注册监控
|
||||||
* @author Luke
|
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
public class ThreadPoolUtil {
|
public class ThreadPoolUtil {
|
||||||
|
|
||||||
private static final int CORE_POOL_SIZE = Runtime.getRuntime().availableProcessors();
|
private static final int CORE_POOL_SIZE = Runtime.getRuntime().availableProcessors() + 1;
|
||||||
private static final int MAX_POOL_SIZE = CORE_POOL_SIZE * 2;
|
private static final int MAX_POOL_SIZE = CORE_POOL_SIZE * 2 + 1;
|
||||||
private static final int QUEUE_CAPACITY = 500;
|
private static final int QUEUE_CAPACITY = 500;
|
||||||
private static final long KEEP_ALIVE_TIME = 60L;
|
private static final long KEEP_ALIVE_TIME = 60L;
|
||||||
|
|
||||||
// 默认线程池
|
|
||||||
private static final ThreadPoolExecutor DEFAULT_THREAD_POOL = new ThreadPoolExecutor(
|
private static final ThreadPoolExecutor DEFAULT_THREAD_POOL = new ThreadPoolExecutor(
|
||||||
CORE_POOL_SIZE,
|
CORE_POOL_SIZE,
|
||||||
MAX_POOL_SIZE,
|
MAX_POOL_SIZE,
|
||||||
@ -28,35 +29,52 @@ public class ThreadPoolUtil {
|
|||||||
);
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行任务,使用默认线程池
|
* 提交默认线程池任务
|
||||||
*/
|
*/
|
||||||
public static void execute(Runnable task) {
|
public static void execute(Runnable task) {
|
||||||
DEFAULT_THREAD_POOL.execute(task);
|
DEFAULT_THREAD_POOL.execute(wrap(task, "default"));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行任务,允许调用方传入自定义线程池
|
* 提交自定义线程池任务(默认不监控)
|
||||||
* @param task Runnable
|
|
||||||
* @param executor 若为 null,则用默认线程池
|
|
||||||
*/
|
*/
|
||||||
public static void execute(Runnable task, ExecutorService executor) {
|
public static void execute(Runnable task, ExecutorService executor) {
|
||||||
if (executor != null) {
|
execute(task, executor, false, "custom");
|
||||||
executor.execute(task);
|
}
|
||||||
} else {
|
|
||||||
DEFAULT_THREAD_POOL.execute(task);
|
/**
|
||||||
|
* 提交任务(带监控选项 + 线程池名称)
|
||||||
|
*/
|
||||||
|
public static void execute(Runnable task, ExecutorService executor, boolean monitor, String poolName) {
|
||||||
|
if (executor == null) {
|
||||||
|
DEFAULT_THREAD_POOL.execute(wrap(task, "default"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
executor.execute(wrap(task, poolName));
|
||||||
|
if (monitor) {
|
||||||
|
ThreadPoolMonitorStarter.getInstance().register(poolName, executor);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private static Runnable wrap(Runnable task, String poolName) {
|
||||||
* 优雅关闭(默认线程池)
|
return () -> {
|
||||||
*/
|
String threadName = Thread.currentThread().getName();
|
||||||
|
try {
|
||||||
|
log.debug("[thread-pool][{}] 执行任务开始", poolName);
|
||||||
|
task.run();
|
||||||
|
log.debug("[thread-pool][{}] 执行任务结束", poolName);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("[thread-pool][{}] 执行异常", poolName, e);
|
||||||
|
} finally {
|
||||||
|
Thread.currentThread().setName(threadName); // 防止线程池复用导致名称混乱
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
public static void shutdownAndAwait() {
|
public static void shutdownAndAwait() {
|
||||||
shutdownAndAwait(DEFAULT_THREAD_POOL);
|
shutdownAndAwait(DEFAULT_THREAD_POOL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 优雅关闭(指定线程池)
|
|
||||||
*/
|
|
||||||
public static void shutdownAndAwait(ExecutorService executor) {
|
public static void shutdownAndAwait(ExecutorService executor) {
|
||||||
if (executor == null) return;
|
if (executor == null) return;
|
||||||
executor.shutdown();
|
executor.shutdown();
|
||||||
@ -70,9 +88,6 @@ public class ThreadPoolUtil {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取默认线程池(如需提交批量任务)
|
|
||||||
*/
|
|
||||||
public static ExecutorService getDefaultThreadPool() {
|
public static ExecutorService getDefaultThreadPool() {
|
||||||
return DEFAULT_THREAD_POOL;
|
return DEFAULT_THREAD_POOL;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -31,7 +31,7 @@ knowledge.base.redis.enable=false
|
|||||||
# 对象存储相关
|
# 对象存储相关
|
||||||
object.storage.use-uuid-prefix=true
|
object.storage.use-uuid-prefix=true
|
||||||
object.storage.public-buckets[0]=kbase
|
object.storage.public-buckets[0]=kbase
|
||||||
object.storage.public-buckets[1]=public-media
|
object.storage.public-buckets[1]=kbase-share-pub
|
||||||
|
|
||||||
# 大模型相关
|
# 大模型相关
|
||||||
llm.provider=anything
|
llm.provider=anything
|
||||||
|
|||||||
@ -0,0 +1,65 @@
|
|||||||
|
package com.knowledge.base.application.service;
|
||||||
|
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
|
import com.knowledge.base.infrastructure.config.DynamicConfig;
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
|
import com.knowledge.base.infrastructure.south.llm.AnythingLLMServiceImpl;
|
||||||
|
import com.knowledge.base.infrastructure.south.llm.AnythingLLMServiceTest;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@SpringBootTest(properties = {
|
||||||
|
"spring.profiles.active=dev-windows"
|
||||||
|
})
|
||||||
|
@Slf4j
|
||||||
|
public class LLMAppServiceImplTest {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private AnythingLLMServiceImpl llmService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private LLMAppService llmAppService;
|
||||||
|
|
||||||
|
private String token;
|
||||||
|
|
||||||
|
private Map<String, Object> llmRepo = Maps.newHashMap();
|
||||||
|
|
||||||
|
public String getSlug() {
|
||||||
|
return AnythingLLMServiceTest.DEFAULT_TEST_SLUG_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
public void setUp() {
|
||||||
|
try {
|
||||||
|
this.token = llmService.fetchToken("lukeye@6");
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("获取 token 失败: {}", e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPinDocsByKeywords() throws Exception {
|
||||||
|
try {
|
||||||
|
List<WorkspaceAttachment> result = llmAppService.pinDocsByKeywords(token, getSlug(), Lists.newArrayList("商业"), "[[商业]] 的本质是什么?");
|
||||||
|
log.info("result: {}", JSONUtil.toJsonStr(result));
|
||||||
|
|
||||||
|
result.forEach(doc -> {
|
||||||
|
try {
|
||||||
|
llmService.updatePin(token, getSlug(), doc.getDocpath(), false);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("unpin失败: {}", doc, e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 pinDocsByKeywords 失败!: {}", e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,159 @@
|
|||||||
|
package com.knowledge.base.infrastructure.south.llm;
|
||||||
|
|
||||||
|
import cn.hutool.core.lang.TypeReference;
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
|
import com.knowledge.base.infrastructure.north.dto.llm.WorkspaceAttachment;
|
||||||
|
import com.knowledge.base.infrastructure.util.BeanTool;
|
||||||
|
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Luke.ye
|
||||||
|
* @date 2025/7/7 06:57
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@SpringBootTest(properties = {
|
||||||
|
"spring.profiles.active=dev-mac"
|
||||||
|
})
|
||||||
|
public class AnythingLLMServiceTest {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private AnythingLLMServiceImpl llmService;
|
||||||
|
|
||||||
|
private String token;
|
||||||
|
|
||||||
|
public static final String DEFAULT_TEST_SLUG_ID = "e6fa5a5a-1220-4eb3-979f-ee37b3711ba2";
|
||||||
|
|
||||||
|
private Map<String, Object> llmRepo = Maps.newHashMap();
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
public void setUp() {
|
||||||
|
try {
|
||||||
|
this.token = llmService.fetchToken("lukeye@6");
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("获取 token 失败: {}", e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSlug() {
|
||||||
|
return DEFAULT_TEST_SLUG_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testImportObsidianToLLM() {
|
||||||
|
try {
|
||||||
|
// 使用 ClassLoader 获取资源目录(obsidian-test)
|
||||||
|
URL resource = getClass().getClassLoader().getResource("obsidian-test");
|
||||||
|
assertNotNull(resource, "obsidian-test 目录不存在,检查 test/resources 路径是否正确");
|
||||||
|
|
||||||
|
File vaultDir = new File(resource.toURI());
|
||||||
|
assertTrue(vaultDir.isDirectory(), "obsidian-test 应为目录");
|
||||||
|
|
||||||
|
Map<String, Object> result = llmService.importObsidianToLLM(token, vaultDir);
|
||||||
|
llmRepo.put("test-vault-name", BeanTool.getMapV("data.destination", result));
|
||||||
|
log.info("导入结果: {}, llmRepo: {}", JSONUtil.toJsonStr(result), JSONUtil.toJsonStr(llmRepo));
|
||||||
|
testRemoveFolderFromLLM();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 importObsidianToLLM 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRemoveFolderFromLLM() {
|
||||||
|
try {
|
||||||
|
String s = llmService.removeFolderFromLLM(token, (String) llmRepo.getOrDefault("test-vault-name", "obsidian-vault-obsidian-test-e511"));
|
||||||
|
log.info("removeResult: {}", s);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 removeFolderFromLLM 异常", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetLocalFileItems() {
|
||||||
|
try {
|
||||||
|
String slug = getSlug();
|
||||||
|
Map<String, Object> result = llmService.getLocalFileItems(token, slug, "custom-documents");
|
||||||
|
llmRepo.put("local-files", result);
|
||||||
|
log.info("本地文件项:{}", JSONUtil.toJsonStr(result));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 getLocalFileItems 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUpdateEmbeddings() {
|
||||||
|
try {
|
||||||
|
String slug = getSlug();
|
||||||
|
testGetLocalFileItems();
|
||||||
|
Map localFilesMap = (Map)llmRepo.getOrDefault("local-files", Maps.newHashMap());
|
||||||
|
String folderName = BeanTool.getMapV("name", localFilesMap);
|
||||||
|
List<Map> items = BeanTool.getMapV("items", localFilesMap, new TypeReference<List<Map>>() {});
|
||||||
|
List<String> files = Lists.newArrayList();
|
||||||
|
items.stream().forEach(e -> {
|
||||||
|
String fileName = (String) e.get("name");
|
||||||
|
if(StrUtil.isNotBlank(fileName)) {
|
||||||
|
files.add(folderName + "/" + fileName);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
List<String> deletes = Arrays.asList();
|
||||||
|
Map result = llmService.updateEmbeddings(token, slug, files, deletes);
|
||||||
|
log.info("result: {}", JSONUtil.toJsonStr(result));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 updateEmbeddings 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUpdatePin() {
|
||||||
|
try {
|
||||||
|
String slug = getSlug();
|
||||||
|
String path = "custom-documents/index.md-76d75c91-d4fe-4fa3-bc4b-0359c47ccae5.json";
|
||||||
|
|
||||||
|
String result = llmService.updatePin(token, slug, path, false);
|
||||||
|
log.info("result: {}", result);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 updatePin 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchAttachments() {
|
||||||
|
try {
|
||||||
|
String slug = getSlug();
|
||||||
|
List<WorkspaceAttachment> attachments = llmService.fetchAttachments(token, slug);
|
||||||
|
attachments.forEach(e -> log.info("Attachment: {} -> {} -> {} ", e.getDocpath(), e.getUrl(), e.isPinned()));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 fetchAttachments 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStreamAnswer() {
|
||||||
|
try {
|
||||||
|
WriterAdapter writer = line -> log.info("回答流:{}", line);
|
||||||
|
|
||||||
|
Map<String, Object> params = new HashMap<>();
|
||||||
|
params.put("slug", getSlug());
|
||||||
|
// llmService.streamAnswer(token, "介绍一下黄金圈法则", params, writer);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("测试 streamAnswer 异常", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
21
src/test/resources/obsidian-test/demo.md
Normal file
21
src/test/resources/obsidian-test/demo.md
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
修订记录
|
||||||
|
|
||||||
|
| 版本 | 时间 | 修订人 | 改动点 |
|
||||||
|
|:-----:|:----------:|:-------:|:---------------------------------:|
|
||||||
|
| 1.0.0 | 2025-06-05 | Luke.Ye | 添加修订记录(新增token校验接口) |
|
||||||
|
| 1.0.1 | 2025-06-06 | Luke.Ye | 重构登录Token校验逻辑, |
|
||||||
|
| 1.0.2 | 2025-06-09 | Luke.Ye | 添加用户角色相关逻辑,重构代码 |
|
||||||
|
| 1.0.3 | 2025-06-16 | Luke.Ye | 新增文档上传至对象存储接口 |
|
||||||
|
| 1.0.4 | 2025-06-17 | Luke.Ye | 上传文档记录入库 |
|
||||||
|
| 1.0.5 | 2025-06-19 | Luke.Ye | 文档导入ES逻辑优化 |
|
||||||
|
| 1.0.6 | 2025-06-20 | Luke.Ye | 完成对象存储删除接口,优化部分代码 |
|
||||||
|
| 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |
|
||||||
|
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
|
||||||
|
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口(ob仓库监听发生删除事件,则同步删除ES) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 开发顺序
|
||||||
|
|
||||||
|
PO ---> Mapper ---> iface ---> persistence
|
||||||
39
src/test/resources/obsidian-test/index.md
Normal file
39
src/test/resources/obsidian-test/index.md
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
|
||||||
|
|
||||||
|
### 提示词权重
|
||||||
|
提示词权重,多个可以级联,权重可以低,但不能过高
|
||||||
|
- `[]`:0.9,最多可以套3层,最多0.729倍
|
||||||
|
- `()`:1.1,最多可以套3层,最多1.331倍。也可以直接将*权重(范围需要控制在0.3~1.5)* 放在提示词后面,例如:`(garden:1.5)`,
|
||||||
|
- `{}`:1.05,最多可以套3层,最多1.15倍
|
||||||
|
- `<>`:调用Lora,
|
||||||
|
|
||||||
|
|
||||||
|
### 提示词顺序
|
||||||
|
越靠前,权重越大
|
||||||
|
|
||||||
|
No1:画质词/画风词,比如 `4k, masterpiece,high quality, highly detailed`
|
||||||
|
No2:主体,比如 `1girl, blue dress`
|
||||||
|
No3:环境/场景/构图,比如 `garden, white background`
|
||||||
|
No4:lora
|
||||||
|
|
||||||
|
### 提示词污染 & 融合
|
||||||
|
污染:不同的提示词(尤其是颜色)会相互渗透,要使用break来隔开提示词,
|
||||||
|
|
||||||
|
融合:`1gril AND cat` 或者 `1girl_cat` 或者 `[1gril|cat]` ,猫女
|
||||||
|
|
||||||
|
### 画的时间
|
||||||
|
|
||||||
|
`{1girl:garden:0.7}`:70%的时间画girl,后面30%画花园
|
||||||
|
|
||||||
|
### 起手式
|
||||||
|
|
||||||
|
正向:4k、8k、masterpiece
|
||||||
|
|
||||||
|
反向:blur(模糊)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 图生图
|
||||||
|
重绘幅度:建议 `0.4~0.7`
|
||||||
Loading…
x
Reference in New Issue
Block a user