lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化llm接口

Browse Source
This commit is contained in:
Luke.Ye 2025-07-07 17:46:26 +08:00
parent 53467e8545
commit da85233841
7 changed files with 274 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Readme.md
View File

@ -13,9 +13,15 @@
| 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口（ob仓库监听发生删除事件，则同步删除ES） |
| 1.1.2 | 2025-07-07 | Luke.Ye | 增加llm后端接口 |
## 开发顺序
PO ---> Mapper ---> iface ---> persistence
PO ---> Mapper ---> iface ---> persistence
## llm接口调用顺序
- importObsidianToLLM(导入本地文档) ---> updateEmbeddings（将本地文档嵌入工作区） ---> updatePin（是否pin住） ---> streamAnswer(开始问答)
- removeFolderFromLLM（移除文档）
- fetchAttachments（获取当前工作区下已经嵌入的附件）

10
src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java
View File

@ -9,7 +9,7 @@ import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import lombok.RequiredArgsConstructor;
import com.knowledge.base.infrastructure.util.BeanConvertUtil;
import com.knowledge.base.infrastructure.util.BeanTool;
import org.springframework.stereotype.Service;
import java.util.List;
@ -57,13 +57,13 @@ public class FileDomainServiceImpl implements FileDomainService {
public PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader) {
PageResult<OSRecord> osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader);
return new PageResult<>(BeanConvertUtil.convertList(osRecordPageResult.getList(), OSRecordDO.class),
return new PageResult<>(BeanTool.convertList(osRecordPageResult.getList(), OSRecordDO.class),
osRecordPageResult.getTotal());
}
@Override
public void save(OSRecordDO osRecordDO) {
osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class));
osRecordRepository.save(BeanTool.convert(osRecordDO, OSRecord.class));
}
@Override
@ -72,7 +72,7 @@ public class FileDomainServiceImpl implements FileDomainService {
return Optional.empty();
}
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
return osRecordOpt.map(osRecord -> BeanConvertUtil.convert(osRecord, OSRecordDO.class));
return osRecordOpt.map(osRecord -> BeanTool.convert(osRecord, OSRecordDO.class));
}
@Override
@ -87,7 +87,7 @@ public class FileDomainServiceImpl implements FileDomainService {
@Override
public List<OSRecordDO> batchQueryOSRecord(List<Long> recordIds) {
return BeanConvertUtil.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class);
return BeanTool.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class);
}
@Override

116
src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java
View File

@ -2,9 +2,10 @@ package com.knowledge.base.infrastructure.south.llm;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.util.HttpHelper;
@ -22,7 +23,9 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Stream;
@Slf4j
@Service
@ -95,48 +98,83 @@ public class AnythingLLMServiceImpl implements LLMService {
}
public void importObsidianToLLM(String llmToken, List<File> mdFiles, String vaultBasePath) {
String url = baseUrl + "/api/ext/obsidian/vault";
List<Map<String, Object>> fileList = new ArrayList<>();
for (File file : mdFiles) {
if (!file.getName().endsWith(".md")) continue;
try {
String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
String relativePath = vaultBasePath == null ? file.getName() :
file.getAbsolutePath().replace(new File(vaultBasePath).getAbsolutePath(), "")
.replaceFirst("^[/\\\\]", "");
Map<String, Object> fileMap = new HashMap<>();
fileMap.put("name", file.getName());
fileMap.put("path", relativePath.replace(File.separator, "/"));
fileMap.put("content", content);
fileList.add(fileMap);
} catch (IOException e) {
log.warn("读取文件失败: {}", file.getAbsolutePath(), e);
}
public Map<String, Object> importObsidianToLLM(String llmToken, File vaultDir) {
if (!vaultDir.exists() || !vaultDir.isDirectory()) {
throw new IllegalArgumentException("无效的 Obsidian vault 目录: " + vaultDir.getAbsolutePath());
}
String url = baseUrl + "/api/ext/obsidian/vault";
List<Map<String, Object>> fileList = new ArrayList<>();
String vaultName = vaultDir.getName();
try (Stream<Path> pathStream = Files.walk(vaultDir.toPath())) {
pathStream.filter(Files::isRegularFile)
.filter(p -> p.toString().endsWith(".md"))
.forEach(path -> {
try {
String content = Files.readString(path, StandardCharsets.UTF_8);
Path relativePath = vaultDir.toPath().relativize(path);
String normalizedPath = vaultName + "/" + relativePath.toString().replace(File.separator, "/");
Map<String, Object> fileMap = new LinkedHashMap<>();
fileMap.put("name", path.getFileName().toString());
fileMap.put("path", normalizedPath);
fileMap.put("content", content);
fileList.add(fileMap);
} catch (IOException e) {
log.warn("读取文件失败: {}", path.toAbsolutePath(), e);
}
});
} catch (IOException e) {
throw new RuntimeException("遍历 Obsidian vault 目录失败: " + e.getMessage(), e);
}
// 构建 JSON 请求体（纯字符串）
Map<String, Object> requestMap = Map.of("files", fileList);
String requestBodyStr = JSONUtil.toJsonStr(requestMap); // 或 objectMapper.writeValueAsString(...)
Map<String, String> headers = new HashMap<>();
headers.put("Authorization", "Bearer " + llmToken);
Map<String, Object> body = new HashMap<>();
body.put("files", fileList);
headers.put("Content-Type", "text/plain;charset=UTF-8");
try {
httpHelper.exchange(
return httpHelper.exchange(
url,
"POST",
headers,
body,
new TypeReference<Map<String, Object>>() {} // 可忽略返回内容
requestBodyStr,
new TypeReference<Map<String, Object>>() {}
);
} catch (IOException e) {
throw new RuntimeException("上传 Obsidian 文件失败", e);
throw new RuntimeException("上传 Obsidian 文件失败: " + e.getMessage(), e);
}
}
public String removeFolderFromLLM(String llmToken, String folderName) {
String url = baseUrl + "/api/system/remove-folder";
Map<String, Object> body = Map.of("name", folderName);
Map<String, String> headers = new HashMap<>();
headers.put("Authorization", "Bearer " + llmToken);
headers.put("Content-Type", "application/json");
try {
return httpHelper.exchange(
url,
"DELETE",
headers,
body,
new TypeReference<String>() {}
);
} catch (IOException e) {
throw new RuntimeException("删除文件夹失败: " + folderName, e);
}
}
@Override
public Map<String, String> fetchAttachments(String llmToken, String workspaceSlug) {
String url = baseUrl + "/api/workspace/" + workspaceSlug;
@ -182,7 +220,7 @@ public class AnythingLLMServiceImpl implements LLMService {
}
public void updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus) {
public String updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus) {
String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-pin";
Map<String, String> headers = new HashMap<>();
headers.put("Authorization", "Bearer " + llmToken);
@ -192,24 +230,23 @@ public class AnythingLLMServiceImpl implements LLMService {
body.put("pinStatus", pinStatus);
try {
httpHelper.exchange(
return httpHelper.exchange(
url,
"POST",
headers,
body,
new TypeReference<Map<String, Object>>() {} // 可忽略返回值，仅验证200
new TypeReference<String>() {}
);
} catch (IOException e) {
throw new RuntimeException("设置 pin 状态失败: " + e.getMessage(), e);
}
}
public List<Map<String, Object>> getLocalFileItems(String llmToken, String workspaceSlug, String keyword) {
public Map<String, Object> getLocalFileItems(String llmToken, String workspaceSlug, String keyword) {
String url = baseUrl + "/api/system/local-files";
Map<String, String> headers = new HashMap<>();
headers.put("Authorization", "Bearer " + llmToken);
headers.put("Referer", baseUrl + "/workspace/" + workspaceSlug);
try {
Map<String, Object> response = httpHelper.exchange(
@ -230,23 +267,20 @@ public class AnythingLLMServiceImpl implements LLMService {
Map folder = (Map) folderObj;
String name = (String) folder.get("name");
if (name != null && name.contains(keyword)) {
Object matchedItems = folder.get("items");
if (matchedItems instanceof List) {
return (List<Map<String, Object>>) matchedItems;
}
return folder;
}
}
}
}
}
return Lists.newArrayList();
return Maps.newHashMap();
} catch (IOException e) {
throw new RuntimeException("获取本地文件失败：" + e.getMessage(), e);
}
}
public void updateEmbeddings(String llmToken, String workspaceSlug, List<String> adds, List<String> deletes) {
public Map<String, Object> updateEmbeddings(String llmToken, String workspaceSlug, List<String> adds, List<String> deletes) {
String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-embeddings";
Map<String, String> headers = new HashMap<>();
@ -257,7 +291,7 @@ public class AnythingLLMServiceImpl implements LLMService {
body.put("deletes", deletes);
try {
httpHelper.exchange(
return httpHelper.exchange(
url,
"POST",
headers,

35
src/main/java/com/knowledge/base/infrastructure/util/BeanConvertUtil.java
View File

@ -1,35 +0,0 @@
package com.knowledge.base.infrastructure.util;
import cn.hutool.core.collection.CollectionUtil;
import org.springframework.beans.BeanUtils;
import java.util.List;
import java.util.stream.Collectors;
public class BeanConvertUtil {
/**
* 将 source 转换为指定类型的目标对象
*/
public static <S, T> T convert(S source, Class<T> targetClass) {
if (source == null) return null;
try {
T target = targetClass.getDeclaredConstructor().newInstance();
BeanUtils.copyProperties(source, target);
return target;
} catch (Exception e) {
throw new RuntimeException("BeanConvertUtil.convert error", e);
}
}
/**
* 批量转换：List<S> → List<T>
*/
public static <S, T> List<T> convertList(List<S> sourceList, Class<T> targetClass) {
if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList();
return sourceList.stream()
.map(source -> convert(source, targetClass))
.collect(Collectors.toList());
}
}

90
src/main/java/com/knowledge/base/infrastructure/util/BeanTool.java Normal file
View File

@ -0,0 +1,90 @@
package com.knowledge.base.infrastructure.util;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.lang.TypeReference;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import org.springframework.beans.BeanUtils;
import java.util.List;
import java.util.stream.Collectors;
public class BeanTool {
/**
* 将 source 转换为指定类型的目标对象
*/
public static <S, T> T convert(S source, Class<T> targetClass) {
if (source == null) return null;
try {
T target = targetClass.getDeclaredConstructor().newInstance();
BeanUtils.copyProperties(source, target);
return target;
} catch (Exception e) {
throw new RuntimeException("BeanConvertUtil.convert error", e);
}
}
/**
* 批量转换：List<S> → List<T>
*/
public static <S, T> List<T> convertList(List<S> sourceList, Class<T> targetClass) {
if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList();
return sourceList.stream()
.map(source -> convert(source, targetClass))
.collect(Collectors.toList());
}
/**
* 获取嵌套字段的值（路径支持.分隔 & 数组索引）
* @param map 原始 Map
* @param key 路径，如 data.destination 或 data.results[0].file
* @param <T> 返回类型
* @return 值（可为 null）
*/
@SuppressWarnings("unchecked")
public static <T> T getMapV(String key, Object map) {
Object value = BeanUtil.getProperty(map, key);
return (T) value;
}
/**
* 获取嵌套字段值并转换为指定类型（通过 JSON 编解码转换）
* @param key 路径
* @param map 原始 Map
* @param clazz 目标类型
* @param <T> 类型参数
* @return 指定类型的值，转换失败或字段不存在时返回 null
*/
public static <T> T getMapV(String key, Object map, Class<T> clazz) {
Object value = BeanUtil.getProperty(map, key);
if (ObjectUtil.isEmpty(value)) {
return null;
}
return JSONUtil.toBean(JSONUtil.toJsonStr(value), clazz);
}
/**
* 获取嵌套字段并反序列化为指定泛型类型
*
* @param key 字段路径（支持嵌套，如 "data.items"）
* @param map 原始 Map 对象
* @param typeReference 泛型类型引用（如 new TypeReference<List<Map<String, Object>>>(){}）
* @return 类型化对象或 null
*/
public static <T> T getMapV(String key, Object map, TypeReference<T> typeReference) {
Object value = BeanUtil.getProperty(map, key);
if (ObjectUtil.isEmpty(value)) {
return null;
}
// value 已经是对象，如果是 Map/List 类型，可直接转换；否则先转 JSON 字符串再解析
if (value instanceof CharSequence) {
return JSONUtil.toBean((String) value, typeReference, false);
} else {
return JSONUtil.toBean(JSONUtil.toJsonStr(value), typeReference, false);
}
}
}

39
src/main/java/com/knowledge/base/infrastructure/util/HttpHelper.java
View File

@ -2,6 +2,7 @@ package com.knowledge.base.infrastructure.util;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.springframework.stereotype.Component;
@ -117,19 +118,27 @@ public class HttpHelper {
* 通用 HTTP 请求方法，支持任意 method、headers、请求体，支持泛型 TypeReference<T> 返回（Jackson）
*/
public <T> T exchange(String url, String method, Map<String, String> headers, Object body, TypeReference<T> typeRef) throws IOException {
if (headers == null) {
headers = Maps.newHashMap();
}
RequestBody requestBody = null;
if (body != null) {
String json = objectMapper.writeValueAsString(body);
requestBody = RequestBody.create(json, MediaType.get("application/json; charset=utf-8"));
boolean hasBody = body != null && !"GET".equalsIgnoreCase(method);
String contentType = headers.getOrDefault("Content-Type", "application/json; charset=utf-8");
if (hasBody) {
String text;
if (body instanceof String) {
text = (String) body;
} else {
text = objectMapper.writeValueAsString(body);
}
requestBody = RequestBody.create(text, MediaType.get(contentType));
}
Request.Builder builder = new Request.Builder().url(url);
if (headers != null) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
builder.addHeader(entry.getKey(), entry.getValue());
}
}
headers.forEach(builder::addHeader);
switch (method.toUpperCase()) {
case "POST":
@ -153,13 +162,21 @@ public class HttpHelper {
if (!response.isSuccessful()) {
throw new IOException("请求失败: " + response.code() + " - " + response.message());
}
String json = response.body().string();
return objectMapper.readValue(json, typeRef);
String responseBody = response.body().string();
// 如果期望的是 String 类型，直接返回
if (typeRef.getType().getTypeName().equals(String.class.getTypeName())) {
return (T) responseBody;
}
return objectMapper.readValue(responseBody, typeRef);
}
}
/**
* 执行请求并解析为 Map
*/

98
src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java
View File

@ -1,7 +1,12 @@
package com.knowledge.base.infrastructure.south.llm;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.util.HttpHelper;
import cn.hutool.core.lang.TypeReference;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.util.BeanTool;
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach;
@ -10,8 +15,12 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.File;
import java.net.URL;
import java.util.*;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* @author Luke.ye
* @date 2025/7/7 06:57
@ -25,14 +34,10 @@ public class AnythingLLMServiceTest {
@Autowired
private AnythingLLMServiceImpl llmService;
@Autowired
private UserCacheService userCacheService;
@Autowired
private HttpHelper httpHelper;
private String token;
private Map<String, Object> llmRepo = Maps.newHashMap();
@BeforeEach
public void setUp() {
try {
@ -42,26 +47,70 @@ public class AnythingLLMServiceTest {
}
}
public static String getSlug() {
return ConstantConfig.DEFAULT_SLUG_ID;
}
@Test
public void testImportObsidianToLLM() {
try {
File file1 = new File("src/test/resources/obsidian-test/index.md");
File file2 = new File("src/test/resources/obsidian-test/demo.md");
// 使用 ClassLoader 获取资源目录（obsidian-test）
URL resource = getClass().getClassLoader().getResource("obsidian-test");
assertNotNull(resource, "obsidian-test 目录不存在，检查 test/resources 路径是否正确");
llmService.importObsidianToLLM(token, Arrays.asList(file1, file2), "src/test/resources/obsidian-test");
File vaultDir = new File(resource.toURI());
assertTrue(vaultDir.isDirectory(), "obsidian-test 应为目录");
Map<String, Object> result = llmService.importObsidianToLLM(token, vaultDir);
llmRepo.put("test-vault-name", BeanTool.getMapV("data.destination", result));
log.info("导入结果: {}, llmRepo: {}", JSONUtil.toJsonStr(result), JSONUtil.toJsonStr(llmRepo));
// testRemoveFolderFromLLM();
} catch (Exception e) {
log.warn("测试 importObsidianToLLM 异常", e);
}
}
@Test
public void testRemoveFolderFromLLM() {
try {
String s = llmService.removeFolderFromLLM(token, (String) llmRepo.getOrDefault("test-vault-name", "obsidian-vault-obsidian-test-e511"));
log.info("removeResult: {}", s);
} catch (Exception e) {
log.warn("测试 removeFolderFromLLM 异常", e);
}
}
@Test
public void testGetLocalFileItems() {
try {
String slug = getSlug();
Map<String, Object> result = llmService.getLocalFileItems(token, slug, "test");
llmRepo.put("local-files", result);
log.info("本地文件项：{}", JSONUtil.toJsonStr(result));
} catch (Exception e) {
log.warn("测试 getLocalFileItems 异常", e);
}
}
@Test
public void testUpdateEmbeddings() {
try {
String slug = "your-workspace-slug";
List<String> adds = Collections.singletonList("public/xx.md");
String slug = getSlug();
testGetLocalFileItems();
Map localFilesMap = (Map)llmRepo.getOrDefault("local-files", Maps.newHashMap());
String folderName = BeanTool.getMapV("name", localFilesMap);
List<Map> items = BeanTool.getMapV("items", localFilesMap, new TypeReference<List<Map>>() {});
List<String> files = Lists.newArrayList();
items.stream().forEach(e -> {
String fileName = (String) e.get("name");
if(StrUtil.isNotBlank(fileName)) {
files.add(folderName + "/" + fileName);
}
});
List<String> deletes = Arrays.asList();
llmService.updateEmbeddings(token, slug, adds, deletes);
Map result = llmService.updateEmbeddings(token, slug, files, deletes);
log.info("result: {}", JSONUtil.toJsonStr(result));
} catch (Exception e) {
log.warn("测试 updateEmbeddings 异常", e);
}
@ -70,10 +119,11 @@ public class AnythingLLMServiceTest {
@Test
public void testUpdatePin() {
try {
String slug = "your-workspace-slug";
String path = "public/xx.md";
String slug = getSlug();
String path = "custom-documents/index.md-76d75c91-d4fe-4fa3-bc4b-0359c47ccae5.json";
llmService.updatePin(token, slug, path, true);
String result = llmService.updatePin(token, slug, path, false);
log.info("result: {}", result);
} catch (Exception e) {
log.warn("测试 updatePin 异常", e);
}
@ -82,7 +132,7 @@ public class AnythingLLMServiceTest {
@Test
public void testFetchAttachments() {
try {
String slug = "your-workspace-slug";
String slug = getSlug();
Map<String, String> attachments = llmService.fetchAttachments(token, slug);
attachments.forEach((k, v) -> log.info("Attachment: {} -> {}", k, v));
} catch (Exception e) {
@ -90,16 +140,6 @@ public class AnythingLLMServiceTest {
}
}
@Test
public void testGetLocalFileItems() {
try {
String slug = "your-workspace-slug";
List<Map<String, Object>> result = llmService.getLocalFileItems(token, slug, "导出");
log.info("本地文件项：{}", result);
} catch (Exception e) {
log.warn("测试 getLocalFileItems 异常", e);
}
}
@Test
public void testStreamAnswer() {