diff --git a/Readme.md b/Readme.md index b1668b6..63c6971 100644 --- a/Readme.md +++ b/Readme.md @@ -13,9 +13,15 @@ | 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt | | 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 | | 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口（ob仓库监听发生删除事件，则同步删除ES） | +| 1.1.2 | 2025-07-07 | Luke.Ye | 增加llm后端接口 | ## 开发顺序 +PO ---> Mapper ---> iface ---> persistence -PO ---> Mapper ---> iface ---> persistence \ No newline at end of file + +## llm接口调用顺序 +- importObsidianToLLM(导入本地文档) ---> updateEmbeddings（将本地文档嵌入工作区） ---> updatePin（是否pin住） ---> streamAnswer(开始问答) +- removeFolderFromLLM（移除文档） +- fetchAttachments（获取当前工作区下已经嵌入的附件） \ No newline at end of file diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java index 23f140c..5c59f1b 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java @@ -9,7 +9,7 @@ import com.knowledge.base.domain.doc.repository.po.File; import com.knowledge.base.domain.doc.repository.po.OSRecord; import com.knowledge.base.domain.doc.service.iface.FileDomainService; import lombok.RequiredArgsConstructor; -import com.knowledge.base.infrastructure.util.BeanConvertUtil; +import com.knowledge.base.infrastructure.util.BeanTool; import org.springframework.stereotype.Service; import java.util.List; @@ -57,13 +57,13 @@ public class FileDomainServiceImpl implements FileDomainService { public PageResult pageQueryOSRecord(int page, int size, String uploader) { PageResult osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader); - return new PageResult<>(BeanConvertUtil.convertList(osRecordPageResult.getList(), OSRecordDO.class), + return new PageResult<>(BeanTool.convertList(osRecordPageResult.getList(), OSRecordDO.class), osRecordPageResult.getTotal()); } @Override public void save(OSRecordDO osRecordDO) { - osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class)); + osRecordRepository.save(BeanTool.convert(osRecordDO, OSRecord.class)); } @Override @@ -72,7 +72,7 @@ public class FileDomainServiceImpl implements FileDomainService { return Optional.empty(); } Optional osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath); - return osRecordOpt.map(osRecord -> BeanConvertUtil.convert(osRecord, OSRecordDO.class)); + return osRecordOpt.map(osRecord -> BeanTool.convert(osRecord, OSRecordDO.class)); } @Override @@ -87,7 +87,7 @@ public class FileDomainServiceImpl implements FileDomainService { @Override public List batchQueryOSRecord(List recordIds) { - return BeanConvertUtil.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class); + return BeanTool.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class); } @Override diff --git a/src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java b/src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java index 72274ae..9377f66 100644 --- a/src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java +++ b/src/main/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceImpl.java @@ -2,9 +2,10 @@ package com.knowledge.base.infrastructure.south.llm; import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.knowledge.base.infrastructure.cache.iface.UserCacheService; import com.knowledge.base.infrastructure.config.ConstantConfig; import com.knowledge.base.infrastructure.util.HttpHelper; @@ -22,7 +23,9 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; +import java.util.stream.Stream; @Slf4j @Service @@ -95,48 +98,83 @@ public class AnythingLLMServiceImpl implements LLMService { } - - public void importObsidianToLLM(String llmToken, List mdFiles, String vaultBasePath) { - String url = baseUrl + "/api/ext/obsidian/vault"; - - List> fileList = new ArrayList<>(); - for (File file : mdFiles) { - if (!file.getName().endsWith(".md")) continue; - try { - String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); - String relativePath = vaultBasePath == null ? file.getName() : - file.getAbsolutePath().replace(new File(vaultBasePath).getAbsolutePath(), "") - .replaceFirst("^[/\\\\]", ""); - - Map fileMap = new HashMap<>(); - fileMap.put("name", file.getName()); - fileMap.put("path", relativePath.replace(File.separator, "/")); - fileMap.put("content", content); - fileList.add(fileMap); - } catch (IOException e) { - log.warn("读取文件失败: {}", file.getAbsolutePath(), e); - } + public Map importObsidianToLLM(String llmToken, File vaultDir) { + if (!vaultDir.exists() || !vaultDir.isDirectory()) { + throw new IllegalArgumentException("无效的 Obsidian vault 目录: " + vaultDir.getAbsolutePath()); } + String url = baseUrl + "/api/ext/obsidian/vault"; + List> fileList = new ArrayList<>(); + String vaultName = vaultDir.getName(); + + try (Stream pathStream = Files.walk(vaultDir.toPath())) { + pathStream.filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".md")) + .forEach(path -> { + try { + String content = Files.readString(path, StandardCharsets.UTF_8); + Path relativePath = vaultDir.toPath().relativize(path); + String normalizedPath = vaultName + "/" + relativePath.toString().replace(File.separator, "/"); + + Map fileMap = new LinkedHashMap<>(); + fileMap.put("name", path.getFileName().toString()); + fileMap.put("path", normalizedPath); + fileMap.put("content", content); + + fileList.add(fileMap); + } catch (IOException e) { + log.warn("读取文件失败: {}", path.toAbsolutePath(), e); + } + }); + } catch (IOException e) { + throw new RuntimeException("遍历 Obsidian vault 目录失败: " + e.getMessage(), e); + } + + // 构建 JSON 请求体（纯字符串） + Map requestMap = Map.of("files", fileList); + String requestBodyStr = JSONUtil.toJsonStr(requestMap); // 或 objectMapper.writeValueAsString(...) + Map headers = new HashMap<>(); headers.put("Authorization", "Bearer " + llmToken); - - Map body = new HashMap<>(); - body.put("files", fileList); + headers.put("Content-Type", "text/plain;charset=UTF-8"); try { - httpHelper.exchange( + return httpHelper.exchange( url, "POST", headers, - body, - new TypeReference>() {} // 可忽略返回内容 + requestBodyStr, + new TypeReference>() {} ); } catch (IOException e) { - throw new RuntimeException("上传 Obsidian 文件失败", e); + throw new RuntimeException("上传 Obsidian 文件失败: " + e.getMessage(), e); } } + public String removeFolderFromLLM(String llmToken, String folderName) { + String url = baseUrl + "/api/system/remove-folder"; + + Map body = Map.of("name", folderName); + + Map headers = new HashMap<>(); + headers.put("Authorization", "Bearer " + llmToken); + headers.put("Content-Type", "application/json"); + + try { + return httpHelper.exchange( + url, + "DELETE", + headers, + body, + new TypeReference() {} + ); + } catch (IOException e) { + throw new RuntimeException("删除文件夹失败: " + folderName, e); + } + } + + + @Override public Map fetchAttachments(String llmToken, String workspaceSlug) { String url = baseUrl + "/api/workspace/" + workspaceSlug; @@ -182,7 +220,7 @@ public class AnythingLLMServiceImpl implements LLMService { } - public void updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus) { + public String updatePin(String llmToken, String workspaceSlug, String docPath, boolean pinStatus) { String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-pin"; Map headers = new HashMap<>(); headers.put("Authorization", "Bearer " + llmToken); @@ -192,24 +230,23 @@ public class AnythingLLMServiceImpl implements LLMService { body.put("pinStatus", pinStatus); try { - httpHelper.exchange( + return httpHelper.exchange( url, "POST", headers, body, - new TypeReference>() {} // 可忽略返回值，仅验证200 + new TypeReference() {} ); } catch (IOException e) { throw new RuntimeException("设置 pin 状态失败: " + e.getMessage(), e); } } - public List> getLocalFileItems(String llmToken, String workspaceSlug, String keyword) { + public Map getLocalFileItems(String llmToken, String workspaceSlug, String keyword) { String url = baseUrl + "/api/system/local-files"; Map headers = new HashMap<>(); headers.put("Authorization", "Bearer " + llmToken); - headers.put("Referer", baseUrl + "/workspace/" + workspaceSlug); try { Map response = httpHelper.exchange( @@ -230,23 +267,20 @@ public class AnythingLLMServiceImpl implements LLMService { Map folder = (Map) folderObj; String name = (String) folder.get("name"); if (name != null && name.contains(keyword)) { - Object matchedItems = folder.get("items"); - if (matchedItems instanceof List) { - return (List>) matchedItems; - } + return folder; } } } } } - return Lists.newArrayList(); + return Maps.newHashMap(); } catch (IOException e) { throw new RuntimeException("获取本地文件失败：" + e.getMessage(), e); } } - public void updateEmbeddings(String llmToken, String workspaceSlug, List adds, List deletes) { + public Map updateEmbeddings(String llmToken, String workspaceSlug, List adds, List deletes) { String url = baseUrl + "/api/workspace/" + workspaceSlug + "/update-embeddings"; Map headers = new HashMap<>(); @@ -257,7 +291,7 @@ public class AnythingLLMServiceImpl implements LLMService { body.put("deletes", deletes); try { - httpHelper.exchange( + return httpHelper.exchange( url, "POST", headers, diff --git a/src/main/java/com/knowledge/base/infrastructure/util/BeanConvertUtil.java b/src/main/java/com/knowledge/base/infrastructure/util/BeanConvertUtil.java deleted file mode 100644 index 722fa48..0000000 --- a/src/main/java/com/knowledge/base/infrastructure/util/BeanConvertUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.knowledge.base.infrastructure.util; - -import cn.hutool.core.collection.CollectionUtil; -import org.springframework.beans.BeanUtils; - -import java.util.List; -import java.util.stream.Collectors; - -public class BeanConvertUtil { - - /** - * 将 source 转换为指定类型的目标对象 - */ - public static T convert(S source, Class targetClass) { - if (source == null) return null; - try { - T target = targetClass.getDeclaredConstructor().newInstance(); - BeanUtils.copyProperties(source, target); - return target; - } catch (Exception e) { - throw new RuntimeException("BeanConvertUtil.convert error", e); - } - } - - /** - * 批量转换：List → List - */ - public static List convertList(List sourceList, Class targetClass) { - if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList(); - return sourceList.stream() - .map(source -> convert(source, targetClass)) - .collect(Collectors.toList()); - } -} - diff --git a/src/main/java/com/knowledge/base/infrastructure/util/BeanTool.java b/src/main/java/com/knowledge/base/infrastructure/util/BeanTool.java new file mode 100644 index 0000000..00d9486 --- /dev/null +++ b/src/main/java/com/knowledge/base/infrastructure/util/BeanTool.java @@ -0,0 +1,90 @@ +package com.knowledge.base.infrastructure.util; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.lang.TypeReference; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.json.JSONUtil; +import org.springframework.beans.BeanUtils; + +import java.util.List; +import java.util.stream.Collectors; + +public class BeanTool { + + /** + * 将 source 转换为指定类型的目标对象 + */ + public static T convert(S source, Class targetClass) { + if (source == null) return null; + try { + T target = targetClass.getDeclaredConstructor().newInstance(); + BeanUtils.copyProperties(source, target); + return target; + } catch (Exception e) { + throw new RuntimeException("BeanConvertUtil.convert error", e); + } + } + + /** + * 批量转换：List → List + */ + public static List convertList(List sourceList, Class targetClass) { + if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList(); + return sourceList.stream() + .map(source -> convert(source, targetClass)) + .collect(Collectors.toList()); + } + + /** + * 获取嵌套字段的值（路径支持.分隔 & 数组索引） + * @param map 原始 Map + * @param key 路径，如 data.destination 或 data.results[0].file + * @param 返回类型 + * @return 值（可为 null） + */ + @SuppressWarnings("unchecked") + public static T getMapV(String key, Object map) { + Object value = BeanUtil.getProperty(map, key); + return (T) value; + } + + /** + * 获取嵌套字段值并转换为指定类型（通过 JSON 编解码转换） + * @param key 路径 + * @param map 原始 Map + * @param clazz 目标类型 + * @param 类型参数 + * @return 指定类型的值，转换失败或字段不存在时返回 null + */ + public static T getMapV(String key, Object map, Class clazz) { + Object value = BeanUtil.getProperty(map, key); + if (ObjectUtil.isEmpty(value)) { + return null; + } + return JSONUtil.toBean(JSONUtil.toJsonStr(value), clazz); + } + + /** + * 获取嵌套字段并反序列化为指定泛型类型 + * + * @param key 字段路径（支持嵌套，如 "data.items"） + * @param map 原始 Map 对象 + * @param typeReference 泛型类型引用（如 new TypeReference>>(){}） + * @return 类型化对象或 null + */ + public static T getMapV(String key, Object map, TypeReference typeReference) { + Object value = BeanUtil.getProperty(map, key); + if (ObjectUtil.isEmpty(value)) { + return null; + } + + // value 已经是对象，如果是 Map/List 类型，可直接转换；否则先转 JSON 字符串再解析 + if (value instanceof CharSequence) { + return JSONUtil.toBean((String) value, typeReference, false); + } else { + return JSONUtil.toBean(JSONUtil.toJsonStr(value), typeReference, false); + } + } +} + diff --git a/src/main/java/com/knowledge/base/infrastructure/util/HttpHelper.java b/src/main/java/com/knowledge/base/infrastructure/util/HttpHelper.java index 55291f2..adae48f 100644 --- a/src/main/java/com/knowledge/base/infrastructure/util/HttpHelper.java +++ b/src/main/java/com/knowledge/base/infrastructure/util/HttpHelper.java @@ -2,6 +2,7 @@ package com.knowledge.base.infrastructure.util; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import org.springframework.stereotype.Component; @@ -117,19 +118,27 @@ public class HttpHelper { * 通用 HTTP 请求方法，支持任意 method、headers、请求体，支持泛型 TypeReference 返回（Jackson） */ public T exchange(String url, String method, Map headers, Object body, TypeReference typeRef) throws IOException { + if (headers == null) { + headers = Maps.newHashMap(); + } + RequestBody requestBody = null; - if (body != null) { - String json = objectMapper.writeValueAsString(body); - requestBody = RequestBody.create(json, MediaType.get("application/json; charset=utf-8")); + boolean hasBody = body != null && !"GET".equalsIgnoreCase(method); + + String contentType = headers.getOrDefault("Content-Type", "application/json; charset=utf-8"); + + if (hasBody) { + String text; + if (body instanceof String) { + text = (String) body; + } else { + text = objectMapper.writeValueAsString(body); + } + requestBody = RequestBody.create(text, MediaType.get(contentType)); } Request.Builder builder = new Request.Builder().url(url); - - if (headers != null) { - for (Map.Entry entry : headers.entrySet()) { - builder.addHeader(entry.getKey(), entry.getValue()); - } - } + headers.forEach(builder::addHeader); switch (method.toUpperCase()) { case "POST": @@ -153,13 +162,21 @@ public class HttpHelper { if (!response.isSuccessful()) { throw new IOException("请求失败: " + response.code() + " - " + response.message()); } - String json = response.body().string(); - return objectMapper.readValue(json, typeRef); + + String responseBody = response.body().string(); + + // 如果期望的是 String 类型，直接返回 + if (typeRef.getType().getTypeName().equals(String.class.getTypeName())) { + return (T) responseBody; + } + + return objectMapper.readValue(responseBody, typeRef); } } + /** * 执行请求并解析为 Map */ diff --git a/src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java b/src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java index e58abb7..d878700 100644 --- a/src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java +++ b/src/test/java/com/knowledge/base/infrastructure/south/llm/AnythingLLMServiceTest.java @@ -1,7 +1,12 @@ package com.knowledge.base.infrastructure.south.llm; -import com.knowledge.base.infrastructure.cache.iface.UserCacheService; -import com.knowledge.base.infrastructure.util.HttpHelper; +import cn.hutool.core.lang.TypeReference; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.knowledge.base.infrastructure.config.ConstantConfig; +import com.knowledge.base.infrastructure.util.BeanTool; import com.knowledge.base.infrastructure.util.http.WriterAdapter; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; @@ -10,8 +15,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import java.io.File; +import java.net.URL; import java.util.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * @author Luke.ye * @date 2025/7/7 06:57 @@ -25,14 +34,10 @@ public class AnythingLLMServiceTest { @Autowired private AnythingLLMServiceImpl llmService; - @Autowired - private UserCacheService userCacheService; - - @Autowired - private HttpHelper httpHelper; - private String token; + private Map llmRepo = Maps.newHashMap(); + @BeforeEach public void setUp() { try { @@ -42,26 +47,70 @@ public class AnythingLLMServiceTest { } } + public static String getSlug() { + return ConstantConfig.DEFAULT_SLUG_ID; + } + @Test public void testImportObsidianToLLM() { try { - File file1 = new File("src/test/resources/obsidian-test/index.md"); - File file2 = new File("src/test/resources/obsidian-test/demo.md"); + // 使用 ClassLoader 获取资源目录（obsidian-test） + URL resource = getClass().getClassLoader().getResource("obsidian-test"); + assertNotNull(resource, "obsidian-test 目录不存在，检查 test/resources 路径是否正确"); - llmService.importObsidianToLLM(token, Arrays.asList(file1, file2), "src/test/resources/obsidian-test"); + File vaultDir = new File(resource.toURI()); + assertTrue(vaultDir.isDirectory(), "obsidian-test 应为目录"); + + Map result = llmService.importObsidianToLLM(token, vaultDir); + llmRepo.put("test-vault-name", BeanTool.getMapV("data.destination", result)); + log.info("导入结果: {}, llmRepo: {}", JSONUtil.toJsonStr(result), JSONUtil.toJsonStr(llmRepo)); +// testRemoveFolderFromLLM(); } catch (Exception e) { log.warn("测试 importObsidianToLLM 异常", e); } } + @Test + public void testRemoveFolderFromLLM() { + try { + String s = llmService.removeFolderFromLLM(token, (String) llmRepo.getOrDefault("test-vault-name", "obsidian-vault-obsidian-test-e511")); + log.info("removeResult: {}", s); + } catch (Exception e) { + log.warn("测试 removeFolderFromLLM 异常", e); + } + + } + + @Test + public void testGetLocalFileItems() { + try { + String slug = getSlug(); + Map result = llmService.getLocalFileItems(token, slug, "test"); + llmRepo.put("local-files", result); + log.info("本地文件项：{}", JSONUtil.toJsonStr(result)); + } catch (Exception e) { + log.warn("测试 getLocalFileItems 异常", e); + } + } + @Test public void testUpdateEmbeddings() { try { - String slug = "your-workspace-slug"; - List adds = Collections.singletonList("public/xx.md"); + String slug = getSlug(); + testGetLocalFileItems(); + Map localFilesMap = (Map)llmRepo.getOrDefault("local-files", Maps.newHashMap()); + String folderName = BeanTool.getMapV("name", localFilesMap); + List items = BeanTool.getMapV("items", localFilesMap, new TypeReference>() {}); + List files = Lists.newArrayList(); + items.stream().forEach(e -> { + String fileName = (String) e.get("name"); + if(StrUtil.isNotBlank(fileName)) { + files.add(folderName + "/" + fileName); + } + }); List deletes = Arrays.asList(); - - llmService.updateEmbeddings(token, slug, adds, deletes); + Map result = llmService.updateEmbeddings(token, slug, files, deletes); + log.info("result: {}", JSONUtil.toJsonStr(result)); } catch (Exception e) { log.warn("测试 updateEmbeddings 异常", e); } @@ -70,10 +119,11 @@ public class AnythingLLMServiceTest { @Test public void testUpdatePin() { try { - String slug = "your-workspace-slug"; - String path = "public/xx.md"; + String slug = getSlug(); + String path = "custom-documents/index.md-76d75c91-d4fe-4fa3-bc4b-0359c47ccae5.json"; - llmService.updatePin(token, slug, path, true); + String result = llmService.updatePin(token, slug, path, false); + log.info("result: {}", result); } catch (Exception e) { log.warn("测试 updatePin 异常", e); } @@ -82,7 +132,7 @@ public class AnythingLLMServiceTest { @Test public void testFetchAttachments() { try { - String slug = "your-workspace-slug"; + String slug = getSlug(); Map attachments = llmService.fetchAttachments(token, slug); attachments.forEach((k, v) -> log.info("Attachment: {} -> {}", k, v)); } catch (Exception e) { @@ -90,16 +140,6 @@ public class AnythingLLMServiceTest { } } - @Test - public void testGetLocalFileItems() { - try { - String slug = "your-workspace-slug"; - List> result = llmService.getLocalFileItems(token, slug, "导出"); - log.info("本地文件项：{}", result); - } catch (Exception e) { - log.warn("测试 getLocalFileItems 异常", e); - } - } @Test public void testStreamAnswer() {