diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java index 4f9d82c..c09e057 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java @@ -215,8 +215,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { String accessUrl = localRelaFilePath; if(localRelaFilePath.endsWith(".md")) { // markdown直接拼接http链接 - accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, localRelaFilePath); - accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html"; + accessUrl = DocAccessUrlTool.buildMarkdownAccessUrl(localRelaFilePath); metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl); } else { // 其它文件从对象存储的DB中获取 @@ -226,7 +225,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { accessUrl = latestRecord.getUrl(); metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl); metaMap.put(DocMetaPropEnum.UPLOADER.code, latestRecord.getUploader()); - metaMap.put(DocMetaPropEnum.EXPIRE_TIME.code, latestRecord.getExpireTime()); + metaMap.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(latestRecord.getExpireTime())); } } return metaMap; diff --git a/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java b/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java index 317e17f..fdafd1a 100644 --- a/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java +++ b/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java @@ -5,8 +5,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; -import java.util.List; - /** * @author Luke.ye * @date 2025/5/8 09:59 @@ -37,4 +35,10 @@ public class DynamicConfig { @Value("${llm.sse.rate.limit: 3}") private String llmSseRateLimit; + + @Value("${markdown.path}") + private String mdDirectoryPath; + + @Value("${exclude.file.path.prefix}") + private String mdExcludePrefix; } diff --git a/src/main/java/com/knowledge/base/infrastructure/north/controller/VaultChangeController.java b/src/main/java/com/knowledge/base/infrastructure/north/controller/VaultChangeController.java index 9213152..967c933 100644 --- a/src/main/java/com/knowledge/base/infrastructure/north/controller/VaultChangeController.java +++ b/src/main/java/com/knowledge/base/infrastructure/north/controller/VaultChangeController.java @@ -1,6 +1,16 @@ package com.knowledge.base.infrastructure.north.controller; +import cn.hutool.json.JSONUtil; import com.knowledge.base.application.service.DocAppService; +import com.knowledge.base.domain.common.enums.DocMetaPropEnum; +import com.knowledge.base.domain.doc.service.FileImporterDispatcher; +import com.knowledge.base.infrastructure.config.DynamicConfig; +import com.knowledge.base.infrastructure.config.ThreadPoolConfig; +import com.knowledge.base.infrastructure.north.dto.doc.ImportFilesReqDTO; +import com.knowledge.base.infrastructure.util.DateUtil; +import com.knowledge.base.infrastructure.util.DocAccessUrlTool; +import com.knowledge.base.infrastructure.util.RateLimiterManager; +import com.knowledge.base.infrastructure.util.ThreadPoolUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; @@ -9,8 +19,13 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; /** * Obisidian Vault仓库内容发生变化， @@ -23,8 +38,15 @@ public class VaultChangeController { private final DocAppService docAppService; + private final FileImporterDispatcher dispatcher; + + private final DynamicConfig dynamicConfig; + + private final RateLimiterManager rateLimiterManager; + /** * git仓库发生删除，需同步删除ES + * * @param relaFilePaths * @return */ @@ -38,4 +60,38 @@ public class VaultChangeController { )); } + @PostMapping("/import-new-md-files-into-es") + public ResponseEntity importNewMarkdownFiles(@RequestBody ImportFilesReqDTO request) { + if (Objects.isNull(request) || !request.isValid()) { + return ResponseEntity.badRequest().body("request is invalid. " + JSONUtil.toJsonStr(request)); + } + log.info("Obsidian Vault中存在新增，开始同步导入ES，relaFilePaths: {}", JSONUtil.toJsonStr(request)); + LocalDateTime expireTime = LocalDateTime.of(9999, 12, 31, 23, 59, 59); + String author = request.getAuthor(); + Set filesRelaPath = request.getFileRelaPaths(); + + String excludePrefix = dynamicConfig.getMdExcludePrefix(); + String directoryPath = dynamicConfig.getMdDirectoryPath(); + + filesRelaPath.stream() + .filter(fileRelaPath -> fileRelaPath.endsWith(".md")) + .forEach(relaPath -> { + rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_IMPORT).acquire(); + Path targetPath = Paths.get(directoryPath, relaPath); + String accessUrl = DocAccessUrlTool.buildMarkdownAccessUrl(relaPath); + Map extInfo = Map.of( + DocMetaPropEnum.UPLOADER.code, author, + DocMetaPropEnum.ACCESS_URL.code, accessUrl, + DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime) + ); + ThreadPoolUtil.execute(() -> { + boolean res = dispatcher.importSingleFile(targetPath.toAbsolutePath(), Paths.get(excludePrefix), extInfo); + log.info("本地文件信息已保存并导入ES: localRelaPath = {}, ESImportRes = {}", relaPath, res); + }, ThreadPoolConfig.IMPORT_DOC_POOL); + }); + log.info("完成Obsidian Vault中新增数据导入ES. {}", JSONUtil.toJsonStr(request)); + return ResponseEntity.ok(Map.of( + "result", true + )); + } } diff --git a/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/ImportFilesReqDTO.java b/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/ImportFilesReqDTO.java new file mode 100644 index 0000000..4a55b26 --- /dev/null +++ b/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/ImportFilesReqDTO.java @@ -0,0 +1,21 @@ +package com.knowledge.base.infrastructure.north.dto.doc; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.StrUtil; +import lombok.Data; + +import java.util.Set; + +/** + * @author Luke.ye + * @date 2025/6/27 11:27 + */ +@Data +public class ImportFilesReqDTO { + private Set fileRelaPaths; + private String author; + + public boolean isValid() { + return CollectionUtil.isNotEmpty(fileRelaPaths) && StrUtil.isNotBlank(author); + } +} diff --git a/src/main/java/com/knowledge/base/infrastructure/util/DocAccessUrlTool.java b/src/main/java/com/knowledge/base/infrastructure/util/DocAccessUrlTool.java new file mode 100644 index 0000000..b71bdc6 --- /dev/null +++ b/src/main/java/com/knowledge/base/infrastructure/util/DocAccessUrlTool.java @@ -0,0 +1,17 @@ +package com.knowledge.base.infrastructure.util; + +import com.knowledge.base.infrastructure.config.ConstantConfig; + +/** + * @author Luke.ye + * @date 2025/6/27 14:33 + */ +public class DocAccessUrlTool { + + public static String buildMarkdownAccessUrl(String relaFilePathWithSuffixName) { + // markdown直接拼接http链接 + String accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relaFilePathWithSuffixName); + accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html"; + return accessUrl; + } +}