lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加markdown文件导入逻辑

Browse Source
This commit is contained in:
luke 2025-06-27 17:06:39 +08:00
parent 8717ad49b7
commit a85610078b
5 changed files with 102 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java
View File

@ -215,8 +215,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
String accessUrl = localRelaFilePath;
if(localRelaFilePath.endsWith(".md")) {
// markdown直接拼接http链接
accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, localRelaFilePath);
accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html";
accessUrl = DocAccessUrlTool.buildMarkdownAccessUrl(localRelaFilePath);
metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl);
} else {
// 其它文件从对象存储的DB中获取
@ -226,7 +225,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
accessUrl = latestRecord.getUrl();
metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl);
metaMap.put(DocMetaPropEnum.UPLOADER.code, latestRecord.getUploader());
metaMap.put(DocMetaPropEnum.EXPIRE_TIME.code, latestRecord.getExpireTime());
metaMap.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(latestRecord.getExpireTime()));
}
}
return metaMap;

8
src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java
View File

@ -5,8 +5,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @author Luke.ye
* @date 2025/5/8 09:59
@ -37,4 +35,10 @@ public class DynamicConfig {
@Value("${llm.sse.rate.limit: 3}")
private String llmSseRateLimit;
@Value("${markdown.path}")
private String mdDirectoryPath;
@Value("${exclude.file.path.prefix}")
private String mdExcludePrefix;
}

56
src/main/java/com/knowledge/base/infrastructure/north/controller/VaultChangeController.java
View File

@ -1,6 +1,16 @@
package com.knowledge.base.infrastructure.north.controller;
import cn.hutool.json.JSONUtil;
import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.doc.service.FileImporterDispatcher;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
import com.knowledge.base.infrastructure.north.dto.doc.ImportFilesReqDTO;
import com.knowledge.base.infrastructure.util.DateUtil;
import com.knowledge.base.infrastructure.util.DocAccessUrlTool;
import com.knowledge.base.infrastructure.util.RateLimiterManager;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
@ -9,8 +19,13 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
* Obisidian Vault仓库内容发生变化，
@ -23,8 +38,15 @@ public class VaultChangeController {
private final DocAppService docAppService;
private final FileImporterDispatcher dispatcher;
private final DynamicConfig dynamicConfig;
private final RateLimiterManager rateLimiterManager;
/**
* git仓库发生删除，需同步删除ES
*
* @param relaFilePaths
* @return
*/
@ -38,4 +60,38 @@ public class VaultChangeController {
));
}
@PostMapping("/import-new-md-files-into-es")
public ResponseEntity<?> importNewMarkdownFiles(@RequestBody ImportFilesReqDTO request) {
if (Objects.isNull(request) || !request.isValid()) {
return ResponseEntity.badRequest().body("request is invalid. " + JSONUtil.toJsonStr(request));
}
log.info("Obsidian Vault中存在新增，开始同步导入ES，relaFilePaths: {}", JSONUtil.toJsonStr(request));
LocalDateTime expireTime = LocalDateTime.of(9999, 12, 31, 23, 59, 59);
String author = request.getAuthor();
Set<String> filesRelaPath = request.getFileRelaPaths();
String excludePrefix = dynamicConfig.getMdExcludePrefix();
String directoryPath = dynamicConfig.getMdDirectoryPath();
filesRelaPath.stream()
.filter(fileRelaPath -> fileRelaPath.endsWith(".md"))
.forEach(relaPath -> {
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_IMPORT).acquire();
Path targetPath = Paths.get(directoryPath, relaPath);
String accessUrl = DocAccessUrlTool.buildMarkdownAccessUrl(relaPath);
Map<String, Object> extInfo = Map.of(
DocMetaPropEnum.UPLOADER.code, author,
DocMetaPropEnum.ACCESS_URL.code, accessUrl,
DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime)
);
ThreadPoolUtil.execute(() -> {
boolean res = dispatcher.importSingleFile(targetPath.toAbsolutePath(), Paths.get(excludePrefix), extInfo);
log.info("本地文件信息已保存并导入ES: localRelaPath = {}, ESImportRes = {}", relaPath, res);
}, ThreadPoolConfig.IMPORT_DOC_POOL);
});
log.info("完成Obsidian Vault中新增数据导入ES. {}", JSONUtil.toJsonStr(request));
return ResponseEntity.ok(Map.of(
"result", true
));
}
}

21
src/main/java/com/knowledge/base/infrastructure/north/dto/doc/ImportFilesReqDTO.java Normal file
View File

@ -0,0 +1,21 @@
package com.knowledge.base.infrastructure.north.dto.doc;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import lombok.Data;
import java.util.Set;
/**
* @author Luke.ye
* @date 2025/6/27 11:27
*/
@Data
public class ImportFilesReqDTO {
private Set<String> fileRelaPaths;
private String author;
public boolean isValid() {
return CollectionUtil.isNotEmpty(fileRelaPaths) && StrUtil.isNotBlank(author);
}
}

17
src/main/java/com/knowledge/base/infrastructure/util/DocAccessUrlTool.java Normal file
View File

@ -0,0 +1,17 @@
package com.knowledge.base.infrastructure.util;
import com.knowledge.base.infrastructure.config.ConstantConfig;
/**
* @author Luke.ye
* @date 2025/6/27 14:33
*/
public class DocAccessUrlTool {
public static String buildMarkdownAccessUrl(String relaFilePathWithSuffixName) {
// markdown直接拼接http链接
String accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relaFilePathWithSuffixName);
accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html";
return accessUrl;
}
}