lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加笨本地å缓存，优化导入性能

Browse Source
This commit is contained in:
luke 2025-05-31 22:52:07 +08:00
parent 94c85afa15
commit e859dd2004
2 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java
View File

@ -1,6 +1,7 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.iface.DocumentImporter;
import com.doc.parser.infrastructure.util.LocalCacheUtil;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
@ -43,47 +44,57 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
.filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
.forEach(path -> {
try {
String content = extractContent(path);
if (content == null || content.isBlank()) {
logger.warn("跳过空内容文件: {}", path);
return;
}
Path absPath = path.toAbsolutePath().normalize();
Path relativePathObj;
if (absPath.startsWith(excludeBase)) {
relativePathObj = excludeBase.relativize(absPath);
} else {
logger.warn("路径未匹配 exclude.prefix，使用全路径: {}", absPath);
relativePathObj = absPath;
}
String relativePath = relativePathObj.toString().replace("\\", "/");
Long localMTime = Files.getLastModifiedTime(path).toMillis();
logger.info("absPath: {}", absPath.toString().replace("\\", "/"));
logger.info("rerelativePath: {}", relativePath);
logger.info("rerelativePath: {}， localMTime： {}", relativePath, localMTime);
// 优先从缓存中加载数据，判断是否文件有修改
Long lastModifiedTime = (Long)LocalCacheUtil.get(relativePath);
if(localMTime.equals(lastModifiedTime)) {
logger.info("文件未变动，跳过导入: {}", relativePath);
return;
}
String content = extractContent(path);
if (content == null || content.isBlank()) {
logger.warn("跳过空内容文件: {}", path);
return;
}
// 以 relativePath 作为文档 ID
GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
boolean skip = false;
long localMTime = Files.getLastModifiedTime(path).toMillis();
boolean fileHasChanged = false;
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime");
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", relativePath);
skip = true;
if(LocalCacheUtil.get(relativePath) == null) {
// 缓存过期
LocalCacheUtil.put(relativePath, localMTime);
}
return;
} else {
// 删除旧版本
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath);
fileHasChanged = true;
}
}
if (skip) return;
if (fileHasChanged) {
// 删除旧版本
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath);
}
// === 构建文档 ===
Map<String, Object> doc = new HashMap<>();
@ -99,6 +110,8 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath);
// 信息入缓存
LocalCacheUtil.put(relativePath, localMTime);
} catch (Exception e) {
logger.error("导入失败: {}", path, e);
}

4
src/main/java/com/doc/parser/infrastructure/north/controller/LoginController.java
View File

@ -15,8 +15,8 @@ import java.util.concurrent.ConcurrentHashMap;
public class LoginController {
private static final Map<String, String> USER_DB = Map.of(
"admin", "123456",
"user", "abc123"
"admin", "Admin@654321",
"lukeye", "lukeye"
);
// token -> timestamp