添加笨本地å缓存,优化导入性能

This commit is contained in:
luke 2025-05-31 22:52:07 +08:00
parent 94c85afa15
commit e859dd2004
2 changed files with 33 additions and 20 deletions

View File

@ -1,6 +1,7 @@
package com.doc.parser.domain.impl; package com.doc.parser.domain.impl;
import com.doc.parser.domain.iface.DocumentImporter; import com.doc.parser.domain.iface.DocumentImporter;
import com.doc.parser.infrastructure.util.LocalCacheUtil;
import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.GetResponse;
@ -43,47 +44,57 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
.filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix())) .filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
.forEach(path -> { .forEach(path -> {
try { try {
String content = extractContent(path);
if (content == null || content.isBlank()) {
logger.warn("跳过空内容文件: {}", path);
return;
}
Path absPath = path.toAbsolutePath().normalize(); Path absPath = path.toAbsolutePath().normalize();
Path relativePathObj; Path relativePathObj;
if (absPath.startsWith(excludeBase)) { if (absPath.startsWith(excludeBase)) {
relativePathObj = excludeBase.relativize(absPath); relativePathObj = excludeBase.relativize(absPath);
} else { } else {
logger.warn("路径未匹配 exclude.prefix使用全路径: {}", absPath); logger.warn("路径未匹配 exclude.prefix使用全路径: {}", absPath);
relativePathObj = absPath; relativePathObj = absPath;
} }
String relativePath = relativePathObj.toString().replace("\\", "/"); String relativePath = relativePathObj.toString().replace("\\", "/");
Long localMTime = Files.getLastModifiedTime(path).toMillis();
logger.info("absPath: {}", absPath.toString().replace("\\", "/")); logger.info("absPath: {}", absPath.toString().replace("\\", "/"));
logger.info("rerelativePath: {}", relativePath); logger.info("rerelativePath: {} localMTime {}", relativePath, localMTime);
// 优先从缓存中加载数据判断是否文件有修改
Long lastModifiedTime = (Long)LocalCacheUtil.get(relativePath);
if(localMTime.equals(lastModifiedTime)) {
logger.info("文件未变动,跳过导入: {}", relativePath);
return;
}
String content = extractContent(path);
if (content == null || content.isBlank()) {
logger.warn("跳过空内容文件: {}", path);
return;
}
// relativePath 作为文档 ID // relativePath 作为文档 ID
GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath); GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
boolean skip = false; boolean fileHasChanged = false;
long localMTime = Files.getLastModifiedTime(path).toMillis();
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) { if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT); GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
Map<String, Object> existingSource = existing.getSourceAsMap(); Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime"); Object esMtime = existingSource.get("mtime");
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动,跳过导入: {}", relativePath); logger.info("文件未变动,跳过导入: {}", relativePath);
skip = true;
if(LocalCacheUtil.get(relativePath) == null) {
// 缓存过期
LocalCacheUtil.put(relativePath, localMTime);
}
return;
} else { } else {
fileHasChanged = true;
}
}
if (fileHasChanged) {
// 删除旧版本 // 删除旧版本
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath); DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
esClient.delete(deleteRequest, RequestOptions.DEFAULT); esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath); logger.info("已删除旧版本文件: {}", relativePath);
} }
}
if (skip) return;
// === 构建文档 === // === 构建文档 ===
Map<String, Object> doc = new HashMap<>(); Map<String, Object> doc = new HashMap<>();
@ -99,6 +110,8 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
esClient.index(request, RequestOptions.DEFAULT); esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath); logger.info("导入成功: {}", relativePath);
// 信息入缓存
LocalCacheUtil.put(relativePath, localMTime);
} catch (Exception e) { } catch (Exception e) {
logger.error("导入失败: {}", path, e); logger.error("导入失败: {}", path, e);
} }

View File

@ -15,8 +15,8 @@ import java.util.concurrent.ConcurrentHashMap;
public class LoginController { public class LoginController {
private static final Map<String, String> USER_DB = Map.of( private static final Map<String, String> USER_DB = Map.of(
"admin", "123456", "admin", "Admin@654321",
"user", "abc123" "lukeye", "lukeye"
); );
// token -> timestamp // token -> timestamp