添加笨本地å缓存,优化导入性能
This commit is contained in:
parent
94c85afa15
commit
e859dd2004
@ -1,6 +1,7 @@
|
|||||||
package com.doc.parser.domain.impl;
|
package com.doc.parser.domain.impl;
|
||||||
|
|
||||||
import com.doc.parser.domain.iface.DocumentImporter;
|
import com.doc.parser.domain.iface.DocumentImporter;
|
||||||
|
import com.doc.parser.infrastructure.util.LocalCacheUtil;
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
import org.elasticsearch.action.get.GetRequest;
|
import org.elasticsearch.action.get.GetRequest;
|
||||||
import org.elasticsearch.action.get.GetResponse;
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
@ -43,47 +44,57 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
|
|||||||
.filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
|
.filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
|
||||||
.forEach(path -> {
|
.forEach(path -> {
|
||||||
try {
|
try {
|
||||||
String content = extractContent(path);
|
|
||||||
if (content == null || content.isBlank()) {
|
|
||||||
logger.warn("跳过空内容文件: {}", path);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Path absPath = path.toAbsolutePath().normalize();
|
Path absPath = path.toAbsolutePath().normalize();
|
||||||
Path relativePathObj;
|
Path relativePathObj;
|
||||||
|
|
||||||
if (absPath.startsWith(excludeBase)) {
|
if (absPath.startsWith(excludeBase)) {
|
||||||
relativePathObj = excludeBase.relativize(absPath);
|
relativePathObj = excludeBase.relativize(absPath);
|
||||||
} else {
|
} else {
|
||||||
logger.warn("路径未匹配 exclude.prefix,使用全路径: {}", absPath);
|
logger.warn("路径未匹配 exclude.prefix,使用全路径: {}", absPath);
|
||||||
relativePathObj = absPath;
|
relativePathObj = absPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
String relativePath = relativePathObj.toString().replace("\\", "/");
|
String relativePath = relativePathObj.toString().replace("\\", "/");
|
||||||
|
Long localMTime = Files.getLastModifiedTime(path).toMillis();
|
||||||
logger.info("absPath: {}", absPath.toString().replace("\\", "/"));
|
logger.info("absPath: {}", absPath.toString().replace("\\", "/"));
|
||||||
logger.info("rerelativePath: {}", relativePath);
|
logger.info("rerelativePath: {}, localMTime: {}", relativePath, localMTime);
|
||||||
|
// 优先从缓存中加载数据,判断是否文件有修改
|
||||||
|
Long lastModifiedTime = (Long)LocalCacheUtil.get(relativePath);
|
||||||
|
if(localMTime.equals(lastModifiedTime)) {
|
||||||
|
logger.info("文件未变动,跳过导入: {}", relativePath);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String content = extractContent(path);
|
||||||
|
if (content == null || content.isBlank()) {
|
||||||
|
logger.warn("跳过空内容文件: {}", path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// 以 relativePath 作为文档 ID
|
// 以 relativePath 作为文档 ID
|
||||||
GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
|
GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
|
||||||
boolean skip = false;
|
boolean fileHasChanged = false;
|
||||||
long localMTime = Files.getLastModifiedTime(path).toMillis();
|
|
||||||
|
|
||||||
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
|
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
|
||||||
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
|
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
|
||||||
Map<String, Object> existingSource = existing.getSourceAsMap();
|
Map<String, Object> existingSource = existing.getSourceAsMap();
|
||||||
Object esMtime = existingSource.get("mtime");
|
Object esMtime = existingSource.get("mtime");
|
||||||
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
|
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
|
||||||
logger.info("文件未变动,跳过导入: {}", relativePath);
|
logger.info("文件未变动,跳过导入: {}", relativePath);
|
||||||
skip = true;
|
|
||||||
|
if(LocalCacheUtil.get(relativePath) == null) {
|
||||||
|
// 缓存过期
|
||||||
|
LocalCacheUtil.put(relativePath, localMTime);
|
||||||
|
}
|
||||||
|
return;
|
||||||
} else {
|
} else {
|
||||||
// 删除旧版本
|
fileHasChanged = true;
|
||||||
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
|
|
||||||
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
|
|
||||||
logger.info("已删除旧版本文件: {}", relativePath);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (skip) return;
|
if (fileHasChanged) {
|
||||||
|
// 删除旧版本
|
||||||
|
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
|
||||||
|
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
|
||||||
|
logger.info("已删除旧版本文件: {}", relativePath);
|
||||||
|
}
|
||||||
|
|
||||||
// === 构建文档 ===
|
// === 构建文档 ===
|
||||||
Map<String, Object> doc = new HashMap<>();
|
Map<String, Object> doc = new HashMap<>();
|
||||||
@ -99,6 +110,8 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
|
|||||||
esClient.index(request, RequestOptions.DEFAULT);
|
esClient.index(request, RequestOptions.DEFAULT);
|
||||||
logger.info("导入成功: {}", relativePath);
|
logger.info("导入成功: {}", relativePath);
|
||||||
|
|
||||||
|
// 信息入缓存
|
||||||
|
LocalCacheUtil.put(relativePath, localMTime);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("导入失败: {}", path, e);
|
logger.error("导入失败: {}", path, e);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -15,8 +15,8 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
public class LoginController {
|
public class LoginController {
|
||||||
|
|
||||||
private static final Map<String, String> USER_DB = Map.of(
|
private static final Map<String, String> USER_DB = Map.of(
|
||||||
"admin", "123456",
|
"admin", "Admin@654321",
|
||||||
"user", "abc123"
|
"lukeye", "lukeye"
|
||||||
);
|
);
|
||||||
|
|
||||||
// token -> timestamp
|
// token -> timestamp
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user