lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opt

Browse Source
This commit is contained in:
luke 2025-06-20 14:41:41 +08:00
parent 25f0ef9cf2
commit 19eec52e52
5 changed files with 116 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java Normal file
View File

@ -0,0 +1,16 @@
package com.knowledge.base.domain.doc.model;
/**
* @author Luke.ye
* @date 2025/6/20 13:57
*/
public class FileEsField {
public static final String INDEX = "documents";
public static final String FILENAME = "filename";
public static final String FILEPATH = "filepath";
public static final String CONTENT = "content";
public static final String MTIME = "mtime";
public static final String UPLOADER = "uploader";
public static final String URL = "url";
public static final String EXPIRE_TIME = "expireTime";
}

32
src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java Normal file
View File

@ -0,0 +1,32 @@
package com.knowledge.base.domain.doc.model;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 文件文档模型（字段应于 {@link FileEsField} 保持一致）
* @author Luke.ye
* @date 2025/6/20 13:45
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileEsModel {
private String filename; // 文档标题
private String filepath; // 相对路径，作为唯一 ID
private String content; // 文本内容
private Long mtime; // 最后修改时间
private String uploader; // 上传人
private String url; // 文件访问链接
private Long expireTime; // 过期时间戳
public String buildDocId() {
return SafeIdUtil.encode(this.filepath);
}
}

69
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java
View File

@ -1,5 +1,6 @@
package com.knowledge.base.domain.doc.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
@ -7,6 +8,8 @@ import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
@ -122,29 +125,30 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
// === Step 4: 检查 ES 是否已有未变动版本 ===
String docId = SafeIdUtil.encode(localRelaFilePath);
GetResponse existing = esGateway.getIfExists(INDEX_NAME, docId);
GetResponse existing = esGateway.getIfExists(docId);
if(Objects.nonNull(existing)) {
Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime");
Object esMtime = existingSource.get(FileEsField.MTIME);
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(existingSource), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true;
}
esGateway.deleteDoc(INDEX_NAME, docId);
esGateway.deleteDoc(docId);
logger.info("[ES] 已删除旧版本文件: {}", localRelaFilePath);
}
logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime);
// === Step 5: 构建待写入文档 ===
Map<String, Object> doc = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo);
FileEsModel fileEsModel = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo);
// === Step 6: 写入 Elasticsearch 并更新缓存 ===
esGateway.saveDoc(INDEX_NAME, docId, doc);
esGateway.saveDoc(fileEsModel);
logger.info("导入成功: {}", localRelaFilePath);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
String metaJson = transToMetaJson(BeanUtil.beanToMap(fileEsModel, false, true));
fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true;
} catch (Exception e) {
@ -153,36 +157,47 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
}
}
private Map<String, Object> buildDocument(String filename, String filepath, String content, Long mtime, Map<String, Object> extInfo) {
Map<String, Object> doc = new HashMap<>();
doc.put("filename", filename);
doc.put("filepath", filepath);
doc.put("content", content);
doc.put("mtime", mtime);
private FileEsModel buildDocument(String filename, String filepath, String content, Long mtime, Map<String, Object> extInfo) {
String uploader = ConstantConfig.DEFAULT_UPLOADER;
String url = "";
Long expireTime = DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME);
Optional<String> metaJsonOpt = fileCacheService.getMeta(filepath);
if (metaJsonOpt.isPresent()) {
// 优先使用缓存
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER));
doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY));
doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)));
uploader = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, uploader);
url = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, url);
expireTime = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, expireTime);
} else {
// 使用传入 extInfo
extInfo = MapUtil.isEmpty(extInfo) ? MapUtil.empty() : extInfo;
boolean hasAllMeta = extInfo.keySet().containsAll(Lists.newArrayList(
DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code));
DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code
));
if (hasAllMeta) {
doc.put("uploader", extInfo.get(DocMetaPropEnum.UPLOADER.code));
doc.put("url", extInfo.get(DocMetaPropEnum.ACCESS_URL.code));
doc.put("expireTime", extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code));
uploader = (String) extInfo.get(DocMetaPropEnum.UPLOADER.code);
url = (String) extInfo.get(DocMetaPropEnum.ACCESS_URL.code);
expireTime = (Long) extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code);
} else {
Map<String, Object> props = buildDocMetaProps(filepath);
doc.put("uploader", props.get(DocMetaPropEnum.UPLOADER.code));
doc.put("url", props.get(DocMetaPropEnum.ACCESS_URL.code));
doc.put("expireTime", props.get(DocMetaPropEnum.EXPIRE_TIME.code));
uploader = (String) props.get(DocMetaPropEnum.UPLOADER.code);
url = (String) props.get(DocMetaPropEnum.ACCESS_URL.code);
expireTime = (Long) props.get(DocMetaPropEnum.EXPIRE_TIME.code);
}
}
return doc;
return FileEsModel.builder()
.filename(filename)
.filepath(filepath)
.content(content)
.mtime(mtime)
.uploader(uploader)
.url(url)
.expireTime(expireTime)
.build();
}
@ -218,10 +233,10 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
private String transToMetaJson(Map<String, Object> esExistingSource) {
return JSONUtil.toJsonStr(Map.of(
DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER),
DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get("mtime"),
DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get("url"),
DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get("expireTime")
DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get(FileEsField.UPLOADER)).orElse(ConstantConfig.DEFAULT_UPLOADER),
DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get(FileEsField.MTIME),
DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get(FileEsField.URL),
DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get(FileEsField.EXPIRE_TIME)
));
}

13
src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java
View File

@ -8,6 +8,7 @@ import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.north.dto.SearchReq;
@ -165,13 +166,13 @@ public class FileQueryController {
for (SearchHit hit : response.getHits()) {
Map<String, Object> source = hit.getSourceAsMap();
Map<String, Object> result = new LinkedHashMap<>();
result.put("filename", source.get("filename"));
result.put("filepath", source.get("filepath"));
result.put("mtime", source.get("mtime"));
result.put("uploader", source.getOrDefault("uploader", ConstantConfig.DEFAULT_UPLOADER));
result.put("url", source.get("url"));
result.put("filename", source.get(FileEsField.FILENAME));
result.put("filepath", source.get(FileEsField.FILEPATH));
result.put("mtime", source.get(FileEsField.MTIME));
result.put("uploader", source.getOrDefault(FileEsField.UPLOADER, ConstantConfig.DEFAULT_UPLOADER));
result.put("url", source.get(FileEsField.URL));
String content = (String) source.get("content");
String content = (String) source.get(FileEsField.CONTENT);
String summary = extractMultiSnippet(content, keywords, 50);
result.put("summary", summary);

33
src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java
View File

@ -1,5 +1,8 @@
package com.knowledge.base.infrastructure.south.es;
import cn.hutool.core.bean.BeanUtil;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
@ -36,7 +39,7 @@ import java.util.Map;
@Component
public class FileElasticsearchGateway {
private static final String INDEX_NAME = "documents";
private static final String INDEX_NAME = FileEsField.INDEX;
@Autowired
private RestHighLevelClient esClient;
@ -50,7 +53,7 @@ public class FileElasticsearchGateway {
.query(query)
.from(from)
.size(size)
.sort("mtime", SortOrder.DESC);
.sort(FileEsField.MTIME, SortOrder.DESC);
SearchRequest request = new SearchRequest(INDEX_NAME).source(builder);
return esClient.search(request, RequestOptions.DEFAULT);
@ -71,8 +74,8 @@ public class FileElasticsearchGateway {
for (String keyword : group) {
groupQuery.must(QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery("filename", keyword).analyzer("ik_smart"))
.should(QueryBuilders.matchQuery("content", keyword).analyzer("ik_smart"))
.should(QueryBuilders.matchQuery(FileEsField.FILENAME, keyword).analyzer("ik_smart"))
.should(QueryBuilders.matchQuery(FileEsField.CONTENT, keyword).analyzer("ik_smart"))
);
}
@ -82,7 +85,7 @@ public class FileElasticsearchGateway {
outerQuery.minimumShouldMatch(1);
// 过滤掉过期文件
outerQuery.filter(QueryBuilders.rangeQuery("expireTime")
outerQuery.filter(QueryBuilders.rangeQuery(FileEsField.EXPIRE_TIME)
.gte(System.currentTimeMillis())
.timeZone("+08:00"));
@ -150,15 +153,15 @@ public class FileElasticsearchGateway {
/**
* 判断文档是否存在并获取内容
*/
public GetResponse getIfExists(String index, String docId) {
public GetResponse getIfExists(String docId) {
try {
GetRequest getRequest = new GetRequest(index, docId);
GetRequest getRequest = new GetRequest(INDEX_NAME, docId);
if (!esClient.exists(getRequest, RequestOptions.DEFAULT)) {
return null;
}
return esClient.get(getRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
log.error("ES 获取文档失败: index={}, docId={}", index, docId, e);
log.error("ES 获取文档失败: index={}, docId={}", INDEX_NAME, docId, e);
return null;
}
}
@ -166,23 +169,25 @@ public class FileElasticsearchGateway {
/**
* 删除文档
*/
public void deleteDoc(String index, String docId) {
public void deleteDoc(String docId) {
try {
esClient.delete(new DeleteRequest(index, docId), RequestOptions.DEFAULT);
esClient.delete(new DeleteRequest(INDEX_NAME, docId), RequestOptions.DEFAULT);
} catch (Exception e) {
log.warn("ES 删除文档失败: index={}, docId={}", index, docId, e);
log.warn("ES 删除文档失败: index={}, docId={}", INDEX_NAME, docId, e);
}
}
/**
* 写入/更新文档
*/
public void saveDoc(String index, String docId, Map<String, Object> source) {
public void saveDoc(FileEsModel docModel) {
try {
IndexRequest request = new IndexRequest(index).id(docId).source(source);
String docId = docModel.buildDocId();
Map<String, Object> source = BeanUtil.beanToMap(docModel, false, true);
IndexRequest request = new IndexRequest(INDEX_NAME).id(docId).source(source);
esClient.index(request, RequestOptions.DEFAULT);
} catch (Exception e) {
log.error("ES 写入文档失败: index={}, docId={}", index, docId, e);
log.error("ES 写入文档失败: doc={}", docModel, e);
}
}