lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opt

Browse Source
This commit is contained in:
luke 2025-06-20 14:41:41 +08:00
parent 25f0ef9cf2
commit 19eec52e52
5 changed files with 116 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java Normal file
View File

@ -0,0 +1,16 @@
package com.knowledge.base.domain.doc.model;
/**
* @author Luke.ye
* @date 2025/6/20 13:57
*/
public class FileEsField {
public static final String INDEX = "documents";
public static final String FILENAME = "filename";
public static final String FILEPATH = "filepath";
public static final String CONTENT = "content";
public static final String MTIME = "mtime";
public static final String UPLOADER = "uploader";
public static final String URL = "url";
public static final String EXPIRE_TIME = "expireTime";
}

32
src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java Normal file
View File

@ -0,0 +1,32 @@
package com.knowledge.base.domain.doc.model;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 文件文档模型（字段应于 {@link FileEsField} 保持一致）
* @author Luke.ye
* @date 2025/6/20 13:45
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileEsModel {
private String filename; // 文档标题
private String filepath; // 相对路径，作为唯一 ID
private String content; // 文本内容
private Long mtime; // 最后修改时间
private String uploader; // 上传人
private String url; // 文件访问链接
private Long expireTime; // 过期时间戳
public String buildDocId() {
return SafeIdUtil.encode(this.filepath);
}
}

69
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java
View File

@ -1,5 +1,6 @@
package com.knowledge.base.domain.doc.service.impl; package com.knowledge.base.domain.doc.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.map.MapUtil; import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON; import cn.hutool.json.JSON;
@ -7,6 +8,8 @@ import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum; import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.domain.doc.model.OSRecordDO; import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter; import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.domain.doc.service.iface.FileDomainService; import com.knowledge.base.domain.doc.service.iface.FileDomainService;
@ -122,29 +125,30 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
// === Step 4: 检查 ES 是否已有未变动版本 === // === Step 4: 检查 ES 是否已有未变动版本 ===
String docId = SafeIdUtil.encode(localRelaFilePath); String docId = SafeIdUtil.encode(localRelaFilePath);
GetResponse existing = esGateway.getIfExists(INDEX_NAME, docId); GetResponse existing = esGateway.getIfExists(docId);
if(Objects.nonNull(existing)) { if(Objects.nonNull(existing)) {
Map<String, Object> existingSource = existing.getSourceAsMap(); Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime"); Object esMtime = existingSource.get(FileEsField.MTIME);
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", localRelaFilePath); logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(existingSource), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(existingSource), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true; return true;
} }
esGateway.deleteDoc(INDEX_NAME, docId); esGateway.deleteDoc(docId);
logger.info("[ES] 已删除旧版本文件: {}", localRelaFilePath); logger.info("[ES] 已删除旧版本文件: {}", localRelaFilePath);
} }
logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime); logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime);
// === Step 5: 构建待写入文档 === // === Step 5: 构建待写入文档 ===
Map<String, Object> doc = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo); FileEsModel fileEsModel = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo);
// === Step 6: 写入 Elasticsearch 并更新缓存 === // === Step 6: 写入 Elasticsearch 并更新缓存 ===
esGateway.saveDoc(INDEX_NAME, docId, doc); esGateway.saveDoc(fileEsModel);
logger.info("导入成功: {}", localRelaFilePath); logger.info("导入成功: {}", localRelaFilePath);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); String metaJson = transToMetaJson(BeanUtil.beanToMap(fileEsModel, false, true));
fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true; return true;
} catch (Exception e) { } catch (Exception e) {
@ -153,36 +157,47 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
} }
} }
private Map<String, Object> buildDocument(String filename, String filepath, String content, Long mtime, Map<String, Object> extInfo) {
Map<String, Object> doc = new HashMap<>(); private FileEsModel buildDocument(String filename, String filepath, String content, Long mtime, Map<String, Object> extInfo) {
doc.put("filename", filename); String uploader = ConstantConfig.DEFAULT_UPLOADER;
doc.put("filepath", filepath); String url = "";
doc.put("content", content); Long expireTime = DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME);
doc.put("mtime", mtime);
Optional<String> metaJsonOpt = fileCacheService.getMeta(filepath); Optional<String> metaJsonOpt = fileCacheService.getMeta(filepath);
if (metaJsonOpt.isPresent()) { if (metaJsonOpt.isPresent()) {
// 优先使用缓存
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER)); uploader = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, uploader);
doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY)); url = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, url);
doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME))); expireTime = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, expireTime);
} else { } else {
// 使用传入 extInfo
extInfo = MapUtil.isEmpty(extInfo) ? MapUtil.empty() : extInfo; extInfo = MapUtil.isEmpty(extInfo) ? MapUtil.empty() : extInfo;
boolean hasAllMeta = extInfo.keySet().containsAll(Lists.newArrayList( boolean hasAllMeta = extInfo.keySet().containsAll(Lists.newArrayList(
DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code)); DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code
));
if (hasAllMeta) { if (hasAllMeta) {
doc.put("uploader", extInfo.get(DocMetaPropEnum.UPLOADER.code)); uploader = (String) extInfo.get(DocMetaPropEnum.UPLOADER.code);
doc.put("url", extInfo.get(DocMetaPropEnum.ACCESS_URL.code)); url = (String) extInfo.get(DocMetaPropEnum.ACCESS_URL.code);
doc.put("expireTime", extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code)); expireTime = (Long) extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code);
} else { } else {
Map<String, Object> props = buildDocMetaProps(filepath); Map<String, Object> props = buildDocMetaProps(filepath);
doc.put("uploader", props.get(DocMetaPropEnum.UPLOADER.code)); uploader = (String) props.get(DocMetaPropEnum.UPLOADER.code);
doc.put("url", props.get(DocMetaPropEnum.ACCESS_URL.code)); url = (String) props.get(DocMetaPropEnum.ACCESS_URL.code);
doc.put("expireTime", props.get(DocMetaPropEnum.EXPIRE_TIME.code)); expireTime = (Long) props.get(DocMetaPropEnum.EXPIRE_TIME.code);
} }
} }
return doc; return FileEsModel.builder()
.filename(filename)
.filepath(filepath)
.content(content)
.mtime(mtime)
.uploader(uploader)
.url(url)
.expireTime(expireTime)
.build();
} }
@ -218,10 +233,10 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
private String transToMetaJson(Map<String, Object> esExistingSource) { private String transToMetaJson(Map<String, Object> esExistingSource) {
return JSONUtil.toJsonStr(Map.of( return JSONUtil.toJsonStr(Map.of(
DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER), DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get(FileEsField.UPLOADER)).orElse(ConstantConfig.DEFAULT_UPLOADER),
DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get("mtime"), DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get(FileEsField.MTIME),
DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get("url"), DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get(FileEsField.URL),
DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get("expireTime") DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get(FileEsField.EXPIRE_TIME)
)); ));
} }

13
src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java
View File

@ -8,6 +8,7 @@ import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService; import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum; import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.common.model.PageResult; import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService; import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig; import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.north.dto.SearchReq; import com.knowledge.base.infrastructure.north.dto.SearchReq;
@ -165,13 +166,13 @@ public class FileQueryController {
for (SearchHit hit : response.getHits()) { for (SearchHit hit : response.getHits()) {
Map<String, Object> source = hit.getSourceAsMap(); Map<String, Object> source = hit.getSourceAsMap();
Map<String, Object> result = new LinkedHashMap<>(); Map<String, Object> result = new LinkedHashMap<>();
result.put("filename", source.get("filename")); result.put("filename", source.get(FileEsField.FILENAME));
result.put("filepath", source.get("filepath")); result.put("filepath", source.get(FileEsField.FILEPATH));
result.put("mtime", source.get("mtime")); result.put("mtime", source.get(FileEsField.MTIME));
result.put("uploader", source.getOrDefault("uploader", ConstantConfig.DEFAULT_UPLOADER)); result.put("uploader", source.getOrDefault(FileEsField.UPLOADER, ConstantConfig.DEFAULT_UPLOADER));
result.put("url", source.get("url")); result.put("url", source.get(FileEsField.URL));
String content = (String) source.get("content"); String content = (String) source.get(FileEsField.CONTENT);
String summary = extractMultiSnippet(content, keywords, 50); String summary = extractMultiSnippet(content, keywords, 50);
result.put("summary", summary); result.put("summary", summary);

33
src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java
View File

@ -1,5 +1,8 @@
package com.knowledge.base.infrastructure.south.es; package com.knowledge.base.infrastructure.south.es;
import cn.hutool.core.bean.BeanUtil;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.infrastructure.util.SafeIdUtil; import com.knowledge.base.infrastructure.util.SafeIdUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequest;
@ -36,7 +39,7 @@ import java.util.Map;
@Component @Component
public class FileElasticsearchGateway { public class FileElasticsearchGateway {
private static final String INDEX_NAME = "documents"; private static final String INDEX_NAME = FileEsField.INDEX;
@Autowired @Autowired
private RestHighLevelClient esClient; private RestHighLevelClient esClient;
@ -50,7 +53,7 @@ public class FileElasticsearchGateway {
.query(query) .query(query)
.from(from) .from(from)
.size(size) .size(size)
.sort("mtime", SortOrder.DESC); .sort(FileEsField.MTIME, SortOrder.DESC);
SearchRequest request = new SearchRequest(INDEX_NAME).source(builder); SearchRequest request = new SearchRequest(INDEX_NAME).source(builder);
return esClient.search(request, RequestOptions.DEFAULT); return esClient.search(request, RequestOptions.DEFAULT);
@ -71,8 +74,8 @@ public class FileElasticsearchGateway {
for (String keyword : group) { for (String keyword : group) {
groupQuery.must(QueryBuilders.boolQuery() groupQuery.must(QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery("filename", keyword).analyzer("ik_smart")) .should(QueryBuilders.matchQuery(FileEsField.FILENAME, keyword).analyzer("ik_smart"))
.should(QueryBuilders.matchQuery("content", keyword).analyzer("ik_smart")) .should(QueryBuilders.matchQuery(FileEsField.CONTENT, keyword).analyzer("ik_smart"))
); );
} }
@ -82,7 +85,7 @@ public class FileElasticsearchGateway {
outerQuery.minimumShouldMatch(1); outerQuery.minimumShouldMatch(1);
// 过滤掉过期文件 // 过滤掉过期文件
outerQuery.filter(QueryBuilders.rangeQuery("expireTime") outerQuery.filter(QueryBuilders.rangeQuery(FileEsField.EXPIRE_TIME)
.gte(System.currentTimeMillis()) .gte(System.currentTimeMillis())
.timeZone("+08:00")); .timeZone("+08:00"));
@ -150,15 +153,15 @@ public class FileElasticsearchGateway {
/** /**
* 判断文档是否存在并获取内容 * 判断文档是否存在并获取内容
*/ */
public GetResponse getIfExists(String index, String docId) { public GetResponse getIfExists(String docId) {
try { try {
GetRequest getRequest = new GetRequest(index, docId); GetRequest getRequest = new GetRequest(INDEX_NAME, docId);
if (!esClient.exists(getRequest, RequestOptions.DEFAULT)) { if (!esClient.exists(getRequest, RequestOptions.DEFAULT)) {
return null; return null;
} }
return esClient.get(getRequest, RequestOptions.DEFAULT); return esClient.get(getRequest, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
log.error("ES 获取文档失败: index={}, docId={}", index, docId, e); log.error("ES 获取文档失败: index={}, docId={}", INDEX_NAME, docId, e);
return null; return null;
} }
} }
@ -166,23 +169,25 @@ public class FileElasticsearchGateway {
/** /**
* 删除文档 * 删除文档
*/ */
public void deleteDoc(String index, String docId) { public void deleteDoc(String docId) {
try { try {
esClient.delete(new DeleteRequest(index, docId), RequestOptions.DEFAULT); esClient.delete(new DeleteRequest(INDEX_NAME, docId), RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
log.warn("ES 删除文档失败: index={}, docId={}", index, docId, e); log.warn("ES 删除文档失败: index={}, docId={}", INDEX_NAME, docId, e);
} }
} }
/** /**
* 写入/更新文档 * 写入/更新文档
*/ */
public void saveDoc(String index, String docId, Map<String, Object> source) { public void saveDoc(FileEsModel docModel) {
try { try {
IndexRequest request = new IndexRequest(index).id(docId).source(source); String docId = docModel.buildDocId();
Map<String, Object> source = BeanUtil.beanToMap(docModel, false, true);
IndexRequest request = new IndexRequest(INDEX_NAME).id(docId).source(source);
esClient.index(request, RequestOptions.DEFAULT); esClient.index(request, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
log.error("ES 写入文档失败: index={}, docId={}", index, docId, e); log.error("ES 写入文档失败: doc={}", docModel, e);
} }
} }