From 19eec52e52e54e25a37ec70c8929f32d4350d4dc Mon Sep 17 00:00:00 2001 From: luke Date: Fri, 20 Jun 2025 14:41:41 +0800 Subject: [PATCH] opt --- .../base/domain/doc/model/FileEsField.java | 16 +++++ .../base/domain/doc/model/FileEsModel.java | 32 +++++++++ .../impl/AbstractBaseFileImporter.java | 69 +++++++++++-------- .../north/controller/FileQueryController.java | 13 ++-- .../south/es/FileElasticsearchGateway.java | 33 +++++---- 5 files changed, 116 insertions(+), 47 deletions(-) create mode 100644 src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java create mode 100644 src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java diff --git a/src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java b/src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java new file mode 100644 index 0000000..b24b411 --- /dev/null +++ b/src/main/java/com/knowledge/base/domain/doc/model/FileEsField.java @@ -0,0 +1,16 @@ +package com.knowledge.base.domain.doc.model; + +/** + * @author Luke.ye + * @date 2025/6/20 13:57 + */ +public class FileEsField { + public static final String INDEX = "documents"; + public static final String FILENAME = "filename"; + public static final String FILEPATH = "filepath"; + public static final String CONTENT = "content"; + public static final String MTIME = "mtime"; + public static final String UPLOADER = "uploader"; + public static final String URL = "url"; + public static final String EXPIRE_TIME = "expireTime"; +} diff --git a/src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java b/src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java new file mode 100644 index 0000000..1c753f7 --- /dev/null +++ b/src/main/java/com/knowledge/base/domain/doc/model/FileEsModel.java @@ -0,0 +1,32 @@ +package com.knowledge.base.domain.doc.model; + +import com.knowledge.base.infrastructure.util.SafeIdUtil; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 文件文档模型（字段应于 {@link FileEsField} 保持一致） + * @author Luke.ye + * @date 2025/6/20 13:45 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FileEsModel { + + private String filename; // 文档标题 + private String filepath; // 相对路径，作为唯一 ID + private String content; // 文本内容 + private Long mtime; // 最后修改时间 + + private String uploader; // 上传人 + private String url; // 文件访问链接 + private Long expireTime; // 过期时间戳 + + public String buildDocId() { + return SafeIdUtil.encode(this.filepath); + } +} diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java index a0ec845..804470a 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java @@ -1,5 +1,6 @@ package com.knowledge.base.domain.doc.service.impl; +import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.map.MapUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSON; @@ -7,6 +8,8 @@ import cn.hutool.json.JSONUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.knowledge.base.domain.common.enums.DocMetaPropEnum; +import com.knowledge.base.domain.doc.model.FileEsField; +import com.knowledge.base.domain.doc.model.FileEsModel; import com.knowledge.base.domain.doc.model.OSRecordDO; import com.knowledge.base.domain.doc.service.iface.DocumentImporter; import com.knowledge.base.domain.doc.service.iface.FileDomainService; @@ -122,29 +125,30 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { // === Step 4: 检查 ES 是否已有未变动版本 === String docId = SafeIdUtil.encode(localRelaFilePath); - GetResponse existing = esGateway.getIfExists(INDEX_NAME, docId); + GetResponse existing = esGateway.getIfExists(docId); if(Objects.nonNull(existing)) { Map existingSource = existing.getSourceAsMap(); - Object esMtime = existingSource.get("mtime"); + Object esMtime = existingSource.get(FileEsField.MTIME); if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { logger.info("文件未变动，跳过导入: {}", localRelaFilePath); fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(existingSource), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); return true; } - esGateway.deleteDoc(INDEX_NAME, docId); + esGateway.deleteDoc(docId); logger.info("[ES] 已删除旧版本文件: {}", localRelaFilePath); } logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime); // === Step 5: 构建待写入文档 === - Map doc = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo); + FileEsModel fileEsModel = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo); // === Step 6: 写入 Elasticsearch 并更新缓存 === - esGateway.saveDoc(INDEX_NAME, docId, doc); + esGateway.saveDoc(fileEsModel); logger.info("导入成功: {}", localRelaFilePath); - fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); + String metaJson = transToMetaJson(BeanUtil.beanToMap(fileEsModel, false, true)); + fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); return true; } catch (Exception e) { @@ -153,36 +157,47 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { } } - private Map buildDocument(String filename, String filepath, String content, Long mtime, Map extInfo) { - Map doc = new HashMap<>(); - doc.put("filename", filename); - doc.put("filepath", filepath); - doc.put("content", content); - doc.put("mtime", mtime); + + private FileEsModel buildDocument(String filename, String filepath, String content, Long mtime, Map extInfo) { + String uploader = ConstantConfig.DEFAULT_UPLOADER; + String url = ""; + Long expireTime = DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME); Optional metaJsonOpt = fileCacheService.getMeta(filepath); if (metaJsonOpt.isPresent()) { + // 优先使用缓存 Map metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); - doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER)); - doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY)); - doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME))); + uploader = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, uploader); + url = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, url); + expireTime = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, expireTime); } else { + // 使用传入 extInfo extInfo = MapUtil.isEmpty(extInfo) ? MapUtil.empty() : extInfo; boolean hasAllMeta = extInfo.keySet().containsAll(Lists.newArrayList( - DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code)); + DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code + )); + if (hasAllMeta) { - doc.put("uploader", extInfo.get(DocMetaPropEnum.UPLOADER.code)); - doc.put("url", extInfo.get(DocMetaPropEnum.ACCESS_URL.code)); - doc.put("expireTime", extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code)); + uploader = (String) extInfo.get(DocMetaPropEnum.UPLOADER.code); + url = (String) extInfo.get(DocMetaPropEnum.ACCESS_URL.code); + expireTime = (Long) extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code); } else { Map props = buildDocMetaProps(filepath); - doc.put("uploader", props.get(DocMetaPropEnum.UPLOADER.code)); - doc.put("url", props.get(DocMetaPropEnum.ACCESS_URL.code)); - doc.put("expireTime", props.get(DocMetaPropEnum.EXPIRE_TIME.code)); + uploader = (String) props.get(DocMetaPropEnum.UPLOADER.code); + url = (String) props.get(DocMetaPropEnum.ACCESS_URL.code); + expireTime = (Long) props.get(DocMetaPropEnum.EXPIRE_TIME.code); } } - return doc; + return FileEsModel.builder() + .filename(filename) + .filepath(filepath) + .content(content) + .mtime(mtime) + .uploader(uploader) + .url(url) + .expireTime(expireTime) + .build(); } @@ -218,10 +233,10 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { private String transToMetaJson(Map esExistingSource) { return JSONUtil.toJsonStr(Map.of( - DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER), - DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get("mtime"), - DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get("url"), - DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get("expireTime") + DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get(FileEsField.UPLOADER)).orElse(ConstantConfig.DEFAULT_UPLOADER), + DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get(FileEsField.MTIME), + DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get(FileEsField.URL), + DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get(FileEsField.EXPIRE_TIME) )); } diff --git a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java index 44bf4d8..4ef17ad 100644 --- a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java +++ b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java @@ -8,6 +8,7 @@ import com.knowledge.base.application.service.DocAppService; import com.knowledge.base.application.service.UserAppService; import com.knowledge.base.domain.common.enums.DocMetaPropEnum; import com.knowledge.base.domain.common.model.PageResult; +import com.knowledge.base.domain.doc.model.FileEsField; import com.knowledge.base.infrastructure.cache.iface.FileCacheService; import com.knowledge.base.infrastructure.config.ConstantConfig; import com.knowledge.base.infrastructure.north.dto.SearchReq; @@ -165,13 +166,13 @@ public class FileQueryController { for (SearchHit hit : response.getHits()) { Map source = hit.getSourceAsMap(); Map result = new LinkedHashMap<>(); - result.put("filename", source.get("filename")); - result.put("filepath", source.get("filepath")); - result.put("mtime", source.get("mtime")); - result.put("uploader", source.getOrDefault("uploader", ConstantConfig.DEFAULT_UPLOADER)); - result.put("url", source.get("url")); + result.put("filename", source.get(FileEsField.FILENAME)); + result.put("filepath", source.get(FileEsField.FILEPATH)); + result.put("mtime", source.get(FileEsField.MTIME)); + result.put("uploader", source.getOrDefault(FileEsField.UPLOADER, ConstantConfig.DEFAULT_UPLOADER)); + result.put("url", source.get(FileEsField.URL)); - String content = (String) source.get("content"); + String content = (String) source.get(FileEsField.CONTENT); String summary = extractMultiSnippet(content, keywords, 50); result.put("summary", summary); diff --git a/src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java b/src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java index 0037bdb..887beed 100644 --- a/src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java +++ b/src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java @@ -1,5 +1,8 @@ package com.knowledge.base.infrastructure.south.es; +import cn.hutool.core.bean.BeanUtil; +import com.knowledge.base.domain.doc.model.FileEsField; +import com.knowledge.base.domain.doc.model.FileEsModel; import com.knowledge.base.infrastructure.util.SafeIdUtil; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.bulk.BulkRequest; @@ -36,7 +39,7 @@ import java.util.Map; @Component public class FileElasticsearchGateway { - private static final String INDEX_NAME = "documents"; + private static final String INDEX_NAME = FileEsField.INDEX; @Autowired private RestHighLevelClient esClient; @@ -50,7 +53,7 @@ public class FileElasticsearchGateway { .query(query) .from(from) .size(size) - .sort("mtime", SortOrder.DESC); + .sort(FileEsField.MTIME, SortOrder.DESC); SearchRequest request = new SearchRequest(INDEX_NAME).source(builder); return esClient.search(request, RequestOptions.DEFAULT); @@ -71,8 +74,8 @@ public class FileElasticsearchGateway { for (String keyword : group) { groupQuery.must(QueryBuilders.boolQuery() - .should(QueryBuilders.matchQuery("filename", keyword).analyzer("ik_smart")) - .should(QueryBuilders.matchQuery("content", keyword).analyzer("ik_smart")) + .should(QueryBuilders.matchQuery(FileEsField.FILENAME, keyword).analyzer("ik_smart")) + .should(QueryBuilders.matchQuery(FileEsField.CONTENT, keyword).analyzer("ik_smart")) ); } @@ -82,7 +85,7 @@ public class FileElasticsearchGateway { outerQuery.minimumShouldMatch(1); // 过滤掉过期文件 - outerQuery.filter(QueryBuilders.rangeQuery("expireTime") + outerQuery.filter(QueryBuilders.rangeQuery(FileEsField.EXPIRE_TIME) .gte(System.currentTimeMillis()) .timeZone("+08:00")); @@ -150,15 +153,15 @@ public class FileElasticsearchGateway { /** * 判断文档是否存在并获取内容 */ - public GetResponse getIfExists(String index, String docId) { + public GetResponse getIfExists(String docId) { try { - GetRequest getRequest = new GetRequest(index, docId); + GetRequest getRequest = new GetRequest(INDEX_NAME, docId); if (!esClient.exists(getRequest, RequestOptions.DEFAULT)) { return null; } return esClient.get(getRequest, RequestOptions.DEFAULT); } catch (Exception e) { - log.error("ES 获取文档失败: index={}, docId={}", index, docId, e); + log.error("ES 获取文档失败: index={}, docId={}", INDEX_NAME, docId, e); return null; } } @@ -166,23 +169,25 @@ public class FileElasticsearchGateway { /** * 删除文档 */ - public void deleteDoc(String index, String docId) { + public void deleteDoc(String docId) { try { - esClient.delete(new DeleteRequest(index, docId), RequestOptions.DEFAULT); + esClient.delete(new DeleteRequest(INDEX_NAME, docId), RequestOptions.DEFAULT); } catch (Exception e) { - log.warn("ES 删除文档失败: index={}, docId={}", index, docId, e); + log.warn("ES 删除文档失败: index={}, docId={}", INDEX_NAME, docId, e); } } /** * 写入/更新文档 */ - public void saveDoc(String index, String docId, Map source) { + public void saveDoc(FileEsModel docModel) { try { - IndexRequest request = new IndexRequest(index).id(docId).source(source); + String docId = docModel.buildDocId(); + Map source = BeanUtil.beanToMap(docModel, false, true); + IndexRequest request = new IndexRequest(INDEX_NAME).id(docId).source(source); esClient.index(request, RequestOptions.DEFAULT); } catch (Exception e) { - log.error("ES 写入文档失败: index={}, docId={}", index, docId, e); + log.error("ES 写入文档失败: doc={}", docModel, e); } }