From f91462c6c1b2dae2cbe8f8d3f81f39ccbaf33f41 Mon Sep 17 00:00:00 2001 From: luke Date: Thu, 19 Jun 2025 01:19:11 +0800 Subject: [PATCH] =?UTF-8?q?=C3=A5=C3=A2=C2=80=E4=BC=98=E5=8C=96=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/es/{01-创建es索引.http => DDL.http} | 12 +++ .../domain/common/enums/DocMetaPropEnum.java | 18 ++++ .../impl/AbstractBaseFileImporter.java | 72 +++++++++++----- .../doc/service/impl/ExcelImporter.java | 5 ++ .../doc/service/impl/MarkdownImporter.java | 8 ++ .../domain/doc/service/impl/PdfImporter.java | 5 ++ .../domain/doc/service/impl/WordImporter.java | 5 ++ .../infrastructure/config/ConstantConfig.java | 6 ++ .../north/controller/FileWriteController.java | 21 ++--- .../base/infrastructure/util/CacheUtil.java | 83 +++++++++++++++++++ .../application-dev-windows.properties | 2 +- .../resources/application-docker.properties | 6 +- 12 files changed, 204 insertions(+), 39 deletions(-) rename config/es/{01-创建es索引.http => DDL.http} (77%) create mode 100644 src/main/java/com/knowledge/base/domain/common/enums/DocMetaPropEnum.java create mode 100644 src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java diff --git a/config/es/01-创建es索引.http b/config/es/DDL.http similarity index 77% rename from config/es/01-创建es索引.http rename to config/es/DDL.http index 0ec77a3..e192bbc 100644 --- a/config/es/01-创建es索引.http +++ b/config/es/DDL.http @@ -1,3 +1,4 @@ +#创建索引 PUT /documents { @@ -28,7 +29,18 @@ PUT /documents }, "mtime": { "type": "date" + }, + "uploader": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + "expireTime": { + "type": "date" } } } } + + diff --git a/src/main/java/com/knowledge/base/domain/common/enums/DocMetaPropEnum.java b/src/main/java/com/knowledge/base/domain/common/enums/DocMetaPropEnum.java new file mode 100644 index 0000000..2d04a8b --- /dev/null +++ b/src/main/java/com/knowledge/base/domain/common/enums/DocMetaPropEnum.java @@ -0,0 +1,18 @@ +package com.knowledge.base.domain.common.enums; + +public enum DocMetaPropEnum { + + UPLOADER("uploader", "上传人"), + ACCESS_URL("url", "可访问的链接"), + UPLOAD_TIME("uploadTime", "上传时间戳（毫秒）"), + EXPIRE_TIME("expireTime", "过期时间戳（毫秒）") + ; + public String code; + + private String desc; + + DocMetaPropEnum(String code, String desc) { + this.code = code; + this.desc = desc; + } +} diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java index 5f7ac5d..3477276 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java @@ -1,8 +1,15 @@ package com.knowledge.base.domain.doc.service.impl; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSON; +import cn.hutool.json.JSONUtil; +import com.knowledge.base.domain.common.enums.DocMetaPropEnum; import com.knowledge.base.domain.doc.service.iface.DocumentImporter; import com.knowledge.base.infrastructure.cache.FileCacheService; +import com.knowledge.base.infrastructure.config.ConstantConfig; import com.knowledge.base.infrastructure.config.ThreadPoolConfig; +import com.knowledge.base.infrastructure.util.CacheUtil; +import com.knowledge.base.infrastructure.util.DateUtil; import com.knowledge.base.infrastructure.util.SafeIdUtil; import com.knowledge.base.infrastructure.util.ThreadPoolUtil; import org.elasticsearch.action.delete.DeleteRequest; @@ -18,6 +25,7 @@ import org.springframework.beans.factory.annotation.Value; import java.io.IOException; import java.nio.file.*; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -28,13 +36,9 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class); private static final String INDEX_NAME = "documents"; - @Autowired protected RestHighLevelClient esClient; - @Value("${exclude.file.path.prefix}") - private String excludePrefix; - @Autowired private FileCacheService fileCacheService; @@ -44,10 +48,12 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { protected abstract String getDocTypeCode(); + protected abstract String getExcludePrefix(); + @Override public void importDocuments() throws IOException { Path basePath = Paths.get(getDirectoryPath()).toAbsolutePath().normalize(); - Path excludeBase = Paths.get(excludePrefix).toAbsolutePath().normalize(); + Path excludeBase = Paths.get(getExcludePrefix()).toAbsolutePath().normalize(); Files.walk(basePath) .filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix())) @@ -74,29 +80,26 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { relativePathObj = absPath; } - String relativePath = relativePathObj.toString().replace("\\", "/"); - String fileName = path.getFileName().toString(); - Optional pathOpt = fileCacheService.getPath(fileName); - if (pathOpt.isEmpty()) { - fileCacheService.cachePath(fileName, relativePath, 24 * 60); - logger.info("新增relativePath: {} 文件，准备开始导入...... ", relativePath); - } - Long localMTime = Files.getLastModifiedTime(path).toMillis(); - // 优先从缓存中加载数据，判断是否文件有修改 - Optional lastModifiedTime = fileCacheService.getMTime(relativePath); - if(lastModifiedTime.isPresent() && localMTime.equals(lastModifiedTime.get())) { - logger.info("文件未变动，跳过导入: {}", relativePath); - return; + String relativePath = relativePathObj.toString().replace("\\", "/"); + String fileNameWithSuffix = path.getFileName().toString(); + Optional metaJsonOpt = fileCacheService.getMeta(relativePath); + if(metaJsonOpt.isPresent()) { + // 有文件信息 + JSON metaJson = JSONUtil.parse(metaJsonOpt.get()); + Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class); + if(localMTime.equals(uploadTime)) { + logger.info("文件未变动，跳过导入: {}", relativePath); + return; + } } String content = extractContent(path); - if (content == null || content.isBlank()) { + if (StrUtil.isBlank(content)) { logger.warn("跳过空内容文件: {}", path); return; } - logger.info("relativePath: {}， localMTime： {}", relativePath, localMTime); String docId = SafeIdUtil.encode(relativePath); GetRequest getRequest = new GetRequest(INDEX_NAME, docId); if (esClient.exists(getRequest, RequestOptions.DEFAULT)) { @@ -105,7 +108,8 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { Object esMtime = existingSource.get("mtime"); if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { logger.info("文件未变动，跳过导入: {}", relativePath); - fileCacheService.cacheMTime(relativePath, localMTime, 24 * 60); + String metaJsonStr = transToMetaJson(existingSource); + fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.CACHE_EXPIRED_MINUTES); return; } DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId); @@ -113,11 +117,24 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { logger.info("已删除旧版本文件: {}", relativePath); } + logger.info("[开始进行文件导入......] relativePath: {}， localMTime： {}", relativePath, localMTime); + Map doc = new HashMap<>(); - doc.put("filename", fileName); + doc.put("filename", fileNameWithSuffix); doc.put("filepath", relativePath); doc.put("content", content); doc.put("mtime", localMTime); + if(metaJsonOpt.isPresent()) { + Map metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); + doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER)); + doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, "")); + doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, + Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME))); + } else { + doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER); + doc.put("url", String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath) ); + doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)); + } IndexRequest request = new IndexRequest(INDEX_NAME) .id(docId) @@ -127,12 +144,21 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { logger.info("导入成功: {}", relativePath); // 信息入缓存 - fileCacheService.cacheMTime(relativePath, localMTime, 24 * 60); + fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.CACHE_EXPIRED_MINUTES); } catch (Exception e) { logger.error("导入失败: {}", path, e); } } + private String transToMetaJson(Map esExistingSource) { + return JSONUtil.toJsonStr(Map.of( + DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER), + DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get("mtime"), + DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get("url"), + DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get("expireTime") + )); + } + @Override public String getType() { diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java index 6462b50..6feedf0 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java @@ -35,6 +35,11 @@ public class ExcelImporter extends AbstractBaseFileImporter { return "excel"; } + @Override + protected String getExcludePrefix() { + return excelPath; + } + @Override protected String extractContent(Path path) throws IOException { try (FileInputStream fis = new FileInputStream(path.toFile()); diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java index 660990a..6052538 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java @@ -18,6 +18,9 @@ public class MarkdownImporter extends AbstractBaseFileImporter { @Value("${markdown.path}") private String directoryPath; + @Value("${exclude.file.path.prefix}") + private String excludePrefix; + @Override protected String getDirectoryPath() { return directoryPath; @@ -33,6 +36,11 @@ public class MarkdownImporter extends AbstractBaseFileImporter { return DocTypeEnum.MARKDOWN.code; } + @Override + protected String getExcludePrefix() { + return excludePrefix; + } + @Override protected String extractContent(Path path) throws IOException { return Files.readString(path, StandardCharsets.UTF_8); diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java index b4ed8b3..482a423 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java @@ -35,6 +35,11 @@ public class PdfImporter extends AbstractBaseFileImporter { return DocTypeEnum.PDF.code; } + @Override + protected String getExcludePrefix() { + return directoryPath; + } + @Override protected String extractContent(Path path) throws IOException { try (PDDocument document = PDDocument.load(path.toFile())) { diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java index 034bf7e..fcc6555 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java @@ -36,6 +36,11 @@ public class WordImporter extends AbstractBaseFileImporter { return DocTypeEnum.WORD.code; } + @Override + protected String getExcludePrefix() { + return directoryPath; + } + @Override protected String extractContent(Path path) throws IOException { try (XWPFDocument document = new XWPFDocument(new FileInputStream(path.toFile()))) { diff --git a/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java b/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java index 64dcdbe..a47bfc3 100644 --- a/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java +++ b/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java @@ -1,5 +1,6 @@ package com.knowledge.base.infrastructure.config; +import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -18,4 +19,9 @@ public class ConstantConfig { public static final LocalDateTime LONG_TERM_EXPIRE_TIME = LocalDateTime.of(9999, 12, 31, 23, 59, 59); + public static final long CACHE_EXPIRED_MINUTES = Duration.ofDays(2).toMinutes(); + + public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public"; + + public static final String DEFAULT_UPLOADER = "ADMIN"; } diff --git a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java index ac93e19..370251d 100644 --- a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java +++ b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java @@ -6,6 +6,7 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.knowledge.base.application.service.DocAppService; import com.knowledge.base.application.service.UserAppService; +import com.knowledge.base.domain.common.enums.DocMetaPropEnum; import com.knowledge.base.domain.doc.service.iface.DocumentImporter; import com.knowledge.base.infrastructure.cache.FileCacheService; import com.knowledge.base.infrastructure.config.ConstantConfig; @@ -31,15 +32,11 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import java.io.File; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.time.Duration; -import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.ZoneId; import java.util.*; @RestController @@ -254,16 +251,16 @@ public class FileWriteController { long mtime = Files.getLastModifiedTime(targetPath).toMillis(); Map cacheMeta = new LinkedHashMap<>(); - cacheMeta.put("fileName", originFileNameWithSuffix); - cacheMeta.put("uploader", uploader); - cacheMeta.put("url", url); - cacheMeta.put("bucket", bucket); - cacheMeta.put("expireTime", DateUtil.toMillis(expireTime)); - cacheMeta.put("localAddTime", mtime); + cacheMeta.put(DocMetaPropEnum.UPLOADER.code, uploader); + cacheMeta.put(DocMetaPropEnum.ACCESS_URL.code, url); + cacheMeta.put(DocMetaPropEnum.UPLOAD_TIME.code, mtime); + cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime)); - fileCacheService.cacheMeta(originFileNameWithSuffix, JSONUtil.toJsonStr(cacheMeta), Duration.ofDays(2).toMinutes()); + String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix); + String metaJson = JSONUtil.toJsonStr(cacheMeta); + fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.CACHE_EXPIRED_MINUTES); - logger.info("本地文件信息已保存并写入缓存: {}", originFileNameWithSuffix); + logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson); } } catch (Exception e) { logger.error("异步写入失败: {}", originFileNameWithSuffix, e); diff --git a/src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java b/src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java new file mode 100644 index 0000000..f1ea864 --- /dev/null +++ b/src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java @@ -0,0 +1,83 @@ +package com.knowledge.base.infrastructure.util; + +import cn.hutool.core.map.MapUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.extra.spring.SpringUtil; +import cn.hutool.json.JSONUtil; +import com.knowledge.base.infrastructure.cache.FileCacheService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.ObjectUtils; + +import java.util.Map; +import java.util.Optional; + +@Slf4j +public class CacheUtil { + + /** + * 从 Redis 缓存中加载指定文件的某个元字段值 + * + * @param filename 文件名 + * @param propKey 属性键名 + * @param propClazz 属性类型 + * @return Optional + */ + public static Optional loadFileMetaProp(String filename, String propKey, Class propClazz) { + try { + Optional metaJsonOpt = SpringUtil.getBean(FileCacheService.class).getMeta(filename); + if (metaJsonOpt.isEmpty() || StrUtil.isBlank(metaJsonOpt.get())) { + return Optional.empty(); + } + + Map metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); + Object value = metaMap.get(propKey); + if (ObjectUtils.isEmpty(value)) { + return Optional.empty(); + } + + // 类型转换（考虑 Long、Integer、String 等类型） + T typedValue = convertType(value, propClazz); + return Optional.ofNullable(typedValue); + } catch (Exception e) { + log.error("loadFileMetaProp error.", e); + return Optional.empty(); + } + } + + public static T getFileMetaProp(Map metaMap, String propKey, Class propClazz, T defaultValue) { + if(MapUtil.isEmpty(metaMap) || !metaMap.containsKey(propKey)) { + return defaultValue; + } + try { + Object value = metaMap.get(propKey); + // 类型转换（考虑 Long、Integer、String 等类型） + return convertType(value, propClazz); + } catch (Exception e) { + log.error("getFileMetaProp error.", e); + return defaultValue; + } + } + + /** + * 类型安全地转换 Object -> T + */ + @SuppressWarnings("unchecked") + private static T convertType(Object value, Class targetType) { + if (targetType.isInstance(value)) { + return (T) value; + } + // 特殊类型处理 + if (targetType == Long.class && value instanceof Number) { + return (T) Long.valueOf(((Number) value).longValue()); + } + if (targetType == Integer.class && value instanceof Number) { + return (T) Integer.valueOf(((Number) value).intValue()); + } + if (targetType == String.class) { + return (T) value.toString(); + } + return null; // 其他类型暂不支持 + } + +} + diff --git a/src/main/resources/application-dev-windows.properties b/src/main/resources/application-dev-windows.properties index a6f103e..02c8edf 100644 --- a/src/main/resources/application-dev-windows.properties +++ b/src/main/resources/application-dev-windows.properties @@ -27,7 +27,7 @@ markdown.path=D:/02-documents/01-ahnx-share-src-public/public pdf.path=D:/02-documents/01-os-uploaded-searchable-files word.path=D:/02-documents/01-os-uploaded-searchable-files excel.path=D:/02-documents/01-os-uploaded-searchable-files -object.storage.local-searchable-path=D:/02-documents/01-os-uploaded-searchable-file +object.storage.local-searchable-path=D:/02-documents/01-os-uploaded-searchable-files # mysql diff --git a/src/main/resources/application-docker.properties b/src/main/resources/application-docker.properties index 93cdcd6..1f40a91 100644 --- a/src/main/resources/application-docker.properties +++ b/src/main/resources/application-docker.properties @@ -23,9 +23,9 @@ arthas.tunnel-server=ws://host.docker.internal:7777/ws # 导入目录（容器内路径） markdown.path=/app/import-data -pdf.path=/app/import-data -word.path=/app/import-data -excel.path=/app/import-data +pdf.path=/app/os-uploaded-searchable-files +word.path=/app/os-uploaded-searchable-files +excel.path=/app/os-uploaded-searchable-files object.storage.local-searchable-path=/app/os-uploaded-searchable-files # 相对路径裁剪