åâ€优化导入逻辑

config/es/DDL.http

@@ -1,3 +1,4 @@
+#创建索引
 PUT /documents
 
 {
@@ -28,7 +29,18 @@ PUT /documents
       },
       "mtime": {
         "type": "date"
+      },
+      "uploader": {
+        "type": "keyword"
+      },
+      "url": {
+        "type": "keyword"
+      },
+      "expireTime": {
+        "type": "date"
       }
     }
   }
 }
+
+

src/main/java/com/knowledge/base/domain/common/enums/DocMetaPropEnum.java

@@ -0,0 +1,18 @@
+package com.knowledge.base.domain.common.enums;
+
+public enum DocMetaPropEnum {
+
+    UPLOADER("uploader", "上传人"),
+    ACCESS_URL("url", "可访问的链接"),
+    UPLOAD_TIME("uploadTime", "上传时间戳（毫秒）"),
+    EXPIRE_TIME("expireTime", "过期时间戳（毫秒）")
+    ;
+    public String code;
+
+    private String desc;
+
+    DocMetaPropEnum(String code, String desc) {
+        this.code = code;
+        this.desc = desc;
+    }
+}

src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java

@@ -1,8 +1,15 @@
 package com.knowledge.base.domain.doc.service.impl;
 
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.json.JSON;
+import cn.hutool.json.JSONUtil;
+import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
 import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
 import com.knowledge.base.infrastructure.cache.FileCacheService;
+import com.knowledge.base.infrastructure.config.ConstantConfig;
 import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
+import com.knowledge.base.infrastructure.util.CacheUtil;
+import com.knowledge.base.infrastructure.util.DateUtil;
 import com.knowledge.base.infrastructure.util.SafeIdUtil;
 import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -18,6 +25,7 @@ import org.springframework.beans.factory.annotation.Value;
 
 import java.io.IOException;
 import java.nio.file.*;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -28,13 +36,9 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
     private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class);
     private static final String INDEX_NAME = "documents";
 
-
     @Autowired
     protected RestHighLevelClient esClient;
 
-    @Value("${exclude.file.path.prefix}")
-    private String excludePrefix;
-
     @Autowired
     private FileCacheService fileCacheService;
 
@@ -44,10 +48,12 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
 
     protected abstract String getDocTypeCode();
 
+    protected abstract String getExcludePrefix();
+
     @Override
     public void importDocuments() throws IOException {
         Path basePath = Paths.get(getDirectoryPath()).toAbsolutePath().normalize();
-        Path excludeBase = Paths.get(excludePrefix).toAbsolutePath().normalize();
+        Path excludeBase = Paths.get(getExcludePrefix()).toAbsolutePath().normalize();
 
         Files.walk(basePath)
                 .filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
@@ -74,29 +80,26 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
                 relativePathObj = absPath;
             }
 
-            String relativePath = relativePathObj.toString().replace("\\", "/");
-            String fileName = path.getFileName().toString();
-            Optional<String> pathOpt = fileCacheService.getPath(fileName);
-            if (pathOpt.isEmpty()) {
-                fileCacheService.cachePath(fileName, relativePath, 24 * 60);
-                logger.info("新增relativePath: {} 文件，准备开始导入...... ", relativePath);
-            }
-
             Long localMTime = Files.getLastModifiedTime(path).toMillis();
-            // 优先从缓存中加载数据，判断是否文件有修改
+            String relativePath = relativePathObj.toString().replace("\\", "/");
-            Optional<Long> lastModifiedTime = fileCacheService.getMTime(relativePath);
+            String fileNameWithSuffix = path.getFileName().toString();
-            if(lastModifiedTime.isPresent() && localMTime.equals(lastModifiedTime.get())) {
+            Optional<String> metaJsonOpt = fileCacheService.getMeta(relativePath);
+            if(metaJsonOpt.isPresent()) {
+                // 有文件信息
+                JSON metaJson = JSONUtil.parse(metaJsonOpt.get());
+                Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class);
+                if(localMTime.equals(uploadTime)) {
                     logger.info("文件未变动，跳过导入: {}", relativePath);
                     return;
                 }
+            }
 
             String content = extractContent(path);
-            if (content == null || content.isBlank()) {
+            if (StrUtil.isBlank(content)) {
                 logger.warn("跳过空内容文件: {}", path);
                 return;
             }
 
-            logger.info("relativePath: {}， localMTime： {}", relativePath, localMTime);
             String docId = SafeIdUtil.encode(relativePath);
             GetRequest getRequest = new GetRequest(INDEX_NAME, docId);
             if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
@@ -105,7 +108,8 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
                 Object esMtime = existingSource.get("mtime");
                 if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
                     logger.info("文件未变动，跳过导入: {}", relativePath);
-                    fileCacheService.cacheMTime(relativePath, localMTime, 24 * 60);
+                    String metaJsonStr = transToMetaJson(existingSource);
+                    fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.CACHE_EXPIRED_MINUTES);
                     return;
                 }
                 DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId);
@@ -113,11 +117,24 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
                 logger.info("已删除旧版本文件: {}", relativePath);
             }
 
+            logger.info("[开始进行文件导入......] relativePath: {}， localMTime： {}", relativePath, localMTime);
+
             Map<String, Object> doc = new HashMap<>();
-            doc.put("filename", fileName);
+            doc.put("filename", fileNameWithSuffix);
             doc.put("filepath", relativePath);
             doc.put("content", content);
             doc.put("mtime", localMTime);
+            if(metaJsonOpt.isPresent()) {
+                Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
+                doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER));
+                doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, ""));
+                doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code,
+                        Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)));
+            } else {
+                doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER);
+                doc.put("url",  String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath) );
+                doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME));
+            }
 
             IndexRequest request = new IndexRequest(INDEX_NAME)
                     .id(docId)
@@ -127,12 +144,21 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
             logger.info("导入成功: {}", relativePath);
 
             // 信息入缓存
-            fileCacheService.cacheMTime(relativePath, localMTime, 24 * 60);
+            fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.CACHE_EXPIRED_MINUTES);
         } catch (Exception e) {
             logger.error("导入失败: {}", path, e);
         }
     }
 
+    private String transToMetaJson(Map<String, Object> esExistingSource) {
+        return JSONUtil.toJsonStr(Map.of(
+                DocMetaPropEnum.UPLOADER.code,  Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER),
+                DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get("mtime"),
+                DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get("url"),
+                DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get("expireTime")
+        ));
+    }
+
 
     @Override
     public String getType() {

src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java

@@ -35,6 +35,11 @@ public class ExcelImporter extends AbstractBaseFileImporter {
         return "excel";
     }
 
+    @Override
+    protected String getExcludePrefix() {
+        return excelPath;
+    }
+
     @Override
     protected String extractContent(Path path) throws IOException {
         try (FileInputStream fis = new FileInputStream(path.toFile());

src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java

@@ -18,6 +18,9 @@ public class MarkdownImporter extends AbstractBaseFileImporter {
     @Value("${markdown.path}")
     private String directoryPath;
 
+    @Value("${exclude.file.path.prefix}")
+    private String excludePrefix;
+
     @Override
     protected String getDirectoryPath() {
         return directoryPath;
@@ -33,6 +36,11 @@ public class MarkdownImporter extends AbstractBaseFileImporter {
         return DocTypeEnum.MARKDOWN.code;
     }
 
+    @Override
+    protected String getExcludePrefix() {
+        return excludePrefix;
+    }
+
     @Override
     protected String extractContent(Path path) throws IOException {
         return Files.readString(path, StandardCharsets.UTF_8);

src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java

@@ -35,6 +35,11 @@ public class PdfImporter extends AbstractBaseFileImporter {
         return DocTypeEnum.PDF.code;
     }
 
+    @Override
+    protected String getExcludePrefix() {
+        return directoryPath;
+    }
+
     @Override
     protected String extractContent(Path path) throws IOException {
         try (PDDocument document = PDDocument.load(path.toFile())) {

src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java

@@ -36,6 +36,11 @@ public class WordImporter extends AbstractBaseFileImporter {
         return DocTypeEnum.WORD.code;
     }
 
+    @Override
+    protected String getExcludePrefix() {
+        return directoryPath;
+    }
+
     @Override
     protected String extractContent(Path path) throws IOException {
         try (XWPFDocument document = new XWPFDocument(new FileInputStream(path.toFile()))) {

src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java

@@ -1,5 +1,6 @@
 package com.knowledge.base.infrastructure.config;
 
+import java.time.Duration;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 
@@ -18,4 +19,9 @@ public class ConstantConfig {
 
     public static final LocalDateTime LONG_TERM_EXPIRE_TIME = LocalDateTime.of(9999, 12, 31, 23, 59, 59);
 
+    public static final long CACHE_EXPIRED_MINUTES = Duration.ofDays(2).toMinutes();
+
+    public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public";
+
+    public static final String DEFAULT_UPLOADER = "ADMIN";
 }

src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java

@@ -6,6 +6,7 @@ import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
 import com.knowledge.base.application.service.DocAppService;
 import com.knowledge.base.application.service.UserAppService;
+import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
 import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
 import com.knowledge.base.infrastructure.cache.FileCacheService;
 import com.knowledge.base.infrastructure.config.ConstantConfig;
@@ -31,15 +32,11 @@ import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.File;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.time.Duration;
-import java.time.LocalDate;
 import java.time.LocalDateTime;
-import java.time.ZoneId;
 import java.util.*;
 
 @RestController
@@ -254,16 +251,16 @@ public class FileWriteController {
                 long mtime = Files.getLastModifiedTime(targetPath).toMillis();
 
                 Map<String, Object> cacheMeta = new LinkedHashMap<>();
-                cacheMeta.put("fileName", originFileNameWithSuffix);
+                cacheMeta.put(DocMetaPropEnum.UPLOADER.code, uploader);
-                cacheMeta.put("uploader", uploader);
+                cacheMeta.put(DocMetaPropEnum.ACCESS_URL.code, url);
-                cacheMeta.put("url", url);
+                cacheMeta.put(DocMetaPropEnum.UPLOAD_TIME.code, mtime);
-                cacheMeta.put("bucket", bucket);
+                cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime));
-                cacheMeta.put("expireTime", DateUtil.toMillis(expireTime));
-                cacheMeta.put("localAddTime", mtime);
 
-                fileCacheService.cacheMeta(originFileNameWithSuffix, JSONUtil.toJsonStr(cacheMeta), Duration.ofDays(2).toMinutes());
+                String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
+                String metaJson = JSONUtil.toJsonStr(cacheMeta);
+                fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.CACHE_EXPIRED_MINUTES);
 
-                logger.info("本地文件信息已保存并写入缓存: {}", originFileNameWithSuffix);
+                logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson);
             }
         } catch (Exception e) {
             logger.error("异步写入失败: {}", originFileNameWithSuffix, e);

src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java

@@ -0,0 +1,83 @@
+package com.knowledge.base.infrastructure.util;
+
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.extra.spring.SpringUtil;
+import cn.hutool.json.JSONUtil;
+import com.knowledge.base.infrastructure.cache.FileCacheService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.ObjectUtils;
+
+import java.util.Map;
+import java.util.Optional;
+
+@Slf4j
+public class CacheUtil {
+
+    /**
+     * 从 Redis 缓存中加载指定文件的某个元字段值
+     *
+     * @param filename   文件名
+     * @param propKey    属性键名
+     * @param propClazz  属性类型
+     * @return Optional<T>
+     */
+    public static <T> Optional<T> loadFileMetaProp(String filename, String propKey, Class<T> propClazz) {
+        try {
+            Optional<String> metaJsonOpt = SpringUtil.getBean(FileCacheService.class).getMeta(filename);
+            if (metaJsonOpt.isEmpty() || StrUtil.isBlank(metaJsonOpt.get())) {
+                return Optional.empty();
+            }
+
+            Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
+            Object value = metaMap.get(propKey);
+            if (ObjectUtils.isEmpty(value)) {
+                return Optional.empty();
+            }
+
+            // 类型转换（考虑 Long、Integer、String 等类型）
+            T typedValue = convertType(value, propClazz);
+            return Optional.ofNullable(typedValue);
+        } catch (Exception e) {
+            log.error("loadFileMetaProp error.", e);
+            return Optional.empty();
+        }
+    }
+
+    public static <T> T getFileMetaProp(Map<String, Object> metaMap, String propKey, Class<T> propClazz, T defaultValue) {
+        if(MapUtil.isEmpty(metaMap) || !metaMap.containsKey(propKey)) {
+            return defaultValue;
+        }
+        try {
+            Object value = metaMap.get(propKey);
+            // 类型转换（考虑 Long、Integer、String 等类型）
+            return convertType(value, propClazz);
+        } catch (Exception e) {
+            log.error("getFileMetaProp error.", e);
+            return defaultValue;
+        }
+    }
+
+    /**
+     * 类型安全地转换 Object -> T
+     */
+    @SuppressWarnings("unchecked")
+    private static <T> T convertType(Object value, Class<T> targetType) {
+        if (targetType.isInstance(value)) {
+            return (T) value;
+        }
+        // 特殊类型处理
+        if (targetType == Long.class && value instanceof Number) {
+            return (T) Long.valueOf(((Number) value).longValue());
+        }
+        if (targetType == Integer.class && value instanceof Number) {
+            return (T) Integer.valueOf(((Number) value).intValue());
+        }
+        if (targetType == String.class) {
+            return (T) value.toString();
+        }
+        return null; // 其他类型暂不支持
+    }
+
+}
+

src/main/resources/application-dev-windows.properties

@@ -27,7 +27,7 @@ markdown.path=D:/02-documents/01-ahnx-share-src-public/public
 pdf.path=D:/02-documents/01-os-uploaded-searchable-files
 word.path=D:/02-documents/01-os-uploaded-searchable-files
 excel.path=D:/02-documents/01-os-uploaded-searchable-files
-object.storage.local-searchable-path=D:/02-documents/01-os-uploaded-searchable-file
+object.storage.local-searchable-path=D:/02-documents/01-os-uploaded-searchable-files
 
 
 # mysql

src/main/resources/application-docker.properties

@@ -23,9 +23,9 @@ arthas.tunnel-server=ws://host.docker.internal:7777/ws
 
 # 导入目录（容器内路径）
 markdown.path=/app/import-data
-pdf.path=/app/import-data
+pdf.path=/app/os-uploaded-searchable-files
-word.path=/app/import-data
+word.path=/app/os-uploaded-searchable-files
-excel.path=/app/import-data
+excel.path=/app/os-uploaded-searchable-files
 object.storage.local-searchable-path=/app/os-uploaded-searchable-files
 
 # 相对路径裁剪