完成导入及搜索优化

Dockerfile/Dockerfile

@@ -0,0 +1,11 @@
+# 使用 JDK 11
+FROM openjdk:11-jdk-slim
+
+# 创建工作目录
+WORKDIR /app
+
+# 拷贝 JAR
+COPY doc-parser-server-0.0.1-SNAPSHOT.jar app.jar
+
+# 启动命令：指定 spring.profiles.active=docker
+ENTRYPOINT ["java", "-jar", "app.jar", "--spring.profiles.active=docker"]

config/01-创建es索引.http

@@ -1,4 +1,5 @@
 PUT /documents
+
 {
   "settings": {
     "analysis": {
@@ -24,6 +25,9 @@ PUT /documents
         "type": "text",
         "analyzer": "ik_max_word",
         "search_analyzer": "ik_smart"
+      },
+      "mtime": {
+        "type": "date"
       }
     }
   }

config/99-常用es查询.http

@@ -0,0 +1,49 @@
+#{
+#  "keywordGroups": [
+#    ["Luke.Ye", "设计模式"],
+#  ]
+#}
+# 表示 (设计模式 AND Luke.Ye)
+POST /documents/_search
+
+{
+  "query": {
+    "bool": {
+      "should": [
+        {
+          "bool": {
+            "must": [
+              { "multi_match": { "query": "设计模式", "fields": ["filename", "content"], "type": "phrase" }},
+              { "multi_match": { "query": "Luke.Ye", "fields": ["filename", "content"], "type": "phrase" }}
+            ]
+          }
+        }
+      ],
+      "minimum_should_match": 1
+    }
+  },
+  "highlight": {
+    "pre_tags": ["<mark>"],
+    "post_tags": ["</mark>"],
+    "fields": {
+      "content": {
+        "fragment_size": 100,
+        "number_of_fragments": 3
+      },
+      "filename": {
+        "fragment_size": 100,
+        "number_of_fragments": 1
+      }
+    }
+  },
+  "sort": [
+    { "mtime": "desc" }
+  ],
+  "size": 10,
+  "from": 0
+}
+
+
+
+
+

src/main/java/com/doc/parser/DocParserApplication.java

@@ -2,10 +2,12 @@ package com.doc.parser;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 /**
  * @author Luke.ye
  */
+@EnableScheduling
 @SpringBootApplication
 public class DocParserApplication {
 

src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java

@@ -1,6 +1,9 @@
 package com.doc.parser.domain.impl;
 
 import com.doc.parser.domain.iface.DocumentImporter;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -47,20 +50,44 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
                             return;
                         }
 
+                        // === 生成 filepath ===
                         String absPath = path.toAbsolutePath().toString().replace("\\", "/");
                         String cleanedPrefix = excludePrefix.replace("\\", "/");
                         if (!cleanedPrefix.endsWith("/")) {
                             cleanedPrefix += "/";
                         }
-
                         String relativePath = absPath.startsWith(cleanedPrefix)
                                 ? absPath.substring(cleanedPrefix.length())
                                 : absPath;
 
+                        // 以 relativePath 作为文档 ID
+                        GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
+                        boolean skip = false;
+                        long localMTime = Files.getLastModifiedTime(path).toMillis();
+
+                        if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
+                            GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
+                            Map<String, Object> existingSource = existing.getSourceAsMap();
+                            Object esMtime = existingSource.get("mtime");
+                            if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
+                                logger.info("文件未变动，跳过导入: {}", relativePath);
+                                skip = true;
+                            } else {
+                                // 删除旧版本
+                                DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
+                                esClient.delete(deleteRequest, RequestOptions.DEFAULT);
+                                logger.info("已删除旧版本文件: {}", relativePath);
+                            }
+                        }
+
+                        if (skip) return;
+
+                        // === 构建文档 ===
                         Map<String, Object> doc = new HashMap<>();
                         doc.put("filename", path.getFileName().toString());
                         doc.put("filepath", relativePath);
                         doc.put("content", content);
+                        doc.put("mtime", localMTime);
 
                         IndexRequest request = new IndexRequest(INDEX_NAME)
                                 .id(relativePath)
@@ -68,6 +95,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
 
                         esClient.index(request, RequestOptions.DEFAULT);
                         logger.info("导入成功: {}", relativePath);
+
                     } catch (Exception e) {
                         logger.error("导入失败: {}", path, e);
                     }

src/main/java/com/doc/parser/domain/schedule/ImportScheduler.java

@@ -0,0 +1,49 @@
+package com.doc.parser.domain.schedule;
+
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/21 14:09
+ */
+@Component
+public class ImportScheduler {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImportScheduler.class);
+
+    @Value("${import.schedule.enabled:true}")
+    private boolean scheduleEnabled;
+
+    private final List<DocumentImporter> importers;
+
+    public ImportScheduler(List<DocumentImporter> importers) {
+        this.importers = importers;
+    }
+
+    @Scheduled(cron = "${import.schedule.cron:0 0 * * * *}")
+    public void scheduledImport() {
+        if (!scheduleEnabled) {
+            logger.info("导入定时任务已禁用");
+            return;
+        }
+
+        logger.info("开始执行定时导入任务");
+        for (DocumentImporter importer : importers) {
+            try {
+                logger.info("执行导入器: {}", importer.getType());
+                importer.importDocuments();
+            } catch (Exception e) {
+                logger.error("导入器执行失败: {}", importer.getType(), e);
+            }
+        }
+        logger.info("定时导入任务执行完成");
+    }
+}
+

src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java

@@ -1,18 +1,21 @@
 package com.doc.parser.infrastructure.north.controller;
 
-
+import com.doc.parser.infrastructure.north.dto.SearchReq;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.*;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.sort.SortOrder;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.*;
 
+import java.io.IOException;
 import java.util.*;
+import java.util.regex.Pattern;
 
 /**
  * @author Luke.ye
@@ -31,66 +34,81 @@ public class SearchController {
         this.esClient = esClient;
     }
 
-    @GetMapping
+    @PostMapping
-    public Map<String, Object> search(
+    public Map<String, Object> search(@RequestBody SearchReq requestBody) throws IOException {
-            @RequestParam String q,
+        int page = requestBody.getPage() != null && requestBody.getPage() > 0 ? requestBody.getPage() : 1;
-            @RequestParam(defaultValue = "1") int page,
+        int size = requestBody.getSize() != null && requestBody.getSize() > 0 ? requestBody.getSize() : defaultPageSize;
-            @RequestParam(required = false) Integer size
+        int from = (page - 1) * size;
-    ) throws Exception {
-        if (page < 1) page = 1;
-        int pageSize = (size != null && size > 0) ? size : defaultPageSize;
 
-        int from = (page - 1) * pageSize;
+        // 构建外层 OR 查询
+        BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();
+        for (List<String> group : requestBody.getKeywordGroups()) {
+            BoolQueryBuilder groupQuery = QueryBuilders.boolQuery();
+            for (String keyword : group) {
+                // 使用 match_phrase 精确匹配短语
+                groupQuery.must(QueryBuilders.multiMatchQuery(keyword, "filename", "content")
+                        .type(MultiMatchQueryBuilder.Type.PHRASE));
+            }
+            outerQuery.should(groupQuery);
+        }
+        outerQuery.minimumShouldMatch(1); // 至少匹配一组
 
-        SearchRequest request = new SearchRequest("documents");
-
-        // 高亮配置
-        HighlightBuilder highlight = new HighlightBuilder()
-                .field("content")
-                .field("filename")
-                .preTags("<mark>")
-                .postTags("</mark>")
-                .fragmentSize(80)
-                .numOfFragments(3)
-                .requireFieldMatch(false);
-
-        // 查询构建
         SearchSourceBuilder builder = new SearchSourceBuilder()
-                .query(QueryBuilders.multiMatchQuery(q, "filename", "content"))
+                .query(outerQuery)
                 .from(from)
-                .size(pageSize)
+                .size(size)
-                .highlighter(highlight);
+                .sort("mtime", SortOrder.DESC);
 
-        request.source(builder);
+        SearchRequest request = new SearchRequest("documents").source(builder);
         SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
 
-        // 结果封装
         List<Map<String, Object>> results = new ArrayList<>();
         for (SearchHit hit : response.getHits()) {
             Map<String, Object> source = hit.getSourceAsMap();
             Map<String, Object> result = new LinkedHashMap<>();
             result.put("filename", source.get("filename"));
             result.put("filepath", source.get("filepath"));
+            result.put("mtime", source.get("mtime"));
 
-            Map<String, Object> highlights = new HashMap<>();
+            String content = (String) source.get("content");
-            for (Map.Entry<String, HighlightField> entry : hit.getHighlightFields().entrySet()) {
+            String summary = extractMultiSnippet(content, requestBody.getFlatKeywords(), 50);
-                List<String> fragments = new ArrayList<>();
+            result.put("summary", summary);
-                Arrays.stream(entry.getValue().fragments())
-                        .forEach(f -> fragments.add(f.string()));
-                highlights.put(entry.getKey(), fragments);
-            }
 
-            result.put("highlight", highlights.isEmpty() ? null : highlights);
             results.add(result);
         }
 
-        // 返回分页结构
         Map<String, Object> responseBody = new LinkedHashMap<>();
         responseBody.put("total", response.getHits().getTotalHits().value);
         responseBody.put("page", page);
-        responseBody.put("size", pageSize);
+        responseBody.put("size", size);
         responseBody.put("results", results);
-
         return responseBody;
     }
+
+    private String extractMultiSnippet(String content, List<String> keywords, int contextLength) {
+        if (content == null || keywords == null || keywords.isEmpty()) return "";
+
+        String lowered = content.toLowerCase();
+        Set<String> matchedSnippets = new LinkedHashSet<>();
+
+        for (String keyword : keywords) {
+            String loweredKeyword = keyword.toLowerCase();
+            int index = lowered.indexOf(loweredKeyword);
+            if (index == -1) continue;
+
+            int start = Math.max(0, index - contextLength);
+            int end = Math.min(content.length(), index + keyword.length() + contextLength);
+            String snippet = content.substring(start, end);
+            snippet = snippet.replaceAll("(?i)" + Pattern.quote(keyword), "<mark>$0</mark>");
+            matchedSnippets.add(snippet);
+        }
+
+        if (matchedSnippets.isEmpty()) {
+            return content.length() <= 100 ? content : content.substring(0, 100) + "...";
+        }
+
+        return String.join(" ... ", matchedSnippets);
+    }
 }
+
+

src/main/java/com/doc/parser/infrastructure/north/dto/SearchReq.java

@@ -0,0 +1,51 @@
+package com.doc.parser.infrastructure.north.dto;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/21 14:31
+ */
+public class SearchReq implements Serializable {
+    private static final long serialVersionUID = 563009883661483142L;
+
+    private List<List<String>> keywordGroups;
+    private Integer page;
+    private Integer size;
+
+    public List<List<String>> getKeywordGroups() {
+        return keywordGroups != null ? keywordGroups : new ArrayList<>();
+    }
+
+    public void setKeywordGroups(List<List<String>> keywordGroups) {
+        this.keywordGroups = keywordGroups;
+    }
+
+    public Integer getPage() {
+        return page;
+    }
+
+    public void setPage(Integer page) {
+        this.page = page;
+    }
+
+    public Integer getSize() {
+        return size;
+    }
+
+    public void setSize(Integer size) {
+        this.size = size;
+    }
+
+    // 所有关键词合并，用于高亮摘要
+    public List<String> getFlatKeywords() {
+        return keywordGroups == null
+                ? new ArrayList<>()
+                : keywordGroups.stream().flatMap(e -> e.stream()).collect(Collectors.toList());
+
+    }
+}

src/main/resources/application-dev-mac.properties

@@ -0,0 +1,28 @@
+# 日志
+logging.config=classpath:log/log4j.xml
+
+# nacos相关
+## nacos公用
+spring.cloud.nacos.config.data-id-without-suffix=nacos
+spring.cloud.nacos.config.file-extension=properties
+spring.config.import=nacos:${spring.cloud.nacos.config.data-id-without-suffix}.${spring.cloud.nacos.config.file-extension}?refresh=true
+## Nacos配置中心
+spring.cloud.nacos.config.username=nacos
+spring.cloud.nacos.config.password=lukeye
+spring.cloud.nacos.config.contextPath=/nacos
+spring.cloud.nacos.config.server-addr=http://101.132.255.39:8848
+spring.cloud.nacos.config.namespace=lukeye
+spring.cloud.nacos.config.refreshEnabled=true
+
+# Arthas配置
+arthas.telnetPort=-1
+arthas.httpPort=-1
+arthas.ip=127.0.0.1
+arthas.appName=${spring.application.name}
+arthas.tunnel-server=ws://101.132.255.39:7777/ws
+
+# 导入的材料路径
+exclude.file.path.prefix=/Users/admin/Desktop/ahnx-share-src-public
+markdown.path=/Users/admin/Desktop/ahnx-share-src-public/public
+pdf.path=/Users/admin/Desktop/ahnx-share-src-public/public
+word.path=/Users/admin/Desktop/ahnx-share-src-public/public

src/main/resources/application-dev-windows.properties

@@ -22,7 +22,7 @@ arthas.appName=${spring.application.name}
 arthas.tunnel-server=ws://101.132.255.39:7777/ws
 
 # 导入的材料路径
-exclude.file.path.prefix=D:/02-documents/
+exclude.file.path.prefix=D:/02-documents/01-ahnx-share-src-public
 markdown.path=D:/02-documents/01-ahnx-share-src-public/public
 pdf.path=D:/02-documents/01-ahnx-share-src-public/public
 word.path=D:/02-documents/01-ahnx-share-src-public/public

src/main/resources/application-docker.properties

@@ -0,0 +1,30 @@
+# 日志
+logging.config=classpath:log/log4j.xml
+
+# nacos相关
+## nacos公用
+spring.cloud.nacos.config.data-id-without-suffix=nacos
+spring.cloud.nacos.config.file-extension=properties
+spring.config.import=nacos:${spring.cloud.nacos.config.data-id-without-suffix}.${spring.cloud.nacos.config.file-extension}?refresh=true
+## Nacos配置中心
+spring.cloud.nacos.config.username=nacos
+spring.cloud.nacos.config.password=lukeye
+spring.cloud.nacos.config.contextPath=/nacos
+spring.cloud.nacos.config.server-addr=http://101.132.255.39:8848
+spring.cloud.nacos.config.namespace=lukeye
+spring.cloud.nacos.config.refreshEnabled=true
+
+# Arthas配置
+arthas.telnetPort=-1
+arthas.httpPort=-1
+arthas.ip=127.0.0.1
+arthas.appName=${spring.application.name}
+arthas.tunnel-server=ws://101.132.255.39:7777/ws
+
+# 导入目录（容器内路径）
+markdown.path=/app/import-data
+pdf.path=/app/import-data
+word.path=/app/import-data
+
+# 相对路径裁剪
+exclude-file-path-prefix=/app/

src/main/resources/application.properties

@@ -11,17 +11,16 @@ logging.level.root=${logging.level.com.doc.parser}
 logging.org.springframework.web=${logging.level.com.doc.parser}
 logging.org.springframework.context=${logging.level.com.doc.parser}
 
-
 # ES相关
 elasticsearch.host=es.wisdompulse.cn
 elasticsearch.port=80
 elasticsearch.scheme=http
-
-
-# 导入的材料路径
-exclude.file.path.prefix=/Users/admin/Desktop
-markdown.path=/Users/admin/Desktop/ahnx-share-src-public
-pdf.path=/Users/admin/Desktop/ahnx-share-src-public
-word.path=/Users/admin/Desktop/ahnx-share-src-public
-
 search.default-page-size=100
+
+# 是否打开定时导入，且指定导入的频率
+import.schedule.enabled=true
+# 每小时执行一次（可改）
+import.schedule.cron=0 0 * * * *
+
+
+