优化导入逻辑“

config/01-创建es索引.http

@@ -13,7 +13,9 @@ PUT /documents
   "mappings": {
     "properties": {
       "filename": {
-        "type": "keyword"
+        "type": "text",
+        "analyzer": "ik_max_word",
+        "search_analyzer": "ik_smart"
       },
       "filepath": {
         "type": "keyword"

src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java

@@ -0,0 +1,83 @@
+package com.doc.parser.domain.impl;
+
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class AbstractBaseFileImporter implements DocumentImporter {
+
+    private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class);
+    private static final String INDEX_NAME = "documents";
+
+    @Autowired
+    protected RestHighLevelClient esClient;
+
+    @Value("${exclude.file.path.prefix}")
+    private String excludePrefix;
+
+    protected abstract String getDirectoryPath();
+
+    protected abstract String getFileSuffix();
+
+    protected abstract String getDocTypeCode();
+
+    @Override
+    public void importDocuments() throws IOException {
+        Path basePath = Paths.get(getDirectoryPath());
+
+        Files.walk(basePath)
+                .filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
+                .forEach(path -> {
+                    try {
+                        String content = extractContent(path);
+                        if (content == null || content.isBlank()) {
+                            logger.warn("跳过空内容文件: {}", path);
+                            return;
+                        }
+
+                        String absPath = path.toAbsolutePath().toString().replace("\\", "/");
+                        String cleanedPrefix = excludePrefix.replace("\\", "/");
+                        if (!cleanedPrefix.endsWith("/")) {
+                            cleanedPrefix += "/";
+                        }
+
+                        String relativePath = absPath.startsWith(cleanedPrefix)
+                                ? absPath.substring(cleanedPrefix.length())
+                                : absPath;
+
+                        Map<String, Object> doc = new HashMap<>();
+                        doc.put("filename", path.getFileName().toString());
+                        doc.put("filepath", relativePath);
+                        doc.put("content", content);
+
+                        IndexRequest request = new IndexRequest(INDEX_NAME)
+                                .id(relativePath)
+                                .source(doc);
+
+                        esClient.index(request, RequestOptions.DEFAULT);
+                        logger.info("导入成功: {}", relativePath);
+                    } catch (Exception e) {
+                        logger.error("导入失败: {}", path, e);
+                    }
+                });
+    }
+
+    @Override
+    public String getType() {
+        return getDocTypeCode();
+    }
+
+    protected abstract String extractContent(Path path) throws IOException;
+}

src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java

@@ -2,6 +2,7 @@ package com.doc.parser.domain.impl;
 
 import com.doc.parser.domain.enums.DocTypeEnum;
 import com.doc.parser.domain.iface.DocumentImporter;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
@@ -12,6 +13,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.*;
 import java.util.HashMap;
 import java.util.Map;
@@ -21,41 +23,28 @@ import java.util.Map;
  * @date 2025/5/20 09:06
  */
 @Component
-public class MarkdownImporter implements DocumentImporter {
+public class MarkdownImporter extends AbstractBaseFileImporter {
-
-    private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class);
-    private static final String INDEX_NAME = "documents";
-
-    @Autowired
-    private RestHighLevelClient esClient;
 
     @Value("${markdown.path}")
     private String directoryPath;
 
     @Override
-    public void importDocuments() throws IOException {
+    protected String getDirectoryPath() {
-        Files.walk(Paths.get(directoryPath))
+        return directoryPath;
-                .filter(p -> p.toString().toLowerCase().endsWith(".md"))
-                .forEach(path -> {
-                    try {
-                        String content = Files.readString(path);
-                        Map<String, Object> doc = new HashMap<>();
-                        doc.put("filename", path.getFileName().toString());
-                        doc.put("filepath", path.toAbsolutePath().toString());
-                        doc.put("content", content);
-
-                        IndexRequest request = new IndexRequest(INDEX_NAME).source(doc);
-                        esClient.index(request, RequestOptions.DEFAULT);
-
-                        logger.info("导入成功: {}", path.getFileName());
-                    } catch (IOException e) {
-                        logger.error("导入失败: {}", path, e);
-                    }
-                });
     }
 
     @Override
-    public String getType() {
+    protected String getFileSuffix() {
+        return ".md";
+    }
+
+    @Override
+    protected String getDocTypeCode() {
         return DocTypeEnum.MARKDOWN.code;
     }
-}
+
+    @Override
+    protected String extractContent(Path path) throws IOException {
+        return Files.readString(path, StandardCharsets.UTF_8);
+    }
+}

src/main/java/com/doc/parser/domain/impl/PdfImporter.java

@@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 
 import java.io.File;
+import java.io.IOException;
 import java.nio.file.*;
 import java.util.*;
 
@@ -22,40 +23,30 @@ import java.util.*;
  * @date 2025/5/20 10:23
  */
 @Component
-public class PdfImporter implements DocumentImporter {
+public class PdfImporter extends AbstractBaseFileImporter {
-    private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class);
-    private final RestHighLevelClient esClient;
 
     @Value("${pdf.path}")
     private String directoryPath;
 
-    public PdfImporter(RestHighLevelClient esClient) {
+    @Override
-        this.esClient = esClient;
+    protected String getDirectoryPath() {
+        return directoryPath;
     }
 
     @Override
-    public void importDocuments() throws Exception {
+    protected String getFileSuffix() {
-        Files.walk(Paths.get(directoryPath))
+        return ".pdf";
-                .filter(p -> p.toString().toLowerCase().endsWith(".pdf"))
-                .forEach(path -> {
-                    try (PDDocument document = PDDocument.load(new File(path.toString()))) {
-                        String content = new PDFTextStripper().getText(document);
-                        Map<String, Object> doc = new HashMap<>();
-                        doc.put("filename", path.getFileName().toString());
-                        doc.put("filepath", path.toAbsolutePath().toString());
-                        doc.put("content", content);
-
-                        IndexRequest request = new IndexRequest("documents").source(doc);
-                        esClient.index(request, RequestOptions.DEFAULT);
-                        LOGGER.info("导入成功: {}", path.getFileName());
-                    } catch (Exception e) {
-                        LOGGER.error("导入失败: {}", path, e);
-                    }
-                });
     }
 
     @Override
-    public String getType() {
+    protected String getDocTypeCode() {
         return DocTypeEnum.PDF.code;
     }
+
+    @Override
+    protected String extractContent(Path path) throws IOException {
+        try (PDDocument document = PDDocument.load(path.toFile())) {
+            return new PDFTextStripper().getText(document);
+        }
+    }
 }

src/main/java/com/doc/parser/domain/impl/WordImporter.java

@@ -1,65 +1,49 @@
 package com.doc.parser.domain.impl;
 
 import com.doc.parser.domain.enums.DocTypeEnum;
-import com.doc.parser.domain.iface.DocumentImporter;
 import org.springframework.stereotype.Component;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.apache.poi.xwpf.usermodel.XWPFParagraph;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.nio.file.*;
-import java.util.*;
+
+
 /**
  * @author Luke.ye
  * @date 2025/5/20 10:25
  */
 @Component
-public class WordImporter implements DocumentImporter {
+public class WordImporter extends AbstractBaseFileImporter {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class);
-
-    private final RestHighLevelClient esClient;
 
     @Value("${word.path}")
     private String directoryPath;
 
-    public WordImporter(RestHighLevelClient esClient) {
+    @Override
-        this.esClient = esClient;
+    protected String getDirectoryPath() {
+        return directoryPath;
     }
 
     @Override
-    public void importDocuments() throws Exception {
+    protected String getFileSuffix() {
-        Files.walk(Paths.get(directoryPath))
+        return ".docx";
-                .filter(p -> p.toString().toLowerCase().endsWith(".docx"))
-                .forEach(path -> {
-                    try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) {
-                        StringBuilder content = new StringBuilder();
-                        for (XWPFParagraph para : docx.getParagraphs()) {
-                            content.append(para.getText()).append("\n");
-                        }
-
-                        Map<String, Object> doc = new HashMap<>();
-                        doc.put("filename", path.getFileName().toString());
-                        doc.put("filepath", path.toAbsolutePath().toString());
-                        doc.put("content", content.toString());
-
-                        IndexRequest request = new IndexRequest("documents").source(doc);
-                        esClient.index(request, RequestOptions.DEFAULT);
-                        LOGGER.info("导入成功: {}", path.getFileName());
-                    } catch (Exception e) {
-                        LOGGER.error("导入失败: {}", path, e);
-                    }
-                });
     }
 
     @Override
-    public String getType() {
+    protected String getDocTypeCode() {
         return DocTypeEnum.WORD.code;
     }
+
+    @Override
+    protected String extractContent(Path path) throws IOException {
+        try (XWPFDocument document = new XWPFDocument(new FileInputStream(path.toFile()))) {
+            StringBuilder content = new StringBuilder();
+            for (XWPFParagraph para : document.getParagraphs()) {
+                content.append(para.getText()).append("\n");
+            }
+            return content.toString().trim();
+        }
+    }
 }

src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java

@@ -9,6 +9,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.SearchHit;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.*;
 
 import java.util.*;
@@ -23,39 +24,73 @@ public class SearchController {
 
     private final RestHighLevelClient esClient;
 
+    @Value("${search.default-page-size:10}")
+    private int defaultPageSize;
+
     public SearchController(RestHighLevelClient esClient) {
         this.esClient = esClient;
     }
 
     @GetMapping
-    public List<Map<String, Object>> search(@RequestParam String q) throws Exception {
+    public Map<String, Object> search(
+            @RequestParam String q,
+            @RequestParam(defaultValue = "1") int page,
+            @RequestParam(required = false) Integer size
+    ) throws Exception {
+        if (page < 1) page = 1;
+        int pageSize = (size != null && size > 0) ? size : defaultPageSize;
+
+        int from = (page - 1) * pageSize;
+
         SearchRequest request = new SearchRequest("documents");
 
+        // 高亮配置
         HighlightBuilder highlight = new HighlightBuilder()
                 .field("content")
+                .field("filename")
                 .preTags("<mark>")
                 .postTags("</mark>")
                 .fragmentSize(80)
-                .numOfFragments(3);
+                .numOfFragments(3)
+                .requireFieldMatch(false);
 
+        // 查询构建
         SearchSourceBuilder builder = new SearchSourceBuilder()
-                .query(QueryBuilders.matchQuery("content", q))
+                .query(QueryBuilders.multiMatchQuery(q, "filename", "content"))
-                .highlighter(highlight)
+                .from(from)
-                .size(10);
+                .size(pageSize)
+                .highlighter(highlight);
 
         request.source(builder);
         SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
 
+        // 结果封装
         List<Map<String, Object>> results = new ArrayList<>();
         for (SearchHit hit : response.getHits()) {
             Map<String, Object> source = hit.getSourceAsMap();
-            Map<String, Object> result = new HashMap<>();
+            Map<String, Object> result = new LinkedHashMap<>();
             result.put("filename", source.get("filename"));
             result.put("filepath", source.get("filepath"));
-            result.put("highlight", hit.getHighlightFields().getOrDefault("content", null));
+
+            Map<String, Object> highlights = new HashMap<>();
+            for (Map.Entry<String, HighlightField> entry : hit.getHighlightFields().entrySet()) {
+                List<String> fragments = new ArrayList<>();
+                Arrays.stream(entry.getValue().fragments())
+                        .forEach(f -> fragments.add(f.string()));
+                highlights.put(entry.getKey(), fragments);
+            }
+
+            result.put("highlight", highlights.isEmpty() ? null : highlights);
             results.add(result);
         }
-        return results;
+
+        // 返回分页结构
+        Map<String, Object> responseBody = new LinkedHashMap<>();
+        responseBody.put("total", response.getHits().getTotalHits().value);
+        responseBody.put("page", page);
+        responseBody.put("size", pageSize);
+        responseBody.put("results", results);
+
+        return responseBody;
     }
 }
-

src/main/resources/application-dev.properties

@@ -19,4 +19,10 @@ arthas.telnetPort=-1
 arthas.httpPort=-1
 arthas.ip=127.0.0.1
 arthas.appName=${spring.application.name}
-arthas.tunnel-server=ws://101.132.255.39:7777/ws
+arthas.tunnel-server=ws://101.132.255.39:7777/ws
+
+# 导入的材料路径
+exclude.file.path.prefix=D:/02-documents/
+markdown.path=D:/02-documents/01-ahnx-share-src-public/public
+pdf.path=D:/02-documents/01-ahnx-share-src-public/public
+word.path=D:/02-documents/01-ahnx-share-src-public/public

src/main/resources/application.properties

@@ -19,6 +19,9 @@ elasticsearch.scheme=http
 
 
 # 导入的材料路径
+exclude.file.path.prefix=/Users/admin/Desktop
 markdown.path=/Users/admin/Desktop/ahnx-share-src-public
 pdf.path=/Users/admin/Desktop/ahnx-share-src-public
-word.path=/Users/admin/Desktop/ahnx-share-src-public
+word.path=/Users/admin/Desktop/ahnx-share-src-public
+
+search.default-page-size=100