新增pdf/doc/markdown解析

pom.xml

@@ -35,6 +35,11 @@
 		<guava.version>31.0.1-jre</guava.version>
 
 		<elasticsearch-rest-high-level-client.version>7.17.10</elasticsearch-rest-high-level-client.version>
+
+		<pdfbox.version>2.0.29</pdfbox.version>
+		<poi.version>5.2.3</poi.version>
+		<poi-ooxml.version>5.2.3</poi-ooxml.version>
+
     </properties>
     <dependencyManagement>
         <dependencies>
@@ -159,6 +164,24 @@
 				<version>${elasticsearch-rest-high-level-client.version}</version>
 			</dependency>
 
+			<!-- pdf/word的解析 -->
+			<dependency>
+				<groupId>org.apache.pdfbox</groupId>
+				<artifactId>pdfbox</artifactId>
+				<version>${pdfbox.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.poi</groupId>
+				<artifactId>poi</artifactId>
+				<version>${poi.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.poi</groupId>
+				<artifactId>poi-ooxml</artifactId>
+				<version>${poi-ooxml.version}</version>
+			</dependency>
+
+
 		</dependencies>
     </dependencyManagement>
 
@@ -267,6 +290,19 @@
 			<artifactId>elasticsearch-rest-high-level-client</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.pdfbox</groupId>
+			<artifactId>pdfbox</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.poi</groupId>
+			<artifactId>poi</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.poi</groupId>
+			<artifactId>poi-ooxml</artifactId>
+		</dependency>
+
     </dependencies>
 
 	<build>

src/main/java/com/doc/parser/domain/enums/DocTypeEnum.java

@@ -0,0 +1,24 @@
+package com.doc.parser.domain.enums;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 09:03
+ */
+public enum DocTypeEnum {
+
+    MARKDOWN("markdown", "markdown"),
+    HTML("html", "html"),
+    PDF("pdf", "pdf"),
+    WORD("word", "word"),
+
+    ;
+
+    public String code;
+
+    public String desc;
+
+    DocTypeEnum(String code, String desc) {
+        this.code = code;
+        this.desc = desc;
+    }
+}

src/main/java/com/doc/parser/domain/iface/DocumentImporter.java

@@ -0,0 +1,15 @@
+package com.doc.parser.domain.iface;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 09:02
+ */
+public interface DocumentImporter {
+    void importDocuments() throws Exception;
+
+    /**
+     * {@link com.doc.parser.domain.enums.DocTypeEnum}
+     * @return
+     */
+    String getType();
+}

src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java

@@ -0,0 +1,61 @@
+package com.doc.parser.domain.impl;
+
+import com.doc.parser.domain.enums.DocTypeEnum;
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.file.*;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 09:06
+ */
+@Component
+public class MarkdownImporter implements DocumentImporter {
+
+    private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class);
+    private static final String INDEX_NAME = "documents";
+
+    @Autowired
+    private RestHighLevelClient esClient;
+
+    @Value("${markdown.path}")
+    private String directoryPath;
+
+    @Override
+    public void importDocuments() throws IOException {
+        Files.walk(Paths.get(directoryPath))
+                .filter(p -> p.toString().toLowerCase().endsWith(".md"))
+                .forEach(path -> {
+                    try {
+                        String content = Files.readString(path);
+                        Map<String, Object> doc = new HashMap<>();
+                        doc.put("filename", path.getFileName().toString());
+                        doc.put("filepath", path.toAbsolutePath().toString());
+                        doc.put("content", content);
+
+                        IndexRequest request = new IndexRequest(INDEX_NAME).source(doc);
+                        esClient.index(request, RequestOptions.DEFAULT);
+
+                        logger.info("导入成功: {}", path.getFileName());
+                    } catch (IOException e) {
+                        logger.error("导入失败: {}", path, e);
+                    }
+                });
+    }
+
+    @Override
+    public String getType() {
+        return DocTypeEnum.MARKDOWN.code;
+    }
+}

src/main/java/com/doc/parser/domain/impl/PdfImporter.java

@@ -0,0 +1,61 @@
+package com.doc.parser.domain.impl;
+
+import com.doc.parser.domain.enums.DocTypeEnum;
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.springframework.stereotype.Component;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.text.PDFTextStripper;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.io.File;
+import java.nio.file.*;
+import java.util.*;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 10:23
+ */
+@Component
+public class PdfImporter implements DocumentImporter {
+    private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class);
+    private final RestHighLevelClient esClient;
+
+    @Value("${pdf.path}")
+    private String directoryPath;
+
+    public PdfImporter(RestHighLevelClient esClient) {
+        this.esClient = esClient;
+    }
+
+    @Override
+    public void importDocuments() throws Exception {
+        Files.walk(Paths.get(directoryPath))
+                .filter(p -> p.toString().toLowerCase().endsWith(".pdf"))
+                .forEach(path -> {
+                    try (PDDocument document = PDDocument.load(new File(path.toString()))) {
+                        String content = new PDFTextStripper().getText(document);
+                        Map<String, Object> doc = new HashMap<>();
+                        doc.put("filename", path.getFileName().toString());
+                        doc.put("filepath", path.toAbsolutePath().toString());
+                        doc.put("content", content);
+
+                        IndexRequest request = new IndexRequest("documents").source(doc);
+                        esClient.index(request, RequestOptions.DEFAULT);
+                        LOGGER.info("导入成功: {}", path.getFileName());
+                    } catch (Exception e) {
+                        LOGGER.error("导入失败: {}", path, e);
+                    }
+                });
+    }
+
+    @Override
+    public String getType() {
+        return DocTypeEnum.PDF.code;
+    }
+}

src/main/java/com/doc/parser/domain/impl/WordImporter.java

@@ -0,0 +1,65 @@
+package com.doc.parser.domain.impl;
+
+import com.doc.parser.domain.enums.DocTypeEnum;
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.springframework.stereotype.Component;
+import org.apache.poi.xwpf.usermodel.XWPFDocument;
+import org.apache.poi.xwpf.usermodel.XWPFParagraph;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.io.FileInputStream;
+import java.nio.file.*;
+import java.util.*;
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 10:25
+ */
+@Component
+public class WordImporter implements DocumentImporter {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class);
+
+    private final RestHighLevelClient esClient;
+
+    @Value("${word.path}")
+    private String directoryPath;
+
+    public WordImporter(RestHighLevelClient esClient) {
+        this.esClient = esClient;
+    }
+
+    @Override
+    public void importDocuments() throws Exception {
+        Files.walk(Paths.get(directoryPath))
+                .filter(p -> p.toString().toLowerCase().endsWith(".docx"))
+                .forEach(path -> {
+                    try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) {
+                        StringBuilder content = new StringBuilder();
+                        for (XWPFParagraph para : docx.getParagraphs()) {
+                            content.append(para.getText()).append("\n");
+                        }
+
+                        Map<String, Object> doc = new HashMap<>();
+                        doc.put("filename", path.getFileName().toString());
+                        doc.put("filepath", path.toAbsolutePath().toString());
+                        doc.put("content", content.toString());
+
+                        IndexRequest request = new IndexRequest("documents").source(doc);
+                        esClient.index(request, RequestOptions.DEFAULT);
+                        LOGGER.info("导入成功: {}", path.getFileName());
+                    } catch (Exception e) {
+                        LOGGER.error("导入失败: {}", path, e);
+                    }
+                });
+    }
+
+    @Override
+    public String getType() {
+        return DocTypeEnum.WORD.code;
+    }
+}

src/main/java/com/doc/parser/infrastructure/north/controller/ImportController.java

@@ -0,0 +1,31 @@
+package com.doc.parser.infrastructure.north.controller;
+
+import com.doc.parser.domain.iface.DocumentImporter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+
+@RestController
+@RequestMapping("/import")
+public class ImportController {
+
+    @Autowired
+    private List<DocumentImporter> importers;
+
+    @GetMapping("/{type}")
+    public ResponseEntity<String> importByType(@PathVariable String type) {
+        for (DocumentImporter importer : importers) {
+            if (importer.getType().equalsIgnoreCase(type)) {
+                try {
+                    importer.importDocuments();
+                    return ResponseEntity.ok(type + " 文件导入成功");
+                } catch (Exception e) {
+                    return ResponseEntity.internalServerError().body("导入失败: " + e.getMessage());
+                }
+            }
+        }
+        return ResponseEntity.badRequest().body("不支持的类型: " + type);
+    }
+}

src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java

@@ -0,0 +1,61 @@
+package com.doc.parser.infrastructure.north.controller;
+
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.*;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.SearchHit;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.*;
+
+/**
+ * @author Luke.ye
+ * @date 2025/5/20 16:56
+ */
+@RestController
+@RequestMapping("/search")
+public class SearchController {
+
+    private final RestHighLevelClient esClient;
+
+    public SearchController(RestHighLevelClient esClient) {
+        this.esClient = esClient;
+    }
+
+    @GetMapping
+    public List<Map<String, Object>> search(@RequestParam String q) throws Exception {
+        SearchRequest request = new SearchRequest("documents");
+
+        HighlightBuilder highlight = new HighlightBuilder()
+                .field("content")
+                .preTags("<mark>")
+                .postTags("</mark>")
+                .fragmentSize(80)
+                .numOfFragments(3);
+
+        SearchSourceBuilder builder = new SearchSourceBuilder()
+                .query(QueryBuilders.matchQuery("content", q))
+                .highlighter(highlight)
+                .size(10);
+
+        request.source(builder);
+        SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
+
+        List<Map<String, Object>> results = new ArrayList<>();
+        for (SearchHit hit : response.getHits()) {
+            Map<String, Object> source = hit.getSourceAsMap();
+            Map<String, Object> result = new HashMap<>();
+            result.put("filename", source.get("filename"));
+            result.put("filepath", source.get("filepath"));
+            result.put("highlight", hit.getHighlightFields().getOrDefault("content", null));
+            results.add(result);
+        }
+        return results;
+    }
+}
+

src/main/resources/application.properties

@@ -13,10 +13,12 @@ logging.org.springframework.context=${logging.level.com.doc.parser}
 
 
 # ES相关
-elasticsearch.host=localhost
+elasticsearch.host=es.wisdompulse.cn
-elasticsearch.port=9200
+elasticsearch.port=80
 elasticsearch.scheme=http
 
 
 # 导入的材料路径
-markdown.path = D:/02-documents/01-ahnx-share-src-public/public/设计模式
+markdown.path=/Users/admin/Desktop/ahnx-share-src-public
+pdf.path=/Users/admin/Desktop/ahnx-share-src-public
+word.path=/Users/admin/Desktop/ahnx-share-src-public