diff --git a/config/01-创建es索引 b/config/01-创建es索引.http similarity index 100% rename from config/01-创建es索引 rename to config/01-创建es索引.http diff --git a/pom.xml b/pom.xml index 31e1d29..ac18a91 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,11 @@ 31.0.1-jre 7.17.10 + + 2.0.29 + 5.2.3 + 5.2.3 + @@ -159,6 +164,24 @@ ${elasticsearch-rest-high-level-client.version} + + + org.apache.pdfbox + pdfbox + ${pdfbox.version} + + + org.apache.poi + poi + ${poi.version} + + + org.apache.poi + poi-ooxml + ${poi-ooxml.version} + + + @@ -267,6 +290,19 @@ elasticsearch-rest-high-level-client + + org.apache.pdfbox + pdfbox + + + org.apache.poi + poi + + + org.apache.poi + poi-ooxml + + diff --git a/src/main/java/com/doc/parser/domain/enums/DocTypeEnum.java b/src/main/java/com/doc/parser/domain/enums/DocTypeEnum.java new file mode 100644 index 0000000..e8a990b --- /dev/null +++ b/src/main/java/com/doc/parser/domain/enums/DocTypeEnum.java @@ -0,0 +1,24 @@ +package com.doc.parser.domain.enums; + +/** + * @author Luke.ye + * @date 2025/5/20 09:03 + */ +public enum DocTypeEnum { + + MARKDOWN("markdown", "markdown"), + HTML("html", "html"), + PDF("pdf", "pdf"), + WORD("word", "word"), + + ; + + public String code; + + public String desc; + + DocTypeEnum(String code, String desc) { + this.code = code; + this.desc = desc; + } +} diff --git a/src/main/java/com/doc/parser/domain/iface/DocumentImporter.java b/src/main/java/com/doc/parser/domain/iface/DocumentImporter.java new file mode 100644 index 0000000..da314db --- /dev/null +++ b/src/main/java/com/doc/parser/domain/iface/DocumentImporter.java @@ -0,0 +1,15 @@ +package com.doc.parser.domain.iface; + +/** + * @author Luke.ye + * @date 2025/5/20 09:02 + */ +public interface DocumentImporter { + void importDocuments() throws Exception; + + /** + * {@link com.doc.parser.domain.enums.DocTypeEnum} + * @return + */ + String getType(); +} diff --git a/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java b/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java new file mode 100644 index 0000000..d852686 --- /dev/null +++ b/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java @@ -0,0 +1,61 @@ +package com.doc.parser.domain.impl; + +import com.doc.parser.domain.enums.DocTypeEnum; +import com.doc.parser.domain.iface.DocumentImporter; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.*; +import java.util.HashMap; +import java.util.Map; + +/** + * @author Luke.ye + * @date 2025/5/20 09:06 + */ +@Component +public class MarkdownImporter implements DocumentImporter { + + private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class); + private static final String INDEX_NAME = "documents"; + + @Autowired + private RestHighLevelClient esClient; + + @Value("${markdown.path}") + private String directoryPath; + + @Override + public void importDocuments() throws IOException { + Files.walk(Paths.get(directoryPath)) + .filter(p -> p.toString().toLowerCase().endsWith(".md")) + .forEach(path -> { + try { + String content = Files.readString(path); + Map doc = new HashMap<>(); + doc.put("filename", path.getFileName().toString()); + doc.put("filepath", path.toAbsolutePath().toString()); + doc.put("content", content); + + IndexRequest request = new IndexRequest(INDEX_NAME).source(doc); + esClient.index(request, RequestOptions.DEFAULT); + + logger.info("导入成功: {}", path.getFileName()); + } catch (IOException e) { + logger.error("导入失败: {}", path, e); + } + }); + } + + @Override + public String getType() { + return DocTypeEnum.MARKDOWN.code; + } +} \ No newline at end of file diff --git a/src/main/java/com/doc/parser/domain/impl/PdfImporter.java b/src/main/java/com/doc/parser/domain/impl/PdfImporter.java new file mode 100644 index 0000000..8453c1c --- /dev/null +++ b/src/main/java/com/doc/parser/domain/impl/PdfImporter.java @@ -0,0 +1,61 @@ +package com.doc.parser.domain.impl; + +import com.doc.parser.domain.enums.DocTypeEnum; +import com.doc.parser.domain.iface.DocumentImporter; +import org.springframework.stereotype.Component; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + +import java.io.File; +import java.nio.file.*; +import java.util.*; + +/** + * @author Luke.ye + * @date 2025/5/20 10:23 + */ +@Component +public class PdfImporter implements DocumentImporter { + private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class); + private final RestHighLevelClient esClient; + + @Value("${pdf.path}") + private String directoryPath; + + public PdfImporter(RestHighLevelClient esClient) { + this.esClient = esClient; + } + + @Override + public void importDocuments() throws Exception { + Files.walk(Paths.get(directoryPath)) + .filter(p -> p.toString().toLowerCase().endsWith(".pdf")) + .forEach(path -> { + try (PDDocument document = PDDocument.load(new File(path.toString()))) { + String content = new PDFTextStripper().getText(document); + Map doc = new HashMap<>(); + doc.put("filename", path.getFileName().toString()); + doc.put("filepath", path.toAbsolutePath().toString()); + doc.put("content", content); + + IndexRequest request = new IndexRequest("documents").source(doc); + esClient.index(request, RequestOptions.DEFAULT); + LOGGER.info("导入成功: {}", path.getFileName()); + } catch (Exception e) { + LOGGER.error("导入失败: {}", path, e); + } + }); + } + + @Override + public String getType() { + return DocTypeEnum.PDF.code; + } +} diff --git a/src/main/java/com/doc/parser/domain/impl/WordImporter.java b/src/main/java/com/doc/parser/domain/impl/WordImporter.java new file mode 100644 index 0000000..e98b1a0 --- /dev/null +++ b/src/main/java/com/doc/parser/domain/impl/WordImporter.java @@ -0,0 +1,65 @@ +package com.doc.parser.domain.impl; + +import com.doc.parser.domain.enums.DocTypeEnum; +import com.doc.parser.domain.iface.DocumentImporter; +import org.springframework.stereotype.Component; +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.apache.poi.xwpf.usermodel.XWPFParagraph; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + +import java.io.FileInputStream; +import java.nio.file.*; +import java.util.*; +/** + * @author Luke.ye + * @date 2025/5/20 10:25 + */ +@Component +public class WordImporter implements DocumentImporter { + + private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class); + + private final RestHighLevelClient esClient; + + @Value("${word.path}") + private String directoryPath; + + public WordImporter(RestHighLevelClient esClient) { + this.esClient = esClient; + } + + @Override + public void importDocuments() throws Exception { + Files.walk(Paths.get(directoryPath)) + .filter(p -> p.toString().toLowerCase().endsWith(".docx")) + .forEach(path -> { + try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) { + StringBuilder content = new StringBuilder(); + for (XWPFParagraph para : docx.getParagraphs()) { + content.append(para.getText()).append("\n"); + } + + Map doc = new HashMap<>(); + doc.put("filename", path.getFileName().toString()); + doc.put("filepath", path.toAbsolutePath().toString()); + doc.put("content", content.toString()); + + IndexRequest request = new IndexRequest("documents").source(doc); + esClient.index(request, RequestOptions.DEFAULT); + LOGGER.info("导入成功: {}", path.getFileName()); + } catch (Exception e) { + LOGGER.error("导入失败: {}", path, e); + } + }); + } + + @Override + public String getType() { + return DocTypeEnum.WORD.code; + } +} diff --git a/src/main/java/com/doc/parser/infrastructure/north/controller/ImportController.java b/src/main/java/com/doc/parser/infrastructure/north/controller/ImportController.java new file mode 100644 index 0000000..a746c29 --- /dev/null +++ b/src/main/java/com/doc/parser/infrastructure/north/controller/ImportController.java @@ -0,0 +1,31 @@ +package com.doc.parser.infrastructure.north.controller; + +import com.doc.parser.domain.iface.DocumentImporter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RestController +@RequestMapping("/import") +public class ImportController { + + @Autowired + private List importers; + + @GetMapping("/{type}") + public ResponseEntity importByType(@PathVariable String type) { + for (DocumentImporter importer : importers) { + if (importer.getType().equalsIgnoreCase(type)) { + try { + importer.importDocuments(); + return ResponseEntity.ok(type + " 文件导入成功"); + } catch (Exception e) { + return ResponseEntity.internalServerError().body("导入失败: " + e.getMessage()); + } + } + } + return ResponseEntity.badRequest().body("不支持的类型: " + type); + } +} \ No newline at end of file diff --git a/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java b/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java new file mode 100644 index 0000000..7e6f1aa --- /dev/null +++ b/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java @@ -0,0 +1,61 @@ +package com.doc.parser.infrastructure.north.controller; + + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.*; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; +import org.elasticsearch.search.SearchHit; +import org.springframework.web.bind.annotation.*; + +import java.util.*; + +/** + * @author Luke.ye + * @date 2025/5/20 16:56 + */ +@RestController +@RequestMapping("/search") +public class SearchController { + + private final RestHighLevelClient esClient; + + public SearchController(RestHighLevelClient esClient) { + this.esClient = esClient; + } + + @GetMapping + public List> search(@RequestParam String q) throws Exception { + SearchRequest request = new SearchRequest("documents"); + + HighlightBuilder highlight = new HighlightBuilder() + .field("content") + .preTags("") + .postTags("") + .fragmentSize(80) + .numOfFragments(3); + + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchQuery("content", q)) + .highlighter(highlight) + .size(10); + + request.source(builder); + SearchResponse response = esClient.search(request, RequestOptions.DEFAULT); + + List> results = new ArrayList<>(); + for (SearchHit hit : response.getHits()) { + Map source = hit.getSourceAsMap(); + Map result = new HashMap<>(); + result.put("filename", source.get("filename")); + result.put("filepath", source.get("filepath")); + result.put("highlight", hit.getHighlightFields().getOrDefault("content", null)); + results.add(result); + } + return results; + } +} + diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a2fd591..79ae0f0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -13,10 +13,12 @@ logging.org.springframework.context=${logging.level.com.doc.parser} # ES相关 -elasticsearch.host=localhost -elasticsearch.port=9200 +elasticsearch.host=es.wisdompulse.cn +elasticsearch.port=80 elasticsearch.scheme=http # 导入的材料路径 -markdown.path = D:/02-documents/01-ahnx-share-src-public/public/设计模式 \ No newline at end of file +markdown.path=/Users/admin/Desktop/ahnx-share-src-public +pdf.path=/Users/admin/Desktop/ahnx-share-src-public +word.path=/Users/admin/Desktop/ahnx-share-src-public \ No newline at end of file