From 580f732ec11565d6178cab253fb26be2994d798c Mon Sep 17 00:00:00 2001 From: luke Date: Wed, 21 May 2025 01:21:13 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AF=BC=E5=85=A5=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E2=80=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/01-创建es索引.http | 4 +- .../domain/impl/AbstractBaseFileImporter.java | 83 +++++++++++++++++++ .../parser/domain/impl/MarkdownImporter.java | 45 ++++------ .../doc/parser/domain/impl/PdfImporter.java | 39 ++++----- .../doc/parser/domain/impl/WordImporter.java | 58 +++++-------- .../north/controller/SearchController.java | 53 ++++++++++-- src/main/resources/application-dev.properties | 8 +- src/main/resources/application.properties | 5 +- 8 files changed, 194 insertions(+), 101 deletions(-) create mode 100644 src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java diff --git a/config/01-创建es索引.http b/config/01-创建es索引.http index b653b27..32afc5b 100644 --- a/config/01-创建es索引.http +++ b/config/01-创建es索引.http @@ -13,7 +13,9 @@ PUT /documents "mappings": { "properties": { "filename": { - "type": "keyword" + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" }, "filepath": { "type": "keyword" diff --git a/src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java b/src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java new file mode 100644 index 0000000..ae873aa --- /dev/null +++ b/src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java @@ -0,0 +1,83 @@ +package com.doc.parser.domain.impl; + +import com.doc.parser.domain.iface.DocumentImporter; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractBaseFileImporter implements DocumentImporter { + + private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class); + private static final String INDEX_NAME = "documents"; + + @Autowired + protected RestHighLevelClient esClient; + + @Value("${exclude.file.path.prefix}") + private String excludePrefix; + + protected abstract String getDirectoryPath(); + + protected abstract String getFileSuffix(); + + protected abstract String getDocTypeCode(); + + @Override + public void importDocuments() throws IOException { + Path basePath = Paths.get(getDirectoryPath()); + + Files.walk(basePath) + .filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix())) + .forEach(path -> { + try { + String content = extractContent(path); + if (content == null || content.isBlank()) { + logger.warn("跳过空内容文件: {}", path); + return; + } + + String absPath = path.toAbsolutePath().toString().replace("\\", "/"); + String cleanedPrefix = excludePrefix.replace("\\", "/"); + if (!cleanedPrefix.endsWith("/")) { + cleanedPrefix += "/"; + } + + String relativePath = absPath.startsWith(cleanedPrefix) + ? absPath.substring(cleanedPrefix.length()) + : absPath; + + Map doc = new HashMap<>(); + doc.put("filename", path.getFileName().toString()); + doc.put("filepath", relativePath); + doc.put("content", content); + + IndexRequest request = new IndexRequest(INDEX_NAME) + .id(relativePath) + .source(doc); + + esClient.index(request, RequestOptions.DEFAULT); + logger.info("导入成功: {}", relativePath); + } catch (Exception e) { + logger.error("导入失败: {}", path, e); + } + }); + } + + @Override + public String getType() { + return getDocTypeCode(); + } + + protected abstract String extractContent(Path path) throws IOException; +} diff --git a/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java b/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java index d852686..5e5b9ed 100644 --- a/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java +++ b/src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java @@ -2,6 +2,7 @@ package com.doc.parser.domain.impl; import com.doc.parser.domain.enums.DocTypeEnum; import com.doc.parser.domain.iface.DocumentImporter; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; @@ -12,6 +13,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.HashMap; import java.util.Map; @@ -21,41 +23,28 @@ import java.util.Map; * @date 2025/5/20 09:06 */ @Component -public class MarkdownImporter implements DocumentImporter { - - private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class); - private static final String INDEX_NAME = "documents"; - - @Autowired - private RestHighLevelClient esClient; +public class MarkdownImporter extends AbstractBaseFileImporter { @Value("${markdown.path}") private String directoryPath; @Override - public void importDocuments() throws IOException { - Files.walk(Paths.get(directoryPath)) - .filter(p -> p.toString().toLowerCase().endsWith(".md")) - .forEach(path -> { - try { - String content = Files.readString(path); - Map doc = new HashMap<>(); - doc.put("filename", path.getFileName().toString()); - doc.put("filepath", path.toAbsolutePath().toString()); - doc.put("content", content); - - IndexRequest request = new IndexRequest(INDEX_NAME).source(doc); - esClient.index(request, RequestOptions.DEFAULT); - - logger.info("导入成功: {}", path.getFileName()); - } catch (IOException e) { - logger.error("导入失败: {}", path, e); - } - }); + protected String getDirectoryPath() { + return directoryPath; } @Override - public String getType() { + protected String getFileSuffix() { + return ".md"; + } + + @Override + protected String getDocTypeCode() { return DocTypeEnum.MARKDOWN.code; } -} \ No newline at end of file + + @Override + protected String extractContent(Path path) throws IOException { + return Files.readString(path, StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/doc/parser/domain/impl/PdfImporter.java b/src/main/java/com/doc/parser/domain/impl/PdfImporter.java index 8453c1c..b83815b 100644 --- a/src/main/java/com/doc/parser/domain/impl/PdfImporter.java +++ b/src/main/java/com/doc/parser/domain/impl/PdfImporter.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import java.io.File; +import java.io.IOException; import java.nio.file.*; import java.util.*; @@ -22,40 +23,30 @@ import java.util.*; * @date 2025/5/20 10:23 */ @Component -public class PdfImporter implements DocumentImporter { - private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class); - private final RestHighLevelClient esClient; +public class PdfImporter extends AbstractBaseFileImporter { @Value("${pdf.path}") private String directoryPath; - public PdfImporter(RestHighLevelClient esClient) { - this.esClient = esClient; + @Override + protected String getDirectoryPath() { + return directoryPath; } @Override - public void importDocuments() throws Exception { - Files.walk(Paths.get(directoryPath)) - .filter(p -> p.toString().toLowerCase().endsWith(".pdf")) - .forEach(path -> { - try (PDDocument document = PDDocument.load(new File(path.toString()))) { - String content = new PDFTextStripper().getText(document); - Map doc = new HashMap<>(); - doc.put("filename", path.getFileName().toString()); - doc.put("filepath", path.toAbsolutePath().toString()); - doc.put("content", content); - - IndexRequest request = new IndexRequest("documents").source(doc); - esClient.index(request, RequestOptions.DEFAULT); - LOGGER.info("导入成功: {}", path.getFileName()); - } catch (Exception e) { - LOGGER.error("导入失败: {}", path, e); - } - }); + protected String getFileSuffix() { + return ".pdf"; } @Override - public String getType() { + protected String getDocTypeCode() { return DocTypeEnum.PDF.code; } + + @Override + protected String extractContent(Path path) throws IOException { + try (PDDocument document = PDDocument.load(path.toFile())) { + return new PDFTextStripper().getText(document); + } + } } diff --git a/src/main/java/com/doc/parser/domain/impl/WordImporter.java b/src/main/java/com/doc/parser/domain/impl/WordImporter.java index e98b1a0..9fd5dbc 100644 --- a/src/main/java/com/doc/parser/domain/impl/WordImporter.java +++ b/src/main/java/com/doc/parser/domain/impl/WordImporter.java @@ -1,65 +1,49 @@ package com.doc.parser.domain.impl; import com.doc.parser.domain.enums.DocTypeEnum; -import com.doc.parser.domain.iface.DocumentImporter; import org.springframework.stereotype.Component; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import java.io.FileInputStream; +import java.io.IOException; import java.nio.file.*; -import java.util.*; + + /** * @author Luke.ye * @date 2025/5/20 10:25 */ @Component -public class WordImporter implements DocumentImporter { - - private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class); - - private final RestHighLevelClient esClient; +public class WordImporter extends AbstractBaseFileImporter { @Value("${word.path}") private String directoryPath; - public WordImporter(RestHighLevelClient esClient) { - this.esClient = esClient; + @Override + protected String getDirectoryPath() { + return directoryPath; } @Override - public void importDocuments() throws Exception { - Files.walk(Paths.get(directoryPath)) - .filter(p -> p.toString().toLowerCase().endsWith(".docx")) - .forEach(path -> { - try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) { - StringBuilder content = new StringBuilder(); - for (XWPFParagraph para : docx.getParagraphs()) { - content.append(para.getText()).append("\n"); - } - - Map doc = new HashMap<>(); - doc.put("filename", path.getFileName().toString()); - doc.put("filepath", path.toAbsolutePath().toString()); - doc.put("content", content.toString()); - - IndexRequest request = new IndexRequest("documents").source(doc); - esClient.index(request, RequestOptions.DEFAULT); - LOGGER.info("导入成功: {}", path.getFileName()); - } catch (Exception e) { - LOGGER.error("导入失败: {}", path, e); - } - }); + protected String getFileSuffix() { + return ".docx"; } @Override - public String getType() { + protected String getDocTypeCode() { return DocTypeEnum.WORD.code; } + + @Override + protected String extractContent(Path path) throws IOException { + try (XWPFDocument document = new XWPFDocument(new FileInputStream(path.toFile()))) { + StringBuilder content = new StringBuilder(); + for (XWPFParagraph para : document.getParagraphs()) { + content.append(para.getText()).append("\n"); + } + return content.toString().trim(); + } + } } diff --git a/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java b/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java index 7e6f1aa..6b8a1a2 100644 --- a/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java +++ b/src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java @@ -9,6 +9,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.SearchHit; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.*; import java.util.*; @@ -23,39 +24,73 @@ public class SearchController { private final RestHighLevelClient esClient; + @Value("${search.default-page-size:10}") + private int defaultPageSize; + public SearchController(RestHighLevelClient esClient) { this.esClient = esClient; } @GetMapping - public List> search(@RequestParam String q) throws Exception { + public Map search( + @RequestParam String q, + @RequestParam(defaultValue = "1") int page, + @RequestParam(required = false) Integer size + ) throws Exception { + if (page < 1) page = 1; + int pageSize = (size != null && size > 0) ? size : defaultPageSize; + + int from = (page - 1) * pageSize; + SearchRequest request = new SearchRequest("documents"); + // 高亮配置 HighlightBuilder highlight = new HighlightBuilder() .field("content") + .field("filename") .preTags("") .postTags("") .fragmentSize(80) - .numOfFragments(3); + .numOfFragments(3) + .requireFieldMatch(false); + // 查询构建 SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchQuery("content", q)) - .highlighter(highlight) - .size(10); + .query(QueryBuilders.multiMatchQuery(q, "filename", "content")) + .from(from) + .size(pageSize) + .highlighter(highlight); request.source(builder); SearchResponse response = esClient.search(request, RequestOptions.DEFAULT); + // 结果封装 List> results = new ArrayList<>(); for (SearchHit hit : response.getHits()) { Map source = hit.getSourceAsMap(); - Map result = new HashMap<>(); + Map result = new LinkedHashMap<>(); result.put("filename", source.get("filename")); result.put("filepath", source.get("filepath")); - result.put("highlight", hit.getHighlightFields().getOrDefault("content", null)); + + Map highlights = new HashMap<>(); + for (Map.Entry entry : hit.getHighlightFields().entrySet()) { + List fragments = new ArrayList<>(); + Arrays.stream(entry.getValue().fragments()) + .forEach(f -> fragments.add(f.string())); + highlights.put(entry.getKey(), fragments); + } + + result.put("highlight", highlights.isEmpty() ? null : highlights); results.add(result); } - return results; + + // 返回分页结构 + Map responseBody = new LinkedHashMap<>(); + responseBody.put("total", response.getHits().getTotalHits().value); + responseBody.put("page", page); + responseBody.put("size", pageSize); + responseBody.put("results", results); + + return responseBody; } } - diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 3918672..74cc099 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -19,4 +19,10 @@ arthas.telnetPort=-1 arthas.httpPort=-1 arthas.ip=127.0.0.1 arthas.appName=${spring.application.name} -arthas.tunnel-server=ws://101.132.255.39:7777/ws \ No newline at end of file +arthas.tunnel-server=ws://101.132.255.39:7777/ws + +# 导入的材料路径 +exclude.file.path.prefix=D:/02-documents/ +markdown.path=D:/02-documents/01-ahnx-share-src-public/public +pdf.path=D:/02-documents/01-ahnx-share-src-public/public +word.path=D:/02-documents/01-ahnx-share-src-public/public \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 79ae0f0..b9c9401 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -19,6 +19,9 @@ elasticsearch.scheme=http # 导入的材料路径 +exclude.file.path.prefix=/Users/admin/Desktop markdown.path=/Users/admin/Desktop/ahnx-share-src-public pdf.path=/Users/admin/Desktop/ahnx-share-src-public -word.path=/Users/admin/Desktop/ahnx-share-src-public \ No newline at end of file +word.path=/Users/admin/Desktop/ahnx-share-src-public + +search.default-page-size=100