lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化导入逻辑“

Browse Source
This commit is contained in:
luke 2025-05-21 01:21:13 +08:00
parent 22426a30d9
commit 580f732ec1
8 changed files with 194 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/01-创建es索引.http
View File

@ -13,7 +13,9 @@ PUT /documents
"mappings": {
"properties": {
"filename": {
"type": "keyword"
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"filepath": {
"type": "keyword"

83
src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java Normal file
View File

@ -0,0 +1,83 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.iface.DocumentImporter;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
public abstract class AbstractBaseFileImporter implements DocumentImporter {
private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class);
private static final String INDEX_NAME = "documents";
@Autowired
protected RestHighLevelClient esClient;
@Value("${exclude.file.path.prefix}")
private String excludePrefix;
protected abstract String getDirectoryPath();
protected abstract String getFileSuffix();
protected abstract String getDocTypeCode();
@Override
public void importDocuments() throws IOException {
Path basePath = Paths.get(getDirectoryPath());
Files.walk(basePath)
.filter(p -> p.toString().toLowerCase().endsWith(getFileSuffix()))
.forEach(path -> {
try {
String content = extractContent(path);
if (content == null || content.isBlank()) {
logger.warn("跳过空内容文件: {}", path);
return;
}
String absPath = path.toAbsolutePath().toString().replace("\\", "/");
String cleanedPrefix = excludePrefix.replace("\\", "/");
if (!cleanedPrefix.endsWith("/")) {
cleanedPrefix += "/";
}
String relativePath = absPath.startsWith(cleanedPrefix)
? absPath.substring(cleanedPrefix.length())
: absPath;
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", relativePath);
doc.put("content", content);
IndexRequest request = new IndexRequest(INDEX_NAME)
.id(relativePath)
.source(doc);
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath);
} catch (Exception e) {
logger.error("导入失败: {}", path, e);
}
});
}
@Override
public String getType() {
return getDocTypeCode();
}
protected abstract String extractContent(Path path) throws IOException;
}

43
src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java
View File

@ -2,6 +2,7 @@ package com.doc.parser.domain.impl;
import com.doc.parser.domain.enums.DocTypeEnum;
import com.doc.parser.domain.iface.DocumentImporter;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
@ -12,6 +13,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.HashMap;
import java.util.Map;
@ -21,41 +23,28 @@ import java.util.Map;
* @date 2025/5/20 09:06
*/
@Component
public class MarkdownImporter implements DocumentImporter {
private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class);
private static final String INDEX_NAME = "documents";
@Autowired
private RestHighLevelClient esClient;
public class MarkdownImporter extends AbstractBaseFileImporter {
@Value("${markdown.path}")
private String directoryPath;
@Override
public void importDocuments() throws IOException {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".md"))
.forEach(path -> {
try {
String content = Files.readString(path);
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content);
IndexRequest request = new IndexRequest(INDEX_NAME).source(doc);
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", path.getFileName());
} catch (IOException e) {
logger.error("导入失败: {}", path, e);
}
});
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public String getType() {
protected String getFileSuffix() {
return ".md";
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.MARKDOWN.code;
}
@Override
protected String extractContent(Path path) throws IOException {
return Files.readString(path, StandardCharsets.UTF_8);
}
}

39
src/main/java/com/doc/parser/domain/impl/PdfImporter.java
View File

@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.util.*;
@ -22,40 +23,30 @@ import java.util.*;
* @date 2025/5/20 10:23
*/
@Component
public class PdfImporter implements DocumentImporter {
private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class);
private final RestHighLevelClient esClient;
public class PdfImporter extends AbstractBaseFileImporter {
@Value("${pdf.path}")
private String directoryPath;
public PdfImporter(RestHighLevelClient esClient) {
this.esClient = esClient;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public void importDocuments() throws Exception {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".pdf"))
.forEach(path -> {
try (PDDocument document = PDDocument.load(new File(path.toString()))) {
String content = new PDFTextStripper().getText(document);
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content);
IndexRequest request = new IndexRequest("documents").source(doc);
esClient.index(request, RequestOptions.DEFAULT);
LOGGER.info("导入成功: {}", path.getFileName());
} catch (Exception e) {
LOGGER.error("导入失败: {}", path, e);
}
});
protected String getFileSuffix() {
return ".pdf";
}
@Override
public String getType() {
protected String getDocTypeCode() {
return DocTypeEnum.PDF.code;
}
@Override
protected String extractContent(Path path) throws IOException {
try (PDDocument document = PDDocument.load(path.toFile())) {
return new PDFTextStripper().getText(document);
}
}
}

58
src/main/java/com/doc/parser/domain/impl/WordImporter.java
View File

@ -1,65 +1,49 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.enums.DocTypeEnum;
import com.doc.parser.domain.iface.DocumentImporter;
import org.springframework.stereotype.Component;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.*;
import java.util.*;
/**
* @author Luke.ye
* @date 2025/5/20 10:25
*/
@Component
public class WordImporter implements DocumentImporter {
private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class);
private final RestHighLevelClient esClient;
public class WordImporter extends AbstractBaseFileImporter {
@Value("${word.path}")
private String directoryPath;
public WordImporter(RestHighLevelClient esClient) {
this.esClient = esClient;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public void importDocuments() throws Exception {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".docx"))
.forEach(path -> {
try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) {
StringBuilder content = new StringBuilder();
for (XWPFParagraph para : docx.getParagraphs()) {
content.append(para.getText()).append("\n");
}
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content.toString());
IndexRequest request = new IndexRequest("documents").source(doc);
esClient.index(request, RequestOptions.DEFAULT);
LOGGER.info("导入成功: {}", path.getFileName());
} catch (Exception e) {
LOGGER.error("导入失败: {}", path, e);
}
});
protected String getFileSuffix() {
return ".docx";
}
@Override
public String getType() {
protected String getDocTypeCode() {
return DocTypeEnum.WORD.code;
}
@Override
protected String extractContent(Path path) throws IOException {
try (XWPFDocument document = new XWPFDocument(new FileInputStream(path.toFile()))) {
StringBuilder content = new StringBuilder();
for (XWPFParagraph para : document.getParagraphs()) {
content.append(para.getText()).append("\n");
}
return content.toString().trim();
}
}
}

53
src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java
View File

@ -9,6 +9,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.SearchHit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import java.util.*;
@ -23,39 +24,73 @@ public class SearchController {
private final RestHighLevelClient esClient;
@Value("${search.default-page-size:10}")
private int defaultPageSize;
public SearchController(RestHighLevelClient esClient) {
this.esClient = esClient;
}
@GetMapping
public List<Map<String, Object>> search(@RequestParam String q) throws Exception {
public Map<String, Object> search(
@RequestParam String q,
@RequestParam(defaultValue = "1") int page,
@RequestParam(required = false) Integer size
) throws Exception {
if (page < 1) page = 1;
int pageSize = (size != null && size > 0) ? size : defaultPageSize;
int from = (page - 1) * pageSize;
SearchRequest request = new SearchRequest("documents");
// 高亮配置
HighlightBuilder highlight = new HighlightBuilder()
.field("content")
.field("filename")
.preTags("<mark>")
.postTags("</mark>")
.fragmentSize(80)
.numOfFragments(3);
.numOfFragments(3)
.requireFieldMatch(false);
// 查询构建
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.matchQuery("content", q))
.highlighter(highlight)
.size(10);
.query(QueryBuilders.multiMatchQuery(q, "filename", "content"))
.from(from)
.size(pageSize)
.highlighter(highlight);
request.source(builder);
SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
// 结果封装
List<Map<String, Object>> results = new ArrayList<>();
for (SearchHit hit : response.getHits()) {
Map<String, Object> source = hit.getSourceAsMap();
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new LinkedHashMap<>();
result.put("filename", source.get("filename"));
result.put("filepath", source.get("filepath"));
result.put("highlight", hit.getHighlightFields().getOrDefault("content", null));
Map<String, Object> highlights = new HashMap<>();
for (Map.Entry<String, HighlightField> entry : hit.getHighlightFields().entrySet()) {
List<String> fragments = new ArrayList<>();
Arrays.stream(entry.getValue().fragments())
.forEach(f -> fragments.add(f.string()));
highlights.put(entry.getKey(), fragments);
}
result.put("highlight", highlights.isEmpty() ? null : highlights);
results.add(result);
}
return results;
// 返回分页结构
Map<String, Object> responseBody = new LinkedHashMap<>();
responseBody.put("total", response.getHits().getTotalHits().value);
responseBody.put("page", page);
responseBody.put("size", pageSize);
responseBody.put("results", results);
return responseBody;
}
}

6
src/main/resources/application-dev.properties
View File

@ -20,3 +20,9 @@ arthas.httpPort=-1
arthas.ip=127.0.0.1
arthas.appName=${spring.application.name}
arthas.tunnel-server=ws://101.132.255.39:7777/ws
# 导入的材料路径
exclude.file.path.prefix=D:/02-documents/
markdown.path=D:/02-documents/01-ahnx-share-src-public/public
pdf.path=D:/02-documents/01-ahnx-share-src-public/public
word.path=D:/02-documents/01-ahnx-share-src-public/public

3
src/main/resources/application.properties
View File

@ -19,6 +19,9 @@ elasticsearch.scheme=http
# 导入的材料路径
exclude.file.path.prefix=/Users/admin/Desktop
markdown.path=/Users/admin/Desktop/ahnx-share-src-public
pdf.path=/Users/admin/Desktop/ahnx-share-src-public
word.path=/Users/admin/Desktop/ahnx-share-src-public
search.default-page-size=100