lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成导入及搜索优化

Browse Source
This commit is contained in:
luke 2025-05-21 16:05:55 +08:00
parent 580f732ec1
commit 35b4981ebf
12 changed files with 322 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Dockerfile/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
# 使用 JDK 11
FROM openjdk:11-jdk-slim
# 创建工作目录
WORKDIR /app
# 拷贝 JAR
COPY doc-parser-server-0.0.1-SNAPSHOT.jar app.jar
# 启动命令：指定 spring.profiles.active=docker
ENTRYPOINT ["java", "-jar", "app.jar", "--spring.profiles.active=docker"]

4
config/01-创建es索引.http
View File

@ -1,4 +1,5 @@
PUT /documents
{
"settings": {
"analysis": {
@ -24,6 +25,9 @@ PUT /documents
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"mtime": {
"type": "date"
}
}
}

49
config/99-常用es查询.http Normal file
View File

@ -0,0 +1,49 @@
#{
# "keywordGroups": [
# ["Luke.Ye", "设计模式"],
# ]
#}
# 表示 (设计模式 AND Luke.Ye)
POST /documents/_search
{
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{ "multi_match": { "query": "设计模式", "fields": ["filename", "content"], "type": "phrase" }},
{ "multi_match": { "query": "Luke.Ye", "fields": ["filename", "content"], "type": "phrase" }}
]
}
}
],
"minimum_should_match": 1
}
},
"highlight": {
"pre_tags": ["<mark>"],
"post_tags": ["</mark>"],
"fields": {
"content": {
"fragment_size": 100,
"number_of_fragments": 3
},
"filename": {
"fragment_size": 100,
"number_of_fragments": 1
}
}
},
"sort": [
{ "mtime": "desc" }
],
"size": 10,
"from": 0
}

2
src/main/java/com/doc/parser/DocParserApplication.java
View File

@ -2,10 +2,12 @@ package com.doc.parser;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* @author Luke.ye
*/
@EnableScheduling
@SpringBootApplication
public class DocParserApplication {

30
src/main/java/com/doc/parser/domain/impl/AbstractBaseFileImporter.java
View File

@ -1,6 +1,9 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.iface.DocumentImporter;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
@ -47,20 +50,44 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
return;
}
// === 生成 filepath ===
String absPath = path.toAbsolutePath().toString().replace("\\", "/");
String cleanedPrefix = excludePrefix.replace("\\", "/");
if (!cleanedPrefix.endsWith("/")) {
cleanedPrefix += "/";
}
String relativePath = absPath.startsWith(cleanedPrefix)
? absPath.substring(cleanedPrefix.length())
: absPath;
// 以 relativePath 作为文档 ID
GetRequest getRequest = new GetRequest(INDEX_NAME, relativePath);
boolean skip = false;
long localMTime = Files.getLastModifiedTime(path).toMillis();
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime");
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", relativePath);
skip = true;
} else {
// 删除旧版本
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, relativePath);
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath);
}
}
if (skip) return;
// === 构建文档 ===
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", relativePath);
doc.put("content", content);
doc.put("mtime", localMTime);
IndexRequest request = new IndexRequest(INDEX_NAME)
.id(relativePath)
@ -68,6 +95,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath);
} catch (Exception e) {
logger.error("导入失败: {}", path, e);
}

49
src/main/java/com/doc/parser/domain/schedule/ImportScheduler.java Normal file
View File

@ -0,0 +1,49 @@
package com.doc.parser.domain.schedule;
import com.doc.parser.domain.iface.DocumentImporter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @author Luke.ye
* @date 2025/5/21 14:09
*/
@Component
public class ImportScheduler {
private static final Logger logger = LoggerFactory.getLogger(ImportScheduler.class);
@Value("${import.schedule.enabled:true}")
private boolean scheduleEnabled;
private final List<DocumentImporter> importers;
public ImportScheduler(List<DocumentImporter> importers) {
this.importers = importers;
}
@Scheduled(cron = "${import.schedule.cron:0 0 * * * *}")
public void scheduledImport() {
if (!scheduleEnabled) {
logger.info("导入定时任务已禁用");
return;
}
logger.info("开始执行定时导入任务");
for (DocumentImporter importer : importers) {
try {
logger.info("执行导入器: {}", importer.getType());
importer.importDocuments();
} catch (Exception e) {
logger.error("导入器执行失败: {}", importer.getType(), e);
}
}
logger.info("定时导入任务执行完成");
}
}

100
src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java
View File

@ -1,18 +1,21 @@
package com.doc.parser.infrastructure.north.controller;
import com.doc.parser.infrastructure.north.dto.SearchReq;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
/**
* @author Luke.ye
@ -31,66 +34,81 @@ public class SearchController {
this.esClient = esClient;
}
@GetMapping
public Map<String, Object> search(
@RequestParam String q,
@RequestParam(defaultValue = "1") int page,
@RequestParam(required = false) Integer size
) throws Exception {
if (page < 1) page = 1;
int pageSize = (size != null && size > 0) ? size : defaultPageSize;
@PostMapping
public Map<String, Object> search(@RequestBody SearchReq requestBody) throws IOException {
int page = requestBody.getPage() != null && requestBody.getPage() > 0 ? requestBody.getPage() : 1;
int size = requestBody.getSize() != null && requestBody.getSize() > 0 ? requestBody.getSize() : defaultPageSize;
int from = (page - 1) * size;
int from = (page - 1) * pageSize;
// 构建外层 OR 查询
BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();
for (List<String> group : requestBody.getKeywordGroups()) {
BoolQueryBuilder groupQuery = QueryBuilders.boolQuery();
for (String keyword : group) {
// 使用 match_phrase 精确匹配短语
groupQuery.must(QueryBuilders.multiMatchQuery(keyword, "filename", "content")
.type(MultiMatchQueryBuilder.Type.PHRASE));
}
outerQuery.should(groupQuery);
}
outerQuery.minimumShouldMatch(1); // 至少匹配一组
SearchRequest request = new SearchRequest("documents");
// 高亮配置
HighlightBuilder highlight = new HighlightBuilder()
.field("content")
.field("filename")
.preTags("<mark>")
.postTags("</mark>")
.fragmentSize(80)
.numOfFragments(3)
.requireFieldMatch(false);
// 查询构建
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.multiMatchQuery(q, "filename", "content"))
.query(outerQuery)
.from(from)
.size(pageSize)
.highlighter(highlight);
.size(size)
.sort("mtime", SortOrder.DESC);
request.source(builder);
SearchRequest request = new SearchRequest("documents").source(builder);
SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
// 结果封装
List<Map<String, Object>> results = new ArrayList<>();
for (SearchHit hit : response.getHits()) {
Map<String, Object> source = hit.getSourceAsMap();
Map<String, Object> result = new LinkedHashMap<>();
result.put("filename", source.get("filename"));
result.put("filepath", source.get("filepath"));
result.put("mtime", source.get("mtime"));
Map<String, Object> highlights = new HashMap<>();
for (Map.Entry<String, HighlightField> entry : hit.getHighlightFields().entrySet()) {
List<String> fragments = new ArrayList<>();
Arrays.stream(entry.getValue().fragments())
.forEach(f -> fragments.add(f.string()));
highlights.put(entry.getKey(), fragments);
}
String content = (String) source.get("content");
String summary = extractMultiSnippet(content, requestBody.getFlatKeywords(), 50);
result.put("summary", summary);
result.put("highlight", highlights.isEmpty() ? null : highlights);
results.add(result);
}
// 返回分页结构
Map<String, Object> responseBody = new LinkedHashMap<>();
responseBody.put("total", response.getHits().getTotalHits().value);
responseBody.put("page", page);
responseBody.put("size", pageSize);
responseBody.put("size", size);
responseBody.put("results", results);
return responseBody;
}
private String extractMultiSnippet(String content, List<String> keywords, int contextLength) {
if (content == null || keywords == null || keywords.isEmpty()) return "";
String lowered = content.toLowerCase();
Set<String> matchedSnippets = new LinkedHashSet<>();
for (String keyword : keywords) {
String loweredKeyword = keyword.toLowerCase();
int index = lowered.indexOf(loweredKeyword);
if (index == -1) continue;
int start = Math.max(0, index - contextLength);
int end = Math.min(content.length(), index + keyword.length() + contextLength);
String snippet = content.substring(start, end);
snippet = snippet.replaceAll("(?i)" + Pattern.quote(keyword), "<mark>$0</mark>");
matchedSnippets.add(snippet);
}
if (matchedSnippets.isEmpty()) {
return content.length() <= 100 ? content : content.substring(0, 100) + "...";
}
return String.join(" ... ", matchedSnippets);
}
}

51
src/main/java/com/doc/parser/infrastructure/north/dto/SearchReq.java Normal file
View File

@ -0,0 +1,51 @@
package com.doc.parser.infrastructure.north.dto;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author Luke.ye
* @date 2025/5/21 14:31
*/
public class SearchReq implements Serializable {
private static final long serialVersionUID = 563009883661483142L;
private List<List<String>> keywordGroups;
private Integer page;
private Integer size;
public List<List<String>> getKeywordGroups() {
return keywordGroups != null ? keywordGroups : new ArrayList<>();
}
public void setKeywordGroups(List<List<String>> keywordGroups) {
this.keywordGroups = keywordGroups;
}
public Integer getPage() {
return page;
}
public void setPage(Integer page) {
this.page = page;
}
public Integer getSize() {
return size;
}
public void setSize(Integer size) {
this.size = size;
}
// 所有关键词合并，用于高亮摘要
public List<String> getFlatKeywords() {
return keywordGroups == null
? new ArrayList<>()
: keywordGroups.stream().flatMap(e -> e.stream()).collect(Collectors.toList());
}
}

28
src/main/resources/application-dev-mac.properties Normal file
View File

@ -0,0 +1,28 @@
# 日志
logging.config=classpath:log/log4j.xml
# nacos相关
## nacos公用
spring.cloud.nacos.config.data-id-without-suffix=nacos
spring.cloud.nacos.config.file-extension=properties
spring.config.import=nacos:${spring.cloud.nacos.config.data-id-without-suffix}.${spring.cloud.nacos.config.file-extension}?refresh=true
## Nacos配置中心
spring.cloud.nacos.config.username=nacos
spring.cloud.nacos.config.password=lukeye
spring.cloud.nacos.config.contextPath=/nacos
spring.cloud.nacos.config.server-addr=http://101.132.255.39:8848
spring.cloud.nacos.config.namespace=lukeye
spring.cloud.nacos.config.refreshEnabled=true
# Arthas配置
arthas.telnetPort=-1
arthas.httpPort=-1
arthas.ip=127.0.0.1
arthas.appName=${spring.application.name}
arthas.tunnel-server=ws://101.132.255.39:7777/ws
# 导入的材料路径
exclude.file.path.prefix=/Users/admin/Desktop/ahnx-share-src-public
markdown.path=/Users/admin/Desktop/ahnx-share-src-public/public
pdf.path=/Users/admin/Desktop/ahnx-share-src-public/public
word.path=/Users/admin/Desktop/ahnx-share-src-public/public

2
src/main/resources/application-dev.properties → src/main/resources/application-dev-windows.properties
View File

@ -22,7 +22,7 @@ arthas.appName=${spring.application.name}
arthas.tunnel-server=ws://101.132.255.39:7777/ws
# 导入的材料路径
exclude.file.path.prefix=D:/02-documents/
exclude.file.path.prefix=D:/02-documents/01-ahnx-share-src-public
markdown.path=D:/02-documents/01-ahnx-share-src-public/public
pdf.path=D:/02-documents/01-ahnx-share-src-public/public
word.path=D:/02-documents/01-ahnx-share-src-public/public

30
src/main/resources/application-docker.properties Normal file
View File

@ -0,0 +1,30 @@
# 日志
logging.config=classpath:log/log4j.xml
# nacos相关
## nacos公用
spring.cloud.nacos.config.data-id-without-suffix=nacos
spring.cloud.nacos.config.file-extension=properties
spring.config.import=nacos:${spring.cloud.nacos.config.data-id-without-suffix}.${spring.cloud.nacos.config.file-extension}?refresh=true
## Nacos配置中心
spring.cloud.nacos.config.username=nacos
spring.cloud.nacos.config.password=lukeye
spring.cloud.nacos.config.contextPath=/nacos
spring.cloud.nacos.config.server-addr=http://101.132.255.39:8848
spring.cloud.nacos.config.namespace=lukeye
spring.cloud.nacos.config.refreshEnabled=true
# Arthas配置
arthas.telnetPort=-1
arthas.httpPort=-1
arthas.ip=127.0.0.1
arthas.appName=${spring.application.name}
arthas.tunnel-server=ws://101.132.255.39:7777/ws
# 导入目录（容器内路径）
markdown.path=/app/import-data
pdf.path=/app/import-data
word.path=/app/import-data
# 相对路径裁剪
exclude-file-path-prefix=/app/

17
src/main/resources/application.properties
View File

@ -11,17 +11,16 @@ logging.level.root=${logging.level.com.doc.parser}
logging.org.springframework.web=${logging.level.com.doc.parser}
logging.org.springframework.context=${logging.level.com.doc.parser}
# ES相关
elasticsearch.host=es.wisdompulse.cn
elasticsearch.port=80
elasticsearch.scheme=http
# 导入的材料路径
exclude.file.path.prefix=/Users/admin/Desktop
markdown.path=/Users/admin/Desktop/ahnx-share-src-public
pdf.path=/Users/admin/Desktop/ahnx-share-src-public
word.path=/Users/admin/Desktop/ahnx-share-src-public
search.default-page-size=100
# 是否打开定时导入，且指定导入的频率
import.schedule.enabled=true
# 每小时执行一次（可改）
import.schedule.cron=0 0 * * * *