lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opt

Browse Source
This commit is contained in:
luke 2025-06-20 12:52:21 +08:00
parent 276773131c
commit 25f0ef9cf2
3 changed files with 81 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java
View File

@ -15,19 +15,11 @@ import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
import com.knowledge.base.infrastructure.util.CacheUtil;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -49,7 +41,7 @@ public class FileQueryController {
private final UserAppService userAppService;
private final RestHighLevelClient esClient;
private final FileElasticsearchGateway esGateway;
@Autowired
private FileCacheService fileCacheService;
@ -64,17 +56,12 @@ public class FileQueryController {
public Map<String, Object> search(@RequestBody SearchReq requestBody) throws IOException {
int page = requestBody.getPage() != null && requestBody.getPage() > 0 ? requestBody.getPage() : 1;
int size = requestBody.getSize() != null && requestBody.getSize() > 0 ? requestBody.getSize() : defaultPageSize;
int from = (page - 1) * size;
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(buildSearchQuery(requestBody.getKeywordGroups()))
.from(from)
.size(size)
.sort("mtime", SortOrder.DESC);
SearchRequest request = new SearchRequest("documents").source(builder);
SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
SearchResponse response = esGateway.searchFileDocuments(
requestBody.getKeywordGroups(),
page,
size
);
return buildSearchResponse(response, requestBody.getFlatKeywords(), page, size);
}
@ -171,26 +158,6 @@ public class FileQueryController {
return ResponseEntity.ok(response);
}
private QueryBuilder buildSearchQuery(List<List<String>> keywordGroups) {
BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();
for (List<String> group : keywordGroups) {
BoolQueryBuilder groupQuery = QueryBuilders.boolQuery();
for (String keyword : group) {
groupQuery.must(QueryBuilders.multiMatchQuery(keyword, "filename", "content")
.type(MultiMatchQueryBuilder.Type.PHRASE));
}
outerQuery.should(groupQuery);
}
outerQuery.minimumShouldMatch(1);
// 东八区时间点
outerQuery.filter(QueryBuilders.rangeQuery("expireTime")
.gte(System.currentTimeMillis())
.timeZone("+08:00"));
return outerQuery;
}
private Map<String, Object> buildSearchResponse(SearchResponse response, List<String> keywords, int page, int size) {
List<Map<String, Object>> results = new ArrayList<>();

14
src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java
View File

@ -14,14 +14,11 @@ import com.knowledge.base.infrastructure.config.DynamicConfig;
import com.knowledge.base.infrastructure.config.ObjectStorageProperties;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
import com.knowledge.base.infrastructure.south.minio.ObjectStorageGateway;
import com.knowledge.base.infrastructure.util.DateUtil;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -43,10 +40,9 @@ import java.util.stream.Collectors;
public class FileWriteController {
private static final Logger logger = LoggerFactory.getLogger(FileWriteController.class);
private final RestHighLevelClient esClient;
private final FileElasticsearchGateway esGateway;
private final FileCacheService fileCacheService;
private static final String INDEX_NAME = "documents";
private final DynamicConfig dynamicConfig;
@ -93,11 +89,7 @@ public class FileWriteController {
fileCacheService.clearAll();
// 2. 清空 ES documents 索引所有文档
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(INDEX_NAME);
deleteRequest.setQuery(QueryBuilders.matchAllQuery());
esClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT);
logger.warn("已清空 ES 和缓存中所有文件相关数据！");
esGateway.clearAllFileDocuments();
return ResponseEntity.ok(Map.of("msg", "所有文件相关数据已清空"));
} catch (Exception e) {
logger.error("全量清空失败", e);

71
src/main/java/com/knowledge/base/infrastructure/south/es/FileElasticsearchGateway.java
View File

@ -10,11 +10,17 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -35,6 +41,55 @@ public class FileElasticsearchGateway {
@Autowired
private RestHighLevelClient esClient;
public SearchResponse searchFileDocuments(List<List<String>> keywordGroups, int page, int size) {
try {
int from = (page - 1) * size;
QueryBuilder query = buildSearchQuery(keywordGroups);
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(query)
.from(from)
.size(size)
.sort("mtime", SortOrder.DESC);
SearchRequest request = new SearchRequest(INDEX_NAME).source(builder);
return esClient.search(request, RequestOptions.DEFAULT);
} catch (Exception e) {
log.error("searchFileDocuments 查询失败", e);
throw new RuntimeException("文件搜索失败", e);
}
}
/**
* 构建支持中文分词的多字段搜索查询（filename + content）
*/
private QueryBuilder buildSearchQuery(List<List<String>> keywordGroups) {
BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();
for (List<String> group : keywordGroups) {
BoolQueryBuilder groupQuery = QueryBuilders.boolQuery();
for (String keyword : group) {
groupQuery.must(QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery("filename", keyword).analyzer("ik_smart"))
.should(QueryBuilders.matchQuery("content", keyword).analyzer("ik_smart"))
);
}
outerQuery.should(groupQuery);
}
outerQuery.minimumShouldMatch(1);
// 过滤掉过期文件
outerQuery.filter(QueryBuilders.rangeQuery("expireTime")
.gte(System.currentTimeMillis())
.timeZone("+08:00"));
return outerQuery;
}
/**
* 根据 filepath 批量删除文档
*
@ -130,5 +185,21 @@ public class FileElasticsearchGateway {
log.error("ES 写入文档失败: index={}, docId={}", index, docId, e);
}
}
public void clearAllFileDocuments() {
try {
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(INDEX_NAME);
deleteRequest.setQuery(QueryBuilders.matchAllQuery());
deleteRequest.setConflicts("proceed");
deleteRequest.setRefresh(true);
BulkByScrollResponse response = esClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT);
log.warn("已清空 ES 索引 [{}]，共删除文档数: {}", INDEX_NAME, response.getDeleted());
} catch (Exception e) {
log.error("清空 ES 索引 [{}] 失败", INDEX_NAME, e);
throw new RuntimeException("清空 ES 索引失败: " + e.getMessage(), e);
}
}
}