lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增pdf/doc/markdown解析

Browse Source
This commit is contained in:
luke 2025-05-20 18:21:28 +08:00
parent 16a6eeab86
commit 22426a30d9
10 changed files with 359 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
config/01-创建es索引 → config/01-创建es索引.http
View File

36
pom.xml
View File

@ -35,6 +35,11 @@
<guava.version>31.0.1-jre</guava.version>
<elasticsearch-rest-high-level-client.version>7.17.10</elasticsearch-rest-high-level-client.version>
<pdfbox.version>2.0.29</pdfbox.version>
<poi.version>5.2.3</poi.version>
<poi-ooxml.version>5.2.3</poi-ooxml.version>
</properties>
<dependencyManagement>
<dependencies>
@ -159,6 +164,24 @@
<version>${elasticsearch-rest-high-level-client.version}</version>
</dependency>
<!-- pdf/word的解析 -->
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi-ooxml.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -267,6 +290,19 @@
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
</dependency>
</dependencies>
<build>

24
src/main/java/com/doc/parser/domain/enums/DocTypeEnum.java Normal file
View File

@ -0,0 +1,24 @@
package com.doc.parser.domain.enums;
/**
* @author Luke.ye
* @date 2025/5/20 09:03
*/
public enum DocTypeEnum {
MARKDOWN("markdown", "markdown"),
HTML("html", "html"),
PDF("pdf", "pdf"),
WORD("word", "word"),
;
public String code;
public String desc;
DocTypeEnum(String code, String desc) {
this.code = code;
this.desc = desc;
}
}

15
src/main/java/com/doc/parser/domain/iface/DocumentImporter.java Normal file
View File

@ -0,0 +1,15 @@
package com.doc.parser.domain.iface;
/**
* @author Luke.ye
* @date 2025/5/20 09:02
*/
public interface DocumentImporter {
void importDocuments() throws Exception;
/**
* {@link com.doc.parser.domain.enums.DocTypeEnum}
* @return
*/
String getType();
}

61
src/main/java/com/doc/parser/domain/impl/MarkdownImporter.java Normal file
View File

@ -0,0 +1,61 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.enums.DocTypeEnum;
import com.doc.parser.domain.iface.DocumentImporter;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.file.*;
import java.util.HashMap;
import java.util.Map;
/**
* @author Luke.ye
* @date 2025/5/20 09:06
*/
@Component
public class MarkdownImporter implements DocumentImporter {
private static final Logger logger = LoggerFactory.getLogger(MarkdownImporter.class);
private static final String INDEX_NAME = "documents";
@Autowired
private RestHighLevelClient esClient;
@Value("${markdown.path}")
private String directoryPath;
@Override
public void importDocuments() throws IOException {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".md"))
.forEach(path -> {
try {
String content = Files.readString(path);
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content);
IndexRequest request = new IndexRequest(INDEX_NAME).source(doc);
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", path.getFileName());
} catch (IOException e) {
logger.error("导入失败: {}", path, e);
}
});
}
@Override
public String getType() {
return DocTypeEnum.MARKDOWN.code;
}
}

61
src/main/java/com/doc/parser/domain/impl/PdfImporter.java Normal file
View File

@ -0,0 +1,61 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.enums.DocTypeEnum;
import com.doc.parser.domain.iface.DocumentImporter;
import org.springframework.stereotype.Component;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import java.io.File;
import java.nio.file.*;
import java.util.*;
/**
* @author Luke.ye
* @date 2025/5/20 10:23
*/
@Component
public class PdfImporter implements DocumentImporter {
private static final Logger LOGGER = LoggerFactory.getLogger(PdfImporter.class);
private final RestHighLevelClient esClient;
@Value("${pdf.path}")
private String directoryPath;
public PdfImporter(RestHighLevelClient esClient) {
this.esClient = esClient;
}
@Override
public void importDocuments() throws Exception {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".pdf"))
.forEach(path -> {
try (PDDocument document = PDDocument.load(new File(path.toString()))) {
String content = new PDFTextStripper().getText(document);
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content);
IndexRequest request = new IndexRequest("documents").source(doc);
esClient.index(request, RequestOptions.DEFAULT);
LOGGER.info("导入成功: {}", path.getFileName());
} catch (Exception e) {
LOGGER.error("导入失败: {}", path, e);
}
});
}
@Override
public String getType() {
return DocTypeEnum.PDF.code;
}
}

65
src/main/java/com/doc/parser/domain/impl/WordImporter.java Normal file
View File

@ -0,0 +1,65 @@
package com.doc.parser.domain.impl;
import com.doc.parser.domain.enums.DocTypeEnum;
import com.doc.parser.domain.iface.DocumentImporter;
import org.springframework.stereotype.Component;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import java.io.FileInputStream;
import java.nio.file.*;
import java.util.*;
/**
* @author Luke.ye
* @date 2025/5/20 10:25
*/
@Component
public class WordImporter implements DocumentImporter {
private static final Logger LOGGER = LoggerFactory.getLogger(WordImporter.class);
private final RestHighLevelClient esClient;
@Value("${word.path}")
private String directoryPath;
public WordImporter(RestHighLevelClient esClient) {
this.esClient = esClient;
}
@Override
public void importDocuments() throws Exception {
Files.walk(Paths.get(directoryPath))
.filter(p -> p.toString().toLowerCase().endsWith(".docx"))
.forEach(path -> {
try (XWPFDocument docx = new XWPFDocument(new FileInputStream(path.toFile()))) {
StringBuilder content = new StringBuilder();
for (XWPFParagraph para : docx.getParagraphs()) {
content.append(para.getText()).append("\n");
}
Map<String, Object> doc = new HashMap<>();
doc.put("filename", path.getFileName().toString());
doc.put("filepath", path.toAbsolutePath().toString());
doc.put("content", content.toString());
IndexRequest request = new IndexRequest("documents").source(doc);
esClient.index(request, RequestOptions.DEFAULT);
LOGGER.info("导入成功: {}", path.getFileName());
} catch (Exception e) {
LOGGER.error("导入失败: {}", path, e);
}
});
}
@Override
public String getType() {
return DocTypeEnum.WORD.code;
}
}

31
src/main/java/com/doc/parser/infrastructure/north/controller/ImportController.java Normal file
View File

@ -0,0 +1,31 @@
package com.doc.parser.infrastructure.north.controller;
import com.doc.parser.domain.iface.DocumentImporter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
@RequestMapping("/import")
public class ImportController {
@Autowired
private List<DocumentImporter> importers;
@GetMapping("/{type}")
public ResponseEntity<String> importByType(@PathVariable String type) {
for (DocumentImporter importer : importers) {
if (importer.getType().equalsIgnoreCase(type)) {
try {
importer.importDocuments();
return ResponseEntity.ok(type + " 文件导入成功");
} catch (Exception e) {
return ResponseEntity.internalServerError().body("导入失败: " + e.getMessage());
}
}
}
return ResponseEntity.badRequest().body("不支持的类型: " + type);
}
}

61
src/main/java/com/doc/parser/infrastructure/north/controller/SearchController.java Normal file
View File

@ -0,0 +1,61 @@
package com.doc.parser.infrastructure.north.controller;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.SearchHit;
import org.springframework.web.bind.annotation.*;
import java.util.*;
/**
* @author Luke.ye
* @date 2025/5/20 16:56
*/
@RestController
@RequestMapping("/search")
public class SearchController {
private final RestHighLevelClient esClient;
public SearchController(RestHighLevelClient esClient) {
this.esClient = esClient;
}
@GetMapping
public List<Map<String, Object>> search(@RequestParam String q) throws Exception {
SearchRequest request = new SearchRequest("documents");
HighlightBuilder highlight = new HighlightBuilder()
.field("content")
.preTags("<mark>")
.postTags("</mark>")
.fragmentSize(80)
.numOfFragments(3);
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.matchQuery("content", q))
.highlighter(highlight)
.size(10);
request.source(builder);
SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
List<Map<String, Object>> results = new ArrayList<>();
for (SearchHit hit : response.getHits()) {
Map<String, Object> source = hit.getSourceAsMap();
Map<String, Object> result = new HashMap<>();
result.put("filename", source.get("filename"));
result.put("filepath", source.get("filepath"));
result.put("highlight", hit.getHighlightFields().getOrDefault("content", null));
results.add(result);
}
return results;
}
}

8
src/main/resources/application.properties
View File

@ -13,10 +13,12 @@ logging.org.springframework.context=${logging.level.com.doc.parser}
# ES相关
elasticsearch.host=localhost
elasticsearch.port=9200
elasticsearch.host=es.wisdompulse.cn
elasticsearch.port=80
elasticsearch.scheme=http
# 导入的材料路径
markdown.path = D:/02-documents/01-ahnx-share-src-public/public/设计模式
markdown.path=/Users/admin/Desktop/ahnx-share-src-public
pdf.path=/Users/admin/Desktop/ahnx-share-src-public
word.path=/Users/admin/Desktop/ahnx-share-src-public