lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持对ppt和txt的导入支持

Browse Source
This commit is contained in:
luke 2025-06-21 23:51:43 +08:00
parent 09d2aac37e
commit 33df9bb0f8
15 changed files with 165 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Readme.md
View File

@ -10,6 +10,7 @@
| 1.0.4 | 2025-06-17 | Luke.Ye | 上传文档记录入库 |
| 1.0.5 | 2025-06-19 | Luke.Ye | 文档导入ES逻辑优化 |
| 1.0.6 | 2025-06-20 | Luke.Ye | 完成对象存储删除接口，优化部分代码 |
| 1.1.0 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |

4
src/main/java/com/knowledge/base/domain/common/enums/DocTypeEnum.java
View File

@ -7,10 +7,12 @@ package com.knowledge.base.domain.common.enums;
public enum DocTypeEnum {
MARKDOWN("markdown", "markdown"),
HTML("xlsx", "excel"),
EXCEL("excel", "excel"),
PDF("pdf", "pdf"),
WORD("word", "word"),
PPT("ppt", "ppt"),
TXT("txt", "txt"),
;
public String code;

2
src/main/java/com/knowledge/base/domain/doc/service/FileImporterDispatcher.java
View File

@ -1,6 +1,6 @@
package com.knowledge.base.domain.doc.service;
import com.knowledge.base.domain.doc.service.impl.AbstractBaseFileImporter;
import com.knowledge.base.domain.doc.service.impl.importer.AbstractBaseFileImporter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

3
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java → src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java
View File

@ -1,4 +1,4 @@
package com.knowledge.base.domain.doc.service.impl;
package com.knowledge.base.domain.doc.service.impl.importer;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.map.MapUtil;
@ -31,7 +31,6 @@ import java.util.*;
public abstract class AbstractBaseFileImporter implements DocumentImporter {
private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class);
private static final String INDEX_NAME = "documents";
@Autowired
private FileElasticsearchGateway esGateway;

5
src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java → src/main/java/com/knowledge/base/domain/doc/service/impl/importer/ExcelImporter.java
View File

@ -1,6 +1,7 @@
package com.knowledge.base.domain.doc.service.impl;
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
@ -35,7 +36,7 @@ public class ExcelImporter extends AbstractBaseFileImporter {
@Override
protected String getDocTypeCode() {
return "excel";
return DocTypeEnum.EXCEL.code;
}
@Override

2
src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java → src/main/java/com/knowledge/base/domain/doc/service/impl/importer/MarkdownImporter.java
View File

@ -1,4 +1,4 @@
package com.knowledge.base.domain.doc.service.impl;
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.beans.factory.annotation.Value;

2
src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java → src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PdfImporter.java
View File

@ -1,4 +1,4 @@
package com.knowledge.base.domain.doc.service.impl;
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.stereotype.Component;

96
src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PowerPointImporter.java Normal file
View File

@ -0,0 +1,96 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFSlide;
import org.apache.poi.hslf.usermodel.HSLFTextParagraph;
import org.apache.poi.xslf.usermodel.XMLSlideShow;
import org.apache.poi.xslf.usermodel.XSLFSlide;
import org.apache.poi.xslf.usermodel.XSLFTextShape;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* PowerPoint 文件导入器，支持 .ppt 和 .pptx 文件。
* 利用 Apache POI 提取幻灯片中的所有文本内容。
*/
@Component
public class PowerPointImporter extends AbstractBaseFileImporter {
@Value("${ppt.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".ppt", ".pptx");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.PPT.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
/**
* 提取 PowerPoint 文件中的全部文字内容。
*
* @param path PPT 文件路径
* @return 幻灯片中全部文字拼接而成的字符串
* @throws IOException 文件读取异常
*/
@Override
protected String extractContent(Path path) throws IOException {
String fileName = path.getFileName().toString().toLowerCase();
try (FileInputStream fis = new FileInputStream(path.toFile())) {
if (fileName.endsWith(".pptx")) {
XMLSlideShow pptx = new XMLSlideShow(fis);
StringBuilder content = new StringBuilder();
for (XSLFSlide slide : pptx.getSlides()) {
slide.getShapes().stream()
.filter(shape -> shape instanceof XSLFTextShape)
.map(shape -> (XSLFTextShape) shape)
.map(XSLFTextShape::getText)
.filter(StrUtil::isNotBlank)
.forEach(text -> content.append(text.trim()).append("\n"));
}
return content.toString();
} else if (fileName.endsWith(".ppt")) {
HSLFSlideShow ppt = new HSLFSlideShow(fis);
StringBuilder content = new StringBuilder();
for (HSLFSlide slide : ppt.getSlides()) {
List<List<HSLFTextParagraph>> textParagraphs = slide.getTextParagraphs();
List<HSLFTextParagraph> paragraphs = textParagraphs.stream()
.flatMap(List::stream)
.collect(Collectors.toList());
String slideText = HSLFTextParagraph.getRawText(paragraphs);
if (StrUtil.isNotBlank(slideText)) {
content.append(slideText.trim()).append("\n");
}
}
return content.toString();
} else {
throw new IllegalArgumentException("Unsupported PowerPoint format: " + fileName);
}
}
}
}

49
src/main/java/com/knowledge/base/domain/doc/service/impl/importer/TxtImporter.java Normal file
View File

@ -0,0 +1,49 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Set;
/**
* 文本文件导入器，支持 .txt 格式。
* 内容按 UTF-8 编码读取，无结构提取，仅用于全文索引。
*/
@Component
public class TxtImporter extends AbstractBaseFileImporter {
@Value("${txt.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".txt");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.TXT.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
@Override
protected String extractContent(Path path) throws IOException {
// 读取 UTF-8 编码文本，去除 BOM 和首尾空白
String content = Files.readString(path, StandardCharsets.UTF_8);
return content.replace("\uFEFF", "").trim();
}
}

2
src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java → src/main/java/com/knowledge/base/domain/doc/service/impl/importer/WordImporter.java
View File

@ -1,4 +1,4 @@
package com.knowledge.base.domain.doc.service.impl;
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hwpf.HWPFDocument;

2
src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java
View File

@ -29,7 +29,7 @@ public class DynamicConfig {
@Value("${token.expire.time:86400000}")
private long tokenExpireTime;
@Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx}")
@Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx,ppt,pptx,txt}")
private String supportedSearchFileSuffix;
@Value("${file.import.rate.limit: 10}")

2
src/main/java/com/knowledge/base/infrastructure/south/minio/MinioOSGatewayImpl.java
View File

@ -40,7 +40,7 @@ public class MinioOSGatewayImpl implements ObjectStorageGateway {
String localEndPoint = "http://localhost:19000";
if("docker".equals(activeProfile)) {
localEndPoint = "http://host.docker.internal:19000";
}else if("mac".equals(activeProfile)) {
}else if("dev-mac".equals(activeProfile)) {
localEndPoint = "http://s3.wisdompulse.cn";
}
this.localMinioClient = MinioClient.builder()

2
src/main/resources/application-dev-mac.properties
View File

@ -27,6 +27,8 @@ markdown.path=/Users/admin/Desktop/Archived/micro-saas
pdf.path=/Users/admin/Desktop/Archived/micro-saas
word.path=/Users/admin/Desktop/Archived/micro-saas
excel.path=/Users/admin/Desktop/Archived/micro-saas
ppt.path=/Users/admin/Desktop/Archived/micro-saas
txt.path=/Users/admin/Desktop/Archived/micro-saas
object.storage.local-searchable-path-prefix=/Users/admin/Desktop/Archived/micro-saas
# mysql

2
src/main/resources/application-dev-windows.properties
View File

@ -31,6 +31,8 @@ markdown.path=D:/02-documents/01-ahnx-share-src-public/public
pdf.path=D:/02-documents/01-os-uploaded-searchable-files
word.path=D:/02-documents/01-os-uploaded-searchable-files
excel.path=D:/02-documents/01-os-uploaded-searchable-files
ppt.path=D:/02-documents/01-os-uploaded-searchable-files
txt.path=D:/02-documents/01-os-uploaded-searchable-files
object.storage.local-searchable-path-prefix=D:/02-documents/01-os-uploaded-searchable-files

2
src/main/resources/application-docker.properties
View File

@ -30,6 +30,8 @@ markdown.path=/app/import-data
pdf.path=/app/os-uploaded-searchable-files
word.path=/app/os-uploaded-searchable-files
excel.path=/app/os-uploaded-searchable-files
ppt.path=/app/os-uploaded-searchable-files
txt.path=/app/os-uploaded-searchable-files
object.storage.local-searchable-path-prefix=/app/os-uploaded-searchable-files
# 相对路径裁剪