diff --git a/Readme.md b/Readme.md index 44ea155..82ed758 100644 --- a/Readme.md +++ b/Readme.md @@ -10,6 +10,7 @@ | 1.0.4 | 2025-06-17 | Luke.Ye | 上传文档记录入库 | | 1.0.5 | 2025-06-19 | Luke.Ye | 文档导入ES逻辑优化 | | 1.0.6 | 2025-06-20 | Luke.Ye | 完成对象存储删除接口，优化部分代码 | +| 1.1.0 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt | diff --git a/src/main/java/com/knowledge/base/domain/common/enums/DocTypeEnum.java b/src/main/java/com/knowledge/base/domain/common/enums/DocTypeEnum.java index 0890d7f..8bb9060 100644 --- a/src/main/java/com/knowledge/base/domain/common/enums/DocTypeEnum.java +++ b/src/main/java/com/knowledge/base/domain/common/enums/DocTypeEnum.java @@ -7,10 +7,12 @@ package com.knowledge.base.domain.common.enums; public enum DocTypeEnum { MARKDOWN("markdown", "markdown"), - HTML("xlsx", "excel"), + EXCEL("excel", "excel"), PDF("pdf", "pdf"), WORD("word", "word"), + PPT("ppt", "ppt"), + TXT("txt", "txt"), ; public String code; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/FileImporterDispatcher.java b/src/main/java/com/knowledge/base/domain/doc/service/FileImporterDispatcher.java index 13318b4..e5b1163 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/FileImporterDispatcher.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/FileImporterDispatcher.java @@ -1,6 +1,6 @@ package com.knowledge.base.domain.doc.service; -import com.knowledge.base.domain.doc.service.impl.AbstractBaseFileImporter; +import com.knowledge.base.domain.doc.service.impl.importer.AbstractBaseFileImporter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java similarity index 99% rename from src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java rename to src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java index 01d9fd2..6cc139c 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/AbstractBaseFileImporter.java @@ -1,4 +1,4 @@ -package com.knowledge.base.domain.doc.service.impl; +package com.knowledge.base.domain.doc.service.impl.importer; import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.map.MapUtil; @@ -31,7 +31,6 @@ import java.util.*; public abstract class AbstractBaseFileImporter implements DocumentImporter { private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class); - private static final String INDEX_NAME = "documents"; @Autowired private FileElasticsearchGateway esGateway; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/ExcelImporter.java similarity index 93% rename from src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java rename to src/main/java/com/knowledge/base/domain/doc/service/impl/importer/ExcelImporter.java index d94faee..46c73f0 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/ExcelImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/ExcelImporter.java @@ -1,6 +1,7 @@ -package com.knowledge.base.domain.doc.service.impl; +package com.knowledge.base.domain.doc.service.impl.importer; +import com.knowledge.base.domain.common.enums.DocTypeEnum; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -35,7 +36,7 @@ public class ExcelImporter extends AbstractBaseFileImporter { @Override protected String getDocTypeCode() { - return "excel"; + return DocTypeEnum.EXCEL.code; } @Override diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/MarkdownImporter.java similarity index 94% rename from src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java rename to src/main/java/com/knowledge/base/domain/doc/service/impl/importer/MarkdownImporter.java index ba5775f..d5dd6dc 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/MarkdownImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/MarkdownImporter.java @@ -1,4 +1,4 @@ -package com.knowledge.base.domain.doc.service.impl; +package com.knowledge.base.domain.doc.service.impl.importer; import com.knowledge.base.domain.common.enums.DocTypeEnum; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PdfImporter.java similarity index 94% rename from src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java rename to src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PdfImporter.java index 2f58e35..1696411 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/PdfImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PdfImporter.java @@ -1,4 +1,4 @@ -package com.knowledge.base.domain.doc.service.impl; +package com.knowledge.base.domain.doc.service.impl.importer; import com.knowledge.base.domain.common.enums.DocTypeEnum; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PowerPointImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PowerPointImporter.java new file mode 100644 index 0000000..2f59a7b --- /dev/null +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/PowerPointImporter.java @@ -0,0 +1,96 @@ +package com.knowledge.base.domain.doc.service.impl.importer; + +import cn.hutool.core.util.StrUtil; +import com.knowledge.base.domain.common.enums.DocTypeEnum; +import org.apache.poi.hslf.usermodel.HSLFSlideShow; +import org.apache.poi.hslf.usermodel.HSLFSlide; +import org.apache.poi.hslf.usermodel.HSLFTextParagraph; +import org.apache.poi.xslf.usermodel.XMLSlideShow; +import org.apache.poi.xslf.usermodel.XSLFSlide; +import org.apache.poi.xslf.usermodel.XSLFTextShape; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * PowerPoint 文件导入器，支持 .ppt 和 .pptx 文件。 + * 利用 Apache POI 提取幻灯片中的所有文本内容。 + */ +@Component +public class PowerPointImporter extends AbstractBaseFileImporter { + + @Value("${ppt.path}") + private String directoryPath; + + @Override + protected String getDirectoryPath() { + return directoryPath; + } + + @Override + public Set getFileSuffixes() { + return Set.of(".ppt", ".pptx"); + } + + @Override + protected String getDocTypeCode() { + return DocTypeEnum.PPT.code; + } + + @Override + protected String getExcludePrefix() { + return directoryPath; + } + + /** + * 提取 PowerPoint 文件中的全部文字内容。 + * + * @param path PPT 文件路径 + * @return 幻灯片中全部文字拼接而成的字符串 + * @throws IOException 文件读取异常 + */ + @Override + protected String extractContent(Path path) throws IOException { + String fileName = path.getFileName().toString().toLowerCase(); + + try (FileInputStream fis = new FileInputStream(path.toFile())) { + if (fileName.endsWith(".pptx")) { + XMLSlideShow pptx = new XMLSlideShow(fis); + StringBuilder content = new StringBuilder(); + for (XSLFSlide slide : pptx.getSlides()) { + slide.getShapes().stream() + .filter(shape -> shape instanceof XSLFTextShape) + .map(shape -> (XSLFTextShape) shape) + .map(XSLFTextShape::getText) + .filter(StrUtil::isNotBlank) + .forEach(text -> content.append(text.trim()).append("\n")); + } + return content.toString(); + } else if (fileName.endsWith(".ppt")) { + HSLFSlideShow ppt = new HSLFSlideShow(fis); + StringBuilder content = new StringBuilder(); + for (HSLFSlide slide : ppt.getSlides()) { + List> textParagraphs = slide.getTextParagraphs(); + List paragraphs = textParagraphs.stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + String slideText = HSLFTextParagraph.getRawText(paragraphs); + if (StrUtil.isNotBlank(slideText)) { + content.append(slideText.trim()).append("\n"); + } + } + return content.toString(); + } else { + throw new IllegalArgumentException("Unsupported PowerPoint format: " + fileName); + } + } + } + + +} diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/TxtImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/TxtImporter.java new file mode 100644 index 0000000..b4f4035 --- /dev/null +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/TxtImporter.java @@ -0,0 +1,49 @@ +package com.knowledge.base.domain.doc.service.impl.importer; +import com.knowledge.base.domain.common.enums.DocTypeEnum; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +/** + * 文本文件导入器，支持 .txt 格式。 + * 内容按 UTF-8 编码读取，无结构提取，仅用于全文索引。 + */ +@Component +public class TxtImporter extends AbstractBaseFileImporter { + + @Value("${txt.path}") + private String directoryPath; + + @Override + protected String getDirectoryPath() { + return directoryPath; + } + + @Override + public Set getFileSuffixes() { + return Set.of(".txt"); + } + + @Override + protected String getDocTypeCode() { + return DocTypeEnum.TXT.code; + } + + @Override + protected String getExcludePrefix() { + return directoryPath; + } + + @Override + protected String extractContent(Path path) throws IOException { + // 读取 UTF-8 编码文本，去除 BOM 和首尾空白 + String content = Files.readString(path, StandardCharsets.UTF_8); + return content.replace("\uFEFF", "").trim(); + } +} + diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/WordImporter.java similarity index 97% rename from src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java rename to src/main/java/com/knowledge/base/domain/doc/service/impl/importer/WordImporter.java index 2d06bbd..1d32824 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/WordImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/importer/WordImporter.java @@ -1,4 +1,4 @@ -package com.knowledge.base.domain.doc.service.impl; +package com.knowledge.base.domain.doc.service.impl.importer; import com.knowledge.base.domain.common.enums.DocTypeEnum; import org.apache.poi.hwpf.HWPFDocument; diff --git a/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java b/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java index dec22bb..83954e7 100644 --- a/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java +++ b/src/main/java/com/knowledge/base/infrastructure/config/DynamicConfig.java @@ -29,7 +29,7 @@ public class DynamicConfig { @Value("${token.expire.time:86400000}") private long tokenExpireTime; - @Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx}") + @Value("${os.supported.searchable.file.suffix: pdf,doc,docx,xls,xlsx,ppt,pptx,txt}") private String supportedSearchFileSuffix; @Value("${file.import.rate.limit: 10}") diff --git a/src/main/java/com/knowledge/base/infrastructure/south/minio/MinioOSGatewayImpl.java b/src/main/java/com/knowledge/base/infrastructure/south/minio/MinioOSGatewayImpl.java index 87a181a..7cb269e 100644 --- a/src/main/java/com/knowledge/base/infrastructure/south/minio/MinioOSGatewayImpl.java +++ b/src/main/java/com/knowledge/base/infrastructure/south/minio/MinioOSGatewayImpl.java @@ -40,7 +40,7 @@ public class MinioOSGatewayImpl implements ObjectStorageGateway { String localEndPoint = "http://localhost:19000"; if("docker".equals(activeProfile)) { localEndPoint = "http://host.docker.internal:19000"; - }else if("mac".equals(activeProfile)) { + }else if("dev-mac".equals(activeProfile)) { localEndPoint = "http://s3.wisdompulse.cn"; } this.localMinioClient = MinioClient.builder() diff --git a/src/main/resources/application-dev-mac.properties b/src/main/resources/application-dev-mac.properties index 9cae15f..3d13f7a 100644 --- a/src/main/resources/application-dev-mac.properties +++ b/src/main/resources/application-dev-mac.properties @@ -27,6 +27,8 @@ markdown.path=/Users/admin/Desktop/Archived/micro-saas pdf.path=/Users/admin/Desktop/Archived/micro-saas word.path=/Users/admin/Desktop/Archived/micro-saas excel.path=/Users/admin/Desktop/Archived/micro-saas +ppt.path=/Users/admin/Desktop/Archived/micro-saas +txt.path=/Users/admin/Desktop/Archived/micro-saas object.storage.local-searchable-path-prefix=/Users/admin/Desktop/Archived/micro-saas # mysql diff --git a/src/main/resources/application-dev-windows.properties b/src/main/resources/application-dev-windows.properties index 4b121bb..5340842 100644 --- a/src/main/resources/application-dev-windows.properties +++ b/src/main/resources/application-dev-windows.properties @@ -31,6 +31,8 @@ markdown.path=D:/02-documents/01-ahnx-share-src-public/public pdf.path=D:/02-documents/01-os-uploaded-searchable-files word.path=D:/02-documents/01-os-uploaded-searchable-files excel.path=D:/02-documents/01-os-uploaded-searchable-files +ppt.path=D:/02-documents/01-os-uploaded-searchable-files +txt.path=D:/02-documents/01-os-uploaded-searchable-files object.storage.local-searchable-path-prefix=D:/02-documents/01-os-uploaded-searchable-files diff --git a/src/main/resources/application-docker.properties b/src/main/resources/application-docker.properties index 0792b00..e81b4a5 100644 --- a/src/main/resources/application-docker.properties +++ b/src/main/resources/application-docker.properties @@ -30,6 +30,8 @@ markdown.path=/app/import-data pdf.path=/app/os-uploaded-searchable-files word.path=/app/os-uploaded-searchable-files excel.path=/app/os-uploaded-searchable-files +ppt.path=/app/os-uploaded-searchable-files +txt.path=/app/os-uploaded-searchable-files object.storage.local-searchable-path-prefix=/app/os-uploaded-searchable-files # 相对路径裁剪