lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化文件导入逻辑

Browse Source
This commit is contained in:
luke 2025-06-19 18:21:07 +08:00
parent 74084d2faf
commit 8567f6bd62
15 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/mysql/init.txt
View File

@ -87,6 +87,7 @@ CREATE TABLE IF NOT EXISTS user_file (
CREATE TABLE IF NOT EXISTS oss_upload_record ( CREATE TABLE IF NOT EXISTS oss_upload_record (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键', id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
file_name VARCHAR(255) NOT NULL COMMENT '文件名', file_name VARCHAR(255) NOT NULL COMMENT '文件名',
local_rela_file_path VARCHAR(512) NOT NULL DEFAULT 'UNKNOWN' COMMENT '本地文件相对路径',
bucket_name VARCHAR(128) NOT NULL COMMENT '目标 bucket 名', bucket_name VARCHAR(128) NOT NULL COMMENT '目标 bucket 名',
uploader VARCHAR(128) NOT NULL COMMENT '上传人用户名', uploader VARCHAR(128) NOT NULL COMMENT '上传人用户名',
upload_time DATETIME NOT NULL COMMENT '上传时间', upload_time DATETIME NOT NULL COMMENT '上传时间',
@ -97,8 +98,10 @@ CREATE TABLE IF NOT EXISTS oss_upload_record (
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_uploader(uploader), INDEX idx_uploader(uploader),
INDEX idx_file_name(file_name), INDEX idx_file_name(file_name),
INDEX idx_bucket(bucket_name) INDEX idx_bucket(bucket_name),
INDEX idx_local_rela_file_path(local_rela_file_path(100))
) COMMENT='对象存储上传记录表'; ) COMMENT='对象存储上传记录表';

1
config/redis/cmd.txt
View File

@ -1,3 +1,4 @@
redis-cli --raw
# 获取redis中的key-value # 获取redis中的key-value
redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done

1
src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java
View File

@ -8,6 +8,7 @@ import java.time.LocalDateTime;
public class OSRecordDO { public class OSRecordDO {
private Long id; private Long id;
private String fileName; private String fileName;
private String localRelaFilePath;
private String bucketName; private String bucketName;
private String uploader; private String uploader;
private String objectPath; private String objectPath;

2
src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java
View File

@ -22,5 +22,5 @@ public interface OSRecordRepository {
* @param * @param
* @return * @return
*/ */
Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath); Optional<OSRecord> getLatestRecordByRelaPath(String localRelaFilePath);
} }

1
src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java
View File

@ -13,6 +13,7 @@ public class OSRecord {
private Long id; private Long id;
private String fileName; private String fileName;
private String localRelaFilePath;
private String bucketName; private String bucketName;
private String uploader; private String uploader;
private String objectPath; private String objectPath;

2
src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java
View File

@ -30,5 +30,5 @@ public interface FileDomainService {
* @param * @param
* @return * @return
*/ */
Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath); Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath);
} }

36
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java
View File

@ -83,15 +83,15 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
} }
Long localMTime = Files.getLastModifiedTime(path).toMillis(); Long localMTime = Files.getLastModifiedTime(path).toMillis();
String relativePath = relativePathObj.toString().replace("\\", "/"); String localRelaFilePath = relativePathObj.toString().replace("\\", "/");
String fileNameWithSuffix = path.getFileName().toString(); String fileNameWithSuffix = path.getFileName().toString();
Optional<String> metaJsonOpt = fileCacheService.getMeta(relativePath); Optional<String> metaJsonOpt = fileCacheService.getMeta(localRelaFilePath);
if(metaJsonOpt.isPresent()) { if(metaJsonOpt.isPresent()) {
// 有文件信息 // 有文件信息
JSON metaJson = JSONUtil.parse(metaJsonOpt.get()); JSON metaJson = JSONUtil.parse(metaJsonOpt.get());
Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class); Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class);
if(localMTime.equals(uploadTime)) { if(localMTime.equals(uploadTime)) {
logger.info("文件未变动，跳过导入: {}", relativePath); logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
return; return;
} }
} }
@ -102,40 +102,40 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
return; return;
} }
String docId = SafeIdUtil.encode(relativePath); String docId = SafeIdUtil.encode(localRelaFilePath);
GetRequest getRequest = new GetRequest(INDEX_NAME, docId); GetRequest getRequest = new GetRequest(INDEX_NAME, docId);
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) { if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT); GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
Map<String, Object> existingSource = existing.getSourceAsMap(); Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime"); Object esMtime = existingSource.get("mtime");
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", relativePath); logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
String metaJsonStr = transToMetaJson(existingSource); String metaJsonStr = transToMetaJson(existingSource);
fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); fileCacheService.cacheMeta(localRelaFilePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return; return;
} }
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId); DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId);
esClient.delete(deleteRequest, RequestOptions.DEFAULT); esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath); logger.info("已删除旧版本文件: {}", localRelaFilePath);
} }
logger.info("[开始进行文件导入......] relativePath: {}， localMTime： {}", relativePath, localMTime); logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime);
Map<String, Object> doc = new HashMap<>(); Map<String, Object> doc = new HashMap<>();
doc.put("filename", fileNameWithSuffix); doc.put("filename", fileNameWithSuffix);
doc.put("filepath", relativePath); doc.put("filepath", localRelaFilePath);
doc.put("content", content); doc.put("content", content);
doc.put("mtime", localMTime); doc.put("mtime", localMTime);
if(metaJsonOpt.isPresent()) { if(metaJsonOpt.isPresent()) {
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER)); doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER));
String cacheFileUrl = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY); String cacheFileUrl = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY);
doc.put("url", buildAccessUrl(cacheFileUrl, relativePath)); doc.put("url", buildAccessUrl(cacheFileUrl, localRelaFilePath, fileNameWithSuffix));
doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code,
Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME))); Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)));
} else { } else {
doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER); doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER);
String url = buildAccessUrl(StrUtil.EMPTY, relativePath); String url = buildAccessUrl(StrUtil.EMPTY, localRelaFilePath, fileNameWithSuffix);
doc.put("url", url); doc.put("url", url);
doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)); doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME));
} }
@ -145,32 +145,32 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
.source(doc); .source(doc);
esClient.index(request, RequestOptions.DEFAULT); esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath); logger.info("导入成功: {}", localRelaFilePath);
// 信息入缓存 // 信息入缓存
fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
} catch (Exception e) { } catch (Exception e) {
logger.error("导入失败: {}", path, e); logger.error("导入失败: {}", path, e);
} }
} }
private String buildAccessUrl(String originUrl, String relativePath) { private String buildAccessUrl(String originUrl, String localRelaFilePath, String fileNameWithSuffix) {
// 优先使用原始链接 // 优先使用原始链接
if(StrUtil.isNotBlank(originUrl)) { if(StrUtil.isNotBlank(originUrl)) {
return originUrl; return originUrl;
} }
if(StrUtil.isBlank(relativePath)) { if(StrUtil.isBlank(localRelaFilePath)) {
return StrUtil.EMPTY; return StrUtil.EMPTY;
} }
String accessUrl = originUrl; String accessUrl = originUrl;
if(relativePath.endsWith(".md")) { if(localRelaFilePath.endsWith(".md")) {
// markdown直接拼接http链接 // markdown直接拼接http链接
accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath); accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, localRelaFilePath);
accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html"; accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html";
} else { } else {
// 其它文件从对象存储的DB中获取 // 其它文件从对象存储的DB中获取
Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByFileName(relativePath); Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByRelaPath(localRelaFilePath);
if(latestRecordOpt.isPresent()) { if(latestRecordOpt.isPresent()) {
OSRecordDO latestRecord = latestRecordOpt.get(); OSRecordDO latestRecord = latestRecordOpt.get();
accessUrl = latestRecord.getUrl(); accessUrl = latestRecord.getUrl();

6
src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java
View File

@ -67,11 +67,11 @@ public class FileDomainServiceImpl implements FileDomainService {
} }
@Override @Override
public Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath) { public Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath) {
if(StrUtil.isBlank(relativeFilePath)) { if(StrUtil.isBlank(localRelaFilePath)) {
return Optional.empty(); return Optional.empty();
} }
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByFileName(relativeFilePath); Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
if(osRecordOpt.isEmpty()) { if(osRecordOpt.isEmpty()) {
return Optional.empty(); return Optional.empty();
} }

8
src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java vendored
View File

@ -10,14 +10,6 @@ import java.util.Optional;
*/ */
public interface FileCacheService { public interface FileCacheService {
/**
* 获取文件名对应的相对路径
*
* @param filename 文件名
* @return 相对路径（可能为 null）
*/
Optional<String> getPath(String filename);
/** /**
* 缓存文件名与文件相关的元信息 * 缓存文件名与文件相关的元信息
* @param fnWithRelativePath 文件名，带相对路径和后缀 * @param fnWithRelativePath 文件名，带相对路径和后缀

6
src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java vendored
View File

@ -27,12 +27,6 @@ public class LocalFileCacheServiceImpl implements FileCacheService {
return String.format("kb:file:%s:%s", prefix, key); return String.format("kb:file:%s:%s", prefix, key);
} }
@Override
public Optional<String> getPath(String filename) {
Object val = LocalCacheUtil.get(key("path", filename));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override @Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) { public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta); LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta);

6
src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java vendored
View File

@ -32,12 +32,6 @@ public class RedisFileCacheServiceImpl implements FileCacheService {
return String.format(FILE_CACHE_PREFIX + ":%s:%s", prefix, key); return String.format(FILE_CACHE_PREFIX + ":%s:%s", prefix, key);
} }
@Override
public Optional<String> getPath(String filename) {
String value = redisTemplate.opsForValue().get(key("path", filename));
return Optional.ofNullable(value);
}
@Override @Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) { public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
redisTemplate.opsForValue().set(key("meta", fnWithRelativePath), jsonMeta, expireMinutes, TimeUnit.MINUTES); redisTemplate.opsForValue().set(key("meta", fnWithRelativePath), jsonMeta, expireMinutes, TimeUnit.MINUTES);

2
src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java
View File

@ -26,4 +26,6 @@ public class ConstantConfig {
public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public"; public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public";
public static final String DEFAULT_UPLOADER = "ADMIN"; public static final String DEFAULT_UPLOADER = "ADMIN";
public static final String UNKNOWN_LOCAL_RELA_PATH = "UNKNOWN";
} }

18
src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java
View File

@ -16,10 +16,8 @@ import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO; import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import com.knowledge.base.infrastructure.south.ObjectStorageGateway; import com.knowledge.base.infrastructure.south.ObjectStorageGateway;
import com.knowledge.base.infrastructure.util.DateUtil; import com.knowledge.base.infrastructure.util.DateUtil;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil; import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
@ -181,7 +179,9 @@ public class FileWriteController {
String uploader = userDTO.get().getUsername(); String uploader = userDTO.get().getUsername();
LocalDateTime now = LocalDateTime.now(); LocalDateTime now = LocalDateTime.now();
String dateFolder = now.toLocalDate().toString(); String dateFolder = now.toLocalDate().toString();
String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix); Set<String> supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx");
boolean allowSaveToLocal = searchable && supportedTypes.contains(suffix);
String localRelaFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
boolean isPublic = Optional.ofNullable(objectStorageProperties.getPublicBuckets()) boolean isPublic = Optional.ofNullable(objectStorageProperties.getPublicBuckets())
.map(buckets -> buckets.contains(bucket)) .map(buckets -> buckets.contains(bucket))
@ -191,7 +191,8 @@ public class FileWriteController {
: now.plusDays(7); : now.plusDays(7);
OSRecordDTO dto = new OSRecordDTO(); OSRecordDTO dto = new OSRecordDTO();
dto.setFileName(relativeFilePath); dto.setFileName(originFileNameWithSuffix);
dto.setLocalRelaFilePath(allowSaveToLocal ? localRelaFilePath : ConstantConfig.UNKNOWN_LOCAL_RELA_PATH);
dto.setUrl(url); dto.setUrl(url);
dto.setUploader(uploader); dto.setUploader(uploader);
dto.setBucketName(bucket); dto.setBucketName(bucket);
@ -201,9 +202,8 @@ public class FileWriteController {
dto.setExpireTime(expireTime.toString()); dto.setExpireTime(expireTime.toString());
docAppService.saveOSUplodRecord(dto); docAppService.saveOSUplodRecord(dto);
// 可检索文件的类型 // 允许保存到本地
Set<String> supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx"); if (allowSaveToLocal) {
if (searchable && supportedTypes.contains(suffix)) {
String localDir = Paths.get(objectStorageProperties.getLocalSearchablePath(), dateFolder).toString(); String localDir = Paths.get(objectStorageProperties.getLocalSearchablePath(), dateFolder).toString();
File localTargetDir = new File(localDir); File localTargetDir = new File(localDir);
if (!localTargetDir.exists()) { if (!localTargetDir.exists()) {
@ -222,9 +222,9 @@ public class FileWriteController {
cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime)); cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime));
String metaJson = JSONUtil.toJsonStr(cacheMeta); String metaJson = JSONUtil.toJsonStr(cacheMeta);
fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson); logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", localRelaFilePath, metaJson);
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("异步写入失败: {}", originFileNameWithSuffix, e); logger.error("异步写入失败: {}", originFileNameWithSuffix, e);

6
src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java
View File

@ -6,6 +6,12 @@ import lombok.Data;
public class OSRecordDTO { public class OSRecordDTO {
private Long id; private Long id;
private String fileName; private String fileName;
/**
* 本地相对路径文件名（不为空，则文件可搜索）
*/
private String localRelaFilePath;
private String bucketName; private String bucketName;
private String uploader; private String uploader;
private String objectPath; private String objectPath;

4
src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java
View File

@ -39,10 +39,10 @@ public class OSRecordRepositoryImpl implements OSRecordRepository {
} }
@Override @Override
public Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath) { public Optional<OSRecord> getLatestRecordByRelaPath(String localRelaFilePath) {
OSRecord fileLatestRecord = osRecordMapper.selectOne( OSRecord fileLatestRecord = osRecordMapper.selectOne(
new QueryWrapper<OSRecord>() new QueryWrapper<OSRecord>()
.eq("file_name", relativeFilePath) .eq("local_rela_file_path", localRelaFilePath)
.orderByDesc("upload_time")); .orderByDesc("upload_time"));
return Optional.ofNullable(fileLatestRecord); return Optional.ofNullable(fileLatestRecord);
} }