lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化文件导入逻辑

Browse Source
This commit is contained in:
luke 2025-06-19 18:21:07 +08:00
parent 74084d2faf
commit 8567f6bd62
15 changed files with 49 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/mysql/init.txt
View File

@ -87,6 +87,7 @@ CREATE TABLE IF NOT EXISTS user_file (
CREATE TABLE IF NOT EXISTS oss_upload_record (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
file_name VARCHAR(255) NOT NULL COMMENT '文件名',
local_rela_file_path VARCHAR(512) NOT NULL DEFAULT 'UNKNOWN' COMMENT '本地文件相对路径',
bucket_name VARCHAR(128) NOT NULL COMMENT '目标 bucket 名',
uploader VARCHAR(128) NOT NULL COMMENT '上传人用户名',
upload_time DATETIME NOT NULL COMMENT '上传时间',
@ -97,8 +98,10 @@ CREATE TABLE IF NOT EXISTS oss_upload_record (
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_uploader(uploader),
INDEX idx_file_name(file_name),
INDEX idx_bucket(bucket_name)
INDEX idx_bucket(bucket_name),
INDEX idx_local_rela_file_path(local_rela_file_path(100))
) COMMENT='对象存储上传记录表';

1
config/redis/cmd.txt
View File

@ -1,3 +1,4 @@
redis-cli --raw
# 获取redis中的key-value
redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done

1
src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java
View File

@ -8,6 +8,7 @@ import java.time.LocalDateTime;
public class OSRecordDO {
private Long id;
private String fileName;
private String localRelaFilePath;
private String bucketName;
private String uploader;
private String objectPath;

2
src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java
View File

@ -22,5 +22,5 @@ public interface OSRecordRepository {
* @param
* @return
*/
Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath);
Optional<OSRecord> getLatestRecordByRelaPath(String localRelaFilePath);
}

1
src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java
View File

@ -13,6 +13,7 @@ public class OSRecord {
private Long id;
private String fileName;
private String localRelaFilePath;
private String bucketName;
private String uploader;
private String objectPath;

2
src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java
View File

@ -30,5 +30,5 @@ public interface FileDomainService {
* @param
* @return
*/
Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath);
Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath);
}

36
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java
View File

@ -83,15 +83,15 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
}
Long localMTime = Files.getLastModifiedTime(path).toMillis();
String relativePath = relativePathObj.toString().replace("\\", "/");
String localRelaFilePath = relativePathObj.toString().replace("\\", "/");
String fileNameWithSuffix = path.getFileName().toString();
Optional<String> metaJsonOpt = fileCacheService.getMeta(relativePath);
Optional<String> metaJsonOpt = fileCacheService.getMeta(localRelaFilePath);
if(metaJsonOpt.isPresent()) {
// 有文件信息
JSON metaJson = JSONUtil.parse(metaJsonOpt.get());
Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class);
if(localMTime.equals(uploadTime)) {
logger.info("文件未变动，跳过导入: {}", relativePath);
logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
return;
}
}
@ -102,40 +102,40 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
return;
}
String docId = SafeIdUtil.encode(relativePath);
String docId = SafeIdUtil.encode(localRelaFilePath);
GetRequest getRequest = new GetRequest(INDEX_NAME, docId);
if (esClient.exists(getRequest, RequestOptions.DEFAULT)) {
GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT);
Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get("mtime");
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", relativePath);
logger.info("文件未变动，跳过导入: {}", localRelaFilePath);
String metaJsonStr = transToMetaJson(existingSource);
fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(localRelaFilePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return;
}
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId);
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
logger.info("已删除旧版本文件: {}", relativePath);
logger.info("已删除旧版本文件: {}", localRelaFilePath);
}
logger.info("[开始进行文件导入......] relativePath: {}， localMTime： {}", relativePath, localMTime);
logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime);
Map<String, Object> doc = new HashMap<>();
doc.put("filename", fileNameWithSuffix);
doc.put("filepath", relativePath);
doc.put("filepath", localRelaFilePath);
doc.put("content", content);
doc.put("mtime", localMTime);
if(metaJsonOpt.isPresent()) {
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER));
String cacheFileUrl = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY);
doc.put("url", buildAccessUrl(cacheFileUrl, relativePath));
doc.put("url", buildAccessUrl(cacheFileUrl, localRelaFilePath, fileNameWithSuffix));
doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code,
Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)));
} else {
doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER);
String url = buildAccessUrl(StrUtil.EMPTY, relativePath);
String url = buildAccessUrl(StrUtil.EMPTY, localRelaFilePath, fileNameWithSuffix);
doc.put("url", url);
doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME));
}
@ -145,32 +145,32 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
.source(doc);
esClient.index(request, RequestOptions.DEFAULT);
logger.info("导入成功: {}", relativePath);
logger.info("导入成功: {}", localRelaFilePath);
// 信息入缓存
fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
} catch (Exception e) {
logger.error("导入失败: {}", path, e);
}
}
private String buildAccessUrl(String originUrl, String relativePath) {
private String buildAccessUrl(String originUrl, String localRelaFilePath, String fileNameWithSuffix) {
// 优先使用原始链接
if(StrUtil.isNotBlank(originUrl)) {
return originUrl;
}
if(StrUtil.isBlank(relativePath)) {
if(StrUtil.isBlank(localRelaFilePath)) {
return StrUtil.EMPTY;
}
String accessUrl = originUrl;
if(relativePath.endsWith(".md")) {
if(localRelaFilePath.endsWith(".md")) {
// markdown直接拼接http链接
accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath);
accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, localRelaFilePath);
accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html";
} else {
// 其它文件从对象存储的DB中获取
Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByFileName(relativePath);
Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByRelaPath(localRelaFilePath);
if(latestRecordOpt.isPresent()) {
OSRecordDO latestRecord = latestRecordOpt.get();
accessUrl = latestRecord.getUrl();

6
src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java
View File

@ -67,11 +67,11 @@ public class FileDomainServiceImpl implements FileDomainService {
}
@Override
public Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath) {
if(StrUtil.isBlank(relativeFilePath)) {
public Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath) {
if(StrUtil.isBlank(localRelaFilePath)) {
return Optional.empty();
}
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByFileName(relativeFilePath);
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
if(osRecordOpt.isEmpty()) {
return Optional.empty();
}

8
src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java vendored
View File

@ -10,14 +10,6 @@ import java.util.Optional;
*/
public interface FileCacheService {
/**
* 获取文件名对应的相对路径
*
* @param filename 文件名
* @return 相对路径（可能为 null）
*/
Optional<String> getPath(String filename);
/**
* 缓存文件名与文件相关的元信息
* @param fnWithRelativePath 文件名，带相对路径和后缀

6
src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java vendored
View File

@ -27,12 +27,6 @@ public class LocalFileCacheServiceImpl implements FileCacheService {
return String.format("kb:file:%s:%s", prefix, key);
}
@Override
public Optional<String> getPath(String filename) {
Object val = LocalCacheUtil.get(key("path", filename));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta);

6
src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java vendored
View File

@ -32,12 +32,6 @@ public class RedisFileCacheServiceImpl implements FileCacheService {
return String.format(FILE_CACHE_PREFIX + ":%s:%s", prefix, key);
}
@Override
public Optional<String> getPath(String filename) {
String value = redisTemplate.opsForValue().get(key("path", filename));
return Optional.ofNullable(value);
}
@Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
redisTemplate.opsForValue().set(key("meta", fnWithRelativePath), jsonMeta, expireMinutes, TimeUnit.MINUTES);

2
src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java
View File

@ -26,4 +26,6 @@ public class ConstantConfig {
public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public";
public static final String DEFAULT_UPLOADER = "ADMIN";
public static final String UNKNOWN_LOCAL_RELA_PATH = "UNKNOWN";
}

18
src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java
View File

@ -16,10 +16,8 @@ import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import com.knowledge.base.infrastructure.south.ObjectStorageGateway;
import com.knowledge.base.infrastructure.util.DateUtil;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
@ -181,7 +179,9 @@ public class FileWriteController {
String uploader = userDTO.get().getUsername();
LocalDateTime now = LocalDateTime.now();
String dateFolder = now.toLocalDate().toString();
String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
Set<String> supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx");
boolean allowSaveToLocal = searchable && supportedTypes.contains(suffix);
String localRelaFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
boolean isPublic = Optional.ofNullable(objectStorageProperties.getPublicBuckets())
.map(buckets -> buckets.contains(bucket))
@ -191,7 +191,8 @@ public class FileWriteController {
: now.plusDays(7);
OSRecordDTO dto = new OSRecordDTO();
dto.setFileName(relativeFilePath);
dto.setFileName(originFileNameWithSuffix);
dto.setLocalRelaFilePath(allowSaveToLocal ? localRelaFilePath : ConstantConfig.UNKNOWN_LOCAL_RELA_PATH);
dto.setUrl(url);
dto.setUploader(uploader);
dto.setBucketName(bucket);
@ -201,9 +202,8 @@ public class FileWriteController {
dto.setExpireTime(expireTime.toString());
docAppService.saveOSUplodRecord(dto);
// 可检索文件的类型
Set<String> supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx");
if (searchable && supportedTypes.contains(suffix)) {
// 允许保存到本地
if (allowSaveToLocal) {
String localDir = Paths.get(objectStorageProperties.getLocalSearchablePath(), dateFolder).toString();
File localTargetDir = new File(localDir);
if (!localTargetDir.exists()) {
@ -222,9 +222,9 @@ public class FileWriteController {
cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime));
String metaJson = JSONUtil.toJsonStr(cacheMeta);
fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson);
logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", localRelaFilePath, metaJson);
}
} catch (Exception e) {
logger.error("异步写入失败: {}", originFileNameWithSuffix, e);

6
src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java
View File

@ -6,6 +6,12 @@ import lombok.Data;
public class OSRecordDTO {
private Long id;
private String fileName;
/**
* 本地相对路径文件名（不为空，则文件可搜索）
*/
private String localRelaFilePath;
private String bucketName;
private String uploader;
private String objectPath;

4
src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java
View File

@ -39,10 +39,10 @@ public class OSRecordRepositoryImpl implements OSRecordRepository {
}
@Override
public Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath) {
public Optional<OSRecord> getLatestRecordByRelaPath(String localRelaFilePath) {
OSRecord fileLatestRecord = osRecordMapper.selectOne(
new QueryWrapper<OSRecord>()
.eq("file_name", relativeFilePath)
.eq("local_rela_file_path", localRelaFilePath)
.orderByDesc("upload_time"));
return Optional.ofNullable(fileLatestRecord);
}