diff --git a/config/mysql/init.txt b/config/mysql/init.txt index 4aa0c58..5627fda 100644 --- a/config/mysql/init.txt +++ b/config/mysql/init.txt @@ -87,6 +87,7 @@ CREATE TABLE IF NOT EXISTS user_file ( CREATE TABLE IF NOT EXISTS oss_upload_record ( id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键', file_name VARCHAR(255) NOT NULL COMMENT '文件名', + local_rela_file_path VARCHAR(512) NOT NULL DEFAULT 'UNKNOWN' COMMENT '本地文件相对路径', bucket_name VARCHAR(128) NOT NULL COMMENT '目标 bucket 名', uploader VARCHAR(128) NOT NULL COMMENT '上传人用户名', upload_time DATETIME NOT NULL COMMENT '上传时间', @@ -97,8 +98,10 @@ CREATE TABLE IF NOT EXISTS oss_upload_record ( update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', INDEX idx_uploader(uploader), INDEX idx_file_name(file_name), - INDEX idx_bucket(bucket_name) + INDEX idx_bucket(bucket_name), + INDEX idx_local_rela_file_path(local_rela_file_path(100)) ) COMMENT='对象存储上传记录表'; + diff --git a/config/redis/cmd.txt b/config/redis/cmd.txt index e075582..4429b8f 100644 --- a/config/redis/cmd.txt +++ b/config/redis/cmd.txt @@ -1,3 +1,4 @@ +redis-cli --raw # 获取redis中的key-value redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done \ No newline at end of file diff --git a/src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java b/src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java index bcd98d1..805cf28 100644 --- a/src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java +++ b/src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java @@ -8,6 +8,7 @@ import java.time.LocalDateTime; public class OSRecordDO { private Long id; private String fileName; + private String localRelaFilePath; private String bucketName; private String uploader; private String objectPath; diff --git a/src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java b/src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java index 3afc892..a676d26 100644 --- a/src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java +++ b/src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java @@ -22,5 +22,5 @@ public interface OSRecordRepository { * @param * @return */ - Optional getLatestRecordByFileName(String relativeFilePath); + Optional getLatestRecordByRelaPath(String localRelaFilePath); } diff --git a/src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java b/src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java index 4332d39..c50bcf1 100644 --- a/src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java +++ b/src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java @@ -13,6 +13,7 @@ public class OSRecord { private Long id; private String fileName; + private String localRelaFilePath; private String bucketName; private String uploader; private String objectPath; diff --git a/src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java b/src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java index 3d11f24..2cf2777 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java @@ -30,5 +30,5 @@ public interface FileDomainService { * @param * @return */ - Optional getLatestRecordByFileName(String relativeFilePath); + Optional getLatestRecordByRelaPath(String localRelaFilePath); } diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java index 0c5e9d5..0231a4d 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java @@ -83,15 +83,15 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { } Long localMTime = Files.getLastModifiedTime(path).toMillis(); - String relativePath = relativePathObj.toString().replace("\\", "/"); + String localRelaFilePath = relativePathObj.toString().replace("\\", "/"); String fileNameWithSuffix = path.getFileName().toString(); - Optional metaJsonOpt = fileCacheService.getMeta(relativePath); + Optional metaJsonOpt = fileCacheService.getMeta(localRelaFilePath); if(metaJsonOpt.isPresent()) { // 有文件信息 JSON metaJson = JSONUtil.parse(metaJsonOpt.get()); Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class); if(localMTime.equals(uploadTime)) { - logger.info("文件未变动，跳过导入: {}", relativePath); + logger.info("文件未变动，跳过导入: {}", localRelaFilePath); return; } } @@ -102,40 +102,40 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { return; } - String docId = SafeIdUtil.encode(relativePath); + String docId = SafeIdUtil.encode(localRelaFilePath); GetRequest getRequest = new GetRequest(INDEX_NAME, docId); if (esClient.exists(getRequest, RequestOptions.DEFAULT)) { GetResponse existing = esClient.get(getRequest, RequestOptions.DEFAULT); Map existingSource = existing.getSourceAsMap(); Object esMtime = existingSource.get("mtime"); if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) { - logger.info("文件未变动，跳过导入: {}", relativePath); + logger.info("文件未变动，跳过导入: {}", localRelaFilePath); String metaJsonStr = transToMetaJson(existingSource); - fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); + fileCacheService.cacheMeta(localRelaFilePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); return; } DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId); esClient.delete(deleteRequest, RequestOptions.DEFAULT); - logger.info("已删除旧版本文件: {}", relativePath); + logger.info("已删除旧版本文件: {}", localRelaFilePath); } - logger.info("[开始进行文件导入......] relativePath: {}， localMTime： {}", relativePath, localMTime); + logger.info("[开始进行文件导入......] localRelaFilePath: {}， localMTime： {}", localRelaFilePath, localMTime); Map doc = new HashMap<>(); doc.put("filename", fileNameWithSuffix); - doc.put("filepath", relativePath); + doc.put("filepath", localRelaFilePath); doc.put("content", content); doc.put("mtime", localMTime); if(metaJsonOpt.isPresent()) { Map metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class); doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER)); String cacheFileUrl = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY); - doc.put("url", buildAccessUrl(cacheFileUrl, relativePath)); + doc.put("url", buildAccessUrl(cacheFileUrl, localRelaFilePath, fileNameWithSuffix)); doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME))); } else { doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER); - String url = buildAccessUrl(StrUtil.EMPTY, relativePath); + String url = buildAccessUrl(StrUtil.EMPTY, localRelaFilePath, fileNameWithSuffix); doc.put("url", url); doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)); } @@ -145,32 +145,32 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter { .source(doc); esClient.index(request, RequestOptions.DEFAULT); - logger.info("导入成功: {}", relativePath); + logger.info("导入成功: {}", localRelaFilePath); // 信息入缓存 - fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); + fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); } catch (Exception e) { logger.error("导入失败: {}", path, e); } } - private String buildAccessUrl(String originUrl, String relativePath) { + private String buildAccessUrl(String originUrl, String localRelaFilePath, String fileNameWithSuffix) { // 优先使用原始链接 if(StrUtil.isNotBlank(originUrl)) { return originUrl; } - if(StrUtil.isBlank(relativePath)) { + if(StrUtil.isBlank(localRelaFilePath)) { return StrUtil.EMPTY; } String accessUrl = originUrl; - if(relativePath.endsWith(".md")) { + if(localRelaFilePath.endsWith(".md")) { // markdown直接拼接http链接 - accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath); + accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, localRelaFilePath); accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html"; } else { // 其它文件从对象存储的DB中获取 - Optional latestRecordOpt = fileDomainService.getLatestRecordByFileName(relativePath); + Optional latestRecordOpt = fileDomainService.getLatestRecordByRelaPath(localRelaFilePath); if(latestRecordOpt.isPresent()) { OSRecordDO latestRecord = latestRecordOpt.get(); accessUrl = latestRecord.getUrl(); diff --git a/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java b/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java index 60645d8..d06ea58 100644 --- a/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java +++ b/src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java @@ -67,11 +67,11 @@ public class FileDomainServiceImpl implements FileDomainService { } @Override - public Optional getLatestRecordByFileName(String relativeFilePath) { - if(StrUtil.isBlank(relativeFilePath)) { + public Optional getLatestRecordByRelaPath(String localRelaFilePath) { + if(StrUtil.isBlank(localRelaFilePath)) { return Optional.empty(); } - Optional osRecordOpt = osRecordRepository.getLatestRecordByFileName(relativeFilePath); + Optional osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath); if(osRecordOpt.isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java b/src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java index 25c3d7b..338401c 100644 --- a/src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java +++ b/src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java @@ -10,14 +10,6 @@ import java.util.Optional; */ public interface FileCacheService { - /** - * 获取文件名对应的相对路径 - * - * @param filename 文件名 - * @return 相对路径（可能为 null） - */ - Optional getPath(String filename); - /** * 缓存文件名与文件相关的元信息 * @param fnWithRelativePath 文件名，带相对路径和后缀 diff --git a/src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java b/src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java index 1a47198..d9cf576 100644 --- a/src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java +++ b/src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java @@ -27,12 +27,6 @@ public class LocalFileCacheServiceImpl implements FileCacheService { return String.format("kb:file:%s:%s", prefix, key); } - @Override - public Optional getPath(String filename) { - Object val = LocalCacheUtil.get(key("path", filename)); - return val instanceof String ? Optional.of((String) val) : Optional.empty(); - } - @Override public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) { LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta); diff --git a/src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java b/src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java index f1f29c8..70a6a09 100644 --- a/src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java +++ b/src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java @@ -32,12 +32,6 @@ public class RedisFileCacheServiceImpl implements FileCacheService { return String.format(FILE_CACHE_PREFIX + ":%s:%s", prefix, key); } - @Override - public Optional getPath(String filename) { - String value = redisTemplate.opsForValue().get(key("path", filename)); - return Optional.ofNullable(value); - } - @Override public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) { redisTemplate.opsForValue().set(key("meta", fnWithRelativePath), jsonMeta, expireMinutes, TimeUnit.MINUTES); diff --git a/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java b/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java index 239dce6..d12d6d2 100644 --- a/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java +++ b/src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java @@ -26,4 +26,6 @@ public class ConstantConfig { public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public"; public static final String DEFAULT_UPLOADER = "ADMIN"; + + public static final String UNKNOWN_LOCAL_RELA_PATH = "UNKNOWN"; } diff --git a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java index 3b26be3..201f40b 100644 --- a/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java +++ b/src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java @@ -16,10 +16,8 @@ import com.knowledge.base.infrastructure.north.dto.user.UserDTO; import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO; import com.knowledge.base.infrastructure.south.ObjectStorageGateway; import com.knowledge.base.infrastructure.util.DateUtil; -import com.knowledge.base.infrastructure.util.SafeIdUtil; import com.knowledge.base.infrastructure.util.ThreadPoolUtil; import lombok.RequiredArgsConstructor; -import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.QueryBuilders; @@ -181,7 +179,9 @@ public class FileWriteController { String uploader = userDTO.get().getUsername(); LocalDateTime now = LocalDateTime.now(); String dateFolder = now.toLocalDate().toString(); - String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix); + Set supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx"); + boolean allowSaveToLocal = searchable && supportedTypes.contains(suffix); + String localRelaFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix); boolean isPublic = Optional.ofNullable(objectStorageProperties.getPublicBuckets()) .map(buckets -> buckets.contains(bucket)) @@ -191,7 +191,8 @@ public class FileWriteController { : now.plusDays(7); OSRecordDTO dto = new OSRecordDTO(); - dto.setFileName(relativeFilePath); + dto.setFileName(originFileNameWithSuffix); + dto.setLocalRelaFilePath(allowSaveToLocal ? localRelaFilePath : ConstantConfig.UNKNOWN_LOCAL_RELA_PATH); dto.setUrl(url); dto.setUploader(uploader); dto.setBucketName(bucket); @@ -201,9 +202,8 @@ public class FileWriteController { dto.setExpireTime(expireTime.toString()); docAppService.saveOSUplodRecord(dto); - // 可检索文件的类型 - Set supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx"); - if (searchable && supportedTypes.contains(suffix)) { + // 允许保存到本地 + if (allowSaveToLocal) { String localDir = Paths.get(objectStorageProperties.getLocalSearchablePath(), dateFolder).toString(); File localTargetDir = new File(localDir); if (!localTargetDir.exists()) { @@ -222,9 +222,9 @@ public class FileWriteController { cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime)); String metaJson = JSONUtil.toJsonStr(cacheMeta); - fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); + fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES); - logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson); + logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", localRelaFilePath, metaJson); } } catch (Exception e) { logger.error("异步写入失败: {}", originFileNameWithSuffix, e); diff --git a/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java b/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java index 2b217d7..9fee96a 100644 --- a/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java +++ b/src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java @@ -6,6 +6,12 @@ import lombok.Data; public class OSRecordDTO { private Long id; private String fileName; + + /** + * 本地相对路径文件名（不为空，则文件可搜索） + */ + private String localRelaFilePath; + private String bucketName; private String uploader; private String objectPath; diff --git a/src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java b/src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java index 60c1212..4d37aa4 100644 --- a/src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java +++ b/src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java @@ -39,10 +39,10 @@ public class OSRecordRepositoryImpl implements OSRecordRepository { } @Override - public Optional getLatestRecordByFileName(String relativeFilePath) { + public Optional getLatestRecordByRelaPath(String localRelaFilePath) { OSRecord fileLatestRecord = osRecordMapper.selectOne( new QueryWrapper() - .eq("file_name", relativeFilePath) + .eq("local_rela_file_path", localRelaFilePath) .orderByDesc("upload_time")); return Optional.ofNullable(fileLatestRecord); }