lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

文档导入ES逻辑优化

Browse Source
This commit is contained in:
luke 2025-06-19 12:36:07 +08:00
parent f91462c6c1
commit 24c12d8a34
21 changed files with 327 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Readme.md
View File

@ -8,6 +8,7 @@
| 1.0.2 | 2025-06-09 | Luke.Ye | 添加用户角色相关逻辑，重构代码 |
| 1.0.3 | 2025-06-16 | Luke.Ye | 新增文档上传至对象存储接口 |
| 1.0.4 | 2025-06-17 | Luke.Ye | 上传文档记录入库 |
| 1.0.5 | 2025-06-19 | Luke.Ye | 文档导入ES逻辑优化 |

2
src/main/java/com/knowledge/base/application/service/UserAppService.java
View File

@ -29,6 +29,8 @@ public interface UserAppService {
boolean isValidToken(String token);
Optional<UserTokenDTO> findToken(String token);
Optional<UserDTO> findUserByToken(String token);
// 用户角色
List<UserRoleDTO> listUserRoles(Long userId);
boolean addUserRole(Long userId, Long roleId);

27
src/main/java/com/knowledge/base/application/service/UserAppServiceImpl.java
View File

@ -1,11 +1,14 @@
package com.knowledge.base.application.service;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.knowledge.base.application.exceptions.AppException;
import com.knowledge.base.domain.user.model.RoleDO;
import com.knowledge.base.domain.user.model.UserDO;
import com.knowledge.base.domain.user.model.UserRoleDO;
import com.knowledge.base.domain.user.model.UserTokenDO;
import com.knowledge.base.domain.user.service.iface.UserDomainService;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.converter.*;
import com.knowledge.base.infrastructure.north.dto.role.RoleDTO;
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
@ -14,6 +17,8 @@ import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import java.util.List;
@ -23,10 +28,13 @@ import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@Slf4j
public class UserAppServiceImpl implements UserAppService {
private final UserDomainService userDomainService;
private final UserCacheService userCacheService;
@Override
public boolean registerUser(String username, String password, List<Long> roleIds) {
boolean ok = userDomainService.registerUser(username, password);
@ -119,6 +127,25 @@ public class UserAppServiceImpl implements UserAppService {
return Optional.ofNullable(UserTokenDtoConverter.toDTO(userDomainService.findToken(token).get()));
}
@Override
public Optional<UserDTO> findUserByToken(String token) {
Optional<String> cachedUserJson = userCacheService.getUserJsonByToken(token);
if(cachedUserJson.isPresent()) {
return Optional.ofNullable(JSONUtil.toBean(cachedUserJson.get(), UserDTO.class));
}
Optional<UserTokenDO> userTokenOpt = userDomainService.findToken(token);
if (userTokenOpt.isEmpty()) {
log.warn("[findUserByToken] token无效: {}", token);
return Optional.empty();
}
Optional<UserDO> userDO = userDomainService.findById(userTokenOpt.get().getUserId());
UserDTO dto = UserDtoConverter.toDTO(userDO.get());
// 数据进缓存
userCacheService.cacheUserJsonByToken(token, JSONUtil.toJsonStr(dto));
return Optional.ofNullable(dto);
}
@Override
public List<UserRoleDTO> listUserRoles(Long userId) {
return Optional.ofNullable(userDomainService.findRolesByUserId(userId)).orElse(Lists.newArrayList()).stream()

9
src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java
View File

@ -3,6 +3,8 @@ package com.knowledge.base.domain.doc.repository.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import java.util.Optional;
public interface OSRecordRepository {
/**
@ -14,4 +16,11 @@ public interface OSRecordRepository {
* 保存上传记录
*/
void save(OSRecord po);
/**
* 获取文档最新版本
* @param
* @return
*/
Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath);
}

8
src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java
View File

@ -3,7 +3,6 @@ package com.knowledge.base.domain.doc.service.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import java.util.List;
import java.util.Optional;
@ -25,4 +24,11 @@ public interface FileDomainService {
* 保存上传记录
*/
void save(OSRecordDO osRecord);
/**
* 获取文档最新版本
* @param
* @return
*/
Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath);
}

45
src/main/java/com/knowledge/base/domain/doc/service/impl/AbstractBaseFileImporter.java
View File

@ -4,8 +4,10 @@ import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONUtil;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
import com.knowledge.base.infrastructure.util.CacheUtil;
@ -21,15 +23,12 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.io.IOException;
import java.nio.file.*;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.*;
public abstract class AbstractBaseFileImporter implements DocumentImporter {
@ -42,6 +41,9 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
@Autowired
private FileCacheService fileCacheService;
@Autowired
private FileDomainService fileDomainService;;
protected abstract String getDirectoryPath();
protected abstract String getFileSuffix();
@ -109,7 +111,7 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动，跳过导入: {}", relativePath);
String metaJsonStr = transToMetaJson(existingSource);
fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(relativePath, metaJsonStr, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return;
}
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId);
@ -127,12 +129,14 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
if(metaJsonOpt.isPresent()) {
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
doc.put("uploader", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, ConstantConfig.DEFAULT_UPLOADER));
doc.put("url", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, ""));
String cacheFileUrl = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, StrUtil.EMPTY);
doc.put("url", buildAccessUrl(cacheFileUrl, relativePath));
doc.put("expireTime", CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code,
Long.class, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)));
} else {
doc.put("uploader",ConstantConfig.DEFAULT_UPLOADER);
doc.put("url", String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath) );
String url = buildAccessUrl(StrUtil.EMPTY, relativePath);
doc.put("url", url);
doc.put("expireTime", DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME));
}
@ -144,12 +148,37 @@ public abstract class AbstractBaseFileImporter implements DocumentImporter {
logger.info("导入成功: {}", relativePath);
// 信息入缓存
fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(relativePath, transToMetaJson(doc), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
} catch (Exception e) {
logger.error("导入失败: {}", path, e);
}
}
private String buildAccessUrl(String originUrl, String relativePath) {
// 优先使用原始链接
if(StrUtil.isNotBlank(originUrl)) {
return originUrl;
}
if(StrUtil.isBlank(relativePath)) {
return StrUtil.EMPTY;
}
String accessUrl = originUrl;
if(relativePath.endsWith(".md")) {
// markdown直接拼接http链接
accessUrl = String.format("%s/%s", ConstantConfig.SHARE_BASE_URL, relativePath);
accessUrl = accessUrl.substring(0, accessUrl.length() - 3) + ".html";
} else {
// 其它文件从对象存储的DB中获取
Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByFileName(relativePath);
if(latestRecordOpt.isPresent()) {
OSRecordDO latestRecord = latestRecordOpt.get();
accessUrl = latestRecord.getUrl();
}
}
return accessUrl;
}
private String transToMetaJson(Map<String, Object> esExistingSource) {
return JSONUtil.toJsonStr(Map.of(
DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get("uploader")).orElse(ConstantConfig.DEFAULT_UPLOADER),

13
src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java
View File

@ -1,5 +1,6 @@
package com.knowledge.base.domain.doc.service.impl;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.iface.FileRepository;
@ -64,4 +65,16 @@ public class FileDomainServiceImpl implements FileDomainService {
public void save(OSRecordDO osRecordDO) {
osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class));
}
@Override
public Optional<OSRecordDO> getLatestRecordByFileName(String relativeFilePath) {
if(StrUtil.isBlank(relativeFilePath)) {
return Optional.empty();
}
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByFileName(relativeFilePath);
if(osRecordOpt.isEmpty()) {
return Optional.empty();
}
return Optional.of(BeanConvertUtil.convert(osRecordOpt.get(), OSRecordDO.class));
}
}

76
src/main/java/com/knowledge/base/infrastructure/cache/FileCacheService.java vendored
View File

@ -1,76 +0,0 @@
package com.knowledge.base.infrastructure.cache;
import java.util.Optional;
/**
* 文件缓存服务接口（支持 Redis 或本地实现）
*
* @author Luke.ye
* @date 2025/6/7
*/
public interface FileCacheService {
/**
* 获取文件名对应的相对路径
*
* @param filename 文件名
* @return 相对路径（可能为 null）
*/
Optional<String> getPath(String filename);
/**
* 缓存文件名与相对路径的映射
*
* @param filename 文件名
* @param relativePath 相对路径
* @param expireMinutes 过期时间（分钟）
*/
void cachePath(String filename, String relativePath, long expireMinutes);
/**
* 缓存文件名与文件相关的元信息
* @param filename 文件名，带后缀
* @param jsonMeta 文件元信息，包含：上传人、可访问链接、可访问链接过期时间、文件写入本地时间等
* @param expireMinutes 过期时间（分钟）
*/
default void cacheMeta(String filename, String jsonMeta, long expireMinutes) {};
/**
* 获取文件名对应的元信息
* @param filename
* @return
*/
default Optional<String> getMeta(String filename) { return null; };
/**
* 获取指定路径的上次修改时间（用于变更检测）
*
* @param relativePath 文件相对路径
* @return 本地上次修改时间（可能为 null）
*/
Optional<Long> getMTime(String relativePath);
/**
* 缓存指定路径的修改时间
*
* @param relativePath 文件路径
* @param mtime 修改时间
* @param expireMinutes 过期时间（分钟）
*/
void cacheMTime(String relativePath, Long mtime, long expireMinutes);
/**
* 清除文件名对应的路径缓存
*/
void clearPath(String filename);
/**
* 清除路径对应的修改时间缓存
*/
void clearMTime(String relativePath);
/**
* 清空所有缓存（注意：某些实现可能未实现）
*/
void clearAll();
}

79
src/main/java/com/knowledge/base/infrastructure/cache/LocalFileCacheService.java vendored
View File

@ -1,79 +0,0 @@
package com.knowledge.base.infrastructure.cache;
import com.knowledge.base.infrastructure.util.LocalCacheUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Optional;
/**
* 使用本地缓存实现的文件缓存服务（非 Redis）
*
* @author Luke.ye
* @date 2025/6/7
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "false")
public class LocalFileCacheService implements FileCacheService {
@Value("${knowledge.base.local-cache.expire-minutes:1440}")
private long expireMinutes;
private String key(String prefix, String key) {
return String.format("kb:file:%s:%s", prefix, key);
}
@Override
public Optional<String> getPath(String filename) {
Object val = LocalCacheUtil.get(key("path", filename));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void cachePath(String filename, String relativePath, long expireMinutes) {
LocalCacheUtil.put(key("path", filename), relativePath);
}
@Override
public Optional<Long> getMTime(String relativePath) {
Object val = LocalCacheUtil.get(key("mtime", relativePath));
if (val instanceof Long) return Optional.of((Long) val);
if (val instanceof Integer) return Optional.of(((Integer) val).longValue());
return Optional.empty();
}
@Override
public void cacheMTime(String relativePath, Long mtime, long expireMinutes) {
LocalCacheUtil.put(key("mtime", relativePath), mtime);
}
@Override
public void cacheMeta(String filename, String jsonMeta, long expireMinutes) {
LocalCacheUtil.put(key("meta", filename), jsonMeta);
}
@Override
public Optional<String> getMeta(String filename) {
Object val = LocalCacheUtil.get(key("meta", filename));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void clearPath(String filename) {
LocalCacheUtil.remove(key("path", filename));
}
@Override
public void clearMTime(String relativePath) {
LocalCacheUtil.remove(key("mtime", relativePath));
}
@Override
public void clearAll() {
LocalCacheUtil.clearAll();
}
}

40
src/main/java/com/knowledge/base/infrastructure/cache/iface/FileCacheService.java vendored Normal file
View File

@ -0,0 +1,40 @@
package com.knowledge.base.infrastructure.cache.iface;
import java.util.Optional;
/**
* 文件缓存服务接口（支持 Redis 或本地实现）
*
* @author Luke.ye
* @date 2025/6/7
*/
public interface FileCacheService {
/**
* 获取文件名对应的相对路径
*
* @param filename 文件名
* @return 相对路径（可能为 null）
*/
Optional<String> getPath(String filename);
/**
* 缓存文件名与文件相关的元信息
* @param fnWithRelativePath 文件名，带相对路径和后缀
* @param jsonMeta 文件元信息，包含：上传人、可访问链接、可访问链接过期时间、文件写入本地时间等
* @param expireMinutes 过期时间（分钟）
*/
default void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {};
/**
* 获取文件名对应的元信息
* @param fnWithRelativePath
* @return
*/
default Optional<String> getMeta(String fnWithRelativePath) { return null; };
/**
* 清空所有缓存（注意：某些实现可能未实现）
*/
void clearAll();
}

25
src/main/java/com/knowledge/base/infrastructure/cache/iface/UserCacheService.java vendored Normal file
View File

@ -0,0 +1,25 @@
package com.knowledge.base.infrastructure.cache.iface;
import java.util.Optional;
/**
* @author Luke.ye
* @date 2025/6/19 09:59
*/
public interface UserCacheService {
/**
* 根据token获取User信息
*
* @param token
* @return UserDTO的Json
*/
Optional<String> getUserJsonByToken(String token);
/**
* 将用户信息缓存
* @param token
* @param userJsonStr
*/
void cacheUserJsonByToken(String token, String userJsonStr);
}

51
src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalFileCacheServiceImpl.java vendored Normal file
View File

@ -0,0 +1,51 @@
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.util.LocalCacheUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Optional;
/**
* 使用本地缓存实现的文件缓存服务（非 Redis）
*
* @author Luke.ye
* @date 2025/6/7
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "false")
public class LocalFileCacheServiceImpl implements FileCacheService {
@Value("${knowledge.base.local-cache.expire-minutes:1440}")
private long expireMinutes;
private String key(String prefix, String key) {
return String.format("kb:file:%s:%s", prefix, key);
}
@Override
public Optional<String> getPath(String filename) {
Object val = LocalCacheUtil.get(key("path", filename));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta);
}
@Override
public Optional<String> getMeta(String fnWithRelativePath) {
Object val = LocalCacheUtil.get(key("meta", fnWithRelativePath));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void clearAll() {
LocalCacheUtil.clearAll();
}
}

27
src/main/java/com/knowledge/base/infrastructure/cache/impl/LocalUserCacheServiceImpl.java vendored Normal file
View File

@ -0,0 +1,27 @@
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Optional;
/**
* @author Luke.ye
* @date 2025/6/19 10:01
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "false")
public class LocalUserCacheServiceImpl implements UserCacheService {
@Override
public Optional<String> getUserJsonByToken(String token) {
return Optional.empty();
}
@Override
public void cacheUserJsonByToken(String token, String userJsonStr) {
}
}

44
src/main/java/com/knowledge/base/infrastructure/cache/RedisFileCacheService.java → src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisFileCacheServiceImpl.java vendored
View File

@ -1,5 +1,6 @@
package com.knowledge.base.infrastructure.cache;
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@ -21,7 +22,7 @@ import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "true", matchIfMissing = true)
public class RedisFileCacheService implements FileCacheService {
public class RedisFileCacheServiceImpl implements FileCacheService {
private final StringRedisTemplate redisTemplate;
@ -38,47 +39,16 @@ public class RedisFileCacheService implements FileCacheService {
}
@Override
public void cachePath(String filename, String relativePath, long expireMinutes) {
redisTemplate.opsForValue().set(key("path", filename), relativePath, expireMinutes, TimeUnit.MINUTES);
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
redisTemplate.opsForValue().set(key("meta", fnWithRelativePath), jsonMeta, expireMinutes, TimeUnit.MINUTES);
}
@Override
public void cacheMeta(String filename, String jsonMeta, long expireMinutes) {
redisTemplate.opsForValue().set(key("meta", filename), jsonMeta, expireMinutes, TimeUnit.MINUTES);
}
@Override
public Optional<String> getMeta(String filename) {
String value = redisTemplate.opsForValue().get(key("meta", filename));
public Optional<String> getMeta(String fnWithRelativePath) {
String value = redisTemplate.opsForValue().get(key("meta", fnWithRelativePath));
return Optional.ofNullable(value);
}
@Override
public Optional<Long> getMTime(String relativePath) {
String val = redisTemplate.opsForValue().get(key("mtime", relativePath));
try {
return val != null ? Optional.of(Long.parseLong(val)) : Optional.empty();
} catch (NumberFormatException e) {
log.warn("mtime 解析失败: {}", val, e);
return Optional.empty();
}
}
@Override
public void cacheMTime(String relativePath, Long mtime, long expireMinutes) {
redisTemplate.opsForValue().set(key("mtime", relativePath), String.valueOf(mtime), expireMinutes, TimeUnit.MINUTES);
}
@Override
public void clearPath(String filename) {
redisTemplate.delete(key("path", filename));
}
@Override
public void clearMTime(String relativePath) {
redisTemplate.delete(key("mtime", relativePath));
}
@Override
public void clearAll() {
log.warn("正在清空 Redis 文件缓存，前缀: {}", FILE_CACHE_PREFIX);

42
src/main/java/com/knowledge/base/infrastructure/cache/impl/RedisUserCacheServiceImpl.java vendored Normal file
View File

@ -0,0 +1,42 @@
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
* @author Luke.ye
* @date 2025/6/19 10:01
*/
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "true")
public class RedisUserCacheServiceImpl implements UserCacheService {
private final StringRedisTemplate redisTemplate;
private static final String USER_CACHE_PREFIX = "kb-user";
private String key(String prefix, String key) {
return String.format(USER_CACHE_PREFIX + ":%s:%s", prefix, key);
}
@Override
public Optional<String> getUserJsonByToken(String token) {
String value = redisTemplate.opsForValue().get(key("token", token));
return Optional.ofNullable(value);
}
@Override
public void cacheUserJsonByToken(String token, String userJsonStr) {
redisTemplate.opsForValue().set(key("token", token), userJsonStr, ConstantConfig.USER_CACHE_EXPIRED_MINUTES, TimeUnit.MINUTES);
}
}

4
src/main/java/com/knowledge/base/infrastructure/config/ConstantConfig.java
View File

@ -19,7 +19,9 @@ public class ConstantConfig {
public static final LocalDateTime LONG_TERM_EXPIRE_TIME = LocalDateTime.of(9999, 12, 31, 23, 59, 59);
public static final long CACHE_EXPIRED_MINUTES = Duration.ofDays(2).toMinutes();
public static final long FILE_META_CACHE_EXPIRED_MINUTES = Duration.ofDays(2).toMinutes();
public static final long USER_CACHE_EXPIRED_MINUTES = Duration.ofDays(10).toMinutes();
public static final String SHARE_BASE_URL = "http://share.wisdompulse.cn/public";

21
src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java
View File

@ -6,14 +6,16 @@ import cn.hutool.json.JSONUtil;
import com.google.common.collect.Maps;
import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.north.dto.SearchReq;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import com.knowledge.base.infrastructure.util.CacheUtil;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
@ -35,7 +37,6 @@ import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.*;
import java.util.regex.Pattern;
@ -89,11 +90,11 @@ public class FileQueryController {
LOGGER.error("SearchController#resolveFilePath Error. Invalid Input: " + JSONUtil.toJsonStr(req));
return Maps.newHashMap();
}
List<String> fileNames = (List)req.get("fileNames");
List<String> namesWithRelativePath = (List)req.get("fileNames");
Map<String, Object> filePaths = Maps.newHashMap();
fileNames.forEach(name -> {
if(fileCacheService.getPath(name).isPresent()) {
filePaths.put(name, fileCacheService.getPath(name));
namesWithRelativePath.forEach(pathFile -> {
if(fileCacheService.getMeta(pathFile).isPresent()) {
filePaths.put(pathFile, CacheUtil.loadFileMetaProp(pathFile, DocMetaPropEnum.ACCESS_URL.code, String.class));
}
});
Map<String, Object> result = Maps.newHashMap();
@ -121,7 +122,6 @@ public class FileQueryController {
PageResult<OSRecordDTO> result = docAppService.pageQueryOSRecord(page, size, userDTO.get().getUsername());
List<Map<String, Object>> data = new ArrayList<>();
LocalDate today = LocalDate.now();
for (OSRecordDTO dto : result.getList()) {
Map<String, Object> item = new LinkedHashMap<>();
@ -183,9 +183,14 @@ public class FileQueryController {
}
outerQuery.minimumShouldMatch(1);
// 东八区时间点
outerQuery.filter(QueryBuilders.rangeQuery("expireTime")
.gte("now")
.timeZone("+08:00"));
return outerQuery;
}
private Map<String, Object> buildSearchResponse(SearchResponse response, List<String> keywords, int page, int size) {
List<Map<String, Object>> results = new ArrayList<>();
@ -195,6 +200,8 @@ public class FileQueryController {
result.put("filename", source.get("filename"));
result.put("filepath", source.get("filepath"));
result.put("mtime", source.get("mtime"));
result.put("uploader", source.getOrDefault("uploader", ConstantConfig.DEFAULT_UPLOADER));
result.put("url", source.get("url"));
String content = (String) source.get("content");
String summary = extractMultiSnippet(content, keywords, 50);

48
src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java
View File

@ -8,7 +8,7 @@ import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.ObjectStorageProperties;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
@ -35,7 +35,6 @@ import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.*;
@ -84,42 +83,6 @@ public class FileWriteController {
return ResponseEntity.badRequest().body("不支持的类型: " + type);
}
@DeleteMapping("/file/{fileName}")
public ResponseEntity<?> deleteByFileName(@PathVariable String fileName) {
boolean esResult = false, cacheResult = false;
try {
// 1. 从缓存获取相对路径
Optional<String> relPathOpt = fileCacheService.getPath(fileName);
if (relPathOpt.isPresent()) {
String relativePath = relPathOpt.get();
String docId = SafeIdUtil.encode(relativePath);
// 2. 删除 ES
DeleteRequest deleteRequest = new DeleteRequest(INDEX_NAME, docId);
// 强制刷新可选：deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
esClient.delete(deleteRequest, RequestOptions.DEFAULT);
esResult = true;
// 3. 删除缓存（文件名和mtime都清除）
fileCacheService.clearPath(fileName);
fileCacheService.clearMTime(relativePath);
cacheResult = true;
logger.info("已删除文件 [{}] 的ES和缓存记录", fileName);
} else {
logger.warn("未找到缓存记录: {}", fileName);
}
return ResponseEntity.ok(Map.of(
"esDeleted", esResult,
"cacheDeleted", cacheResult
));
} catch (Exception e) {
logger.error("删除失败: {}", fileName, e);
return ResponseEntity.status(500).body("删除失败: " + e.getMessage());
}
}
@DeleteMapping("/files/clear-all")
public ResponseEntity<?> clearAllFiles() {
try {
@ -217,6 +180,9 @@ public class FileWriteController {
String uploader = userDTO.get().getUsername();
LocalDateTime now = LocalDateTime.now();
String dateFolder = now.toLocalDate().toString();
String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
boolean isPublic = Optional.ofNullable(objectStorageProperties.getPublicBuckets())
.map(buckets -> buckets.contains(bucket))
.orElse(false);
@ -225,7 +191,7 @@ public class FileWriteController {
: now.plusDays(7);
OSRecordDTO dto = new OSRecordDTO();
dto.setFileName(originFileNameWithSuffix);
dto.setFileName(relativeFilePath);
dto.setUrl(url);
dto.setUploader(uploader);
dto.setBucketName(bucket);
@ -238,7 +204,6 @@ public class FileWriteController {
// 可检索文件的类型
Set<String> supportedTypes = Set.of("pdf", "doc", "docx", "xls", "xlsx");
if (searchable && supportedTypes.contains(suffix)) {
String dateFolder = now.toLocalDate().toString();
String localDir = Paths.get(objectStorageProperties.getLocalSearchablePath(), dateFolder).toString();
File localTargetDir = new File(localDir);
if (!localTargetDir.exists()) {
@ -256,9 +221,8 @@ public class FileWriteController {
cacheMeta.put(DocMetaPropEnum.UPLOAD_TIME.code, mtime);
cacheMeta.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(expireTime));
String relativeFilePath = String.format("%s/%s", dateFolder, originFileNameWithSuffix);
String metaJson = JSONUtil.toJsonStr(cacheMeta);
fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.CACHE_EXPIRED_MINUTES);
fileCacheService.cacheMeta(relativeFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
logger.info("本地文件信息已保存并写入缓存: key = {}, value = {}", relativeFilePath, metaJson);
}

3
src/main/java/com/knowledge/base/infrastructure/north/controller/UserQueryController.java
View File

@ -3,6 +3,7 @@ package com.knowledge.base.infrastructure.north.controller;
import cn.hutool.core.util.StrUtil;
import com.google.common.collect.Lists;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
import com.knowledge.base.infrastructure.north.dto.role.UserRoleDTO;
@ -29,6 +30,8 @@ public class UserQueryController {
private final UserAppService userAppService;
private final FileCacheService fileCacheService;
@GetMapping("/check")
public ResponseEntity<?> check(
@CookieValue(value = ConstantConfig.COOKIE_KEY, required = false) String cookieToken,

12
src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java
View File

@ -2,6 +2,7 @@ package com.knowledge.base.infrastructure.repository.persistence.doc;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.repository.iface.OSRecordRepository;
@ -10,6 +11,8 @@ import com.knowledge.base.infrastructure.repository.mapper.doc.OSRecordMapper;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
@RequiredArgsConstructor
public class OSRecordRepositoryImpl implements OSRecordRepository {
@ -34,4 +37,13 @@ public class OSRecordRepositoryImpl implements OSRecordRepository {
public void save(OSRecord po) {
osRecordMapper.insert(po);
}
@Override
public Optional<OSRecord> getLatestRecordByFileName(String relativeFilePath) {
OSRecord fileLatestRecord = osRecordMapper.selectOne(
new QueryWrapper<OSRecord>()
.eq("file_name", relativeFilePath)
.orderByDesc("upload_time"));
return Optional.ofNullable(fileLatestRecord);
}
}

2
src/main/java/com/knowledge/base/infrastructure/util/CacheUtil.java
View File

@ -4,7 +4,7 @@ import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.extra.spring.SpringUtil;
import cn.hutool.json.JSONUtil;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.ObjectUtils;