lukeye/doc-parser-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成对象存储上传、查询逻辑

Browse Source
This commit is contained in:
luke 2025-06-17 00:49:53 +08:00
parent fa5fa75b0d
commit deab62880d
17 changed files with 441 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config/mysql/init.txt
View File

@ -92,6 +92,7 @@ CREATE TABLE IF NOT EXISTS oss_upload_record (
upload_time DATETIME NOT NULL COMMENT '上传时间',
object_path VARCHAR(512) NOT NULL COMMENT 'MinIO 存储路径',
url VARCHAR(1024) COMMENT '访问链接（如CDN代理后的地址）',
expire_time DATETIME NOT NULL DEFAULT '9999-12-31 23:59:59' COMMENT '链接过期时间（默认永久）',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_uploader(uploader),

13
src/main/java/com/knowledge/base/application/service/DocAppService.java
View File

@ -1,8 +1,21 @@
package com.knowledge.base.application.service;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
/**
* @author Luke.ye
* @date 2025/6/16 19:13
*/
public interface DocAppService {
/**
* 分页查询上传记录（按上传人）
*/
PageResult<OSRecordDTO> pageQueryOSRecord(int page, int size, String uploader);
/**
* 保存上传记录
*/
void save(OSRecordDTO dto);
}

46
src/main/java/com/knowledge/base/application/service/DocAppServiceImpl.java Normal file
View File

@ -0,0 +1,46 @@
package com.knowledge.base.application.service;
import com.google.common.collect.Lists;
import com.knowledge.base.application.exceptions.AppException;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import com.knowledge.base.infrastructure.converter.FileDtoConverter;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@RequiredArgsConstructor
public class DocAppServiceImpl implements DocAppService{
private final FileDomainService fileDomainService;
@Override
public PageResult<OSRecordDTO> pageQueryOSRecord(int page, int size, String uploader) {
if(page < 1 || size < 1) {
throw new AppException("分页参数错误！");
}
PageResult<OSRecordDO> osRecordDOPageResult = null;
final List<OSRecordDTO> dtoList = Lists.newArrayList();
try {
osRecordDOPageResult = fileDomainService.pageQueryOSRecord(page, size, uploader);
osRecordDOPageResult.getList().forEach(osRecordDO ->
dtoList.add(FileDtoConverter.toDTO(osRecordDO))
);
} catch (Exception e) {
throw new AppException("分页获取对象存储记录失败！", e);
}
return new PageResult<>(dtoList, osRecordDOPageResult.getTotal());
}
@Override
public void save(OSRecordDTO dto) {
try {
fileDomainService.save(FileDtoConverter.toDO(dto));
} catch (Exception e) {
throw new AppException("保存对象存储记录失败！", e);
}
}
}

15
src/main/java/com/knowledge/base/domain/common/model/PageResult.java Normal file
View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.common.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class PageResult<T> {
private List<T> list;
private long total;
}

18
src/main/java/com/knowledge/base/domain/doc/model/OSRecordDO.java Normal file
View File

@ -0,0 +1,18 @@
package com.knowledge.base.domain.doc.model;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class OSRecordDO {
private Long id;
private String fileName;
private String bucketName;
private String uploader;
private String objectPath;
private String url;
private LocalDateTime uploadTime;
private LocalDateTime expireTime;
private LocalDateTime addTime;
}

17
src/main/java/com/knowledge/base/domain/doc/repository/iface/OSRecordRepository.java Normal file
View File

@ -0,0 +1,17 @@
package com.knowledge.base.domain.doc.repository.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
public interface OSRecordRepository {
/**
* 分页查询上传记录（按上传人）
*/
PageResult<OSRecord> pageQuery(int page, int size, String uploader);
/**
* 保存上传记录
*/
void save(OSRecord po);
}

28
src/main/java/com/knowledge/base/domain/doc/repository/po/OSRecord.java Normal file
View File

@ -0,0 +1,28 @@
package com.knowledge.base.domain.doc.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("oss_upload_record")
public class OSRecord {
@TableId(type = IdType.AUTO)
private Long id;
private String fileName;
private String bucketName;
private String uploader;
private String objectPath;
private String url;
private LocalDateTime uploadTime;
private LocalDateTime expireTime;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

13
src/main/java/com/knowledge/base/domain/doc/service/iface/FileDomainService.java
View File

@ -1,6 +1,9 @@
package com.knowledge.base.domain.doc.service.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import java.util.List;
import java.util.Optional;
@ -12,4 +15,14 @@ public interface FileDomainService {
Optional<File> findById(Long id);
List<File> findAll();
List<File> findByFileNameLike(String pattern);
/**
* 分页查询上传记录（按上传人）
*/
PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader);
/**
* 保存上传记录
*/
void save(OSRecordDO osRecord);
}

20
src/main/java/com/knowledge/base/domain/doc/service/impl/FileDomainServiceImpl.java
View File

@ -1,9 +1,14 @@
package com.knowledge.base.domain.doc.service.impl;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.iface.FileRepository;
import com.knowledge.base.domain.doc.repository.iface.OSRecordRepository;
import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import lombok.RequiredArgsConstructor;
import com.knowledge.base.infrastructure.util.BeanConvertUtil;
import org.springframework.stereotype.Service;
import java.util.List;
@ -15,6 +20,8 @@ public class FileDomainServiceImpl implements FileDomainService {
private final FileRepository fileRepository;
private final OSRecordRepository osRecordRepository;
@Override
public boolean save(File file) {
return fileRepository.save(file);
@ -44,4 +51,17 @@ public class FileDomainServiceImpl implements FileDomainService {
public List<File> findByFileNameLike(String pattern) {
return fileRepository.findByFileNameLike(pattern);
}
@Override
public PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader) {
PageResult<OSRecord> osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader);
return new PageResult<>(BeanConvertUtil.convertList(osRecordPageResult.getList(), OSRecordDO.class),
osRecordPageResult.getTotal());
}
@Override
public void save(OSRecordDO osRecordDO) {
osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class));
}
}

39
src/main/java/com/knowledge/base/infrastructure/converter/FileDtoConverter.java
View File

@ -1,7 +1,9 @@
package com.knowledge.base.infrastructure.converter;
import com.knowledge.base.domain.doc.model.FileDO;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.infrastructure.north.dto.doc.FileDTO;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import org.springframework.beans.BeanUtils;
import java.time.LocalDateTime;
@ -27,4 +29,41 @@ public class FileDtoConverter {
dto.updateTime = doObj.updateTime == null ? null : dtf.format(doObj.updateTime);
return dto;
}
public static OSRecordDO toDO(OSRecordDTO dto) {
if (dto == null){
return null;
}
OSRecordDO doObj = new OSRecordDO();
BeanUtils.copyProperties(dto, doObj);
if (dto.getAddTime() != null) {
doObj.setAddTime(LocalDateTime.parse(dto.getAddTime()));
}
if (dto.getUploadTime() != null) {
doObj.setUploadTime(LocalDateTime.parse(dto.getUploadTime()));
}
if (dto.getExpireTime() != null) {
doObj.setExpireTime(LocalDateTime.parse(dto.getExpireTime()));
}
return doObj;
}
public static OSRecordDTO toDTO(OSRecordDO doObj) {
if (doObj == null){
return null;
}
OSRecordDTO dto = new OSRecordDTO();
BeanUtils.copyProperties(doObj, dto);
if (doObj.getAddTime() != null) {
dto.setAddTime(doObj.getAddTime().toString());
}
if (doObj.getUploadTime() != null) {
dto.setUploadTime(doObj.getUploadTime().toString());
}
if (doObj.getExpireTime() != null) {
dto.setExpireTime(doObj.getExpireTime().toString());
}
return dto;
}
}

59
src/main/java/com/knowledge/base/infrastructure/north/controller/FileQueryController.java
View File

@ -1,12 +1,18 @@
package com.knowledge.base.infrastructure.north.controller;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Maps;
import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.infrastructure.north.dto.SearchReq;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
@ -27,6 +33,9 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.*;
import java.util.regex.Pattern;
@ -43,6 +52,9 @@ public class FileQueryController {
@Autowired
private FileCacheService fileCacheService;
@Autowired
private DocAppService docAppService;
@Value("${search.default-page-size:10}")
private int defaultPageSize;
@ -70,7 +82,7 @@ public class FileQueryController {
return ResponseEntity.ok(userAppService.listUserFiles(userId));
}
@PostMapping("file-path")
@PostMapping("/file-path")
public Map<String, Object> resolveFilePath(@RequestBody Map<String, Object> req) {
if (MapUtil.isEmpty(req) || Objects.isNull(req.get("fileNames")) || !(req.get("fileNames") instanceof List)) {
LOGGER.error("SearchController#resolveFilePath Error. Invalid Input: " + JSONUtil.toJsonStr(req));
@ -88,6 +100,51 @@ public class FileQueryController {
return result;
}
@GetMapping("/fetch-os-upload-history")
public ResponseEntity<?> listUploadedFiles(
@RequestHeader("Authorization") String token,
@RequestParam(defaultValue = "1") int page,
@RequestParam(defaultValue = "10") int size
) {
Optional<UserTokenDTO> userTokenDTO = userAppService.findToken(token);
if (userTokenDTO.isEmpty()) {
return ResponseEntity.status(401).body("未登录或token无效");
}
Optional<UserDTO> userDTO = userAppService.findById(userTokenDTO.get().getUserId());
if(userDTO.isEmpty() || StrUtil.isBlank(userDTO.get().getUsername())) {
return ResponseEntity.badRequest().body(Map.of("msg", "无法找到token对应的用户"));
}
PageResult<OSRecordDTO> result = docAppService.pageQueryOSRecord(page, size, userDTO.get().getUsername());
List<Map<String, Object>> data = new ArrayList<>();
LocalDate today = LocalDate.now();
for (OSRecordDTO dto : result.getList()) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("fileName", dto.getFileName());
item.put("url", dto.getUrl());
if (dto.getExpireTime() != null) {
long days = Duration.between(
today.atStartOfDay(),
LocalDateTime.parse(dto.getExpireTime()).toLocalDate().atStartOfDay()
).toDays();
item.put("daysRemaining", Math.max(days, 0));
} else {
item.put("daysRemaining", -1);
}
data.add(item);
}
Map<String, Object> response = new LinkedHashMap<>();
response.put("total", result.getTotal());
response.put("currentCount", result.getList().size());
response.put("lastPage", (page * size) >= result.getTotal());
response.put("records", data);
return ResponseEntity.ok(response);
}
private QueryBuilder buildSearchQuery(List<List<String>> keywordGroups) {
BoolQueryBuilder outerQuery = QueryBuilders.boolQuery();

72
src/main/java/com/knowledge/base/infrastructure/north/controller/FileWriteController.java
View File

@ -1,12 +1,18 @@
package com.knowledge.base.infrastructure.north.controller;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.application.service.DocAppService;
import com.knowledge.base.application.service.UserAppService;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.infrastructure.cache.FileCacheService;
import com.knowledge.base.infrastructure.config.ObjectStorageProperties;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import com.knowledge.base.infrastructure.south.ObjectStorageGateway;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import lombok.RequiredArgsConstructor;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.client.RequestOptions;
@ -20,10 +26,8 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.time.LocalDateTime;
import java.util.*;
@RestController
@RequestMapping("/api/v1/doc")
@ -37,6 +41,7 @@ public class FileWriteController {
private static final String INDEX_NAME = "documents";
private final UserAppService userAppService;
private final DocAppService docAppService;
private final ObjectStorageGateway objectStorageGateway;
private final ObjectStorageProperties objectStorageProperties;
@ -131,25 +136,23 @@ public class FileWriteController {
@PostMapping("/upload-files")
public ResponseEntity<?> uploadFolderToOS(
@RequestHeader("Authorization") String token,
@RequestParam("files") MultipartFile[] files,
@RequestParam("bucket") String bucket
) {
try {
objectStorageGateway.ensureBucketExists(bucket);
Map<String, String> filePathMap = new LinkedHashMap<>();
for (MultipartFile file : files) {
String originalName = file.getOriginalFilename();
if (originalName == null || originalName.isBlank()) {
originalName = file.getName();
}
// 获取原始文件名
String originalName = Optional.ofNullable(file.getOriginalFilename())
.filter(s -> !s.isBlank())
.orElse(file.getName());
// 路径前缀
String folderPrefix = "default/";
// 构造安全路径
String safeName = originalName.replace("\\", "/");
// 是否启用 UUID 前缀
String folderPrefix = "default/";
if (objectStorageProperties.isUseUuidPrefix()) {
String uuid = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
safeName = folderPrefix + uuid + "_" + Paths.get(safeName).getFileName().toString();
@ -157,13 +160,52 @@ public class FileWriteController {
safeName = folderPrefix + Paths.get(safeName).getFileName().toString();
}
// 上传到 MinIO
// 上传文件到 MinIO
objectStorageGateway.uploadFile(bucket, safeName, file);
// 使用原始文件名作为 key，MinIO 对象路径作为 value
// 获取访问 URL
String fileKey = Paths.get(originalName).getFileName().toString();
String url = objectStorageGateway.generateUrl(bucket, safeName);
filePathMap.put(fileKey, url);
// 异步写入 OSS 上传记录
String finalSafeName = safeName;
ThreadPoolUtil.execute(() -> {
try {
Optional<UserTokenDTO> userTokenDTO = userAppService.findToken(token);
if (userTokenDTO.isEmpty()) {
logger.warn("异步写入失败：无效 token");
return;
}
Optional<UserDTO> userDTO = userAppService.findById(userTokenDTO.get().getUserId());
if (userDTO.isEmpty() || StrUtil.isBlank(userDTO.get().getUsername())) {
logger.warn("异步写入失败：无法获取用户名");
return;
}
LocalDateTime now = LocalDateTime.now();
boolean isPublic = objectStorageProperties.getPublicBuckets() != null &&
objectStorageProperties.getPublicBuckets().contains(bucket);
LocalDateTime expireTime = isPublic
? LocalDateTime.of(9999, 12, 31, 23, 59, 59)
: now.plusDays(7);
OSRecordDTO dto = new OSRecordDTO();
dto.setFileName(fileKey);
dto.setUrl(url);
dto.setUploader(userDTO.get().getUsername());
dto.setBucketName(bucket);
dto.setObjectPath(finalSafeName);
dto.setAddTime(now.toString());
dto.setUploadTime(now.toString());
dto.setExpireTime(expireTime.toString());
docAppService.save(dto);
} catch (Exception e) {
logger.error("异步写入 OSRecord 失败", e);
}
});
}
return ResponseEntity.ok(Map.of(

16
src/main/java/com/knowledge/base/infrastructure/north/dto/doc/OSRecordDTO.java Normal file
View File

@ -0,0 +1,16 @@
package com.knowledge.base.infrastructure.north.dto.doc;
import lombok.Data;
@Data
public class OSRecordDTO {
private Long id;
private String fileName;
private String bucketName;
private String uploader;
private String objectPath;
private String url;
private String uploadTime;
private String expireTime;
private String addTime;
}

19
src/main/java/com/knowledge/base/infrastructure/repository/config/MybatisPlusConfig.java Normal file
View File

@ -0,0 +1,19 @@
package com.knowledge.base.infrastructure.repository.config;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MybatisPlusConfig {
@Bean
public MybatisPlusInterceptor mybatisPlusInterceptor() {
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL)); // 指定数据库类型
return interceptor;
}
}

9
src/main/java/com/knowledge/base/infrastructure/repository/mapper/doc/OSRecordMapper.java Normal file
View File

@ -0,0 +1,9 @@
package com.knowledge.base.infrastructure.repository.mapper.doc;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface OSRecordMapper extends BaseMapper<OSRecord> {
}

37
src/main/java/com/knowledge/base/infrastructure/repository/persistence/doc/OSRecordRepositoryImpl.java Normal file
View File

@ -0,0 +1,37 @@
package com.knowledge.base.infrastructure.repository.persistence.doc;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.repository.iface.OSRecordRepository;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import com.knowledge.base.infrastructure.repository.mapper.doc.OSRecordMapper;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Repository;
@Repository
@RequiredArgsConstructor
public class OSRecordRepositoryImpl implements OSRecordRepository {
private final OSRecordMapper osRecordMapper;
@Override
public PageResult<OSRecord> pageQuery(int page, int size, String uploader) {
LambdaQueryWrapper<OSRecord> wrapper = new LambdaQueryWrapper<>();
if (StrUtil.isNotBlank(uploader)) {
wrapper.eq(OSRecord::getUploader, uploader);
}
wrapper.orderByDesc(OSRecord::getUploadTime);
Page<OSRecord> mpPage = new Page<>(page, size);
Page<OSRecord> resultPage = osRecordMapper.selectPage(mpPage, wrapper);
return new PageResult<>(resultPage.getRecords(), resultPage.getTotal());
}
@Override
public void save(OSRecord po) {
osRecordMapper.insert(po);
}
}

35
src/main/java/com/knowledge/base/infrastructure/util/BeanConvertUtil.java Normal file
View File

@ -0,0 +1,35 @@
package com.knowledge.base.infrastructure.util;
import cn.hutool.core.collection.CollectionUtil;
import org.springframework.beans.BeanUtils;
import java.util.List;
import java.util.stream.Collectors;
public class BeanConvertUtil {
/**
* 将 source 转换为指定类型的目标对象
*/
public static <S, T> T convert(S source, Class<T> targetClass) {
if (source == null) return null;
try {
T target = targetClass.getDeclaredConstructor().newInstance();
BeanUtils.copyProperties(source, target);
return target;
} catch (Exception e) {
throw new RuntimeException("BeanConvertUtil.convert error", e);
}
}
/**
* 批量转换：List<S> → List<T>
*/
public static <S, T> List<T> convertList(List<S> sourceList, Class<T> targetClass) {
if (sourceList == null || sourceList.isEmpty()) return CollectionUtil.newArrayList();
return sourceList.stream()
.map(source -> convert(source, targetClass))
.collect(Collectors.toList());
}
}