Compare commits

...

119 Commits

Author SHA1 Message Date
7614196f5e Merge branch 'dev_luke' of http://git.wisdompulse.cn/lukeye/doc-parser-server into dev_luke 2025-07-04 14:19:32 +08:00
d33e32e770 add config 2025-07-04 14:19:05 +08:00
2cd8b180d6 fix 2025-07-03 23:32:02 +08:00
daa7767ecb add log 2025-07-03 21:36:31 +08:00
fd99c7692a opt 2025-06-28 13:27:31 +08:00
a85610078b 添加markdown文件导入逻辑 2025-06-27 17:06:39 +08:00
8717ad49b7 提供删除ES数据的接口 2025-06-27 00:56:33 +08:00
1231f0c826 fix 2025-06-24 18:16:07 +08:00
1236799852 fix 2025-06-24 17:13:46 +08:00
f4dc2800a7 opt 2025-06-24 17:08:43 +08:00
56e988435f fix 2025-06-23 19:29:40 +08:00
58a78d33c1 fix 2025-06-23 19:26:53 +08:00
eb3a858681 fix 2025-06-23 19:20:32 +08:00
87198e42e4 fix 2025-06-23 18:34:22 +08:00
09b2db0df3 fix 2025-06-23 18:27:28 +08:00
69a71dfa43 fix 2025-06-23 18:19:08 +08:00
bf2363b7ce 优化返回逻辑 2025-06-23 18:00:49 +08:00
e6cf18fa45 添加llm的debug日志 2025-06-23 17:05:04 +08:00
2cd64981a1 fix SSE格式 2025-06-23 16:28:06 +08:00
0901ff78a5 fix 2025-06-23 16:22:03 +08:00
73ebf1fb66 opt 2025-06-23 16:16:53 +08:00
45537ab339 opt 2025-06-23 16:12:24 +08:00
a222e64d24 fix 2025-06-22 14:50:07 +08:00
fe00238532 opt 2025-06-22 14:40:28 +08:00
647ef43655 完成llm接口迁移至后端 2025-06-22 08:32:51 +08:00
ff8a2c5089 fix 2025-06-22 00:23:39 +08:00
33df9bb0f8 支持对ppt和txt的导入支持 2025-06-21 23:51:43 +08:00
09d2aac37e opt 2025-06-21 22:36:21 +08:00
d90217c0ee fix 2025-06-21 22:31:18 +08:00
b82f44543e fix 2025-06-21 22:02:55 +08:00
9670477f51 兼容doc/xls格式的文件 2025-06-21 10:00:35 +08:00
9ec83eec49 fix 2025-06-21 08:11:03 +08:00
9360afa83b fix 2025-06-21 01:25:27 +08:00
06c3882190 fix 2025-06-21 01:16:52 +08:00
343472f5cb opt 2025-06-21 01:06:10 +08:00
31da1625f5 fix 2025-06-21 01:00:54 +08:00
1380115e72 fix 2025-06-21 00:54:00 +08:00
0062ebdbac opt,上传s3逻辑优化 2025-06-21 00:48:57 +08:00
e174c085f0 opt 2025-06-21 00:30:19 +08:00
d949e034d7 opt. docker中的es直接访问本地 2025-06-20 23:54:01 +08:00
5928f23176 fix 2025-06-20 17:32:52 +08:00
f2bd42005a 新增可搜索状态字段 2025-06-20 17:13:18 +08:00
4f439361b7 opt 2025-06-20 16:18:58 +08:00
19eec52e52 opt 2025-06-20 14:41:41 +08:00
25f0ef9cf2 opt 2025-06-20 12:52:41 +08:00
276773131c add recordId 2025-06-20 11:17:22 +08:00
c1376a2e68 完成对象存储删除接口,优化部分代码 2025-06-20 11:15:05 +08:00
59d1a9de3a opt 2025-06-20 08:44:10 +08:00
e98486c4f8 fix 2025-06-20 01:18:46 +08:00
b0ea0f5ffa fix 2025-06-20 01:05:21 +08:00
f1d54c107e opt 2025-06-19 18:27:14 +08:00
8567f6bd62 优化文件导入逻辑 2025-06-19 18:24:01 +08:00
74084d2faf fix 2025-06-19 12:50:28 +08:00
24c12d8a34 文档导入ES逻辑优化 2025-06-19 12:36:07 +08:00
f91462c6c1 åâ€优化导入逻辑 2025-06-19 01:19:11 +08:00
916e42bdf7 上传文件至S3的同时,保存到本地 2025-06-18 18:41:24 +08:00
c1460acb64 fix 2025-06-18 10:39:55 +08:00
bf4037e86f add updateRoleFileRule iface 2025-06-18 09:23:59 +08:00
aa4e995410 fix 2025-06-18 00:47:41 +08:00
6c22b72094 fix 2025-06-17 10:07:34 +08:00
eba7cc53bb fix 2025-06-17 08:22:09 +08:00
48ad6da53a fix 2025-06-17 08:16:09 +08:00
618652784e fix 2025-06-17 08:08:50 +08:00
deab62880d 完成对象存储上传、查询逻辑 2025-06-17 00:49:53 +08:00
fa5fa75b0d fix 2025-06-16 19:14:20 +08:00
d87266b3e2 新增对象存储上传接口 2025-06-16 01:38:44 +08:00
6440a1c97e fix 2025-06-12 18:28:19 +08:00
58c75ce332 新增修改代码逻辑 2025-06-12 18:12:36 +08:00
089da5da2a fix 2025-06-12 17:43:27 +08:00
78f4c659ad 添加接口 2025-06-12 16:38:27 +08:00
6aaab95126 fix 2025-06-12 00:06:46 +08:00
ac5c66199b 优化接口 2025-06-11 23:41:18 +08:00
57ebe17bb2 fix 2025-06-11 15:35:01 +08:00
a89cca5909 opt 2025-06-11 15:32:17 +08:00
08bfb9cfff fix 2025-06-11 14:32:19 +08:00
9064f18000 add list all users 2025-06-10 23:55:57 +08:00
ff8b6251e1 fix 2025-06-10 00:33:01 +08:00
e8e5deda8d 添加异常处理 2025-06-09 23:17:31 +08:00
f70867e2c1 fix 2025-06-09 19:55:09 +08:00
c491ab9cc2 添加权限校验 2025-06-09 19:33:04 +08:00
7886ec3267 fix 2025-06-09 14:03:55 +08:00
a8c8682920 fix 2025-06-09 13:42:39 +08:00
d638a36b0f 优化controller 2025-06-09 12:50:12 +08:00
5deadeb99e fix 2025-06-09 00:47:28 +08:00
7e089c66bd 添加用户æ©角色相关逻辑 2025-06-09 00:40:26 +08:00
cb0883bcda fix 2025-06-07 17:17:56 +08:00
a406696cfd fix 2025-06-07 14:38:04 +08:00
b24f0f4793 fix 2025-06-07 14:33:23 +08:00
9dda80b4dd 添加日志&清空ESå/redis的接口 2025-06-07 13:12:48 +08:00
119c25060a fix redis bug 2025-06-07 02:11:18 +08:00
915fcd61ab 集成redis 2025-06-07 01:53:50 +08:00
4be29b3d85 fix 2025-06-07 00:59:37 +08:00
6a135421f5 添å加安全相关的çš的代码 + fix 2025-06-07 00:27:12 +08:00
f91475281c fix lombok bug 2025-06-06 23:18:53 +08:00
3a6df99b65 集成mysql,重构用户逻辑 2025-06-06 22:17:14 +08:00
e132958d1b 修改项目名称 2025-06-06 10:44:22 +08:00
51eea73f98 修改项目名称 2025-06-06 10:14:22 +08:00
bbe5b6ccab 完成æ°设置cookie设置“ 2025-06-05 23:52:01 +08:00
c95565e1dd 将基于Authentification校验替换成基于Cookie校验 2025-06-05 15:57:05 +08:00
6b042eed63 添加token校验接口 2025-06-05 14:36:46 +08:00
8e28cdc692 fix 2025-06-03 14:48:19 +08:00
b6c86679fd ä¿重构获取文件路径接口 2025-06-03 14:30:33 +08:00
49d51c371d 中间件迁回内网 2025-06-03 14:28:54 +08:00
e19d6097e5 添加获取文件路径接口 2025-06-01 16:23:17 +08:00
e859dd2004 添加笨本地å缓存,优化导入性能 2025-05-31 22:52:07 +08:00
94c85afa15 fix 2025-05-25 15:38:38 +08:00
f423a9df10 fix docker path bug 2025-05-25 15:33:25 +08:00
fe74eb879d 调整es中存放的文件路径,确保各端导入的路径是一样的 2025-05-25 15:07:52 +08:00
1c8b29e249 添加登录&token校验逻辑 2025-05-25 10:13:16 +08:00
0db7d8370b change path 2025-05-24 16:22:12 +08:00
957d97cf98 解决跨域访问问题 2025-05-24 09:23:34 +08:00
62791e5553 add single query method 2025-05-21 22:54:40 +08:00
21a1179765 定时自动导入 2025-05-21 18:37:54 +08:00
78cabcb107 添加es语句 2025-05-21 18:01:36 +08:00
c448c24cd2 fix docker bug 2025-05-21 17:49:04 +08:00
35b4981ebf 完成导入及搜索优化 2025-05-21 16:05:55 +08:00
580f732ec1 优化导入逻辑“ 2025-05-21 01:21:13 +08:00
22426a30d9 新增pdf/doc/markdown解析 2025-05-20 18:21:28 +08:00
16a6eeab86 add es 2025-05-20 08:50:26 +08:00
193 changed files with 9496 additions and 215 deletions

2
.gitignore vendored
View File

@ -35,3 +35,5 @@ build/
### Mac OS ###
.DS_Store
/knowledge-base-server-log/
knowledge-base-server.iml

13
Dockerfile/Dockerfile Normal file
View File

@ -0,0 +1,13 @@
# 使用 JDK 11
FROM openjdk:11-jdk-slim
# 创建工作目录
WORKDIR /app
# 拷贝 JAR
COPY doc-parser-server-0.0.1-SNAPSHOT.jar /app/
EXPOSE 18080
# 启动命令:指定 spring.profiles.active=docker
CMD ["java", "-Djdk.attach.allowAttachSelf=true", "-Dspring.profiles.active=docker", "-jar", "doc-parser-server-0.0.1-SNAPSHOT.jar"]

View File

@ -0,0 +1,17 @@
version: '3'
services:
document-parser:
image: document-parser:latest
build:
context: ./docker-document-parser-java
container_name: document-parser
volumes:
- "D:/02-documents/01-ahnx-share-src-public/public:/app/import-data"
- "D:/00-projects/00-micro-sass/docker-document-parser-java/logs:/app/doc-parser-server-log"
environment:
- SPRING_PROFILES_ACTIVE=docker
ports:
- "18080:18080"
command: java -Dspring.profiles.active=docker -Djdk.attach.allowAttachSelf=true -jar doc-parser-server-0.0.1-SNAPSHOT.jar
restart: always

View File

@ -1,3 +1,21 @@
#TODO
修订记录
| 版本 | 时间 | 修订人 | 改动点 |
|:-----:|:----------:|:-------:|:---------------------------------:|
| 1.0.0 | 2025-06-05 | Luke.Ye | 添加修订记录新增token校验接口 |
| 1.0.1 | 2025-06-06 | Luke.Ye | 重构登录Token校验逻辑 |
| 1.0.2 | 2025-06-09 | Luke.Ye | 添加用户角色相关逻辑,重构代码 |
| 1.0.3 | 2025-06-16 | Luke.Ye | 新增文档上传至对象存储接口 |
| 1.0.4 | 2025-06-17 | Luke.Ye | 上传文档记录入库 |
| 1.0.5 | 2025-06-19 | Luke.Ye | 文档导入ES逻辑优化 |
| 1.0.6 | 2025-06-20 | Luke.Ye | 完成对象存储删除接口,优化部分代码 |
| 1.0.7 | 2025-06-22 | Luke.Ye | 导入文档类型新增支持ppt & txt |
| 1.1.0 | 2025-06-22 | Luke.Ye | 完成llm接口迁移至后端 |
| 1.1.1 | 2025-06-26 | Luke.Ye | 提供删除ES数据的接口ob仓库监听发生删除事件则同步删除ES |
## 开发顺序
PO ---> Mapper ---> iface ---> persistence

View File

@ -0,0 +1,63 @@
#{
# "keywordGroups": [
# ["Luke.Ye", "设计模式"],
# ]
#}
# 表示 (设计模式 AND Luke.Ye)
POST /documents/_search
{
"query": {
"bool": {
"should": [
{
"bool": {
"must": [
{ "multi_match": { "query": "设计模式", "fields": ["filename", "content"], "type": "phrase" }},
{ "multi_match": { "query": "Luke.Ye", "fields": ["filename", "content"], "type": "phrase" }}
]
}
}
],
"minimum_should_match": 1
}
},
"highlight": {
"pre_tags": ["<mark>"],
"post_tags": ["</mark>"],
"fields": {
"content": {
"fragment_size": 100,
"number_of_fragments": 3
},
"filename": {
"fragment_size": 100,
"number_of_fragments": 1
}
}
},
"sort": [
{ "mtime": "desc" }
],
"size": 10,
"from": 0
}
# 清空索引中的全部内容
POST /documents/_delete_by_query
{
"query": {
"match_all": {}
}
}
# 从索引中删除某个文件
POST /documents/_delete_by_query
{
"query": {
"match_phrase": {
"filename": "服务定位器模式"
}
}
}

46
config/es/DDL.http Normal file
View File

@ -0,0 +1,46 @@
#创建索引
PUT /documents
{
"settings": {
"analysis": {
"analyzer": {
"my_chinese_analyzer": {
"type": "custom",
"tokenizer": "ik_max_word"
}
}
}
},
"mappings": {
"properties": {
"filename": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"filepath": {
"type": "keyword"
},
"content": {
"type": "text",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"mtime": {
"type": "date"
},
"uploader": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"expireTime": {
"type": "date"
}
}
}
}

20
config/mysql/dml.txt Normal file
View File

@ -0,0 +1,20 @@
INSERT INTO role (role_code, role_name) VALUES ('ADMIN', '管理员');
INSERT INTO role (role_code, role_name) VALUES ('USER', '普通用户');
INSERT INTO role (role_code, role_name) VALUES ('GUEST', '访客');
-- 管理员全部可看
INSERT INTO role_file_rule (role_id, file_pattern, remark) VALUES
(1, '/', '管理员可访问所有文件');
-- 普通用户可访问自己目录和公共目录
INSERT INTO role_file_rule (role_id, file_pattern, remark) VALUES
(2, '/public/${username}-', '用户可访问以用户名开头的文件');
INSERT INTO role_file_rule (role_id, file_pattern, remark) VALUES
(2, '/public/2025年', '用户可访问公共文件');
-- 访客只能访问公共目录
INSERT INTO role_file_rule (role_id, file_pattern, remark) VALUES
(3, '/public/2025年', '访客可访问公共文件');
---- 为用户赋予权限 ----
INSERT INTO user_role (user_id, role_id) values ('1', '1');
INSERT INTO user_role (user_id, role_id) values ('2', '2');

108
config/mysql/init.txt Normal file
View File

@ -0,0 +1,108 @@
CREATE DATABASE IF NOT EXISTS kbase DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
USE kbase;
-- 1. 用户基础表
CREATE TABLE IF NOT EXISTS user (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '用户ID主键',
username VARCHAR(64) NOT NULL UNIQUE COMMENT '登录用户名,唯一',
password VARCHAR(128) NOT NULL COMMENT '用户密码(建议加密存储)',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间'
) COMMENT='用户基础表';
-- 2. 角色表
CREATE TABLE IF NOT EXISTS role (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '角色ID',
role_code VARCHAR(64) NOT NULL UNIQUE COMMENT '角色编码(如 admin/user/guest',
role_name VARCHAR(64) NOT NULL COMMENT '角色名称',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间'
) COMMENT='系统角色表';
-- 3. 用户-角色关联表
CREATE TABLE IF NOT EXISTS user_role (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
user_id BIGINT NOT NULL COMMENT '用户ID',
role_id BIGINT NOT NULL COMMENT '角色ID',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_user_id(user_id),
INDEX idx_role_id(role_id),
UNIQUE KEY uk_user_role (user_id, role_id)
) COMMENT='用户与角色关联表';
-- 4. 文件表
CREATE TABLE IF NOT EXISTS file (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '文件ID',
file_name VARCHAR(255) NOT NULL COMMENT '文件名或路径',
file_path VARCHAR(512) NOT NULL COMMENT '存储路径',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间'
) COMMENT='知识库文件表';
-- 5. 角色-文件可见规则表(适合批量规则分配权限)
CREATE TABLE IF NOT EXISTS role_file_rule (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
role_id BIGINT NOT NULL COMMENT '角色ID',
file_pattern VARCHAR(255) NOT NULL COMMENT '可访问文件名/路径模式(如前缀、正则)',
remark VARCHAR(255) COMMENT '规则说明',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_role_id(role_id),
UNIQUE KEY uk_role_pattern (role_id, file_pattern)
) COMMENT='角色-文件可访问规则表';
-- 6. 用户token表
CREATE TABLE IF NOT EXISTS user_token (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增',
user_id BIGINT NOT NULL COMMENT '用户ID业务逻辑关联user表',
token VARCHAR(128) NOT NULL UNIQUE COMMENT '用户登录token',
login_at DATETIME NOT NULL COMMENT '登录时间',
expired_at DATETIME NOT NULL COMMENT '过期时间',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_user_id(user_id),
INDEX idx_token(token)
) COMMENT='用户登录Token表';
-- 7. 用户-文件直接授权关联表
CREATE TABLE IF NOT EXISTS user_file (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
user_id BIGINT NOT NULL COMMENT '用户ID',
file_id BIGINT NOT NULL COMMENT '文件ID',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_user_id(user_id),
INDEX idx_file_id(file_id),
UNIQUE INDEX uniq_user_file(user_id, file_id)
) COMMENT='用户-文件直接授权关联表(用于特殊文件个别授权)';
-- 8. 对象存储上传记录表
CREATE TABLE IF NOT EXISTS oss_upload_record (
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
file_name VARCHAR(255) NOT NULL COMMENT '文件名',
local_rela_file_path VARCHAR(512) NOT NULL DEFAULT 'UNKNOWN' COMMENT '本地文件相对路径',
searchable_status INT NOT NULL DEFAULT 0 COMMENT '可搜索状态0-初始状态1-可搜索, 2-导入失败, 99-禁止搜索'
bucket_name VARCHAR(128) NOT NULL COMMENT '目标 bucket 名',
uploader VARCHAR(128) NOT NULL COMMENT '上传人用户名',
upload_time DATETIME NOT NULL COMMENT '上传时间',
object_path VARCHAR(512) NOT NULL COMMENT 'MinIO 存储路径',
url VARCHAR(1024) COMMENT '访问链接如CDN代理后的地址',
expire_time DATETIME NOT NULL DEFAULT '9999-12-31 23:59:59' COMMENT '链接过期时间(默认永久)',
add_time DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
INDEX idx_uploader(uploader),
INDEX idx_file_name(file_name),
INDEX idx_bucket(bucket_name),
INDEX idx_local_rela_file_path(local_rela_file_path(100))
) COMMENT='对象存储上传记录表';

4
config/redis/cmd.txt Normal file
View File

@ -0,0 +1,4 @@
redis-cli --raw
# 获取redis中的key-value
redis-cli keys "kb*" | while read key; do echo "$key : $(redis-cli get "$key")"; done

168
pom.xml
View File

@ -4,9 +4,9 @@
<modelVersion>4.0.0</modelVersion>
<groupId>luke.micro.saas</groupId>
<artifactId>doc-parser-server</artifactId>
<artifactId>knowledge-base-server</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>doc-parser-server</name>
<name>knowledge-base-server</name>
<description>文档资源解析服务</description>
<properties>
@ -29,10 +29,25 @@
<jackson-databind.version>2.13.4.2</jackson-databind.version>
<jackson-core.version>2.13.4</jackson-core.version>
<lombok.version>1.18.24</lombok.version>
<lombok.version>1.18.30</lombok.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
<hutool-all.version>5.7.20</hutool-all.version>
<guava.version>31.0.1-jre</guava.version>
<elasticsearch-rest-high-level-client.version>7.17.10</elasticsearch-rest-high-level-client.version>
<pdfbox.version>2.0.29</pdfbox.version>
<poi.version>5.2.3</poi.version>
<poi-ooxml.version>5.2.3</poi-ooxml.version>
<poi-scratchpad.version>5.2.3</poi-scratchpad.version>
<mysql-connector-java.version>8.0.33</mysql-connector-java.version>
<mybatis-plus-boot-starter.version>3.5.5</mybatis-plus-boot-starter.version>
<org.bouncycastle.version>1.70</org.bouncycastle.version>
<minio.version>8.5.3</minio.version>
<okhttp.version>4.12.0</okhttp.version>
</properties>
<dependencyManagement>
<dependencies>
@ -57,6 +72,11 @@
<scope>test</scope>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<!-- #日志依赖,参考 https://cdn.jsdelivr.net/gh/lukeyewu/pictures@main/uPic/202411/YhMx1M.png -->
<!-- 日志_抽象API层 -->
@ -130,7 +150,6 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
<version>${lombok.version}</version>
</dependency>
<dependency>
@ -149,6 +168,80 @@
<version>${guava.version}</version>
</dependency>
<!-- Elasticsearch 客户端 -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch-rest-high-level-client.version}</version>
</dependency>
<!-- pdf/word的解析 -->
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi-ooxml.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>${poi-scratchpad.version}</version>
</dependency>
<!-- MySQL JDBC 驱动 -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql-connector-java.version}</version>
</dependency>
<!-- MyBatis-Plus 启动器 -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus-boot-starter.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15to18</artifactId>
<version>${org.bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15to18</artifactId>
<version>${org.bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
<exclusions>
<exclusion>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -188,6 +281,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
@ -237,7 +334,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@ -252,6 +349,57 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<!-- MyBatis-Plus 启动器 -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15to18</artifactId>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15to18</artifactId>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</dependency>
</dependencies>
<build>
@ -264,11 +412,13 @@
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>UTF-8</encoding>
<release>${java.version}</release>
<annotationProcessorPaths>
<!-- lombok 支持 -->
<path>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.17.1</version>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
@ -278,7 +428,7 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<mainClass>com.doc.parser.DocParserApplication</mainClass>
<mainClass>com.knowledge.base.KnowledgeBaseApplication</mainClass>
</configuration>
<executions>
<execution>

View File

@ -1,16 +0,0 @@
package com.doc.parser;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author Luke.ye
*/
@SpringBootApplication
public class DocParserApplication {
public static void main(String[] args) {
SpringApplication.run(DocParserApplication.class, args);
}
}

View File

@ -1,24 +0,0 @@
package com.doc.parser.application.exceptions;
/**
* @author Luke.ye
* @date 2025/5/6 08:47
*/
public class BizException extends RuntimeException {
private int code;
public BizException(int code, String message) {
super(message);
this.code = code;
}
public BizException(int code, String message, Throwable cause) {
super(message, cause);
this.code = code;
}
public int getCode() {
return code;
}
}

View File

@ -1,98 +0,0 @@
package com.doc.parser.application.filter;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.doc.parser.application.enums.MDCKeyEnum;
import com.doc.parser.infrastructure.config.DynamicConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;
import org.springframework.web.filter.OncePerRequestFilter;
import org.springframework.web.util.ContentCachingRequestWrapper;
import org.springframework.web.util.ContentCachingResponseWrapper;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.InetAddress;
import java.util.Optional;
import java.util.UUID;
/**
* Description: 交易日志记录
*/
@Component
public class LogFilter extends OncePerRequestFilter {
private final static Logger LOGGER = LoggerFactory.getLogger(LogFilter.class);
@Autowired
private DynamicConfig dynamicConfig;
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException {
ContentCachingRequestWrapper contentCachingRequestWrapper = new ContentCachingRequestWrapper(request);
ContentCachingResponseWrapper contentCachingResponseWrapper = new ContentCachingResponseWrapper(response);
if (!"Y".equals(dynamicConfig.getRecordMsgBody())) {
// 不记录请求和响应信息
filterChain.doFilter(contentCachingRequestWrapper, contentCachingResponseWrapper);
contentCachingResponseWrapper.copyBodyToResponse();
return;
}
StopWatch stopWatch = new StopWatch();
try {
stopWatch.start();
//获取http header 中的traceid,没有默认生成一个,放入到mdc中,进行日志打印
String traceId = Optional.ofNullable(request.getHeader("traceId"))
.orElse(UUID.randomUUID().toString().replace("-", ""));
InetAddress addr = InetAddress.getLocalHost();
MDC.put(MDCKeyEnum.HOST_NAME.name, addr.getHostName());
MDC.put(MDCKeyEnum.TRACE_ID.name, traceId);
filterChain.doFilter(contentCachingRequestWrapper, contentCachingResponseWrapper);
} catch (Exception e) {
LOGGER.warn("交易跟踪号[{}]记录交易报文信息异常:{}", MDC.get(MDCKeyEnum.TRACE_ID.name), e.getMessage());
} finally {
byte[] requestBody = contentCachingRequestWrapper.getContentAsByteArray();
byte[] responseBody = contentCachingResponseWrapper.getContentAsByteArray();
try {
//获取请求路径
String url = contentCachingRequestWrapper.getRequestURL().toString();
//获取报文头交易码
String tranCode = Optional.ofNullable(contentCachingRequestWrapper.getHeader("TRANCODE")).orElse("UNKOWN");
//获取IP地址
String remoteIp = getRemortIP(contentCachingRequestWrapper);
//获取IP端口
int remotePort = contentCachingRequestWrapper.getRemotePort();
stopWatch.stop();
LOGGER.info("交易跟踪号[{}]\n请求路径[{}]\n交易码[{}]\n远程地址[{}]\n远程端口[{}]\n请求内容:[{}]\n响应内容:[{}]\n响应时间:[{}]毫秒",
MDC.get(MDCKeyEnum.TRACE_ID.name),
url,
tranCode,
remoteIp,
remotePort,
JSONUtil.toJsonStr(new String(requestBody)),
JSONUtil.toJsonStr(new String(responseBody)),
stopWatch.getTotalTimeMillis());
} catch (Exception e) {
LOGGER.warn("登记交易信息异常:" + e.getMessage());
}
MDC.remove(MDCKeyEnum.HOST_NAME.name);
MDC.remove(MDCKeyEnum.TRACE_ID.name);
contentCachingResponseWrapper.copyBodyToResponse();
}
}
private String getRemortIP(HttpServletRequest request) {
return StrUtil.isBlank(request.getHeader("x-forwarded-for")) ?
request.getRemoteAddr() :
request.getHeader("x-forwarded-for");
}
}

View File

@ -1,9 +0,0 @@
package com.doc.parser.infrastructure.config;
/**
* @author Luke.ye
* @date 2025/5/19 11:18
*/
public class ConstantConfig {
}

View File

@ -1,21 +0,0 @@
package com.doc.parser.infrastructure.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Component;
/**
* @author Luke.ye
* @date 2025/5/8 09:59
*/
@Component
@RefreshScope
public class DynamicConfig {
//是否记录请求和响应信息 Y-记录 N-不记录 默认记录
@Value("${micro.saas.doc.parser.recordMsgBody:Y}")
private String recordMsgBody;
public String getRecordMsgBody() {
return recordMsgBody;
}
}

View File

@ -0,0 +1,25 @@
package com.knowledge.base;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
import java.security.Security;
/**
* @author Luke.ye
*/
@EnableScheduling
@SpringBootApplication
public class KnowledgeBaseApplication {
static {
Security.addProvider(new BouncyCastleProvider());
}
public static void main(String[] args) {
SpringApplication.run(KnowledgeBaseApplication.class, args);
}
}

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.anno;
package com.knowledge.base.application.anno;
import java.lang.annotation.*;

View File

@ -0,0 +1,32 @@
package com.knowledge.base.application.aspect;
import com.knowledge.base.application.exceptions.AppException;
import com.knowledge.base.domain.common.exception.DomainException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import java.util.Map;
@Slf4j
@RestControllerAdvice
public class GlobalExceptionHandler {
@ExceptionHandler(DomainException.class)
public ResponseEntity<?> handleDomainException(DomainException e) {
return ResponseEntity.badRequest().body(Map.of("code", 400, "msg", e.getMessage()));
}
@ExceptionHandler(AppException.class)
public ResponseEntity<?> handleAppException(AppException e) {
return ResponseEntity.status(500).body(Map.of("code", 500, "msg", "系统异常:" + e.getMessage()));
}
@ExceptionHandler(Exception.class)
public ResponseEntity<?> handleOtherException(Exception e) {
log.error(e.getMessage(), e);
return ResponseEntity.status(500).body(Map.of("code", 500, "msg", "服务器繁忙,请稍后重试"));
}
}

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.aspect;
package com.knowledge.base.application.aspect;
import cn.hutool.json.JSONUtil;
import org.aspectj.lang.ProceedingJoinPoint;
@ -21,7 +21,7 @@ import java.lang.reflect.Method;
public class OperationLogAspect {
private static final Logger LOGGER = LoggerFactory.getLogger(OperationLogAspect.class);
@Around("@annotation(com.doc.parser.application.anno.OperationLog)")
@Around("@annotation(com.knowledge.base.application.anno.OperationLog)")
public Object logOperation(ProceedingJoinPoint joinPoint) throws Throwable {
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.enums;
package com.knowledge.base.application.enums;
/**
* @Author: luke ye

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.enums;
package com.knowledge.base.application.enums;
/**
* @author Luke.ye

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.enums;
package com.knowledge.base.application.enums;
/**
* @author Luke.ye

View File

@ -0,0 +1,6 @@
package com.knowledge.base.application.exceptions;
public class AppException extends RuntimeException {
public AppException(String msg) { super(msg); }
public AppException(String msg, Throwable cause) { super(msg, cause); }
}

View File

@ -0,0 +1,62 @@
package com.knowledge.base.application.filter;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.domain.user.service.iface.UserDomainService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.servlet.*;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
/**
* @author Luke.ye
* @date 2025/5/25 10:05
*/
@Component
public class AuthFilter implements Filter {
@Autowired
private UserDomainService userDomainService;
private static final String COOKIE_KEY = "auth_token";
@Override
public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
throws IOException, ServletException {
HttpServletRequest request = (HttpServletRequest) req;
HttpServletResponse response = (HttpServletResponse) res;
String path = request.getRequestURI();
// 需要鉴权的路径前缀
if (path.startsWith("/api/v1/doc")
|| path.startsWith("/api/v1/llm")
) {
String token = request.getHeader("Authorization");
if (StrUtil.isBlank(token)) {
Cookie[] cookies = request.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
if (COOKIE_KEY.equals(cookie.getName())) {
token = cookie.getValue();
break;
}
}
}
}
if (token == null || !userDomainService.isValidToken(token)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
response.getWriter().write("Invalid or expired token");
return;
}
}
chain.doFilter(req, res);
}
}

View File

@ -0,0 +1,155 @@
package com.knowledge.base.application.filter;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.knowledge.base.application.enums.MDCKeyEnum;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;
import org.springframework.web.filter.OncePerRequestFilter;
import org.springframework.web.util.ContentCachingRequestWrapper;
import org.springframework.web.util.ContentCachingResponseWrapper;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
/**
* 通用交易日志过滤器
*/
@Component
public class LogFilter extends OncePerRequestFilter {
private static final Logger LOGGER = LoggerFactory.getLogger(LogFilter.class);
private static final int MAX_LOG_BODY_LENGTH = 3000;
@Autowired
private DynamicConfig dynamicConfig;
@Override
protected void doFilterInternal(HttpServletRequest request,
HttpServletResponse response,
FilterChain filterChain) throws ServletException, IOException {
ContentCachingRequestWrapper requestWrapper = new ContentCachingRequestWrapper(request);
ContentCachingResponseWrapper responseWrapper = new ContentCachingResponseWrapper(response);
String accept = request.getHeader("Accept");
String contentType = response.getContentType();
String requestURI = requestWrapper.getRequestURI();
logger.info("requestURI: " + requestURI);
if ((accept != null && accept.contains("text/event-stream"))
|| (contentType != null && contentType.contains("text/event-stream"))
|| ("/api/v1/llm/ask".equals(requestURI))
) {
filterChain.doFilter(request, response);
return;
}
if (!"Y".equalsIgnoreCase(dynamicConfig.getRecordMsgBody())) {
filterChain.doFilter(requestWrapper, responseWrapper);
responseWrapper.copyBodyToResponse();
return;
}
StopWatch stopWatch = new StopWatch();
String traceId = null;
try {
stopWatch.start();
// traceId 生成或提取
traceId = Optional.ofNullable(request.getHeader("traceId"))
.orElse(IdUtil.fastSimpleUUID());
InetAddress addr = InetAddress.getLocalHost();
MDC.put(MDCKeyEnum.HOST_NAME.name, addr.getHostName());
MDC.put(MDCKeyEnum.TRACE_ID.name, traceId);
filterChain.doFilter(requestWrapper, responseWrapper);
} catch (Exception e) {
LOGGER.warn("记录交易信息异常(traceId={}): {}", MDC.get(MDCKeyEnum.TRACE_ID.name), e.getMessage());
} finally {
responseWrapper.setHeader(MDCKeyEnum.TRACE_ID.name, traceId);
stopWatch.stop();
logTraceMessage(requestWrapper, responseWrapper, stopWatch.getTotalTimeMillis());
MDC.clear();
responseWrapper.copyBodyToResponse();
}
}
private void logTraceMessage(ContentCachingRequestWrapper request,
ContentCachingResponseWrapper response,
long timeCostMs) {
String url = request.getRequestURL().toString();
String tranCode = Optional.ofNullable(request.getHeader("TRANCODE")).orElse("UNKNOWN");
String remoteIp = getRemoteIP(request);
int remotePort = request.getRemotePort();
String traceId = MDC.get(MDCKeyEnum.TRACE_ID.name);
String reqBody = extractRequestBody(request);
String respBody = extractResponseBody(response);
LOGGER.info("交易跟踪号[{}]\n请求路径[{}]\n交易码[{}]\n远程地址[{}]\n远程端口[{}]\n请求内容:[{}]\n响应内容:[{}]\n响应时间:[{}]毫秒",
traceId,
url,
tranCode,
remoteIp,
remotePort,
reqBody,
respBody,
timeCostMs
);
Map<String, Object> jsonLog = new HashMap<>();
jsonLog.put("traceId", traceId);
jsonLog.put("url", url);
jsonLog.put("tranCode", tranCode);
jsonLog.put("ip", remoteIp);
jsonLog.put("port", remotePort);
jsonLog.put("request", reqBody);
jsonLog.put("response", respBody);
jsonLog.put("timeMs", timeCostMs);
LOGGER.info("log_trace_record: {}", JSONUtil.toJsonStr(jsonLog));
}
private String extractRequestBody(ContentCachingRequestWrapper request) {
String contentType = request.getContentType();
byte[] content = request.getContentAsByteArray();
if (StrUtil.isNotBlank(contentType) && contentType.toLowerCase().contains("multipart/form-data")) {
// 表单上传只记录参数不打印文件体
Map<String, String[]> paramMap = request.getParameterMap();
return "[multipart/form-data] " + JSONUtil.toJsonStr(paramMap);
}
if (content.length == 0) return "<empty>";
String body = new String(content, StandardCharsets.UTF_8);
return body.length() > MAX_LOG_BODY_LENGTH ? body.substring(0, MAX_LOG_BODY_LENGTH) + "...[truncated]" : body;
}
private String extractResponseBody(ContentCachingResponseWrapper response) {
byte[] content = response.getContentAsByteArray();
if (content.length == 0) return "<empty>";
String body = new String(content, StandardCharsets.UTF_8);
return body.length() > MAX_LOG_BODY_LENGTH ? body.substring(0, MAX_LOG_BODY_LENGTH) + "...[truncated]" : body;
}
private String getRemoteIP(HttpServletRequest request) {
String xff = request.getHeader("x-forwarded-for");
return StrUtil.isBlank(xff) ? request.getRemoteAddr() : xff;
}
}

View File

@ -1,4 +1,4 @@
package com.doc.parser.application.filter;
package com.knowledge.base.application.filter;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@ -8,7 +8,7 @@ import javax.servlet.ServletResponse;
import java.io.IOException;
import cn.hutool.core.util.IdUtil;
import com.doc.parser.application.enums.MDCKeyEnum;
import com.knowledge.base.application.enums.MDCKeyEnum;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;

View File

@ -0,0 +1,55 @@
package com.knowledge.base.application.service;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import java.util.List;
/**
* @author Luke.ye
* @date 2025/6/16 19:13
*/
public interface DocAppService {
/**
* 分页查询上传记录按上传人
*/
PageResult<OSRecordDTO> pageQueryOSRecord(int page, int size, String uploader);
/**
* 批量获取上传记录
* @param recordIds
* @return
*/
List<OSRecordDTO> batchQueryOSRecord(List<Long> recordIds);
/**
* 批量删除上传记录
*
* @param clearAll 是否清空所有
* true: 清空ES/缓存/MySQL/对象存储/本地存储中所有相关的数据
* false仅清除 MySQL+对象存储 中的数据
* @param recordIds 本地存储路径
* @return
*/
boolean batchRemoveOSFiles(boolean clearAll, List<Long> recordIds);
/**
* 保存上传记录
*/
void saveOSUplodRecord(OSRecordDTO dto);
/**
* 将同路径下上传的历史同名文件全部抛弃
* @param localRelaFilePath
* @param latestMillis --- 最新版本文件的上传时间毫秒
*/
int markOldSearchableRecordsAsDiscarded(String localRelaFilePath, long latestMillis);
boolean batchDeleteFilesInES(List<String> filePaths);
}

View File

@ -0,0 +1,136 @@
package com.knowledge.base.application.service;
import cn.hutool.core.collection.CollectionUtil;
import com.google.common.collect.Lists;
import com.knowledge.base.application.exceptions.AppException;
import com.knowledge.base.domain.common.enums.SearchableStatusEnum;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ObjectStorageProperties;
import com.knowledge.base.infrastructure.converter.FileDtoConverter;
import com.knowledge.base.infrastructure.north.dto.doc.OSRecordDTO;
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
import com.knowledge.base.infrastructure.south.minio.ObjectStorageGateway;
import com.knowledge.base.infrastructure.util.LocalFileUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
@Component
@RequiredArgsConstructor
@Slf4j
public class DocAppServiceImpl implements DocAppService{
private final FileDomainService fileDomainService;
private final ObjectStorageProperties osProperties;
private final FileElasticsearchGateway esGateway;
private final FileCacheService fileCacheService;
private final ObjectStorageGateway osGateway;
@Override
public PageResult<OSRecordDTO> pageQueryOSRecord(int page, int size, String uploader) {
if(page < 1 || size < 1) {
throw new AppException("分页参数错误!");
}
PageResult<OSRecordDO> osRecordDOPageResult = null;
final List<OSRecordDTO> dtoList = Lists.newArrayList();
try {
osRecordDOPageResult = fileDomainService.pageQueryOSRecord(page, size, uploader);
osRecordDOPageResult.getList().forEach(osRecordDO ->
dtoList.add(FileDtoConverter.toDTO(osRecordDO))
);
} catch (Exception e) {
throw new AppException("分页获取对象存储记录失败!", e);
}
return new PageResult<>(dtoList, osRecordDOPageResult.getTotal());
}
@Override
public List<OSRecordDTO> batchQueryOSRecord(List<Long> recordIds) {
if(CollectionUtil.isEmpty(recordIds)) {
throw new AppException("参数recordIds不能为空");
}
List<OSRecordDO> osRecordDOList = null;
try {
osRecordDOList = fileDomainService.batchQueryOSRecord(recordIds);
} catch (Exception e) {
throw new AppException("批量获取对象存储记录失败请检查recordIds", e);
}
return FileDtoConverter.toDTOs(osRecordDOList);
}
@Override
public boolean batchRemoveOSFiles(boolean clearAll, List<Long> recordIds) {
if(CollectionUtil.isEmpty(recordIds)) {
throw new AppException("参数recordIds不能为空");
}
try {
List<OSRecordDTO> osRecordDTOS = Optional.ofNullable(batchQueryOSRecord(recordIds)).orElse(Lists.newArrayList());
Map<String, List<String>> bucketToObjectsMap = osRecordDTOS.stream()
.collect(Collectors.groupingBy(
OSRecordDTO::getBucketName,
Collectors.mapping(OSRecordDTO::getObjectPath, Collectors.toList())
));
// 删除关联数据库记录
fileDomainService.batchRemoveOSFiles(recordIds);
// 删除对象存储中的文件
bucketToObjectsMap.forEach((bucket, objPaths) -> {
try {
osGateway.deleteObjects(bucket, objPaths);
} catch (Exception e) {
log.error("删除对象存储中的文件失败. bucket: {}", bucket, e);
log.error("待删除路径如下: {}", objPaths);
}
});
if(clearAll) {
List<String> allSearchableFilesRelaPath = osRecordDTOS.stream()
.filter(e -> e.getSearchableStatus() == SearchableStatusEnum.SUCCESS.code)
.map(OSRecordDTO::getLocalRelaFilePath).collect(Collectors.toList());
// 删除本地文件
LocalFileUtil.deleteFilesByRelativePath(osProperties.getLocalSearchablePathPrefix(), allSearchableFilesRelaPath);
// 删除ES
esGateway.deleteByFilepaths(allSearchableFilesRelaPath);
// 删除缓存
fileCacheService.removeMetaCacheBatch(allSearchableFilesRelaPath);
}
return true;
} catch (Exception e) {
throw new AppException("批量删除对象存储记录失败请检查recordIds", e);
}
}
@Override
public void saveOSUplodRecord(OSRecordDTO dto) {
try {
fileDomainService.save(FileDtoConverter.toDO(dto));
} catch (Exception e) {
throw new AppException("保存对象存储记录失败!", e);
}
}
@Override
public int markOldSearchableRecordsAsDiscarded(String localRelaFilePath, long latestMillis) {
return fileDomainService.updateOldSearchableRecordsByRelaPathAndTime(localRelaFilePath, latestMillis,
SearchableStatusEnum.SUCCESS.code, SearchableStatusEnum.DISCARDED.code);
}
@Override
public boolean batchDeleteFilesInES(List<String> filePaths) {
esGateway.deleteByFilepaths(filePaths);
return true;
}
}

View File

@ -0,0 +1,28 @@
package com.knowledge.base.application.service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.Map;
public interface LLMAppService {
/**
* 获取llmtoken
* @param password
* @return
* @throws Exception
*/
String getToken(String password) throws Exception;
/**
* 问llm问题
* @param llmToken
* @param question
* @param params
* @return
* @throws Exception
*/
SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception;
}

View File

@ -0,0 +1,54 @@
package com.knowledge.base.application.service;
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
import com.knowledge.base.infrastructure.south.llm.LLMServiceFactory;
import com.knowledge.base.infrastructure.util.RateLimiterManager;
import com.knowledge.base.infrastructure.util.ThreadPoolUtil;
import com.knowledge.base.infrastructure.util.http.FilteredSseOutputAdapter;
import com.knowledge.base.infrastructure.util.http.WriterAdapter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class LLMAppServiceImpl implements LLMAppService {
private final LLMServiceFactory llmServiceFactory;
private final RateLimiterManager rateLimiterManager;
@Override
public String getToken(String password) throws Exception {
return llmServiceFactory.current().fetchToken(password);
}
@Override
public SseEmitter ask(String llmToken, String question, Map<String, Object> params) throws Exception {
SseEmitter emitter = new SseEmitter(300 * 1000L); // 超时时间设为5分钟
ThreadPoolUtil.execute(() -> {
try {
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_LLM_ASK).acquire();
WriterAdapter adapter = new FilteredSseOutputAdapter(emitter);
llmServiceFactory.current().streamAnswer(llmToken, question, params, adapter);
emitter.complete();
} catch (Exception e) {
log.error("LLM调用异常", e);
try {
emitter.send(SseEmitter.event().data("{\"error\": \"LLM异常\"}"));
emitter.completeWithError(e);
} catch (IOException ioException) {
log.warn("SSE发送错误信息失败", ioException);
}
}
}, ThreadPoolConfig.SSE_POOL);
return emitter;
}
}

View File

@ -0,0 +1,60 @@
package com.knowledge.base.application.service;
import com.knowledge.base.infrastructure.north.dto.role.RoleDTO;
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
import com.knowledge.base.infrastructure.north.dto.role.UserRoleDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import java.util.List;
import java.util.Optional;
public interface UserAppService {
// 用户注册
boolean registerUser(String username, String password, List<Long> roleIds);
boolean verifyPassword(String username, String plainText);
boolean changePassword(Long userId, String oldPassword, String newPassword);
boolean deleteUser(Long id);
boolean updateUser(Long id, UserDTO userDTO);
// 用户资料
Optional<UserDTO> findByUsername(String username);
Optional<UserDTO> findById(Long id);
List<UserDTO> findAll();
// Token 相关
UserTokenDTO createToken(Long userId, long expireMs);
void removeToken(String token);
boolean isValidToken(String token);
Optional<UserTokenDTO> findToken(String token);
Optional<UserDTO> findUserByToken(String token);
// 用户角色
List<UserRoleDTO> listUserRoles(Long userId);
boolean addUserRole(Long userId, Long roleId);
boolean removeUserRole(Long userId, Long roleId);
// 用户特殊授权
List<UserFileDTO> listUserFiles(Long userId);
boolean addUserFileAuth(Long userId, Long fileId);
boolean removeUserFileAuth(Long userId, Long fileId);
// 角色
List<RoleDTO> listRoles();
Optional<RoleDTO> getRoleById(Long roleId);
boolean addRole(RoleDTO role);
boolean updateRole(RoleDTO role);
boolean deleteRole(Long id);
// 角色授权信息
List<RoleFileRuleDTO> listRoleFileRules(Long roleId);
List<RoleFileRuleDTO> listAllRules();
boolean addRoleFileRule(RoleFileRuleDTO rule);
boolean updateRoleFileRule(RoleFileRuleDTO rule);
boolean removeRoleFileRule(Long id);
}

View File

@ -0,0 +1,238 @@
package com.knowledge.base.application.service;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.knowledge.base.application.exceptions.AppException;
import com.knowledge.base.domain.user.model.RoleDO;
import com.knowledge.base.domain.user.model.UserDO;
import com.knowledge.base.domain.user.model.UserRoleDO;
import com.knowledge.base.domain.user.model.UserTokenDO;
import com.knowledge.base.domain.user.service.iface.UserDomainService;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import com.knowledge.base.infrastructure.converter.*;
import com.knowledge.base.infrastructure.north.dto.role.RoleDTO;
import com.knowledge.base.infrastructure.north.dto.role.RoleFileRuleDTO;
import com.knowledge.base.infrastructure.north.dto.role.UserRoleDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserFileDTO;
import com.knowledge.base.infrastructure.north.dto.user.UserTokenDTO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@Slf4j
public class UserAppServiceImpl implements UserAppService {
private final UserDomainService userDomainService;
private final UserCacheService userCacheService;
@Override
public boolean registerUser(String username, String password, List<Long> roleIds) {
boolean ok = userDomainService.registerUser(username, password);
if (!ok) return false;
Optional<UserDO> userOpt = userDomainService.findByUsername(username);
if (userOpt.isEmpty()) return false;
Long userId = userOpt.get().getId();
for (Long roleId : roleIds) {
userDomainService.addUserRole(userId, roleId);
}
return true;
}
@Override
public boolean verifyPassword(String username, String plainText) {
return userDomainService.verifyPassword(username, plainText);
}
@Override
public boolean changePassword(Long userId, String oldPassword, String newPassword) {
return userDomainService.changePassword(userId, oldPassword, newPassword);
}
@Override
public boolean deleteUser(Long userId) {
return userDomainService.deleteUser(userId);
}
@Override
public boolean updateUser(Long userId, UserDTO userDTO) {
UserDO userDO = UserDtoConverter.toDO(userDTO);
try {
userDomainService.updateUser(userDO);
userDomainService.removeUserRoles(userId);
userDomainService.batchAddUserRoles(userId,
userDTO.getUserRoles().stream().map(RoleDTO::getId).collect(Collectors.toList()));
} catch (Exception e) {
throw new AppException("更新用户信息失败", e);
}
return true;
}
@Override
public Optional<UserDTO> findByUsername(String username) {
return Optional.ofNullable(UserDtoConverter.toDTO(userDomainService.findByUsername(username).get()));
}
@Override
public Optional<UserDTO> findById(Long id) {
return Optional.ofNullable(UserDtoConverter.toDTO(userDomainService.findById(id).get()));
}
@Override
public List<UserDTO> findAll() {
// userIds
List<UserDO> allUsers = userDomainService.findAllUsers();
List<Long> allUserIds = allUsers.stream().map(e -> e.getId()).collect(Collectors.toList());
// userId ---> roleId
Map<Long, List<UserRoleDO>> userRolesRelationMap = userDomainService.batchFindRolesByUserIds(allUserIds);
// roleId ---> role详情
Map<Long, RoleDO> roleMap = userDomainService.findAllRoles().stream().collect(Collectors.toMap(RoleDO::getId, e -> e));
// userId ---> role详情
Map<Long, List<RoleDO>> userRolesMap = allUserIds.stream().collect(Collectors.toMap(userId -> userId, userId -> {
List<Long> roleIds = Optional.ofNullable(userRolesRelationMap.get(userId)).orElse(Lists.newArrayList()).stream()
.map(UserRoleDO::getRoleId).collect(Collectors.toList());
return roleIds.stream().map(roleId -> roleMap.get(roleId)).collect(Collectors.toList());
}));
return Optional.ofNullable(allUsers).orElse(Lists.newArrayList()).stream()
.map(e -> UserDtoConverter.toDTO(e, userRolesMap.get(e.getId())))
.collect(Collectors.toList());
}
@Override
public UserTokenDTO createToken(Long userId, long expireMs) {
return UserTokenDtoConverter.toDTO(userDomainService.createToken(userId, expireMs));
}
@Override
public void removeToken(String token) {
userDomainService.removeToken(token);
}
@Override
public boolean isValidToken(String token) {
return userDomainService.isValidToken(token);
}
@Override
public Optional<UserTokenDTO> findToken(String token) {
return Optional.ofNullable(UserTokenDtoConverter.toDTO(userDomainService.findToken(token).get()));
}
@Override
public Optional<UserDTO> findUserByToken(String token) {
Optional<String> cachedUserJson = userCacheService.getUserJsonByToken(token);
if(cachedUserJson.isPresent()) {
return Optional.ofNullable(JSONUtil.toBean(cachedUserJson.get(), UserDTO.class));
}
Optional<UserTokenDO> userTokenOpt = userDomainService.findToken(token);
if (userTokenOpt.isEmpty()) {
log.warn("[findUserByToken] token无效: {}", token);
return Optional.empty();
}
Optional<UserDO> userDO = userDomainService.findById(userTokenOpt.get().getUserId());
UserDTO dto = UserDtoConverter.toDTO(userDO.get());
// 数据进缓存
userCacheService.cacheUserJsonByToken(token, JSONUtil.toJsonStr(dto));
return Optional.ofNullable(dto);
}
@Override
public List<UserRoleDTO> listUserRoles(Long userId) {
return Optional.ofNullable(userDomainService.findRolesByUserId(userId)).orElse(Lists.newArrayList()).stream()
.map(e -> UserRoleDtoConverter.toDTO(e))
.collect(Collectors.toList());
}
@Override
public boolean addUserRole(Long userId, Long roleId) {
return userDomainService.addUserRole(userId, roleId);
}
@Override
public boolean removeUserRole(Long userId, Long roleId) {
return userDomainService.removeUserRole(userId, roleId);
}
@Override
public List<UserFileDTO> listUserFiles(Long userId) {
return Optional.ofNullable(userDomainService.findUserFiles(userId)).orElse(Lists.newArrayList()).stream()
.map(e -> UserFileDtoConverter.toDTO(e))
.collect(Collectors.toList());
}
@Override
public boolean addUserFileAuth(Long userId, Long fileId) {
return userDomainService.addUserFileAuth(userId, fileId);
}
@Override
public boolean removeUserFileAuth(Long userId, Long fileId) {
return userDomainService.removeUserFileAuth(userId, fileId);
}
@Override
public List<RoleDTO> listRoles() {
return Optional.ofNullable(userDomainService.findAllRoles()).orElse(Lists.newArrayList()).stream()
.map(e -> RoleDtoConverter.toDTO(e))
.collect(Collectors.toList());
}
@Override
public Optional<RoleDTO> getRoleById(Long roleId) {
return Optional.ofNullable(RoleDtoConverter.toDTO(userDomainService.findRoleById(roleId).get()));
}
@Override
public boolean addRole(RoleDTO role) {
return userDomainService.addRole(RoleDtoConverter.toDO(role));
}
@Override
public boolean updateRole(RoleDTO role) {
return userDomainService.updateRole(RoleDtoConverter.toDO(role));
}
@Override
public boolean deleteRole(Long id) {
return userDomainService.deleteRole(id);
}
@Override
public List<RoleFileRuleDTO> listRoleFileRules(Long roleId) {
return Optional.ofNullable(userDomainService.findRoleFileRules(roleId)).orElse(Lists.newArrayList()).stream()
.map(e -> RoleFileRuleDtoConverter.toDTO(e))
.collect(Collectors.toList());
}
@Override
public List<RoleFileRuleDTO> listAllRules() {
return Optional.ofNullable(userDomainService.listAllRules()).orElse(Lists.newArrayList()).stream()
.map(e -> RoleFileRuleDtoConverter.toDTO(e))
.collect(Collectors.toList());
}
@Override
public boolean addRoleFileRule(RoleFileRuleDTO rule) {
return userDomainService.addRoleFileRule(RoleFileRuleDtoConverter.toDO(rule));
}
@Override
public boolean updateRoleFileRule(RoleFileRuleDTO rule) {
return userDomainService.updateRoleFileRule(RoleFileRuleDtoConverter.toDO(rule));
}
@Override
public boolean removeRoleFileRule(Long id) {
return userDomainService.removeRoleFileRule(id);
}
}

View File

@ -0,0 +1,18 @@
package com.knowledge.base.domain.common.enums;
public enum DocMetaPropEnum {
UPLOADER("uploader", "上传人"),
ACCESS_URL("url", "可访问的链接"),
UPLOAD_TIME("uploadTime", "上传时间戳(毫秒)"),
EXPIRE_TIME("expireTime", "过期时间戳(毫秒)")
;
public String code;
private String desc;
DocMetaPropEnum(String code, String desc) {
this.code = code;
this.desc = desc;
}
}

View File

@ -0,0 +1,26 @@
package com.knowledge.base.domain.common.enums;
/**
* @author Luke.ye
* @date 2025/5/20 09:03
*/
public enum DocTypeEnum {
MARKDOWN("markdown", "markdown"),
EXCEL("excel", "excel"),
PDF("pdf", "pdf"),
WORD("word", "word"),
PPT("ppt", "ppt"),
TXT("txt", "txt"),
;
public String code;
public String desc;
DocTypeEnum(String code, String desc) {
this.code = code;
this.desc = desc;
}
}

View File

@ -0,0 +1,23 @@
package com.knowledge.base.domain.common.enums;
/**
* @author Luke.ye
* @date 2025/6/20 16:41
*/
public enum SearchableStatusEnum {
INIT(0, "未开始进行导入"),
SUCCESS(1, "成功"),
FAILED(2, "导入失败"),
DISCARDED(3, "已过期,改文件不允许被检索;同人当天多次上次同名文件会进入该状态"),
PROHIBITED(99, "用户禁止文件被检索"),
;
public int code;
private String desc;
SearchableStatusEnum(int code, String desc) {
this.code = code;
this.desc = desc;
}
}

View File

@ -0,0 +1,6 @@
package com.knowledge.base.domain.common.exception;
public class DomainException extends RuntimeException {
public DomainException(String msg) { super(msg); }
public DomainException(String msg, Throwable cause) { super(msg, cause); }
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.common.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class PageResult<T> {
private List<T> list;
private long total;
}

View File

@ -0,0 +1,76 @@
package com.knowledge.base.domain.common.schedule;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.infrastructure.config.DynamicConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.Trigger;
import org.springframework.scheduling.TriggerContext;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.SchedulingConfigurer;
import org.springframework.scheduling.config.ScheduledTaskRegistrar;
import org.springframework.scheduling.support.CronTrigger;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
/**
* 动态定时任务调度器读取 cron 表达式来自 Nacos
*/
@Component
@EnableScheduling
public class ImportScheduler implements SchedulingConfigurer {
private static final Logger logger = LoggerFactory.getLogger(ImportScheduler.class);
@Value("${import.schedule.enabled:true}")
private boolean scheduleEnabled;
@Resource
private List<DocumentImporter> importers;
@Resource
private DynamicConfig dynamicConfig;
@Override
public void configureTasks(ScheduledTaskRegistrar taskRegistrar) {
taskRegistrar.addTriggerTask(
this::doImportTask,
new Trigger() {
@Override
public Date nextExecutionTime(TriggerContext triggerContext) {
String cron = dynamicConfig.getImportScheduleCron();
try {
return new CronTrigger(cron).nextExecutionTime(triggerContext);
} catch (Exception e) {
logger.error("动态 cron 表达式解析失败: {},使用默认值 0 0 * * * *", cron, e);
return new CronTrigger("0 0 * * * *").nextExecutionTime(triggerContext);
}
}
}
);
}
private void doImportTask() {
if (!scheduleEnabled) {
logger.info("导入定时任务已禁用");
return;
}
logger.info("开始执行动态定时导入任务");
for (DocumentImporter importer : importers) {
try {
logger.info("执行导入器: {}", importer.getType());
importer.importDocuments();
} catch (Exception e) {
logger.error("导入器执行失败: {}", importer.getType(), e);
}
}
logger.info("定时导入任务执行完成");
}
}

View File

@ -0,0 +1,18 @@
package com.knowledge.base.domain.doc.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/9 10:51
*/
@Data
public class FileDO {
public Long id;
public String fileName;
public String filePath;
public LocalDateTime addTime;
public LocalDateTime updateTime;
}

View File

@ -0,0 +1,16 @@
package com.knowledge.base.domain.doc.model;
/**
* @author Luke.ye
* @date 2025/6/20 13:57
*/
public class FileEsField {
public static final String INDEX = "documents";
public static final String FILENAME = "filename";
public static final String FILEPATH = "filepath";
public static final String CONTENT = "content";
public static final String MTIME = "mtime";
public static final String UPLOADER = "uploader";
public static final String URL = "url";
public static final String EXPIRE_TIME = "expireTime";
}

View File

@ -0,0 +1,32 @@
package com.knowledge.base.domain.doc.model;
import com.knowledge.base.infrastructure.util.SafeIdUtil;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 文件文档模型字段应于 {@link FileEsField} 保持一致
* @author Luke.ye
* @date 2025/6/20 13:45
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileEsModel {
private String filename; // 文档标题
private String filepath; // 相对路径作为唯一 ID
private String content; // 文本内容
private Long mtime; // 最后修改时间
private String uploader; // 上传人
private String url; // 文件访问链接
private Long expireTime; // 过期时间戳
public String buildDocId() {
return SafeIdUtil.encode(this.filepath);
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.doc.model;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class OSRecordDO {
private Long id;
private String fileName;
private String localRelaFilePath;
private int searchableStatus;
private String bucketName;
private String uploader;
private String objectPath;
private String url;
private LocalDateTime uploadTime;
private LocalDateTime expireTime;
private LocalDateTime addTime;
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.doc.repository.converter;
import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.model.FileDO;
import org.springframework.beans.BeanUtils;
public class FileDomainConverter {
public static FileDO toDO(File po) {
if (po == null) return null;
FileDO doObj = new FileDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static File toPO(FileDO doObj) {
if (doObj == null) return null;
File po = new File();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,14 @@
package com.knowledge.base.domain.doc.repository.iface;
import com.knowledge.base.domain.doc.repository.po.File;
import java.util.List;
public interface FileRepository {
boolean save(File file);
boolean update(File file);
boolean deleteById(Long id);
File findById(Long id);
List<File> findAll();
List<File> findByFileNameLike(String pattern);
}

View File

@ -0,0 +1,59 @@
package com.knowledge.base.domain.doc.repository.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import java.util.List;
import java.util.Optional;
public interface OSRecordRepository {
/**
* 分页查询上传记录按上传人
*/
PageResult<OSRecord> pageQuery(int page, int size, String uploader);
/**
* 保存上传记录
*/
void save(OSRecord po);
/**
* 获取文档最新版本
* @param
* @return
*/
Optional<OSRecord> getLatestRecordByRelaPath(String localRelaFilePath);
/**
* 更新上传记录中文档的可搜索状态
* @param localRelaFilePath
* @param oldSearchableStatus --- 为空时表示所有数据都需要更新
* @param newSearchableStatus
* @return
*/
boolean updateSearchableStatusByRelaPath(String localRelaFilePath, Integer oldSearchableStatus, int newSearchableStatus);
/**
* 跟新latestMills时间戳之前的记录可搜索状态为searchableStatus
* @param localRelaFilePath
* @param oldSearchableStatus --- 为空时表示所有数据都需要更新
* @param newSearchableStatus
* @param latestMills
* @return
*/
int updateOldSearchableRecordsByRelaPathAndTime(String localRelaFilePath, long latestMills, Integer oldSearchableStatus, int newSearchableStatus);
/**
* 批量获取
* @param recordIds
* @return
*/
List<OSRecord> batchQueryOSRecord(List<Long> recordIds);
/**
* 批量删除
* @param recordIds
*/
boolean batchRemoveOSFiles(List<Long> recordIds);
}

View File

@ -0,0 +1,28 @@
package com.knowledge.base.domain.doc.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 知识库文件表
* @author Luke.Ye
*/
@Data
@TableName("file")
public class File {
@TableId(type = IdType.AUTO)
private Long id;
private String fileName;
private String filePath;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,30 @@
package com.knowledge.base.domain.doc.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("oss_upload_record")
public class OSRecord {
@TableId(type = IdType.AUTO)
private Long id;
private String fileName;
private String localRelaFilePath;
private int searchableStatus;
private String bucketName;
private String uploader;
private String objectPath;
private String url;
private LocalDateTime uploadTime;
private LocalDateTime expireTime;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,45 @@
package com.knowledge.base.domain.doc.service;
import com.knowledge.base.domain.doc.service.impl.importer.AbstractBaseFileImporter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.nio.file.Path;
import java.util.*;
/**
* 文件导入分派器可根据文件后缀名的不同执行不同的导入逻辑
*/
@Component
public class FileImporterDispatcher {
@Autowired
private List<AbstractBaseFileImporter> fileImporters;
private final Map<String, AbstractBaseFileImporter> suffixToImporter = new HashMap<>();
@PostConstruct
public void init() {
for (AbstractBaseFileImporter importer : fileImporters) {
for (String suffix : importer.getFileSuffixes()) {
suffixToImporter.put(suffix.toLowerCase(), importer);
}
}
}
public boolean importSingleFile(Path filePath, Path excludePrefix, Map<String, Object> extInfo) {
String fileName = filePath.getFileName().toString().toLowerCase();
Optional<String> matchedSuffix = suffixToImporter.keySet().stream()
.filter(fileName::endsWith)
.findFirst();
if (matchedSuffix.isEmpty()) {
throw new IllegalArgumentException("不支持的文件后缀: " + fileName);
}
AbstractBaseFileImporter importer = suffixToImporter.get(matchedSuffix.get());
return importer.insertOrUpdateOneFileIntoES(filePath, excludePrefix, extInfo);
}
}

View File

@ -0,0 +1,28 @@
package com.knowledge.base.domain.doc.service.iface;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import java.nio.file.Path;
import java.util.Map;
/**
* @author Luke.ye
* @date 2025/5/20 09:02
*/
public interface DocumentImporter {
void importDocuments() throws Exception;
/**
* {@link DocTypeEnum}
* @return
*/
String getType();
/**
* 将文件插入/更新到ES中
* @param absoluteFilePath
* @param excludeFilePrefix
* @return
*/
default boolean insertOrUpdateOneFileIntoES(Path absoluteFilePath, Path excludeFilePrefix, Map<String, Object> extInfo) { return true; }
}

View File

@ -0,0 +1,69 @@
package com.knowledge.base.domain.doc.service.iface;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.po.File;
import java.util.List;
import java.util.Optional;
public interface FileDomainService {
boolean save(File file);
boolean update(File file);
boolean deleteById(Long id);
Optional<File> findById(Long id);
List<File> findAll();
List<File> findByFileNameLike(String pattern);
/**
* 分页查询上传记录按上传人
*/
PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader);
/**
* 保存上传记录
*/
void save(OSRecordDO osRecord);
/**
* 获取文档最新版本
* @param
* @return
*/
Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath);
/**
* 更新上传记录中文档的可搜索状态
* @param localRelaFilePath
* @param oldSearchableStatus
* @param newSearchableStatus
* @return
*/
boolean updateSearchableStatusByRelaPath(String localRelaFilePath, Integer oldSearchableStatus, int newSearchableStatus);
/**
* 跟新latestMills时间戳之前的记录可搜索状态为searchableStatus
* @param localRelaFilePath
* @param oldSearchableStatus --- 为空时表示所有数据都需要更新
* @param newSearchableStatus
* @param latestMills
* @return
*/
int updateOldSearchableRecordsByRelaPathAndTime(String localRelaFilePath, long latestMills, Integer oldSearchableStatus, int newSearchableStatus);
/**
* 批量获取上传记录
* @param recordIds
* @return
*/
List<OSRecordDO> batchQueryOSRecord(List<Long> recordIds);
/**
* 批量删除上传记录
* @param recordIds
* @return
*/
boolean batchRemoveOSFiles(List<Long> recordIds);
}

View File

@ -0,0 +1,98 @@
package com.knowledge.base.domain.doc.service.impl;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.domain.common.model.PageResult;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.repository.iface.FileRepository;
import com.knowledge.base.domain.doc.repository.iface.OSRecordRepository;
import com.knowledge.base.domain.doc.repository.po.File;
import com.knowledge.base.domain.doc.repository.po.OSRecord;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import lombok.RequiredArgsConstructor;
import com.knowledge.base.infrastructure.util.BeanConvertUtil;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
@Service
@RequiredArgsConstructor
public class FileDomainServiceImpl implements FileDomainService {
private final FileRepository fileRepository;
private final OSRecordRepository osRecordRepository;
@Override
public boolean save(File file) {
return fileRepository.save(file);
}
@Override
public boolean update(File file) {
return fileRepository.update(file);
}
@Override
public boolean deleteById(Long id) {
return fileRepository.deleteById(id);
}
@Override
public Optional<File> findById(Long id) {
return Optional.ofNullable(fileRepository.findById(id));
}
@Override
public List<File> findAll() {
return fileRepository.findAll();
}
@Override
public List<File> findByFileNameLike(String pattern) {
return fileRepository.findByFileNameLike(pattern);
}
@Override
public PageResult<OSRecordDO> pageQueryOSRecord(int page, int size, String uploader) {
PageResult<OSRecord> osRecordPageResult = osRecordRepository.pageQuery(page, size, uploader);
return new PageResult<>(BeanConvertUtil.convertList(osRecordPageResult.getList(), OSRecordDO.class),
osRecordPageResult.getTotal());
}
@Override
public void save(OSRecordDO osRecordDO) {
osRecordRepository.save(BeanConvertUtil.convert(osRecordDO, OSRecord.class));
}
@Override
public Optional<OSRecordDO> getLatestRecordByRelaPath(String localRelaFilePath) {
if(StrUtil.isBlank(localRelaFilePath)) {
return Optional.empty();
}
Optional<OSRecord> osRecordOpt = osRecordRepository.getLatestRecordByRelaPath(localRelaFilePath);
return osRecordOpt.map(osRecord -> BeanConvertUtil.convert(osRecord, OSRecordDO.class));
}
@Override
public boolean updateSearchableStatusByRelaPath(String localRelaFilePath, Integer oldSearchableStatus, int newSearchableStatus) {
return osRecordRepository.updateSearchableStatusByRelaPath(localRelaFilePath, oldSearchableStatus, newSearchableStatus);
}
@Override
public int updateOldSearchableRecordsByRelaPathAndTime(String localRelaFilePath, long latestMills, Integer oldSearchableStatus, int newSearchableStatus) {
return osRecordRepository.updateOldSearchableRecordsByRelaPathAndTime(localRelaFilePath, latestMills, oldSearchableStatus, newSearchableStatus);
}
@Override
public List<OSRecordDO> batchQueryOSRecord(List<Long> recordIds) {
return BeanConvertUtil.convertList(osRecordRepository.batchQueryOSRecord(recordIds), OSRecordDO.class);
}
@Override
public boolean batchRemoveOSFiles(List<Long> recordIds) {
osRecordRepository.batchRemoveOSFiles(recordIds);
return true;
}
}

View File

@ -0,0 +1,250 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.knowledge.base.domain.common.enums.DocMetaPropEnum;
import com.knowledge.base.domain.common.enums.SearchableStatusEnum;
import com.knowledge.base.domain.doc.model.FileEsField;
import com.knowledge.base.domain.doc.model.FileEsModel;
import com.knowledge.base.domain.doc.model.OSRecordDO;
import com.knowledge.base.domain.doc.service.iface.DocumentImporter;
import com.knowledge.base.domain.doc.service.iface.FileDomainService;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.config.ConstantConfig;
import com.knowledge.base.infrastructure.config.ThreadPoolConfig;
import com.knowledge.base.infrastructure.south.es.FileElasticsearchGateway;
import com.knowledge.base.infrastructure.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.IOException;
import java.nio.file.*;
import java.util.*;
public abstract class AbstractBaseFileImporter implements DocumentImporter {
private static final Logger logger = LoggerFactory.getLogger(AbstractBaseFileImporter.class);
@Autowired
private FileElasticsearchGateway esGateway;
@Autowired
private FileCacheService fileCacheService;
@Autowired
private FileDomainService fileDomainService;;
@Autowired
private RateLimiterManager rateLimiterManager;
protected abstract String getDirectoryPath();
public abstract Set<String> getFileSuffixes();
protected abstract String getDocTypeCode();
protected abstract String getExcludePrefix();
@Override
public void importDocuments() throws IOException {
Path basePath = Paths.get(getDirectoryPath()).toAbsolutePath().normalize();
Path excludeBase = Paths.get(getExcludePrefix()).toAbsolutePath().normalize();
Files.walk(basePath)
.filter(Files::isRegularFile)
.filter(path -> {
String fileName = path.getFileName().toString().toLowerCase();
return getFileSuffixes().stream().anyMatch(fileName::endsWith);
})
.forEach(path -> {
rateLimiterManager.getRateLimiter(RateLimiterManager.RATE_LIMIT_SCENE_IMPORT).acquire();
ThreadPoolUtil.execute(() -> insertOrUpdateOneFileIntoES(path, excludeBase, Maps.newHashMap()), ThreadPoolConfig.IMPORT_DOC_POOL);
});
ThreadPoolUtil.shutdownAndAwait();
}
/**
* 将单个文件导入或更新到 Elasticsearch 并缓存元信息至 Redis
*
* 方法逻辑流程如下
* 1. 获取文件相对路径及最后修改时间
* 2. 检查 Redis 中的缓存元信息是否存在且未过期
* 3. 检查 ES 中是否已有相同文档且未变动
* 4. 若有变化或首次导入则提取文件内容构建文档
* 5. 将文档写入 ES并缓存元信息
* 6. 支持通过 extInfo 参数手动提供 uploaderurlexpireTime 信息用于无缓存情况
*
* @param absoluteFilePath 文件绝对路径
* @param excludeFilePrefix 排除前缀用于计算相对路径
* @param extInfo 可选附加元信息当缓存未命中时使用
* @return 导入是否成功
*/
@Override
public boolean insertOrUpdateOneFileIntoES(Path absoluteFilePath, Path excludeFilePrefix, Map<String, Object> extInfo) {
try {
// === Step 1: 计算相对路径 & 获取文件信息 ===
Path absPath = absoluteFilePath.toAbsolutePath().normalize();
Path relativePathObj = absPath.startsWith(excludeFilePrefix)
? excludeFilePrefix.relativize(absPath)
: absPath;
if (!absPath.startsWith(excludeFilePrefix)) {
logger.warn("路径未匹配 exclude.prefix使用全路径: {}", absPath);
}
Long localMTime = Files.getLastModifiedTime(absoluteFilePath).toMillis();
String localRelaFilePath = relativePathObj.toString().replace("\\", "/");
String fileNameWithSuffix = absoluteFilePath.getFileName().toString();
// === Step 2: 检查 Redis 缓存是否已是最新 ===
Optional<String> metaJsonOpt = fileCacheService.getMeta(localRelaFilePath);
if (metaJsonOpt.isPresent()) {
JSON metaJson = JSONUtil.parse(metaJsonOpt.get());
Long uploadTime = metaJson.getByPath(DocMetaPropEnum.UPLOAD_TIME.code, Long.class);
if (localMTime.equals(uploadTime)) {
logger.info("文件未变动,跳过导入: {}", localRelaFilePath);
return true;
}
fileCacheService.removeMetaCache(localRelaFilePath);
logger.info("[Redis] 已移除旧版本文件缓存Meta信息: {}", localRelaFilePath);
}
// === Step 3: 提取文件内容 ===
String content = extractContent(absoluteFilePath);
if (StrUtil.isBlank(content)) {
logger.warn("跳过空内容文件: {}", absoluteFilePath);
return true;
}
// === Step 4: 检查 ES 是否已有未变动版本 ===
String docId = SafeIdUtil.encode(localRelaFilePath);
GetResponse existing = esGateway.getIfExists(docId);
if(Objects.nonNull(existing)) {
Map<String, Object> existingSource = existing.getSourceAsMap();
Object esMtime = existingSource.get(FileEsField.MTIME);
if (esMtime != null && Long.parseLong(esMtime.toString()) == localMTime) {
logger.info("文件未变动,跳过导入: {}", localRelaFilePath);
fileCacheService.cacheMeta(localRelaFilePath, transToMetaJson(existingSource), ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true;
}
esGateway.deleteDoc(docId);
logger.info("[ES] 已删除旧版本文件: {}", localRelaFilePath);
}
logger.info("[开始进行文件导入......] localRelaFilePath: {} localMTime {}", localRelaFilePath, localMTime);
// === Step 5: 构建待写入文档 ===
FileEsModel fileEsModel = buildDocument(fileNameWithSuffix, localRelaFilePath, content, localMTime, extInfo);
// === Step 6: 写入 Elasticsearch 并更新 导入状态&缓存 ===
boolean saved = esGateway.saveDoc(fileEsModel);
logger.info("导入结束: relativePah: {}, status: {}", localRelaFilePath, saved);
fileDomainService.updateSearchableStatusByRelaPath(localRelaFilePath, SearchableStatusEnum.INIT.code, saved ? SearchableStatusEnum.SUCCESS.code : SearchableStatusEnum.FAILED.code);
String metaJson = transToMetaJson(BeanUtil.beanToMap(fileEsModel, false, true));
fileCacheService.cacheMeta(localRelaFilePath, metaJson, ConstantConfig.FILE_META_CACHE_EXPIRED_MINUTES);
return true;
} catch (Exception e) {
logger.error("导入失败: {}", absoluteFilePath, e);
return false;
}
}
private FileEsModel buildDocument(String filename, String filepath, String content, Long mtime, Map<String, Object> extInfo) {
String uploader = ConstantConfig.DEFAULT_UPLOADER;
String url = "";
Long expireTime = DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME);
Optional<String> metaJsonOpt = fileCacheService.getMeta(filepath);
if (metaJsonOpt.isPresent()) {
// 优先使用缓存
Map<String, Object> metaMap = JSONUtil.toBean(metaJsonOpt.get(), Map.class);
uploader = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.UPLOADER.code, String.class, uploader);
url = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.ACCESS_URL.code, String.class, url);
expireTime = CacheUtil.getFileMetaProp(metaMap, DocMetaPropEnum.EXPIRE_TIME.code, Long.class, expireTime);
} else {
// 使用传入 extInfo
extInfo = MapUtil.isEmpty(extInfo) ? MapUtil.empty() : extInfo;
boolean hasAllMeta = extInfo.keySet().containsAll(Lists.newArrayList(
DocMetaPropEnum.UPLOADER.code, DocMetaPropEnum.ACCESS_URL.code, DocMetaPropEnum.EXPIRE_TIME.code
));
if (hasAllMeta) {
uploader = (String) extInfo.get(DocMetaPropEnum.UPLOADER.code);
url = (String) extInfo.get(DocMetaPropEnum.ACCESS_URL.code);
expireTime = (Long) extInfo.get(DocMetaPropEnum.EXPIRE_TIME.code);
} else {
Map<String, Object> props = buildDocMetaProps(filepath);
uploader = (String) props.get(DocMetaPropEnum.UPLOADER.code);
url = (String) props.get(DocMetaPropEnum.ACCESS_URL.code);
expireTime = (Long) props.get(DocMetaPropEnum.EXPIRE_TIME.code);
}
}
return FileEsModel.builder()
.filename(filename)
.filepath(filepath)
.content(content)
.mtime(mtime)
.uploader(uploader)
.url(url)
.expireTime(expireTime)
.build();
}
private Map<String, Object> buildDocMetaProps(String localRelaFilePath) {
Map<String, Object> metaMap = new HashMap<>(Map.of(
DocMetaPropEnum.UPLOADER.code, ConstantConfig.DEFAULT_UPLOADER,
DocMetaPropEnum.ACCESS_URL.code, StrUtil.EMPTY,
DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(ConstantConfig.LONG_TERM_EXPIRE_TIME)
));
if(StrUtil.isBlank(localRelaFilePath)) {
return metaMap;
}
String accessUrl = localRelaFilePath;
if(localRelaFilePath.endsWith(".md")) {
// markdown直接拼接http链接
accessUrl = DocAccessUrlTool.buildMarkdownAccessUrl(localRelaFilePath);
metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl);
} else {
// 其它文件从对象存储的DB中获取
Optional<OSRecordDO> latestRecordOpt = fileDomainService.getLatestRecordByRelaPath(localRelaFilePath);
if(latestRecordOpt.isPresent()) {
OSRecordDO latestRecord = latestRecordOpt.get();
accessUrl = latestRecord.getUrl();
metaMap.put(DocMetaPropEnum.ACCESS_URL.code, accessUrl);
metaMap.put(DocMetaPropEnum.UPLOADER.code, latestRecord.getUploader());
metaMap.put(DocMetaPropEnum.EXPIRE_TIME.code, DateUtil.toMillis(latestRecord.getExpireTime()));
}
}
return metaMap;
}
private String transToMetaJson(Map<String, Object> esExistingSource) {
return JSONUtil.toJsonStr(Map.of(
DocMetaPropEnum.UPLOADER.code, Optional.ofNullable((String)esExistingSource.get(FileEsField.UPLOADER)).orElse(ConstantConfig.DEFAULT_UPLOADER),
DocMetaPropEnum.UPLOAD_TIME.code, esExistingSource.get(FileEsField.MTIME),
DocMetaPropEnum.ACCESS_URL.code, esExistingSource.get(FileEsField.URL),
DocMetaPropEnum.EXPIRE_TIME.code, esExistingSource.get(FileEsField.EXPIRE_TIME)
));
}
@Override
public String getType() {
return getDocTypeCode();
}
protected abstract String extractContent(Path path) throws IOException;
}

View File

@ -0,0 +1,84 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Set;
/**
* @author Luke.ye
* @date 2025/5/25 14:54
*/
@Component
public class ExcelImporter extends AbstractBaseFileImporter {
@Value("${excel.path}")
private String excelPath;
@Override
protected String getDirectoryPath() {
return excelPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".xlsx", ".xls");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.EXCEL.code;
}
@Override
protected String getExcludePrefix() {
return excelPath;
}
@Override
protected String extractContent(Path path) throws IOException {
String fileName = path.getFileName().toString().toLowerCase();
boolean isXlsx = fileName.endsWith(".xlsx");
try (FileInputStream fis = new FileInputStream(path.toFile());
Workbook workbook = isXlsx ? new XSSFWorkbook(fis) : new HSSFWorkbook(fis)) {
Sheet sheet = workbook.getSheetAt(0);
StringBuilder sb = new StringBuilder();
Iterator<Row> rowIterator = sheet.iterator();
if (rowIterator.hasNext()) rowIterator.next(); // 跳过表头
while (rowIterator.hasNext()) {
Row row = rowIterator.next();
String title = getCellString(row.getCell(0));
String content = getCellString(row.getCell(1));
if (!title.isBlank()) {
sb.append("【标题】").append(title).append("\n");
}
if (!content.isBlank()) {
sb.append(content).append("\n\n");
}
}
return sb.toString().trim();
}
}
private String getCellString(Cell cell) {
if (cell == null) return "";
cell.setCellType(CellType.STRING);
return cell.getStringCellValue().trim();
}
}

View File

@ -0,0 +1,49 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.Set;
/**
* @author Luke.ye
* @date 2025/5/20 09:06
*/
@Component
public class MarkdownImporter extends AbstractBaseFileImporter {
@Value("${markdown.path}")
private String directoryPath;
@Value("${exclude.file.path.prefix}")
private String excludePrefix;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".md");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.MARKDOWN.code;
}
@Override
protected String getExcludePrefix() {
return excludePrefix;
}
@Override
protected String extractContent(Path path) throws IOException {
return Files.readString(path, StandardCharsets.UTF_8);
}
}

View File

@ -0,0 +1,50 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.stereotype.Component;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.springframework.beans.factory.annotation.Value;
import java.io.IOException;
import java.nio.file.*;
import java.util.Set;
/**
* @author Luke.ye
* @date 2025/5/20 10:23
*/
@Component
public class PdfImporter extends AbstractBaseFileImporter {
@Value("${pdf.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".pdf");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.PDF.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
@Override
protected String extractContent(Path path) throws IOException {
try (PDDocument document = PDDocument.load(path.toFile())) {
return new PDFTextStripper().getText(document);
}
}
}

View File

@ -0,0 +1,96 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import cn.hutool.core.util.StrUtil;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFSlide;
import org.apache.poi.hslf.usermodel.HSLFTextParagraph;
import org.apache.poi.xslf.usermodel.XMLSlideShow;
import org.apache.poi.xslf.usermodel.XSLFSlide;
import org.apache.poi.xslf.usermodel.XSLFTextShape;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* PowerPoint 文件导入器支持 .ppt .pptx 文件
* 利用 Apache POI 提取幻灯片中的所有文本内容
*/
@Component
public class PowerPointImporter extends AbstractBaseFileImporter {
@Value("${ppt.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".ppt", ".pptx");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.PPT.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
/**
* 提取 PowerPoint 文件中的全部文字内容
*
* @param path PPT 文件路径
* @return 幻灯片中全部文字拼接而成的字符串
* @throws IOException 文件读取异常
*/
@Override
protected String extractContent(Path path) throws IOException {
String fileName = path.getFileName().toString().toLowerCase();
try (FileInputStream fis = new FileInputStream(path.toFile())) {
if (fileName.endsWith(".pptx")) {
XMLSlideShow pptx = new XMLSlideShow(fis);
StringBuilder content = new StringBuilder();
for (XSLFSlide slide : pptx.getSlides()) {
slide.getShapes().stream()
.filter(shape -> shape instanceof XSLFTextShape)
.map(shape -> (XSLFTextShape) shape)
.map(XSLFTextShape::getText)
.filter(StrUtil::isNotBlank)
.forEach(text -> content.append(text.trim()).append("\n"));
}
return content.toString();
} else if (fileName.endsWith(".ppt")) {
HSLFSlideShow ppt = new HSLFSlideShow(fis);
StringBuilder content = new StringBuilder();
for (HSLFSlide slide : ppt.getSlides()) {
List<List<HSLFTextParagraph>> textParagraphs = slide.getTextParagraphs();
List<HSLFTextParagraph> paragraphs = textParagraphs.stream()
.flatMap(List::stream)
.collect(Collectors.toList());
String slideText = HSLFTextParagraph.getRawText(paragraphs);
if (StrUtil.isNotBlank(slideText)) {
content.append(slideText.trim()).append("\n");
}
}
return content.toString();
} else {
throw new IllegalArgumentException("Unsupported PowerPoint format: " + fileName);
}
}
}
}

View File

@ -0,0 +1,49 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Set;
/**
* 文本文件导入器支持 .txt 格式
* 内容按 UTF-8 编码读取无结构提取仅用于全文索引
*/
@Component
public class TxtImporter extends AbstractBaseFileImporter {
@Value("${txt.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".txt");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.TXT.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
@Override
protected String extractContent(Path path) throws IOException {
// 读取 UTF-8 编码文本去除 BOM 和首尾空白
String content = Files.readString(path, StandardCharsets.UTF_8);
return content.replace("\uFEFF", "").trim();
}
}

View File

@ -0,0 +1,72 @@
package com.knowledge.base.domain.doc.service.impl.importer;
import com.knowledge.base.domain.common.enums.DocTypeEnum;
import org.apache.poi.hwpf.HWPFDocument;
import org.springframework.stereotype.Component;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.springframework.beans.factory.annotation.Value;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.*;
import java.util.Set;
/**
* @author Luke.ye
* @date 2025/5/20 10:25
*/
@Component
public class WordImporter extends AbstractBaseFileImporter {
@Value("${word.path}")
private String directoryPath;
@Override
protected String getDirectoryPath() {
return directoryPath;
}
@Override
public Set<String> getFileSuffixes() {
return Set.of(".doc", ".docx");
}
@Override
protected String getDocTypeCode() {
return DocTypeEnum.WORD.code;
}
@Override
protected String getExcludePrefix() {
return directoryPath;
}
@Override
protected String extractContent(Path path) throws IOException {
String fileName = path.getFileName().toString().toLowerCase();
if (fileName.endsWith(".docx")) {
// 处理 .docx
try (FileInputStream fis = new FileInputStream(path.toFile());
XWPFDocument document = new XWPFDocument(fis)) {
StringBuilder content = new StringBuilder();
for (XWPFParagraph para : document.getParagraphs()) {
content.append(para.getText()).append("\n");
}
return content.toString().trim();
}
} else if (fileName.endsWith(".doc")) {
// 处理 .doc
try (FileInputStream fis = new FileInputStream(path.toFile());
HWPFDocument doc = new HWPFDocument(fis)) {
return doc.getDocumentText().trim();
}
} else {
throw new IllegalArgumentException("不支持的 Word 文件类型: " + fileName);
}
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/9 10:37
*/
@Data
public class RoleDO {
private Long id;
private String roleCode;
private String roleName;
private LocalDateTime addTime;
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,19 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/9 10:39
*/
@Data
public class RoleFileRuleDO {
private Long id;
private Long roleId;
private String filePattern;
private String remark;
private LocalDateTime addTime;
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,19 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/9 10:36
*/
@Data
public class UserDO {
private Long id;
private String username;
private String password;
private LocalDateTime addTime;
// 领域层需要的字段可适当扩展
}

View File

@ -0,0 +1,14 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
/**
* @author Luke.ye
* @date 2025/6/9 10:38
*/
@Data
public class UserFileDO {
public Long id;
public Long userId;
public Long fileId;
}

View File

@ -0,0 +1,14 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
/**
* @author Luke.ye
* @date 2025/6/9 10:38
*/
@Data
public class UserRoleDO {
public Long id;
public Long userId;
public Long roleId;
}

View File

@ -0,0 +1,18 @@
package com.knowledge.base.domain.user.model;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/9 10:40
*/
@Data
public class UserTokenDO {
public Long id;
public Long userId;
public String token;
public LocalDateTime loginAt;
public LocalDateTime expiredAt;
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.Role;
import com.knowledge.base.domain.user.model.RoleDO;
import org.springframework.beans.BeanUtils;
public class RoleDomainConverter {
public static RoleDO toDO(Role po) {
if (po == null) return null;
RoleDO doObj = new RoleDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static Role toPO(RoleDO doObj) {
if (doObj == null) return null;
Role po = new Role();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.RoleFileRule;
import com.knowledge.base.domain.user.model.RoleFileRuleDO;
import org.springframework.beans.BeanUtils;
public class RoleFileRuleDomainConverter {
public static RoleFileRuleDO toDO(RoleFileRule po) {
if (po == null) return null;
RoleFileRuleDO doObj = new RoleFileRuleDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static RoleFileRule toPO(RoleFileRuleDO doObj) {
if (doObj == null) return null;
RoleFileRule po = new RoleFileRule();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.User;
import com.knowledge.base.domain.user.model.UserDO;
import org.springframework.beans.BeanUtils;
public class UserDomainConverter {
public static UserDO toDO(User po) {
if (po == null) return null;
UserDO doObj = new UserDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static User toPO(UserDO doObj) {
if (doObj == null) return null;
User po = new User();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.UserFile;
import com.knowledge.base.domain.user.model.UserFileDO;
import org.springframework.beans.BeanUtils;
public class UserFileDomainConverter {
public static UserFileDO toDO(UserFile po) {
if (po == null) return null;
UserFileDO doObj = new UserFileDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static UserFile toPO(UserFileDO doObj) {
if (doObj == null) return null;
UserFile po = new UserFile();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.UserRole;
import com.knowledge.base.domain.user.model.UserRoleDO;
import org.springframework.beans.BeanUtils;
public class UserRoleDomainConverter {
public static UserRoleDO toDO(UserRole po) {
if (po == null) return null;
UserRoleDO doObj = new UserRoleDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static UserRole toPO(UserRoleDO doObj) {
if (doObj == null) return null;
UserRole po = new UserRole();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.converter;
import com.knowledge.base.domain.user.repository.po.UserToken;
import com.knowledge.base.domain.user.model.UserTokenDO;
import org.springframework.beans.BeanUtils;
public class UserTokenDomainConverter {
public static UserTokenDO toDO(UserToken po) {
if (po == null) return null;
UserTokenDO doObj = new UserTokenDO();
BeanUtils.copyProperties(po, doObj);
return doObj;
}
public static UserToken toPO(UserTokenDO doObj) {
if (doObj == null) return null;
UserToken po = new UserToken();
BeanUtils.copyProperties(doObj, po);
return po;
}
}

View File

@ -0,0 +1,14 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.RoleFileRule;
import java.util.List;
public interface RoleFileRuleRepository {
boolean save(RoleFileRule rule);
boolean update(RoleFileRule rule);
boolean deleteById(Long id);
RoleFileRule findById(Long id);
List<RoleFileRule> findByRoleId(Long roleId);
List<RoleFileRule> findAll();
}

View File

@ -0,0 +1,13 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.Role;
import java.util.List;
public interface RoleRepository {
boolean save(Role role);
boolean update(Role role);
boolean deleteById(Long id);
Role findById(Long id);
List<Role> findAll();
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.UserFile;
import java.util.List;
public interface UserFileRepository {
boolean save(UserFile userFile);
boolean update(UserFile userFile);
boolean deleteById(Long id);
boolean deleteByUserIdAndFileId(Long userId, Long fileId);
UserFile findById(Long id);
List<UserFile> findByUserId(Long userId);
List<UserFile> findAll();
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.User;
import java.util.Optional;
import java.util.List;
public interface UserRepository {
boolean save(User user);
boolean update(User user);
boolean deleteById(Long id);
Optional<User> findById(Long id);
Optional<User> findByUsername(String username);
List<User> findAll();
}

View File

@ -0,0 +1,20 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.UserRole;
import java.util.Collection;
import java.util.List;
public interface UserRoleRepository {
boolean save(UserRole userRole);
boolean update(UserRole userRole);
boolean deleteById(Long id);
boolean deleteByUserIdAndRoleId(Long userId, Long roleId);
UserRole findById(Long id);
List<UserRole> findByUserId(Long userId);
List<UserRole> findAll();
List<UserRole> findByUserIdIn(List<Long> userIds);
boolean removeUserRoles(Long userId);
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.user.repository.iface;
import com.knowledge.base.domain.user.repository.po.UserToken;
import java.util.Optional;
import java.util.List;
public interface UserTokenRepository {
boolean save(UserToken userToken);
boolean deleteById(Long id);
void removeByToken(String token);
Optional<UserToken> findById(Long id);
Optional<UserToken> findByToken(String token);
List<UserToken> findByUserId(Long userId);
}

View File

@ -0,0 +1,27 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 角色表
*/
@Data
@TableName("role")
public class Role {
@TableId(type = IdType.AUTO)
private Long id;
private String roleCode;
private String roleName;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,29 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 角色-文件可访问规则表
*/
@Data
@TableName("role_file_rule")
public class RoleFileRule {
@TableId(type = IdType.AUTO)
private Long id;
private Long roleId;
private String filePattern;
private String remark;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,27 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* @author Luke.ye
* @date 2025/6/6 12:16
*/
@Data
@TableName("user")
public class User {
@TableId(type = IdType.AUTO)
private Long id;
private String username;
private String password;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,27 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 用户-文件直接授权关联表
*/
@Data
@TableName("user_file")
public class UserFile {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private Long fileId;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,27 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* 用户-角色关联表
*/
@Data
@TableName("user_role")
public class UserRole {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private Long roleId;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,30 @@
package com.knowledge.base.domain.user.repository.po;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("user_token")
public class UserToken {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String token;
@TableField("login_at")
private LocalDateTime loginAt;
@TableField("expired_at")
private LocalDateTime expiredAt;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime addTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.RoleDO;
import com.knowledge.base.domain.user.repository.po.Role;
import java.util.List;
import java.util.Optional;
public interface RoleDomainSupport {
Optional<RoleDO> findById(Long id);
List<RoleDO> findAll();
boolean addRole(RoleDO role);
boolean updateRole(RoleDO role);
boolean deleteRole(Long id);
}

View File

@ -0,0 +1,15 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.RoleFileRuleDO;
import java.util.List;
public interface RoleFileRuleDomainSupport {
List<RoleFileRuleDO> findByRoleId(Long roleId);
boolean addRoleFileRule(RoleFileRuleDO rule);
boolean update(RoleFileRuleDO rule);
boolean removeRoleFileRule(Long ruleId);
List<RoleFileRuleDO> listAllRules();
}

View File

@ -0,0 +1,17 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.UserTokenDO;
import com.knowledge.base.domain.user.repository.po.UserToken;
import java.util.Optional;
public interface UserAuthDomainSupport {
boolean verifyPassword(String username, String plainText);
boolean changePassword(Long userId, String oldPassword, String newPassword);
UserTokenDO createToken(Long userId, long expireMs);
boolean isValidToken(String token);
void removeToken(String token);
Optional<UserTokenDO> findToken(String token);
}

View File

@ -0,0 +1,55 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.*;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public interface UserDomainService {
// 用户资料
Optional<UserDO> findByUsername(String username);
Optional<UserDO> findById(Long id);
List<UserDO> findAllUsers();
boolean registerUser(String username, String plainPassword);
boolean deleteUser(Long id);
boolean updateUser(UserDO user);
// 认证/Token
Optional<UserTokenDO> findToken(String token);
boolean verifyPassword(String username, String plainText);
boolean changePassword(Long userId, String oldPassword, String newPassword);
UserTokenDO createToken(Long userId, long expireMs);
boolean isValidToken(String token);
void removeToken(String token);
// 角色
List<RoleDO> findAllRoles();
Optional<RoleDO> findRoleById(Long id);
boolean addRole(RoleDO role);
boolean updateRole(RoleDO role);
boolean deleteRole(Long id);
// 用户-角色
List<UserRoleDO> findRolesByUserId(Long userId);
Map<Long, List<UserRoleDO>> batchFindRolesByUserIds(List<Long> userIds);
boolean batchAddUserRoles(Long userId, List<Long> roleIds);
boolean removeUserRoles(Long userId);
boolean addUserRole(Long userId, Long roleId);
boolean removeUserRole(Long userId, Long roleId);
// 角色-文件规则
List<RoleFileRuleDO> findRoleFileRules(Long roleId);
List<RoleFileRuleDO> listAllRules();
boolean addRoleFileRule(RoleFileRuleDO rule);
boolean updateRoleFileRule(RoleFileRuleDO rule);
boolean removeRoleFileRule(Long ruleId);
// 用户特殊授权
List<UserFileDO> findUserFiles(Long userId);
boolean addUserFileAuth(Long userId, Long fileId);
boolean removeUserFileAuth(Long userId, Long fileId);
}

View File

@ -0,0 +1,12 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.UserFileDO;
import com.knowledge.base.domain.user.repository.po.UserFile;
import java.util.List;
public interface UserFileDomainSupport {
List<UserFileDO> findByUserId(Long userId);
boolean addUserFileAuth(Long userId, Long fileId);
boolean removeUserFileAuth(Long userId, Long fileId);
}

View File

@ -0,0 +1,16 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.UserDO;
import com.knowledge.base.domain.user.repository.po.User;
import java.util.Optional;
import java.util.List;
public interface UserProfileDomainSupport {
Optional<UserDO> findByUsername(String username);
Optional<UserDO> findById(Long id);
List<UserDO> findAll();
boolean registerUser(String username, String plainPassword);
boolean updateUser(UserDO user);
boolean deleteUser(Long id);
}

View File

@ -0,0 +1,19 @@
package com.knowledge.base.domain.user.service.iface;
import com.knowledge.base.domain.user.model.UserRoleDO;
import com.knowledge.base.domain.user.repository.po.UserRole;
import java.util.List;
import java.util.Map;
public interface UserRoleDomainSupport {
List<UserRoleDO> findByUserId(Long userId);
boolean addUserRole(Long userId, Long roleId);
boolean removeUserRole(Long userId, Long roleId);
Map<Long, List<UserRoleDO>> batchFindRolesByUserIds(List<Long> userIds);
boolean batchAddUserRoles(Long userId, List<Long> roleIds);
boolean removeUserRoles(Long userId);
}

View File

@ -0,0 +1,49 @@
package com.knowledge.base.domain.user.service.impl;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.user.model.RoleDO;
import com.knowledge.base.domain.user.repository.converter.RoleDomainConverter;
import com.knowledge.base.domain.user.repository.iface.RoleRepository;
import com.knowledge.base.domain.user.service.iface.RoleDomainSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class RoleDomainSupportImpl implements RoleDomainSupport {
private final RoleRepository roleRepository;
@Override
public Optional<RoleDO> findById(Long id) {
return Optional.ofNullable(
RoleDomainConverter.toDO(roleRepository.findById(id))
);
}
@Override
public List<RoleDO> findAll() {
return Optional.ofNullable(roleRepository.findAll()).orElse(Lists.newArrayList()).stream()
.map(e -> RoleDomainConverter.toDO(e))
.collect(Collectors.toList());
}
@Override
public boolean addRole(RoleDO role) {
return roleRepository.save(RoleDomainConverter.toPO(role));
}
@Override
public boolean updateRole(RoleDO role) {
return roleRepository.update(RoleDomainConverter.toPO(role));
}
@Override
public boolean deleteRole(Long id) {
return roleRepository.deleteById(id);
}
}

View File

@ -0,0 +1,52 @@
package com.knowledge.base.domain.user.service.impl;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.user.model.RoleFileRuleDO;
import com.knowledge.base.domain.user.repository.converter.RoleFileRuleDomainConverter;
import com.knowledge.base.domain.user.repository.iface.RoleFileRuleRepository;
import com.knowledge.base.domain.user.repository.po.RoleFileRule;
import com.knowledge.base.domain.user.service.iface.RoleFileRuleDomainSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class RoleFileRuleDomainSupportImpl implements RoleFileRuleDomainSupport {
private final RoleFileRuleRepository roleFileRuleRepository;
@Override
public List<RoleFileRuleDO> findByRoleId(Long roleId) {
return Optional.ofNullable(roleFileRuleRepository.findByRoleId(roleId)).orElse(Lists.newArrayList()).stream()
.map(e -> RoleFileRuleDomainConverter.toDO(e))
.collect(Collectors.toList());
}
@Override
public boolean addRoleFileRule(RoleFileRuleDO rule) {
return roleFileRuleRepository.save(RoleFileRuleDomainConverter.toPO(rule));
}
@Override
public boolean update(RoleFileRuleDO rule) {
return roleFileRuleRepository.update(RoleFileRuleDomainConverter.toPO(rule));
}
@Override
public boolean removeRoleFileRule(Long ruleId) {
return roleFileRuleRepository.deleteById(ruleId);
}
@Override
public List<RoleFileRuleDO> listAllRules() {
return Optional.ofNullable(roleFileRuleRepository.findAll()).orElse(Lists.newArrayList()).stream()
.map(e -> RoleFileRuleDomainConverter.toDO(e))
.collect(Collectors.toList());
}
}

View File

@ -0,0 +1,76 @@
package com.knowledge.base.domain.user.service.impl;
import com.knowledge.base.domain.common.exception.DomainException;
import com.knowledge.base.domain.user.model.UserTokenDO;
import com.knowledge.base.domain.user.repository.converter.UserTokenDomainConverter;
import com.knowledge.base.domain.user.repository.iface.UserRepository;
import com.knowledge.base.domain.user.repository.iface.UserTokenRepository;
import com.knowledge.base.domain.user.repository.po.User;
import com.knowledge.base.domain.user.repository.po.UserToken;
import com.knowledge.base.domain.user.service.iface.UserAuthDomainSupport;
import com.knowledge.base.infrastructure.util.crypto.SM3Util;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.UUID;
@Service
@RequiredArgsConstructor
public class UserAuthDomainSupportImpl implements UserAuthDomainSupport {
private final UserRepository userRepository;
private final UserTokenRepository userTokenRepository;
@Override
public boolean verifyPassword(String username, String plainText) {
Optional<User> userOpt = userRepository.findByUsername(username);
if (userOpt.isEmpty()) return false;
User user = userOpt.get();
String inputHash = SM3Util.digest(plainText);
return inputHash.equals(user.getPassword());
}
@Override
public boolean changePassword(Long userId, String oldPassword, String newPassword) {
Optional<User> userOpt = userRepository.findById(userId);
if(userOpt.isEmpty()) return false;
User user = userOpt.get();
String oldInputHash = SM3Util.digest(oldPassword);
String newInputHash = SM3Util.digest(newPassword);
if(!oldInputHash.equals(user.getPassword())) {
throw new DomainException("旧密码错误!");
}
user.setPassword(newInputHash);
userRepository.update(user);
return true;
}
@Override
public UserTokenDO createToken(Long userId, long expireMs) {
String token = UUID.randomUUID().toString().replace("-", "");
UserToken userToken = new UserToken();
userToken.setUserId(userId);
userToken.setToken(token);
userToken.setLoginAt(LocalDateTime.now());
userToken.setExpiredAt(LocalDateTime.now().plusSeconds(expireMs / 1000));
userTokenRepository.save(userToken);
return UserTokenDomainConverter.toDO(userToken);
}
@Override
public boolean isValidToken(String token) {
Optional<UserToken> userTokenOpt = userTokenRepository.findByToken(token);
return userTokenOpt.filter(tk -> tk.getExpiredAt().isAfter(LocalDateTime.now())).isPresent();
}
@Override
public void removeToken(String token) {
userTokenRepository.removeByToken(token);
}
@Override
public Optional<UserTokenDO> findToken(String token) {
return Optional.ofNullable(UserTokenDomainConverter.toDO(userTokenRepository.findByToken(token).get()));
}
}

View File

@ -0,0 +1,183 @@
package com.knowledge.base.domain.user.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.map.MapUtil;
import com.knowledge.base.domain.user.model.*;
import com.knowledge.base.domain.user.service.iface.*;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Service
@RequiredArgsConstructor
public class UserDomainServiceImpl implements UserDomainService {
private final UserProfileDomainSupport userProfileDomainSupport;
private final UserAuthDomainSupport userAuthDomainSupport;
private final UserRoleDomainSupport userRoleDomainSupport;
private final RoleDomainSupport roleDomainSupport;
private final RoleFileRuleDomainSupport roleFileRuleDomainSupport;
private final UserFileDomainSupport userFileDomainSupport;
// 用户资料
@Override
public Optional<UserDO> findByUsername(String username) {
return userProfileDomainSupport.findByUsername(username);
}
@Override
public Optional<UserDO> findById(Long id) {
return userProfileDomainSupport.findById(id);
}
@Override
public List<UserDO> findAllUsers() {
return userProfileDomainSupport.findAll();
}
@Override
public boolean registerUser(String username, String plainPassword) {
userProfileDomainSupport.registerUser(username, plainPassword);
return true;
}
@Override
public boolean updateUser(UserDO userDO) {
return userProfileDomainSupport.updateUser(userDO);
}
@Override
public boolean deleteUser(Long id) {
return userProfileDomainSupport.deleteUser(id);
}
// 认证/Token
@Override
public Optional<UserTokenDO> findToken(String token) {
return userAuthDomainSupport.findToken(token);
}
@Override
public boolean verifyPassword(String username, String plainText) {
return userAuthDomainSupport.verifyPassword(username, plainText);
}
@Override
public boolean changePassword(Long userId, String oldPassword, String newPassword) {
return userAuthDomainSupport.changePassword(userId, oldPassword, newPassword);
}
@Override
public UserTokenDO createToken(Long userId, long expireMs) {
return userAuthDomainSupport.createToken(userId, expireMs);
}
@Override
public boolean isValidToken(String token) {
return userAuthDomainSupport.isValidToken(token);
}
@Override
public void removeToken(String token) {
userAuthDomainSupport.removeToken(token);
}
// 角色
@Override
public List<RoleDO> findAllRoles() {
return roleDomainSupport.findAll();
}
@Override
public Optional<RoleDO> findRoleById(Long id) {
return roleDomainSupport.findById(id);
}
@Override
public boolean addRole(RoleDO role) {
return roleDomainSupport.addRole(role);
}
@Override
public boolean updateRole(RoleDO role) {
return roleDomainSupport.updateRole(role);
}
@Override
public boolean deleteRole(Long id) {
return roleDomainSupport.deleteRole(id);
}
// 用户-角色
@Override
public List<UserRoleDO> findRolesByUserId(Long userId) {
return userRoleDomainSupport.findByUserId(userId);
}
@Override
public Map<Long, List<UserRoleDO>> batchFindRolesByUserIds(List<Long> userIds) {
if(CollectionUtil.isEmpty(userIds)) {
return MapUtil.newHashMap();
}
return userRoleDomainSupport.batchFindRolesByUserIds(userIds);
}
@Override
public boolean batchAddUserRoles(Long userId, List<Long> roleIds) {
return userRoleDomainSupport.batchAddUserRoles(userId, roleIds);
}
@Override
public boolean removeUserRoles(Long userId) {
return userRoleDomainSupport.removeUserRoles(userId);
}
@Override
public boolean addUserRole(Long userId, Long roleId) {
return userRoleDomainSupport.addUserRole(userId, roleId);
}
@Override
public boolean removeUserRole(Long userId, Long roleId) {
return userRoleDomainSupport.removeUserRole(userId, roleId);
}
// 角色-文件规则
@Override
public List<RoleFileRuleDO> findRoleFileRules(Long roleId) {
return roleFileRuleDomainSupport.findByRoleId(roleId);
}
@Override
public List<RoleFileRuleDO> listAllRules() {
return roleFileRuleDomainSupport.listAllRules();
}
@Override
public boolean addRoleFileRule(RoleFileRuleDO rule) {
return roleFileRuleDomainSupport.addRoleFileRule(rule);
}
@Override
public boolean updateRoleFileRule(RoleFileRuleDO rule) {
return roleFileRuleDomainSupport.update(rule);
}
@Override
public boolean removeRoleFileRule(Long ruleId) {
return roleFileRuleDomainSupport.removeRoleFileRule(ruleId);
}
// 用户特殊授权
@Override
public List<UserFileDO> findUserFiles(Long userId) {
return userFileDomainSupport.findByUserId(userId);
}
@Override
public boolean addUserFileAuth(Long userId, Long fileId) {
return userFileDomainSupport.addUserFileAuth(userId, fileId);
}
@Override
public boolean removeUserFileAuth(Long userId, Long fileId) {
return userFileDomainSupport.removeUserFileAuth(userId, fileId);
}
}

View File

@ -0,0 +1,41 @@
package com.knowledge.base.domain.user.service.impl;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.user.model.UserFileDO;
import com.knowledge.base.domain.user.repository.converter.UserFileDomainConverter;
import com.knowledge.base.domain.user.repository.iface.UserFileRepository;
import com.knowledge.base.domain.user.repository.po.UserFile;
import com.knowledge.base.domain.user.service.iface.UserFileDomainSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class UserFileDomainSupportImpl implements UserFileDomainSupport {
private final UserFileRepository userFileRepository;
@Override
public List<UserFileDO> findByUserId(Long userId) {
return Optional.ofNullable(userFileRepository.findByUserId(userId)).orElse(Lists.newArrayList()).stream()
.map(e -> UserFileDomainConverter.toDO(e))
.collect(Collectors.toList());
}
@Override
public boolean addUserFileAuth(Long userId, Long fileId) {
UserFile userFile = new UserFile();
userFile.setUserId(userId);
userFile.setFileId(fileId);
return userFileRepository.save(userFile);
}
@Override
public boolean removeUserFileAuth(Long userId, Long fileId) {
return userFileRepository.deleteByUserIdAndFileId(userId, fileId);
}
}

View File

@ -0,0 +1,57 @@
package com.knowledge.base.domain.user.service.impl;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.user.model.UserDO;
import com.knowledge.base.domain.user.repository.converter.UserDomainConverter;
import com.knowledge.base.domain.user.repository.iface.UserRepository;
import com.knowledge.base.domain.user.repository.po.User;
import com.knowledge.base.domain.user.service.iface.UserProfileDomainSupport;
import com.knowledge.base.infrastructure.util.crypto.SM3Util;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class UserProfileDomainSupportImpl implements UserProfileDomainSupport {
private final UserRepository userRepository;
@Override
public Optional<UserDO> findByUsername(String username) {
return Optional.ofNullable(UserDomainConverter.toDO(userRepository.findByUsername(username).get()));
}
@Override
public Optional<UserDO> findById(Long id) {
return Optional.ofNullable(UserDomainConverter.toDO(userRepository.findById(id).get()));
}
@Override
public List<UserDO> findAll() {
return Optional.ofNullable(userRepository.findAll()).orElse(Lists.newArrayList()).stream()
.map(e -> UserDomainConverter.toDO(e))
.collect(Collectors.toList());
}
@Override
public boolean registerUser(String username, String plainPassword) {
if (userRepository.findByUsername(username).isPresent()) return false;
User user = new User();
user.setUsername(username);
user.setPassword(SM3Util.digest(plainPassword));
return userRepository.save(user);
}
@Override
public boolean updateUser(UserDO user) {
return userRepository.update(UserDomainConverter.toPO(user));
}
@Override
public boolean deleteUser(Long id) {
return userRepository.deleteById(id);
}
}

View File

@ -0,0 +1,64 @@
package com.knowledge.base.domain.user.service.impl;
import com.google.common.collect.Lists;
import com.knowledge.base.domain.user.model.UserRoleDO;
import com.knowledge.base.domain.user.repository.converter.UserRoleDomainConverter;
import com.knowledge.base.domain.user.repository.iface.UserRoleRepository;
import com.knowledge.base.domain.user.repository.po.UserRole;
import com.knowledge.base.domain.user.service.iface.UserRoleDomainSupport;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class UserRoleDomainSupportImpl implements UserRoleDomainSupport {
private final UserRoleRepository userRoleRepository;
@Override
public List<UserRoleDO> findByUserId(Long userId) {
return Optional.ofNullable(userRoleRepository.findByUserId(userId)).orElse(Lists.newArrayList()).stream()
.map(e -> UserRoleDomainConverter.toDO(e))
.collect(Collectors.toList());
}
@Override
public boolean addUserRole(Long userId, Long roleId) {
UserRole userRole = new UserRole();
userRole.setUserId(userId);
userRole.setRoleId(roleId);
return userRoleRepository.save(userRole);
}
@Override
public boolean removeUserRole(Long userId, Long roleId) {
return userRoleRepository.deleteByUserIdAndRoleId(userId, roleId);
}
@Override
public Map<Long, List<UserRoleDO>> batchFindRolesByUserIds(List<Long> userIds) {
return userRoleRepository.findByUserIdIn(userIds).stream()
.map(e -> UserRoleDomainConverter.toDO(e))
.collect(Collectors.groupingBy(UserRoleDO::getUserId));
}
@Override
public boolean batchAddUserRoles(Long userId, List<Long> roleIds) {
boolean res = true;
for (Long roleId : roleIds) {
boolean b = addUserRole(userId, roleId);
res = res && b;
}
return res;
}
@Override
public boolean removeUserRoles(Long userId) {
return userRoleRepository.removeUserRoles(userId);
}
}

View File

@ -0,0 +1,41 @@
package com.knowledge.base.infrastructure.cache.iface;
import java.util.Collection;
import java.util.Optional;
/**
* 文件缓存服务接口支持 Redis 或本地实现
*
* @author Luke.ye
* @date 2025/6/7
*/
public interface FileCacheService {
/**
* 缓存文件名与文件相关的元信息
* @param fnWithRelativePath 文件名带相对路径和后缀
* @param jsonMeta 文件元信息包含上传人可访问链接可访问链接过期时间文件写入本地时间等
* @param expireMinutes 过期时间分钟
*/
default void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {};
/**
* 获取文件名对应的元信息
* @param fnWithRelativePath
* @return
*/
default Optional<String> getMeta(String fnWithRelativePath) { return null; };
/**
* 移除指定key对应对的缓存
* @param fnWithRelativePath
*/
default void removeMetaCache(String fnWithRelativePath) {};
default public void removeMetaCacheBatch(Collection<String> fileRelativePaths) {}
/**
* 清空所有缓存注意某些实现可能未实现
*/
void clearAll();
}

View File

@ -0,0 +1,30 @@
package com.knowledge.base.infrastructure.cache.iface;
import java.util.Optional;
/**
* @author Luke.ye
* @date 2025/6/19 09:59
*/
public interface UserCacheService {
/**
* 根据token获取User信息
*
* @param token
* @return UserDTO的Json
*/
Optional<String> getUserJsonByToken(String token);
/**
* 将用户信息缓存
* @param token
* @param userJsonStr
*/
void cacheUserJsonByToken(String token, String userJsonStr);
default void cacheAnythingLLMSlugId(String wsName, String slugId) {}
default Optional<String> getAnythingLLMSlugIdByWorkspaceName(String wsName) {return Optional.empty();}
}

View File

@ -0,0 +1,45 @@
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.FileCacheService;
import com.knowledge.base.infrastructure.util.LocalCacheUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Optional;
/**
* 使用本地缓存实现的文件缓存服务 Redis
*
* @author Luke.ye
* @date 2025/6/7
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "false")
public class LocalFileCacheServiceImpl implements FileCacheService {
@Value("${knowledge.base.local-cache.expire-minutes:1440}")
private long expireMinutes;
private String key(String prefix, String key) {
return String.format("kb:file:%s:%s", prefix, key);
}
@Override
public void cacheMeta(String fnWithRelativePath, String jsonMeta, long expireMinutes) {
LocalCacheUtil.put(key("meta", fnWithRelativePath), jsonMeta);
}
@Override
public Optional<String> getMeta(String fnWithRelativePath) {
Object val = LocalCacheUtil.get(key("meta", fnWithRelativePath));
return val instanceof String ? Optional.of((String) val) : Optional.empty();
}
@Override
public void clearAll() {
LocalCacheUtil.clearAll();
}
}

View File

@ -0,0 +1,27 @@
package com.knowledge.base.infrastructure.cache.impl;
import com.knowledge.base.infrastructure.cache.iface.UserCacheService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Optional;
/**
* @author Luke.ye
* @date 2025/6/19 10:01
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "knowledge.base.redis", name = "enable", havingValue = "false")
public class LocalUserCacheServiceImpl implements UserCacheService {
@Override
public Optional<String> getUserJsonByToken(String token) {
return Optional.empty();
}
@Override
public void cacheUserJsonByToken(String token, String userJsonStr) {
}
}

Some files were not shown because too many files have changed in this diff Show More