Merge branch 'main' into develop/hsc/clean

# Conflicts:
#	backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/entity/CleaningTask.java
#	scripts/db/data-cleaning-init.sql

Author: hhhhsc701

README-zh.md

@@ -8,6 +8,7 @@
 ![GitHub Forks](https://img.shields.io/github/forks/ModelEngine-Group/DataMate)
 ![GitHub Issues](https://img.shields.io/github/issues/ModelEngine-Group/DataMate)
 ![GitHub License](https://img.shields.io/github/license/ModelEngine-Group/datamate-docs)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ModelEngine-Group/DataMate)
 
 **DataMate是面向模型微调与RAG检索的企业级数据处理平台，支持数据归集、数据管理、算子市场、数据清洗、数据合成、数据标注、数据评估、知识生成等核心功能。**
 

README.md

@@ -8,6 +8,7 @@
 ![GitHub Forks](https://img.shields.io/github/forks/ModelEngine-Group/DataMate)
 ![GitHub Issues](https://img.shields.io/github/issues/ModelEngine-Group/DataMate)
 ![GitHub License](https://img.shields.io/github/license/ModelEngine-Group/datamate-docs)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ModelEngine-Group/DataMate)
 
 **DataMate is an enterprise-level data processing platform for model fine-tuning and RAG retrieval, supporting core
 functions such as data collection, data management, operator marketplace, data cleaning, data synthesis, data

…way/common/filter/UserContextFilter.java …te/gateway/common/filter/AuthFilter.javabackend/api-gateway/src/main/java/com/datamate/gateway/common/filter/UserContextFilter.java renamed to backend/api-gateway/src/main/java/com/datamate/gateway/common/filter/AuthFilter.java backend/api-gateway/src/main/java/com/datamate/gateway/common/filter/UserContextFilter.java renamed to backend/api-gateway/src/main/java/com/datamate/gateway/common/filter/AuthFilter.java

@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cloud.gateway.filter.GatewayFilterChain;
 import org.springframework.cloud.gateway.filter.GlobalFilter;
@@ -22,17 +23,19 @@
 import java.nio.charset.StandardCharsets;
 
 /**
- * 用户信息过滤器
+ * 鉴权过滤器
  *
  */
 @Slf4j
 @Component
 @RequiredArgsConstructor
-public class UserContextFilter implements GlobalFilter {
+public class AuthFilter implements GlobalFilter {
     private static final String AUTH_HEADER = "Authorization";
 
     private static final String TOKEN_PREFIX = "Bearer ";
 
+    private static final String USER_HEADER = "User";
+
     private final UserService userService;
 
     @Value("${datamate.jwt.enable:false}")
@@ -55,10 +58,22 @@ public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
                 return sendUnauthorizedResponse(exchange);
             }
             String token = authHeader.substring(TOKEN_PREFIX.length());
-            if (!userService.validateToken(token)) {
+            String user = userService.validateToken(token);
+            if (StringUtils.isBlank(user)) {
                 return sendUnauthorizedResponse(exchange);
             }
-            return chain.filter(exchange);
+            // 4. 创建新的请求
+            ServerHttpRequest mutatedRequest = request.mutate()
+                    .headers(httpHeaders -> {
+                        // 或者直接操作headers
+                        httpHeaders.add(USER_HEADER, user);
+                    })
+                    .build();
+            // 5. 使用新的请求创建新的exchange
+            ServerWebExchange mutatedExchange = exchange.mutate()
+                    .request(mutatedRequest)
+                    .build();
+            return chain.filter(mutatedExchange);
         } catch (Exception e) {
             log.error("get current user info error", e);
             return sendUnauthorizedResponse(exchange);

backend/api-gateway/src/main/java/com/datamate/gateway/domain/service/UserService.java

@@ -3,9 +3,7 @@
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.datamate.gateway.domain.entity.User;
 import com.datamate.gateway.domain.repository.UserRepository;
-import io.jsonwebtoken.JwtException;
-import io.jsonwebtoken.Jwts;
-import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.*;
 import io.jsonwebtoken.security.Keys;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.factory.annotation.Value;
@@ -26,6 +24,8 @@
 @Service
 @RequiredArgsConstructor
 public class UserService {
+    private static final String SYSTEM_USER = "system";
+
     private final UserRepository userRepository;
 
     @Value("${datamate.jwt.expiration-seconds:3600}")
@@ -70,12 +70,12 @@ private String generateToken(User user) {
                 .compact();
     }
 
-    public boolean validateToken(String token) {
+    public String validateToken(String token) {
         try {
-            Jwts.parser().setSigningKey(secret.getBytes()).parseClaimsJws(token);
-            return true;
+            Jws<Claims> claimsJws = Jwts.parserBuilder().setSigningKey(Keys.hmacShaKeyFor(secret.getBytes(StandardCharsets.UTF_8))).build().parseClaimsJws(token);
+            return claimsJws.getBody().getSubject();
         } catch (JwtException | IllegalArgumentException ex) {
-            return false;
+            return null;
         }
     }
 
@@ -89,7 +89,7 @@ public Optional<User> register(RegisterRequest registerRequest) {
         // Check if username already exists
         LambdaQueryWrapper<User> usernameQuery = new LambdaQueryWrapper<>();
         usernameQuery.eq(User::getUsername, registerRequest.getUsername());
-        if (userRepository.getOne(usernameQuery) != null) {
+        if (userRepository.getOne(usernameQuery) != null || SYSTEM_USER.equals(registerRequest.getUsername())) {
             return Optional.empty();
         }
 

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/entity/CleaningTask.java

@@ -2,6 +2,7 @@
 
 import com.baomidou.mybatisplus.annotation.TableName;
 import com.datamate.cleaning.common.enums.CleaningTaskStatusEnum;
+import com.datamate.common.domain.model.base.BaseEntity;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -14,9 +15,7 @@
 @Getter
 @Setter
 @TableName(value = "t_clean_task", autoResultMap = true)
-public class CleaningTask {
-    private String id;
-
+public class CleaningTask extends BaseEntity<String> {
     private String name;
 
     private String description;
@@ -39,8 +38,6 @@ public class CleaningTask {
 
     private Integer retryCount;
 
-    private LocalDateTime createdAt;
-
     private LocalDateTime startedAt;
 
     private LocalDateTime finishedAt;

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/CleaningResultMapper.java

@@ -2,8 +2,10 @@
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.datamate.cleaning.domain.model.entity.CleaningResult;
+import com.datamate.common.infrastructure.config.IgnoreDataScopeAnnotation;
 import org.apache.ibatis.annotations.Mapper;
 
 @Mapper
+@IgnoreDataScopeAnnotation
 public interface CleaningResultMapper extends BaseMapper<CleaningResult> {
 }

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/CleaningTemplateMapper.java

@@ -5,13 +5,15 @@
 import com.baomidou.mybatisplus.core.toolkit.Constants;
 import com.datamate.cleaning.domain.model.entity.TemplateWithInstance;
 import com.datamate.cleaning.domain.model.entity.CleaningTemplate;
+import com.datamate.common.infrastructure.config.IgnoreDataScopeAnnotation;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
 
 import java.util.List;
 
 @Mapper
+@IgnoreDataScopeAnnotation
 public interface CleaningTemplateMapper extends BaseMapper<CleaningTemplate> {
     @Select("SELECT t.id AS id, name, description, created_at, updated_at, created_by, operator_id, op_index, " +
         "settings_override FROM t_clean_template t LEFT JOIN t_operator_instance o ON t.id = o.instance_id " +

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java

@@ -2,6 +2,7 @@
 
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.datamate.cleaning.domain.model.entity.OperatorInstance;
+import com.datamate.common.infrastructure.config.IgnoreDataScopeAnnotation;
 import com.datamate.operator.domain.model.OperatorView;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Select;
@@ -10,6 +11,7 @@
 
 
 @Mapper
+@IgnoreDataScopeAnnotation
 public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
     @Select("SELECT o.operator_id as id, o.operator_name as name, o.description, o.version, o.inputs, o.outputs, " +
             "o.runtime, o.settings, o.created_at, o.updated_at, " +

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -244,7 +244,7 @@ public AllDatasetStatisticsResponse getAllDatasetStatistics() {
     public void processDataSourceAsync(String datasetId, String dataSourceId) {
         try {
             log.info("Initiating data source file scanning, dataset ID: {}, collection task ID: {}", datasetId, dataSourceId);
-            List<String> filePaths = getFilePaths(dataSourceId);
+            List<String> filePaths = getFilePaths(dataSourceId, datasetRepository.getById(datasetId));
             if (CollectionUtils.isEmpty(filePaths)) {
                 return;
             }
@@ -255,8 +255,8 @@ public void processDataSourceAsync(String datasetId, String dataSourceId) {
         }
     }
 
-    private List<String> getFilePaths(String dataSourceId) {
-        CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId).getData();
+    private List<String> getFilePaths(String dataSourceId, Dataset dataset) {
+        CollectionTaskDetailResponse taskDetail = collectionTaskClient.getTaskDetail(dataSourceId, dataset.getCreatedBy()).getData();
         if (taskDetail == null) {
             log.warn("Fail to get collection task detail, task ID: {}", dataSourceId);
             return Collections.emptyList();

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java

@@ -137,7 +137,7 @@ public PagedResponse<DatasetFile> getDatasetFilesWithDirectory(String datasetId,
 
             return new PagedResponse<>(page, size, total, totalPages, datasetFiles);
         } catch (IOException e) {
-            log.error("list dataset path error", e);
+            log.warn("list dataset path error");
             return PagedResponse.of(new Page<>(page, size));
         }
     }