[data] feat: add datachecker

rl_insight/data/__init__.py

@@ -14,20 +14,12 @@
 
 """Data module for RL-Insight."""
 
-from .base import (
-    BaseData,
-    DataValidationError,
-    ValidationRule,
+from .data_checker import (
+    DataChecker,
+    DataEnum
 )
-from .multi_json import MultiJsonData
-from .rules import PathExistsRule
-from .enums import DataEnum
 
 __all__ = [
-    "BaseData",
-    "MultiJsonData",
-    "DataValidationError",
-    "ValidationRule",
-    "PathExistsRule",
+    "DataChecker",
     "DataEnum",
 ]

rl_insight/data/data_checker.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2025 verl-project authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base data definitions for RL-Insight."""
+
+
+from typing import List
+from .rules import ValidationRule, PathExistsRule, DataValidationError
+from enum import Enum
+import logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+
+class DataEnum(Enum):
+    """Enum for data types in RL-Insight."""
+    # input data type of parser
+    MULTI_JSON = "multi_json"
+    VERL_LOG = "verl_log"
+    # output data type of parser, input data type of visualizer
+    SUMMARY_EVENT = "summary_event"
+    # other data type
+    UNKNOWN = "unknown"
+
+
+class DataChecker():
+    """Base data class for RL-Insight."""
+    rules: dict[DataEnum, List[ValidationRule]] = {
+        DataEnum.MULTI_JSON: [PathExistsRule()],
+        DataEnum.SUMMARY_EVENT: [],
+    }
+
+    def __init__(self, type: DataEnum, data: str|dict):
+        self.type = type
+        self.data = data
+
+    def run(self):
+        """Validate the data"""
+        errors = []
+        rules = self.rules[self.type]
+        for rule in rules:
+            if not rule.check(self.data):
+                errors.append(rule.error_message)
+        if errors:
+            raise DataValidationError("Data validation failed", errors)
+        logger.info(f"Data validation passed for {self.type}")

rl_insight/data/rules.py

@@ -13,21 +13,56 @@
 # limitations under the License.
 
 from typing import List
+from abc import ABC, abstractmethod 
+from pathlib import Path
+from typing import Optional
+import pandas as pd
 
-from rl_insight.data.base import ValidationRule
 
 
-class PathExistsRule(ValidationRule):
-    _error_message: List[str] = []
+class DataValidationError(Exception):
+    """Exception raised when data validation fails."""
+
+    def __init__(self, message: str, errors: Optional[List[List[str]]] = None):
+        super().__init__(message)
+        self.errors = errors or []
+
+    def __str__(self) -> str:
+        if self.errors:
+            return f"{super().__str__()}\n  - " + "\n  - ".join(
+                ["\n    ".join(err) for err in self.errors]
+            )
+        return super().__str__()
 
+
+class ValidationRule(ABC):
+    """Validation rule base class"""
+
+    @abstractmethod
     def check(self, data) -> bool:
-        if not hasattr(data, "path"):
-            self._error_message = ["Data object does not have 'path' attribute"]
+        pass
+
+    @property
+    @abstractmethod
+    def error_message(self) -> List[str]:
+        pass
+
+
+class PathExistsRule(ValidationRule):
+    _error_message: str
+    def check(self, data: str|dict|pd.DataFrame) -> bool:
+        if not isinstance(data, str):
+            self._error_message = "Data object is not a path"
             return False
-        if not data.path.exists():
-            self._error_message = [f"Source path does not exist: {data.path}"]
+        try:
+            path = Path(data)
+            if not path.is_dir():
+                self._error_message = f"Source path does not exist: {data}"
+                return False
+            return True
+        except Exception as e:
+            self._error_message = f"Source path does not exist: {data}"
             return False
-        return True
 
     @property
     def error_message(self) -> List[str]:

rl_insight/main.py

@@ -29,7 +29,7 @@ def run_pipeline(config, pipeline_class=None):
 def main():
     arg_parser = argparse.ArgumentParser(description="Cluster scheduling visualization")
     arg_parser.add_argument(
-        "--input-path", default="test", help="Raw path of profiling data"
+        "--input-path", default=r"C:\Users\Tardis\Documents\profile_data\discrete_analyse_false_json_only", help="Raw path of profiling data"
     )
     arg_parser.add_argument(
         "--input-type",