support extract data from tecent meeting

Coffee/Data/DataLoader.py

@@ -68,6 +68,8 @@ def start(self):
         except ValueError:
             base_datetime = datetime.utcnow()
 
+        prev_datetime = None
+
         # get lines of this log
         lines = 0
         for _, _ in enumerate(io.open(self.log_path, encoding='utf-8', errors='ignore')):
@@ -85,26 +87,28 @@ def start(self):
                 if not line:
                     break
 
+                # match time
+                ts = None
+                for t in PTS:
+                    ts = t.match(line)
+                    # early break when find ts
+                    if ts:
+                        break
+
+                if not ts:
+                    if not prev_datetime:
+                        continue
+                    dt = prev_datetime
+                else:
+                    dt = merge_datetime(base_datetime, ts)
+                    prev_datetime = dt
+
                 for p in PDT:
                     # match data
                     r = p.match(line)
                     if not r:
                         continue
 
-                    # match time
-                    ts = None
-                    for t in PTS:
-                        ts = t.match(line)
-                        # early break when find ts
-                        if ts:
-                            break
-
-                    if not ts:
-                        # not find ts
-                        continue
-
-                    dt = merge_datetime(base_datetime, ts)
-
                     dp = DataPoint(
                             name=p.get_name(),
                             timestamp=dt,

Coffee/Data/DataProcessor.py

@@ -114,12 +114,15 @@ class DataAggregator(DataSink):
     aggregate all points in an object, don't process one by one
     """
 
-    def __init__(self):
+    def __init__(self, append_timestamp: bool = False):
         super(DataAggregator, self).__init__()
         self.all_points = []
+        self.append_timestamp = append_timestamp
 
     def on_data(self, datapoint: DataPoint) -> DataPoint:
-        self.all_points.append(datapoint.value)
+        value = datapoint.value
+        value['timestamp'] = datapoint.timestamp
+        self.all_points.append(value)
         return datapoint
 
     def finish(self, datapoint: DataPoint) -> DataPoint:

examples/extract_tencent_meeting.py

@@ -0,0 +1,28 @@
+from Coffee.Data import *
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+
+
+agg = DataAggregator(append_timestamp=True)
+
+LogFileDataLoader(
+    os.path.join(os.path.expanduser('~'),
+                 'Library/Containers/com.tencent.meeting/Data/Library/Global/Logs/xcast_2022110620.log')
+).add_ts_pattern(
+    RegexPattern('', r'(\d+):(\d+):(\d+)\.(\d\d\d)', {
+        'hour': int, 'minute': int, 'second': int, 'millisecond': int
+    })
+).add_pattern(
+    RegexPattern(
+        name='Tencent_Sys_Info',
+        pattern=r'Sys\[CPU:([\d\.]+)%\(App\) ([\d\.]+)%\(Sys\) '
+        r'.*CpuLevel:(\d+)\]',
+        fields={'app': float, 'sys': float, 'cpu_level': int}
+    )
+).add_sink(agg).start()
+
+df = pd.DataFrame(agg.all_points)
+print(df)
+df.plot(x='timestamp', y=['app', 'sys', 'cpu_level'], kind='line')
+plt.show()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pycoffee"
-version = "1.0.4"
+version = "1.0.5"
 authors = [
   { name="tkorays", email="[email protected]" },
 ]

requirements.txt

@@ -13,6 +13,4 @@ python-daemon
 django
 celery
 django-rest-framework
-redis
-pandas
-matplotlib
+redis

setup.py

@@ -1,5 +1,5 @@
 from setuptools import setup, find_packages
-version = '1.0.3'
+version = '1.0.5'
 
 
 setup(
@@ -36,8 +36,8 @@
         'django',
         'celery',
         'redis',
-        'pandas',
-        'matplotlib'
+        # 'pandas',
+        # 'matplotlib'
     ],
     dependency_links=[
     ],