Skip to content

Commit

Permalink
rename datapattern to dataextractor and add SplitPattern
Browse files Browse the repository at this point in the history
  • Loading branch information
tkorays committed Nov 28, 2022
1 parent 46fcb1e commit cbc990c
Show file tree
Hide file tree
Showing 6 changed files with 47 additions and 4 deletions.
41 changes: 41 additions & 0 deletions Coffee/Data/DataPattern.py → Coffee/Data/DataExtractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,47 @@ class GrokPattern(PatternInterface):
pass


class SplitPattern(PatternInterface):
def __init__(self, name: str, keyword: str, splitter: str, kv_splitter: str, fields: dict, tags: dict = [], processors=[], tests=[]):
self.name = name
self.keyword = keyword
self.splitter = splitter
self.kv_splitter = kv_splitter
self.fields = fields
self.tags = tags
self.processors = processors
self.tests = tests
self.match_count = 0

def get_unique_id(self):
return self.name

def get_name(self):
return self.name

def get_tags(self):
return self.tags

def get_match_count(self):
return self.match_count

def match(self, s):
fields = s.split(self.splitter)
if self.keyword not in fields:
return None
li = [kv for kv in [f.split(self.kv_splitter, 1) for f in fields] if len(kv) == 2]
result = {L[0]: self.fields[L[0]](L[1]) for L in li if L[0] in self.fields.keys()}

self.match_count = self.match_count + 1

for p in self.processors:
if isinstance(p, types.FunctionType):
result = p(self.name, result)
else:
result = p.process(self.name, result)
return result


class PatternGroupBuilder:
"""
build a pattern group
Expand Down
4 changes: 2 additions & 2 deletions Coffee/Data/DataLoader.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
from rich.progress import Progress
import io

from Coffee.Data.DataPattern import RegexPattern, PatternGroupBuilder
from Coffee.Data.DataExtractor import RegexPattern, PatternGroupBuilder
from Coffee.Core.Utils import merge_datetime
from Coffee.Data.DataPattern import PatternGroup
from Coffee.Data.DataExtractor import PatternGroup
from Coffee.Data.DataFlow import DataPoint, DataLoader


Expand Down
2 changes: 1 addition & 1 deletion Coffee/Data/TimestampPatterns.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright 2022 tkorays. All Rights Reserved.
# Licensed to MIT under a Contributor Agreement.

from Coffee.Data.DataPattern import RegexPattern
from Coffee.Data.DataExtractor import RegexPattern

DEFAULT_TS_PATTERNS = [
RegexPattern(
Expand Down
2 changes: 1 addition & 1 deletion Coffee/Data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
LogFileDataLoader
)
from .DataModel import DataModel
from .DataPattern import (
from .DataExtractor import (
PatternInterface, RegexPattern,
PatternGroup, PatternGroupBuilder
)
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ pid
python-daemon
django
h5py
numpy
ddt
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
'python-daemon',
'django',
'h5py',
'numpy',
'ddt',
],
dependency_links=[
Expand Down

0 comments on commit cbc990c

Please sign in to comment.