Skip to content

Commit

Permalink
Merged in TG-107 (pull request #30)
Browse files Browse the repository at this point in the history 
TG-107
  • Loading branch information
@ahmadjd94
ahmadjd94 committed Jan 28, 2017
2 parents f92dd5a + e55ce86 commit 7c9cca9
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 87 deletions.
66 changes: 33 additions & 33 deletions PredefnedQueries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,52 +4,52 @@ def initQueries(Key):
if Key=='http':
http=[
# QueryStatment('select * from http',1,'http',['uid','ts']),
QueryStatment('select uid,ts,`user_agent` from http',2,'http',['uid','ts','user agent']),
QueryStatment('select uid,ts,`request_body_len` from http',3,'http',['uid','ts','request length']),
QueryStatment('select uid,ts,`uri` from http',4,'http',['uid','ts','uri']),
QueryStatment('select uid,ts,`method` from http',5,'http',['uid','ts','method']),
QueryStatment('select uid,ts,`status_code` from http',6,'http',['uid','ts','status_code'])
QueryStatment("select uid,DateTime(ts,'unixepoch'),`user_agent` from http",2,'http',['uid','ts','user agent']),
QueryStatment("select uid,DateTime(ts,'unixepoch'),`request_body_len` from http",3,'http',['uid','ts','request length']),
QueryStatment("select uid,DateTime(ts,'unixepoch'),`uri` from http",4,'http',['uid','ts','uri']),
QueryStatment("select uid,DateTime(ts,'unixepoch'),`method` from http",5,'http',['uid','ts','method']),
QueryStatment("select uid,DateTime(ts,'unixepoch'),`status_code` from http",6,'http',['uid','ts','status_code'])
]
return http
elif Key == 'dns':
dns= [QueryStatment("SELECT uid,ts,`QUERY` FROM DNS",7,'dns',['uid','ts','dns']),
QueryStatment("SELECT uid,ts,`ANSWER` FROM DNS_ANSWERS",8,'dns',['uid','ts','answer']),
QueryStatment("SELECT uid,ts,`RESP_H` FROM IDS",9,'dns',['uid','ts','RESP_H']),
QueryStatment("SELECT uid,ts,`QTYPE` FROM DNS ",10,'dns',['uid','ts','QTYPE'])
dns= [QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`QUERY` FROM DNS",7,'dns',['uid','ts','dns']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`ANSWER` FROM DNS_ANSWERS",8,'dns',['uid','ts','answer']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`RESP_H` FROM IDS",9,'dns',['uid','ts','RESP_H']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`QTYPE` FROM DNS ",10,'dns',['uid','ts','QTYPE'])
]
return dns
elif Key=='conn':
conn=[QueryStatment("SELECT uid,ts,DURATION FROM CONN",11,'conn',['uid','ts','duration']),
QueryStatment("SELECT uid,ts FROM CONN",12,'conn',['uid','ts']),
QueryStatment("SELECT uid,ts,ORIG_H , ORIG_P FROM IDS",13,'conn',['uid','ts','orig_h','orig_p']),
QueryStatment("SELECT uid,ts,RESP_H ,RESP_P FROM IDS",14,'conn',['uid','ts','resp_h','resp_p']),
QueryStatment("SELECT uid,ts,PROTO FROM CONN",15,'conn',['uid','ts','proto']),
QueryStatment("SELECT uid,ts,ORIG_BYTES FROM CONN",16,'conn',['uid','ts','orig_bytes']),
QueryStatment("SELECT uid,ts,RESP_BYTES FROM CONN",17,'conn',['uid','ts','resp_bytes']),
QueryStatment("SELECT uid,ts,CONN_STATE FROM CONN",18,'conn',['uid','ts','conn_state']),
QueryStatment("SELECT uid,ts,ORIG_H,RESP_H FROM ids", 19, 'conn', ['uid', 'ts', 'origin host','respondent host'])]
conn=[QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),DURATION FROM CONN",11,'conn',['uid','ts','duration']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch') FROM CONN",12,'conn',['uid','ts']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),ORIG_H , ORIG_P FROM IDS",13,'conn',['uid','ts','orig_h','orig_p']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),RESP_H ,RESP_P FROM IDS",14,'conn',['uid','ts','resp_h','resp_p']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),PROTO FROM CONN",15,'conn',['uid','ts','proto']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),ORIG_BYTES FROM CONN",16,'conn',['uid','ts','orig_bytes']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),RESP_BYTES FROM CONN",17,'conn',['uid','ts','resp_bytes']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),CONN_STATE FROM CONN",18,'conn',['uid','ts','conn_state']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),ORIG_H,RESP_H FROM ids", 19, 'conn', ['uid', 'ts', 'origin host','respondent host'])]
return conn
elif Key == 'ssl':
ssl=[QueryStatment("SELECT uid,ts,VERSION FROM SSL",19,'ssl',['uid','ts','version']),
QueryStatment("SELECT uid,ts,CIPHER FROM SSL",20,'ssl',['uid','ts','CIPHER']),
QueryStatment("SELECT uid,ts,`SERVER_NAME` FROM SSL",21,'ssl',['uid','ts','SERVER_NAME']),
QueryStatment("SELECT uid,ts,`SUBJECT` FROM SSL",22,'ssl',['uid','ts','SUBJECT']),
QueryStatment("SELECT uid,ts,`ISSUER_SUBJECT` FROM SSL",23,'ssl',['uid','ts','ISSUER_SUBJECT']),]
ssl=[QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),VERSION FROM SSL",19,'ssl',['uid','ts','version']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),CIPHER FROM SSL",20,'ssl',['uid','ts','CIPHER']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`SERVER_NAME` FROM SSL",21,'ssl',['uid','ts','SERVER_NAME']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`SUBJECT` FROM SSL",22,'ssl',['uid','ts','SUBJECT']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`ISSUER_SUBJECT` FROM SSL",23,'ssl',['uid','ts','ISSUER_SUBJECT']),]
return ssl
elif Key == 'ssh':
ssh= [QueryStatment("SELECT uid,ts,`host_key` FROM SSH",24,'ssh',['uid','ts','host key']),
QueryStatment("SELECT uid,ts,DIRECTION FROM SSH",24,'ssh',['uid','ts','direction']),
QueryStatment("SELECT uid,ts,CLIENT FROM SSH",24,'ssh',['uid','ts','client']),
QueryStatment("SELECT uid,ts,SERVER FROM SSH",24,'ssh',['uid','ts','server']),
QueryStatment("SELECT uid,ts,CIPHER_ALG FROM SSH",24,'ssh',['uid','ts','cipher']),
QueryStatment("SELECT uid,ts,VERSION FROM SSH",24,'ssh',['uid','ts','version'])]
ssh= [QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`host_key` FROM SSH",24,'ssh',['uid','ts','host key']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),DIRECTION FROM SSH",24,'ssh',['uid','ts','direction']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),CLIENT FROM SSH",24,'ssh',['uid','ts','client']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),SERVER FROM SSH",24,'ssh',['uid','ts','server']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),CIPHER_ALG FROM SSH",24,'ssh',['uid','ts','cipher']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),VERSION FROM SSH",24,'ssh',['uid','ts','version'])]
return ssh
elif Key == 'weird':
weird= [
QueryStatment("SELECT uid,ts,`NAME` FROM WEIRD",25,'weird',['uid','ts','NAME']),
QueryStatment("SELECT uid,ts,`ADDI` FROM WEIRD",26,'weird',['uid','ts','ADDI']),
QueryStatment("SELECT uid,ts,`NOTICE` FROM WEIRD",27,'weird',['uid','ts','NOTICE']),
QueryStatment("SELECT uid,ts,`PEER` FROM WEIRD",28,'weird',['uid','ts','PEER'])
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`NAME` FROM WEIRD",25,'weird',['uid','ts','NAME']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`ADDI` FROM WEIRD",26,'weird',['uid','ts','ADDI']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`NOTICE` FROM WEIRD",27,'weird',['uid','ts','NOTICE']),
QueryStatment("SELECT uid,DateTime(ts,'unixepoch'),`PEER` FROM WEIRD",28,'weird',['uid','ts','PEER'])
]
return weird
else :
Expand Down
39 changes: 37 additions & 2 deletions SMTPfiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,46 @@ def smtp_files(connection):

labels = ['smtp', 'other']
values = [mailed_files, all_files]

fig = {
"data": [
{
"values": values,
"labels": labels,
"domain": {"x": [0, .48]},
"name": "status codes of HTTP log",
"hoverinfo": "label+percent+name",
"hole": .1,
"type": "pie"
}],
"layout": {
"autosize": True,
"title": "ratio of files origin according to smtp",
"annotations": [
{
"font": {
"size": 14
},
"showarrow": False,
"text": "ratio",
"x": 0.20,
"y": 0.5
}
]
}
}



now = time.now().strftime('%Y-%m-%d %H:%m:%S')
file = 'BILA-smtp-pie-%s.html' % now
trace = go.Pie(labels=labels, values=values)

py.plot([trace],filename=file,auto_open=False)





py.plot(fig,filename=file,auto_open=False)
return file


Expand Down
161 changes: 119 additions & 42 deletions code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(759, 518)
self.single = False # indicates if user is dealing with a signle file / DIR
self.DBmode=False
MainWindow.setStyleSheet("background-color: rgb(51, 51, 51);")
self.progress = 0 # indicate the level of progress bar
self.centralWidget = QtWidgets.QWidget(MainWindow)
Expand Down Expand Up @@ -464,24 +465,19 @@ def retranslateUi(self, MainWindow):
# self.textEdit.textChanged.connect(self.uMan)
self.pushButton_5.clicked.connect(self.executeSQL)
self.comboBox.currentIndexChanged.connect(self.selected_query)
self.tab_2.setEnabled(True)
self.tab_2.setEnabled(False)
self.tab_4.setEnabled(False)
self.tab_3.setEnabled(False)
self.comboBox_2.addItem('--select a plot type--')
self.comboBox_2.addItem('files statistics')
self.comboBox_2.addItem('weird bars')
self.comboBox_2.addItem('DNS Graph')
self.comboBox_2.addItem('smtp and files relation')

self.comboBox_2.addItem('ssl subjects')
self.comboBox_2.addItem('connections map')
self.comboBox_2.addItem('HTTP status code piechart')

self.radioButton.click()
def connect_plot(self):
self.webview.setUrl(QUrl("BILA.html"))
self.webview.load()
self.m = None

def loadDB(self):

global connection # defines the global connection
DBpath = QFileDialog.getOpenFileName(None, 'connect to a database', '/home',
'BILA*.db') # open files that follow a certain regex
Expand All @@ -497,8 +493,12 @@ def loadDB(self):
self.tab.setEnabled(False)
self.lineEdit.setDisabled(True)
self.lineEdit_2.setDisabled(True)
self.tab_3.setEnabled(False)
self.tab_2.setEnabled(True)
self.tab_4.setEnabled(True)
self.tab_3.setEnabled(True)
self.label_db.setVisible(True)
self.setup_combobox('DB',True,False)
self.DBmode = True
else:
pass

Expand All @@ -507,38 +507,39 @@ def pier(self):
if self.comboBox_2.currentText()=='--select a plot type--':
self.label_4.setText('please select a valid option')
elif self.comboBox_2.currentText()=='files statistics':
if self.single==True:
if self.single==True or self.DBmode==True:
self.label_4.setText('files statistics not available in single files mode')
self.label_4.show()
return
# Data to plot
else :
self.label_4.setVisible(False)
self.m=PlotCanvas(self, width=9, height=3)

self.m=PlotCanvas(self)
self.m.show()

elif self.comboBox_2.currentText() == 'weird bars': #completed
elif self.comboBox_2.currentText() == 'weird bars': #completed and tested
page = plotbars(connection)


elif self.comboBox_2.currentText() == 'DNS Graph': #completed
elif self.comboBox_2.currentText() == 'DNS Graph': #completed and tested
page=graph_plot(connection)

elif self.comboBox_2.currentText() == 'connections map': #completed
elif self.comboBox_2.currentText() == 'connections map': #completed and tested
os.chdir(OriDir)
page = plotlyMap.map(connection,finder)

elif self.comboBox_2.currentText() == 'HTTP status code piechart': #completed
elif self.comboBox_2.currentText() == 'HTTP status code piechart': #completed and tested
page = plot_http_status_pir(connection)

elif self.comboBox_2.currentText() == 'ssl subjects': #completed
page = ssl_subjects_pie(connection)
elif self.comboBox_2.currentText() == 'smtp and files relation': #completed
elif self.comboBox_2.currentText() == 'smtp and files relation': #completed and tested
page = smtp_files(connection)
self.webview.setZoomFactor(.5)

self.webview.load(QUrl('file://' + os.getcwd() + '/%s' % page))
if self.comboBox_2.currentText() !=('files statistics'):
self.webview.load(QUrl('file://' + os.getcwd() + '/%s' % page))
self.analysis.setCurrentIndex(3)



def uMan(self):
Expand Down Expand Up @@ -641,33 +642,103 @@ def traverse(self, fname): # this function will traverse the file that is based
def selected_query(self):
self.clear_table()
for i in AllowedQueries:
for query in i:
if query.Query == self.comboBox.currentText():
self.currentQuery = query
print(query.Headers[0])
self.model.setColumnCount(len(query.Headers[0]))
self.model.setHorizontalHeaderLabels(query.Headers[0])
self.model.show()
if i != None:
for query in i:
if query.Query == self.comboBox.currentText():
self.currentQuery = query
print(query.Headers[0])
self.model.setColumnCount(len(query.Headers[0]))
self.model.setHorizontalHeaderLabels(query.Headers[0])
self.model.show()

def clear_table(self):
while(self.model.rowCount()>0):
self.model.removeRow(0)
def add_queries(self,who):
pass

def setup_combobox(self,who,Dbmode,single):
self.comboBox_2.clear()
self.comboBox.clear()
self.comboBox_2.addItem('--select a plot type--')
if Dbmode==False and single==False:
self.comboBox_2.addItem('files statistics')


IDSmap = False # this variable indicates if the IDS connection map option has been inserted into combobox
table=[]

def setup_combobox(self):
try:
print (type(AllowedQueries))
print (AllowedQueries)
try :
if who =='loader':

try :
for obj in AllowedQueries:
if obj !=None:
for query in obj:
self.comboBox.addItem(query.Query)

if query.Table not in table:
table.append(query.Table)

print (query.Query)

if table_created['smtp'] and table_created['files']:
table.append('smtp')
table.append('files')

except Exception as s1:
print (s1)

elif who =='DB':


if table_created['smtp'] and table_created['files']:
table.append('smtp')
table.append('files')
query='Select distinct name from sqlite_master'
connection.DBquery.exec_(query)
while (connection.DBquery.next()):
table.append(connection.DBquery.value(0).lower())
print (table)
for i in table:

AllowedQueries.append(initQueries(i))
print (AllowedQueries)
for obj in AllowedQueries:
for query in obj:
self.comboBox.addItem(query.Query)
print (query.Query)
self.comboBox.setEnabled(True)
except Exception as s1:
print (s1)
if obj != None:
for query in obj:
self.comboBox.addItem(query.Query)
print ("Query Table:",query.Table)


print(query.Query)

for each in table:

if IDSmap == False and (each in ['weird', 'http', 'ssl', 'conn','dns']):
self.comboBox_2.addItem('connections map')
IDSmap = True
if each == 'http':
self.comboBox_2.addItem('HTTP status code piechart')

elif each == 'dns':
self.comboBox_2.addItem('DNS Graph')

elif each == 'ssl':
self.comboBox_2.addItem('ssl subjects')

elif each == 'weird':
self.comboBox_2.addItem('weird bars')
if 'smtp' and 'files' in table:
self.comboBox_2.addItem('smtp and files relation')

self.comboBox.setEnabled(True)
self.comboBox_2.setEnabled(True)
print ("DONE here")
except Exception as A:
print('eroro adding to combo box ', A)
print('error adding to combo box ', A)




def executeSQL(self): # this function performs the SQL queries in the SQL panel
Expand Down Expand Up @@ -747,11 +818,16 @@ def load_files(self,connection): # this function loads the content of the log f
table_created[key] = True
AllowedQueries.append(initQueries(fName.split('.')[0]))
self.traverse(fName)
self.setup_combobox()
self.setup_combobox('loader',False,True)
self.tab_2.setEnabled(True)
self.tab_3.setEnabled(True)
self.tab_4.setEnabled(True)


elif self.radioButton_2.isChecked() and self.lineEdit_2.text() != "": # user choosed to load multiple files
for each in self.validFiles:
each = str.lower(each)
print (each.split('.'))
AllowedQueries.append(initQueries(each.split('.')[0]))
if table_created[each.split('.')[0]] == False:
if each.split('.')[0] in ['weird', 'dns', 'conn', 'http', 'dhcp', 'irc', 'ssl'] and \
Expand All @@ -769,13 +845,14 @@ def load_files(self,connection): # this function loads the content of the log f
connection.DBquery.exec_(queries[query])
table_created[query] = True
self.traverse(each) # load every file in the dir
print (each,"wtffffff")

self.setup_combobox()
self.setup_combobox('loader',False,False)

self.analysis.setTabEnabled(1, True) #enable plotting tab after loading
self.loaded=True
self.tab_2.setEnabled(True)
self.analysis.setTabEnabled(2, True) #enable query tab after loading
self.tab_3.setEnabled(True)
self.tab_4.setEnabled(True) #enable query tab after loading
# self.loaded = True # this flag indicates the program and database are loaded with data
else:
self.message.setText("please specifiy a file to load or a directory")
Expand Down
Loading

0 comments on commit 7c9cca9

Please sign in to comment.