Skip to content

Commit

Permalink
Adding several changes to the dashboard webpages and buttons, also ad…
Browse files Browse the repository at this point in the history 
…ding the collction script to the daily automation so it runs on a daily schedule
  • Loading branch information
Dale Brennan committed Feb 5, 2024
1 parent f9f1406 commit 5b0ca9b
Show file tree
Hide file tree
Showing 22 changed files with 241 additions and 322 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ study-metrics.db
essential-metrics.log
python_errors.log
.vscode
virtual-env
virtual-env
TaskSchedulerView.cfg
9 changes: 0 additions & 9 deletions EM_2_software_register.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,17 +75,8 @@ def update_install_date(cell_value):
except Exception as e:
logger.error(f'Could not drop duplicates from the software frame, continuing on: {e}')

# import pdb; pdb.set_trace()

df['InstallDate'] = df['InstallDate'].apply(update_install_date)


'''
for index, row in df.iterrows():
if row.isna().any():
logger.debug(f"Row {index}:")
logger.debug(row)
'''
with DatabaseManager() as db:
db.add_new_rows('em_2_software_register', df, ['Publisher', 'DisplayName', 'DisplayVersion'])
db.remove_old_rows('em_2_software_register', df, ['Publisher', 'DisplayName', 'DisplayVersion'])
Expand Down
2 changes: 1 addition & 1 deletion EM_3_firewall.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# We first need to validate the firewall is enabled or our application will not work.
# https://support.microsoft.com/en-us/windows/risks-of-allowing-apps-through-windows-defender-firewall-654559af-3f54-3dcf-349f-71ccd90bcc5c

# Check windows advanced firewall before completeing automation.
# Check windows advanced firewall before completing automation.

import pandas as pd
import json
Expand Down
Binary file added UPDATE.md
View file
Binary file not shown.
3 changes: 2 additions & 1 deletion gather_metrics_daily.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ Set-Location C:\opt\essential-metrics
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\EM_13_antivirus_working.py"
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\EM_15_external_ports.py"
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\EM_18_rdp_settings.py"
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\EM_19_usb_attached.py"
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\EM_19_usb_attached.py"
& "C:\\opt\\essential-metrics\\virtual-env\\Scripts\\python.exe" "C:\opt\essential-metrics\collect_system_metrics.py"
72 changes: 22 additions & 50 deletions pages/EM_11_vulnerability_patching_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

def generate_patching_graphs(df):
try:
#import pdb; pdb.set_trace()
filtered_df = df[df['Software'].str.contains('Update for Windows 11', na=False)]

# We first need to convert the Microsoft DateTime stamp format: 20231215060522.240996-000 into a format we can perform calculations with in pandas
Expand Down Expand Up @@ -111,20 +110,24 @@ def _categorize_software(software):


model_id = 'em_11_vulnerability_patching'
training_modal_graph = cgf.training_modal(model_id, 'Manage Patching', 'https://www.youtube-nocookie.com/embed/xtsNTwRg7iM?si=40yydd3SmHuglvlL')
patching_bar_chart, mean_time_to_patch = generate_patching_graphs(em_11_vulnerability_patching)
system_updates = generate_updates_graph(em_11_vulnerability_patching)


layout = html.Div([
html.H2('Patch Management', style={'textAlign': 'center'}),
dbc.Button("Patching Help", id=f"{model_id}-open-model", n_clicks=0, style={
'width': '200px',
'height': '56px',
'position': 'absolute',
'top': '10px',
'right': '10px'
}),
html.A(
dbc.Button("Patching Advice", id=f"{model_id}-help", color="danger", n_clicks=0, style={
'width': '200px',
'height': '56px',
'position': 'absolute',
'top': '10px',
'right': '10px'
}),
href=f'https://www.ncsc.gov.uk/collection/top-tips-for-staying-secure-online/install-the-latest-software-and-app-updates',
target="_blank"
),
html.Div(id=f'{model_id}-dummy-div', style={'display': 'none'}),
dbc.Button("Manage Patching", id=f"{model_id}-manage", color="success", n_clicks=0, style= {
'width': '200px',
'height': '56px',
Expand All @@ -142,7 +145,7 @@ def _categorize_software(software):
html.Br(),
f'Your Mean time to patch, that is mean time from when you downloaded a patch until it was successfully installed was:',
html.Br(),
f'{mean_time_to_patch} Hours'
dcc.Markdown(f'**{mean_time_to_patch} hours**')
],
style={'textAlign': 'center'}
),
Expand All @@ -161,52 +164,21 @@ def _categorize_software(software):
html.Br(),
html.Div([dcc.Graph(figure=system_updates, style={'height': '800px', 'width': '100%'})]),
html.Div(id=f"{model_id}-hidden-output", style={"display": "none"}),
training_modal_graph,
dcc.Loading(id=f'loading-{model_id}', type='default', children = [
dash_table.DataTable(
id=f'{model_id}-table',
style_cell=dict(textAlign='left', maxWidth='500px'),
style_table={
'overflow-y': 'hidden',
'overflow-x': 'auto',
},
css=[{
'selector': '.dash-spreadsheet td div',
'rule': '''
line-height: 15px,
max-height: 30px, min-height: 30px, height: 30px;
display: block;
overflow-y: hidden;
'''
}],
export_format='csv',
columns=[ {'name': i, 'id': i} for i in em_11_vulnerability_patching.columns],
data=em_11_vulnerability_patching.to_dict('records'),
tooltip_data=[
{column: {'value': str(value), 'type': 'markdown'} for column, value in row.items()}
for row in em_11_vulnerability_patching.to_dict('records')
],
tooltip_duration=None,
sort_action='native',
sort_mode='single',
filter_action='native',
sort_by=[{'column_id': 'captured_at', 'direction': 'asc'}],
page_size=10,
),
]),
html.H4("These are the system update events we have collected from your event logs", style={'textAlign': 'center'}),
cgf.generate_dash_table(em_11_vulnerability_patching, 'em_11_vulnerability_patching'),
])


@callback(
Output(model_id, "is_open"),
[Input(f"{model_id}-open-model", "n_clicks"), Input(f"{model_id}-close-modal", "n_clicks")],
[State(model_id, "is_open")],
Output(f'{model_id}-dummy-div', 'children'),
[Input(f"{model_id}-help", "n_clicks")]
)
def toggle_modal(n1, n2, is_open):
if n1 or n2:
def toggle_modal(n_clicks):
if n_clicks > 0:
logger.info(f'{model_id} Help button pressed')
return not is_open
return is_open
return ""
return ""


@callback(
Output(f"{model_id}-hidden-output", "children"),
Expand Down
82 changes: 14 additions & 68 deletions pages/EM_12_reboot_analysis_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ def patch_to_reboot_analysis(df, df_v):
combined_df['UnifiedTime'] = combined_df['TimeGenerated'].fillna(combined_df['StartTime'])
combined_df_sorted = combined_df.sort_values(by='UnifiedTime')

# This will find all consequative rows that are not unique in the 'Software' column
# this is necessary as I found multiple software versions without a coorosponding event ID 12 so these ned to be removed from the analysis as they cannot be paired
# This will find all consecutive rows that are not unique in the 'Software' column
# this is necessary as I found multiple software versions without a corresponding event ID 12 so these ned to be removed from the analysis as they cannot be paired
mask = (
(combined_df_sorted['Software'] != combined_df_sorted['Software'].shift()) &
~(combined_df_sorted['Software'].isna() & combined_df_sorted['Software'].shift().isna())
Expand Down Expand Up @@ -127,110 +127,56 @@ def patch_to_reboot_analysis(df, df_v):

fig.update_layout(xaxis_tickangle=-45, xaxis_title='Software Update', yaxis_title='Install started -> reboot completed (Hours)')

return(fig)
return(fig, new_df['TimeDifferenceHours'].mean())
except Exception as e:
logger.error(f'The bar graph could not be rendered, sending back generic graph: {e}')
return(cgf.set_no_results_found_figure())
return(cgf.set_no_results_found_figure(), 0)



model_id = 'em_12_reboot_analysis'
training_modal_graph = cgf.training_modal(model_id, 'Reboot management', 'https://www.youtube-nocookie.com/embed/xtsNTwRg7iM?si=40yydd3SmHuglvlL')
new_windows_version = generate_reboot_graph(em_12_kernel_versions)
patch_reboot_analysis = patch_to_reboot_analysis(em_12_kernel_versions, em_11_vulnerability_patching)
patch_reboot_analysis, mean_time_to_reboot = patch_to_reboot_analysis(em_12_kernel_versions, em_11_vulnerability_patching)


layout = html.Div([
html.H2('Patch Management (Reboot Analysis)', style={'textAlign': 'center'}),
dbc.Button("Patching Help", id=f"{model_id}-open-model", n_clicks=0, style={
'width': '200px',
'height': '56px',
'position': 'absolute',
'top': '10px',
'right': '10px'
}),
dbc.Button("Manage Patching", id=f"{model_id}-manage", color="success", n_clicks=0, style= {
'width': '200px',
'height': '56px',
'position': 'absolute',
'top': '10px',
'right': '230px'
'right': '10px'
}),
html.Br(),
html.P([
'Rebooting is an integral part of patch management as it allows the new kernel code to be loaded in the operating system.',
html.Br(),
'This patches critical security bugs in the operating system and decreases your chances of harm from a vulnerability as the window of opportunity for an attacker is reduced.',
html.Br(),
'These patches generally have security updates in them and are vital for keeping your system secure.',
html.Br(),
f'Your Mean time to patch, that is mean time from when you downloaded a patch until it was successfully installed was:',
f'Your Mean time to reboot, that is mean time from when you started installing a patch until you rebooted your system was:',
html.Br(),
dcc.Markdown(f'**{mean_time_to_reboot} hours**')
],
style={'textAlign': 'center'}
),
html.Br(),
html.Div([dcc.Graph(figure=new_windows_version, style={'height': '600px', 'width': '100%'})]),
html.Div([dcc.Graph(figure=patch_reboot_analysis, style={'height': '1000px', 'width': '100%'})]),
html.Br(),
html.P([
'If you notice on the below graph that you have not been receiving updates recently (past month) this could be an indication there is an issue with your system patching.',
html.Br(),
'The Microsoft Defender Updates are very busy so you should see a lot of activity there, the Windows System update is your Monthly Patch Tuesday patches.',
'This is your reboot timelines for each of your patching versions, lower is better here.',
html.Br(),
'The .NET Framework is less busy and the AntiMalware updates are only a few updates a year.',
],
style={'textAlign': 'center'}
),
'We recommend rebooting as close to the reboot prompt as possible to keep your system secure.',
], style={'textAlign': 'center'}),
html.Div([dcc.Graph(figure=patch_reboot_analysis, style={'height': '1000px', 'width': '100%'})]),
html.Br(),
html.Div(id=f"{model_id}-hidden-output", style={"display": "none"}),
training_modal_graph,
dcc.Loading(id=f'loading-{model_id}', type='default', children = [
dash_table.DataTable(
id=f'{model_id}-table',
style_cell=dict(textAlign='left', maxWidth='500px'),
style_table={
'overflow-y': 'hidden',
'overflow-x': 'auto',
},
css=[{
'selector': '.dash-spreadsheet td div',
'rule': '''
line-height: 15px,
max-height: 30px, min-height: 30px, height: 30px;
display: block;
overflow-y: hidden;
'''
}],
export_format='csv',
columns=[ {'name': i, 'id': i} for i in em_12_reboot_analysis.columns],
data=em_12_reboot_analysis.to_dict('records'),
tooltip_data=[
{column: {'value': str(value), 'type': 'markdown'} for column, value in row.items()}
for row in em_12_reboot_analysis.to_dict('records')
],
tooltip_duration=None,
sort_action='native',
sort_mode='single',
filter_action='native',
sort_by=[{'column_id': 'captured_at', 'direction': 'asc'}],
page_size=10,
),
]),
html.H4("These are the reboot events we have collected from your event logs", style={'textAlign': 'center'}),
cgf.generate_dash_table(em_12_reboot_analysis, 'em_12_reboot_analysis'),
])


@callback(
Output(model_id, "is_open"),
[Input(f"{model_id}-open-model", "n_clicks"), Input(f"{model_id}-close-modal", "n_clicks")],
[State(model_id, "is_open")],
)
def toggle_modal(n1, n2, is_open):
if n1 or n2:
logger.info(f'{model_id} Help button pressed')
return not is_open
return is_open

@callback(
Output(f"{model_id}-hidden-output", "children"),
[Input(f"{model_id}-manage", "n_clicks")]
Expand Down
21 changes: 11 additions & 10 deletions pages/EM_13_14_antivirus_working_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,13 @@ def gen_threat_count_over_time(df):


layout = html.Div([
#dcc.Location(id='url', refresh=False),
dcc.Interval(
id='refresh-dashboard',
interval=1*60*1000, # in milliseconds, 1*60*1000 for 1 minute
n_intervals=0
),
html.A(
dbc.Button("AntiVirus EICAR Help", id=f"{model_id}-open-model", color="danger", n_clicks=0, style={
dbc.Button("AntiVirus EICAR Help", id=f"{model_id}-help", color="danger", n_clicks=0, style={
'width': '200px',
'height': '56px',
'position': 'absolute',
Expand All @@ -89,6 +88,7 @@ def gen_threat_count_over_time(df):
href=f'https://www.eicar.org/download-anti-malware-testfile/',
target="_blank"
),
html.Div(id=f'{model_id}-dummy-div', style={'display': 'none'}),
dbc.Button("Manage AntiVirus", id=f"{model_id}-manage", color="success", n_clicks=0, style= {
'width': '200px',
'height': '56px',
Expand Down Expand Up @@ -147,16 +147,17 @@ def refresh_table(n):
else:
return html.H4(f"We found {len(df)} new threats over the past week:", style={'textAlign': 'center'}), cgf.generate_dash_table(df, 'em_14_threats_found_past_week')


@callback(
Output(model_id, "is_open"),
[Input(f"{model_id}-open-model", "n_clicks"), Input(f"{model_id}-close-modal", "n_clicks")],
[State(model_id, "is_open")],
Output(f'{model_id}-dummy-div', 'children'),
[Input(f"{model_id}-help", "n_clicks")]
)
def toggle_modal(n1, n2, is_open):
if n1 or n2:
logger.info(f'{model_id} Help button pressed')
return not is_open
return is_open
def toggle_modal(n_clicks):
if n_clicks > 0:
logger.info(f'{model_id} Advice button pressed')
return ""
return ""


@callback(
Output(f"{model_id}-hidden-output", "children"),
Expand Down
13 changes: 12 additions & 1 deletion pages/EM_15_16_int_ext_ports_web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ def _get_asset_name(df, ip):
href=f'http://{gateway_url}',
target="_blank"
),
html.Div(id=f'{model_id}-dummy-div', style={'display': 'none'}),
training_modal_graph,
html.H2('External/ Internal port Management', style={'textAlign': 'center'}),
html.P([
Expand Down Expand Up @@ -258,4 +259,14 @@ def toggle_modal(n1, n2, is_open):
if n1 or n2:
logger.info(f'{model_id} Help button pressed')
return not is_open
return is_open
return is_open

@callback(
Output(f'{model_id}-dummy-div', 'children'),
[Input(f"{model_id}-manage", "n_clicks")]
)
def toggle_modal(n_clicks):
if n_clicks > 0:
logger.info(f'{model_id} Connect to router pressed')
return ""
return ""
Loading

0 comments on commit 5b0ca9b

Please sign in to comment.