Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multi threaded processing #54

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Multi threaded processing #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4131d6d
added - parallel processing for multi files
aviadtamir Dec 15, 2020
5a49ac9
fixed the parallel processing of sessions and files
aviadtamir Dec 15, 2020
ab3b5cb
added parallel processing feature as option to UIs
aviadtamir Dec 19, 2020
4b0b626
WIP
aviadtamir Dec 29, 2020
0e6336c
fixed parallel processing, TODO - support pcapng
aviadtamir Dec 31, 2020
ffd7eec
refactored multi processing and pcap processor mechanism
aviadtamir Jan 1, 2021
6b5e3c0
added pcapng files
aviadtamir Jan 2, 2021
18756c4
merged master
aviadtamir Jan 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions BruteShark/BruteSharkCli/BruteSharkCli.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ internal class BruteSharkCli
private ulong _udpPacketsCount;
private int _tcpSessionsCount;
private int _udpStreamsCount;
private PcapProcessor.Processor _processor;
private PcapProcessor.ProcessorEngine _processor;
private PcapAnalyzer.Analyzer _analyzer;
private List<string> _files;
private HashSet<PcapAnalyzer.NetworkPassword> _passwords;
Expand All @@ -38,7 +38,7 @@ public BruteSharkCli()
_files = new List<string>();
_connections = new HashSet<NetworkConnection>();

_processor = new PcapProcessor.Processor();
_processor = new PcapProcessor.ProcessorEngine(false);
_analyzer = new PcapAnalyzer.Analyzer();
_shell = new CliShell(seperator:"Brute-Shark > ");

Expand Down Expand Up @@ -67,6 +67,7 @@ public BruteSharkCli()
_shell.AddCommand(new CliShellCommand("show-networkmap", p => PrintNetworkMap(), "Prints the network map as a json string. Usage: show-networkmap"));
_shell.AddCommand(new CliShellCommand("export-hashes", p => ExportHashes(p), "Export all Hashes to Hascat format input files. Usage: export-hashes <OUTPUT-DIRECTORY>"));
_shell.AddCommand(new CliShellCommand("export-networkmap", p => ExportNetworkMap(p), "Export network map to a json file for neo4j. Usage: export-networkmap <OUTPUT-file>"));
_shell.AddCommand(new CliShellCommand("process-files-parallel", p => this._processor.ProcessFilesParallel = true, "Processes the pcap files in paralell"));

}

Expand Down
23 changes: 19 additions & 4 deletions BruteShark/BruteSharkDesktop/MainForm.Designer.cs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 17 additions & 2 deletions BruteShark/BruteSharkDesktop/MainForm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace BruteSharkDesktop
public partial class MainForm : Form
{
private HashSet<string> _files;
private PcapProcessor.Processor _processor;
private PcapProcessor.ProcessorEngine _processor;
private PcapAnalyzer.Analyzer _analyzer;

private GenericTableUserControl _passwordsUserControl;
Expand All @@ -35,8 +35,9 @@ public MainForm()
_files = new HashSet<string>();

// Create the DAL and BLL objects.
_processor = new PcapProcessor.Processor();
_processor = new PcapProcessor.ProcessorEngine(false) ;
_analyzer = new PcapAnalyzer.Analyzer();
_processor.ProcessFilesParallel = false;
_processor.BuildTcpSessions = true;
_processor.BuildUdpSessions = true;

Expand Down Expand Up @@ -313,6 +314,20 @@ private void buildTcpSessionsCheckBox_CheckedChanged(object sender, EventArgs e)
messageOnBuildSessionsConfigurationChanged();
}
}

private void ProcessFilesParallel_CheckedChanged(object sender, EventArgs e)
{
if (ProcessFilesParallelCheckBox.CheckState == CheckState.Checked)
{
ProcessFilesParallelCheckBox.Text = "Process Files Parallel : ON";
this._processor.ProcessFilesParallel = true;
}
else if (ProcessFilesParallelCheckBox.CheckState == CheckState.Unchecked)
{
ProcessFilesParallelCheckBox.Text = "Process Files Parallel : OFF";
this._processor.ProcessFilesParallel = false;
}
}

private void buildUdpSessionsCheckBox_CheckedChanged(object sender, EventArgs e)
{
Expand Down
6 changes: 4 additions & 2 deletions BruteShark/PcapProcessor/ProcessingPrecentsPredicator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
using System.IO;
using System.Linq;
using System.Text;
using System.Collections.Concurrent;

namespace PcapProcessor
{
class ProcessingPrecentsPredicator
public class ProcessingPrecentsPredicator
{
public delegate void ProcessingPrecentsChangedEventHandler(object sender, ProcessingPrecentsChangedEventArgs e);
public event ProcessingPrecentsChangedEventHandler ProcessingPrecentsChanged;
Expand Down Expand Up @@ -60,7 +61,8 @@ public void NotifyAboutProcessedData(long additionalData)
public void NotifyAboutProcessedFile(FileInfo fileProcessed)
{
this.FilesProcessed.Add(fileProcessed);
this.DataProcessed = this.FilesProcessed.Sum(fi => fi.Length);
//this.DataProcessed = this.FilesProcessed.Sum(fi => fi.Length);
this.DataProcessed += fileProcessed.Length;
}

private void CheckIfProcessingPrecentsChanged(long additionalData)
Expand Down
110 changes: 51 additions & 59 deletions BruteShark/PcapProcessor/Processor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
using System.Linq;
using System.Threading;
using Haukcode.PcapngUtils;
using Haukcode.PcapngUtils.Common;
using Haukcode.PcapngUtils.Common;


namespace PcapProcessor
Expand All @@ -23,6 +23,8 @@ public class Processor
{
public delegate void FileProcessingStatusChangedEventHandler(object sender, FileProcessingStatusChangedEventArgs e);
public event FileProcessingStatusChangedEventHandler FileProcessingStatusChanged;
public delegate void ProcessingPrecentsChangedEventHandler(object sender, ProcessingPrecentsChangedEventArgs e);
private ProcessingPrecentsPredicator _processingPrecentsPredicator;
public delegate void UdpPacketArivedEventHandler(object sender, UdpPacketArivedEventArgs e);
public event UdpPacketArivedEventHandler UdpPacketArived;
public delegate void UdpSessionArrivedEventHandler(object sender, UdpSessionArrivedEventArgs e);
Expand All @@ -31,93 +33,81 @@ public class Processor
public event TcpPacketArivedEventHandler TcpPacketArived;
public delegate void TcpSessionArivedEventHandler(object sender, TcpSessionArivedEventArgs e);
public event TcpSessionArivedEventHandler TcpSessionArrived;
public delegate void ProcessingPrecentsChangedEventHandler(object sender, ProcessingPrecentsChangedEventArgs e);
public event ProcessingPrecentsChangedEventHandler ProcessingPrecentsChanged;
public event EventHandler ProcessingFinished;

public bool ProcessFilesParallel { get; set; }
public bool BuildTcpSessions { get; set; }
public bool BuildUdpSessions { get; set; }

private TcpSessionsBuilder _tcpSessionsBuilder;
private UdpStreamBuilder _udpStreamBuilder;
private ProcessingPrecentsPredicator _processingPrecentsPredicator;
private UdpStreamBuilder _udpStreamsBuilder;
private string _filepath;



public Processor()
public Processor(ProcessingPrecentsPredicator processingPrecentsPredicator, string filepath)
{

this._filepath = filepath;
_processingPrecentsPredicator = processingPrecentsPredicator;
this.ProcessFilesParallel = false;
this.BuildTcpSessions = false;
this.BuildUdpSessions = false;
_tcpSessionsBuilder = new TcpSessionsBuilder();
_udpStreamBuilder = new UdpStreamBuilder();
_processingPrecentsPredicator = new ProcessingPrecentsPredicator();
_processingPrecentsPredicator.ProcessingPrecentsChanged += OnPredicatorProcessingPrecentsChanged;
this._tcpSessionsBuilder = new TcpSessionsBuilder();
this._udpStreamsBuilder = new UdpStreamBuilder();


}

private void OnPredicatorProcessingPrecentsChanged(object sender, ProcessingPrecentsChangedEventArgs e)
private void invokeAndClear(object session)
{
// TODO: think of make this check in a dedicated extention method for events (e.g SafeInvoke())
if (ProcessingPrecentsChanged is null)
return;

ProcessingPrecentsChanged.Invoke(this, new ProcessingPrecentsChangedEventArgs()
if(session is TcpSession)
{
Precents = e.Precents
});
}

public void ProcessPcaps(IEnumerable<string> filesPaths)
{
_processingPrecentsPredicator.AddFiles(new HashSet<FileInfo>(filesPaths.Select(fp => new FileInfo(fp))));

foreach (var filePath in filesPaths)
this._tcpSessionsBuilder.ClearSession((TcpSession)session);
TcpSessionArrived?.Invoke(this, new TcpSessionArivedEventArgs()
{
TcpSession = (TcpSession)session
});
}

if (session is UdpSession)
{
this.ProcessPcap(filePath);
this._udpStreamsBuilder.ClearSession((UdpSession)session);
UdpSessionArrived?.Invoke(this, new UdpSessionArrivedEventArgs()
{
UdpSession = (UdpSession)session
});

}

ProcessingFinished?.Invoke(this, new EventArgs());
}


public void ProcessPcap(string filePath)
public void ProcessPcap()
{
try
{
RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Started, filePath);
_tcpSessionsBuilder.Clear();
_udpStreamBuilder.Clear();
RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Started, this._filepath);

switch (GetFileType(filePath))
switch (GetFileType(_filepath))
{
case FileType.Pcap:
ReadPcapFile(filePath);
ReadPcapFile(_filepath);
break;
case FileType.PcapNG:
ReadPcapNGFile(filePath);
ReadPcapNGFile(_filepath);
break;
}

// Raise event for each Tcp session that was built.
// TODO: think about detecting complete sesions on the fly and raising
// events accordingly.
foreach (var session in this._tcpSessionsBuilder.Sessions)
{
TcpSessionArrived?.Invoke(this, new TcpSessionArivedEventArgs()
{
TcpSession = session
});
}
foreach (var session in this._udpStreamBuilder.Sessions)
{
UdpSessionArrived?.Invoke(this, new UdpSessionArrivedEventArgs()
{
UdpSession = session
});
}
this._udpStreamsBuilder.Sessions.AsParallel().ForAll(session => invokeAndClear(session));
this._tcpSessionsBuilder.Sessions.AsParallel().ForAll(session => invokeAndClear(session));


_processingPrecentsPredicator.NotifyAboutProcessedFile(new FileInfo(filePath));
RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Finished, filePath);

_processingPrecentsPredicator.NotifyAboutProcessedFile(new FileInfo(this._filepath));

RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Finished, _filepath);
}
catch (Exception ex)
{
RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Faild, filePath);
RaiseFileProcessingStatusChangedEvent(FileProcessingStatus.Faild, _filepath);
}
}

Expand Down Expand Up @@ -162,6 +152,7 @@ private void ReadPcapFile(string filepath)
}
private void ConvertPacket(object sender, IPacket packet)
{

var _packet_ether = PacketDotNet.Packet.ParsePacket(PacketDotNet.LinkLayers.Ethernet, packet.Data);
var _packet_raw = PacketDotNet.Packet.ParsePacket(PacketDotNet.LinkLayers.Raw, packet.Data);

Expand Down Expand Up @@ -226,7 +217,8 @@ void ProcessPacket(PacketDotNet.Packet packet)

if (this.BuildUdpSessions)
{
this._udpStreamBuilder.HandlePacket(udpPacket);

_udpStreamsBuilder.HandlePacket(udpPacket);
}
_processingPrecentsPredicator.NotifyAboutProcessedData(packet.Bytes.Length);
}
Expand All @@ -249,7 +241,7 @@ void ProcessPacket(PacketDotNet.Packet packet)

if (this.BuildTcpSessions)
{
this._tcpSessionsBuilder.HandlePacket(tcpPacket);
_tcpSessionsBuilder.HandlePacket(tcpPacket);
}

_processingPrecentsPredicator.NotifyAboutProcessedData(packet.Bytes.Length);
Expand Down
Loading