Skip to content

Commit

Permalink
Added functionality to lookup malware
Browse files Browse the repository at this point in the history 
on Objective-See.
  • Loading branch information
xorhex committed Oct 18, 2021
1 parent 8a89892 commit af48679
Show file tree
Hide file tree
Showing 5 changed files with 206 additions and 27 deletions.
34 changes: 27 additions & 7 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,15 @@ const (
Malshare
VirusTotal
Polyswarm
ObjectiveSee

//UploadMWDB must always be last, or other things won't work as expected
UploadMWDB
)

var MalwareRepoList = []MalwareRepoType{JoeSandbox, MWDB, HybridAnalysis, CapeSandbox, InQuest, MalwareBazaar, Triage, Malshare, VirusTotal, Polyswarm, UploadMWDB}
var MalwareRepoList = []MalwareRepoType{JoeSandbox, MWDB, HybridAnalysis, CapeSandbox, InQuest, MalwareBazaar, Triage, Malshare, VirusTotal, Polyswarm, ObjectiveSee, UploadMWDB}

func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool) (bool, string) {
func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, osq ObjectiveSeeQuery) (bool, string) {
matchingConfigRepos := getConfigsByType(malrepo, repos)
if len(matchingConfigRepos) == 0 {
fmt.Printf(" [!] %s is not found in the yml config file\n", malrepo)
Expand All @@ -93,7 +94,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
fmt.Printf(" [*] %s: %s\n", mcr.Type, mcr.Host)
switch malrepo {
case MalwareBazaar:
found, filename = malwareBazaar(mcr.Host, hash, doNotExtract)
found, filename = malwareBazaar(mcr.Host, hash, doNotExtract, "infected")
case MWDB:
found, filename = mwdb(mcr.Host, mcr.Api, hash)
case Malshare:
Expand All @@ -112,6 +113,10 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
found, filename = joesandbox(mcr.Host, mcr.Api, hash)
case CapeSandbox:
found, filename = capesandbox(mcr.Host, mcr.Api, hash)
case ObjectiveSee:
if len(osq.Malware) > 0 {
found, filename = objectivesee(osq, hash, doNotExtract, "infect3d")
}
case UploadMWDB:
found, filename = mwdb(mcr.Host, mcr.Api, hash)
}
Expand All @@ -130,6 +135,10 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool
if repo.Host != "" {
return true
}
case ObjectiveSee:
if repo.Host != "" {
return true
}
default:
if repo.Host != "" && repo.Api != "" {
return true
Expand Down Expand Up @@ -167,6 +176,8 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
default_url = "https://www.virustotal.com/api/v3"
case Polyswarm:
default_url = "https://api.polyswarm.network/v2"
case ObjectiveSee:
default_url = "https://objective-see.com/malware.json"
}
if default_url != "" {
fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url)
Expand Down Expand Up @@ -209,6 +220,8 @@ func (malrepo MalwareRepoType) String() string {
return "VirusTotal"
case Polyswarm:
return "Polyswarm"
case ObjectiveSee:
return "ObjectiveSee"
case UploadMWDB:
return "UploadMWDB"

Expand All @@ -229,24 +242,26 @@ func printAllowedMalwareRepoTypeOptions() {
}
}

func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUploadMWDBEntries bool) (bool, string) {
func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUploadMWDBEntries bool, osq ObjectiveSeeQuery) (bool, string) {
found := false
filename := ""
sort.Slice(repos[:], func(i, j int) bool {
return repos[i].QueryOrder < repos[j].QueryOrder
})

// Hack for now
// Due to Multiple entries of the same type, for each type instance in the config it will
// try to download for type (number of entries for type in config squared)
// Due to Multiple entries of the same type, for each type instance in the config it will
// try to download for type the number of entries for type in config squared
// This array is meant to ensure that for each type it will only try it once
var completedTypes []MalwareRepoType

for _, repo := range repos {
if repo.Type == UploadMWDB.String() && skipUploadMWDBEntries {
continue
}
mr := getMalwareRepoByName(repo.Type)
if !contains(completedTypes, mr) {
found, filename = mr.QueryAndDownload(repos, hash, doNotExtract)
found, filename = mr.QueryAndDownload(repos, hash, doNotExtract, osq)
if found {
break
}
Expand Down Expand Up @@ -278,6 +293,8 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType {
return VirusTotal
case strings.ToLower("ps"):
return Polyswarm
case strings.ToLower("os"):
return ObjectiveSee
}
return NotSupported
}
Expand All @@ -304,6 +321,8 @@ func getMalwareRepoByName(name string) MalwareRepoType {
return VirusTotal
case strings.ToLower("Polyswarm"):
return Polyswarm
case strings.ToLower("ObjectiveSee"):
return ObjectiveSee
case strings.ToLower("UploadMWDB"):
return UploadMWDB
}
Expand All @@ -325,6 +344,7 @@ type RepositoryConfigEntry struct {
Host string `yaml:"url"`
Api string `yaml:"api"`
QueryOrder int `yaml:"queryorder"`
Password string `yaml:"pwd"`
}

func LoadConfig(filename string) ([]RepositoryConfigEntry, error) {
Expand Down
125 changes: 120 additions & 5 deletions download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"net/http"
"net/url"
"os"
"regexp"

"github.com/yeka/zip"
)
Expand Down Expand Up @@ -63,6 +64,111 @@ type TriageQueryData struct {
Filename string `json:"filename"`
}

type ObjectiveSeeQuery struct {
Malware []ObjectiveSeeData `json:"malware"`
}

type ObjectiveSeeData struct {
Name string `json:"name"`
Type string `json:"type"`
VirusTotal string `json:"virusTotal"`
MoreInfo string `json:"moreInfo"`
Download string `json:"download"`
Sha256 string
}

func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) {

fmt.Printf("Downloading Objective-See Malware json from: %s\n\n", uri)

client := &http.Client{}
response, error := client.Get(uri)
if error != nil {
fmt.Println(error)
return ObjectiveSeeQuery{}, error
}

defer response.Body.Close()

if response.StatusCode == http.StatusOK {
byteValue, _ := ioutil.ReadAll(response.Body)

var data = ObjectiveSeeQuery{}
error = json.Unmarshal(byteValue, &data)

var unmarshalTypeError *json.UnmarshalTypeError
if errors.As(error, &unmarshalTypeError) {
fmt.Printf(" [!] Failed unmarshaling json. Likely due to the format of the Objective-See json file changing\n")
fmt.Printf(" %s\n", byteValue)

} else if error != nil {
fmt.Println(error)
return ObjectiveSeeQuery{}, error
}

fmt.Printf(" Parsing VirusTotal Links for sha256 hashes\n")
re := regexp.MustCompile("[A-Fa-f0-9]{64}")
for k, item := range data.Malware {
if len(item.VirusTotal) > 0 {
matches := re.FindStringSubmatch(item.VirusTotal)
if len(matches) == 1 {
data.Malware[k].Sha256 = matches[0]
}
}
if len(data.Malware[k].Sha256) == 0 {
fmt.Printf(" [!] SHA256 not found for %s : %s\n VirusTotal Link: %s\n", item.Name, item.Type, item.VirusTotal)
}
}

return data, nil
} else {
return ObjectiveSeeQuery{}, fmt.Errorf("unable to download objective-see json file")
}
}

func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool, password string) (bool, string) {
if hash.HashType != sha256 {
fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n")
}

item, found := findHashInObjectiveSeeList(data.Malware, hash)

if !found {
return false, ""
}

if !doNotExtract {
fmt.Printf(" [!] Extraction is not supported for Objective-See\n Try again but with the --noextraction flag\n")
return false, ""
}

client := &http.Client{}
response, error := client.Get(item.Download)
if error != nil {
fmt.Println(error)
return false, ""
}

defer response.Body.Close()

if response.StatusCode != http.StatusOK {
return false, ""
}

error = writeToFile(response.Body, hash.Hash+".zip")
if error != nil {
fmt.Println(error)
return false, ""
}

fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip")
if doNotExtract {
return true, hash.Hash + ".zip"
} else {
return false, ""
}
}

func joesandbox(uri string, api string, hash Hash) (bool, string) {
if api == "" {
fmt.Println(" [!] !! Missing Key !!")
Expand Down Expand Up @@ -701,7 +807,7 @@ func malshareDownload(uri string, api string, hash Hash) (bool, string) {
}
}

func malwareBazaar(url string, hash Hash, doNotExtract bool) (bool, string) {
func malwareBazaar(url string, hash Hash, doNotExtract bool, password string) (bool, string) {
if hash.HashType != sha256 {
fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash)

Expand Down Expand Up @@ -747,12 +853,12 @@ func malwareBazaar(url string, hash Hash, doNotExtract bool) (bool, string) {
}

if hash.HashType == sha256 {
return malwareBazaarDownload(url, hash, doNotExtract)
return malwareBazaarDownload(url, hash, doNotExtract, password)
}
return false, ""
}

func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool) (bool, string) {
func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) {
query := "query=get_file&sha256_hash=" + hash.Hash
values, error := url.ParseQuery(query)
if error != nil {
Expand Down Expand Up @@ -784,7 +890,7 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool) (bool, stri
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
files, err := extractPwdZip(hash.Hash)
files, err := extractPwdZip(hash.Hash, password)
if err != nil {
fmt.Println(err)
return false, ""
Expand Down Expand Up @@ -813,7 +919,7 @@ func extractGzip(hash string) error {
return err
}

func extractPwdZip(hash string) ([]*zip.File, error) {
func extractPwdZip(hash string, password string) ([]*zip.File, error) {

r, err := zip.OpenReader(hash + ".zip")
if err != nil {
Expand Down Expand Up @@ -846,3 +952,12 @@ func extractPwdZip(hash string) ([]*zip.File, error) {
}
return files, nil
}

func findHashInObjectiveSeeList(list []ObjectiveSeeData, hash Hash) (ObjectiveSeeData, bool) {
for _, item := range list {
if item.Sha256 == hash.Hash {
return item, true
}
}
return ObjectiveSeeData{}, false
}
21 changes: 17 additions & 4 deletions mlget.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func usage() {
}

func init() {
flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - tg (Triage)\n - mb (Malware Bazaar)\n - ms (Malshare)\n - ha (Hybird Anlysis)\n - vt (VirusTotal)\n - cp (Cape Sandbox)\n - mw (Malware Database)\n - ps (PolySwarm)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\nIf omitted, all services will be tried.")
flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - tg (Triage)\n - mb (Malware Bazaar)\n - ms (Malshare)\n - ha (Hybird Anlysis)\n - vt (VirusTotal)\n - cp (Cape Sandbox)\n - mw (Malware Database)\n - ps (PolySwarm)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - os (Objective-See)\nIf omitted, all services will be tried.")
flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)")
flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)")
flag.BoolVar(&helpFlag, "help", false, "Print the help message")
Expand Down Expand Up @@ -90,6 +90,19 @@ func main() {
return
}

var osq ObjectiveSeeQuery
osConfigs := getConfigsByType(ObjectiveSee, cfg)
// Can have multiple Objective-See configs but only the first one to load will be used
for _, osc := range osConfigs {
osq, err = loadObjectiveSeeJson(osc.Host)
if err != nil {
fmt.Println("Unable to load Objective-See json data. Skipping...")
continue
}
fmt.Println("")
break
}

hashes := parseArgHashes(args, tagsFlag, commentsFlag)

if inputFileFlag != "" {
Expand Down Expand Up @@ -138,9 +151,9 @@ func main() {
continue
}

fmt.Printf("Looking on %s\n", apiFlag)
fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag))

found, filename := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag)
found, filename := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag, osq)
if !found {
fmt.Println(" [!] Not Found")
notFoundHashes, _ = addHash(notFoundHashes, h)
Expand All @@ -156,7 +169,7 @@ func main() {
} else {
fmt.Println("Querying all services")

found, filename := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag)
found, filename := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag, osq)
if found {
if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag {
err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag)
Expand Down
Loading

0 comments on commit af48679

Please sign in to comment.