From ca84d431e47570181529dcf1dd9ebd37429d46d6 Mon Sep 17 00:00:00 2001 From: xorhex Date: Sat, 13 Nov 2021 18:19:56 +0100 Subject: [PATCH] Added two more sources: FileScanIO and VxShare Fixed a bug with the Inquest downloader. Fixed a bug with the UnpacMe downloader. Fixed a bug with the Malpedia downloader. Added some sanity checks when parsing an input file. --- README.md | 2 + config.go | 26 +++++++- download.go | 163 ++++++++++++++++++++++++++++++++++++++++++++++++-- history.go | 46 +++++++------- mlget.go | 11 ++-- mlget_test.go | 67 ++++++++++++++++++++- 6 files changed, 278 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 0571b64..2f2e1da 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Use mlget to query multiple sources for a given malware hash and download it. T Currently queries: - cp (Cape Sandbox) + - fs (File Scan) - ha (Hybrid Analysis) - iq (Inquest Labs) - js (Joe Sandbox) @@ -45,6 +46,7 @@ Currently queries: - tg (Triage) - um (UnpacMe) - vt (VirusTotal) + - vx (VxShare) Only Malware Bazaar and Objective-See does not require a key, the rest require a key. The config file needs to be placed in the user's home directory (essentially where `os.UserHomeDir()` resolves to). diff --git a/config.go b/config.go index 71bd8ab..a10453c 100644 --- a/config.go +++ b/config.go @@ -67,6 +67,7 @@ const ( NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected CapeSandbox + FileScanIo HybridAnalysis InQuest JoeSandbox @@ -79,6 +80,7 @@ const ( Triage UnpacMe VirusTotal + VxShare //UploadMWDB must always be last, or other things won't work as expected UploadMWDB @@ -127,12 +129,16 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h found, filename = capesandbox(mcr.Host, mcr.Api, hash) case ObjectiveSee: if len(osq.Malware) > 0 { - found, filename = objectivesee(osq, hash, doNotExtract, "infect3d") + found, filename = objectivesee(osq, hash, doNotExtract) } case UnpacMe: found, filename = unpacme(mcr.Host, mcr.Api, hash) case Malpedia: found, filename = malpedia(mcr.Host, mcr.Api, hash) + case VxShare: + found, filename = vxshare(mcr.Host, mcr.Api, hash, doNotExtract, "infected") + case FileScanIo: + found, filename = filescanio(mcr.Host, mcr.Api, hash, doNotExtract) case UploadMWDB: found, filename = mwdb(mcr.Host, mcr.Api, hash) } @@ -181,7 +187,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { case CapeSandbox: default_url = "https://www.capesandbox.com/apiv2" case JoeSandbox: - default_url = "https://jbxcloud.joesecurity.org/api" + default_url = "https://joesecurity.org/api" case InQuest: default_url = "https://labs.inquest.net/api" case HybridAnalysis: @@ -198,6 +204,10 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { default_url = "https://api.unpac.me/api/v1" case Malpedia: default_url = "https://malpedia.caad.fkie.fraunhofer.de/api" + case VxShare: + default_url = "https://virusshare.com/apiv2" + case FileScanIo: + default_url = "https://www.filescan.io/api" } if default_url != "" { fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url) @@ -246,6 +256,10 @@ func (malrepo MalwareRepoType) String() string { return "UnpacMe" case Malpedia: return "Malpedia" + case VxShare: + return "VxShare" + case FileScanIo: + return "FileScanIo" case UploadMWDB: return "UploadMWDB" @@ -323,6 +337,10 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType { return UnpacMe case strings.ToLower("mp"): return Malpedia + case strings.ToLower("vx"): + return VxShare + case strings.ToLower("fs"): + return FileScanIo } return NotSupported } @@ -355,6 +373,10 @@ func getMalwareRepoByName(name string) MalwareRepoType { return UnpacMe case strings.ToLower("Malpedia"): return Malpedia + case strings.ToLower("VxShare"): + return VxShare + case strings.ToLower("FileScanIo"): + return FileScanIo case strings.ToLower("UploadMWDB"): return UploadMWDB } diff --git a/download.go b/download.go index 0dd4090..e3537b6 100644 --- a/download.go +++ b/download.go @@ -26,8 +26,8 @@ type JoeSandboxQueryData struct { } type InquestLabsQuery struct { - Data *InquestLabsQueryData `json:"data"` - Success string `json:"success"` + Data []InquestLabsQueryData `json:"data"` + Success bool `json:"success"` } type InquestLabsQueryData struct { @@ -126,7 +126,7 @@ func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) { } } -func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool, password string) (bool, string) { +func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool) (bool, string) { if hash.HashType != sha256 { fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n") } @@ -373,11 +373,19 @@ func inquestlabs(uri string, api string, hash Hash) (bool, string) { return false, "" } - if data.Data.Sha256 == "" { + if !data.Success { + return false, "" + } + + if len(data.Data) == 0 { + return false, "" + } + + if data.Data[0].Sha256 == "" { return false, "" } hash.HashType = sha256 - hash.Hash = data.Data.Sha256 + hash.Hash = data.Data[0].Sha256 fmt.Printf(" [-] Using hash %s\n", hash.Hash) } else if response.StatusCode == http.StatusForbidden { @@ -902,6 +910,134 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password st } } +func filescanio(uri string, api string, hash Hash, doNotExtract bool) (bool, string) { + if api == "" { + fmt.Println(" [!] !! Missing Key !!") + return false, "" + } + return filescaniodownload(uri, api, hash, doNotExtract) +} + +func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool) (bool, string) { + query := "type=raw" + _, error := url.ParseQuery(query) + if error != nil { + fmt.Println(error) + return false, "" + } + + request, error := http.NewRequest("GET", uri+"/files/"+url.PathEscape(hash.Hash)+"?"+query, nil) + if error != nil { + fmt.Println(error) + return false, "" + } + + request.Header.Set("X-Api-Key", api) + + client := &http.Client{} + response, error := client.Do(request) + if error != nil { + fmt.Println(error) + return false, "" + } + + defer response.Body.Close() + + if response.StatusCode == 404 { + return false, "" + } else if response.StatusCode == 422 { + fmt.Printf(" [!] Validation Error.\n") + return false, "" + } else if response.StatusCode == http.StatusForbidden { + fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") + return false, "" + } + + error = writeToFile(response.Body, hash.Hash+".zip") + if error != nil { + fmt.Println(error) + return false, "" + } + + fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") + if doNotExtract { + return true, hash.Hash + ".zip" + } else { + fmt.Println(" [-] Extracting...") + files, err := extractPwdZip(hash.Hash, "infected") + if err != nil { + fmt.Println(err) + return false, "" + } else { + for _, f := range files { + fmt.Printf(" [-] Extracted %s\n", f.Name) + } + } + os.Remove(hash.Hash + ".zip") + return true, hash.Hash + } +} + +func vxshare(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { + if api == "" { + fmt.Println(" [!] !! Missing Key !!") + return false, "" + } + return vxsharedownload(uri, api, hash, doNotExtract, password) +} + +func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { + query := "apikey=" + url.QueryEscape(api) + "&hash=" + url.QueryEscape(hash.Hash) + _, error := url.ParseQuery(query) + if error != nil { + fmt.Println(error) + return false, "" + } + + client := &http.Client{} + response, error := client.Get(uri + "/download?" + query) + if error != nil { + fmt.Println(error) + return false, "" + } + + defer response.Body.Close() + + if response.StatusCode == 404 { + return false, "" + } else if response.StatusCode == 204 { + fmt.Printf(" [!] Request rate limit exceeded. You are making more requests than are allowed or have exceeded your quota.\n") + return false, "" + } else if response.StatusCode == http.StatusForbidden { + fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") + return false, "" + } + + error = writeToFile(response.Body, hash.Hash+".zip") + if error != nil { + fmt.Println(error) + return false, "" + } + + fmt.Printf(" [+] Downloaded %s\n", hash.Hash) + if doNotExtract { + return true, hash.Hash + ".zip" + } else { + fmt.Println(" [-] Extracting...") + files, err := extractPwdZip(hash.Hash, password) + if err != nil { + fmt.Println(err) + return false, "" + } else { + for _, f := range files { + fmt.Printf(" [-] Extracted %s\n", f.Name) + } + } + os.Remove(hash.Hash + ".zip") + return true, hash.Hash + } +} + func unpacme(uri string, api string, hash Hash) (bool, string) { if api == "" { fmt.Println(" [!] !! Missing Key !!") @@ -910,6 +1046,7 @@ func unpacme(uri string, api string, hash Hash) (bool, string) { if hash.HashType != sha256 { fmt.Printf(" [!] UnpacMe only supports SHA256\n Skipping\n") + return false, "" } return unpacmeDownload(uri, api, hash) @@ -933,6 +1070,13 @@ func unpacmeDownload(uri string, api string, hash Hash) (bool, string) { defer response.Body.Close() + if response.StatusCode == 404 { + return false, "" + } else if response.StatusCode == http.StatusForbidden { + fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") + return false, "" + } + error = writeToFile(response.Body, hash.Hash) if error != nil { fmt.Println(error) @@ -975,6 +1119,13 @@ func malpediaDownload(uri string, api string, hash Hash) (bool, string) { defer response.Body.Close() + if response.StatusCode == 404 { + return false, "" + } else if response.StatusCode == http.StatusForbidden { + fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") + return false, "" + } + error = writeToFile(response.Body, hash.Hash) if error != nil { fmt.Println(error) @@ -1012,7 +1163,7 @@ func extractPwdZip(hash string, password string) ([]*zip.File, error) { for _, f := range r.File { if f.IsEncrypted() { - f.SetPassword("infected") + f.SetPassword(password) } r, err := f.Open() diff --git a/history.go b/history.go index 328d574..114c503 100644 --- a/history.go +++ b/history.go @@ -44,30 +44,34 @@ func parseFileForHashEntries(filename string) ([]Hash, error) { scanner := bufio.NewScanner(file) for scanner.Scan() { // internally, it advances token based on sperator text := scanner.Text() - hash := strings.FieldsFunc(text, f)[0] - tags := []string{} - comments := []string{} - if len(strings.FieldsFunc(text, f)) > 1 { - fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))] - tagSection := false - commentSection := false - for _, f := range fields { - if f == "TAGS" { - tagSection = true - commentSection = false - } else if f == "COMMENTS" { - tagSection = false - commentSection = true - } else if f != "TAGS" && f != "COMMENTS" && tagSection { - tags = append(tags, f) - } else if f != "TAGS" && f != "COMMENTS" && commentSection { - comments = append(comments, f) + if len(strings.TrimSpace(text)) > 0 { + hash := strings.FieldsFunc(strings.TrimSpace(text), f)[0] + tags := []string{} + comments := []string{} + if len(strings.FieldsFunc(text, f)) > 1 { + fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))] + tagSection := false + commentSection := false + for _, f := range fields { + if f == "TAGS" { + tagSection = true + commentSection = false + } else if f == "COMMENTS" { + tagSection = false + commentSection = true + } else if f != "TAGS" && f != "COMMENTS" && tagSection { + tags = append(tags, f) + } else if f != "TAGS" && f != "COMMENTS" && commentSection { + comments = append(comments, f) + } } } + pHash := Hash{} + pHash, err = parseFileHashEntry(hash, tags, comments) + if err == nil { + hashes = append(hashes, pHash) + } } - pHash := Hash{} - pHash, err = parseFileHashEntry(hash, tags, comments) - hashes = append(hashes, pHash) } return hashes, nil } diff --git a/mlget.go b/mlget.go index 08c0836..18f61cd 100644 --- a/mlget.go +++ b/mlget.go @@ -24,7 +24,7 @@ var tagsFlag []string var commentsFlag []string var versionFlag bool -var version string = "2.4" +var version string = "2.5" func usage() { fmt.Println("mlget - A command line tool to download malware from a variety of sources") @@ -40,7 +40,7 @@ func usage() { } func init() { - flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - cp (Cape Sandbox)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - vt (VirusTotal)\nIf omitted, all services will be tried.") + flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - cp (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)") flag.BoolVar(&helpFlag, "help", false, "Print the help message") @@ -129,7 +129,8 @@ func main() { hashes, _ = addHash(hashes, hsh) } } - } else if readFromFileAndUpdateWithNotFoundHashesFlag != "" { + } + if readFromFileAndUpdateWithNotFoundHashesFlag != "" { hshs, err := parseFileForHashEntries(readFromFileAndUpdateWithNotFoundHashesFlag) if err != nil { fmt.Printf("Error reading from %s\n", readFromFileAndUpdateWithNotFoundHashesFlag) @@ -152,8 +153,8 @@ func main() { return } - for _, h := range hashes.Hashes { - fmt.Printf("\nLook up %s (%s)\n", h.Hash, h.HashType) + for idx, h := range hashes.Hashes { + fmt.Printf("\nLook up %s (%s) - (%d of %d)\n", h.Hash, h.HashType, idx+1, len(hashes.Hashes)) if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { if SyncSampleAcrossUploadMWDBsIfExists(cfg, h) { diff --git a/mlget_test.go b/mlget_test.go index 72baf9b..f1e9de2 100644 --- a/mlget_test.go +++ b/mlget_test.go @@ -67,7 +67,7 @@ func TestCapeSandbox(t *testing.T) { } } -func TestInquestLabs(t *testing.T) { +func TestInquestLabsLookUp(t *testing.T) { home, _ := os.UserHomeDir() cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) if err != nil { @@ -75,7 +75,27 @@ func TestInquestLabs(t *testing.T) { t.Errorf("%v", err) } - hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} + hash := Hash{HashType: md5, Hash: "b3f868fa1af24f270e3ecc0ecb79325e"} + + var osq ObjectiveSeeQuery + result, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) + + if !result { + t.Errorf("InquestLabs failed") + } else { + os.Remove(hash.Hash) + } +} + +func TestInquestLabsNoLookUp(t *testing.T) { + home, _ := os.UserHomeDir() + cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + hash := Hash{HashType: sha256, Hash: "6b425804d43bb369211bbec59808807730a908804ca9b8c09081139179bbc868"} var osq ObjectiveSeeQuery result, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) @@ -215,7 +235,8 @@ func TestMalwareBazaar(t *testing.T) { t.Errorf("%v", err) } - hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} + //hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} + hash := Hash{HashType: sha256, Hash: "bbe855f9259345af18de5f2cfd759eb78782b664bb22c43f19177dab51d782da"} var osq ObjectiveSeeQuery result, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) @@ -266,3 +287,43 @@ func TestUnpacme(t *testing.T) { os.Remove(hash.Hash) } } + +func TestVxShare(t *testing.T) { + home, _ := os.UserHomeDir() + cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + hash := Hash{HashType: sha256, Hash: "1c11c963a417674e1414bac05fdbfa5cfa09f92c7b0d9882aeb55ce2a058d668"} + + var osq ObjectiveSeeQuery + result, _ := VxShare.QueryAndDownload(cfg, hash, false, osq) + + if !result { + t.Errorf("VxShare failed") + } else { + os.Remove(hash.Hash) + } +} + +func TestFileScanIo(t *testing.T) { + home, _ := os.UserHomeDir() + cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + hash := Hash{HashType: sha256, Hash: "2799af2efd698da215afc9c88da3b1e84b00137433d9444a5c11d69092b3f80d"} + + var osq ObjectiveSeeQuery + result, _ := FileScanIo.QueryAndDownload(cfg, hash, false, osq) + + if !result { + t.Errorf("FileScanIo failed") + } else { + os.Remove(hash.Hash) + } +}