diff --git a/config.go b/config.go index c430c57..6e2060d 100644 --- a/config.go +++ b/config.go @@ -75,14 +75,15 @@ const ( Malshare VirusTotal Polyswarm + ObjectiveSee //UploadMWDB must always be last, or other things won't work as expected UploadMWDB ) -var MalwareRepoList = []MalwareRepoType{JoeSandbox, MWDB, HybridAnalysis, CapeSandbox, InQuest, MalwareBazaar, Triage, Malshare, VirusTotal, Polyswarm, UploadMWDB} +var MalwareRepoList = []MalwareRepoType{JoeSandbox, MWDB, HybridAnalysis, CapeSandbox, InQuest, MalwareBazaar, Triage, Malshare, VirusTotal, Polyswarm, ObjectiveSee, UploadMWDB} -func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool) (bool, string) { +func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, osq ObjectiveSeeQuery) (bool, string) { matchingConfigRepos := getConfigsByType(malrepo, repos) if len(matchingConfigRepos) == 0 { fmt.Printf(" [!] %s is not found in the yml config file\n", malrepo) @@ -93,7 +94,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h fmt.Printf(" [*] %s: %s\n", mcr.Type, mcr.Host) switch malrepo { case MalwareBazaar: - found, filename = malwareBazaar(mcr.Host, hash, doNotExtract) + found, filename = malwareBazaar(mcr.Host, hash, doNotExtract, "infected") case MWDB: found, filename = mwdb(mcr.Host, mcr.Api, hash) case Malshare: @@ -112,6 +113,10 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h found, filename = joesandbox(mcr.Host, mcr.Api, hash) case CapeSandbox: found, filename = capesandbox(mcr.Host, mcr.Api, hash) + case ObjectiveSee: + if len(osq.Malware) > 0 { + found, filename = objectivesee(osq, hash, doNotExtract, "infect3d") + } case UploadMWDB: found, filename = mwdb(mcr.Host, mcr.Api, hash) } @@ -130,6 +135,10 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool if repo.Host != "" { return true } + case ObjectiveSee: + if repo.Host != "" { + return true + } default: if repo.Host != "" && repo.Api != "" { return true @@ -167,6 +176,8 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { default_url = "https://www.virustotal.com/api/v3" case Polyswarm: default_url = "https://api.polyswarm.network/v2" + case ObjectiveSee: + default_url = "https://objective-see.com/malware.json" } if default_url != "" { fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url) @@ -209,6 +220,8 @@ func (malrepo MalwareRepoType) String() string { return "VirusTotal" case Polyswarm: return "Polyswarm" + case ObjectiveSee: + return "ObjectiveSee" case UploadMWDB: return "UploadMWDB" @@ -229,24 +242,26 @@ func printAllowedMalwareRepoTypeOptions() { } } -func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUploadMWDBEntries bool) (bool, string) { +func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUploadMWDBEntries bool, osq ObjectiveSeeQuery) (bool, string) { found := false filename := "" sort.Slice(repos[:], func(i, j int) bool { return repos[i].QueryOrder < repos[j].QueryOrder }) + // Hack for now - // Due to Multiple entries of the same type, for each type instance in the config it will - // try to download for type (number of entries for type in config squared) + // Due to Multiple entries of the same type, for each type instance in the config it will + // try to download for type the number of entries for type in config squared // This array is meant to ensure that for each type it will only try it once var completedTypes []MalwareRepoType + for _, repo := range repos { if repo.Type == UploadMWDB.String() && skipUploadMWDBEntries { continue } mr := getMalwareRepoByName(repo.Type) if !contains(completedTypes, mr) { - found, filename = mr.QueryAndDownload(repos, hash, doNotExtract) + found, filename = mr.QueryAndDownload(repos, hash, doNotExtract, osq) if found { break } @@ -278,6 +293,8 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType { return VirusTotal case strings.ToLower("ps"): return Polyswarm + case strings.ToLower("os"): + return ObjectiveSee } return NotSupported } @@ -304,6 +321,8 @@ func getMalwareRepoByName(name string) MalwareRepoType { return VirusTotal case strings.ToLower("Polyswarm"): return Polyswarm + case strings.ToLower("ObjectiveSee"): + return ObjectiveSee case strings.ToLower("UploadMWDB"): return UploadMWDB } @@ -325,6 +344,7 @@ type RepositoryConfigEntry struct { Host string `yaml:"url"` Api string `yaml:"api"` QueryOrder int `yaml:"queryorder"` + Password string `yaml:"pwd"` } func LoadConfig(filename string) ([]RepositoryConfigEntry, error) { diff --git a/download.go b/download.go index 15e78ec..7f8996f 100644 --- a/download.go +++ b/download.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "os" + "regexp" "github.com/yeka/zip" ) @@ -63,6 +64,111 @@ type TriageQueryData struct { Filename string `json:"filename"` } +type ObjectiveSeeQuery struct { + Malware []ObjectiveSeeData `json:"malware"` +} + +type ObjectiveSeeData struct { + Name string `json:"name"` + Type string `json:"type"` + VirusTotal string `json:"virusTotal"` + MoreInfo string `json:"moreInfo"` + Download string `json:"download"` + Sha256 string +} + +func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) { + + fmt.Printf("Downloading Objective-See Malware json from: %s\n\n", uri) + + client := &http.Client{} + response, error := client.Get(uri) + if error != nil { + fmt.Println(error) + return ObjectiveSeeQuery{}, error + } + + defer response.Body.Close() + + if response.StatusCode == http.StatusOK { + byteValue, _ := ioutil.ReadAll(response.Body) + + var data = ObjectiveSeeQuery{} + error = json.Unmarshal(byteValue, &data) + + var unmarshalTypeError *json.UnmarshalTypeError + if errors.As(error, &unmarshalTypeError) { + fmt.Printf(" [!] Failed unmarshaling json. Likely due to the format of the Objective-See json file changing\n") + fmt.Printf(" %s\n", byteValue) + + } else if error != nil { + fmt.Println(error) + return ObjectiveSeeQuery{}, error + } + + fmt.Printf(" Parsing VirusTotal Links for sha256 hashes\n") + re := regexp.MustCompile("[A-Fa-f0-9]{64}") + for k, item := range data.Malware { + if len(item.VirusTotal) > 0 { + matches := re.FindStringSubmatch(item.VirusTotal) + if len(matches) == 1 { + data.Malware[k].Sha256 = matches[0] + } + } + if len(data.Malware[k].Sha256) == 0 { + fmt.Printf(" [!] SHA256 not found for %s : %s\n VirusTotal Link: %s\n", item.Name, item.Type, item.VirusTotal) + } + } + + return data, nil + } else { + return ObjectiveSeeQuery{}, fmt.Errorf("unable to download objective-see json file") + } +} + +func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool, password string) (bool, string) { + if hash.HashType != sha256 { + fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n") + } + + item, found := findHashInObjectiveSeeList(data.Malware, hash) + + if !found { + return false, "" + } + + if !doNotExtract { + fmt.Printf(" [!] Extraction is not supported for Objective-See\n Try again but with the --noextraction flag\n") + return false, "" + } + + client := &http.Client{} + response, error := client.Get(item.Download) + if error != nil { + fmt.Println(error) + return false, "" + } + + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return false, "" + } + + error = writeToFile(response.Body, hash.Hash+".zip") + if error != nil { + fmt.Println(error) + return false, "" + } + + fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") + if doNotExtract { + return true, hash.Hash + ".zip" + } else { + return false, "" + } +} + func joesandbox(uri string, api string, hash Hash) (bool, string) { if api == "" { fmt.Println(" [!] !! Missing Key !!") @@ -701,7 +807,7 @@ func malshareDownload(uri string, api string, hash Hash) (bool, string) { } } -func malwareBazaar(url string, hash Hash, doNotExtract bool) (bool, string) { +func malwareBazaar(url string, hash Hash, doNotExtract bool, password string) (bool, string) { if hash.HashType != sha256 { fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) @@ -747,12 +853,12 @@ func malwareBazaar(url string, hash Hash, doNotExtract bool) (bool, string) { } if hash.HashType == sha256 { - return malwareBazaarDownload(url, hash, doNotExtract) + return malwareBazaarDownload(url, hash, doNotExtract, password) } return false, "" } -func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool) (bool, string) { +func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) { query := "query=get_file&sha256_hash=" + hash.Hash values, error := url.ParseQuery(query) if error != nil { @@ -784,7 +890,7 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool) (bool, stri return true, hash.Hash + ".zip" } else { fmt.Println(" [-] Extracting...") - files, err := extractPwdZip(hash.Hash) + files, err := extractPwdZip(hash.Hash, password) if err != nil { fmt.Println(err) return false, "" @@ -813,7 +919,7 @@ func extractGzip(hash string) error { return err } -func extractPwdZip(hash string) ([]*zip.File, error) { +func extractPwdZip(hash string, password string) ([]*zip.File, error) { r, err := zip.OpenReader(hash + ".zip") if err != nil { @@ -846,3 +952,12 @@ func extractPwdZip(hash string) ([]*zip.File, error) { } return files, nil } + +func findHashInObjectiveSeeList(list []ObjectiveSeeData, hash Hash) (ObjectiveSeeData, bool) { + for _, item := range list { + if item.Sha256 == hash.Hash { + return item, true + } + } + return ObjectiveSeeData{}, false +} diff --git a/mlget.go b/mlget.go index e286a84..c877f6a 100644 --- a/mlget.go +++ b/mlget.go @@ -37,7 +37,7 @@ func usage() { } func init() { - flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - tg (Triage)\n - mb (Malware Bazaar)\n - ms (Malshare)\n - ha (Hybird Anlysis)\n - vt (VirusTotal)\n - cp (Cape Sandbox)\n - mw (Malware Database)\n - ps (PolySwarm)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\nIf omitted, all services will be tried.") + flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - tg (Triage)\n - mb (Malware Bazaar)\n - ms (Malshare)\n - ha (Hybird Anlysis)\n - vt (VirusTotal)\n - cp (Cape Sandbox)\n - mw (Malware Database)\n - ps (PolySwarm)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - os (Objective-See)\nIf omitted, all services will be tried.") flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)") flag.BoolVar(&helpFlag, "help", false, "Print the help message") @@ -90,6 +90,19 @@ func main() { return } + var osq ObjectiveSeeQuery + osConfigs := getConfigsByType(ObjectiveSee, cfg) + // Can have multiple Objective-See configs but only the first one to load will be used + for _, osc := range osConfigs { + osq, err = loadObjectiveSeeJson(osc.Host) + if err != nil { + fmt.Println("Unable to load Objective-See json data. Skipping...") + continue + } + fmt.Println("") + break + } + hashes := parseArgHashes(args, tagsFlag, commentsFlag) if inputFileFlag != "" { @@ -138,9 +151,9 @@ func main() { continue } - fmt.Printf("Looking on %s\n", apiFlag) + fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag)) - found, filename := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag) + found, filename := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag, osq) if !found { fmt.Println(" [!] Not Found") notFoundHashes, _ = addHash(notFoundHashes, h) @@ -156,7 +169,7 @@ func main() { } else { fmt.Println("Querying all services") - found, filename := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag) + found, filename := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag, osq) if found { if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag) diff --git a/mlget_test.go b/mlget_test.go index 1310d55..2c249ed 100644 --- a/mlget_test.go +++ b/mlget_test.go @@ -17,7 +17,8 @@ func TestJoeSandbox(t *testing.T) { hash := Hash{HashType: md5, Hash: "28eefc36104bebb595fb38cae21a7d0a"} - result, _ := JoeSandbox.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := JoeSandbox.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("JoeSandbox failed") @@ -26,6 +27,26 @@ func TestJoeSandbox(t *testing.T) { } } +func TestObjectiveSee(t *testing.T) { + home, _ := os.UserHomeDir() + cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + hash := Hash{HashType: sha256, Hash: "458a9ac086116fa011c1a7bd49ac15f386cd95e39eb6b7cd5c5125aef516c78c"} + + osq, _ := loadObjectiveSeeJson(getConfigsByType(ObjectiveSee, cfg)[0].Host) + result, _ := ObjectiveSee.QueryAndDownload(cfg, hash, true, osq) + + if !result { + t.Errorf("Objective-See failed") + } else { + os.Remove(hash.Hash) + } +} + func TestCapeSandbox(t *testing.T) { home, _ := os.UserHomeDir() cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) @@ -34,9 +55,10 @@ func TestCapeSandbox(t *testing.T) { t.Errorf("%v", err) } + var osq ObjectiveSeeQuery hash := Hash{HashType: md5, Hash: "28eefc36104bebb595fb38cae21a7d0a"} - result, _ := CapeSandbox.QueryAndDownload(cfg, hash, false) + result, _ := CapeSandbox.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("CapeSandbox failed") @@ -55,7 +77,8 @@ func TestInquestLabs(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := InQuest.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("InquestLabs failed") @@ -74,7 +97,8 @@ func TestVirusTotal(t *testing.T) { hash := Hash{HashType: sha256, Hash: "21cc9c0ae5f97b66d69f1ff99a4fed264551edfe0a5ce8d5449942bf8f0aefb2"} - result, _ := VirusTotal.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := VirusTotal.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("VirusTotal failed") @@ -93,7 +117,8 @@ func TestMWDB(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := MWDB.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := MWDB.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("MWDB failed") @@ -112,7 +137,8 @@ func TestPolyswarm(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := Polyswarm.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := Polyswarm.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("PolySwarm failed") @@ -131,7 +157,8 @@ func TestHybridAnalysis(t *testing.T) { hash := Hash{HashType: sha256, Hash: "ed2f501408a7a6e1a854c29c4b0bc5648a6aa8612432df829008931b3e34bf56"} - result, _ := HybridAnalysis.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := HybridAnalysis.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("HybridAnalysis failed") @@ -150,7 +177,8 @@ func TestTriage(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := Triage.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := Triage.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("Triage failed") @@ -169,7 +197,8 @@ func TestMalShare(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := Malshare.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := Malshare.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("Malshare failed") @@ -188,7 +217,8 @@ func TestMalwareBazaar(t *testing.T) { hash := Hash{HashType: sha256, Hash: "75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18"} - result, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false) + var osq ObjectiveSeeQuery + result, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) if !result { t.Errorf("Malshare failed") diff --git a/upload.go b/upload.go index bbf8784..ffc22f4 100644 --- a/upload.go +++ b/upload.go @@ -30,7 +30,8 @@ func SyncSampleAcrossUploadMWDBsIfExists(repos []RepositoryConfigEntry, hash Has } // A sample was found, now download and upload to the remaining instances if len(matchingConfigRepos) > 1 { - downloaded, filename := UploadMWDB.QueryAndDownload(matchingConfigRepos, hash, false) + var osq ObjectiveSeeQuery + downloaded, filename := UploadMWDB.QueryAndDownload(matchingConfigRepos, hash, false, osq) if downloaded { UploadSampleToMWDBs(matchingConfigRepos, filename, hash, true) }