Skip to content

Commit

Permalink
Added two more sources: FileScanIO and VxShare
Browse files Browse the repository at this point in the history 
Fixed a bug with the Inquest downloader.
Fixed a bug with the UnpacMe downloader.
Fixed a bug with the Malpedia downloader.
Added some sanity checks when parsing an input file.
  • Loading branch information
xorhex committed Nov 13, 2021
1 parent ac29bfd commit ca84d43
Show file tree
Hide file tree
Showing 6 changed files with 278 additions and 37 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ Use mlget to query multiple sources for a given malware hash and download it. T
Currently queries:

- cp (Cape Sandbox)
- fs (File Scan)
- ha (Hybrid Analysis)
- iq (Inquest Labs)
- js (Joe Sandbox)
Expand All @@ -45,6 +46,7 @@ Currently queries:
- tg (Triage)
- um (UnpacMe)
- vt (VirusTotal)
- vx (VxShare)

Only Malware Bazaar and Objective-See does not require a key, the rest require a key. The config file needs to be placed in the user's home directory (essentially where `os.UserHomeDir()` resolves to).

Expand Down
26 changes: 24 additions & 2 deletions config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ const (
NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected

CapeSandbox
FileScanIo
HybridAnalysis
InQuest
JoeSandbox
Expand All @@ -79,6 +80,7 @@ const (
Triage
UnpacMe
VirusTotal
VxShare

//UploadMWDB must always be last, or other things won't work as expected
UploadMWDB
Expand Down Expand Up @@ -127,12 +129,16 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
found, filename = capesandbox(mcr.Host, mcr.Api, hash)
case ObjectiveSee:
if len(osq.Malware) > 0 {
found, filename = objectivesee(osq, hash, doNotExtract, "infect3d")
found, filename = objectivesee(osq, hash, doNotExtract)
}
case UnpacMe:
found, filename = unpacme(mcr.Host, mcr.Api, hash)
case Malpedia:
found, filename = malpedia(mcr.Host, mcr.Api, hash)
case VxShare:
found, filename = vxshare(mcr.Host, mcr.Api, hash, doNotExtract, "infected")
case FileScanIo:
found, filename = filescanio(mcr.Host, mcr.Api, hash, doNotExtract)
case UploadMWDB:
found, filename = mwdb(mcr.Host, mcr.Api, hash)
}
Expand Down Expand Up @@ -181,7 +187,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
case CapeSandbox:
default_url = "https://www.capesandbox.com/apiv2"
case JoeSandbox:
default_url = "https://jbxcloud.joesecurity.org/api"
default_url = "https://joesecurity.org/api"
case InQuest:
default_url = "https://labs.inquest.net/api"
case HybridAnalysis:
Expand All @@ -198,6 +204,10 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
default_url = "https://api.unpac.me/api/v1"
case Malpedia:
default_url = "https://malpedia.caad.fkie.fraunhofer.de/api"
case VxShare:
default_url = "https://virusshare.com/apiv2"
case FileScanIo:
default_url = "https://www.filescan.io/api"
}
if default_url != "" {
fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url)
Expand Down Expand Up @@ -246,6 +256,10 @@ func (malrepo MalwareRepoType) String() string {
return "UnpacMe"
case Malpedia:
return "Malpedia"
case VxShare:
return "VxShare"
case FileScanIo:
return "FileScanIo"
case UploadMWDB:
return "UploadMWDB"

Expand Down Expand Up @@ -323,6 +337,10 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType {
return UnpacMe
case strings.ToLower("mp"):
return Malpedia
case strings.ToLower("vx"):
return VxShare
case strings.ToLower("fs"):
return FileScanIo
}
return NotSupported
}
Expand Down Expand Up @@ -355,6 +373,10 @@ func getMalwareRepoByName(name string) MalwareRepoType {
return UnpacMe
case strings.ToLower("Malpedia"):
return Malpedia
case strings.ToLower("VxShare"):
return VxShare
case strings.ToLower("FileScanIo"):
return FileScanIo
case strings.ToLower("UploadMWDB"):
return UploadMWDB
}
Expand Down
163 changes: 157 additions & 6 deletions download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ type JoeSandboxQueryData struct {
}

type InquestLabsQuery struct {
Data *InquestLabsQueryData `json:"data"`
Success string `json:"success"`
Data []InquestLabsQueryData `json:"data"`
Success bool `json:"success"`
}

type InquestLabsQueryData struct {
Expand Down Expand Up @@ -126,7 +126,7 @@ func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) {
}
}

func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool, password string) (bool, string) {
func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool) (bool, string) {
if hash.HashType != sha256 {
fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n")
}
Expand Down Expand Up @@ -373,11 +373,19 @@ func inquestlabs(uri string, api string, hash Hash) (bool, string) {
return false, ""
}

if data.Data.Sha256 == "" {
if !data.Success {
return false, ""
}

if len(data.Data) == 0 {
return false, ""
}

if data.Data[0].Sha256 == "" {
return false, ""
}
hash.HashType = sha256
hash.Hash = data.Data.Sha256
hash.Hash = data.Data[0].Sha256
fmt.Printf(" [-] Using hash %s\n", hash.Hash)

} else if response.StatusCode == http.StatusForbidden {
Expand Down Expand Up @@ -902,6 +910,134 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password st
}
}

func filescanio(uri string, api string, hash Hash, doNotExtract bool) (bool, string) {
if api == "" {
fmt.Println(" [!] !! Missing Key !!")
return false, ""
}
return filescaniodownload(uri, api, hash, doNotExtract)
}

func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool) (bool, string) {
query := "type=raw"
_, error := url.ParseQuery(query)
if error != nil {
fmt.Println(error)
return false, ""
}

request, error := http.NewRequest("GET", uri+"/files/"+url.PathEscape(hash.Hash)+"?"+query, nil)
if error != nil {
fmt.Println(error)
return false, ""
}

request.Header.Set("X-Api-Key", api)

client := &http.Client{}
response, error := client.Do(request)
if error != nil {
fmt.Println(error)
return false, ""
}

defer response.Body.Close()

if response.StatusCode == 404 {
return false, ""
} else if response.StatusCode == 422 {
fmt.Printf(" [!] Validation Error.\n")
return false, ""
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}

error = writeToFile(response.Body, hash.Hash+".zip")
if error != nil {
fmt.Println(error)
return false, ""
}

fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip")
if doNotExtract {
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
files, err := extractPwdZip(hash.Hash, "infected")
if err != nil {
fmt.Println(err)
return false, ""
} else {
for _, f := range files {
fmt.Printf(" [-] Extracted %s\n", f.Name)
}
}
os.Remove(hash.Hash + ".zip")
return true, hash.Hash
}
}

func vxshare(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) {
if api == "" {
fmt.Println(" [!] !! Missing Key !!")
return false, ""
}
return vxsharedownload(uri, api, hash, doNotExtract, password)
}

func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) {
query := "apikey=" + url.QueryEscape(api) + "&hash=" + url.QueryEscape(hash.Hash)
_, error := url.ParseQuery(query)
if error != nil {
fmt.Println(error)
return false, ""
}

client := &http.Client{}
response, error := client.Get(uri + "/download?" + query)
if error != nil {
fmt.Println(error)
return false, ""
}

defer response.Body.Close()

if response.StatusCode == 404 {
return false, ""
} else if response.StatusCode == 204 {
fmt.Printf(" [!] Request rate limit exceeded. You are making more requests than are allowed or have exceeded your quota.\n")
return false, ""
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}

error = writeToFile(response.Body, hash.Hash+".zip")
if error != nil {
fmt.Println(error)
return false, ""
}

fmt.Printf(" [+] Downloaded %s\n", hash.Hash)
if doNotExtract {
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
files, err := extractPwdZip(hash.Hash, password)
if err != nil {
fmt.Println(err)
return false, ""
} else {
for _, f := range files {
fmt.Printf(" [-] Extracted %s\n", f.Name)
}
}
os.Remove(hash.Hash + ".zip")
return true, hash.Hash
}
}

func unpacme(uri string, api string, hash Hash) (bool, string) {
if api == "" {
fmt.Println(" [!] !! Missing Key !!")
Expand All @@ -910,6 +1046,7 @@ func unpacme(uri string, api string, hash Hash) (bool, string) {

if hash.HashType != sha256 {
fmt.Printf(" [!] UnpacMe only supports SHA256\n Skipping\n")
return false, ""
}

return unpacmeDownload(uri, api, hash)
Expand All @@ -933,6 +1070,13 @@ func unpacmeDownload(uri string, api string, hash Hash) (bool, string) {

defer response.Body.Close()

if response.StatusCode == 404 {
return false, ""
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}

error = writeToFile(response.Body, hash.Hash)
if error != nil {
fmt.Println(error)
Expand Down Expand Up @@ -975,6 +1119,13 @@ func malpediaDownload(uri string, api string, hash Hash) (bool, string) {

defer response.Body.Close()

if response.StatusCode == 404 {
return false, ""
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}

error = writeToFile(response.Body, hash.Hash)
if error != nil {
fmt.Println(error)
Expand Down Expand Up @@ -1012,7 +1163,7 @@ func extractPwdZip(hash string, password string) ([]*zip.File, error) {

for _, f := range r.File {
if f.IsEncrypted() {
f.SetPassword("infected")
f.SetPassword(password)
}

r, err := f.Open()
Expand Down
46 changes: 25 additions & 21 deletions history.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,30 +44,34 @@ func parseFileForHashEntries(filename string) ([]Hash, error) {
scanner := bufio.NewScanner(file)
for scanner.Scan() { // internally, it advances token based on sperator
text := scanner.Text()
hash := strings.FieldsFunc(text, f)[0]
tags := []string{}
comments := []string{}
if len(strings.FieldsFunc(text, f)) > 1 {
fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))]
tagSection := false
commentSection := false
for _, f := range fields {
if f == "TAGS" {
tagSection = true
commentSection = false
} else if f == "COMMENTS" {
tagSection = false
commentSection = true
} else if f != "TAGS" && f != "COMMENTS" && tagSection {
tags = append(tags, f)
} else if f != "TAGS" && f != "COMMENTS" && commentSection {
comments = append(comments, f)
if len(strings.TrimSpace(text)) > 0 {
hash := strings.FieldsFunc(strings.TrimSpace(text), f)[0]
tags := []string{}
comments := []string{}
if len(strings.FieldsFunc(text, f)) > 1 {
fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))]
tagSection := false
commentSection := false
for _, f := range fields {
if f == "TAGS" {
tagSection = true
commentSection = false
} else if f == "COMMENTS" {
tagSection = false
commentSection = true
} else if f != "TAGS" && f != "COMMENTS" && tagSection {
tags = append(tags, f)
} else if f != "TAGS" && f != "COMMENTS" && commentSection {
comments = append(comments, f)
}
}
}
pHash := Hash{}
pHash, err = parseFileHashEntry(hash, tags, comments)
if err == nil {
hashes = append(hashes, pHash)
}
}
pHash := Hash{}
pHash, err = parseFileHashEntry(hash, tags, comments)
hashes = append(hashes, pHash)
}
return hashes, nil
}
Expand Down
Loading

0 comments on commit ca84d43

Please sign in to comment.