From 47315c4c7c257138435110e91f43167d7dce0eaa Mon Sep 17 00:00:00 2001 From: Sebastian Winkler Date: Wed, 5 Dec 2018 20:02:24 +0100 Subject: [PATCH] improves handling of links in whitelisted channels --- common.go | 17 +++++++++++ extract.go | 4 +-- handler.go | 59 +++++++++++++++++++++++++++++++++++ main.go | 90 ++++++++++++++++++++++-------------------------------- models.go | 9 ++++++ vars.go | 2 +- 6 files changed, 125 insertions(+), 56 deletions(-) create mode 100644 handler.go create mode 100644 models.go diff --git a/common.go b/common.go index 4eb59cf..54587ad 100644 --- a/common.go +++ b/common.go @@ -13,3 +13,20 @@ func isDiscordEmoji(link string) bool { return false } + +// deduplicateDownloadItems removes duplicates from a slice of *DownloadItem s identified by the Link +func deduplicateDownloadItems(DownloadItems []*DownloadItem) []*DownloadItem { + var result []*DownloadItem + seen := map[string]bool{} + + for _, item := range DownloadItems { + if seen[item.Link] { + continue + } + + seen[item.Link] = true + result = append(result, item) + } + + return result +} diff --git a/extract.go b/extract.go index 4150c89..6c23f4e 100644 --- a/extract.go +++ b/extract.go @@ -144,9 +144,9 @@ func getDownloadLinks(inputURL string, channelID string, interactive bool) map[s parsedURL.RawQuery = "" inputURLWithoutQueries := parsedURL.String() if inputURLWithoutQueries != inputURL { - return getDownloadLinks(inputURLWithoutQueries, channelID, interactive) + return skipDuplicateLinks(getDownloadLinks(inputURLWithoutQueries, channelID, interactive), channelID, interactive) } } - return map[string]string{inputURL: ""} + return skipDuplicateLinks(map[string]string{inputURL: ""}, channelID, interactive) } diff --git a/handler.go b/handler.go new file mode 100644 index 0000000..ca7d937 --- /dev/null +++ b/handler.go @@ -0,0 +1,59 @@ +package main + +import ( + "github.com/bwmarrin/discordgo" + "mvdan.cc/xurls" +) + +func getRawLinksOfMessage(message *discordgo.Message) []*DownloadItem { + var links []*DownloadItem + + if message.Author == nil { + message.Author = new(discordgo.User) + } + + for _, attachment := range message.Attachments { + links = append(links, &DownloadItem{ + Link: attachment.URL, + Filename: attachment.Filename, + }) + } + + foundLinks := xurls.Strict.FindAllString(message.Content, -1) + for _, foundLink := range foundLinks { + links = append(links, &DownloadItem{ + Link: foundLink, + }) + } + + for _, embed := range message.Embeds { + if embed.URL != "" { + links = append(links, &DownloadItem{ + Link: embed.URL, + }) + } + + if embed.Description != "" { + foundLinks = xurls.Strict.FindAllString(embed.Description, -1) + for _, foundLink := range foundLinks { + links = append(links, &DownloadItem{ + Link: foundLink, + }) + } + } + + if embed.Image != nil && embed.Image.URL != "" { + links = append(links, &DownloadItem{ + Link: embed.Image.URL, + }) + } + + if embed.Video != nil && embed.Video.URL != "" { + links = append(links, &DownloadItem{ + Link: embed.Video.URL, + }) + } + } + + return links +} diff --git a/main.go b/main.go index 39eda6b..d481c04 100644 --- a/main.go +++ b/main.go @@ -265,62 +265,46 @@ func skipDuplicateLinks(linkList map[string]string, channelID string, interactiv func handleDiscordMessage(m *discordgo.Message) { if folderName, ok := ChannelWhitelist[m.ChannelID]; ok { - fileTime := time.Now() - var err error - if m.Timestamp != "" { - fileTime, err = m.Timestamp.Parse() - if err != nil { - fmt.Println(err) - } - } - if m.Author == nil { - m.Author = new(discordgo.User) - } - for _, iAttachment := range m.Attachments { - startDownload(iAttachment.URL, iAttachment.Filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } - foundUrls := xurls.Strict.FindAllString(m.Content, -1) - for _, iFoundUrl := range foundUrls { - links := getDownloadLinks(iFoundUrl, m.ChannelID, false) - for link, filename := range links { - startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } - } - if m.Embeds != nil && len(m.Embeds) > 0 { - for _, embed := range m.Embeds { - if embed.Provider != nil { - // skip preview official embeds to prevent duplictes - continue - } - if embed.URL != "" { - links := getDownloadLinks(embed.URL, m.ChannelID, false) - for link, filename := range links { - startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } - } - if embed.Description != "" { - foundUrls := xurls.Strict.FindAllString(embed.Description, -1) - for _, iFoundUrl := range foundUrls { - links := getDownloadLinks(iFoundUrl, m.ChannelID, false) - for link, filename := range links { - startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } - } + // download from whitelisted channels + var downloadItems []*DownloadItem + rawLinks := getRawLinksOfMessage(m) + for _, rawLink := range rawLinks { + downloadLinks := getDownloadLinks( + rawLink.Link, + m.ChannelID, + false, + ) + for link, filename := range downloadLinks { + if rawLink.Filename != "" { + filename = rawLink.Filename } - if embed.Image != nil && embed.Image.URL != "" { - links := getDownloadLinks(embed.Image.URL, m.ChannelID, false) - for link, filename := range links { - startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } - } - if embed.Video != nil && embed.Video.URL != "" { - links := getDownloadLinks(embed.Video.URL, m.ChannelID, false) - for link, filename := range links { - startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime) - } + + linkTime, err := m.Timestamp.Parse() + if err != nil { + linkTime = time.Now() } + + downloadItems = append(downloadItems, &DownloadItem{ + Link: link, + Filename: filename, + Time: linkTime, + }) } } + + downloadItems = deduplicateDownloadItems(downloadItems) + + for _, downloadItem := range downloadItems { + startDownload( + downloadItem.Link, + downloadItem.Filename, + folderName, + m.ChannelID, + m.Author.ID, + downloadItem.Time, + ) + } + } else if folderName, ok := InteractiveChannelWhitelist[m.ChannelID]; ok { if DiscordUserId != "" && m.Author != nil && m.Author.ID != DiscordUserId { dg.ChannelTyping(m.ChannelID) @@ -1447,9 +1431,9 @@ func Pagify(text string, delimiter string) []string { } func filepathExtension(filepath string) string { - filepath = path.Ext(filepath) if strings.Contains(filepath, "?") { filepath = strings.Split(filepath, "?")[0] } + filepath = path.Ext(filepath) return filepath } diff --git a/models.go b/models.go new file mode 100644 index 0000000..49db54e --- /dev/null +++ b/models.go @@ -0,0 +1,9 @@ +package main + +import "time" + +type DownloadItem struct { + Link string + Filename string + Time time.Time +} diff --git a/vars.go b/vars.go index 0066f79..52f0b9c 100644 --- a/vars.go +++ b/vars.go @@ -1,7 +1,7 @@ package main const ( - VERSION = "1.33" + VERSION = "1.33.1" DATABASE_DIR = "database" RELEASE_URL = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest" RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest"