improves handling of links in whitelisted channels

common.go

@@ -13,3 +13,20 @@ func isDiscordEmoji(link string) bool {
 
 	return false
 }
+
+// deduplicateDownloadItems removes duplicates from a slice of *DownloadItem s identified by the Link
+func deduplicateDownloadItems(DownloadItems []*DownloadItem) []*DownloadItem {
+	var result []*DownloadItem
+	seen := map[string]bool{}
+
+	for _, item := range DownloadItems {
+		if seen[item.Link] {
+			continue
+		}
+
+		seen[item.Link] = true
+		result = append(result, item)
+	}
+
+	return result
+}

extract.go

@@ -144,9 +144,9 @@ func getDownloadLinks(inputURL string, channelID string, interactive bool) map[s
 		parsedURL.RawQuery = ""
 		inputURLWithoutQueries := parsedURL.String()
 		if inputURLWithoutQueries != inputURL {
-			return getDownloadLinks(inputURLWithoutQueries, channelID, interactive)
+			return skipDuplicateLinks(getDownloadLinks(inputURLWithoutQueries, channelID, interactive), channelID, interactive)
 		}
 	}
 
-	return map[string]string{inputURL: ""}
+	return skipDuplicateLinks(map[string]string{inputURL: ""}, channelID, interactive)
 }

handler.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"github.com/bwmarrin/discordgo"
+	"mvdan.cc/xurls"
+)
+
+func getRawLinksOfMessage(message *discordgo.Message) []*DownloadItem {
+	var links []*DownloadItem
+
+	if message.Author == nil {
+		message.Author = new(discordgo.User)
+	}
+
+	for _, attachment := range message.Attachments {
+		links = append(links, &DownloadItem{
+			Link:     attachment.URL,
+			Filename: attachment.Filename,
+		})
+	}
+
+	foundLinks := xurls.Strict.FindAllString(message.Content, -1)
+	for _, foundLink := range foundLinks {
+		links = append(links, &DownloadItem{
+			Link: foundLink,
+		})
+	}
+
+	for _, embed := range message.Embeds {
+		if embed.URL != "" {
+			links = append(links, &DownloadItem{
+				Link: embed.URL,
+			})
+		}
+
+		if embed.Description != "" {
+			foundLinks = xurls.Strict.FindAllString(embed.Description, -1)
+			for _, foundLink := range foundLinks {
+				links = append(links, &DownloadItem{
+					Link: foundLink,
+				})
+			}
+		}
+
+		if embed.Image != nil && embed.Image.URL != "" {
+			links = append(links, &DownloadItem{
+				Link: embed.Image.URL,
+			})
+		}
+
+		if embed.Video != nil && embed.Video.URL != "" {
+			links = append(links, &DownloadItem{
+				Link: embed.Video.URL,
+			})
+		}
+	}
+
+	return links
+}

main.go

@@ -265,62 +265,46 @@ func skipDuplicateLinks(linkList map[string]string, channelID string, interactiv
 
 func handleDiscordMessage(m *discordgo.Message) {
 	if folderName, ok := ChannelWhitelist[m.ChannelID]; ok {
-		fileTime := time.Now()
-		var err error
-		if m.Timestamp != "" {
-			fileTime, err = m.Timestamp.Parse()
-			if err != nil {
-				fmt.Println(err)
-			}
-		}
-		if m.Author == nil {
-			m.Author = new(discordgo.User)
-		}
-		for _, iAttachment := range m.Attachments {
-			startDownload(iAttachment.URL, iAttachment.Filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-		}
-		foundUrls := xurls.Strict.FindAllString(m.Content, -1)
-		for _, iFoundUrl := range foundUrls {
-			links := getDownloadLinks(iFoundUrl, m.ChannelID, false)
-			for link, filename := range links {
-				startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-			}
-		}
-		if m.Embeds != nil && len(m.Embeds) > 0 {
-			for _, embed := range m.Embeds {
-				if embed.Provider != nil {
-					// skip preview official embeds to prevent duplictes
-					continue
-				}
-				if embed.URL != "" {
-					links := getDownloadLinks(embed.URL, m.ChannelID, false)
-					for link, filename := range links {
-						startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-					}
-				}
-				if embed.Description != "" {
-					foundUrls := xurls.Strict.FindAllString(embed.Description, -1)
-					for _, iFoundUrl := range foundUrls {
-						links := getDownloadLinks(iFoundUrl, m.ChannelID, false)
-						for link, filename := range links {
-							startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-						}
-					}
+		// download from whitelisted channels
+		var downloadItems []*DownloadItem
+		rawLinks := getRawLinksOfMessage(m)
+		for _, rawLink := range rawLinks {
+			downloadLinks := getDownloadLinks(
+				rawLink.Link,
+				m.ChannelID,
+				false,
+			)
+			for link, filename := range downloadLinks {
+				if rawLink.Filename != "" {
+					filename = rawLink.Filename
 				}
-				if embed.Image != nil && embed.Image.URL != "" {
-					links := getDownloadLinks(embed.Image.URL, m.ChannelID, false)
-					for link, filename := range links {
-						startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-					}
-				}
-				if embed.Video != nil && embed.Video.URL != "" {
-					links := getDownloadLinks(embed.Video.URL, m.ChannelID, false)
-					for link, filename := range links {
-						startDownload(link, filename, folderName, m.ChannelID, m.Author.ID, fileTime)
-					}
+
+				linkTime, err := m.Timestamp.Parse()
+				if err != nil {
+					linkTime = time.Now()
 				}
+
+				downloadItems = append(downloadItems, &DownloadItem{
+					Link:     link,
+					Filename: filename,
+					Time:     linkTime,
+				})
 			}
 		}
+
+		downloadItems = deduplicateDownloadItems(downloadItems)
+
+		for _, downloadItem := range downloadItems {
+			startDownload(
+				downloadItem.Link,
+				downloadItem.Filename,
+				folderName,
+				m.ChannelID,
+				m.Author.ID,
+				downloadItem.Time,
+			)
+		}
+
 	} else if folderName, ok := InteractiveChannelWhitelist[m.ChannelID]; ok {
 		if DiscordUserId != "" && m.Author != nil && m.Author.ID != DiscordUserId {
 			dg.ChannelTyping(m.ChannelID)
@@ -1447,9 +1431,9 @@ func Pagify(text string, delimiter string) []string {
 }
 
 func filepathExtension(filepath string) string {
-	filepath = path.Ext(filepath)
 	if strings.Contains(filepath, "?") {
 		filepath = strings.Split(filepath, "?")[0]
 	}
+	filepath = path.Ext(filepath)
 	return filepath
 }

models.go

@@ -0,0 +1,9 @@
+package main
+
+import "time"
+
+type DownloadItem struct {
+	Link     string
+	Filename string
+	Time     time.Time
+}

vars.go

@@ -1,7 +1,7 @@
 package main
 
 const (
-	VERSION         = "1.33"
+	VERSION         = "1.33.1"
 	DATABASE_DIR    = "database"
 	RELEASE_URL     = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest"
 	RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest"