feat: Add blobstore package

This commit introduces a `blobstore` package and refactors the existing
upload mechanism. Upload is now handled by `providers` and the two
bundled providers are `S3` and `Filesystem`. `app.Blobstore` initialises
the correct provider based on the configuration and handles `Put`,
`Delete` and `Get` operations.

Author: mr-karan

.dockerignore

@@ -0,0 +1,25 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.gitignore

@@ -6,3 +6,4 @@ frontend/yarn.lock
 config.toml
 node_modules
 listmonk
+dist/*

admin.go

@@ -28,7 +28,6 @@ func handleGetConfigScript(c echo.Context) error {
 		app = c.Get("app").(*App)
 		out = configScript{
 			RootURL:    app.Constants.RootURL,
-			UploadURI:  app.Constants.UploadURI,
 			FromEmail:  app.Constants.FromEmail,
 			Messengers: app.Manager.GetMessengerNames(),
 		}

config.toml.sample

@@ -24,14 +24,6 @@ from_email = "listmonk <[email protected]>"
 # To disable notifications, set an empty list, eg: notify_emails = []
 notify_emails = ["[email protected]", "[email protected]"]
 
-# Path to the uploads directory where media will be uploaded.
-upload_path = "uploads"
-
-# Upload URI that's visible to the outside world. The media
-# uploaded to upload_path will be made available publicly
-# under this URI, for instance, list.yoursite.com/uploads.
-upload_uri = "/uploads"
-
 # Maximum concurrent workers that will attempt to send messages
 # simultaneously. This should depend on the number of CPUs the
 # machine has and also the number of simultaenous e-mails the
@@ -110,3 +102,33 @@ ssl_mode = "disable"
 
         # Maximum concurrent connections to the SMTP server.
         max_conns = 10
+
+# Upload settings
+[upload]
+# Provider which will be used to host uploaded media. Bundled providers are "filesystem" and "s3".
+provider = "filesystem"
+
+# S3 Provider settings
+[upload.s3]
+# (Optional). AWS Access Key and Secret Key for the user to access the bucket. Leaving it empty would default to use
+# instance IAM role.
+aws_access_key_id = ""
+aws_secret_access_key = ""
+# AWS Region where S3 bucket is hosted.
+aws_default_region="ap-south-1"
+# Specify bucket name.
+bucket=""
+# Path where the files will be stored inside bucket. Empty value ("") means the root of bucket.
+bucket_path=""
+# Bucket type can be "private" or "public".
+bucket_type="public"
+# (Optional) Specify TTL (in seconds) for the generated presigned URL. Expiry value is used only if the bucket is private.
+expiry="86400"
+
+# Filesystem provider settings
+[upload.filesystem]
+# Path to the uploads directory where media will be uploaded. Leaving it empty ("") means current working directory.
+upload_path=""
+# Upload URI that's visible to the outside world. The media uploaded to upload_path will be made available publicly
+# under this URI, for instance, list.yoursite.com/uploads.
+upload_uri = "/uploads"

go.mod

@@ -2,6 +2,7 @@ module github.com/knadh/listmonk
 
 require (
 	github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
+	github.com/aws/aws-sdk-go v1.25.12
 	github.com/disintegration/imaging v1.5.0
 	github.com/jinzhu/gorm v1.9.1
 	github.com/jmoiron/sqlx v1.2.0
@@ -15,6 +16,7 @@ require (
 	github.com/lib/pq v1.0.0
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.4 // indirect
+	github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/pflag v1.0.3
 	github.com/stretchr/objx v0.2.0 // indirect
@@ -27,3 +29,5 @@ require (
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/volatiletech/null.v6 v6.0.0-20170828023728-0bef4e07ae1b
 )
+
+go 1.13

go.sum

@@ -2,6 +2,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.25.12 h1:a4h2FxoUJq9h+hajSE/dsRiqoOniIh6BkzhxMjkepzY=
+github.com/aws/aws-sdk-go v1.25.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -16,6 +18,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jinzhu/gorm v1.9.1 h1:lDSDtsCt5AGGSKTs8AHlSDbbgif4G4+CKJ8ETBDVHTA=
 github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jordan-wright/email v0.0.0-20181027021455-480bedc4908b h1:veTPVnbkOijplSJVywDYKDRPoZEN39kfuMDzzRKP0FA=
@@ -57,6 +61,10 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rhnvrm/simples3 v0.2.3 h1:qNXPynabu8M3F4+69fspA5aWZR8jqVV1RQtv2xc1OVk=
+github.com/rhnvrm/simples3 v0.2.3/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
+github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727 h1:2josYcx2gm3CT0WMqi0jBagvg50V3UMWlYN/CnBEbSI=
+github.com/rhnvrm/simples3 v0.2.4-0.20191018074503-3d5b071ef727/go.mod h1:iphavgjkW1uvoIiqLUX6D42XuuI9Cr+B/63xw3gb9qA=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
@@ -73,6 +81,7 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM
 golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b h1:VHyIDlv3XkfCa5/a81uzaoDkHH4rr81Z62g+xlnO8uM=
 golang.org/x/image v0.0.0-20181116024801-cd38e8056d9b/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg=
 golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

main.go

@@ -19,6 +19,9 @@ import (
 	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/providers/posflag"
 	"github.com/knadh/listmonk/manager"
+	"github.com/knadh/listmonk/media"
+	"github.com/knadh/listmonk/media/providers/filesystem"
+	"github.com/knadh/listmonk/media/providers/s3"
 	"github.com/knadh/listmonk/messenger"
 	"github.com/knadh/listmonk/subimporter"
 	"github.com/knadh/stuffbin"
@@ -30,8 +33,6 @@ type constants struct {
 	RootURL      string         `koanf:"root"`
 	LogoURL      string         `koanf:"logo_url"`
 	FaviconURL   string         `koanf:"favicon_url"`
-	UploadPath   string         `koanf:"upload_path"`
-	UploadURI    string         `koanf:"upload_uri"`
 	FromEmail    string         `koanf:"from_email"`
 	NotifyEmails []string       `koanf:"notify_emails"`
 	Privacy      privacyOptions `koanf:"privacy"`
@@ -56,6 +57,7 @@ type App struct {
 	Logger    *log.Logger
 	NotifTpls *template.Template
 	Messenger messenger.Messenger
+	Media     media.Store
 }
 
 var (
@@ -195,6 +197,33 @@ func initMessengers(r *manager.Manager) messenger.Messenger {
 	return msgr
 }
 
+// initMediaStore initializes Upload manager with a custom backend.
+func initMediaStore() media.Store {
+	switch provider := ko.String("upload.provider"); provider {
+	case "s3":
+		var opts s3.Opts
+		ko.Unmarshal("upload.s3", &opts)
+		uplder, err := s3.NewS3Store(opts)
+		if err != nil {
+			logger.Fatalf("error initializing s3 upload provider %s", err)
+		}
+		return uplder
+	case "filesystem":
+		var opts filesystem.Opts
+		ko.Unmarshal("upload.filesystem", &opts)
+		opts.UploadPath = filepath.Clean(opts.UploadPath)
+		opts.UploadURI = filepath.Clean(opts.UploadURI)
+		uplder, err := filesystem.NewDiskStore(opts)
+		if err != nil {
+			logger.Fatalf("error initializing filesystem upload provider %s", err)
+		}
+		return uplder
+	default:
+		logger.Fatalf("unknown provider. please select one of either filesystem or s3")
+	}
+	return nil
+}
+
 func main() {
 	// Connect to the DB.
 	db, err := connectDB(ko.String("db.host"),
@@ -216,8 +245,6 @@ func main() {
 		log.Fatalf("error loading app config: %v", err)
 	}
 	c.RootURL = strings.TrimRight(c.RootURL, "/")
-	c.UploadURI = filepath.Clean(c.UploadURI)
-	c.UploadPath = filepath.Clean(c.UploadPath)
 	c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
 
 	// Initialize the static file system into which all
@@ -299,6 +326,9 @@ func main() {
 	// Add messengers.
 	app.Messenger = initMessengers(app.Manager)
 
+	// Add uploader
+	app.Media = initMediaStore()
+
 	// Initialize the workers that push out messages.
 	go m.Run(time.Second * 5)
 	m.SpawnWorkers()
@@ -330,7 +360,9 @@ func main() {
 	fSrv := app.FS.FileServer()
 	srv.GET("/public/*", echo.WrapHandler(fSrv))
 	srv.GET("/frontend/*", echo.WrapHandler(fSrv))
-	srv.Static(c.UploadURI, c.UploadURI)
+	if ko.String("upload.provider") == "filesystem" {
+		srv.Static(ko.String("upload.filesystem.upload_uri"), ko.String("upload.filesystem.upload_path"))
+	}
 	registerHandlers(srv)
 	srv.Logger.Fatal(srv.Start(ko.String("app.address")))
 }

media.go

@@ -3,12 +3,9 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"os"
-	"path/filepath"
 	"strconv"
 
-	"github.com/disintegration/imaging"
-	"github.com/knadh/listmonk/models"
+	"github.com/knadh/listmonk/media"
 	"github.com/labstack/echo"
 	uuid "github.com/satori/go.uuid"
 )
@@ -26,53 +23,72 @@ func handleUploadMedia(c echo.Context) error {
 		app     = c.Get("app").(*App)
 		cleanUp = false
 	)
-
+	file, err := c.FormFile("file")
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Invalid file uploaded: %v", err))
+	}
+	// Validate MIME type with the list of allowed types.
+	var typ = file.Header.Get("Content-type")
+	ok := validateMIME(typ, imageMimes)
+	if !ok {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Unsupported file type (%s) uploaded.", typ))
+	}
+	// Generate filename
+	fName := generateFileName(file.Filename)
+	// Read file contents in memory
+	src, err := file.Open()
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest,
+			fmt.Sprintf("Error reading file: %s", err))
+	}
+	defer src.Close()
 	// Upload the file.
-	fName, err := uploadFile("file", app.Constants.UploadPath, "", imageMimes, c)
+	fName, err = app.Media.Put(fName, typ, src)
 	if err != nil {
+		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error uploading file: %s", err))
 	}
-	path := filepath.Join(app.Constants.UploadPath, fName)
 
 	defer func() {
 		// If any of the subroutines in this function fail,
 		// the uploaded image should be removed.
 		if cleanUp {
-			os.Remove(path)
+			app.Media.Delete(fName)
+			app.Media.Delete(thumbPrefix + fName)
 		}
 	}()
 
-	// Create a thumbnail.
-	src, err := imaging.Open(path)
+	// Create thumbnail from file.
+	thumbFile, err := createThumbnail(file)
 	if err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error opening image for resizing: %s", err))
 	}
-
-	t := imaging.Resize(src, thumbnailSize, 0, imaging.Lanczos)
-	if err := imaging.Save(t, fmt.Sprintf("%s/%s%s", app.Constants.UploadPath, thumbPrefix, fName)); err != nil {
+	// Upload thumbnail.
+	thumbfName, err := app.Media.Put(thumbPrefix+fName, typ, thumbFile)
+	if err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error saving thumbnail: %s", err))
 	}
-
 	// Write to the DB.
-	if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, fmt.Sprintf("%s%s", thumbPrefix, fName), 0, 0); err != nil {
+	if _, err := app.Queries.InsertMedia.Exec(uuid.NewV4(), fName, thumbfName, 0, 0); err != nil {
 		cleanUp = true
 		return echo.NewHTTPError(http.StatusInternalServerError,
-			fmt.Sprintf("Error saving uploaded file: %s", pqErrMsg(err)))
+			fmt.Sprintf("Error saving uploaded file to db: %s", pqErrMsg(err)))
 	}
-
 	return c.JSON(http.StatusOK, okResp{true})
 }
 
 // handleGetMedia handles retrieval of uploaded media.
 func handleGetMedia(c echo.Context) error {
 	var (
 		app = c.Get("app").(*App)
-		out []models.Media
+		out []media.Media
 	)
 
 	if err := app.Queries.GetMedia.Select(&out); err != nil {
@@ -81,8 +97,8 @@ func handleGetMedia(c echo.Context) error {
 	}
 
 	for i := 0; i < len(out); i++ {
-		out[i].URI = fmt.Sprintf("%s/%s", app.Constants.UploadURI, out[i].Filename)
-		out[i].ThumbURI = fmt.Sprintf("%s/%s%s", app.Constants.UploadURI, thumbPrefix, out[i].Filename)
+		out[i].URI = app.Media.Get(out[i].Filename)
+		out[i].ThumbURI = app.Media.Get(thumbPrefix + out[i].Filename)
 	}
 
 	return c.JSON(http.StatusOK, okResp{out})
@@ -99,13 +115,14 @@ func handleDeleteMedia(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID.")
 	}
 
-	var m models.Media
+	var m media.Media
 	if err := app.Queries.DeleteMedia.Get(&m, id); err != nil {
 		return echo.NewHTTPError(http.StatusInternalServerError,
 			fmt.Sprintf("Error deleting media: %s", pqErrMsg(err)))
 	}
-	os.Remove(filepath.Join(app.Constants.UploadPath, m.Filename))
-	os.Remove(filepath.Join(app.Constants.UploadPath, fmt.Sprintf("%s%s", thumbPrefix, m.Filename)))
+
+	app.Media.Delete(m.Filename)
+	app.Media.Delete(thumbPrefix + m.Filename)
 
 	return c.JSON(http.StatusOK, okResp{true})
 }

media/media.go

@@ -0,0 +1,26 @@
+package media
+
+import (
+	"io"
+
+	"gopkg.in/volatiletech/null.v6"
+)
+
+// Media represents an uploaded object.
+type Media struct {
+	ID        int       `db:"id" json:"id"`
+	UUID      string    `db:"uuid" json:"uuid"`
+	Filename  string    `db:"filename" json:"filename"`
+	Width     int       `db:"width" json:"width"`
+	Height    int       `db:"height" json:"height"`
+	CreatedAt null.Time `db:"created_at" json:"created_at"`
+	ThumbURI  string    `json:"thumb_uri"`
+	URI       string    `json:"uri"`
+}
+
+// Store represents set of methods to perform upload/delete operations.
+type Store interface {
+	Put(string, string, io.ReadSeeker) (string, error)
+	Delete(string) error
+	Get(string) string
+}