introduce a way to remain below a threshold for the mail throughput

Author: barais

Dockerfile

@@ -0,0 +1,25 @@
+FROM golang:1.22-alpine AS build
+WORKDIR /app
+
+
+
+COPY go.mod ./
+COPY go.sum ./
+
+RUN apk add git
+
+RUN go mod download
+
+COPY *.go ./
+
+RUN go build -o smtprelay
+
+FROM golang:1.22-alpine
+
+WORKDIR /app
+
+COPY --from=build /app/smtprelay ./
+
+EXPOSE 25
+
+CMD ["./smtprelay", "-config", "smtprelay.ini"]

README.md

@@ -3,7 +3,7 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/decke/smtprelay)](https://goreportcard.com/report/github.com/decke/smtprelay)
 
 Simple Golang based SMTP relay/proxy server that accepts mail via SMTP
-and forwards it directly to another SMTP server.
+and forwards it directly to another SMTP server. Fork to add the ability to cache mail that can not be sent due to rate limit. Mail are sent when the the service will not exceed the rate limit. 
 
 
 ## Why another SMTP server?
@@ -30,3 +30,4 @@ device which produces mail.
 * Forwards all mail to a smarthost (any SMTP server)
 * Small codebase
 * IPv6 support
+* Cache mail to avoid exceeding the rate limit per remote

go.mod

@@ -3,19 +3,23 @@ module github.com/decke/smtprelay
 require (
 	github.com/chrj/smtpd v0.3.1
 	github.com/google/uuid v1.6.0
+	github.com/maypok86/otter v1.2.4
 	github.com/peterbourgon/ff/v3 v3.4.0
+	github.com/sethvargo/go-limiter v1.0.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
-	golang.org/x/crypto v0.28.0
+	golang.org/x/crypto v0.29.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dolthub/maphash v0.1.0 // indirect
+	github.com/gammazero/deque v1.0.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.20
+go 1.22

go.sum

@@ -4,6 +4,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
+github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
+github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
+github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -13,24 +17,40 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/maypok86/otter v1.2.3 h1:jxyPD4ofCwtrQM5is5JNrdAs+6+JQkf/PREZd7JCVgg=
+github.com/maypok86/otter v1.2.3/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4=
+github.com/maypok86/otter v1.2.4-0.20241122154217-c7fa1631301b h1:OcjzyR4TevoH7W/4WIH4ymBR0RCVoRJrvRFU1bW/SmI=
+github.com/maypok86/otter v1.2.4-0.20241122154217-c7fa1631301b/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4=
+github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc=
+github.com/maypok86/otter v1.2.4/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4=
 github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc=
 github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/scalalang2/golang-fifo v1.0.2 h1:sfOJBB86iXuqB5WoLtVI7+wxn8UOEOr9SnJaTakinBA=
+github.com/scalalang2/golang-fifo v1.0.2/go.mod h1:TsyVkLbka5m8tmfqsWBXwJ7Om1jV/uuOuvoPulZbMmA=
+github.com/sethvargo/go-limiter v1.0.0 h1:JqW13eWEMn0VFv86OKn8wiYJY/m250WoXdrjRV0kLe4=
+github.com/sethvargo/go-limiter v1.0.0/go.mod h1:01b6tW25Ap+MeLYBuD4aHunMrJoNO5PVUFdS9rac3II=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
+golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

main.go

@@ -195,7 +195,7 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error {
 		environ = append(environ, fmt.Sprintf("%s=%s", "SMTPRELAY_PEER", peerIP))
 
 		cmd := exec.Cmd{
-			Env: environ,
+			Env:  environ,
 			Path: *command,
 		}
 
@@ -211,7 +211,7 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error {
 
 		cmdLogger.Info("pipe command successful: " + stdout.String())
 	}
-
+	var smtpError *smtpd.Error
 	for _, remote := range envRemotes {
 		logger = logger.WithField("host", remote.Addr)
 		logger.Info("delivering mail from peer using smarthost")
@@ -223,30 +223,28 @@ func mailHandler(peer smtpd.Peer, env smtpd.Envelope) error {
 			env.Data,
 		)
 		if err != nil {
-			var smtpError smtpd.Error
-
 			switch err := err.(type) {
 			case *textproto.Error:
-				smtpError = smtpd.Error{Code: err.Code, Message: err.Msg}
-
+				smtpError = &smtpd.Error{Code: err.Code, Message: err.Msg}
 				logger.WithFields(logrus.Fields{
 					"err_code": err.Code,
 					"err_msg":  err.Msg,
 				}).Error("delivery failed")
 			default:
-				smtpError = smtpd.Error{Code: 554, Message: "Forwarding failed"}
-
+				smtpError = &smtpd.Error{Code: 554, Message: "Forwarding failed"}
 				logger.WithError(err).
 					Error("delivery failed")
 			}
-
-			return smtpError
 		}
-
+	}
+	if smtpError == nil {
 		logger.Debug("delivery successful")
+		return nil
+	} else {
+		logger.Debug("do not direct send")
+		return *smtpError
 	}
 
-	return nil
 }
 
 func generateUUID() string {

ratelimiter.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/maypok86/otter"
+)
+
+var lock = &sync.Mutex{}
+
+type single struct {
+	context context.Context
+	cache   otter.Cache[string, string]
+	r       *Remote
+}
+
+var remoteCache map[*Remote]*single = make(map[*Remote]*single)
+
+func getContext(r *Remote) *single {
+
+	if remoteCache[r] == nil {
+		lock.Lock()
+		defer lock.Unlock()
+		if remoteCache[r] == nil {
+			remoteCache[r] = &single{}
+
+			remoteCache[r].context = context.Background()
+			remoteCache[r].r = r
+
+			cache, err := otter.MustBuilder[string, string](10_000).
+				CollectStats().
+				Cost(func(key string, value string) uint32 {
+					return 1
+				}).DeletionListener(func(key, value string, cause otter.DeletionCause) {
+				log.Infof("Evicted %s %s %v ", key, value, cause)
+
+				parts := strings.Split(value, ";")
+				if len(parts) < 3 {
+					log.Info("Should have had at least three parts")
+				} else {
+					msg, err := os.ReadFile("/tmp/" + key + ".mail")
+					if err != nil {
+						log.Errorf("cannot read file %s", key+".mail")
+
+					} else {
+						from := parts[1]
+						to := parts[2:]
+						SendMail(r, from, to, msg)
+						os.Remove("/tmp/" + key + ".mail")
+					}
+				}
+			}).
+				WithTTL(time.Minute).
+				Build()
+
+			if err != nil {
+				panic(err)
+			}
+			remoteCache[r].cache = cache
+		}
+	}
+	return remoteCache[r]
+}

remotes.go

@@ -4,16 +4,23 @@ import (
 	"fmt"
 	"net/smtp"
 	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/sethvargo/go-limiter"
+	"github.com/sethvargo/go-limiter/memorystore"
 )
 
 type Remote struct {
-	SkipVerify bool
-	Auth       smtp.Auth
-	Scheme     string
-	Hostname   string
-	Port       string
-	Addr       string
-	Sender     string
+	SkipVerify  bool
+	Auth        smtp.Auth
+	Scheme      string
+	Hostname    string
+	Port        string
+	Addr        string
+	Sender      string
+	RateLimiter *limiter.Store
 }
 
 // ParseRemote creates a remote from a given url in the following format:
@@ -79,5 +86,22 @@ func ParseRemote(remoteURL string) (*Remote, error) {
 		r.Sender = u.Path[1:]
 	}
 
+	if hasVal, rate := q.Has("rate"), q.Get("rate"); hasVal && strings.Contains(rate, "/") {
+		i, err := strconv.ParseInt(strings.Split(rate, "/")[0], 10, 32)
+		if err == nil {
+			t, err := time.ParseDuration(strings.Split(rate, "/")[1])
+			log.Infof("Configuring rate limiter %v/%v", i, t)
+			if err == nil {
+				store, err := memorystore.New(&memorystore.Config{
+					Tokens:   uint64(i),
+					Interval: t,
+				})
+				if err == nil {
+					r.RateLimiter = &store
+				}
+			}
+		}
+	}
+
 	return r, nil
 }

send4mail.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+swaks --to [email protected]   --cc [email protected]    --from [email protected]       --header "Subject: Test Email"       --body "This is a test email sent using swaks."       --server 127.0.0.1       --port 1025
+swaks --to [email protected]    --cc [email protected]     --from [email protected]       --header "Subject: Test Email"       --body "This is a test email sent using swaks."       --server 127.0.0.1       --port 1025
+swaks --to [email protected]    --cc [email protected]     --from [email protected]       --header "Subject: Test Email"       --body "This is a test email sent using swaks."       --server 127.0.0.1       --port 1025
+swaks --to [email protected]    --cc [email protected]     --from [email protected]       --header "Subject: Test Email"       --body "This is a test email sent using swaks."       --server 127.0.0.1       --port 1025
+

smtp.go

@@ -26,7 +26,11 @@ import (
 	"net"
 	"net/smtp"
 	"net/textproto"
+	"os"
 	"strings"
+	"time"
+
+	"github.com/chrj/smtpd"
 )
 
 // A Client represents a client connection to an SMTP server.
@@ -320,7 +324,28 @@ var testHookStartTLS func(*tls.Config) // nil, except for tests
 // attachments (see the mime/multipart package), or other mail
 // functionality. Higher-level packages exist outside of the standard
 // library.
+
 func SendMail(r *Remote, from string, to []string, msg []byte) error {
+	if r.RateLimiter != nil {
+		// Do the background in the main
+		tokens, remaining, _, ok, err := (*r.RateLimiter).Take(getContext(r).context, "")
+		log.Infof("Remaining %v tokens of %v", remaining, tokens)
+
+		if err != nil || !ok {
+			//			return smtpd.Error{Code: 452, Message: "Rate limit reached"}
+			theTime := time.Now()
+			filename := theTime.Format("2006-1-2-15-4-5") + ";" + from + ";" + strings.Join(to, ";")
+			filenameb64 := base64.URLEncoding.EncodeToString([]byte(filename))
+			err := os.WriteFile("/tmp/"+filenameb64+".mail", msg, 0644)
+			getContext(r).cache.Set(filenameb64, filename)
+			if err != nil {
+				// handle error
+			}
+			return smtpd.Error{Code: 452, Message: "Rate limit reached"}
+
+		}
+		log.Debugf("Remaining %v tokens of %v", remaining, tokens)
+	}
 	if r.Sender != "" {
 		from = r.Sender
 	}

smtprelay.ini

@@ -8,20 +8,20 @@
 ;logfile = 
 
 ; Log format: default, plain (no timestamp), json
-;log_format = default
+log_format = default
 
 ; Log level: panic, fatal, error, warn, info, debug, trace
-;log_level = info
+log_level = info
 
 ; Hostname for this SMTP server
-;hostname = localhost.localdomain
+hostname = localhost.localdomain
 
 ; Welcome message for clients
 ;welcome_msg = <hostname> ESMTP ready.
 
 ; Listen on the following addresses for incoming
 ; unencrypted connections.
-;listen = 127.0.0.1:25 [::1]:25
+listen = 127.0.0.1:1025
 
 ; STARTTLS and TLS are also supported but need a
 ; SSL certificate and key.
@@ -37,7 +37,6 @@
 ; Only use remotes where FROM EMail address in received
 ; EMail matches remote_sender.
 ;strict_sender = false
-
 ; Socket timeout for read operations
 ; Duration string as sequence of decimal numbers,
 ; each with optional fraction and a unit suffix.
@@ -126,5 +125,10 @@
 ; Multiple remotes, space delimited
 ;remotes = smtp://127.0.0.1:1025 starttls://user:[email protected]:587
 
+; rate limit
+; remotes = smtp://127.0.0.1:2525?rate=99/21m
+remotes = smtp://127.0.0.1:2525?rate=1/1m smtp://127.0.0.1:2527?rate=6/1m
+
+
 ; Pipe messages to external command
 ;command = /usr/local/bin/script