optimize buffer log trace when serve large file

README.md

@@ -4,6 +4,7 @@
 [![Build Status](https://github.com/golangid/candi/workflows/build/badge.svg)](https://github.com/golangid/candi/actions)
 [![Go Report Card](https://goreportcard.com/badge/github.com/golangid/candi)](https://goreportcard.com/report/github.com/golangid/candi)
 [![codecov](https://codecov.io/gh/golangid/candi/branch/master/graph/badge.svg)](https://codecov.io/gh/golangid/candi)
+[![golang](https://img.shields.io/badge/golang%20%3E=-v1.18-green.svg?logo=go)](https://golang.org/doc/devel/release.html#go1.18)
 
 ## Build with :heart: and
 <p align="center">

candihelper/const.go

@@ -26,6 +26,8 @@ const (
 	MByte = KByte * 1024
 	// GByte ...
 	GByte = MByte * 1024
+	// TByte ...
+	TByte = GByte * 1024
 
 	// WORKDIR const for workdir environment
 	WORKDIR = "WORKDIR"

candihelper/file_loader.go

@@ -9,9 +9,7 @@ import (
 
 // LoadAllFile from path
 func LoadAllFile(path, formatFile string) []byte {
-
 	var buff bytes.Buffer
-
 	filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			panic(err)

candihelper/helper.go

@@ -662,3 +662,26 @@ func ParseTimeToString(date time.Time, format string) (res string) {
 	}
 	return res
 }
+
+// TransformSizeToByte helper
+func TransformSizeToByte(size uint64) string {
+	var unit string
+	if size >= TByte {
+		unit = "TB"
+		size /= TByte
+	} else if size >= GByte {
+		unit = "GB"
+		size /= GByte
+	} else if size >= MByte {
+		unit = "MB"
+		size /= MByte
+	} else if size >= KByte {
+		unit = "KB"
+		size /= KByte
+	} else {
+		unit = "B"
+		size /= Byte
+	}
+
+	return fmt.Sprintf("%d %s", size, unit)
+}

candihelper/interfaces.go

@@ -15,3 +15,10 @@ type MultiError interface {
 	Merge(MultiError) MultiError
 	Error() string
 }
+
+// FilterStreamer abstract interface
+type FilterStreamer interface {
+	GetPage() int
+	IncrPage()
+	GetLimit() int
+}

candihelper/streamer.go

@@ -0,0 +1,25 @@
+package candihelper
+
+import (
+	"context"
+	"math"
+)
+
+// StreamAllBatch helper func for stream data
+func StreamAllBatch[T any, F FilterStreamer](ctx context.Context, totalData int, filter F, fetchAllFunc func(context.Context, F) ([]T, error), handleFunc func(idx int, data *T) error) error {
+	totalPages := int(math.Ceil(float64(totalData) / float64(filter.GetLimit())))
+	for filter.GetPage() <= totalPages {
+		list, err := fetchAllFunc(ctx, filter)
+		if err != nil {
+			return err
+		}
+		for i, data := range list {
+			offset := (filter.GetPage() - 1) * filter.GetLimit()
+			if err := handleFunc(offset+i, &data); err != nil {
+				return err
+			}
+		}
+		filter.IncrPage()
+	}
+	return nil
+}

cmd/candi/template_usecase.go

@@ -496,6 +496,7 @@ func TestNew{{upper (camel .ModuleName)}}Usecase(t *testing.T) {
 	mockDeps := &mockdeps.Dependency{}
 	mockDeps.On("GetRedisPool").Return(mockRedisPool)
 	mockDeps.On("GetBroker", mock.Anything).Return(mockBroker)
+	mockDeps.On("GetLocker").Return(&mockinterfaces.Locker{})
 
 	uc, setFunc := New{{upper (camel .ModuleName)}}Usecase(mockDeps)
 	setFunc(nil)

init.go

@@ -2,5 +2,5 @@ package candi
 
 const (
 	// Version of this library
-	Version = "v1.14.10"
+	Version = "v1.14.11"
 )

middleware/cache.go

@@ -20,15 +20,12 @@ const (
 
 // HTTPCache middleware for cache
 func (m *Middleware) HTTPCache(next http.Handler) http.Handler {
-
 	type cacheData struct {
-		Body       []byte      `json:"body,omitempty"`
-		StatusCode int         `json:"statusCode,omitempty"`
-		Header     http.Header `json:"header,omitempty"`
+		Body   []byte      `json:"body,omitempty"`
+		Header http.Header `json:"header,omitempty"`
 	}
 
 	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
-
 		if m.cache == nil {
 			next.ServeHTTP(res, req)
 			return
@@ -67,8 +64,8 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler {
 		defer trace.Finish()
 
 		cacheKey := req.Method + ":" + req.URL.String()
+		trace.SetTag("key", cacheKey)
 		if cacheVal, err := m.cache.Get(ctx, cacheKey); err == nil {
-
 			if ttl, err := m.cache.GetTTL(ctx, cacheKey); err == nil {
 				res.Header().Add(HeaderExpires, time.Now().In(time.UTC).Add(ttl).Format(time.RFC1123))
 			}
@@ -84,7 +81,6 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler {
 				res.Header().Set(k, data.Header.Get(k))
 			}
 			res.Write(data.Body)
-			res.WriteHeader(data.StatusCode)
 			return
 		}
 
@@ -96,9 +92,8 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler {
 		if respWriter.StatusCode() < http.StatusBadRequest {
 			m.cache.Set(ctx, cacheKey, candihelper.ToBytes(
 				cacheData{
-					Body:       resBody.Bytes(),
-					StatusCode: respWriter.StatusCode(),
-					Header:     res.Header(),
+					Body:   resBody.Bytes(),
+					Header: res.Header(),
 				},
 			), maxAge)
 		}

wrapper/http_handler.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/golangid/candi/candihelper"
@@ -26,9 +27,18 @@ type HTTPMiddlewareTracerConfig struct {
 
 // HTTPMiddlewareTracer middleware wrapper for tracer
 func HTTPMiddlewareTracer(cfg HTTPMiddlewareTracerConfig) func(http.Handler) http.Handler {
+	bPool := &sync.Pool{
+		New: func() interface{} {
+			return bytes.NewBuffer(make([]byte, 256))
+		},
+	}
+	releasePool := func(buff *bytes.Buffer) {
+		buff.Reset()
+		bPool.Put(buff)
+	}
+
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-
 			if _, isExcludePath := cfg.ExcludePath[req.URL.Path]; isExcludePath {
 				next.ServeHTTP(rw, req)
 				return
@@ -54,32 +64,39 @@ func HTTPMiddlewareTracer(cfg HTTPMiddlewareTracerConfig) func(http.Handler) htt
 			}()
 
 			httpDump, _ := httputil.DumpRequest(req, false)
+			trace.Log("http.request", httpDump)
 			trace.SetTag("http.url_path", req.URL.Path)
 			trace.SetTag("http.method", req.Method)
-			trace.Log("http.request", httpDump)
 
-			body, _ := io.ReadAll(req.Body)
-			if len(body) < cfg.MaxLogSize {
-				trace.Log("request.body", body)
-			} else {
-				trace.Log("request.body.size", len(body))
+			if contentLength, err := strconv.Atoi(req.Header.Get("Content-Length")); err == nil {
+				if contentLength < cfg.MaxLogSize {
+					reqBody := bPool.Get().(*bytes.Buffer)
+					reqBody.Reset()
+					reqBody.ReadFrom(req.Body)
+					trace.Log("request.body", reqBody.String())
+					req.Body = io.NopCloser(bytes.NewReader(reqBody.Bytes())) // reuse body
+					releasePool(reqBody)
+				} else {
+					trace.Log("request.body.size", candihelper.TransformSizeToByte(uint64(contentLength)))
+				}
 			}
-			req.Body = io.NopCloser(bytes.NewBuffer(body)) // reuse body
 
-			resBody := &bytes.Buffer{}
+			resBody := bPool.Get().(*bytes.Buffer)
+			resBody.Reset()
+			defer releasePool(resBody)
 			respWriter := NewWrapHTTPResponseWriter(resBody, rw)
-
+			respWriter.SetMaxWriteSize(cfg.MaxLogSize)
 			next.ServeHTTP(respWriter, req.WithContext(ctx))
 
 			trace.SetTag("http.status_code", respWriter.statusCode)
 			if respWriter.statusCode >= http.StatusBadRequest {
 				trace.SetError(fmt.Errorf("resp.code:%d", respWriter.statusCode))
 			}
-
-			if resBody.Len() < cfg.MaxLogSize {
+			trace.Log("response.header", respWriter.Header())
+			if respWriter.contentLength < cfg.MaxLogSize {
 				trace.Log("response.body", resBody.String())
 			} else {
-				trace.Log("response.body.size", resBody.Len())
+				trace.Log("response.body.size", candihelper.TransformSizeToByte(uint64(respWriter.contentLength)))
 			}
 		})
 	}
@@ -91,7 +108,6 @@ func HTTPMiddlewareCORS(
 	exposeHeaders []string,
 	allowCredential bool,
 ) func(http.Handler) http.Handler {
-
 	if len(allowOrigins) == 0 {
 		allowOrigins = []string{"*"}
 	}
@@ -101,9 +117,7 @@ func HTTPMiddlewareCORS(
 	exposeHeader := strings.Join(exposeHeaders, ",")
 
 	return func(next http.Handler) http.Handler {
-
 		return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
-
 			origin := req.Header.Get("Origin")
 			allowOrigin := ""
 

wrapper/http_response_writer.go

@@ -1,21 +1,41 @@
 package wrapper
 
 import (
-	"io"
+	"bytes"
 	"net/http"
 )
 
 // WrapHTTPResponseWriter wrapper
 type WrapHTTPResponseWriter struct {
-	statusCode int
-	writer     io.Writer
+	statusCode     int
+	buff           *bytes.Buffer
+	maxWriteSize   int
+	limitWriteSize bool
+	contentLength  int
 	http.ResponseWriter
 }
 
 // NewWrapHTTPResponseWriter init new wrapper for http response writter
-func NewWrapHTTPResponseWriter(w io.Writer, httpResponseWriter http.ResponseWriter) *WrapHTTPResponseWriter {
-	// Default the status code to 200
-	return &WrapHTTPResponseWriter{statusCode: http.StatusOK, writer: io.MultiWriter(w, httpResponseWriter), ResponseWriter: httpResponseWriter}
+func NewWrapHTTPResponseWriter(responseBuff *bytes.Buffer, httpResponseWriter http.ResponseWriter) *WrapHTTPResponseWriter {
+	return &WrapHTTPResponseWriter{
+		statusCode: http.StatusOK, buff: responseBuff, ResponseWriter: httpResponseWriter,
+	}
+}
+
+// SetMaxWriteSize set max write size to buffer
+func (w *WrapHTTPResponseWriter) SetMaxWriteSize(max int) {
+	w.maxWriteSize = max
+	w.limitWriteSize = true
+}
+
+// GetContentLength get response content length
+func (w *WrapHTTPResponseWriter) GetContentLength() int {
+	return w.contentLength
+}
+
+// GetContent get response content
+func (w *WrapHTTPResponseWriter) GetContent() []byte {
+	return w.buff.Bytes()
 }
 
 // StatusCode give a way to get the Code
@@ -30,7 +50,12 @@ func (w *WrapHTTPResponseWriter) Header() http.Header {
 
 func (w *WrapHTTPResponseWriter) Write(data []byte) (int, error) {
 	// Store response body to writer
-	return w.writer.Write(data)
+	n, err := w.ResponseWriter.Write(data)
+	w.contentLength += n
+	if !w.limitWriteSize || w.contentLength < w.maxWriteSize {
+		w.buff.Write(data)
+	}
+	return n, err
 }
 
 // WriteHeader method