From b1775f94cf3e11092c94bb74b494a14ec55aabc2 Mon Sep 17 00:00:00 2001 From: agungdwiprasetyo Date: Fri, 2 Jun 2023 15:05:07 +0700 Subject: [PATCH] optimize buffer log trace when serve large file --- README.md | 1 + candihelper/const.go | 2 + candihelper/file_loader.go | 2 - candihelper/helper.go | 23 ++++++++++++ candihelper/interfaces.go | 7 ++++ candihelper/streamer.go | 25 +++++++++++++ cmd/candi/template_usecase.go | 1 + init.go | 2 +- middleware/cache.go | 15 +++----- mocks/candihelper/FilterStreamer.go | 58 +++++++++++++++++++++++++++++ wrapper/http_handler.go | 46 +++++++++++++++-------- wrapper/http_response_writer.go | 39 +++++++++++++++---- 12 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 candihelper/streamer.go create mode 100644 mocks/candihelper/FilterStreamer.go diff --git a/README.md b/README.md index 3e08d404..ea3058e0 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Build Status](https://github.com/golangid/candi/workflows/build/badge.svg)](https://github.com/golangid/candi/actions) [![Go Report Card](https://goreportcard.com/badge/github.com/golangid/candi)](https://goreportcard.com/report/github.com/golangid/candi) [![codecov](https://codecov.io/gh/golangid/candi/branch/master/graph/badge.svg)](https://codecov.io/gh/golangid/candi) +[![golang](https://img.shields.io/badge/golang%20%3E=-v1.18-green.svg?logo=go)](https://golang.org/doc/devel/release.html#go1.18) ## Build with :heart: and

diff --git a/candihelper/const.go b/candihelper/const.go index be1b757c..5504ee9c 100644 --- a/candihelper/const.go +++ b/candihelper/const.go @@ -26,6 +26,8 @@ const ( MByte = KByte * 1024 // GByte ... GByte = MByte * 1024 + // TByte ... + TByte = GByte * 1024 // WORKDIR const for workdir environment WORKDIR = "WORKDIR" diff --git a/candihelper/file_loader.go b/candihelper/file_loader.go index 6203cae6..69585809 100644 --- a/candihelper/file_loader.go +++ b/candihelper/file_loader.go @@ -9,9 +9,7 @@ import ( // LoadAllFile from path func LoadAllFile(path, formatFile string) []byte { - var buff bytes.Buffer - filepath.Walk(path, func(p string, info os.FileInfo, err error) error { if err != nil { panic(err) diff --git a/candihelper/helper.go b/candihelper/helper.go index 705ea4d2..822d3ae9 100644 --- a/candihelper/helper.go +++ b/candihelper/helper.go @@ -662,3 +662,26 @@ func ParseTimeToString(date time.Time, format string) (res string) { } return res } + +// TransformSizeToByte helper +func TransformSizeToByte(size uint64) string { + var unit string + if size >= TByte { + unit = "TB" + size /= TByte + } else if size >= GByte { + unit = "GB" + size /= GByte + } else if size >= MByte { + unit = "MB" + size /= MByte + } else if size >= KByte { + unit = "KB" + size /= KByte + } else { + unit = "B" + size /= Byte + } + + return fmt.Sprintf("%d %s", size, unit) +} diff --git a/candihelper/interfaces.go b/candihelper/interfaces.go index a59a13e2..5cc20788 100644 --- a/candihelper/interfaces.go +++ b/candihelper/interfaces.go @@ -15,3 +15,10 @@ type MultiError interface { Merge(MultiError) MultiError Error() string } + +// FilterStreamer abstract interface +type FilterStreamer interface { + GetPage() int + IncrPage() + GetLimit() int +} diff --git a/candihelper/streamer.go b/candihelper/streamer.go new file mode 100644 index 00000000..1dd1e78a --- /dev/null +++ b/candihelper/streamer.go @@ -0,0 +1,25 @@ +package candihelper + +import ( + "context" + "math" +) + +// StreamAllBatch helper func for stream data +func StreamAllBatch[T any, F FilterStreamer](ctx context.Context, totalData int, filter F, fetchAllFunc func(context.Context, F) ([]T, error), handleFunc func(idx int, data *T) error) error { + totalPages := int(math.Ceil(float64(totalData) / float64(filter.GetLimit()))) + for filter.GetPage() <= totalPages { + list, err := fetchAllFunc(ctx, filter) + if err != nil { + return err + } + for i, data := range list { + offset := (filter.GetPage() - 1) * filter.GetLimit() + if err := handleFunc(offset+i, &data); err != nil { + return err + } + } + filter.IncrPage() + } + return nil +} diff --git a/cmd/candi/template_usecase.go b/cmd/candi/template_usecase.go index b8619957..452b83aa 100644 --- a/cmd/candi/template_usecase.go +++ b/cmd/candi/template_usecase.go @@ -496,6 +496,7 @@ func TestNew{{upper (camel .ModuleName)}}Usecase(t *testing.T) { mockDeps := &mockdeps.Dependency{} mockDeps.On("GetRedisPool").Return(mockRedisPool) mockDeps.On("GetBroker", mock.Anything).Return(mockBroker) + mockDeps.On("GetLocker").Return(&mockinterfaces.Locker{}) uc, setFunc := New{{upper (camel .ModuleName)}}Usecase(mockDeps) setFunc(nil) diff --git a/init.go b/init.go index 2c1b3d99..98e15bd4 100644 --- a/init.go +++ b/init.go @@ -2,5 +2,5 @@ package candi const ( // Version of this library - Version = "v1.14.10" + Version = "v1.14.11" ) diff --git a/middleware/cache.go b/middleware/cache.go index bdb79daa..083cb070 100644 --- a/middleware/cache.go +++ b/middleware/cache.go @@ -20,15 +20,12 @@ const ( // HTTPCache middleware for cache func (m *Middleware) HTTPCache(next http.Handler) http.Handler { - type cacheData struct { - Body []byte `json:"body,omitempty"` - StatusCode int `json:"statusCode,omitempty"` - Header http.Header `json:"header,omitempty"` + Body []byte `json:"body,omitempty"` + Header http.Header `json:"header,omitempty"` } return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - if m.cache == nil { next.ServeHTTP(res, req) return @@ -67,8 +64,8 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler { defer trace.Finish() cacheKey := req.Method + ":" + req.URL.String() + trace.SetTag("key", cacheKey) if cacheVal, err := m.cache.Get(ctx, cacheKey); err == nil { - if ttl, err := m.cache.GetTTL(ctx, cacheKey); err == nil { res.Header().Add(HeaderExpires, time.Now().In(time.UTC).Add(ttl).Format(time.RFC1123)) } @@ -84,7 +81,6 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler { res.Header().Set(k, data.Header.Get(k)) } res.Write(data.Body) - res.WriteHeader(data.StatusCode) return } @@ -96,9 +92,8 @@ func (m *Middleware) HTTPCache(next http.Handler) http.Handler { if respWriter.StatusCode() < http.StatusBadRequest { m.cache.Set(ctx, cacheKey, candihelper.ToBytes( cacheData{ - Body: resBody.Bytes(), - StatusCode: respWriter.StatusCode(), - Header: res.Header(), + Body: resBody.Bytes(), + Header: res.Header(), }, ), maxAge) } diff --git a/mocks/candihelper/FilterStreamer.go b/mocks/candihelper/FilterStreamer.go new file mode 100644 index 00000000..f840c2b6 --- /dev/null +++ b/mocks/candihelper/FilterStreamer.go @@ -0,0 +1,58 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// FilterStreamer is an autogenerated mock type for the FilterStreamer type +type FilterStreamer struct { + mock.Mock +} + +// GetLimit provides a mock function with given fields: +func (_m *FilterStreamer) GetLimit() int { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// GetPage provides a mock function with given fields: +func (_m *FilterStreamer) GetPage() int { + ret := _m.Called() + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// IncrPage provides a mock function with given fields: +func (_m *FilterStreamer) IncrPage() { + _m.Called() +} + +type mockConstructorTestingTNewFilterStreamer interface { + mock.TestingT + Cleanup(func()) +} + +// NewFilterStreamer creates a new instance of FilterStreamer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFilterStreamer(t mockConstructorTestingTNewFilterStreamer) *FilterStreamer { + mock := &FilterStreamer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/wrapper/http_handler.go b/wrapper/http_handler.go index 0993d85a..11f63a3b 100644 --- a/wrapper/http_handler.go +++ b/wrapper/http_handler.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/golangid/candi/candihelper" @@ -26,9 +27,18 @@ type HTTPMiddlewareTracerConfig struct { // HTTPMiddlewareTracer middleware wrapper for tracer func HTTPMiddlewareTracer(cfg HTTPMiddlewareTracerConfig) func(http.Handler) http.Handler { + bPool := &sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 256)) + }, + } + releasePool := func(buff *bytes.Buffer) { + buff.Reset() + bPool.Put(buff) + } + return func(next http.Handler) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if _, isExcludePath := cfg.ExcludePath[req.URL.Path]; isExcludePath { next.ServeHTTP(rw, req) return @@ -54,32 +64,39 @@ func HTTPMiddlewareTracer(cfg HTTPMiddlewareTracerConfig) func(http.Handler) htt }() httpDump, _ := httputil.DumpRequest(req, false) + trace.Log("http.request", httpDump) trace.SetTag("http.url_path", req.URL.Path) trace.SetTag("http.method", req.Method) - trace.Log("http.request", httpDump) - body, _ := io.ReadAll(req.Body) - if len(body) < cfg.MaxLogSize { - trace.Log("request.body", body) - } else { - trace.Log("request.body.size", len(body)) + if contentLength, err := strconv.Atoi(req.Header.Get("Content-Length")); err == nil { + if contentLength < cfg.MaxLogSize { + reqBody := bPool.Get().(*bytes.Buffer) + reqBody.Reset() + reqBody.ReadFrom(req.Body) + trace.Log("request.body", reqBody.String()) + req.Body = io.NopCloser(bytes.NewReader(reqBody.Bytes())) // reuse body + releasePool(reqBody) + } else { + trace.Log("request.body.size", candihelper.TransformSizeToByte(uint64(contentLength))) + } } - req.Body = io.NopCloser(bytes.NewBuffer(body)) // reuse body - resBody := &bytes.Buffer{} + resBody := bPool.Get().(*bytes.Buffer) + resBody.Reset() + defer releasePool(resBody) respWriter := NewWrapHTTPResponseWriter(resBody, rw) - + respWriter.SetMaxWriteSize(cfg.MaxLogSize) next.ServeHTTP(respWriter, req.WithContext(ctx)) trace.SetTag("http.status_code", respWriter.statusCode) if respWriter.statusCode >= http.StatusBadRequest { trace.SetError(fmt.Errorf("resp.code:%d", respWriter.statusCode)) } - - if resBody.Len() < cfg.MaxLogSize { + trace.Log("response.header", respWriter.Header()) + if respWriter.contentLength < cfg.MaxLogSize { trace.Log("response.body", resBody.String()) } else { - trace.Log("response.body.size", resBody.Len()) + trace.Log("response.body.size", candihelper.TransformSizeToByte(uint64(respWriter.contentLength))) } }) } @@ -91,7 +108,6 @@ func HTTPMiddlewareCORS( exposeHeaders []string, allowCredential bool, ) func(http.Handler) http.Handler { - if len(allowOrigins) == 0 { allowOrigins = []string{"*"} } @@ -101,9 +117,7 @@ func HTTPMiddlewareCORS( exposeHeader := strings.Join(exposeHeaders, ",") return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - origin := req.Header.Get("Origin") allowOrigin := "" diff --git a/wrapper/http_response_writer.go b/wrapper/http_response_writer.go index f502994f..74460ed6 100644 --- a/wrapper/http_response_writer.go +++ b/wrapper/http_response_writer.go @@ -1,21 +1,41 @@ package wrapper import ( - "io" + "bytes" "net/http" ) // WrapHTTPResponseWriter wrapper type WrapHTTPResponseWriter struct { - statusCode int - writer io.Writer + statusCode int + buff *bytes.Buffer + maxWriteSize int + limitWriteSize bool + contentLength int http.ResponseWriter } // NewWrapHTTPResponseWriter init new wrapper for http response writter -func NewWrapHTTPResponseWriter(w io.Writer, httpResponseWriter http.ResponseWriter) *WrapHTTPResponseWriter { - // Default the status code to 200 - return &WrapHTTPResponseWriter{statusCode: http.StatusOK, writer: io.MultiWriter(w, httpResponseWriter), ResponseWriter: httpResponseWriter} +func NewWrapHTTPResponseWriter(responseBuff *bytes.Buffer, httpResponseWriter http.ResponseWriter) *WrapHTTPResponseWriter { + return &WrapHTTPResponseWriter{ + statusCode: http.StatusOK, buff: responseBuff, ResponseWriter: httpResponseWriter, + } +} + +// SetMaxWriteSize set max write size to buffer +func (w *WrapHTTPResponseWriter) SetMaxWriteSize(max int) { + w.maxWriteSize = max + w.limitWriteSize = true +} + +// GetContentLength get response content length +func (w *WrapHTTPResponseWriter) GetContentLength() int { + return w.contentLength +} + +// GetContent get response content +func (w *WrapHTTPResponseWriter) GetContent() []byte { + return w.buff.Bytes() } // StatusCode give a way to get the Code @@ -30,7 +50,12 @@ func (w *WrapHTTPResponseWriter) Header() http.Header { func (w *WrapHTTPResponseWriter) Write(data []byte) (int, error) { // Store response body to writer - return w.writer.Write(data) + n, err := w.ResponseWriter.Write(data) + w.contentLength += n + if !w.limitWriteSize || w.contentLength < w.maxWriteSize { + w.buff.Write(data) + } + return n, err } // WriteHeader method