diff --git a/assets/go-licenses.json b/assets/go-licenses.json index a20494184bd40..817c84bc04f05 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -439,6 +439,11 @@ "path": "github.com/felixge/fgprof/LICENSE.txt", "licenseText": "The MIT License (MIT)\nCopyright © 2020 Felix Geisendörfer \u003cfelix@felixge.de\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/felixge/httpsnoop", + "path": "github.com/felixge/httpsnoop/LICENSE.txt", + "licenseText": "Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in\n all copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n THE SOFTWARE.\n" + }, { "name": "github.com/fsnotify/fsnotify", "path": "github.com/fsnotify/fsnotify/LICENSE", diff --git a/go.mod b/go.mod index 0ee4257f13061..88e495d9ed971 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/common v0.60.1 github.com/quasoft/websspi v1.1.2 github.com/redis/go-redis/v9 v9.7.0 github.com/robfig/cron/v3 v3.0.1 @@ -241,6 +242,7 @@ require ( github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -268,7 +270,6 @@ require ( github.com/pjbgf/sha1cd v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rhysd/actionlint v1.7.3 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/modules/cache/cache.go b/modules/cache/cache.go index f7828e3cae2df..e1af57faa7b79 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -10,10 +10,39 @@ import ( "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + _ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache" ) -var defaultCache StringCache +var ( + defaultCache StringCache + + // TODO: Combine hit and miss into one + hitCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "hit"}, + }) + missCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "miss"}, + }) + latencyHistogram = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "gitea", + Help: "Cache latency", + Subsystem: "cache", + Name: "duration", + }, + ) +) // Init start cache service func Init() error { diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 4f659616f501e..328cb6d2135f3 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -6,6 +6,7 @@ package cache import ( "errors" "strings" + "time" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" @@ -63,10 +64,15 @@ func (sc *stringCache) Ping() error { } func (sc *stringCache) Get(key string) (string, bool) { + start := time.Now() v := sc.chiCache.Get(key) + elapsed := time.Since(start).Seconds() + latencyHistogram.Observe(elapsed) if v == nil { + missCounter.Add(1) return "", false } + hitCounter.Add(1) s, ok := v.(string) return s, ok } diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 230260ff94896..3701ba9ec7335 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -11,10 +11,15 @@ import ( "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" ) const namespace = "gitea_" +func init() { + model.NameValidationScheme = model.UTF8Validation +} + // Collector implements the prometheus.Collector interface and // exposes gitea metrics for prometheus type Collector struct { @@ -89,7 +94,7 @@ func NewCollector() Collector { Issues: prometheus.NewDesc( namespace+"issues", "Number of Issues", - nil, nil, + []string{"state"}, nil, ), IssuesByLabel: prometheus.NewDesc( namespace+"issues_by_label", @@ -103,12 +108,12 @@ func NewCollector() Collector { ), IssuesOpen: prometheus.NewDesc( namespace+"issues_open", - "Number of open Issues", + "DEPRECATED: Use Issues with state: open", nil, nil, ), IssuesClosed: prometheus.NewDesc( namespace+"issues_closed", - "Number of closed Issues", + "DEPRECATED: Use Issues with state: closed", nil, nil, ), Labels: prometheus.NewDesc( @@ -272,8 +277,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Issues, prometheus.GaugeValue, - float64(stats.Counter.Issue), + float64(stats.Counter.IssueOpen), "open", + ) + ch <- prometheus.MustNewConstMetric( + c.Issues, + prometheus.GaugeValue, + float64(stats.Counter.IssueClosed), "closed", ) + for _, il := range stats.Counter.IssueByLabel { ch <- prometheus.MustNewConstMetric( c.IssuesByLabel, diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 2ba02de8edc48..5d7a2ee03e296 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -6,7 +6,9 @@ package common import ( "fmt" "net/http" + "strconv" "strings" + "time" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/gtprof" @@ -19,6 +21,45 @@ import ( "gitea.com/go-chi/session" "github.com/chi-middleware/proxy" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + httpRequestMethod = "http_request_method" + httpResponseStatusCode = "http_response_status_code" + httpRoute = "http_route" +) + +var ( + // reqInflightGauge tracks the amount of currently handled requests + reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "http", + Subsystem: "server", + Name: "active_requests", + Help: "Number of active HTTP server requests.", + }, []string{httpRequestMethod}) + // reqDurationHistogram tracks the time taken by http request + reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server", + Name: "request_duration", + Help: "Measures the latency of HTTP requests processed by the server", + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) + // reqSizeHistogram tracks the size of request + reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_request", + Name: "body_size", + Help: "Size of HTTP server request bodies.", + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) + // respSizeHistogram tracks the size of the response + respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_response", + Name: "body_size", + Help: "Size of HTTP server response bodies.", + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) ) // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery @@ -38,6 +79,9 @@ func ProtocolMiddlewares() (handlers []any) { if setting.IsAccessLogEnabled() { handlers = append(handlers, context.AccessLogger()) } + if setting.Metrics.Enabled { + handlers = append(handlers, RouteMetrics()) + } return handlers } @@ -107,6 +151,30 @@ func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Han return proxy.ForwardedHeaders(opt) } +// RouteMetrics instruments http requests and responses +func RouteMetrics() func(h http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + inflight := reqInflightGauge.WithLabelValues(req.Method) + inflight.Inc() + defer inflight.Dec() + start := time.Now() + + next.ServeHTTP(resp, req) + m := context.WrapResponseWriter(resp) + route := chi.RouteContext(req.Context()).RoutePattern() + code := strconv.Itoa(m.WrittenStatus()) + reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds()) + respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.WrittenSize())) + size := req.ContentLength + if size < 0 { + size = 0 + } + reqSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(size)) + }) + } +} + func Sessioner() func(next http.Handler) http.Handler { return session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go new file mode 100644 index 0000000000000..b2ec73c565d49 --- /dev/null +++ b/routers/common/middleware_test.go @@ -0,0 +1,38 @@ +package common + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetricsMiddlewere(t *testing.T) { + middleware := RouteMetrics() + r := chi.NewRouter() + r.Use(middleware) + r.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("test")) + time.Sleep(5 * time.Millisecond) + })) + testServer := httptest.NewServer(r) + // Check all defined metrics + verify := func(i int) { + assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration"), i) + assert.Equal(t, testutil.CollectAndCount(reqSizeHistogram, "http_server_request_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(respSizeHistogram, "http_server_response_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(reqInflightGauge, "http_server_active_requests"), i) + } + + // Check they don't exist before making a request + verify(0) + _, err := http.Get(testServer.URL) + require.NoError(t, err) + // Check they do exist after making the request + verify(1) +} diff --git a/routers/web/web.go b/routers/web/web.go index ef26dd9a7438a..02167b60d7558 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -52,6 +52,7 @@ import ( "github.com/go-chi/cors" "github.com/klauspost/compress/gzhttp" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" ) var GzipMinSize = 1400 // min size to compress for the body size of response @@ -258,6 +259,7 @@ func Routes() *web.Router { } if setting.Metrics.Enabled { + prometheus.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{Namespace: "gitea"})) prometheus.MustRegister(metrics.NewCollector()) routes.Get("/metrics", append(mid, Metrics)...) } diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 51b22d6111b15..edff7d4a959cd 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -21,6 +21,14 @@ import ( base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + repoMigrationsInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "inflight_migrations", Help: "Number of inflight repository migrations"}) + repoMigrationsCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "migrations", Help: "Total migrations"}, []string{"result"}) ) // MigrateOptions is equal to base.MigrateOptions @@ -124,6 +132,9 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str return nil, err } + repoMigrationsInflightGauge.Inc() + defer repoMigrationsInflightGauge.Dec() + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType @@ -134,8 +145,10 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { log.Error("create respotiry notice failed: ", err2) } + repoMigrationsCounter.WithLabelValues("fail").Inc() return nil, err } + repoMigrationsCounter.WithLabelValues("success").Inc() return uploader.repo, nil }