feat: adds proper support for goroutines, block, mutex profiling (#1178)

* feat: adds proper support for goroutines, block, mutex profiling

* moves type definitions to default scrape config

* updates default types mapping

* enables more self-profiling

* Update upstream.go

* updates client version

* cleanup

examples/golang-pull/static/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.18.3
+
+WORKDIR /go/src/app
+
+COPY main.go ./main.go
+
+RUN go mod init github.com/pyroscope-io/pyroscope/examples/golang-pull/static
+RUN go get -d ./
+RUN go build -o main .
+
+RUN adduser --disabled-password --gecos --quiet pyroscope
+USER pyroscope
+
+CMD ["./main"]

examples/golang-pull/static/docker-compose.yml

@@ -26,3 +26,8 @@ services:
     ports:
       - '6831:6831/udp'
       - '16686:16686'
+
+  app:
+    build: .
+    ports:
+      - 6061:6060

examples/golang-pull/static/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/pyroscope-io/client/pyroscope"
+)
+
+//go:noinline
+func work(n int) {
+	// revive:disable:empty-block this is fine because this is a example app, not real production code
+	for i := 0; i < n; i++ {
+	}
+	// revive:enable:empty-block
+}
+
+var m *sync.Mutex
+
+func init() {
+	m = &sync.Mutex{}
+	runtime.SetMutexProfileFraction(5)
+	runtime.SetBlockProfileRate(5)
+}
+
+func fastFunction(c context.Context, wg *sync.WaitGroup) {
+	m.Lock()
+	defer m.Unlock()
+	pyroscope.TagWrapper(c, pyroscope.Labels("function", "fast"), func(c context.Context) {
+		work(20000000)
+	})
+	wg.Done()
+}
+
+func slowFunction(c context.Context, wg *sync.WaitGroup) {
+	m.Lock()
+	defer m.Unlock()
+	// standard pprof.Do wrappers work as well
+	pprof.Do(c, pprof.Labels("function", "slow"), func(c context.Context) {
+		work(80000000)
+	})
+	wg.Done()
+}
+
+func main() {
+	go func() {
+		log.Println(http.ListenAndServe(":6060", nil))
+	}()
+	serverAddress := os.Getenv("PYROSCOPE_SERVER_ADDRESS")
+	if serverAddress == "" {
+		serverAddress = "http://localhost:4040"
+	}
+	pyroscope.TagWrapper(context.Background(), pyroscope.Labels("foo", "bar"), func(c context.Context) {
+		for {
+			wg := sync.WaitGroup{}
+			wg.Add(6)
+			go fastFunction(c, &wg)
+			go fastFunction(c, &wg)
+			go fastFunction(c, &wg)
+			go slowFunction(c, &wg)
+			go slowFunction(c, &wg)
+			go slowFunction(c, &wg)
+			wg.Wait()
+		}
+	})
+}

examples/golang-pull/static/server.yml

@@ -2,11 +2,17 @@
 log-level: debug
 scrape-configs:
   - job-name: testing
-    enabled-profiles: [cpu, mem]
+    enabled-profiles: [cpu, mem, goroutines, mutex, block]
     static-configs:
       - application: hotrod
         spy-name: gospy
         targets:
           - hotrod:6060
         labels:
           env: dev
+      - application: last-app
+        spy-name: gospy
+        targets:
+          - app:6060
+        labels:
+          env: dev

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
 	github.com/prometheus/common v0.34.0
-	github.com/pyroscope-io/client v0.0.0-20211206204731-3fd0a4b8239c
+	github.com/pyroscope-io/client v0.3.0
 	github.com/pyroscope-io/dotnetdiag v1.2.1
 	github.com/pyroscope-io/goldga v0.4.2-0.20220218190441-817afcc3a7f1
 	github.com/pyroscope-io/jfr-parser v0.5.2
@@ -175,5 +175,3 @@ require (
 )
 
 replace github.com/mgechev/revive v1.0.3 => github.com/pyroscope-io/revive v1.0.6-0.20210330033039-4a71146f9dc1
-
-replace github.com/pyroscope-io/client => github.com/pyroscope-io/client v0.2.4-0.20220607180407-0ba26860ce5b

go.sum

@@ -660,8 +660,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/statsd_exporter v0.15.0/go.mod h1:Dv8HnkoLQkeEjkIE4/2ndAA7WL1zHKK7WMqFQqu72rw=
-github.com/pyroscope-io/client v0.2.4-0.20220607180407-0ba26860ce5b h1:5YuLgMAQ7XbW8Kmccwv0C1vkl85SG7ItLC8AdMtCFOs=
-github.com/pyroscope-io/client v0.2.4-0.20220607180407-0ba26860ce5b/go.mod h1:zRdQXIGxy0H2QbKEkCmZBR6KOLLIFYLWsdzVI0MRm2E=
+github.com/pyroscope-io/client v0.3.0 h1:H+1pvKpdMOESX9jYuJKbO0+7yWY7aQNjIKuQ2rpnZIY=
+github.com/pyroscope-io/client v0.3.0/go.mod h1:zRdQXIGxy0H2QbKEkCmZBR6KOLLIFYLWsdzVI0MRm2E=
 github.com/pyroscope-io/dotnetdiag v1.2.1 h1:3XEMrfFJnZ87BiEhozyQKmCUAuMd/Spq7KChPuD2Cf0=
 github.com/pyroscope-io/dotnetdiag v1.2.1/go.mod h1:eFUEHCp4eD1TgcXMlJihC+R4MrqGf7nTRdWxNADbDHA=
 github.com/pyroscope-io/goldga v0.4.2-0.20220218190441-817afcc3a7f1 h1:T1fDdt3E3UpaGZ7tRF2IYrUFwNyuPlIWGeCOjfINp1s=

packages/pyroscope-flamegraph/src/FlameGraph/FlameGraphComponent/Header.tsx

@@ -29,6 +29,7 @@ export default function Header(props: HeaderProps) {
 
   const unitsToFlamegraphTitle = {
     objects: 'number of objects in RAM per function',
+    goroutines: 'number of goroutines',
     bytes: 'amount of RAM per function',
     samples: 'CPU time per function',
     lock_nanoseconds: 'time spent waiting on locks per function',

packages/pyroscope-flamegraph/src/format/format.ts

@@ -20,6 +20,8 @@ export function getFormatter(max: number, sampleRate: number, unit: Units) {
       return new DurationFormatter(max / sampleRate);
     case 'objects':
       return new ObjectsFormatter(max);
+    case 'goroutines':
+      return new ObjectsFormatter(max);
     case 'bytes':
       return new BytesFormatter(max);
     case 'lock_nanoseconds':

packages/pyroscope-models/src/flamebearer.ts

@@ -23,6 +23,7 @@ export type Flamebearer = {
   units:
     | 'samples'
     | 'objects'
+    | 'goroutines'
     | 'bytes'
     | 'lock_samples'
     | 'lock_nanoseconds'

packages/pyroscope-models/src/profile.ts

@@ -13,6 +13,7 @@ const UnitsSchema = z.preprocess((u) => {
   const units = [
     'samples',
     'objects',
+    'goroutines',
     'bytes',
     'lock_samples',
     'lock_nanoseconds',
@@ -23,7 +24,7 @@ const UnitsSchema = z.preprocess((u) => {
     }
   }
   return '';
-}, z.enum(['samples', 'objects', 'bytes', 'lock_samples', 'lock_nanoseconds', '']));
+}, z.enum(['samples', 'objects', 'goroutines', 'bytes', 'lock_samples', 'lock_nanoseconds', '']));
 
 export const MetadataSchema = z.object({
   // Optional fields since adhoc may be missing them

pkg/scrape/config/config.go

@@ -75,6 +75,49 @@ func DefaultConfig() *Config {
 					},
 				},
 			},
+			"goroutines": {
+				Path:   "/debug/pprof/goroutine",
+				Params: nil,
+				SampleTypes: map[string]*profile.SampleTypeConfig{
+					"goroutine": {
+						DisplayName: "goroutines",
+						Units:       metadata.GoroutinesUnits,
+						Aggregation: metadata.AverageAggregationType,
+					},
+				},
+			},
+			"mutex": {
+				Path:   "/debug/pprof/mutex",
+				Params: nil,
+				SampleTypes: map[string]*profile.SampleTypeConfig{
+					"contentions": {
+						DisplayName: "mutex_count",
+						Units:       metadata.LockSamplesUnits,
+						Cumulative:  true,
+					},
+					"delay": {
+						DisplayName: "mutex_duration",
+						Units:       metadata.LockNanosecondsUnits,
+						Cumulative:  true,
+					},
+				},
+			},
+			"block": {
+				Path:   "/debug/pprof/block",
+				Params: nil,
+				SampleTypes: map[string]*profile.SampleTypeConfig{
+					"contentions": {
+						DisplayName: "block_count",
+						Units:       metadata.LockSamplesUnits,
+						Cumulative:  true,
+					},
+					"delay": {
+						DisplayName: "block_duration",
+						Units:       metadata.LockNanosecondsUnits,
+						Cumulative:  true,
+					},
+				},
+			},
 		},
 
 		HTTPClientConfig: DefaultHTTPClientConfig,

pkg/selfprofiling/upstream.go

@@ -2,6 +2,7 @@ package selfprofiling
 
 import (
 	"context"
+	"runtime"
 	"runtime/debug"
 	"time"
 
@@ -15,14 +16,28 @@ import (
 )
 
 func NewSession(logger pyroscope.Logger, ingester ingestion.Ingester, appName string, tags map[string]string) *pyroscope.Session {
+	runtime.SetMutexProfileFraction(5)
+	runtime.SetBlockProfileRate(5)
+
 	session, _ := pyroscope.NewSession(pyroscope.SessionConfig{
-		Upstream:       NewUpstream(logger, ingester),
-		AppName:        appName,
-		ProfilingTypes: pyroscope.DefaultProfileTypes,
-		SampleRate:     100,
-		UploadRate:     10 * time.Second,
-		Logger:         logger,
-		Tags:           tags,
+		Upstream: NewUpstream(logger, ingester),
+		AppName:  appName,
+		ProfilingTypes: []pyroscope.ProfileType{
+			pyroscope.ProfileCPU,
+			pyroscope.ProfileInuseObjects,
+			pyroscope.ProfileAllocObjects,
+			pyroscope.ProfileInuseSpace,
+			pyroscope.ProfileAllocSpace,
+			pyroscope.ProfileGoroutines,
+			pyroscope.ProfileMutexCount,
+			pyroscope.ProfileMutexDuration,
+			pyroscope.ProfileBlockCount,
+			pyroscope.ProfileBlockDuration,
+		},
+		SampleRate: 100,
+		UploadRate: 10 * time.Second,
+		Logger:     logger,
+		Tags:       tags,
 	})
 	return session
 }

pkg/storage/metadata/metadata.go

@@ -9,6 +9,7 @@ func (u Units) String() string {
 const (
 	SamplesUnits         Units = "samples"
 	ObjectsUnits               = "objects"
+	GoroutinesUnits            = "goroutines"
 	BytesUnits                 = "bytes"
 	LockNanosecondsUnits       = "lock_nanoseconds"
 	LockSamplesUnits           = "lock_samples"

pkg/storage/tree/pprof.go

@@ -40,25 +40,44 @@ var DefaultSampleTypeMapping = map[string]*SampleTypeConfig{
 	// Sample types specific to Go.
 	"samples": {
 		DisplayName: "cpu",
-		Units:       "samples",
+		Units:       metadata.SamplesUnits,
 		Sampled:     true,
 	},
 	"inuse_objects": {
-		Units:       "objects",
-		Aggregation: "average",
+		Units:       metadata.ObjectsUnits,
+		Aggregation: metadata.AverageAggregationType,
 	},
 	"alloc_objects": {
-		Units:      "objects",
+		Units:      metadata.ObjectsUnits,
 		Cumulative: true,
 	},
 	"inuse_space": {
-		Units:       "bytes",
-		Aggregation: "average",
+		Units:       metadata.BytesUnits,
+		Aggregation: metadata.AverageAggregationType,
 	},
 	"alloc_space": {
-		Units:      "bytes",
+		Units:      metadata.BytesUnits,
 		Cumulative: true,
 	},
+	"goroutine": {
+		DisplayName: "goroutines",
+		Units:       metadata.GoroutinesUnits,
+		Aggregation: metadata.AverageAggregationType,
+	},
+	"contentions": {
+		// TODO(petethepig): technically block profiles have the same name
+		//   so this might be a block profile, need better heuristic
+		DisplayName: "mutex_count",
+		Units:       metadata.LockSamplesUnits,
+		Cumulative:  true,
+	},
+	"delay": {
+		// TODO(petethepig): technically block profiles have the same name
+		//   so this might be a block profile, need better heuristic
+		DisplayName: "mutex_duration",
+		Units:       metadata.LockNanosecondsUnits,
+		Cumulative:  true,
+	},
 }
 
 type pprof struct {