scope.yml config file support

agent/agent.go

@@ -19,7 +19,7 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/opentracing/opentracing-go"
 
-	"go.undefinedlabs.com/scopeagent/env"
+	"go.undefinedlabs.com/scopeagent/config"
 	scopeError "go.undefinedlabs.com/scopeagent/errors"
 	"go.undefinedlabs.com/scopeagent/instrumentation"
 	scopetesting "go.undefinedlabs.com/scopeagent/instrumentation/testing"
@@ -66,6 +66,8 @@ var (
 
 	testingModeFrequency    = time.Second
 	nonTestingModeFrequency = time.Minute
+
+	cfg = config.Get()
 )
 
 func WithApiKey(apiKey string) Option {
@@ -210,13 +212,13 @@ func NewAgent(options ...Option) (*Agent, error) {
 		agent.logger = log.New(ioutil.Discard, "", 0)
 	}
 
-	agent.debugMode = agent.debugMode || env.ScopeDebug.Value
+	agent.debugMode = agent.debugMode || *cfg.Debug
 
 	configProfile := GetConfigCurrentProfile()
 
 	if agent.apiKey == "" || agent.apiEndpoint == "" {
-		if dsn, set := env.ScopeDsn.Tuple(); set && dsn != "" {
-			dsnApiKey, dsnApiEndpoint, dsnErr := parseDSN(dsn)
+		if cfg.Dsn != nil && *cfg.Dsn != "" {
+			dsnApiKey, dsnApiEndpoint, dsnErr := parseDSN(*cfg.Dsn)
 			if dsnErr != nil {
 				agent.logger.Printf("Error parsing dsn value: %v\n", dsnErr)
 			} else {
@@ -229,8 +231,8 @@ func NewAgent(options ...Option) (*Agent, error) {
 	}
 
 	if agent.apiKey == "" {
-		if apiKey, set := env.ScopeApiKey.Tuple(); set && apiKey != "" {
-			agent.apiKey = apiKey
+		if cfg.ApiKey != nil && *cfg.ApiKey != "" {
+			agent.apiKey = *cfg.ApiKey
 		} else if configProfile != nil {
 			agent.logger.Println("API key found in the native app configuration")
 			agent.apiKey = configProfile.ApiKey
@@ -242,12 +244,13 @@ func NewAgent(options ...Option) (*Agent, error) {
 	}
 
 	if agent.apiEndpoint == "" {
-		if endpoint, set := env.ScopeApiEndpoint.Tuple(); set && endpoint != "" {
-			agent.apiEndpoint = endpoint
+		if cfg.ApiEndpoint != nil && *cfg.ApiEndpoint != "" {
+			agent.apiEndpoint = *cfg.ApiEndpoint
 		} else if configProfile != nil {
 			agent.logger.Println("API endpoint found in the native app configuration")
 			agent.apiEndpoint = configProfile.ApiEndpoint
 		} else {
+			endpoint := "https://app.scope.dev"
 			agent.logger.Printf("using default endpoint: %v\n", endpoint)
 			agent.apiEndpoint = endpoint
 		}
@@ -287,13 +290,19 @@ func NewAgent(options ...Option) (*Agent, error) {
 	agent.metadata[tags.GoVersion] = runtime.Version()
 
 	// Service name
-	addElementToMapIfEmpty(agent.metadata, tags.Service, env.ScopeService.Value)
+	if cfg.Service != nil {
+		addElementToMapIfEmpty(agent.metadata, tags.Service, *cfg.Service)
+	}
 
 	// Configurations
-	addElementToMapIfEmpty(agent.metadata, tags.ConfigurationKeys, env.ScopeConfiguration.Value)
+	if cfg.Configuration != nil {
+		addElementToMapIfEmpty(agent.metadata, tags.ConfigurationKeys, cfg.Configuration)
+	}
 
 	// Metadata
-	addToMapIfEmpty(agent.metadata, env.ScopeMetadata.Value)
+	if cfg.Metadata != nil {
+		addToMapIfEmpty(agent.metadata, cfg.Metadata)
+	}
 
 	// Git data
 	addToMapIfEmpty(agent.metadata, getGitInfoFromEnv())
@@ -323,17 +332,19 @@ func NewAgent(options ...Option) (*Agent, error) {
 	agent.metadata[tags.SourceRoot] = sourceRoot
 
 	if !agent.testingMode {
-		if env.ScopeTestingMode.IsSet {
-			agent.testingMode = env.ScopeTestingMode.Value
+		if cfg.TestingMode != nil {
+			agent.testingMode = *cfg.TestingMode
 		} else {
 			agent.testingMode = agent.metadata[tags.CI].(bool)
 		}
 	}
 
-	if agent.failRetriesCount == 0 {
-		agent.failRetriesCount = env.ScopeTestingFailRetries.Value
+	if agent.failRetriesCount == 0 && cfg.Instrumentation.TestsFrameworks.FailRetries != nil {
+		agent.failRetriesCount = *cfg.Instrumentation.TestsFrameworks.FailRetries
+	}
+	if cfg.Instrumentation.TestsFrameworks.PanicAsFail != nil {
+		agent.panicAsFail = agent.panicAsFail || *cfg.Instrumentation.TestsFrameworks.PanicAsFail
 	}
-	agent.panicAsFail = agent.panicAsFail || env.ScopeTestingPanicAsFail.Value
 
 	if agent.debugMode {
 		agent.logMetadata()
@@ -362,7 +373,7 @@ func NewAgent(options ...Option) (*Agent, error) {
 	instrumentation.SetTracer(agent.tracer)
 	instrumentation.SetLogger(agent.logger)
 	instrumentation.SetSourceRoot(sourceRoot)
-	if agent.setGlobalTracer || env.ScopeTracerGlobal.Value {
+	if agent.setGlobalTracer || (cfg.Tracer.Global != nil && *cfg.Tracer.Global) {
 		opentracing.SetGlobalTracer(agent.Tracer())
 	}
 
@@ -457,8 +468,8 @@ func generateAgentID() string {
 }
 
 func getLogPath() (string, error) {
-	if env.ScopeLoggerRoot.IsSet {
-		return env.ScopeLoggerRoot.Value, nil
+	if cfg.Logger.Root != nil {
+		return *cfg.Logger.Root, nil
 	}
 
 	logFolder := ""

agent/agent_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"go.undefinedlabs.com/scopeagent/env"
+	"go.undefinedlabs.com/scopeagent/config"
 	"go.undefinedlabs.com/scopeagent/tags"
 )
 
@@ -134,7 +134,9 @@ func sameElements(a, b []string) bool {
 }
 
 func TestTildeExpandRaceMetadata(t *testing.T) {
-	env.ScopeSourceRoot.Value = "~/scope"
+	cfg := config.Get()
+	sroot := "~/scope"
+	cfg.SourceRoot = &sroot
 	agent, err := NewAgent(WithApiKey("123"), WithTestingModeEnabled())
 	if err != nil {
 		t.Fatal(err)

agent/git.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	"github.com/google/uuid"
-	"go.undefinedlabs.com/scopeagent/env"
 	"go.undefinedlabs.com/scopeagent/tags"
 )
 
@@ -163,6 +162,10 @@ func getGitFolder() (string, error) {
 }
 
 func getGitDiff() *GitDiff {
+	if cfg.Instrumentation.DiffSummary != nil && !*cfg.Instrumentation.DiffSummary {
+		return nil
+	}
+
 	var diff string
 	if diffBytes, err := exec.Command("git", "diff", "--numstat").Output(); err == nil {
 		diff = string(diffBytes)
@@ -229,20 +232,20 @@ func getGitInfoFromGitFolder() map[string]interface{} {
 func getGitInfoFromEnv() map[string]interface{} {
 	gitInfo := map[string]interface{}{}
 
-	if repository, set := env.ScopeRepository.Tuple(); set && repository != "" {
-		gitInfo[tags.Repository] = repository
+	if cfg.Repository != nil && *cfg.Repository != "" {
+		gitInfo[tags.Repository] = *cfg.Repository
 	}
-	if commit, set := env.ScopeCommitSha.Tuple(); set && commit != "" {
-		gitInfo[tags.Commit] = commit
+	if cfg.CommitSha != nil && *cfg.CommitSha != "" {
+		gitInfo[tags.Commit] = *cfg.CommitSha
 	}
-	if sourceRoot, set := env.ScopeSourceRoot.Tuple(); set && sourceRoot != "" {
+	if cfg.SourceRoot != nil && *cfg.SourceRoot != "" {
 		// We check if is a valid and existing folder
-		if fInfo, err := os.Stat(sourceRoot); err == nil && fInfo.IsDir() {
-			gitInfo[tags.SourceRoot] = sourceRoot
+		if fInfo, err := os.Stat(*cfg.SourceRoot); err == nil && fInfo.IsDir() {
+			gitInfo[tags.SourceRoot] = *cfg.SourceRoot
 		}
 	}
-	if branch, set := env.ScopeBranch.Tuple(); set && branch != "" {
-		gitInfo[tags.Branch] = branch
+	if cfg.Branch != nil && *cfg.Branch != "" {
+		gitInfo[tags.Branch] = *cfg.Branch
 	}
 
 	return gitInfo

agent/recorder.go

@@ -38,8 +38,10 @@ type (
 		debugMode   bool
 		metadata    map[string]interface{}
 
-		payloadSpans  []PayloadSpan
-		payloadEvents []PayloadEvent
+		payloadSpans        []PayloadSpan
+		payloadEvents       []PayloadEvent
+		maxSpansPerPayload  int
+		maxEventsPerPayload int
 
 		flushFrequency time.Duration
 		url            string
@@ -82,6 +84,16 @@ func NewSpanRecorder(agent *Agent) *SpanRecorder {
 	r.url = agent.getUrl("api/agent/ingest")
 	r.client = &http.Client{}
 	r.stats = &RecorderStats{}
+	if cfg.Tracer.Dispatcher.Events.MaxPayloadSize != nil {
+		r.maxEventsPerPayload = *cfg.Tracer.Dispatcher.Events.MaxPayloadSize
+	} else {
+		r.maxEventsPerPayload = 1000
+	}
+	if cfg.Tracer.Dispatcher.Spans.MaxPayloadSize != nil {
+		r.maxSpansPerPayload = *cfg.Tracer.Dispatcher.Spans.MaxPayloadSize
+	} else {
+		r.maxSpansPerPayload = 1000
+	}
 	r.t.Go(r.loop)
 	return r
 }
@@ -142,11 +154,10 @@ func (r *SpanRecorder) loop() error {
 // Sends the spans in the buffer to Scope
 func (r *SpanRecorder) sendSpans() (error, bool) {
 	atomic.AddInt64(&r.stats.sendSpansCalls, 1)
-	const batchSize = 1000
 	var lastError error
 	for {
-		spans, spMore, spTotal := r.popPayloadSpan(batchSize)
-		events, evMore, evTotal := r.popPayloadEvents(batchSize)
+		spans, spMore, spTotal := r.popPayloadSpan(r.maxSpansPerPayload)
+		events, evMore, evTotal := r.popPayloadEvents(r.maxEventsPerPayload)
 
 		payload := map[string]interface{}{
 			"metadata":   r.metadata,

config/types.go

@@ -0,0 +1,70 @@
+package config
+
+type (
+	ScopeConfig struct {
+		Dsn             *string                `env:"SCOPE_DSN"`
+		ApiKey          *string                `env:"SCOPE_APIKEY"`
+		ApiEndpoint     *string                `env:"SCOPE_API_ENDPOINT"`
+		Service         *string                `yaml:"service" env:"SCOPE_SERVICE" default:"default"`
+		Repository      *string                `yaml:"repository" env:"SCOPE_REPOSITORY"`
+		CommitSha       *string                `yaml:"commit_sha" env:"SCOPE_COMMIT_SHA"`
+		Branch          *string                `yaml:"branch" env:"SCOPE_BRANCH"`
+		SourceRoot      *string                `yaml:"source_root" env:"SCOPE_SOURCE_ROOT"`
+		Logger          LoggerConfig           `yaml:"logger"`
+		Metadata        map[string]interface{} `yaml:"metadata" env:"SCOPE_METADATA"`
+		Configuration   []string               `yaml:"configuration" env:"SCOPE_CONFIGURATION" default:"platform.name, platform.architecture, go.version"`
+		TestingMode     *bool                  `yaml:"testing_mode" env:"SCOPE_TESTING_MODE" default:"false"`
+		Instrumentation InstrumentationConfig  `yaml:"instrumentation"`
+		Tracer          TracerConfig           `yaml:"tracer"`
+		Debug           *bool                  `env:"SCOPE_DEBUG" default:"false"`
+		ConfigPath      *string
+		LoadError       error
+	}
+	LoggerConfig struct {
+		Root *string `yaml:"root" env:"SCOPE_LOGGER_ROOT, SCOPE_LOG_ROOT_PATH"`
+	}
+	InstrumentationConfig struct {
+		DiffSummary     *bool                                `yaml:"diff_summary" env:"SCOPE_INSTRUMENTATION_DIFF_SUMMARY" default:"true"`
+		TestsFrameworks InstrumentationTestsFrameworksConfig `yaml:"tests_frameworks"`
+		DB              InstrumentationDatabaseConfig        `yaml:"db"`
+		Http            InstrumentationHttpConfig            `yaml:"http"`
+		Logger          InstrumentationLoggerConfig          `yaml:"logger"`
+	}
+	InstrumentationTestsFrameworksConfig struct {
+		FailRetries *int  `yaml:"fail_retries" env:"SCOPE_INSTRUMENTATION_TESTS_FRAMEWORKS_FAIL_RETRIES" default:"0"`
+		PanicAsFail *bool `yaml:"panic_as_fail" env:"SCOPE_INSTRUMENTATION_TESTS_FRAMEWORKS_PANIC_AS_FAIL" default:"false"`
+	}
+	InstrumentationDatabaseConfig struct {
+		StatementValues *bool `yaml:"statement_values" env:"SCOPE_INSTRUMENTATION_DB_STATEMENT_VALUES" default:"false"`
+		Stacktrace      *bool `yaml:"stacktrace" env:"SCOPE_INSTRUMENTATION_DB_STACKTRACE" default:"false"`
+	}
+	InstrumentationHttpConfig struct {
+		Client     *bool    `yaml:"client" env:"SCOPE_INSTRUMENTATION_HTTP_CLIENT" default:"true"`
+		Server     *bool    `yaml:"server" env:"SCOPE_INSTRUMENTATION_HTTP_SERVER" default:"true"`
+		Payloads   *bool    `yaml:"payloads" env:"SCOPE_INSTRUMENTATION_HTTP_PAYLOADS" default:"false"`
+		Stacktrace *bool    `yaml:"stacktrace" env:"SCOPE_INSTRUMENTATION_HTTP_STACKTRACE" default:"false"`
+		Headers    []string `yaml:"headers" env:"SCOPE_INSTRUMENTATION_HTTP_HEADERS"`
+	}
+	InstrumentationLoggerConfig struct {
+		StandardLogger *bool `yaml:"standard_logger" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_LOGGER" default:"true"`
+		StandardOutput *bool `yaml:"standard_output" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_OUTPUT" default:"false"`
+		StandardError  *bool `yaml:"standard_error" env:"SCOPE_INSTRUMENTATION_LOGGER_STANDARD_ERROR" default:"false"`
+	}
+	TracerConfig struct {
+		Global     *bool                  `yaml:"global" env:"SCOPE_TRACER_GLOBAL, SCOPE_SET_GLOBAL_TRACER" default:"false"`
+		Dispatcher TracerDispatcherConfig `yaml:"dispatcher"`
+	}
+	TracerDispatcherConfig struct {
+		HealthCheckFrequency           *int                         `yaml:"healthcheck_frecuency" env:"SCOPE_TRACER_DISPATCHER_HEALTHCHECK_FRECUENCY" default:"1000"`
+		HealthCheckFrequencyInTestMode *int                         `yaml:"healthcheck_frecuency_in_testmode" env:"SCOPE_TRACER_DISPATCHER_HEALTHCHECK_FRECUENCY_IN_TESTMODE" default:"60000"`
+		ConcurrencyLevel               *int                         `yaml:"concurrency_level" env:"SCOPE_TRACER_DISPATCHER_CONCURRENCY_LEVEL" default:"1"`
+		Spans                          TracerDispatcherSpansConfig  `yaml:"spans"`
+		Events                         TracerDispatcherEventsConfig `yaml:"events"`
+	}
+	TracerDispatcherSpansConfig struct {
+		MaxPayloadSize *int `yaml:"max_payload_size" env:"SCOPE_TRACER_DISPATCHER_SPANS_MAX_PAYLOAD_SIZE" default:"1000"`
+	}
+	TracerDispatcherEventsConfig struct {
+		MaxPayloadSize *int `yaml:"max_payload_size" env:"SCOPE_TRACER_DISPATCHER_EVENTS_MAX_PAYLOAD_SIZE" default:"1000"`
+	}
+)

config/vars.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"gopkg.in/yaml.v2"
+
+	"github.com/undefinedlabs/go-env"
+)
+
+var (
+	current *ScopeConfig
+	m       sync.RWMutex
+)
+
+func Get() *ScopeConfig {
+	// We check is already loaded with a reader lock
+	m.RLock()
+	if current != nil {
+		defer m.RUnlock()
+		return current
+	}
+	m.RUnlock()
+
+	// Is not loaded we block to load it
+	m.Lock()
+	defer m.Unlock()
+	if current != nil {
+		return current
+	}
+	var config ScopeConfig
+	content, path, err := readConfigurationFile()
+	if err == nil {
+		config.ConfigPath = path
+		_ = yaml.Unmarshal(content, &config)
+		if config.Metadata != nil {
+			for k, v := range config.Metadata {
+				if str, ok := v.(string); ok {
+					config.Metadata[k] = os.ExpandEnv(str)
+				}
+			}
+		}
+	} else {
+		config.LoadError = err
+	}
+	_, err = env.UnmarshalFromEnviron(&config)
+	if err != nil {
+		config.LoadError = err
+	}
+	current = &config
+	return current
+}
+
+func readConfigurationFile() ([]byte, *string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return nil, nil, err
+	}
+	for {
+		rel, _ := filepath.Rel("/", dir)
+		// Exit the loop once we reach the basePath.
+		if rel == "." {
+			break
+		}
+
+		path := fmt.Sprintf("%v/scope.yml", dir)
+		dat, err := ioutil.ReadFile(path)
+		if err == nil {
+			return dat, &path, nil
+		}
+
+		// Going up!
+		dir += "/.."
+	}
+	return nil, nil, errors.New("configuration not found")
+}