From 126c9b407b2e20821739627b6c297a113a0b20ed Mon Sep 17 00:00:00 2001 From: Yu Lei Date: Wed, 21 Jul 2021 18:01:36 +0800 Subject: [PATCH] telemetry: Add telemetry information about builtin functions usage (#26234) --- planner/core/expression_rewriter.go | 18 +++++--- session/session.go | 27 +++++++---- sessionctx/context.go | 3 ++ telemetry/data_window.go | 70 ++++++++++++++++++++++++++--- util/mock/context.go | 5 +++ 5 files changed, 104 insertions(+), 19 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 3a1d7391965a1..dd590daf63f94 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/telemetry" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" @@ -1178,14 +1179,21 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok } // newFunction chooses which expression.NewFunctionImpl() will be used. -func (er *expressionRewriter) newFunction(funcName string, retType *types.FieldType, args ...expression.Expression) (expression.Expression, error) { +func (er *expressionRewriter) newFunction(funcName string, retType *types.FieldType, args ...expression.Expression) (ret expression.Expression, err error) { if er.disableFoldCounter > 0 { - return expression.NewFunctionBase(er.sctx, funcName, retType, args...) + ret, err = expression.NewFunctionBase(er.sctx, funcName, retType, args...) + } else if er.tryFoldCounter > 0 { + ret, err = expression.NewFunctionTryFold(er.sctx, funcName, retType, args...) + } else { + ret, err = expression.NewFunction(er.sctx, funcName, retType, args...) } - if er.tryFoldCounter > 0 { - return expression.NewFunctionTryFold(er.sctx, funcName, retType, args...) + if err != nil { + return } - return expression.NewFunction(er.sctx, funcName, retType, args...) + if scalarFunc, ok := ret.(*expression.ScalarFunction); ok { + telemetry.BuiltinFunctionsUsage(er.b.ctx.GetBuiltinFunctionUsage()).Inc(scalarFunc.Function.PbCode().String()) + } + return } func (er *expressionRewriter) checkTimePrecision(ft *types.FieldType) error { diff --git a/session/session.go b/session/session.go index cef9fa7695e53..297b314639e2a 100644 --- a/session/session.go +++ b/session/session.go @@ -225,6 +225,8 @@ type session struct { idxUsageCollector *handle.SessionIndexUsageCollector cache [1]ast.StmtNode + + builtinFunctionUsage telemetry.BuiltinFunctionsUsage } var parserPool = &sync.Pool{New: func() interface{} { return parser.New() }} @@ -2244,6 +2246,7 @@ func (s *session) Close() { if s.idxUsageCollector != nil { s.idxUsageCollector.Delete() } + telemetry.GlobalBuiltinFunctionsUsage.Collect(s.GetBuiltinFunctionUsage()) bindValue := s.Value(bindinfo.SessionBindInfoKeyType) if bindValue != nil { bindValue.(*bindinfo.SessionHandle).Close() @@ -2659,11 +2662,12 @@ func createSessionWithOpt(store kv.Storage, opt *Opt) (*session, error) { return nil, err } s := &session{ - store: store, - sessionVars: variable.NewSessionVars(), - ddlOwnerChecker: dom.DDL().OwnerManager(), - client: store.GetClient(), - mppClient: store.GetMPPClient(), + store: store, + sessionVars: variable.NewSessionVars(), + ddlOwnerChecker: dom.DDL().OwnerManager(), + client: store.GetClient(), + mppClient: store.GetMPPClient(), + builtinFunctionUsage: make(telemetry.BuiltinFunctionsUsage), } if plannercore.PreparedPlanCacheEnabled() { if opt != nil && opt.PreparedPlanCache != nil { @@ -2692,10 +2696,11 @@ func createSessionWithOpt(store kv.Storage, opt *Opt) (*session, error) { // a lock context, which cause we can't call createSession directly. func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, error) { s := &session{ - store: store, - sessionVars: variable.NewSessionVars(), - client: store.GetClient(), - mppClient: store.GetMPPClient(), + store: store, + sessionVars: variable.NewSessionVars(), + client: store.GetClient(), + mppClient: store.GetMPPClient(), + builtinFunctionUsage: make(telemetry.BuiltinFunctionsUsage), } if plannercore.PreparedPlanCacheEnabled() { s.preparedPlanCache = kvcache.NewSimpleLRUCache(plannercore.PreparedPlanCacheCapacity, @@ -3126,3 +3131,7 @@ func (s *session) updateTelemetryMetric(es *executor.ExecStmt) { telemetryCTEUsage.WithLabelValues("notCTE").Inc() } } + +func (s *session) GetBuiltinFunctionUsage() map[string]uint32 { + return s.builtinFunctionUsage +} diff --git a/sessionctx/context.go b/sessionctx/context.go index 0b7413178a221..2a25292b3a17b 100644 --- a/sessionctx/context.go +++ b/sessionctx/context.go @@ -128,6 +128,9 @@ type Context interface { StoreIndexUsage(tblID int64, idxID int64, rowsSelected int64) // GetTxnWriteThroughputSLI returns the TxnWriteThroughputSLI. GetTxnWriteThroughputSLI() *sli.TxnWriteThroughputSLI + // GetBuiltinFunctionUsage returns the BuiltinFunctionUsage of current Context, which is not thread safe. + // Use primitive map type to prevent circular import. Should convert it to telemetry.BuiltinFunctionUsage before using. + GetBuiltinFunctionUsage() map[string]uint32 } type basicCtxType int diff --git a/telemetry/data_window.go b/telemetry/data_window.go index b03eed8b8d5b0..54e7f8bcc3daf 100644 --- a/telemetry/data_window.go +++ b/telemetry/data_window.go @@ -61,11 +61,12 @@ const ( ) type windowData struct { - BeginAt time.Time `json:"beginAt"` - ExecuteCount uint64 `json:"executeCount"` - TiFlashUsage tiFlashUsageData `json:"tiFlashUsage"` - CoprCacheUsage coprCacheUsageData `json:"coprCacheUsage"` - SQLUsage sqlUsageData `json:"SQLUsage"` + BeginAt time.Time `json:"beginAt"` + ExecuteCount uint64 `json:"executeCount"` + TiFlashUsage tiFlashUsageData `json:"tiFlashUsage"` + CoprCacheUsage coprCacheUsageData `json:"coprCacheUsage"` + SQLUsage sqlUsageData `json:"SQLUsage"` + BuiltinFunctionsUsage map[string]uint32 `json:"builtinFunctionsUsage"` } type sqlType map[string]uint64 @@ -90,6 +91,60 @@ type tiFlashUsageData struct { ExchangePushDown uint64 `json:"exchangePushDown"` } +// builtinFunctionsUsageCollector collects builtin functions usage information and dump it into windowData. +type builtinFunctionsUsageCollector struct { + sync.Mutex + + // Should acquire lock to access this + usageData BuiltinFunctionsUsage +} + +// Merge BuiltinFunctionsUsage data +func (b *builtinFunctionsUsageCollector) Collect(usageData BuiltinFunctionsUsage) { + // TODO(leiysky): use multi-worker to collect the usage information so we can make this asynchronous + b.Lock() + defer b.Unlock() + b.usageData.Merge(usageData) +} + +// Dump BuiltinFunctionsUsage data +func (b *builtinFunctionsUsageCollector) Dump() map[string]uint32 { + b.Lock() + ret := b.usageData + b.usageData = make(map[string]uint32) + b.Unlock() + + return ret +} + +// BuiltinFunctionsUsage is a map from ScalarFuncSig_name(string) to usage count(uint32) +type BuiltinFunctionsUsage map[string]uint32 + +// Inc will increase the usage count of scalar function by 1 +func (b BuiltinFunctionsUsage) Inc(scalarFuncSigName string) { + v, ok := b[scalarFuncSigName] + if !ok { + b[scalarFuncSigName] = 1 + } else { + b[scalarFuncSigName] = v + 1 + } +} + +// Merge BuiltinFunctionsUsage data +func (b BuiltinFunctionsUsage) Merge(usageData BuiltinFunctionsUsage) { + for k, v := range usageData { + prev, ok := b[k] + if !ok { + b[k] = v + } else { + b[k] = prev + v + } + } +} + +// GlobalBuiltinFunctionsUsage is used to collect builtin functions usage information +var GlobalBuiltinFunctionsUsage = &builtinFunctionsUsageCollector{usageData: make(BuiltinFunctionsUsage)} + var ( rotatedSubWindows []*windowData subWindowsLock = sync.RWMutex{} @@ -186,6 +241,7 @@ func RotateSubWindow() { SQLTotal: 0, SQLType: make(sqlType), }, + BuiltinFunctionsUsage: GlobalBuiltinFunctionsUsage.Dump(), } err := readSQLMetric(time.Now(), &thisSubWindow.SQLUsage) @@ -243,6 +299,10 @@ func getWindowData() []*windowData { thisWindow.CoprCacheUsage.GTE100 += rotatedSubWindows[i].CoprCacheUsage.GTE100 thisWindow.SQLUsage.SQLTotal = rotatedSubWindows[i].SQLUsage.SQLTotal - startWindow.SQLUsage.SQLTotal thisWindow.SQLUsage.SQLType = calDeltaSQLTypeMap(rotatedSubWindows[i].SQLUsage.SQLType, startWindow.SQLUsage.SQLType) + + mergedBuiltinFunctionsUsage := BuiltinFunctionsUsage(thisWindow.BuiltinFunctionsUsage) + mergedBuiltinFunctionsUsage.Merge(BuiltinFunctionsUsage(rotatedSubWindows[i].BuiltinFunctionsUsage)) + thisWindow.BuiltinFunctionsUsage = mergedBuiltinFunctionsUsage aggregatedSubWindows++ i++ } diff --git a/util/mock/context.go b/util/mock/context.go index dd4019c28db41..bfaf2abbdcb84 100644 --- a/util/mock/context.go +++ b/util/mock/context.go @@ -158,6 +158,11 @@ func (c *Context) GetInfoSchema() sessionctx.InfoschemaMetaVersion { return nil } +// GetBuiltinFunctionUsage implements sessionctx.Context GetBuiltinFunctionUsage interface. +func (c *Context) GetBuiltinFunctionUsage() map[string]uint32 { + return make(map[string]uint32) +} + // GetGlobalSysVar implements GlobalVarAccessor GetGlobalSysVar interface. func (c *Context) GetGlobalSysVar(ctx sessionctx.Context, name string) (string, error) { v := variable.GetSysVar(name)