diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index ae54a5917..3ccb755a2 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -355,6 +355,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/config/crd/bases/config.nri_templatepolicies.yaml b/config/crd/bases/config.nri_templatepolicies.yaml index 36e7f3123..4b9d8cecb 100644 --- a/config/crd/bases/config.nri_templatepolicies.yaml +++ b/config/crd/bases/config.nri_templatepolicies.yaml @@ -84,6 +84,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/config/crd/bases/config.nri_topologyawarepolicies.yaml b/config/crd/bases/config.nri_topologyawarepolicies.yaml index 7abd698db..2f71f326a 100644 --- a/config/crd/bases/config.nri_topologyawarepolicies.yaml +++ b/config/crd/bases/config.nri_topologyawarepolicies.yaml @@ -98,6 +98,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index ae54a5917..3ccb755a2 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -355,6 +355,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/deployment/helm/template/crds/config.nri_templatepolicies.yaml b/deployment/helm/template/crds/config.nri_templatepolicies.yaml index 36e7f3123..4b9d8cecb 100644 --- a/deployment/helm/template/crds/config.nri_templatepolicies.yaml +++ b/deployment/helm/template/crds/config.nri_templatepolicies.yaml @@ -84,6 +84,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml index 7abd698db..2f71f326a 100644 --- a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml +++ b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml @@ -98,6 +98,12 @@ spec: classes: additionalProperties: properties: + disabledCstates: + description: DisabledCstates lists C-states disabled + for this class. + items: + type: string + type: array energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 20d1634c0..19f0ca5fa 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -187,7 +187,7 @@ Balloons policy parameters: request less. - `cpuClass` specifies the name of the CPU class according to which CPUs of balloons are configured. Class properties are defined in - separate `cpu.classes` objects, see below. + separate `control.cpu.classes` objects, see below. - `pinMemory` overrides policy-level `pinMemory` in balloons of this type. - `memoryTypes` is a list of allowed memory types for containers in @@ -310,6 +310,15 @@ Balloons policy parameters: important than avoiding balloon's own load. - `control.cpu.classes`: defines CPU classes and their properties. Class names are keys followed by properties: + - `disabledCstates` is a list of c-state names that are disabled + for CPUs in this class. Disabling deepest c-states lowers + latencies by preventing CPUs from entering deepest powersaving + states while processes are idle or wait for data. C-states + available in a system can be listed with `grep + . /sys/devices/system/cpu/cpu0/cpuidle/state*/name`. C-states + not listed in `disabledCstates` will be enabled. Disabling + non-existent c-states is silently ignored. Disabling c-states + does not affect min/max frequencies or vice versa. - `minFreq` minimum frequency for CPUs in this class (kHz). - `maxFreq` maximum frequency for CPUs in this class (kHz). - `uncoreMinFreq` minimum uncore frequency for CPUs in this @@ -390,6 +399,7 @@ spec: maxFreq: 3600000 uncoreMinFreq: 2000000 uncoreMaxFreq: 2400000 + disabledCstates: [C6, C8, C10] instrumentation: httpEndpoint: :8891 prometheusExport: true diff --git a/go.mod b/go.mod index c61713921..47ddef455 100644 --- a/go.mod +++ b/go.mod @@ -102,6 +102,7 @@ require ( replace ( github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology + github.com/intel/goresctrl => github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322 github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 ) diff --git a/go.sum b/go.sum index 82bc7d7b2..5f1c596d0 100644 --- a/go.sum +++ b/go.sum @@ -610,6 +610,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f h1:AKRIaPPDqBRhpWnvxhvtdbVtkV/3XrboabuFaLyp1kw= github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk= +github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322 h1:3PNNMiCNmZUlSglA/sj9yP9shj/R3Aa5b7bf02KzyBA= +github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322/go.mod h1:1S8GDqL46GuKb525bxNhIEEkhf4rhVcbSf9DuKhp7mw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -838,8 +840,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/intel/goresctrl v0.9.0 h1:IKI4ZrPTazLyFgdnWEkR9LS+DDATapOgoBtGxVMHePs= -github.com/intel/goresctrl v0.9.0/go.mod h1:1S8GDqL46GuKb525bxNhIEEkhf4rhVcbSf9DuKhp7mw= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/pkg/apis/config/v1alpha1/resmgr/control/cpu/config.go b/pkg/apis/config/v1alpha1/resmgr/control/cpu/config.go index 493e78fce..94b3c88c3 100644 --- a/pkg/apis/config/v1alpha1/resmgr/control/cpu/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/control/cpu/config.go @@ -32,4 +32,19 @@ type Class struct { UncoreMaxFreq uint `json:"uncoreMaxFreq,omitempty"` // CPUFreq Governor for this class. FreqGovernor string `json:"freqGovernor,omitempty"` + // DisabledCstates lists C-states disabled for this class. + DisabledCstates []string `json:"disabledCstates,omitempty"` +} + +// DeepCopy makes a deep copy of the Class. +func (c *Class) DeepCopy() *Class { + if c == nil { + return nil + } + cc := *c + if c.DisabledCstates != nil { + cc.DisabledCstates = make([]string, len(c.DisabledCstates)) + copy(cc.DisabledCstates, c.DisabledCstates) + } + return &cc } diff --git a/pkg/apis/config/v1alpha1/resmgr/control/cpu/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/control/cpu/zz_generated.deepcopy.go index 6bbbdaa84..d38afcb05 100644 --- a/pkg/apis/config/v1alpha1/resmgr/control/cpu/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/control/cpu/zz_generated.deepcopy.go @@ -27,7 +27,7 @@ func (in *Config) DeepCopyInto(out *Config) { in, out := &in.Classes, &out.Classes *out = make(map[string]Class, len(*in)) for key, val := range *in { - (*out)[key] = val + (*out)[key] = *val.DeepCopy() } } } diff --git a/pkg/resmgr/control/cpu/api.go b/pkg/resmgr/control/cpu/api.go index 610277c0e..01447353e 100644 --- a/pkg/resmgr/control/cpu/api.go +++ b/pkg/resmgr/control/cpu/api.go @@ -61,6 +61,9 @@ func Assign(c cache.Cache, class string, cpus ...int) error { if err := ctl.enforceCpufreq(class, cpus...); err != nil { log.Error("cpufreq enforcement failed: %v", err) } + if err := ctl.enforceCstates(class, cpus...); err != nil { + log.Error("cstate enforcement failed: %v", err) + } if err := ctl.enforceUncore(assignments, cpus...); err != nil { log.Error("uncore frequency enforcement failed: %v", err) } diff --git a/pkg/resmgr/control/cpu/cpu.go b/pkg/resmgr/control/cpu/cpu.go index 6631b20b7..57e8d3bcf 100644 --- a/pkg/resmgr/control/cpu/cpu.go +++ b/pkg/resmgr/control/cpu/cpu.go @@ -25,6 +25,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/control" "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/intel/goresctrl/pkg/cstates" "github.com/intel/goresctrl/pkg/utils" ) @@ -41,6 +42,7 @@ type cpuctl struct { cache cache.Cache // resource manager cache system sysfs.System // system topology classes map[string]Class // configured CPU classes + cstates *cstates.Cstates // C-states handler uncoreEnabled bool // whether we need to care about uncore started bool } @@ -153,6 +155,43 @@ func (ctl *cpuctl) enforceCpufreq(class string, cpus ...int) error { return nil } +// enforceStates enforces a class-specific C-state configuration to a cpuset +func (ctl *cpuctl) enforceCstates(class string, cpus ...int) error { + c, ok := ctl.classes[class] + if !ok { + return fmt.Errorf("non-existent cpu class %q", class) + } + if ctl.cstates == nil || len(cpus) == 0 { + return nil + } + enabledCstates := []string{} + for _, name := range ctl.cstates.Names() { + enabled := true + for _, dname := range c.DisabledCstates { + if name == dname { + enabled = false + break + } + } + if enabled { + enabledCstates = append(enabledCstates, name) + } + } + cpuCstates := ctl.cstates.Copy(cstates.FilterCPUs(cpus...)) + enCpuCstates := cpuCstates.Copy(cstates.FilterNames(enabledCstates...)) + disCpuCstates := cpuCstates.Copy(cstates.FilterNames(c.DisabledCstates...)) + enCpuCstates.SetAttrs(cstates.DISABLE, "0") + disCpuCstates.SetAttrs(cstates.DISABLE, "1") + log.Debug("enforcing cstates: enable: %v disable: %v from class %q on cpus %v", enabledCstates, c.DisabledCstates, class, cpus) + if err := enCpuCstates.Apply(); err != nil { + return fmt.Errorf("cannot enable cstates %v on cpus %v: %w", enabledCstates, cpus, err) + } + if err := disCpuCstates.Apply(); err != nil { + return fmt.Errorf("cannot disable cstates %v on cpus %v: %w", c.DisabledCstates, cpus, err) + } + return nil +} + // enforceUncore enforces uncore frequency limits func (ctl *cpuctl) enforceUncore(assignments cpuClassAssignments, affectedCPUs ...int) error { if !ctl.uncoreEnabled { @@ -251,6 +290,7 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error { log.Debug("applying cpu controller configuration:\n%s", utils.DumpJSON(ctl.classes)) // Sanity check + cstatesNeeded := map[string]struct{}{} uncoreAvailable := utils.UncoreFreqAvailable() for name, conf := range ctl.classes { if conf.UncoreMinFreq != 0 || conf.UncoreMaxFreq != 0 { @@ -260,6 +300,21 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error { ctl.uncoreEnabled = true break } + for _, cstate := range conf.DisabledCstates { + cstatesNeeded[cstate] = struct{}{} + } + } + if len(cstatesNeeded) != 0 { + var err error + filter := cstates.FilterAttrs(cstates.DISABLE) + if cstatesEnvOverridesJson != "" { + ctl.cstates, err = NewCstatesFromOverride(filter) + } else { + ctl.cstates, err = cstates.NewCstatesFromSysfs(filter) + } + if err != nil { + return fmt.Errorf("failed to read C-states: %w", err) + } } // Configure the system @@ -269,6 +324,9 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error { if err := ctl.enforceCpufreq(class, cpus.SortedMembers()...); err != nil { log.Error("cpufreq enforcement on re-configure failed: %v", err) } + if err := ctl.enforceCstates(class, cpus.SortedMembers()...); err != nil { + log.Error("cpufreq enforcement on re-configure failed: %v", err) + } } else { // TODO: what should we really do with classes that do not exist in // the configuration anymore? Now we remember the CPUs assigned to diff --git a/pkg/resmgr/control/cpu/cstates_overridefs.go b/pkg/resmgr/control/cpu/cstates_overridefs.go new file mode 100644 index 000000000..1ad5cb06c --- /dev/null +++ b/pkg/resmgr/control/cpu/cstates_overridefs.go @@ -0,0 +1,162 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpu + +import ( + "encoding/json" + "fmt" + "maps" + "os" + "slices" + "strconv" + "strings" + + "github.com/intel/goresctrl/pkg/cstates" + "github.com/intel/goresctrl/pkg/utils" +) + +var ( + cstatesEnvOverridesVar = "OVERRIDE_SYS_CSTATES" + cstatesEnvOverridesJson = os.Getenv(cstatesEnvOverridesVar) +) + +type cstatesOverrides []cstatesOverride +type cstatesOverride struct { + Cpus string `json:"cpus"` // CPU ids in list format + Names []string `json:"names"` // list of C-state names, lexical order defines state number + Files map[string]string `json:"files"` // map of attribute name to value for all above CPUs and C-states +} + +type overrideFs struct { + overrides cstatesOverrides + stateName map[int]string + nameState map[string]int + cpuStateFile map[utils.ID]map[int]map[string]string // cpu -> state -> attr -> value +} + +func NewCstatesFromOverride(filters ...cstates.CstatesFilter) (*cstates.Cstates, error) { + cs := cstates.NewCstates() + ofs, err := NewOverrideFs() + if err != nil { + return nil, fmt.Errorf("failed to create override fs from %s: %v", cstatesEnvOverridesVar, err) + } + cs.SetFs(ofs) + err = cs.Read(filters...) + if err != nil { + return nil, fmt.Errorf("failed to refresh cstates from %s overrides: %v", cstatesEnvOverridesVar, err) + } + return cs, nil +} + +func NewOverrideFs() (*overrideFs, error) { + ofs := &overrideFs{ + stateName: make(map[int]string), + nameState: make(map[string]int), + cpuStateFile: make(map[utils.ID]map[int]map[string]string), + } + if err := json.Unmarshal([]byte(cstatesEnvOverridesJson), &ofs.overrides); err != nil { + return nil, err + } + if len(ofs.overrides) == 0 { + return nil, fmt.Errorf("no overrides found in %s", cstatesEnvOverridesVar) + } + // Collect unique C-state names from all overrides and assign state numbers + names := make(map[string]struct{}) + for _, o := range ofs.overrides { + for _, name := range o.Names { + names[name] = struct{}{} + } + } + orderedNames := make([]string, 0, len(names)) + for name := range names { + orderedNames = append(orderedNames, name) + } + slices.Sort(orderedNames) + for state, name := range orderedNames { + ofs.stateName[state] = name + ofs.nameState[name] = state + } + + // Build cpuStateFile map for reading and writing attribute values by cpu and state + for _, o := range ofs.overrides { + cpus, err := utils.NewIDSetFromString(o.Cpus) + if err != nil { + return nil, fmt.Errorf("invalid CPU list %q in %s: %v", o.Cpus, cstatesEnvOverridesVar, err) + } + for cpu := range cpus { + cpuid := utils.ID(cpu) + if _, ok := ofs.cpuStateFile[cpuid]; !ok { + ofs.cpuStateFile[cpuid] = make(map[int]map[string]string) + } + for _, name := range o.Names { + state := ofs.nameState[name] + if _, ok := ofs.cpuStateFile[cpuid][state]; !ok { + ofs.cpuStateFile[cpuid][state] = make(map[string]string) + } + maps.Copy(ofs.cpuStateFile[cpuid][state], o.Files) + ofs.cpuStateFile[cpuid][state]["name"] = name // always have name attribute + } + } + } + log.Debugf("cstates override fs: loaded overrides for %d CPUs C-states: %s", len(ofs.cpuStateFile), strings.Join(orderedNames, ", ")) + return ofs, nil +} + +func (fs *overrideFs) PossibleCpus() (string, error) { + maxCpu := utils.ID(-1) + for cpu := range fs.cpuStateFile { + if cpu > maxCpu { + maxCpu = cpu + } + } + if maxCpu < 0 { + return "", nil + } + return "0-" + strconv.Itoa(maxCpu), nil +} + +func (fs *overrideFs) CpuidleStates(cpuID utils.ID) ([]int, error) { + states := []int{} + for state := range fs.stateName { + states = append(states, state) + } + slices.Sort(states) + return states, nil +} + +func (fs *overrideFs) CpuidleStateAttrRead(cpu utils.ID, state int, attribute string) (string, error) { + if stateFiles, ok := fs.cpuStateFile[cpu]; ok { + if files, ok := stateFiles[state]; ok { + if val, ok := files[attribute]; ok { + log.Debugf("cstates override fs: read cpu%d cstate=%s %s=%q", cpu, fs.stateName[state], attribute, val) + return val, nil + } + } + } + log.Errorf("cstates override fs: cannot read cpu%d cstate=%s attribute %q", cpu, fs.stateName[state], attribute) + return "", os.ErrNotExist +} + +func (fs *overrideFs) CpuidleStateAttrWrite(cpu utils.ID, state int, attribute string, value string) error { + if stateFiles, ok := fs.cpuStateFile[cpu]; ok { + if files, ok := stateFiles[state]; ok { + files[attribute] = value + log.Debugf("cstates override fs: wrote cpu%d cstate=%s %s=%q", cpu, fs.stateName[state], attribute, value) + return nil + } + } + log.Errorf("cstates override fs: write to non-existing cpu%d cstate=%d %s=%q ignored", cpu, state, attribute, value) + return nil +} diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg new file mode 100644 index 000000000..fe7f80b55 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg @@ -0,0 +1,30 @@ +config: + agent: + nodeResourceTopology: true + allocatorTopologyBalancing: false + availableResources: + cpu: cpuset:2-7,10-13 # policy must not touch C-states of unavailable CPUs + reservedResources: + cpu: 750m + + pinCPU: true + + idleCPUClass: default-class + + balloonTypes: + - name: lowlatency-bln + cpuClass: lowlatency-class + + control: + cpu: + classes: + lowlatency-class: + disabledCstates: [C4, C6, C8, C10] + default-class: + disabledCstates: [] + log: + debug: + - policy + - cpu +extraEnv: + OVERRIDE_SYS_CSTATES: '''[{"cpus": "0-15", "names": ["C1E", "C2", "C4", "C8"], "files": {"disable": "0"}}]''' diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh new file mode 100644 index 000000000..a5c250696 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh @@ -0,0 +1,100 @@ +# Test balloons with certain CPU c-states disabled + +helm-terminate +helm_config=$TEST_DIR/balloons-cstates.cfg helm-launch balloons + +# cpuids-of-container returns CPU ids a container is allowed to use, e.g. "1 2 4" +cpuids-of() { + local ctr=$1 # e.g. pod0c0 + # return only cpu ids without zero-fill: replace cpu01 -> 1, cpu11 -> 11 + pyexec "for cpu in cpus['$ctr']: print(cpu.replace('cpu0','').replace('cpu',''))" +} + +# verify-cstates checks the last writes to "disable" files in the +# override fs. +verify-cstates() { + local cpu_ids=$1 # e.g. "1 2 4" + local enabled_cstates=$2 # e.g. "C1E C2" + local disabled_cstates=$3 # e.g. "C6 C8" + local last_n_writes=$4 # expect the write within last N writes, e.g. 6 + + vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | nl | grep 'cstates override fs: wrote' | tail -n $last_n_writes | nl" + for cpu_id in $cpu_ids; do + for cstate in $enabled_cstates; do + echo "verify last write to cpu$cpu_id cstate=$cstate disable is 0" + grep "cpu${cpu_id} cstate=$cstate disable=" <<< $COMMAND_OUTPUT | tail -n 1 | grep -q 'disable="0"' || { + command-error "expected write 0 not found" + } + done + for cstate in $disabled_cstates; do + echo "verify last write to cpu$cpu_id cstate=$cstate disable is 1" + grep "cpu${cpu_id} cstate=$cstate disable=" <<< $COMMAND_OUTPUT | tail -n 1 | grep -q 'disable="1"' || { + command-error "expected write 1 not found" + } + done + done +} + +# verify-cstates-no-writes checks that any c-states of given CPUs have not been written +verify-cstates-no-writes() { + local cpu_ids=$1 # e.g. "1 2 4" + local last_n_writes=$2 # e.g. 100 + echo "verify no writes to c-states of CPUs $cpu_ids" + cpu_ids="(${cpu_ids// /|})" + vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | nl | grep -E 'cstates override fs: wrote cpu${cpu_ids} cstate='" + grep -q wrote <<< $COMMAND_OUTPUT && { + command-error "writes to forbidden CPUs found" + } +} + +cleanup() { + vm-command "kubectl delete pods --all --now" +} + +echo "verify that all c-states of all available CPUs are enabled" +verify-cstates "2 3 4 5 6 7 11 12 13" "C1E C2 C4 C8" "" 36 + +echo "verify that c-states of CPUs outside AvailableResources have not been written" +verify-cstates-no-writes "0 1 8 9 14 15" + +CPUREQ="750m" MEMREQ="100M" CPULIM="750m" MEMLIM="" +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOUNT=1 create balloons-busybox +report allowed +verify 'len(cpus["pod0c0"]) == 1' +echo "verify that CPUs of low-latency pod0 cannot enter C4 or C8" +verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 4 + +CPUREQ="3" MEMREQ="100M" CPULIM="" MEMLIM="" +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOUNT=1 create balloons-busybox +report allowed +verify 'cpus["pod0c0"] == cpus["pod1c0"]' \ + 'len(cpus["pod0c0"]) == 4' +echo "verify that CPUs of low-latency pods pod0 and pod1 cannot enter C4 or C8" +verify-cstates "$(cpuids-of pod1c0)" "C1E C2" "C4 C8" 16 + +# store CPU ids of maximal cpuset before deleting pods +max_lowlatency_cpus="$(echo $(cpuids-of pod1c0) )" + +vm-command 'kubectl delete pod pod1' +report allowed +verify 'len(cpus["pod0c0"]) == 1' + +# spaces around each id helps ensuring grep " 1 " never matches cpu 11 but always matches cpu 1 +pod0cpus=" $(echo $(cpuids-of pod0c0) ) " + +echo "verify that c-states of freed CPUs are enabled again after balloon was deflated" +freed_cpus="" +for cpu_id in $max_lowlatency_cpus; do + grep -q " $cpu_id " <<< $pod0cpus || freed_cpus+=" $cpu_id" +done +echo "verify that all c-states of freed CPUs $freed_cpus (= {$max_lowlatency_cpus} - {$pod0cpus}) are enabled after the balloon got deflated" +verify-cstates "$freed_cpus" "C1E C2 C4 C8" "" 24 + +echo "verify that c-states of the remaining CPU $(cpuids-of pod0c0) are still configured for low-latency" +verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 16 + +vm-command 'kubectl delete pod pod0' +report allowed + +echo "verify that after all containers are gone and their destroyed, c-states of all used CPUs are enabled again" +verify-cstates "$max_lowlatency_cpus" "C1E C2 C4 C8" "" 32