Balloons: enabling disabling c-states in latency-critical balloons

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -355,6 +355,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

config/crd/bases/config.nri_templatepolicies.yaml

@@ -84,6 +84,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

config/crd/bases/config.nri_topologyawarepolicies.yaml

@@ -98,6 +98,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -355,6 +355,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

deployment/helm/template/crds/config.nri_templatepolicies.yaml

@@ -84,6 +84,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml

@@ -98,6 +98,12 @@ spec:
                       classes:
                         additionalProperties:
                           properties:
+                            disabledCstates:
+                              description: DisabledCstates lists C-states disabled
+                                for this class.
+                              items:
+                                type: string
+                              type: array
                             energyPerformancePreference:
                               description: EnergyPerformancePreference for CPUs in
                                 this class.

docs/resource-policy/policy/balloons.md

@@ -187,7 +187,7 @@ Balloons policy parameters:
     request less.
   - `cpuClass` specifies the name of the CPU class according to which
     CPUs of balloons are configured. Class properties are defined in
-    separate `cpu.classes` objects, see below.
+    separate `control.cpu.classes` objects, see below.
   - `pinMemory` overrides policy-level `pinMemory` in balloons of this
     type.
   - `memoryTypes` is a list of allowed memory types for containers in
@@ -310,6 +310,15 @@ Balloons policy parameters:
     important than avoiding balloon's own load.
 - `control.cpu.classes`: defines CPU classes and their
     properties. Class names are keys followed by properties:
+    - `disabledCstates` is a list of c-state names that are disabled
+      for CPUs in this class. Disabling deepest c-states lowers
+      latencies by preventing CPUs from entering deepest powersaving
+      states while processes are idle or wait for data. C-states
+      available in a system can be listed with `grep
+      . /sys/devices/system/cpu/cpu0/cpuidle/state*/name`. C-states
+      not listed in `disabledCstates` will be enabled. Disabling
+      non-existent c-states is silently ignored. Disabling c-states
+      does not affect min/max frequencies or vice versa.
     - `minFreq` minimum frequency for CPUs in this class (kHz).
     - `maxFreq` maximum frequency for CPUs in this class (kHz).
     - `uncoreMinFreq` minimum uncore frequency for CPUs in this
@@ -390,6 +399,7 @@ spec:
           maxFreq: 3600000
           uncoreMinFreq: 2000000
           uncoreMaxFreq: 2400000
+          disabledCstates: [C6, C8, C10]
   instrumentation:
     httpEndpoint: :8891
     prometheusExport: true

go.mod

@@ -102,6 +102,7 @@ require (
 
 replace (
 	github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology
+	github.com/intel/goresctrl => github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322
 	github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809
 )
 

go.sum

@@ -610,6 +610,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f h1:AKRIaPPDqBRhpWnvxhvtdbVtkV/3XrboabuFaLyp1kw=
 github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk=
+github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322 h1:3PNNMiCNmZUlSglA/sj9yP9shj/R3Aa5b7bf02KzyBA=
+github.com/askervin/goresctrl v0.0.0-20250917152138-0b2fbc956322/go.mod h1:1S8GDqL46GuKb525bxNhIEEkhf4rhVcbSf9DuKhp7mw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -838,8 +840,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/intel/goresctrl v0.9.0 h1:IKI4ZrPTazLyFgdnWEkR9LS+DDATapOgoBtGxVMHePs=
-github.com/intel/goresctrl v0.9.0/go.mod h1:1S8GDqL46GuKb525bxNhIEEkhf4rhVcbSf9DuKhp7mw=
 github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4=
 github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=

pkg/apis/config/v1alpha1/resmgr/control/cpu/config.go

@@ -32,4 +32,19 @@ type Class struct {
 	UncoreMaxFreq uint `json:"uncoreMaxFreq,omitempty"`
 	// CPUFreq Governor for this class.
 	FreqGovernor string `json:"freqGovernor,omitempty"`
+	// DisabledCstates lists C-states disabled for this class.
+	DisabledCstates []string `json:"disabledCstates,omitempty"`
+}
+
+// DeepCopy makes a deep copy of the Class.
+func (c *Class) DeepCopy() *Class {
+	if c == nil {
+		return nil
+	}
+	cc := *c
+	if c.DisabledCstates != nil {
+		cc.DisabledCstates = make([]string, len(c.DisabledCstates))
+		copy(cc.DisabledCstates, c.DisabledCstates)
+	}
+	return &cc
 }

pkg/resmgr/control/cpu/api.go

@@ -61,6 +61,9 @@ func Assign(c cache.Cache, class string, cpus ...int) error {
 		if err := ctl.enforceCpufreq(class, cpus...); err != nil {
 			log.Error("cpufreq enforcement failed: %v", err)
 		}
+		if err := ctl.enforceCstates(class, cpus...); err != nil {
+			log.Error("cstate enforcement failed: %v", err)
+		}
 		if err := ctl.enforceUncore(assignments, cpus...); err != nil {
 			log.Error("uncore frequency enforcement failed: %v", err)
 		}

pkg/resmgr/control/cpu/cpu.go

@@ -25,6 +25,7 @@ import (
 	"github.com/containers/nri-plugins/pkg/resmgr/cache"
 	"github.com/containers/nri-plugins/pkg/resmgr/control"
 	"github.com/containers/nri-plugins/pkg/sysfs"
+	"github.com/intel/goresctrl/pkg/cstates"
 	"github.com/intel/goresctrl/pkg/utils"
 )
 
@@ -41,6 +42,7 @@ type cpuctl struct {
 	cache         cache.Cache      // resource manager cache
 	system        sysfs.System     // system topology
 	classes       map[string]Class // configured CPU classes
+	cstates       *cstates.Cstates // C-states handler
 	uncoreEnabled bool             // whether we need to care about uncore
 	started       bool
 }
@@ -153,6 +155,43 @@ func (ctl *cpuctl) enforceCpufreq(class string, cpus ...int) error {
 	return nil
 }
 
+// enforceStates enforces a class-specific C-state configuration to a cpuset
+func (ctl *cpuctl) enforceCstates(class string, cpus ...int) error {
+	c, ok := ctl.classes[class]
+	if !ok {
+		return fmt.Errorf("non-existent cpu class %q", class)
+	}
+	if ctl.cstates == nil || len(cpus) == 0 {
+		return nil
+	}
+	enabledCstates := []string{}
+	for _, name := range ctl.cstates.Names() {
+		enabled := true
+		for _, dname := range c.DisabledCstates {
+			if name == dname {
+				enabled = false
+				break
+			}
+		}
+		if enabled {
+			enabledCstates = append(enabledCstates, name)
+		}
+	}
+	cpuCstates := ctl.cstates.Copy(cstates.FilterCPUs(cpus...))
+	enCpuCstates := cpuCstates.Copy(cstates.FilterNames(enabledCstates...))
+	disCpuCstates := cpuCstates.Copy(cstates.FilterNames(c.DisabledCstates...))
+	enCpuCstates.SetAttrs(cstates.DISABLE, "0")
+	disCpuCstates.SetAttrs(cstates.DISABLE, "1")
+	log.Debug("enforcing cstates: enable: %v disable: %v from class %q on cpus %v", enabledCstates, c.DisabledCstates, class, cpus)
+	if err := enCpuCstates.Apply(); err != nil {
+		return fmt.Errorf("cannot enable cstates %v on cpus %v: %w", enabledCstates, cpus, err)
+	}
+	if err := disCpuCstates.Apply(); err != nil {
+		return fmt.Errorf("cannot disable cstates %v on cpus %v: %w", c.DisabledCstates, cpus, err)
+	}
+	return nil
+}
+
 // enforceUncore enforces uncore frequency limits
 func (ctl *cpuctl) enforceUncore(assignments cpuClassAssignments, affectedCPUs ...int) error {
 	if !ctl.uncoreEnabled {
@@ -251,6 +290,7 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
 	log.Debug("applying cpu controller configuration:\n%s", utils.DumpJSON(ctl.classes))
 
 	// Sanity check
+	cstatesNeeded := map[string]struct{}{}
 	uncoreAvailable := utils.UncoreFreqAvailable()
 	for name, conf := range ctl.classes {
 		if conf.UncoreMinFreq != 0 || conf.UncoreMaxFreq != 0 {
@@ -260,6 +300,21 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
 			ctl.uncoreEnabled = true
 			break
 		}
+		for _, cstate := range conf.DisabledCstates {
+			cstatesNeeded[cstate] = struct{}{}
+		}
+	}
+	if len(cstatesNeeded) != 0 {
+		var err error
+		filter := cstates.FilterAttrs(cstates.DISABLE)
+		if cstatesEnvOverridesJson != "" {
+			ctl.cstates, err = NewCstatesFromOverride(filter)
+		} else {
+			ctl.cstates, err = cstates.NewCstatesFromSysfs(filter)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to read C-states: %w", err)
+		}
 	}
 
 	// Configure the system
@@ -269,6 +324,9 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
 			if err := ctl.enforceCpufreq(class, cpus.SortedMembers()...); err != nil {
 				log.Error("cpufreq enforcement on re-configure failed: %v", err)
 			}
+			if err := ctl.enforceCstates(class, cpus.SortedMembers()...); err != nil {
+				log.Error("cpufreq enforcement on re-configure failed: %v", err)
+			}
 		} else {
 			// TODO: what should we really do with classes that do not exist in
 			// the configuration anymore? Now we remember the CPUs assigned to