feat(state): allow storing of state in tmpfs instead of disk

cmd/katalyst-agent/app/options/qrm/qrm_base.go

@@ -19,28 +19,29 @@ package qrm
 import (
 	cliflag "k8s.io/component-base/cli/flag"
 
+	"github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/options/qrm/statedirectory"
 	qrmconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm"
 )
 
 type GenericQRMPluginOptions struct {
 	QRMPluginSocketDirs         []string
-	StateFileDirectory          string
 	ExtraStateFileAbsPath       string
 	PodDebugAnnoKeys            []string
 	UseKubeletReservedConfig    bool
 	PodAnnotationKeptKeys       []string
 	PodLabelKeptKeys            []string
 	EnableReclaimNUMABinding    bool
 	EnableSNBHighNumaPreference bool
+	*statedirectory.StateDirectoryOptions
 }
 
 func NewGenericQRMPluginOptions() *GenericQRMPluginOptions {
 	return &GenericQRMPluginOptions{
 		QRMPluginSocketDirs:   []string{"/var/lib/kubelet/plugins_registry"},
-		StateFileDirectory:    "/var/lib/katalyst/qrm_advisor",
 		PodDebugAnnoKeys:      []string{},
 		PodAnnotationKeptKeys: []string{},
 		PodLabelKeptKeys:      []string{},
+		StateDirectoryOptions: statedirectory.NewStateDirectoryOptions(),
 	}
 }
 
@@ -49,7 +50,6 @@ func (o *GenericQRMPluginOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 
 	fs.StringSliceVar(&o.QRMPluginSocketDirs, "qrm-socket-dirs",
 		o.QRMPluginSocketDirs, "socket file directories that qrm plugins communicate witch other components")
-	fs.StringVar(&o.StateFileDirectory, "qrm-state-dir", o.StateFileDirectory, "Directory that qrm plugins are using")
 	fs.StringVar(&o.ExtraStateFileAbsPath, "qrm-extra-state-file", o.ExtraStateFileAbsPath, "The absolute path to an extra state file to specify cpuset.mems for specific pods")
 	fs.StringSliceVar(&o.PodDebugAnnoKeys, "qrm-pod-debug-anno-keys",
 		o.PodDebugAnnoKeys, "pod annotations keys to identify the pod is a debug pod, and qrm plugins will apply specific strategy to it")
@@ -63,11 +63,11 @@ func (o *GenericQRMPluginOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.EnableReclaimNUMABinding, "if set true, reclaim pod will be allocated on a specific NUMA node best-effort, otherwise, reclaim pod will be allocated on multi NUMA nodes")
 	fs.BoolVar(&o.EnableSNBHighNumaPreference, "enable-snb-high-numa-preference",
 		o.EnableSNBHighNumaPreference, "default false,if set true, snb pod will be preferentially allocated on high numa node")
+	o.StateDirectoryOptions.AddFlags(fss)
 }
 
 func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfiguration) error {
 	conf.QRMPluginSocketDirs = o.QRMPluginSocketDirs
-	conf.StateFileDirectory = o.StateFileDirectory
 	conf.ExtraStateFileAbsPath = o.ExtraStateFileAbsPath
 	conf.PodDebugAnnoKeys = o.PodDebugAnnoKeys
 	conf.UseKubeletReservedConfig = o.UseKubeletReservedConfig
@@ -76,6 +76,10 @@ func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfig
 	conf.EnableReclaimNUMABinding = o.EnableReclaimNUMABinding
 	conf.EnableSNBHighNumaPreference = o.EnableSNBHighNumaPreference
 
+	if err := o.StateDirectoryOptions.ApplyTo(conf.StateDirectoryConfiguration); err != nil {
+		return err
+	}
+
 	return nil
 }
 

cmd/katalyst-agent/app/options/qrm/statedirectory/statedirectory_base.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statedirectory
+
+import (
+	cliflag "k8s.io/component-base/cli/flag"
+
+	"github.com/kubewharf/katalyst-core/pkg/config/agent/qrm/statedirectory"
+)
+
+type StateDirectoryOptions struct {
+	StateFileDirectory         string
+	InMemoryStateFileDirectory string
+	EnableInMemoryState        bool
+	HasPreStop                 bool
+}
+
+func NewStateDirectoryOptions() *StateDirectoryOptions {
+	return &StateDirectoryOptions{
+		StateFileDirectory:         "/var/lib/katalyst/qrm_advisor",
+		InMemoryStateFileDirectory: "/dev/shm/qrm/state",
+	}
+}
+
+func (o *StateDirectoryOptions) AddFlags(fss *cliflag.NamedFlagSets) {
+	fs := fss.FlagSet("qrm_state_directory")
+
+	fs.StringVar(&o.StateFileDirectory, "qrm-state-dir", o.StateFileDirectory, "The directory to store the state file.")
+	fs.StringVar(&o.InMemoryStateFileDirectory, "qrm-state-dir-in-memory",
+		o.InMemoryStateFileDirectory, "The in memory directory to store the state file.")
+	fs.BoolVar(&o.EnableInMemoryState, "qrm-enable-in-memory-state",
+		o.EnableInMemoryState, "if set true, the state will be stored in the in-memory directory.")
+	fs.BoolVar(&o.HasPreStop, "qrm-has-pre-stop",
+		o.HasPreStop, "if set true, there is a pre-stop script in place.")
+}
+
+func (o *StateDirectoryOptions) ApplyTo(conf *statedirectory.StateDirectoryConfiguration) error {
+	conf.StateFileDirectory = o.StateFileDirectory
+	conf.InMemoryStateFileDirectory = o.InMemoryStateFileDirectory
+	conf.EnableInMemoryState = o.EnableInMemoryState
+	conf.HasPreStop = o.HasPreStop
+	return nil
+}

pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpueviction/cpu_eviciton_test.go

@@ -27,6 +27,7 @@ import (
 
 	qrmstate "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state"
 	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/config/agent/qrm/statedirectory"
 	"github.com/kubewharf/katalyst-core/pkg/metaserver"
 	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent"
 	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric"
@@ -50,10 +51,13 @@ func makeMetaServer(metricsFetcher types.MetricsFetcher, cpuTopology *machine.CP
 
 func makeState(topo *machine.CPUTopology) (qrmstate.State, error) {
 	tmpDir, err := os.MkdirTemp("", "checkpoint-makeState")
+	stateDirectoryConfig := &statedirectory.StateDirectoryConfiguration{
+		StateFileDirectory: tmpDir,
+	}
 	if err != nil {
 		return nil, fmt.Errorf("make tmp dir for checkpoint failed with error: %v", err)
 	}
-	return qrmstate.NewCheckpointState(tmpDir, "test", "test", topo, false, qrmstate.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
+	return qrmstate.NewCheckpointState(stateDirectoryConfig, "test", "test", topo, false, qrmstate.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
 }
 
 func TestNewCPUPressureEviction(t *testing.T) {

pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpueviction/strategy/pressure_load_test.go

@@ -41,6 +41,7 @@ import (
 	qrmstate "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/util"
 	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/config/agent/qrm/statedirectory"
 	"github.com/kubewharf/katalyst-core/pkg/consts"
 	"github.com/kubewharf/katalyst-core/pkg/metaserver"
 	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent"
@@ -62,7 +63,9 @@ const (
 	defaultReservedForSystem                        = 0
 )
 
-func makeMetaServer(metricsFetcher metrictypes.MetricsFetcher, cpuTopology *machine.CPUTopology) *metaserver.MetaServer {
+func makeMetaServer(
+	metricsFetcher metrictypes.MetricsFetcher, cpuTopology *machine.CPUTopology,
+) *metaserver.MetaServer {
 	metaServer := &metaserver.MetaServer{
 		MetaAgent: &agent.MetaAgent{},
 	}
@@ -75,7 +78,8 @@ func makeMetaServer(metricsFetcher metrictypes.MetricsFetcher, cpuTopology *mach
 	return metaServer
 }
 
-func makeConf(metricRingSize int, gracePeriod int64, loadUpperBoundRatio, loadLowerBoundRatio,
+func makeConf(
+	metricRingSize int, gracePeriod int64, loadUpperBoundRatio, loadLowerBoundRatio,
 	loadThresholdMetPercentage float64, reservedForReclaim, reservedForAllocate string, reservedForSystem int,
 ) *config.Configuration {
 	conf := config.NewConfiguration()
@@ -103,10 +107,13 @@ func makeConf(metricRingSize int, gracePeriod int64, loadUpperBoundRatio, loadLo
 
 func makeState(topo *machine.CPUTopology) (qrmstate.State, error) {
 	tmpDir, err := os.MkdirTemp("", "checkpoint-makeState")
+	stateDirectoryConfig := &statedirectory.StateDirectoryConfiguration{
+		StateFileDirectory: tmpDir,
+	}
 	if err != nil {
 		return nil, fmt.Errorf("make tmp dir for checkpoint failed with error: %v", err)
 	}
-	return qrmstate.NewCheckpointState(tmpDir, "test", "test", topo, false, qrmstate.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
+	return qrmstate.NewCheckpointState(stateDirectoryConfig, "test", "test", topo, false, qrmstate.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
 }
 
 func TestNewCPUPressureLoadEviction(t *testing.T) {

pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpueviction/strategy/pressure_numa_test.go

@@ -25,6 +25,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kubewharf/katalyst-core/pkg/config/agent/qrm/statedirectory"
+
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -96,7 +98,10 @@ func TestNumaCPUPressureEviction_update(t *testing.T) {
 	defer os.RemoveAll(testingDir)
 
 	cpuTopology, _ := machine.GenerateDummyCPUTopology(16, 2, 4)
-	state1, _ := state.NewCheckpointState(testingDir, "test", "test", cpuTopology, false, state.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
+	stateDirectoryConfig := &statedirectory.StateDirectoryConfiguration{
+		StateFileDirectory: testingDir,
+	}
+	state1, _ := state.NewCheckpointState(stateDirectoryConfig, "test", "test", cpuTopology, false, state.GenerateMachineStateFromPodEntries, metrics.DummyMetrics{})
 
 	podEntry := state.PodEntries{
 		"pod1": state.ContainerEntries{