Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: allow custom kwok instance-types at runtime #1847

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions kwok/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,8 @@ After doing this, you can create a deployment to test node scaling with kwok pro
## Specifying Instance Types

By default, the KWOK provider will create a hypothetical set of instance types that it uses for node provisioning. You
can specify a custom set of instance types by providing a JSON file with the list of supported instance options. This
set of instance types is embedded into the binary on creation; if you want to change the instance types that
Karpenter+KWOK support, you will need to adjust the embedded data and recompile.
can specify a custom set of instance types by providing a JSON file with the list of supported instance options. To do so,
set the `--instance-types-file-path` flag or `INSTANCE_TYPES_FILE_PATH` environment variable to your custom file's path.

There is an example instance types file in [examples/instance\_types.json](examples/instance_types.json) that you can
regenerate with `make gen_instance_types`.
Expand Down
16 changes: 14 additions & 2 deletions kwok/cloudprovider/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,19 @@ limitations under the License.
package kwok

import (
"context"
_ "embed"
"encoding/json"
"fmt"
"os"
"regexp"

"github.com/samber/lo"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"

"sigs.k8s.io/karpenter/kwok/apis/v1alpha1"
"sigs.k8s.io/karpenter/kwok/options"
v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
"sigs.k8s.io/karpenter/pkg/cloudprovider"
"sigs.k8s.io/karpenter/pkg/scheduling"
Expand Down Expand Up @@ -61,13 +64,22 @@ type InstanceTypeOptions struct {
}

//go:embed instance_types.json
var rawInstanceTypes []byte
var defaultRawInstanceTypes []byte

// ConstructInstanceTypes create many instance types based on the embedded instance type data
func ConstructInstanceTypes() ([]*cloudprovider.InstanceType, error) {
func ConstructInstanceTypes(ctx context.Context) ([]*cloudprovider.InstanceType, error) {
var instanceTypes []*cloudprovider.InstanceType
var instanceTypeOptions []InstanceTypeOptions

rawInstanceTypes := defaultRawInstanceTypes
if customInstanceTypes := options.FromContext(ctx).InstanceTypesFilePath; customInstanceTypes != "" {
customRawInstanceTypes, err := os.ReadFile(customInstanceTypes)
if err != nil {
return nil, fmt.Errorf("could not read custom instance types file: %w", err)
}
rawInstanceTypes = customRawInstanceTypes
}

if err := json.Unmarshal(rawInstanceTypes, &instanceTypeOptions); err != nil {
return nil, fmt.Errorf("could not parse JSON data: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion kwok/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

func main() {
ctx, op := operator.NewOperator()
instanceTypes, err := kwok.ConstructInstanceTypes()
instanceTypes, err := kwok.ConstructInstanceTypes(ctx)
if err != nil {
log.FromContext(ctx).Error(err, "failed constructing instance types")
}
Expand Down
69 changes: 69 additions & 0 deletions kwok/options/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
"context"
"errors"
"flag"
"fmt"
"os"

"sigs.k8s.io/karpenter/pkg/operator/options"
"sigs.k8s.io/karpenter/pkg/utils/env"
)

func init() {
options.Injectables = append(options.Injectables, &Options{})
}

type optionsKey struct{}

// Options contains all CLI flags / env vars for the KWOK cloudprovider.
type Options struct {
InstanceTypesFilePath string
}

func (o *Options) AddFlags(fs *options.FlagSet) {
fs.StringVar(&o.InstanceTypesFilePath, "instance-types-file-path", env.WithDefaultString("INSTANCE_TYPES_FILE_PATH", ""), "Path to a custom instance-types file")
}

func (o *Options) Parse(fs *options.FlagSet, args ...string) error {
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
os.Exit(0)
}
return fmt.Errorf("parsing flags, %w", err)
}
return nil
}

func (o *Options) ToContext(ctx context.Context) context.Context {
return ToContext(ctx, o)
}

func ToContext(ctx context.Context, opts *Options) context.Context {
return context.WithValue(ctx, optionsKey{}, opts)
}

func FromContext(ctx context.Context) *Options {
retval := ctx.Value(optionsKey{})
if retval == nil {
return nil
}
return retval.(*Options)
}