Skip to content

Commit 504eff2

Browse files
ArielLahianyArielLahiany
committed
Kubernetes Driver: switch to 'StatefulSet' from 'Deployment'.
Signed-off-by: ArielLahiany <[email protected]>
1 parent b4a0dee commit 504eff2

File tree

4 files changed

+103
-96
lines changed

4 files changed

+103
-96
lines changed

Diff for: ‎driver/kubernetes/driver.go

+23-23
Original file line numberDiff line numberDiff line change
@@ -43,16 +43,16 @@ type Driver struct {
4343

4444
// if you add fields, remember to update docs:
4545
// https://github.com/docker/docs/blob/main/content/build/drivers/kubernetes.md
46-
minReplicas int
47-
deployment *appsv1.Deployment
48-
configMaps []*corev1.ConfigMap
49-
clientset *kubernetes.Clientset
50-
deploymentClient clientappsv1.DeploymentInterface
51-
podClient clientcorev1.PodInterface
52-
configMapClient clientcorev1.ConfigMapInterface
53-
podChooser podchooser.PodChooser
54-
defaultLoad bool
55-
timeout time.Duration
46+
minReplicas int
47+
statefulSet *appsv1.StatefulSet
48+
configMaps []*corev1.ConfigMap
49+
clientset *kubernetes.Clientset
50+
statefulSetClient clientappsv1.StatefulSetInterface
51+
podClient clientcorev1.PodInterface
52+
configMapClient clientcorev1.ConfigMapInterface
53+
podChooser podchooser.PodChooser
54+
defaultLoad bool
55+
timeout time.Duration
5656
}
5757

5858
func (d *Driver) IsMobyDriver() bool {
@@ -65,10 +65,10 @@ func (d *Driver) Config() driver.InitConfig {
6565

6666
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
6767
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
68-
_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
68+
_, err := d.statefulSetClient.Get(ctx, d.statefulSet.Name, metav1.GetOptions{})
6969
if err != nil {
7070
if !apierrors.IsNotFound(err) {
71-
return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
71+
return errors.Wrapf(err, "error for bootstrap %q", d.statefulSet.Name)
7272
}
7373

7474
for _, cfg := range d.configMaps {
@@ -85,9 +85,9 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
8585
}
8686
}
8787

88-
_, err = d.deploymentClient.Create(ctx, d.deployment, metav1.CreateOptions{})
88+
_, err = d.statefulSetClient.Create(ctx, d.statefulSet, metav1.CreateOptions{})
8989
if err != nil {
90-
return errors.Wrapf(err, "error while calling deploymentClient.Create for %q", d.deployment.Name)
90+
return errors.Wrapf(err, "error while calling statefulSetClient.Create for %q", d.statefulSet.Name)
9191
}
9292
}
9393
return sub.Wrap(
@@ -102,7 +102,7 @@ func (d *Driver) wait(ctx context.Context) error {
102102
// TODO: use watch API
103103
var (
104104
err error
105-
depl *appsv1.Deployment
105+
stat *appsv1.StatefulSet
106106
)
107107

108108
timeoutChan := time.After(d.timeout)
@@ -116,31 +116,31 @@ func (d *Driver) wait(ctx context.Context) error {
116116
case <-timeoutChan:
117117
return err
118118
case <-ticker.C:
119-
depl, err = d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
119+
stat, err = d.statefulSetClient.Get(ctx, d.statefulSet.Name, metav1.GetOptions{})
120120
if err == nil {
121-
if depl.Status.ReadyReplicas >= int32(d.minReplicas) {
121+
if stat.Status.ReadyReplicas >= int32(d.minReplicas) {
122122
return nil
123123
}
124-
err = errors.Errorf("expected %d replicas to be ready, got %d", d.minReplicas, depl.Status.ReadyReplicas)
124+
err = errors.Errorf("expected %d replicas to be ready, got %d", d.minReplicas, stat.Status.ReadyReplicas)
125125
}
126126
}
127127
}
128128
}
129129

130130
func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
131-
depl, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
131+
stat, err := d.statefulSetClient.Get(ctx, d.statefulSet.Name, metav1.GetOptions{})
132132
if err != nil {
133133
// TODO: return err if err != ErrNotFound
134134
return &driver.Info{
135135
Status: driver.Inactive,
136136
}, nil
137137
}
138-
if depl.Status.ReadyReplicas <= 0 {
138+
if stat.Status.ReadyReplicas <= 0 {
139139
return &driver.Info{
140140
Status: driver.Stopped,
141141
}, nil
142142
}
143-
pods, err := podchooser.ListRunningPods(ctx, d.podClient, depl)
143+
pods, err := podchooser.ListRunningPods(ctx, d.podClient, stat)
144144
if err != nil {
145145
return nil, err
146146
}
@@ -182,9 +182,9 @@ func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error {
182182
return nil
183183
}
184184

185-
if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
185+
if err := d.statefulSetClient.Delete(ctx, d.statefulSet.Name, metav1.DeleteOptions{}); err != nil {
186186
if !apierrors.IsNotFound(err) {
187-
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
187+
return errors.Wrapf(err, "error while calling statefulSetClient.Delete for %q", d.statefulSet.Name)
188188
}
189189
}
190190
for _, cfg := range d.configMaps {

Diff for: ‎driver/kubernetes/factory.go

+35-35
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
104104
}
105105
}
106106

107-
deploymentName, err := buildxNameToDeploymentName(cfg.Name)
107+
statefulSetName, err := buildxNameToStatefulSetName(cfg.Name)
108108
if err != nil {
109109
return nil, err
110110
}
@@ -128,44 +128,44 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
128128
clientset: clientset,
129129
}
130130

131-
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)
131+
statefulSetOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(statefulSetName, namespace, cfg)
132132
if nil != err {
133133
return nil, err
134134
}
135135

136136
d.defaultLoad = defaultLoad
137137
d.timeout = timeout
138138

139-
d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
139+
d.statefulSet, d.configMaps, err = manifest.NewStatefulSet(statefulSetOpt)
140140
if err != nil {
141141
return nil, err
142142
}
143143

144-
d.minReplicas = deploymentOpt.Replicas
144+
d.minReplicas = statefulSetOpt.Replicas
145145

146-
d.deploymentClient = clientset.AppsV1().Deployments(namespace)
146+
d.statefulSetClient = clientset.AppsV1().StatefulSets(namespace)
147147
d.podClient = clientset.CoreV1().Pods(namespace)
148148
d.configMapClient = clientset.CoreV1().ConfigMaps(namespace)
149149

150150
switch loadbalance {
151151
case LoadbalanceSticky:
152152
d.podChooser = &podchooser.StickyPodChooser{
153-
Key: cfg.ContextPathHash,
154-
PodClient: d.podClient,
155-
Deployment: d.deployment,
153+
Key: cfg.ContextPathHash,
154+
PodClient: d.podClient,
155+
StatefulSet: d.statefulSet,
156156
}
157157
case LoadbalanceRandom:
158158
d.podChooser = &podchooser.RandomPodChooser{
159-
PodClient: d.podClient,
160-
Deployment: d.deployment,
159+
PodClient: d.podClient,
160+
StatefulSet: d.statefulSet,
161161
}
162162
}
163163
return d, nil
164164
}
165165

166-
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, error) {
167-
deploymentOpt := &manifest.DeploymentOpt{
168-
Name: deploymentName,
166+
func (f *factory) processDriverOpts(statefulSetName string, namespace string, cfg driver.InitConfig) (*manifest.StatefulSetOpt, string, string, bool, time.Duration, error) {
167+
statefulSetOpt := &manifest.StatefulSetOpt{
168+
Name: statefulSetName,
169169
Image: bkimage.DefaultImage,
170170
Replicas: 1,
171171
BuildkitFlags: cfg.BuildkitdFlags,
@@ -177,7 +177,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
177177
defaultLoad := false
178178
timeout := defaultTimeout
179179

180-
deploymentOpt.Qemu.Image = bkimage.QemuImage
180+
statefulSetOpt.Qemu.Image = bkimage.QemuImage
181181

182182
loadbalance := LoadbalanceSticky
183183
var err error
@@ -186,57 +186,57 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
186186
switch k {
187187
case "image":
188188
if v != "" {
189-
deploymentOpt.Image = v
189+
statefulSetOpt.Image = v
190190
}
191191
case "namespace":
192192
namespace = v
193193
case "replicas":
194-
deploymentOpt.Replicas, err = strconv.Atoi(v)
194+
statefulSetOpt.Replicas, err = strconv.Atoi(v)
195195
if err != nil {
196196
return nil, "", "", false, 0, err
197197
}
198198
case "requests.cpu":
199-
deploymentOpt.RequestsCPU = v
199+
statefulSetOpt.RequestsCPU = v
200200
case "requests.memory":
201-
deploymentOpt.RequestsMemory = v
201+
statefulSetOpt.RequestsMemory = v
202202
case "requests.ephemeral-storage":
203-
deploymentOpt.RequestsEphemeralStorage = v
203+
statefulSetOpt.RequestsEphemeralStorage = v
204204
case "limits.cpu":
205-
deploymentOpt.LimitsCPU = v
205+
statefulSetOpt.LimitsCPU = v
206206
case "limits.memory":
207-
deploymentOpt.LimitsMemory = v
207+
statefulSetOpt.LimitsMemory = v
208208
case "limits.ephemeral-storage":
209-
deploymentOpt.LimitsEphemeralStorage = v
209+
statefulSetOpt.LimitsEphemeralStorage = v
210210
case "rootless":
211-
deploymentOpt.Rootless, err = strconv.ParseBool(v)
211+
statefulSetOpt.Rootless, err = strconv.ParseBool(v)
212212
if err != nil {
213213
return nil, "", "", false, 0, err
214214
}
215215
if _, isImage := cfg.DriverOpts["image"]; !isImage {
216-
deploymentOpt.Image = bkimage.DefaultRootlessImage
216+
statefulSetOpt.Image = bkimage.DefaultRootlessImage
217217
}
218218
case "schedulername":
219-
deploymentOpt.SchedulerName = v
219+
statefulSetOpt.SchedulerName = v
220220
case "serviceaccount":
221-
deploymentOpt.ServiceAccountName = v
221+
statefulSetOpt.ServiceAccountName = v
222222
case "nodeselector":
223-
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
223+
statefulSetOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
224224
if err != nil {
225225
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse node selector")
226226
}
227227
case "annotations":
228-
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
228+
statefulSetOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
229229
if err != nil {
230230
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse annotations")
231231
}
232232
case "labels":
233-
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
233+
statefulSetOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
234234
if err != nil {
235235
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse labels")
236236
}
237237
case "tolerations":
238238
ts := strings.Split(v, ";")
239-
deploymentOpt.Tolerations = []corev1.Toleration{}
239+
statefulSetOpt.Tolerations = []corev1.Toleration{}
240240
for i := range ts {
241241
kvs := strings.Split(ts[i], ",")
242242

@@ -267,7 +267,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
267267
}
268268
}
269269

270-
deploymentOpt.Tolerations = append(deploymentOpt.Tolerations, t)
270+
statefulSetOpt.Tolerations = append(statefulSetOpt.Tolerations, t)
271271
}
272272
case "loadbalance":
273273
switch v {
@@ -278,13 +278,13 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
278278
}
279279
loadbalance = v
280280
case "qemu.install":
281-
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
281+
statefulSetOpt.Qemu.Install, err = strconv.ParseBool(v)
282282
if err != nil {
283283
return nil, "", "", false, 0, err
284284
}
285285
case "qemu.image":
286286
if v != "" {
287-
deploymentOpt.Qemu.Image = v
287+
statefulSetOpt.Qemu.Image = v
288288
}
289289
case "default-load":
290290
defaultLoad, err = strconv.ParseBool(v)
@@ -301,7 +301,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
301301
}
302302
}
303303

304-
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, nil
304+
return statefulSetOpt, loadbalance, namespace, defaultLoad, timeout, nil
305305
}
306306

307307
func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {
@@ -324,7 +324,7 @@ func (f *factory) AllowsInstances() bool {
324324
// buildxNameToDeploymentName converts buildx name to Kubernetes Deployment name.
325325
//
326326
// eg. "buildx_buildkit_loving_mendeleev0" -> "loving-mendeleev0"
327-
func buildxNameToDeploymentName(bx string) (string, error) {
327+
func buildxNameToStatefulSetName(bx string) (string, error) {
328328
// TODO: commands.util.go should not pass "buildx_buildkit_" prefix to drivers
329329
s, err := driver.ParseBuilderName(bx)
330330
if err != nil {

0 commit comments

Comments
 (0)