Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate msi/subnet client to track2 one #1588

Merged
merged 1 commit into from
Sep 27, 2024
Merged

Migrate msi/subnet client to track2 one #1588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
github.com/Azure/go-autorest/autorest v0.11.29
github.com/Azure/go-autorest/autorest/adal v0.9.24
github.com/container-storage-interface/spec v1.9.0
github.com/go-ini/ini v1.67.0
github.com/golang/protobuf v1.5.4
Expand All @@ -38,7 +39,7 @@ require (
k8s.io/pod-security-admission v0.31.1
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27
sigs.k8s.io/yaml v1.4.0
)
Expand All @@ -47,14 +48,14 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
Expand Down Expand Up @@ -189,7 +190,7 @@ replace (
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.7
k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.7
k8s.io/endpointslice => k8s.io/endpointslice v0.29.7
k8s.io/kms => k8s.io/kms v0.31.1
k8s.io/kms => k8s.io/kms v0.29.7
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.7
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.7
k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.7
Expand Down
10 changes: 6 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthoriza
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q=
Expand Down Expand Up @@ -431,8 +433,8 @@ k8s.io/csi-translation-lib v0.29.7 h1:6z1iFhTmVMK9mebK2eodvDCKv3bfL0OFu5z2C8YNvM
k8s.io/csi-translation-lib v0.29.7/go.mod h1:+5ZOwRS5LUQOghtqv6QWWmadixbm697xNHZC318oVf4=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kms v0.31.1 h1:cGLyV3cIwb0ovpP/jtyIe2mEuQ/MkbhmeBF2IYCA9Io=
k8s.io/kms v0.31.1/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
k8s.io/kms v0.29.7 h1:4ELQdx7T4EPKbN/QMj6SeZizrEKapza5YF8e5XtZPv0=
k8s.io/kms v0.29.7/go.mod h1:vWVImKkJd+1BQY4tBwdfSwjQBiLrnbNtHADcDEDQFtk=
k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo=
k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE=
k8s.io/kubectl v0.29.7 h1:D+Jheug9M++zlt67cROZgxaIjrDdLqp9jkW/EYrXAoM=
Expand All @@ -451,8 +453,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775 h1:0YqezUI2dBm+Y+XgoXA0+Atd2CDEGFq6PS/8vtgwbJI=
sigs.k8s.io/cloud-provider-azure v1.31.1-0.20240914065912-f4dd79d54775/go.mod h1:ZMuwABqLK6ICPch/wMIeMdTs15yH1lkPlwenTVzaB2A=
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56 h1:k71HScdrMkpf04udgySK7Jsw+bw90eQbaRssItA+ej4=
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.56/go.mod h1:kMZIHUHyI3TejvPoPVC9bPJgmOs3Wu7/dz0hxInU03o=
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57 h1:Gt0aHqpju4eEtO9DoLLSZbKCjfH5fLmfCES7VGsiHHo=
sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.57/go.mod h1:pCcUbyidPO6qrplCGARQY70n0E7ANUjmwR1xtAz/nng=
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27 h1:o1LU+o0hAuY3esYQ5gzGElsCfkUNKCXmAIcBvf4CxZo=
sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.27/go.mod h1:g/XTYItaIrR2AX3CGoFR0jIwitKedKBf6WwNJYXGoDw=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
Expand Down
73 changes: 27 additions & 46 deletions pkg/blob/azure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,19 @@ import (
"os"
"strings"

kv "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
"github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest"
azure2 "github.com/Azure/go-autorest/autorest/azure"
"golang.org/x/net/context"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"k8s.io/utils/ptr"
"sigs.k8s.io/cloud-provider-azure/pkg/azclient"
"sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader"
azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
azure "sigs.k8s.io/cloud-provider-azure/pkg/provider"
providerconfig "sigs.k8s.io/cloud-provider-azure/pkg/provider/config"
"sigs.k8s.io/cloud-provider-azure/pkg/retry"
)

var (
Expand Down Expand Up @@ -147,46 +146,28 @@ func GetCloudProvider(ctx context.Context, kubeClient kubernetes.Interface, node

// getKeyVaultSecretContent get content of the keyvault secret
func (d *Driver) getKeyVaultSecretContent(ctx context.Context, vaultURL string, secretName string, secretVersion string) (content string, err error) {
kvClient, err := d.initializeKvClient()
var authProvider *azclient.AuthProvider
authProvider, err = azclient.NewAuthProvider(&d.cloud.AzureAuthConfig.ARMClientConfig, &d.cloud.AzureAuthConfig.AzureAuthConfig)
if err != nil {
return "", err
}
kvClient, err := azsecrets.NewClient(vaultURL, authProvider.GetAzIdentity(), nil)
if err != nil {
return "", fmt.Errorf("failed to get keyvaultClient: %w", err)
}

klog.V(2).Infof("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v)", vaultURL, secretName, secretVersion)
secret, err := kvClient.GetSecret(ctx, vaultURL, secretName, secretVersion)
secret, err := kvClient.GetSecret(ctx, secretName, secretVersion, nil)
if err != nil {
return "", fmt.Errorf("get secret from vaultURL(%v), sercretName(%v), secretVersion(%v) failed with error: %w", vaultURL, secretName, secretVersion, err)
}
return *secret.Value, nil
}

func (d *Driver) initializeKvClient() (*kv.BaseClient, error) {
kvClient := kv.New()
token, err := d.getKeyvaultToken()
if err != nil {
return nil, err
}

kvClient.Authorizer = token
return &kvClient, nil
}

// getKeyvaultToken retrieves a new service principal token to access keyvault
func (d *Driver) getKeyvaultToken() (authorizer autorest.Authorizer, err error) {
env := d.getCloudEnvironment()
kvEndPoint := strings.TrimSuffix(env.KeyVaultEndpoint, "/")
servicePrincipalToken, err := providerconfig.GetServicePrincipalToken(&d.cloud.AzureAuthConfig, &env, kvEndPoint)
if err != nil {
return nil, err
}
authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
return authorizer, nil
}

func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceGroup, vnetName, subnetName string) ([]string, error) {
var vnetResourceIDs []string
if d.cloud.SubnetsClient == nil {
return vnetResourceIDs, fmt.Errorf("SubnetsClient is nil")
if d.networkClientFactory == nil {
return vnetResourceIDs, fmt.Errorf("networkClientFactory is nil")
}

if vnetResourceGroup == "" {
Expand Down Expand Up @@ -220,21 +201,21 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
d.subnetLockMap.LockEntry(lockKey)
defer d.subnetLockMap.UnlockEntry(lockKey)

var subnets []network.Subnet
var subnets []*network.Subnet
if subnetName != "" {
// list multiple subnets separated by comma
subnetNames := strings.Split(subnetName, ",")
for _, sn := range subnetNames {
sn = strings.TrimSpace(sn)
subnet, rerr := d.cloud.SubnetsClient.Get(ctx, vnetResourceGroup, vnetName, sn, "")
subnet, rerr := d.networkClientFactory.GetSubnetClient().Get(ctx, vnetResourceGroup, vnetName, sn, nil)
if rerr != nil {
return vnetResourceIDs, fmt.Errorf("failed to get the subnet %s under rg %s vnet %s: %v", subnetName, vnetResourceGroup, vnetName, rerr.Error())
}
subnets = append(subnets, subnet)
}
} else {
var rerr *retry.Error
subnets, rerr = d.cloud.SubnetsClient.List(ctx, vnetResourceGroup, vnetName)
var rerr error
subnets, rerr = d.networkClientFactory.GetSubnetClient().List(ctx, vnetResourceGroup, vnetName)
if rerr != nil {
return vnetResourceIDs, fmt.Errorf("failed to list the subnets under rg %s vnet %s: %v", vnetResourceGroup, vnetName, rerr.Error())
}
Expand All @@ -249,19 +230,19 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG
klog.V(2).Infof("set vnetResourceID %s", vnetResourceID)
vnetResourceIDs = append(vnetResourceIDs, vnetResourceID)

endpointLocaions := []string{location}
storageServiceEndpoint := network.ServiceEndpointPropertiesFormat{
endpointLocaions := []*string{to.Ptr(location)}
storageServiceEndpoint := &network.ServiceEndpointPropertiesFormat{
Service: &storageService,
Locations: &endpointLocaions,
Locations: endpointLocaions,
}
storageServiceExists := false
if subnet.SubnetPropertiesFormat == nil {
subnet.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
if subnet.Properties == nil {
subnet.Properties = &network.SubnetPropertiesFormat{}
}
if subnet.SubnetPropertiesFormat.ServiceEndpoints == nil {
subnet.SubnetPropertiesFormat.ServiceEndpoints = &[]network.ServiceEndpointPropertiesFormat{}
if subnet.Properties.ServiceEndpoints == nil {
subnet.Properties.ServiceEndpoints = []*network.ServiceEndpointPropertiesFormat{}
}
serviceEndpoints := *subnet.SubnetPropertiesFormat.ServiceEndpoints
serviceEndpoints := subnet.Properties.ServiceEndpoints
for _, v := range serviceEndpoints {
if strings.HasPrefix(ptr.Deref(v.Service, ""), storageService) {
storageServiceExists = true
Expand All @@ -272,10 +253,10 @@ func (d *Driver) updateSubnetServiceEndpoints(ctx context.Context, vnetResourceG

if !storageServiceExists {
serviceEndpoints = append(serviceEndpoints, storageServiceEndpoint)
subnet.SubnetPropertiesFormat.ServiceEndpoints = &serviceEndpoints
subnet.Properties.ServiceEndpoints = serviceEndpoints

klog.V(2).Infof("begin to update the subnet %s under vnet %s in rg %s", sn, vnetName, vnetResourceGroup)
if err := d.cloud.SubnetsClient.CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, subnet); err != nil {
if _, err := d.networkClientFactory.GetSubnetClient().CreateOrUpdate(ctx, vnetResourceGroup, vnetName, sn, *subnet); err != nil {
return vnetResourceIDs, fmt.Errorf("failed to update the subnet %s under vnet %s: %v", sn, vnetName, err)
}
}
Expand Down
Loading
Loading