diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index e02f5aa0e..ec042850c 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -1388,6 +1388,7 @@ spec: - dynatrace - keptn - splunk + - azuremonitor address: description: API address of this provider type: string diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index e02f5aa0e..ec042850c 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -1388,6 +1388,7 @@ spec: - dynatrace - keptn - splunk + - azuremonitor address: description: API address of this provider type: string diff --git a/docs/gitbook/usage/metrics.md b/docs/gitbook/usage/metrics.md index 0cceec77f..13c1e3a0d 100644 --- a/docs/gitbook/usage/metrics.md +++ b/docs/gitbook/usage/metrics.md @@ -798,6 +798,111 @@ Reference the template in the canary analysis: interval: 1m ``` +## Azure Monitor Workspace + +You can create custom metric checks using the Azure Monitor provider, which queries the +[Prometheus compatible API](https://learn.microsoft.com/azure/azure-monitor/metrics/prometheus-api-promql) +of an [Azure Monitor Workspace](https://learn.microsoft.com/azure/azure-monitor/metrics/azure-monitor-workspace-overview). + +Queries are authenticated with a Microsoft Entra ID token, so no authentication proxy is +required in front of the workspace. Set `address` to the workspace **Query endpoint** and grant +the identity that Flagger uses the `Monitoring Data Reader` role on the workspace. Azure +Government and Azure China workspaces are detected from the endpoint. + +With no `secretRef`, Flagger authenticates with the +[workload identity](https://learn.microsoft.com/azure/aks/workload-identity-overview) of its own +pod. This requires a federated credential for the `system:serviceaccount:istio-system:flagger` +subject, and a Flagger install that carries the annotation and label the webhook looks for: + +```yaml +serviceAccount: + annotations: + azure.workload.identity/client-id: your-managed-identity-client-id +podLabels: + azure.workload.identity/use: "true" +``` + +The metric template then needs no credentials: + +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: request-success-rate + namespace: istio-system +spec: + provider: + type: azuremonitor + address: https://flagger-abc1.eastus.prometheus.monitor.azure.com + query: | + 100 - sum( + rate( + istio_requests_total{ + reporter="destination", + destination_workload_namespace="{{ namespace }}", + destination_workload="{{ target }}", + response_code=~"5.." + }[{{ interval }}] + ) + ) + / + sum( + rate( + istio_requests_total{ + reporter="destination", + destination_workload_namespace="{{ namespace }}", + destination_workload="{{ target }}" + }[{{ interval }}] + ) + ) * 100 +``` + +Reference the template in the canary analysis: + +```yaml + analysis: + metrics: + - name: "request success rate" + templateRef: + name: request-success-rate + namespace: istio-system + thresholdRange: + min: 99 + interval: 1m +``` + +To use a different identity, reference a secret with `secretRef`. The keys it contains select +the identity: + +| Secret keys | Identity used | +| --- | --- | +| `clientId`, `tenantId`, `clientSecret` | Microsoft Entra ID application (service principal) | +| `clientId`, `tenantId` | Workload identity, for an identity other than the pod default | + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: azure-monitor + namespace: istio-system +stringData: + clientId: your-application-id + tenantId: your-tenant-id + clientSecret: your-client-secret +``` + +Because Flagger can authenticate with its own identity, `address` is restricted to an `https` +workspace query endpoint and `insecureSkipVerify` is not supported. + +Troubleshooting: + +* `no token file specified` or `no client ID specified` means the workload identity webhook did + not project a token into the Flagger pod. Check the service account annotation and pod label. +* A `403` response means the identity is missing the `Monitoring Data Reader` role assignment + on the Azure Monitor Workspace. +* A `429` response means the workspace query limits have been reached. Flagger treats this as a + failed metric check, which can cause a canary to be rolled back. + ## Kubernetes External Metrics You can query an external metrics provider that implements the diff --git a/go.mod b/go.mod index 8cd0f3b09..72e0757c1 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.26.0 require ( cloud.google.com/go/monitoring v1.29.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go v1.55.8 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -39,6 +41,8 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -51,6 +55,7 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-containerregistry v0.20.3 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -69,6 +74,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect diff --git a/go.sum b/go.sum index 262c45793..665d673e8 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,15 @@ cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizX cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= @@ -64,6 +73,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= @@ -134,6 +145,8 @@ github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7o github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -231,6 +244,7 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index e02f5aa0e..ec042850c 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -1388,6 +1388,7 @@ spec: - dynatrace - keptn - splunk + - azuremonitor address: description: API address of this provider type: string diff --git a/pkg/metrics/providers/azuremonitor.go b/pkg/metrics/providers/azuremonitor.go new file mode 100644 index 000000000..fa3b429ac --- /dev/null +++ b/pkg/metrics/providers/azuremonitor.go @@ -0,0 +1,209 @@ +/* +Copyright 2025 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "context" + "crypto/sha256" + "fmt" + "net/url" + "strings" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + + corev1 "k8s.io/api/core/v1" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" +) + +const ( + azureClientIDSecretKey = "clientId" + azureTenantIDSecretKey = "tenantId" + azureClientSecretSecretKey = "clientSecret" + + azureTokenTimeout = 30 * time.Second +) + +// credentials are cached because a provider is built for every metric on every +// analysis interval and each credential holds its own token cache +var ( + azureCredentialCacheMu sync.Mutex + azureCredentialCache = map[string]azcore.TokenCredential{} +) + +var azureClouds = []struct { + suffix string + config cloud.Configuration +}{ + {suffix: ".prometheus.monitor.azure.us", config: cloud.AzureGovernment}, + {suffix: ".prometheus.monitor.azure.cn", config: cloud.AzureChina}, + {suffix: ".prometheus.monitor.azure.com", config: cloud.AzurePublic}, +} + +// AzureMonitorProvider executes promQL queries against an Azure Monitor Workspace +type AzureMonitorProvider struct { + *PrometheusProvider +} + +// NewAzureMonitorProvider takes a provider spec and the credentials map, validates the +// address and returns a Prometheus client holding a Microsoft Entra ID token +// for the workspace +func NewAzureMonitorProvider(provider flaggerv1.MetricTemplateProvider, credentials map[string][]byte) (*AzureMonitorProvider, error) { + if provider.InsecureSkipVerify { + return nil, fmt.Errorf("%s provider does not support insecureSkipVerify", provider.Type) + } + + address, err := url.Parse(provider.Address) + if err != nil { + return nil, fmt.Errorf("%s address %s is not a valid URL: %w", provider.Type, provider.Address, err) + } + + if !strings.EqualFold(address.Scheme, "https") { + return nil, fmt.Errorf("%s address %s must use https", provider.Type, provider.Address) + } + + cloudConfig, audience, err := azureCloudForHost(address.Hostname()) + if err != nil { + return nil, fmt.Errorf("%s address %s %w", provider.Type, provider.Address, err) + } + + cred, err := azureCredential(provider.Type, credentials, cloudConfig) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), azureTokenTimeout) + defer cancel() + + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{azureScope(audience)}}) + if err != nil { + return nil, fmt.Errorf("%s token request failed: %w", provider.Type, err) + } + + // RunQuery adds the bearer token to the header map it was given, and that map + // is shared by every canary using this template, so it is copied first + promProvider := provider + promProvider.Headers = provider.Headers.Clone() + + // the Prometheus provider only reads credentials when a secret is referenced + if promProvider.SecretRef == nil { + promProvider.SecretRef = &corev1.LocalObjectReference{} + } + + prom, err := NewPrometheusProvider(promProvider, map[string][]byte{"token": []byte(token.Token)}) + if err != nil { + return nil, err + } + + return &AzureMonitorProvider{PrometheusProvider: prom}, nil +} + +func azureCredential(providerType string, credentials map[string][]byte, + cloudConfig cloud.Configuration, +) (azcore.TokenCredential, error) { + key := azureCredentialKey(credentials, cloudConfig) + + azureCredentialCacheMu.Lock() + defer azureCredentialCacheMu.Unlock() + + if cred, ok := azureCredentialCache[key]; ok { + return cred, nil + } + + cred, err := newAzureCredential(providerType, credentials, cloudConfig) + if err != nil { + return nil, err + } + + azureCredentialCache[key] = cred + return cred, nil +} + +func azureCredentialKey(credentials map[string][]byte, cloudConfig cloud.Configuration) string { + return fmt.Sprintf("%s|%s|%s|%x", + cloudConfig.ActiveDirectoryAuthorityHost, + credentials[azureClientIDSecretKey], + credentials[azureTenantIDSecretKey], + sha256.Sum256(credentials[azureClientSecretSecretKey])) +} + +// newAzureCredential selects an identity from the keys found in the provider +// secret, defaulting to the workload identity of the Flagger pod +func newAzureCredential(providerType string, credentials map[string][]byte, + cloudConfig cloud.Configuration, +) (azcore.TokenCredential, error) { + clientOptions := azcore.ClientOptions{Cloud: cloudConfig} + + clientID := string(credentials[azureClientIDSecretKey]) + tenantID := string(credentials[azureTenantIDSecretKey]) + clientSecret := string(credentials[azureClientSecretSecretKey]) + + switch { + case clientSecret != "": + if clientID == "" { + return nil, fmt.Errorf("%s credentials does not contain a clientId", providerType) + } + if tenantID == "" { + return nil, fmt.Errorf("%s credentials does not contain a tenantId", providerType) + } + return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret, + &azidentity.ClientSecretCredentialOptions{ClientOptions: clientOptions}) + case clientID != "" && tenantID != "": + return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + ClientOptions: clientOptions, + ClientID: clientID, + TenantID: tenantID, + }) + case clientID != "": + return nil, fmt.Errorf("%s credentials does not contain a tenantId", providerType) + case tenantID != "": + return nil, fmt.Errorf("%s credentials does not contain a clientId", providerType) + default: + return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + ClientOptions: clientOptions, + }) + } +} + +// azureCloudForHost returns the Azure cloud and the token audience matching +// the query endpoint +func azureCloudForHost(host string) (cloud.Configuration, string, error) { + host = strings.ToLower(host) + for _, c := range azureClouds { + if strings.HasSuffix(host, c.suffix) { + return c.config, "https://" + strings.TrimPrefix(c.suffix, "."), nil + } + } + suffixes := make([]string, len(azureClouds)) + for i, c := range azureClouds { + suffixes[i] = c.suffix + } + return cloud.Configuration{}, "", fmt.Errorf( + "is not an Azure Monitor Workspace query endpoint, expected a host ending in %s", strings.Join(suffixes, ", ")) +} + +func azureScope(audience string) string { + if strings.HasSuffix(audience, "/.default") { + return audience + } + return strings.TrimSuffix(audience, "/") + "/.default" +} diff --git a/pkg/metrics/providers/azuremonitor_test.go b/pkg/metrics/providers/azuremonitor_test.go new file mode 100644 index 000000000..6605c6d9e --- /dev/null +++ b/pkg/metrics/providers/azuremonitor_test.go @@ -0,0 +1,442 @@ +/* +Copyright 2025 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" +) + +const azureMonitorTestAddress = "https://flagger-abc1.eastus.prometheus.monitor.azure.com" + +type fakeAzureCredential struct { + token string + scopes []string + err error +} + +func (c *fakeAzureCredential) GetToken(_ context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { + c.scopes = opts.Scopes + if c.err != nil { + return azcore.AccessToken{}, c.err + } + return azcore.AccessToken{Token: c.token, ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +// azureWorkloadIdentityEnv sets the variables injected by the workload identity webhook +func azureWorkloadIdentityEnv(t *testing.T) { + t.Helper() + t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/azure/tokens/azure-identity-token") + t.Setenv("AZURE_CLIENT_ID", "pod-client-id") + t.Setenv("AZURE_TENANT_ID", "pod-tenant-id") +} + +func azureMonitorTestProvider(address string) flaggerv1.MetricTemplateProvider { + return flaggerv1.MetricTemplateProvider{ + Type: "azuremonitor", + Address: address, + SecretRef: &corev1.LocalObjectReference{Name: "azuremonitor"}, + } +} + +// azureUseCredential seeds the credential cache so the constructor mints its token from cred +func azureUseCredential(t *testing.T, cred azcore.TokenCredential) { + t.Helper() + azureWorkloadIdentityEnv(t) + + key := azureCredentialKey(nil, cloud.AzurePublic) + + azureCredentialCacheMu.Lock() + azureCredentialCache[key] = cred + azureCredentialCacheMu.Unlock() + + t.Cleanup(func() { + azureCredentialCacheMu.Lock() + delete(azureCredentialCache, key) + azureCredentialCacheMu.Unlock() + }) +} + +// azureMonitorTestClient points a provider at the test server, since only workspace addresses are accepted +func azureMonitorTestClient(t *testing.T, serverURL string, provider flaggerv1.MetricTemplateProvider, + cred azcore.TokenCredential, +) *AzureMonitorProvider { + t.Helper() + azureUseCredential(t, cred) + + az, err := NewAzureMonitorProvider(provider, nil) + require.NoError(t, err) + + u, err := url.Parse(serverURL) + require.NoError(t, err) + az.url = *u + + return az +} + +func TestNewAzureMonitorProvider(t *testing.T) { + azureWorkloadIdentityEnv(t) + + t.Run("ok", func(t *testing.T) { + cred := &fakeAzureCredential{token: "token"} + azureUseCredential(t, cred) + + az, err := NewAzureMonitorProvider(azureMonitorTestProvider(azureMonitorTestAddress), nil) + require.NoError(t, err) + + assert.Equal(t, azureMonitorTestAddress, az.url.String()) + assert.Equal(t, "token", az.token) + assert.Equal(t, []string{"https://prometheus.monitor.azure.com/.default"}, cred.scopes) + }) + + t.Run("no secret", func(t *testing.T) { + azureUseCredential(t, &fakeAzureCredential{token: "token"}) + + provider := azureMonitorTestProvider(azureMonitorTestAddress) + provider.SecretRef = nil + + az, err := NewAzureMonitorProvider(provider, nil) + require.NoError(t, err) + assert.Equal(t, "token", az.token) + }) + + t.Run("token error", func(t *testing.T) { + azureUseCredential(t, &fakeAzureCredential{err: assert.AnError}) + + _, err := NewAzureMonitorProvider(azureMonitorTestProvider(azureMonitorTestAddress), nil) + require.ErrorContains(t, err, "azuremonitor token request failed") + }) + + t.Run("invalid address", func(t *testing.T) { + _, err := NewAzureMonitorProvider(azureMonitorTestProvider(""), nil) + require.Error(t, err) + }) + + t.Run("address outside of azure monitor", func(t *testing.T) { + _, err := NewAzureMonitorProvider(azureMonitorTestProvider("https://prometheus.example.com"), nil) + require.EqualError(t, err, + "azuremonitor address https://prometheus.example.com is not an Azure Monitor Workspace query endpoint, "+ + "expected a host ending in .prometheus.monitor.azure.us, .prometheus.monitor.azure.cn, .prometheus.monitor.azure.com") + }) + + t.Run("address without tls", func(t *testing.T) { + address := "http://flagger-abc1.eastus.prometheus.monitor.azure.com" + + _, err := NewAzureMonitorProvider(azureMonitorTestProvider(address), nil) + require.EqualError(t, err, "azuremonitor address "+address+" must use https") + }) + + t.Run("insecure skip verify", func(t *testing.T) { + provider := azureMonitorTestProvider(azureMonitorTestAddress) + provider.InsecureSkipVerify = true + + _, err := NewAzureMonitorProvider(provider, nil) + require.EqualError(t, err, "azuremonitor provider does not support insecureSkipVerify") + }) +} + +func TestAzureMonitorProvider_RunQuery(t *testing.T) { + t.Run("ok", func(t *testing.T) { + expected := `sum(envoy_cluster_upstream_rq)` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v1/query", r.URL.Path) + assert.Equal(t, expected, r.URL.Query()["query"][0]) + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + cred := &fakeAzureCredential{token: "token"} + az := azureMonitorTestClient(t, ts.URL, azureMonitorTestProvider(azureMonitorTestAddress), cred) + + val, err := az.RunQuery(expected) + require.NoError(t, err) + assert.Equal(t, float64(100), val) + assert.Equal(t, []string{"https://prometheus.monitor.azure.com/.default"}, cred.scopes) + }) +} + +func TestAzureMonitorProvider_IsOnline(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + az := azureMonitorTestClient(t, ts.URL, azureMonitorTestProvider(azureMonitorTestAddress), &fakeAzureCredential{token: "token"}) + + ok, err := az.IsOnline() + require.NoError(t, err) + assert.True(t, ok) +} + +func TestAzureMonitorProvider_Headers(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + assert.Equal(t, "bar", r.Header.Get("Foo")) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + provider := azureMonitorTestProvider(azureMonitorTestAddress) + provider.Headers = http.Header{"Foo": []string{"bar"}} + + az := azureMonitorTestClient(t, ts.URL, provider, &fakeAzureCredential{token: "token"}) + + for i := 0; i < 2; i++ { + _, err := az.RunQuery("vector(1)") + require.NoError(t, err) + } + + assert.Empty(t, provider.Headers.Get("Authorization"), + "the header map comes from the informer cache and must not be modified") +} + +// TestAzureMonitorProvider_Redirect relies on the http client dropping the Authorization +// header when a redirect leaves the workspace host. The hosts are faked because the +// client compares host names and every test server listens on the loopback address. +func TestAzureMonitorProvider_Redirect(t *testing.T) { + var received string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + received = r.Header.Get("Authorization") + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}` + w.Write([]byte(json)) + })) + defer target.Close() + + workspace := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + http.Redirect(w, r, "http://elsewhere.invalid/api/v1/query?query=vector(1)", http.StatusFound) + })) + defer workspace.Close() + + az := azureMonitorTestClient(t, "http://workspace.invalid", azureMonitorTestProvider(azureMonitorTestAddress), + &fakeAzureCredential{token: "token"}) + + hosts := map[string]string{ + "workspace.invalid:80": workspace.Listener.Addr().String(), + "elsewhere.invalid:80": target.Listener.Addr().String(), + } + az.client = &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if mapped, ok := hosts[addr]; ok { + addr = mapped + } + return (&net.Dialer{}).DialContext(ctx, network, addr) + }, + }} + + val, err := az.RunQuery("vector(1)") + require.NoError(t, err) + assert.Equal(t, float64(1), val, "the redirect must have been followed") + assert.Empty(t, received, "the token must not be sent to another host") +} + +func TestNewAzureCredential(t *testing.T) { + tests := []struct { + name string + credentials map[string][]byte + expectedType any + expectedErr string + }{ + { + name: "service principal", + credentials: map[string][]byte{ + azureClientIDSecretKey: []byte("client-id"), + azureTenantIDSecretKey: []byte("tenant-id"), + azureClientSecretSecretKey: []byte("client-secret"), + }, + expectedType: &azidentity.ClientSecretCredential{}, + }, + { + name: "workload identity", + credentials: map[string][]byte{ + azureClientIDSecretKey: []byte("client-id"), + azureTenantIDSecretKey: []byte("tenant-id"), + }, + expectedType: &azidentity.WorkloadIdentityCredential{}, + }, + { + name: "pod workload identity", + credentials: nil, + expectedType: &azidentity.WorkloadIdentityCredential{}, + }, + { + name: "client secret without client id", + credentials: map[string][]byte{ + azureTenantIDSecretKey: []byte("tenant-id"), + azureClientSecretSecretKey: []byte("client-secret"), + }, + expectedErr: "azuremonitor credentials does not contain a clientId", + }, + { + name: "client secret without tenant id", + credentials: map[string][]byte{ + azureClientIDSecretKey: []byte("client-id"), + azureClientSecretSecretKey: []byte("client-secret"), + }, + expectedErr: "azuremonitor credentials does not contain a tenantId", + }, + { + name: "client id without tenant id", + credentials: map[string][]byte{ + azureClientIDSecretKey: []byte("client-id"), + }, + expectedErr: "azuremonitor credentials does not contain a tenantId", + }, + { + name: "tenant id without client id", + credentials: map[string][]byte{ + azureTenantIDSecretKey: []byte("tenant-id"), + }, + expectedErr: "azuremonitor credentials does not contain a clientId", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + azureWorkloadIdentityEnv(t) + + cred, err := newAzureCredential("azuremonitor", tt.credentials, cloud.AzurePublic) + if tt.expectedErr != "" { + require.EqualError(t, err, tt.expectedErr) + return + } + require.NoError(t, err) + assert.IsType(t, tt.expectedType, cred) + }) + } +} + +func TestAzureCredentialCache(t *testing.T) { + azureWorkloadIdentityEnv(t) + + credentials := map[string][]byte{ + azureClientIDSecretKey: []byte("cache-test-client-id"), + azureTenantIDSecretKey: []byte("tenant-id"), + } + + first, err := azureCredential("azuremonitor", credentials, cloud.AzurePublic) + require.NoError(t, err) + + second, err := azureCredential("azuremonitor", credentials, cloud.AzurePublic) + require.NoError(t, err) + + assert.Same(t, first, second) + + other, err := azureCredential("azuremonitor", map[string][]byte{ + azureClientIDSecretKey: []byte("another-client-id"), + azureTenantIDSecretKey: []byte("tenant-id"), + }, cloud.AzurePublic) + require.NoError(t, err) + + assert.NotSame(t, first, other) +} + +func TestAzureCloudForHost(t *testing.T) { + tests := []struct { + host string + expectedCloud cloud.Configuration + expectedAudience string + expectedErr bool + }{ + { + host: "flagger-abcd.eastus.prometheus.monitor.azure.com", + expectedCloud: cloud.AzurePublic, + expectedAudience: "https://prometheus.monitor.azure.com", + }, + { + host: "flagger-abcd.usgovvirginia.prometheus.monitor.azure.us", + expectedCloud: cloud.AzureGovernment, + expectedAudience: "https://prometheus.monitor.azure.us", + }, + { + host: "flagger-abcd.chinaeast.prometheus.monitor.azure.cn", + expectedCloud: cloud.AzureChina, + expectedAudience: "https://prometheus.monitor.azure.cn", + }, + { + host: "FLAGGER-ABCD.EASTUS.PROMETHEUS.MONITOR.AZURE.US", + expectedCloud: cloud.AzureGovernment, + expectedAudience: "https://prometheus.monitor.azure.us", + }, + { + host: "prometheus.example.com", + expectedErr: true, + }, + { + host: "prometheus.monitor.azure.com.example.com", + expectedErr: true, + }, + { + host: "notprometheus.monitor.azure.com", + expectedErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + config, audience, err := azureCloudForHost(tt.host) + if tt.expectedErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expectedCloud, config) + assert.Equal(t, tt.expectedAudience, audience) + }) + } +} + +func TestAzureScope(t *testing.T) { + tests := []struct { + audience string + expected string + }{ + {audience: "https://prometheus.monitor.azure.com", expected: "https://prometheus.monitor.azure.com/.default"}, + {audience: "https://prometheus.monitor.azure.com/", expected: "https://prometheus.monitor.azure.com/.default"}, + {audience: "https://prometheus.monitor.azure.com/.default", expected: "https://prometheus.monitor.azure.com/.default"}, + } + + for _, tt := range tests { + t.Run(tt.audience, func(t *testing.T) { + assert.Equal(t, tt.expected, azureScope(tt.audience)) + }) + } +} diff --git a/pkg/metrics/providers/factory.go b/pkg/metrics/providers/factory.go index 21daa5004..7a8fb0518 100644 --- a/pkg/metrics/providers/factory.go +++ b/pkg/metrics/providers/factory.go @@ -56,6 +56,8 @@ func (factory Factory) Provider(metricInterval string, provider flaggerv1.Metric return NewKeptnProvider(config) case "splunk": return NewSplunkProvider(metricInterval, provider, credentials) + case "azuremonitor": + return NewAzureMonitorProvider(provider, credentials) default: factory.logger.Warnf("unknown metrics provider '%s', using prometheus", provider.Type) return NewPrometheusProvider(provider, credentials)