diff --git a/docs/faq.md b/docs/faq.md index 1b5d72ac0d..cf238b170b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -30,6 +30,7 @@ this document. Few of the answers assume that the MCM being used is in conjuctio - [How to delete machine object immedietly if I don't have access to it?](#how-to-delete-machine-object-immedietly-if-i-dont-have-access-to-it) - [How to avoid garbage collection of your node?](#how-to-avoid-garbage-collection-of-your-node) - [How to trigger rolling update of a machinedeployment?](#how-to-trigger-rolling-update-of-a-machinedeployment) + - [How to override a MachineDeployment's MachineCreationTimeout ?](#how-to-override-a-machinedeployments-machinecreationtimeout) - [Internals](#internals) - [What is the high level design of MCM?](#what-is-the-high-level-design-of-mcm) - [What are the different configuration options in MCM?](#what-are-the-different-configuration-options-in-mcm) @@ -181,6 +182,17 @@ Rolling update can be triggered for a machineDeployment by updating one of the f - `.spec.template.metadata.annotations` - `.spec.template.spec.class.name` + +### How to override a MachineDeployment's MachineCreationTimeout? + +In production, the MachineDeployment's `.Spec.Template.Spec.MachineCreationTimeout` is generally +set from an orchestrating controller during its reconcile cycle. (Ex: in gardener, this is the shoot controller.) + +As an operator, you would like to override the machine creation timeout for Machines of a bunch of MachineDeployment(s) +without touching the orchestrating controller logic or configuration. + +This can be done by setting the annotation `node.machine.sapcloud.io/effective-creation-timeout` on the `MachineDeployment`. +This will take effect for any newly launched Machine(s) belonging to this `MachineDeployment`. # Internals diff --git a/pkg/apis/machine/v1alpha1/constants.go b/pkg/apis/machine/v1alpha1/constants.go index 3ae59312aa..f0141ddb12 100644 --- a/pkg/apis/machine/v1alpha1/constants.go +++ b/pkg/apis/machine/v1alpha1/constants.go @@ -7,7 +7,13 @@ package v1alpha1 const ( // AnnotationKeyMachineUpdateFailedReason is the annotation key that indicates the reason for a machine update failure. AnnotationKeyMachineUpdateFailedReason = "node.machine.sapcloud.io/update-failed-reason" - + // AnnotationKeyMachineEffectiveCreationTimeout is the annotation key set on the MachineDeployment that indicates + // the effective creation timeout for all Machine's belonging to this MachineDeployment. If specified, the value for this + // annotation takes precedence over the MachineDeployment.Spec.Template.Spec.MachineCreationTimeout. + AnnotationKeyMachineEffectiveCreationTimeout = "node.machine.sapcloud.io/effective-creation-timeout" + // AnnotationKeyMachineJoinDuration is the annotation key set on the Machine that indicates the amount of time Machine + // took to join the cluster and is useful for diagnosis. The value is a Go Duration string. + AnnotationKeyMachineJoinDuration = "node.machine.sapcloud.io/machine-join-duration" // LabelKeyNodeCandidateForUpdate is the label key that indicates a node is a candidate for update. LabelKeyNodeCandidateForUpdate = "node.machine.sapcloud.io/candidate-for-update" // LabelKeyNodeSelectedForUpdate is the label key that indicates a node has been selected for update. diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index 4bfda891f9..238865a4aa 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -505,9 +505,13 @@ func getMachinesFinalizers(template *v1alpha1.MachineTemplateSpec) []string { return desiredFinalizers } -func getMachinesAnnotationSet(template *v1alpha1.MachineTemplateSpec, _ runtime.Object) labels.Set { +func getMachinesAnnotationSet(template *v1alpha1.MachineTemplateSpec, parentObject metav1.Object) labels.Set { desiredAnnotations := make(labels.Set) maps.Copy(desiredAnnotations, template.Annotations) + effectiveCreationTimeoutStr, ok := parentObject.GetAnnotations()[v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout] + if ok { + desiredAnnotations[v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout] = effectiveCreationTimeoutStr + } return desiredAnnotations } @@ -535,13 +539,13 @@ func GetMachineFromTemplate(template *v1alpha1.MachineTemplateSpec, parentObject desiredLabels := getMachinesLabelSet(template) //klog.Info(desiredLabels) desiredFinalizers := getMachinesFinalizers(template) - desiredAnnotations := getMachinesAnnotationSet(template, parentObject) - accessor, err := meta.Accessor(parentObject) + parentMetaObj, err := meta.Accessor(parentObject) if err != nil { return nil, fmt.Errorf("parentObject does not have ObjectMeta, %v", err) } - prefix := getMachinesPrefix(accessor.GetName()) + prefix := getMachinesPrefix(parentMetaObj.GetName()) + desiredAnnotations := getMachinesAnnotationSet(template, parentMetaObj) machine := &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ @@ -550,15 +554,25 @@ func GetMachineFromTemplate(template *v1alpha1.MachineTemplateSpec, parentObject GenerateName: prefix, Finalizers: desiredFinalizers, }, - Spec: v1alpha1.MachineSpec{ - Class: template.Spec.Class, - }, + } + machine.Spec = *template.Spec.DeepCopy() + effectiveCreationTimeout, err := annotationsutils.GetEffectiveMachineCreationTimeout(parentObject) + if err != nil { + klog.Warningf("Failed to obtain effective creation timeout from annotation %q on machine set %q due to %s", + v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout, controllerRef, err) + } + if effectiveCreationTimeout != nil { + if machine.Spec.MachineConfiguration == nil { + machine.Spec.MachineConfiguration = &v1alpha1.MachineConfiguration{MachineCreationTimeout: effectiveCreationTimeout} + } else { + machine.Spec.MachineConfiguration.MachineCreationTimeout = effectiveCreationTimeout + } + klog.V(2).Infof("MachineCreationTimeout overridden on new Machine with GenerateName %q and parent MachineSet %q to %q", + machine.ObjectMeta.GenerateName, parentMetaObj.GetName(), effectiveCreationTimeout.Duration) } if controllerRef != nil { machine.OwnerReferences = append(machine.OwnerReferences, *controllerRef) } - machine.Spec = *template.Spec.DeepCopy() - return machine, nil } @@ -696,13 +710,13 @@ func GetFakeMachineFromTemplate(template *v1alpha1.MachineTemplateSpec, parentOb desiredLabels := getMachinesLabelSet(template) desiredFinalizers := getMachinesFinalizers(template) - desiredAnnotations := getMachinesAnnotationSet(template, parentObject) - - accessor, err := meta.Accessor(parentObject) + parentMetaObj, err := meta.Accessor(parentObject) if err != nil { return nil, fmt.Errorf("parentObject does not have ObjectMeta, %v", err) } - prefix := getMachinesPrefix(accessor.GetName()) + desiredAnnotations := getMachinesAnnotationSet(template, parentMetaObj) + + prefix := getMachinesPrefix(parentMetaObj.GetName()) prefix = prefix + "-" + uuid.New().String()[:5] machine := &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/util/annotations/annotations.go b/pkg/util/annotations/annotations.go index 5fc817d0c5..57fb3799f8 100644 --- a/pkg/util/annotations/annotations.go +++ b/pkg/util/annotations/annotations.go @@ -8,10 +8,14 @@ package annotations import ( "slices" "strings" + "time" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) // AddOrUpdateAnnotation tries to add an annotation. Returns a new copy of updated Node and true if something was updated @@ -87,3 +91,21 @@ func CreateMachinesTriggeredForDeletionAnnotValue(machineNames []string) string slices.Sort(machineNames) return strings.Join(machineNames, ",") } + +// GetEffectiveMachineCreationTimeout gets the value of the annotation [v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout] +// as a [metav1.Duration] if present. +func GetEffectiveMachineCreationTimeout(object runtime.Object) (*metav1.Duration, error) { + metaObject, err := meta.Accessor(object) + if err != nil { + return nil, err + } + effectiveMachineCreationTimeoutStr, ok := metaObject.GetAnnotations()[v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout] + if !ok { + return nil, nil + } + effectiveMachineCreationTimeout, err := time.ParseDuration(effectiveMachineCreationTimeoutStr) + if err != nil { + return nil, err + } + return &metav1.Duration{Duration: effectiveMachineCreationTimeout}, nil +} diff --git a/pkg/util/annotations/annotations_test.go b/pkg/util/annotations/annotations_test.go index 6ba89fc79a..5ea80697ee 100644 --- a/pkg/util/annotations/annotations_test.go +++ b/pkg/util/annotations/annotations_test.go @@ -5,10 +5,14 @@ package annotations import ( + "time" + + "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ) var _ = Describe("annotations", func() { @@ -219,4 +223,93 @@ var _ = Describe("annotations", func() { ) }) + Describe("#GetEffectiveMachineCreationTimeout", func() { + type setup struct { + object any + } + type expect struct { + timeout *metav1.Duration + err bool + } + type data struct { + setup setup + expect expect + } + + DescribeTable("##table", + func(data *data) { + timeout, err := GetEffectiveMachineCreationTimeout(data.setup.object.(runtime.Object)) + + if data.expect.err { + Expect(err).To(HaveOccurred()) + return + } + + Expect(err).NotTo(HaveOccurred()) + Expect(timeout).To(Equal(data.expect.timeout)) + }, + + Entry("annotation is absent", &data{ + setup: setup{ + object: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + Annotations: map[string]string{}, + }, + }, + }, + expect: expect{ + timeout: nil, + err: false, + }, + }), + + Entry("annotation contains a valid duration", &data{ + setup: setup{ + object: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + Annotations: map[string]string{ + v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout: "15m30s", + }, + }, + }, + }, + expect: expect{ + timeout: &metav1.Duration{ + Duration: 15*time.Minute + 30*time.Second, + }, + err: false, + }, + }), + + Entry("annotation contains an invalid duration", &data{ + setup: setup{ + object: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + Annotations: map[string]string{ + v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout: "definitely-not-a-duration", + }, + }, + }, + }, + expect: expect{ + timeout: nil, + err: true, + }, + }), + + Entry("object does not implement metav1.Object", &data{ + setup: setup{ + object: &corev1.NodeList{}, + }, + expect: expect{ + timeout: nil, + err: true, + }, + }), + ) + }) + }) diff --git a/pkg/util/provider/drain/drain.go b/pkg/util/provider/drain/drain.go index 2dba1a1966..2c99541dd1 100644 --- a/pkg/util/provider/drain/drain.go +++ b/pkg/util/provider/drain/drain.go @@ -1187,6 +1187,11 @@ func (o *Options) RunCordonOrUncordon(ctx context.Context, desired bool) error { return nil } +// GetDrainDuration gets the duration of drain activity +func (o *Options) GetDrainDuration() time.Duration { + return o.drainEndedOn.Sub(o.drainStartedOn) +} + func getPdbForPod(pdbLister policyv1listers.PodDisruptionBudgetLister, pod *corev1.Pod) *policyv1.PodDisruptionBudget { // GetPodPodDisruptionBudgets returns an error only if no PodDisruptionBudgets are found. // We don't return that as an error to the caller. diff --git a/pkg/util/provider/machinecontroller/controller.go b/pkg/util/provider/machinecontroller/controller.go index 896e43446c..3292b8b48e 100644 --- a/pkg/util/provider/machinecontroller/controller.go +++ b/pkg/util/provider/machinecontroller/controller.go @@ -301,8 +301,7 @@ type controller struct { machineClassSynced cache.InformerSynced machineSynced cache.InformerSynced podSynced cache.InformerSynced - - resourceExhaustedRetry machineutils.RetryPeriod + resourceExhaustedRetry machineutils.RetryPeriod } func (dc *controller) Run(workers int, stopCh <-chan struct{}) { diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index fc05c38c76..e785f77059 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/gardener/machine-controller-manager/pkg/util/provider/metrics" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -327,6 +328,7 @@ func (c *controller) triggerCreationFlow(ctx context.Context, createMachineReque var ( // Declarations nodeName, providerID string + createDuration time.Duration // Initializations machine = createMachineRequest.Machine @@ -392,7 +394,9 @@ func (c *controller) triggerCreationFlow(ctx context.Context, createMachineReque providerID = createMachineResponse.ProviderID addresses.Insert(createMachineResponse.Addresses...) // Creation was successful - klog.V(2).Infof("Created new VM for machine: %q with ProviderID: %q and backing node: %q", machine.Name, providerID, nodeName) + createDuration = time.Since(machine.CreationTimestamp.Time) + klog.V(2).Infof("Created new VM for machine: %q with ProviderID: %q and backing node: %q, createDuration: %s", machine.Name, providerID, nodeName, createDuration) + metrics.UpdateMetricsForMachineDurations(machine, createMachineRequest.MachineClass, metrics.MachineDurations{Create: createDuration}) if c.nodeLister != nil { // If a node obj already exists by the same nodeName, treat it as a stale node and trigger machine deletion. @@ -489,12 +493,19 @@ func (c *controller) triggerCreationFlow(ctx context.Context, createMachineReque } //initialize VM if not initialized if uninitializedMachine { - var retryPeriod machineutils.RetryPeriod - var initResponse *driver.InitializeMachineResponse + var ( + retryPeriod machineutils.RetryPeriod + initResponse *driver.InitializeMachineResponse + initializeBeginTime = time.Now() + initializeDuration time.Duration + ) initResponse, retryPeriod, err = c.initializeMachine(ctx, clone, createMachineRequest.MachineClass, createMachineRequest.Secret) if err != nil { return retryPeriod, err } + initializeDuration = time.Since(initializeBeginTime) + klog.V(2).Infof("Machine %q was initialized in %q", machine.Name, initializeDuration) + metrics.UpdateMetricsForMachineDurations(machine, createMachineRequest.MachineClass, metrics.MachineDurations{Initialize: initializeDuration}) if c.targetCoreClient == nil { // persist addresses from the InitializeMachine and CreateMachine responses diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index 7c08640b5c..6a22832bf8 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -35,6 +35,8 @@ import ( "strings" "time" + annotationsutils "github.com/gardener/machine-controller-manager/pkg/util/annotations" + machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" "github.com/gardener/machine-controller-manager/pkg/util/nodeops" @@ -43,6 +45,7 @@ import ( "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/codes" "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/status" "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" + "github.com/gardener/machine-controller-manager/pkg/util/provider/metrics" utilstrings "github.com/gardener/machine-controller-manager/pkg/util/strings" utiltime "github.com/gardener/machine-controller-manager/pkg/util/time" @@ -923,7 +926,10 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph description string lastOperationType v1alpha1.MachineOperationType ) - + machineClass, err := c.machineClassLister.MachineClasses(c.namespace).Get(machine.Spec.Class.Name) + if err != nil { + klog.Warningf("unable to get MachineClass %q for Machine %q: %v", machine.Spec.Class.Name, machine.Name, err) + } node, err := c.nodeLister.Get(machine.Labels[v1alpha1.NodeLabelKey]) if err != nil { if !apierrors.IsNotFound(err) { @@ -973,8 +979,9 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph if clone.Status.CurrentStatus.Phase != v1alpha1.MachineRunning && !isPendingMachineWithCriticalComponentsNotReadyTaint(clone, node) { if clone.Status.LastOperation.Type == v1alpha1.MachineOperationCreate && clone.Status.LastOperation.State != v1alpha1.MachineStateSuccessful { + joinDuration := time.Since(machine.CreationTimestamp.Time) // When machine creation went through - description = fmt.Sprintf("Machine %s successfully joined the cluster", clone.Name) + description = fmt.Sprintf("Machine %s successfully joined the cluster in %s", clone.Name, joinDuration) lastOperationType = v1alpha1.MachineOperationCreate // Delete the bootstrap token @@ -982,6 +989,9 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph if err != nil { klog.Warning(err) } + klog.V(2).Infof("Machine %q joined the cluster in %q", machine.Name, joinDuration) + metrics.UpdateMetricsForMachineDurations(machine, machineClass, metrics.MachineDurations{Join: joinDuration}) + metav1.SetMetaDataAnnotation(&clone.ObjectMeta, v1alpha1.AnnotationKeyMachineJoinDuration, joinDuration.String()) } else { // Machine rejoined the cluster after a health-check description = fmt.Sprintf("Machine %s successfully re-joined the cluster", clone.Name) @@ -1099,7 +1109,7 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph timeOutDuration, machine.Status.Conditions, ) - machineDeployName := getMachineDeploymentName(machine) + machineDeployName := machineutils.GetMachineDeploymentName(machine) // creating lock for machineDeployment, if not allocated c.permitGiver.RegisterPermits(machineDeployName, 1) return c.tryMarkingMachineFailed(ctx, machine, clone, machineDeployName, description, lockAcquireTimeout) @@ -1147,6 +1157,9 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph PreserveExpiryTime: machine.Status.CurrentStatus.PreserveExpiryTime, } cloneDirty = true + if machineClass != nil { + metrics.IncrementNumFailedToJoin(machine, machineClass) + } } } else { // If timeout has not occurred, re-enqueue the machine @@ -1544,6 +1557,7 @@ func (c *controller) drainNodeForInPlace(ctx context.Context, machine *v1alpha1. description = fmt.Sprintf("Drain successful. %s", machineutils.NodeReadyForUpdate) } state = v1alpha1.MachineStateProcessing + metrics.UpdateMetricsForMachineDurations(machine, nil, metrics.MachineDurations{Drain: drainOptions.GetDrainDuration()}) } else { klog.Warningf("Drain failed for machine %q , providerID %q ,backing node %q. \nBuf:%v \nErrBuf:%v \nErr-Message:%v", machine.Name, getProviderID(machine), getNodeName(machine), buf, errBuf, err) @@ -1708,6 +1722,7 @@ func (c *controller) drainNode(ctx context.Context, deleteMachineRequest *driver } err = fmt.Errorf("%s", description) state = v1alpha1.MachineStateProcessing + metrics.UpdateMetricsForMachineDurations(machine, nil, metrics.MachineDurations{Drain: drainOptions.GetDrainDuration()}) // Return error even when machine object is updated } else if err != nil && forceDeleteMachine { @@ -1820,6 +1835,7 @@ func (c *controller) deleteVM(ctx context.Context, deleteMachineRequest *driver. description string state v1alpha1.MachineState lastKnownState string + deleteDuration time.Duration ) deleteMachineResponse, err := c.driver.DeleteMachine(ctx, deleteMachineRequest) @@ -1853,6 +1869,11 @@ func (c *controller) deleteVM(ctx context.Context, deleteMachineRequest *driver. description = fmt.Sprintf("VM deletion was successful. %s", machineutils.InitiateNodeDeletion) state = v1alpha1.MachineStateProcessing + if machine.DeletionTimestamp != nil { + deleteDuration = time.Since(machine.DeletionTimestamp.Time) + metrics.UpdateMetricsForMachineDurations(machine, deleteMachineRequest.MachineClass, metrics.MachineDurations{Delete: deleteDuration}) + } + err = fmt.Errorf("Machine deletion in process. %s", description) } @@ -2014,11 +2035,20 @@ func (c *controller) getEffectiveHealthTimeout(machine *v1alpha1.Machine) *metav // getEffectiveCreationTimeout returns the creationTimeout set on the machine-object, otherwise returns the timeout set using the global-flag. func (c *controller) getEffectiveCreationTimeout(machine *v1alpha1.Machine) *metav1.Duration { var effectiveCreationTimeout *metav1.Duration + effectiveCreationTimeout, err := annotationsutils.GetEffectiveMachineCreationTimeout(machine) + if effectiveCreationTimeout != nil { + return effectiveCreationTimeout + } + if err != nil { + klog.Warningf("error obtaining effectiveCreationTimeout from annotation %q (if any) on machine %q: %v", + v1alpha1.AnnotationKeyMachineEffectiveCreationTimeout, machine.Name, err) + } if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineCreationTimeout != nil { effectiveCreationTimeout = machine.Spec.MachineCreationTimeout } else { effectiveCreationTimeout = &c.safetyOptions.MachineCreationTimeout } + klog.V(4).Infof("obtained effectiveCreationTimeout %q for machine %q", effectiveCreationTimeout, machine.Name) return effectiveCreationTimeout } @@ -2280,10 +2310,6 @@ func getNodeName(machine *v1alpha1.Machine) string { return machine.Labels[v1alpha1.NodeLabelKey] } -func getMachineDeploymentName(machine *v1alpha1.Machine) string { - return machine.Labels["name"] -} - func (c *controller) updateMachineNodeLabel(ctx context.Context, machine *v1alpha1.Machine, nodeName string) error { klog.V(2).Infof("Updating %q label on machine %q to %q", v1alpha1.NodeLabelKey, machine.Name, nodeName) clone := machine.DeepCopy() @@ -2675,5 +2701,6 @@ func (c *controller) drainPreservedNode(ctx context.Context, machine *v1alpha1.M } else { klog.V(3).Infof("Drain successful for machine %q , providerID %q ,backing node %q.", machine.Name, getProviderID(machine), getNodeName(machine)) } + metrics.UpdateMetricsForMachineDurations(machine, nil, metrics.MachineDurations{Drain: drainOptions.GetDrainDuration()}) return nil } diff --git a/pkg/util/provider/machineutils/utils.go b/pkg/util/provider/machineutils/utils.go index b56da7cc23..4fe47258ee 100644 --- a/pkg/util/provider/machineutils/utils.go +++ b/pkg/util/provider/machineutils/utils.go @@ -7,6 +7,8 @@ package machineutils import ( "context" + "time" + "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" v1alpha1client "github.com/gardener/machine-controller-manager/pkg/client/clientset/versioned/typed/machine/v1alpha1" v1alpha1listers "github.com/gardener/machine-controller-manager/pkg/client/listers/machine/v1alpha1" @@ -16,7 +18,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" - "time" ) const ( @@ -174,6 +175,11 @@ func IsMachinePreservationExpired(m *v1alpha1.Machine) bool { return t != nil && !t.After(time.Now()) } +// GetMachineDeploymentName gets the name of the MachineDeployment associated with this Machine +func GetMachineDeploymentName(machine *v1alpha1.Machine) string { + return machine.Labels["name"] +} + // see https://github.com/kubernetes/kubernetes/issues/21479 type updateMachineFunc func(machine *v1alpha1.Machine) error diff --git a/pkg/util/provider/metrics/metrics.go b/pkg/util/provider/metrics/metrics.go index b44aed9a0a..88df6a5fd4 100644 --- a/pkg/util/provider/metrics/metrics.go +++ b/pkg/util/provider/metrics/metrics.go @@ -5,7 +5,14 @@ package metrics import ( + "time" + + "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" ) const ( @@ -46,6 +53,60 @@ var ( Name: "status_condition", Help: "Information of the mcm managed Machines' status conditions.", }, []string{"name", "namespace", "condition"}) + + // MachineCreateDurationSeconds is the Prometheus gauge metric representing the time duration + // in seconds to create a Machine of a MachineDeployment from the time of Machine creation. + MachineCreateDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "create_duration_seconds", + Help: "Duration in seconds to create a Machine of a MachineDeployment.", + }, []string{"namespace", "machine_deployment", "instance_type", "zone"}) + + // MachineInitializeDurationSeconds is the Prometheus gauge metric representing the time duration + // in seconds to initialize a Machine of a MachineDeployment. + MachineInitializeDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "initialize_duration_seconds", + Help: "Duration in seconds to initialize a Machine of a MachineDeployment.", + }, []string{"namespace", "machine_deployment", "instance_type", "zone"}) + + // MachineJoinDurationSeconds is the Prometheus gauge metric representing the time duration + // in seconds for a Machine of a MachineDeployment to join the cluster from the time of Machine creation. + MachineJoinDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "join_duration_seconds", + Help: "Duration in seconds for a Machine of a MachineDeployment to join the cluster", + }, []string{"namespace", "machine_deployment", "instance_type", "zone"}) + + // MachineDrainDurationSeconds is the Prometheus gauge metric representing the time duration + // in seconds to drain a Machine of a MachineDeployment. + MachineDrainDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "drain_duration_seconds", + Help: "Duration in seconds to drain a Machine of a MachineDeployment.", + }, []string{"namespace", "machine_deployment"}) + + // MachineDeleteDurationSeconds is the Prometheus gauge metric representing the time duration + // in seconds to delete a Machine for a MachineDeployment. + MachineDeleteDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "delete_duration_seconds", + Help: "Duration in seconds to delete a Machine of a MachineDeployment.", + }, []string{"namespace", "machine_deployment", "instance_type", "zone"}) + + // MachineNumFailedJoin is the Prometheus counter metric representing the number of machines that + // failed to join the cluster for a MachineDeployment. + MachineNumFailedJoin = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: machineSubsystem, + Name: "num_failed_join", + Help: "Number of Machines of a MachineDeployment that failed to join the cluster.", + }, []string{"namespace", "machine_deployment", "instance_type", "zone"}) ) // variables for subsystem: cloud_api @@ -108,9 +169,78 @@ var ( }, []string{"kind"}) ) +// MachineDurations encapsulates duration data for a machine and used as input to update prometheus metrics. +type MachineDurations struct { + // Create is the duration to create a Machine from the time of its creation timestamp. + Create time.Duration + // Initialize is the duration to initialize a Machine after creation. + Initialize time.Duration + // Join is the duration for a Machine to join the cluster ie associated Node becomes Ready + Join time.Duration + // Drain is the duration for the machine-controller to drain the Machine or rather the Node associated with a Machine + Drain time.Duration + // Delete is the duration for the machine-controller to delete the Machine from the time its deletion timestamp was set. + Delete time.Duration +} + +// UpdateMetricsForMachineDurations updates the prometheus metrics relevant for machine activity durations such as +// create/initialize/join/drain/delete using the values specified in the given [MachineDurations] object. +func UpdateMetricsForMachineDurations(machine *v1alpha1.Machine, machineClass *v1alpha1.MachineClass, newDurations MachineDurations) { + metricLabels := getDeploymentLabels(machine) + if metricLabels == nil { + return + } + addPlacementLabels(metricLabels, machine, machineClass) + metricLabelsStr := labels.FormatLabels(metricLabels) + if newDurations.Create != 0 { + numSecs := newDurations.Create.Round(time.Second).Seconds() + MachineCreateDurationSeconds.With(metricLabels).Set(numSecs) + klog.V(3).Infof("updated machine_create_duration_seconds metric to %f with labels %s", numSecs, metricLabelsStr) + } + if newDurations.Initialize != 0 { + numSecs := newDurations.Initialize.Round(time.Second).Seconds() + MachineInitializeDurationSeconds.With(metricLabels).Set(numSecs) + klog.V(3).Infof("updated machine_initialize_duration_seconds metric to %f with labels %s", numSecs, metricLabelsStr) + } + if newDurations.Join != 0 { + numSecs := newDurations.Join.Round(time.Second).Seconds() + MachineJoinDurationSeconds.With(metricLabels).Set(numSecs) + klog.V(3).Infof("updated machine_join_duration_seconds metric to %f with labels %s", numSecs, metricLabelsStr) + } + if newDurations.Drain != 0 { + numSecs := newDurations.Drain.Round(time.Second).Seconds() + MachineDrainDurationSeconds.With(metricLabels).Set(numSecs) + klog.V(3).Infof("updated machine_drain_duration_seconds metric to %f with labels %s", numSecs, metricLabelsStr) + } + if newDurations.Delete != 0 { + numSecs := newDurations.Delete.Round(time.Second).Seconds() + MachineDeleteDurationSeconds.With(metricLabels).Set(numSecs) + klog.V(3).Infof("updated machine_delete_duration_seconds metric to %f with labels %s", numSecs, metricLabelsStr) + } +} + +// IncrementNumFailedToJoin increments the prometheus metric for the number of machines that failed to join the cluster, +// deriving label classifiers from the given Machine and MachineClass +func IncrementNumFailedToJoin(machine *v1alpha1.Machine, machineClass *v1alpha1.MachineClass) { + metricLabels := getDeploymentLabels(machine) + if metricLabels == nil { + return + } + addPlacementLabels(metricLabels, machine, machineClass) + metricLabelsStr := labels.FormatLabels(metricLabels) + MachineNumFailedJoin.With(metricLabels).Inc() + klog.V(3).Infof("incremented num_failed_join metric due to %q with labels %s", machine.Name, metricLabelsStr) +} + func registerMachineSubsystemMetrics() { prometheus.MustRegister(MachineInfo) prometheus.MustRegister(MachineStatusCondition) + prometheus.MustRegister(MachineCreateDurationSeconds) + prometheus.MustRegister(MachineInitializeDurationSeconds) + prometheus.MustRegister(MachineJoinDurationSeconds) + prometheus.MustRegister(MachineDrainDurationSeconds) + prometheus.MustRegister(MachineDeleteDurationSeconds) + prometheus.MustRegister(MachineNumFailedJoin) } func registerCloudAPISubsystemMetrics() { @@ -125,6 +255,36 @@ func registerMiscellaneousMetrics() { prometheus.MustRegister(ScrapeFailedCounter) } +// getDeploymentLabels returns prometheus Labels populated with machine deployment related labels or nil if labels +// cannot be populated. Presently, this is the machine deployment name and namespace. +func getDeploymentLabels(machine *v1alpha1.Machine) prometheus.Labels { + mcdName := machineutils.GetMachineDeploymentName(machine) + if mcdName == "" { + klog.Warningf("Machine %q does not possess 'name' label which is its MachineDeployment name", machine.Name) + return nil + } + metricLabels := prometheus.Labels{ + "namespace": machine.GetNamespace(), + "machine_deployment": mcdName, + } + return metricLabels +} + +// addPlacementLabels adds metric labels related to placement, currently instance_type and zone to the given prometheus metricLabels. +func addPlacementLabels(metricLabels prometheus.Labels, machine *v1alpha1.Machine, machineClass *v1alpha1.MachineClass) { + if machine == nil || machineClass == nil { + return + } + if machineClass.NodeTemplate == nil { + klog.Warningf("MachineClass %q does not have NodeTemplate", machineClass.Name) + return + } + instanceType := machineClass.NodeTemplate.InstanceType + zone := machine.Spec.NodeTemplateSpec.Labels[corev1.LabelTopologyZone] + metricLabels["instance_type"] = instanceType + metricLabels["zone"] = zone +} + func init() { registerMachineSubsystemMetrics() registerCloudAPISubsystemMetrics()