From 2753f4e2b2426dba51c19595189fa64c26155979 Mon Sep 17 00:00:00 2001
From: Johannes Scheuermann <johscheuer@users.noreply.github.com>
Date: Thu, 4 Jul 2024 12:50:31 +0200
Subject: [PATCH] fdb-kubernetes-monitor: Ensure that the annotations are
 updated when the correct configuration is already loaded (#11486)

---
 fdbkubernetesmonitor/kubernetes.go | 26 ++++++++++++++++----------
 fdbkubernetesmonitor/monitor.go    | 11 ++++++-----
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/fdbkubernetesmonitor/kubernetes.go b/fdbkubernetesmonitor/kubernetes.go
index d6c9bb127c..2cf3f17c1d 100644
--- a/fdbkubernetesmonitor/kubernetes.go
+++ b/fdbkubernetesmonitor/kubernetes.go
@@ -239,11 +239,6 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 	if podClient.podMetadata == nil {
 		return fmt.Errorf("pod client has no metadata present")
 	}
-	
-	annotations := podClient.podMetadata.Annotations
-	if len(annotations) == 0 {
-		annotations = map[string]string{}
-	}
 
 	if !podClient.podMetadata.DeletionTimestamp.IsZero() {
 		return fmt.Errorf("pod is marked for deletion, cannot update annotations")
@@ -264,13 +259,24 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 
 		return true
 	}, func() error {
-		annotations := podClient.podMetadata.Annotations
-		if len(annotations) == 0 {
-			annotations = map[string]string{}
+		currentAnnotations := podClient.podMetadata.Annotations
+		if len(currentAnnotations) == 0 {
+			currentAnnotations = map[string]string{}
 		}
 
+		var hasChanges bool
 		for key, val := range annotationChanges {
-			annotations[key] = val
+			currentValue, present := currentAnnotations[key]
+			if !present || currentValue != val {
+				podClient.Logger.Info("update annotation with new value", "annotation", key, "currentValue", currentValue, "newValue", val, "present", present)
+				currentAnnotations[key] = val
+				hasChanges = true
+			}
+		}
+
+		// If no changes are present, we can skip the patch.
+		if !hasChanges {
+			return nil
 		}
 
 		return podClient.Patch(context.Background(), &corev1.Pod{
@@ -281,7 +287,7 @@ func (podClient *kubernetesClient) updateAnnotationsOnPod(annotationChanges map[
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace:   podClient.podMetadata.Namespace,
 				Name:        podClient.podMetadata.Name,
-				Annotations: annotations,
+				Annotations: currentAnnotations,
 			},
 		}, client.Apply, client.FieldOwner("fdb-kubernetes-monitor"), client.ForceOwnership)
 	})
diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go
index 82cb283b5a..908e6d665b 100644
--- a/fdbkubernetesmonitor/monitor.go
+++ b/fdbkubernetesmonitor/monitor.go
@@ -283,6 +283,12 @@ func (monitor *monitor) loadConfiguration() {
 	}
 
 	monitor.acceptConfiguration(configuration, configurationBytes)
+	// Always update the annotations if needed to handle cases where the fdb-kubernetes-monitor
+	// has loaded the new configuration but was not able to update the annotations.
+	err := monitor.podClient.updateAnnotations(monitor)
+	if err != nil {
+		monitor.logger.Error(err, "Error updating pod annotations")
+	}
 }
 
 // checkOwnerExecutable validates that a path is a file that exists and is
@@ -332,11 +338,6 @@ func (monitor *monitor) acceptConfiguration(configuration *api.ProcessConfigurat
 	if hasRunningProcesses && !monitor.activeConfiguration.ShouldRunServers() {
 		monitor.sendSignalToProcesses(syscall.SIGTERM)
 	}
-
-	err := monitor.podClient.updateAnnotations(monitor)
-	if err != nil {
-		monitor.logger.Error(err, "Error updating pod annotations")
-	}
 }
 
 // getBackoffDuration returns the backoff duration. The backoff time will increase exponential with a maximum of 60 seconds.