233 do not upgrade statefulset when only replicas differ (#236)

* scale only when only the replicas have changed #233

* extract image tag correct #233

* extend triggers logging to quorum up/down-grades #233

* bump charts, manifests and versions #233

Author: akyriako

Makefile

@@ -52,7 +52,7 @@ OPERATOR_SDK_VERSION ?= v1.38.0
 # Image URL to use all building/pushing image targets
 DOCKER_HUB_NAME ?= $(shell docker info | sed '/Username:/!d;s/.* //')
 IMG_NAME ?= typesense-operator
-IMG_TAG ?= 0.3.8-rc.2
+IMG_TAG ?= 0.3.8-rc.3
 IMG ?= $(DOCKER_HUB_NAME)/$(IMG_NAME):$(IMG_TAG)
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.

README.md

@@ -228,7 +228,7 @@ Join the conversation:
 ## 📦 Project Status
 
 TyKO is an **independently maintained** project (not affiliated with Typesense, Inc.).
-- Latest version: **0.3.8-rc.2**
+- Latest version: **0.3.8-rc.3**
 - Tested on: Kubernetes 1.35 (earliest 1.26), Typesense 30.0 (earliest 26.0)
 - Contributions welcome! See [FAQ](https://akyriako.github.io/typesense-operator-docs/docs/faq) and [Development](https://akyriako.github.io/typesense-operator-docs/docs/development)
 

charts/typesense-operator/Chart.yaml

@@ -14,12 +14,12 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.8-rc.2
+version: 0.3.8-rc.3
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.3.8-rc.2"
+appVersion: "0.3.8-rc.3"
 
 maintainers:
   - name: Kyriakos Akriotis

charts/typesense-operator/values.yaml

@@ -12,7 +12,7 @@ controllerManager:
         - ALL
     image:
       repository: akyriako78/typesense-operator
-      tag: 0.3.8-rc.2
+      tag: 0.3.8-rc.3
     imagePullPolicy: IfNotPresent
     resources:
       limits:

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: akyriako78/typesense-operator
-  newTag: 0.3.8-rc.2
+  newTag: 0.3.8-rc.3

internal/controller/typesensecluster_quorum.go

@@ -216,14 +216,20 @@ func (r *TypesenseClusterReconciler) ReconcileQuorum(ctx context.Context, ts *ts
 	return ConditionReasonQuorumReady, 0, nil
 }
 
+var (
+	QuorumDowngraded UpdateStatefulSetTrigger = "QuorumDowngraded"
+	QuorumUpgraded   UpdateStatefulSetTrigger = "QuorumUpgraded"
+)
+
 func (r *TypesenseClusterReconciler) downgradeQuorum(
 	ctx context.Context,
 	ts *tsv1alpha1.TypesenseCluster,
 	cm *v1.ConfigMap,
 	stsObjectKey client.ObjectKey,
 	healthyNodes, minRequiredNodes int32,
 ) (ConditionQuorum, int, error) {
-	r.logger.Info("downgrading quorum")
+	//r.logger.Info("downgrading quorum")
+	r.logger.V(debugLevel).Info("scaling statefulset", "sts", stsObjectKey.Name, "triggers", QuorumDowngraded)
 
 	sts, err := r.GetFreshStatefulSet(ctx, stsObjectKey)
 	if err != nil {
@@ -264,7 +270,8 @@ func (r *TypesenseClusterReconciler) upgradeQuorum(
 	cm *v1.ConfigMap,
 	stsObjectKey client.ObjectKey,
 ) (ConditionQuorum, int, error) {
-	r.logger.Info("upgrading quorum", "incremental", ts.Spec.IncrementalQuorumRecovery)
+	//r.logger.Info("upgrading quorum", "incremental", ts.Spec.IncrementalQuorumRecovery)
+	r.logger.V(debugLevel).Info("scaling statefulset", "sts", stsObjectKey.Name, "triggers", QuorumUpgraded, "incremental", ts.Spec.IncrementalQuorumRecovery)
 
 	sts, err := r.GetFreshStatefulSet(ctx, stsObjectKey)
 	if err != nil {

internal/controller/typesensecluster_statefulset.go

@@ -90,17 +90,17 @@ func (r *TypesenseClusterReconciler) ReconcileStatefulSet(ctx context.Context, t
 					r.logger.Error(err, "building statefulset failed", "sts", stsObjectKey.Name)
 				}
 
-				update, triggers := r.shouldUpdateStatefulSet(sts, desiredSts, ts)
+				update, scaleOnly, triggers := r.shouldUpdateStatefulSet(sts, desiredSts, ts)
 				if update {
-					r.logger.V(debugLevel).Info("updating statefulset", "sts", sts.Name, "triggers", triggers)
-
-					oldImage := strings.Replace(sts.Spec.Template.Spec.Containers[0].Image, "typesense/typesense:", "", -1)
-					newImage := strings.Replace(desiredSts.Spec.Template.Spec.Containers[0].Image, "typesense/typesense:", "", -1)
+					oldImage := getImageTag(sts.Spec.Template.Spec.Containers[0].Image)
+					newImage := getImageTag(desiredSts.Spec.Template.Spec.Containers[0].Image)
 					if oldImage != newImage {
-						r.logger.V(debugLevel).Info("scheduling typesense update", "current", oldImage, "target", newImage)
+						triggers = append(triggers, SpecTypesenseVersionChanged)
 						r.Recorder.Eventf(ts, "Normal", "TypesenseVersionUpdate", "Scheduled update from %s to %s", oldImage, newImage)
 					}
 
+					r.logger.V(debugLevel).Info("updating statefulset", "sts", sts.Name, "triggers", triggers)
+
 					updatedSts, err := r.updateStatefulSet(ctx, sts, desiredSts)
 					if err != nil {
 						r.logger.Error(err, "updating statefulset failed", "sts", stsObjectKey.Name)
@@ -126,6 +126,33 @@ func (r *TypesenseClusterReconciler) ReconcileStatefulSet(ctx context.Context, t
 
 					r.logLagThresholds(updatedSts)
 					return updatedSts, nil
+				} else if !update && scaleOnly {
+					r.logger.V(debugLevel).Info("scaling statefulset", "sts", sts.Name, "triggers", triggers)
+
+					size := ts.Spec.Replicas
+					err = r.ScaleStatefulSet(ctx, stsObjectKey, size)
+					if err != nil {
+						return desiredSts, nil
+					}
+
+					configMapName := fmt.Sprintf(ClusterNodesConfigMap, ts.Name)
+					configMapObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: configMapName}
+
+					var cm = &corev1.ConfigMap{}
+					if err := r.Get(ctx, configMapObjectKey, cm); err != nil {
+						r.logger.V(debugLevel).Error(err, fmt.Sprintf("unable to fetch config map: %s", configMapName))
+					}
+					_, _, updated, err := r.updateConfigMap(ctx, ts, cm, &size, true)
+					if err != nil {
+						return desiredSts, nil
+					}
+
+					if updated && ts.Spec.ForceResetPeersConfigOnUpdate {
+						_ = r.forcePodsConfigMapUpdate(ctx, ts)
+					}
+
+					r.logLagThresholds(desiredSts)
+					return desiredSts, nil
 				}
 			}
 		}
@@ -520,33 +547,35 @@ func (r *TypesenseClusterReconciler) buildStatefulSet(ctx context.Context, key c
 type UpdateStatefulSetTrigger string
 
 var (
-	BelowSpecReplicas               UpdateStatefulSetTrigger = "BelowSpecReplicas"
+	SpecReplicasChanged             UpdateStatefulSetTrigger = "SpecReplicasChanged"
 	HashAnnotationChanged           UpdateStatefulSetTrigger = "HashAnnotationChanged"
 	PodAnnotationsChanged           UpdateStatefulSetTrigger = "PodAnnotationsChanged"
 	StatefulSetAnnotationsChanged   UpdateStatefulSetTrigger = "StatefulSetAnnotationsChanged"
 	SpecResourcesChanged            UpdateStatefulSetTrigger = "SpecResourcesChanged"
 	PodSecurityContextChanged       UpdateStatefulSetTrigger = "PodSecurityContextChanged"
 	InvalidContainerCount           UpdateStatefulSetTrigger = "InvalidContainerCount"
 	ContainerSecurityContextChanged UpdateStatefulSetTrigger = "ContainerSecurityContextChanged"
+	SpecTypesenseVersionChanged     UpdateStatefulSetTrigger = "SpecTypesenseVersionChanged"
 )
 
-func (r *TypesenseClusterReconciler) shouldUpdateStatefulSet(sts *appsv1.StatefulSet, desired *appsv1.StatefulSet, ts *tsv1alpha1.TypesenseCluster) (update bool, triggers []UpdateStatefulSetTrigger) {
+func (r *TypesenseClusterReconciler) shouldUpdateStatefulSet(sts *appsv1.StatefulSet, desired *appsv1.StatefulSet, ts *tsv1alpha1.TypesenseCluster) (update bool, scaleOnly bool, triggers []UpdateStatefulSetTrigger) {
 	update = false
+	scaleOnly = false
 
 	if sts == nil || ts == nil {
-		return false, nil
+		return false, false, nil
 	}
 
 	condition := r.getConditionReady(ts)
 	if condition == nil {
-		return false, nil
+		return false, false, nil
 	}
 
-	// BelowSpecReplicas
+	// SpecReplicasChanged
 	if *sts.Spec.Replicas != ts.Spec.Replicas &&
 		(condition.Reason != string(ConditionReasonQuorumDowngraded) || condition.Reason != string(ConditionReasonQuorumQueuedWrites)) {
-		triggers = append(triggers, BelowSpecReplicas)
-		update = true
+		triggers = append(triggers, SpecReplicasChanged)
+		scaleOnly = true
 	}
 
 	// HashAnnotationChanged
@@ -606,7 +635,7 @@ func (r *TypesenseClusterReconciler) shouldUpdateStatefulSet(sts *appsv1.Statefu
 		update = true
 	}
 
-	return update, triggers
+	return update, scaleOnly, triggers
 }
 
 func (r *TypesenseClusterReconciler) shouldEmergencyUpdateStatefulSet(sts *appsv1.StatefulSet, ts *tsv1alpha1.TypesenseCluster) bool {

internal/controller/utils.go

@@ -211,3 +211,11 @@ func filterAnnotations(annotations map[string]string, filters ...string) map[str
 
 	return filtered
 }
+
+func getImageTag(image string) string {
+	pos := strings.LastIndex(image, ":")
+	if pos == -1 {
+		return image
+	}
+	return image[pos+1:]
+}