Harden primary protection recovery

Require a full healthy window after process or leader startup, expose the blocking metric in check responses, and publish recovery latch state.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 0fbc396f-32f9-449a-8537-6264e8665c31

Author: pradeepkintali

doc/mysql.md

@@ -203,7 +203,9 @@ The ProxySQL query above is illustrative: deployments must provide a scalar quer
 - `FailOnNoHosts` changes the legacy no-host response from `HTTP 200` to `HTTP 500`. Enable it for primary and ProxySQL safety probes, where an empty roster cannot prove that work is safe.
 - `RecoveryThreshold` is the lower threshold that begins recovery after a cluster has throttled. It must not exceed `ThrottleThreshold`.
 - `RecoveryDurationMillis` is the continuous time the metric must remain at or below `RecoveryThreshold` before checks return `HTTP 200` again. A new error or threshold violation resets the recovery window.
+- A new process or leader starts configured recovery metrics in the recovering state. It must observe a complete healthy window before admitting work.
 - Metric sampling remains centralized in the `freno` leader and uses the existing metric cache. Application checks read the aggregated decision rather than querying the primary or ProxySQL for every batch.
+- Check responses include `MetricName`, identifying the replica, primary, or ProxySQL metric that blocked a composite decision. `recovery.mysql.<cluster>.active` reports whether the recovery latch is active.
 
 Configuration loading rejects missing `RequiredClusters` references and dependency cycles.
 

pkg/throttle/check.go

@@ -103,7 +103,9 @@ func (check *ThrottlerCheck) checkAppMetricResult(appName string, storeType stri
 		// all good!
 		statusCode = http.StatusOK // 200
 	}
-	return NewCheckResult(statusCode, value, threshold, err)
+	checkResult = NewCheckResult(statusCode, value, threshold, err)
+	checkResult.MetricName = metricName
+	return checkResult
 }
 
 func (check *ThrottlerCheck) checkMySQLCluster(
@@ -134,10 +136,12 @@ func (check *ThrottlerCheck) checkMySQLCluster(
 			continue
 		}
 		if requiredResult.StatusCode == http.StatusNotFound {
-			return NewErrorCheckResult(
+			checkResult := NewErrorCheckResult(
 				http.StatusInternalServerError,
 				fmt.Errorf("required MySQL cluster metric %q is unavailable", requiredCluster),
 			)
+			checkResult.MetricName = fmt.Sprintf("mysql/%s", requiredCluster)
+			return checkResult
 		}
 		return requiredResult
 	}

pkg/throttle/check_result.go

@@ -11,6 +11,7 @@ type CheckResult struct {
 	StatusCode int     `json:"StatusCode"`
 	Value      float64 `json:"Value"`
 	Threshold  float64 `json:"Threshold"`
+	MetricName string  `json:"MetricName,omitempty"`
 	Error      error   `json:"-"`
 	Message    string  `json:"Message"`
 }

pkg/throttle/throttler.go

@@ -486,16 +486,21 @@ func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResul
 
 	state, ok := throttler.mysqlRecoveryState[clusterName]
 	if !ok {
-		state = &mysqlClusterRecoveryState{}
+		// A new process or leader has no recovery history. Require a complete
+		// healthy window before admitting work rather than assuming the metric
+		// was healthy before this process began observing it.
+		state = &mysqlClusterRecoveryState{throttled: true}
 		throttler.mysqlRecoveryState[clusterName] = state
 	}
 
 	if err != nil || value > clusterSettings.ThrottleThreshold {
 		state.throttled = true
 		state.healthySince = time.Time{}
+		metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1)
 		return metricResult
 	}
 	if !state.throttled {
+		metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(0)
 		return metricResult
 	}
 
@@ -505,20 +510,24 @@ func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResul
 	}
 	if value > recoveryThreshold {
 		state.healthySince = time.Time{}
+		metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1)
 		return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError)
 	}
 	if state.healthySince.IsZero() {
 		state.healthySince = now
+		metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1)
 		return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError)
 	}
 
 	recoveryDuration := time.Duration(clusterSettings.RecoveryDurationMillis) * time.Millisecond
 	if now.Sub(state.healthySince) < recoveryDuration {
+		metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1)
 		return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError)
 	}
 
 	state.throttled = false
 	state.healthySince = time.Time{}
+	metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(0)
 	return metricResult
 }
 

pkg/throttle/throttler_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/github/freno/pkg/base"
 	"github.com/github/freno/pkg/config"
+	metrics "github.com/rcrowley/go-metrics"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -247,6 +248,7 @@ func TestCheckMySQLRequiredClusters(t *testing.T) {
 		assert.Equal(t, http.StatusTooManyRequests, result.StatusCode)
 		assert.Equal(t, 60.0, result.Value)
 		assert.Equal(t, 50.0, result.Threshold)
+		assert.Equal(t, "mysql/primary", result.MetricName)
 	})
 
 	t.Run("ProxySQL cluster blocks admission", func(t *testing.T) {
@@ -275,6 +277,7 @@ func TestCheckMySQLRequiredClusters(t *testing.T) {
 		result := check.Check("transitions", "mysql", "replicas", "", &CheckFlags{OKIfNotExists: true})
 		assert.Equal(t, http.StatusInternalServerError, result.StatusCode)
 		assert.EqualError(t, result.Error, `required MySQL cluster metric "primary" is unavailable`)
+		assert.Equal(t, "mysql/primary", result.MetricName)
 	})
 }
 
@@ -337,3 +340,31 @@ func TestStabilizeMySQLMetricResetsRecoveryWindow(t *testing.T) {
 	_, err := result.Get()
 	assert.Equal(t, base.RecoveryNotCompleteError, err)
 }
+
+func TestStabilizeMySQLMetricRequiresHealthyWindowAfterStartup(t *testing.T) {
+	originalSettings := config.Settings().Stores.MySQL
+	defer func() {
+		config.Settings().Stores.MySQL = originalSettings
+	}()
+
+	config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{
+		Clusters: map[string]*config.MySQLClusterConfigurationSettings{
+			"primary": {
+				ThrottleThreshold:      50.0,
+				RecoveryDurationMillis: 5000,
+			},
+		},
+	}
+
+	throttler := NewThrottler()
+	start := time.Date(2026, time.September, 23, 12, 0, 0, 0, time.UTC)
+
+	result := throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(30), start)
+	_, err := result.Get()
+	assert.Equal(t, base.RecoveryNotCompleteError, err)
+
+	result = throttler.stabilizeMySQLMetric("primary", base.NewSimpleMetricResult(30), start.Add(5*time.Second))
+	_, err = result.Get()
+	assert.Nil(t, err)
+	assert.Equal(t, int64(0), metrics.Get("recovery.mysql.primary.active").(metrics.Gauge).Value())
+}