Add composite MySQL safety checks and stable recovery

doc/memcache.md

@@ -37,6 +37,10 @@ Optionally set `MemcachePath` (default is `"freno"`):
 - The value will be of the form `<epochmillis>:<aggregated-value>`.
   - As example, it might be `1497418678836:0.54` where `1497418678836` is the unix epoch in milliseconds, and `0.54` is the aggregated value.
   - Embedding the epoch within the value allows the app to double-check the validity of the value, or go into more granular validation.
+- For a MySQL cluster with `RequiredClusters`, the key is removed while any
+  required metric is unavailable, recovering, or above its configured
+  threshold. Direct memcache clients therefore observe the same composite
+  safety gate as HTTP clients.
 
 ### Runtime access to memcache configuration
 

doc/mysql.md

@@ -96,7 +96,8 @@ Looking at clusters configuration:
     ],
     "VitessSettings": {
       "API": "https://vtctld.example.com/api/",
-      "Keyspace": "my_sharded_ks"
+      "Keyspace": "my_sharded_ks",
+      "TabletType": "REPLICA"
     }
   },
   "local": {
@@ -120,6 +121,7 @@ Noteworthy:
 
 - `prod4` chooses to (but doesn't have to) override the `ThrottleThreshold` to `0.8` seconds
 - `prod4` list of servers is dictated by `HAProxy`. `freno` will routinely and dynamically poll given HAProxy server for list of hosts. These will include any hosts not in `NOLB`.
+- `sharded` discovers `REPLICA` tablets by default. Set `TabletType` to `MASTER` (or `PRIMARY`) for a separately configured primary safety metric.
 - `local` cluster chooses to override `User`, `Password` and `IgnoreHostsCount`.
 - `local` cluster defines a static list of hosts.
 
@@ -148,3 +150,67 @@ Noteworthy:
 `freno` explicitly recognizes `show global ...` statements and reads the result's numeric value.
 
 Otherwise you may provide any query that returns a single row, single numeric column.
+
+### Composing replica, primary, and ProxySQL protection
+
+A cluster can require additional cluster metrics to pass before `freno` permits work. This allows the normal replica-lag check to depend on separately sampled primary-load and ProxySQL-capacity checks:
+
+```json
+"Clusters": {
+  "prod4": {
+    "RequiredClusters": [
+      "prod4-primary",
+      "prod4-proxysql"
+    ],
+    "HAProxySettings": {
+      "Host": "my.haproxy.mydomain.com",
+      "Port": 1001,
+      "PoolName": "my_prod4_pool"
+    }
+  },
+  "prod4-primary": {
+    "MetricQuery": "show global status like 'Threads_running'",
+    "CacheMillis": 500,
+    "ThrottleThreshold": 50,
+    "RecoveryThreshold": 35,
+    "RecoveryDurationMillis": 5000,
+    "FailOnNoHosts": true,
+    "StaticHostsSettings": {
+      "Hosts": [
+        "my.prod4.primary.vip.example.com:3306"
+      ]
+    }
+  },
+  "prod4-proxysql": {
+    "User": "${proxysql_stats_user}",
+    "Password": "${proxysql_stats_password}",
+    "MetricQuery": "select connection_pressure_percent from operational_metrics.proxysql_capacity limit 1",
+    "CacheMillis": 500,
+    "ThrottleThreshold": 80,
+    "RecoveryThreshold": 60,
+    "RecoveryDurationMillis": 5000,
+    "FailOnNoHosts": true,
+    "StaticHostsSettings": {
+      "Hosts": [
+        "my.prod4.proxysql.example.com:6032"
+      ]
+    }
+  }
+}
+```
+
+The ProxySQL query above is illustrative: deployments must provide a scalar query or operational view whose value increases as usable connection capacity is consumed.
+
+- `RequiredClusters` lists additional configured metrics that must return `HTTP 200`. Missing required runtime metrics fail closed.
+- `FailOnNoHosts` changes the legacy no-host response from `HTTP 200` to `HTTP 500`. Enable it for primary and ProxySQL safety probes, where an empty roster cannot prove that work is safe.
+- `RecoveryThreshold` is the lower threshold that begins recovery after a cluster has throttled. It must not exceed `ThrottleThreshold`.
+- `RecoveryDurationMillis` is the continuous time the metric must remain at or below `RecoveryThreshold` before checks return `HTTP 200` again. A new error or threshold violation resets the recovery window.
+- A new process or leader starts configured recovery metrics in the recovering state. It must observe a complete healthy window before admitting work.
+- Metric sampling remains centralized in the `freno` leader and uses the existing metric cache. Application checks read the aggregated decision rather than querying the primary or ProxySQL for every batch.
+- Check responses include `MetricName`, identifying the replica, primary, or ProxySQL metric that blocked a composite decision. `recovery.mysql.<cluster>.active` reports whether the recovery latch is active.
+
+Configuration loading rejects missing `RequiredClusters` references and dependency cycles.
+
+### Scope boundaries
+
+These settings provide reusable admission checks and recovery behavior. Deployments remain responsible for choosing appropriate metrics and thresholds, configuring their host-discovery sources, monitoring probe health and rejection rates, and controlling application concurrency or fallback behavior.

pkg/base/throttle_metric.go

@@ -18,6 +18,7 @@ type MetricResultFunc func() (metricResult MetricResult, threshold float64)
 
 var ThresholdExceededError = errors.New("Threshold exceeded")
 var NoHostsError = errors.New("No hosts found")
+var RecoveryNotCompleteError = errors.New("Recovery period not complete")
 var noResultYetError = errors.New("Metric not collected yet")
 var NoSuchMetricError = errors.New("No such metric")
 
@@ -63,3 +64,16 @@ func NewSimpleMetricResult(value float64) MetricResult {
 func (metricResult *simpleMetricResult) Get() (float64, error) {
 	return metricResult.Value, nil
 }
+
+type errorMetricResult struct {
+	Value float64
+	Err   error
+}
+
+func NewErrorMetricResult(value float64, err error) MetricResult {
+	return &errorMetricResult{Value: value, Err: err}
+}
+
+func (metricResult *errorMetricResult) Get() (float64, error) {
+	return metricResult.Value, metricResult.Err
+}

pkg/config/config_test.go

@@ -112,6 +112,141 @@ func TestMySQLFallbackCluster(t *testing.T) {
 	}
 }
 
+func TestMySQLRequiredClusters(t *testing.T) {
+	tests := []struct {
+		name     string
+		clusters map[string]*MySQLClusterConfigurationSettings
+		wantErr  string
+	}{
+		{
+			name: "valid dependencies",
+			clusters: map[string]*MySQLClusterConfigurationSettings{
+				"replicas": {RequiredClusters: []string{"primary", "proxysql"}},
+				"primary":  {},
+				"proxysql": {},
+			},
+		},
+		{
+			name: "unknown dependency",
+			clusters: map[string]*MySQLClusterConfigurationSettings{
+				"replicas": {RequiredClusters: []string{"primary"}},
+			},
+			wantErr: `Stores.MySQL.Clusters.replicas.RequiredClusters references unknown cluster "primary"`,
+		},
+		{
+			name: "dependency cycle",
+			clusters: map[string]*MySQLClusterConfigurationSettings{
+				"replicas": {RequiredClusters: []string{"primary"}},
+				"primary":  {RequiredClusters: []string{"replicas"}},
+			},
+			wantErr: `Stores.MySQL.Clusters contains a RequiredClusters cycle involving "primary"`,
+		},
+		{
+			name: "null cluster",
+			clusters: map[string]*MySQLClusterConfigurationSettings{
+				"replicas": nil,
+			},
+			wantErr: `Stores.MySQL.Clusters.replicas must not be null`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			settings := MySQLConfigurationSettings{Clusters: test.clusters}
+			err := settings.postReadAdjustments()
+			if test.wantErr == "" && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if test.wantErr != "" && (err == nil || err.Error() != test.wantErr) {
+				t.Fatalf("error = %v, want %q", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func TestMySQLRecoveryConfiguration(t *testing.T) {
+	recoveryThreshold := 5.0
+	tooHighRecoveryThreshold := 11.0
+	tests := []struct {
+		name            string
+		clusterSettings *MySQLClusterConfigurationSettings
+		wantErr         string
+	}{
+		{name: "disabled", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10}},
+		{name: "valid", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryThreshold: &recoveryThreshold, RecoveryDurationMillis: 5000}},
+		{name: "default recovery threshold", clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryDurationMillis: 5000}},
+		{
+			name:            "negative duration",
+			clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryDurationMillis: -1},
+			wantErr:         "Stores.MySQL.Clusters.primary.RecoveryDurationMillis must not be negative",
+		},
+		{
+			name:            "recovery threshold above throttle threshold",
+			clusterSettings: &MySQLClusterConfigurationSettings{ThrottleThreshold: 10, RecoveryThreshold: &tooHighRecoveryThreshold, RecoveryDurationMillis: 5000},
+			wantErr:         "Stores.MySQL.Clusters.primary.RecoveryThreshold must not exceed ThrottleThreshold",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			settings := MySQLConfigurationSettings{
+				Clusters: map[string]*MySQLClusterConfigurationSettings{"primary": test.clusterSettings},
+			}
+			err := settings.postReadAdjustments()
+			if test.wantErr == "" && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if test.wantErr != "" && (err == nil || err.Error() != test.wantErr) {
+				t.Fatalf("error = %v, want %q", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func TestVitessTabletTypeConfiguration(t *testing.T) {
+	tests := []struct {
+		name       string
+		tabletType string
+		want       string
+		wantErr    string
+	}{
+		{name: "default replica", want: "REPLICA"},
+		{name: "replica", tabletType: "replica", want: "REPLICA"},
+		{name: "master", tabletType: "master", want: "MASTER"},
+		{name: "primary alias", tabletType: "primary", want: "MASTER"},
+		{name: "invalid", tabletType: "rdonly", wantErr: `Stores.MySQL.Clusters.vitess.VitessSettings: unsupported Vitess tablet type "rdonly"`},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			settings := MySQLConfigurationSettings{
+				Clusters: map[string]*MySQLClusterConfigurationSettings{
+					"vitess": {
+						VitessSettings: VitessConfigurationSettings{
+							API:        "https://vtctld.example.com/api",
+							Keyspace:   "test",
+							TabletType: test.tabletType,
+						},
+					},
+				},
+			}
+			err := settings.postReadAdjustments()
+			if test.wantErr == "" && err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if test.wantErr != "" {
+				if err == nil || err.Error() != test.wantErr {
+					t.Fatalf("error = %v, want %q", err, test.wantErr)
+				}
+				return
+			}
+			if got := settings.Clusters["vitess"].VitessSettings.TabletType; got != test.want {
+				t.Fatalf("TabletType = %q, want %q", got, test.want)
+			}
+		})
+	}
+}
+
 func dump(path string, contents *ConfigurationSettings) error {
 	json, _ := json.Marshal(contents)
 	err := ioutil.WriteFile(path, json, 0644)