//! K8s Lease grace period time source tests //! //! These tests verify the behavior and document the limitations of grace period //! calculations in the K8s Lease-based leader elector. //! //! # Background //! //! The K8sLeaderElector uses a grace period after losing leadership before //! attempting re-acquisition. This prevents rapid flip-flopping during //! transient network issues. //! //! # Clock Skew Consideration //! //! Unlike the Redis-based leader elector which uses Redis TIME for grace period //! calculations, the K8s version uses local `tokio::time::Instant`. This is a //! deliberate design choice because: //! //! 1. **Grace period is an optimization, not a safety guarantee**: The purpose //! is to reduce unnecessary API calls and log noise, not to prevent split-brain. //! Split-brain is already prevented by K8s Lease's `resourceVersion` optimistic //! locking. //! //! 2. **K8s API server time is not readily available**: Unlike Redis TIME, there's //! no lightweight way to get K8s API server time. The `renew_time` field in //! Lease is set by the client, not the server. //! //! 3. **Impact is limited**: Clock skew affects only the fairness of grace period //! expiration, not correctness. In the worst case, a node with a fast clock //! may exit grace period slightly early and attempt acquisition. //! //! # Monitoring Recommendation //! //! Deployments should monitor `synctv_cluster_leader_election_consecutive_failures` //! metric. Rising values may indicate clock skew or network issues. #![allow(clippy::unwrap_used)] use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; // Test 5: Document exponential backoff calculation /// Test the exponential backoff formula for grace period. /// /// Formula: grace = base * 2^(consecutive_losses - 1), capped at max /// /// - consecutive_losses = 0: grace = base (5s) /// - consecutive_losses = 1: grace = base * 2^0 = 5s /// - consecutive_losses = 2: grace = base * 2^1 = 10s /// - consecutive_losses = 3: grace = base * 2^2 = 20s /// - consecutive_losses = 4: grace = base * 2^3 = 40s /// - consecutive_losses = 5: grace = min(base * 2^4, max) = min(80, 60) = 60s #[test] fn test_exponential_backoff_formula() { let base_secs = 5u64; let max_secs = 60u64; let test_cases = [ (0, 5), // base (1, 5), // base * 2^0 = 5 (2, 10), // base * 2^1 = 10 (3, 20), // base * 2^2 = 20 (4, 40), // base * 2^3 = 40 (5, 60), // base * 2^4 = 80, capped at 60 (6, 60), // capped at max (10, 60), // capped at max ]; for (consecutive_losses, expected_secs) in test_cases { let multiplier = if consecutive_losses == 0 { 1u64 } else { 1u64 << (consecutive_losses - 1).min(6) }; let grace_secs = (base_secs * multiplier).min(max_secs); assert_eq!( grace_secs, expected_secs, "Failed for consecutive_losses = {consecutive_losses}" ); } } // Test 6: Document that consecutive_losses is reset on leadership gain /// Test that consecutive_losses counter is reset when leadership is gained. /// /// This ensures that the exponential backoff doesn't persist across /// successful leadership tenures. #[test] fn test_consecutive_losses_resets_on_leadership_gain() { let consecutive_losses = Arc::new(AtomicU64::new(0)); // Simulate multiple losses consecutive_losses.fetch_add(1, Ordering::AcqRel); // 1 consecutive_losses.fetch_add(1, Ordering::AcqRel); // 2 consecutive_losses.fetch_add(1, Ordering::AcqRel); // 3 assert_eq!( consecutive_losses.load(Ordering::Relaxed), 3, "Should have 3 consecutive losses" ); // Simulate leadership gain (reset) let previous = consecutive_losses.swap(0, Ordering::AcqRel); assert_eq!(previous, 3, "Previous value should be 3"); assert_eq!( consecutive_losses.load(Ordering::Relaxed), 0, "Should be reset to 0 after leadership gain" ); } // Test 7: Document limitation and monitoring recommendation // This test block documents the limitation of local time usage and the // recommended monitoring approach. // Limitation: // The grace period uses local `tokio::time::Instant` which is monotonic // but represents the node's local time. If the node's clock is significantly // skewed, the grace period duration in wall-clock time may differ from // other nodes' perspectives. // Impact: // - Minimal: The grace period is an optimization, not a correctness guarantee. // - `tokio::time::Instant` is monotonic, so at least it's consistent within a node. // - Split-brain is prevented by K8s resourceVersion, not grace period. // Monitoring: // Monitor the `synctv_cluster_leader_election_consecutive_failures` metric. // Rising values may indicate network issues, RBAC problems, or K8s API overload. // Test 8: Verify checked_sub for Instant arithmetic /// Test that we can safely subtract from Instant for grace period calculations. /// /// Note: `tokio::time::Instant` doesn't support `std::ops::Sub` directly. /// Instead, we use `checked_sub` on Durations or compare elapsed time. #[test] fn test_instant_arithmetic_for_grace_period() { let now = tokio::time::Instant::now(); // Simulate a past instant (5 seconds ago) // We can't directly subtract from Instant, but we can work with elapsed let grace_period = Duration::from_secs(10); std::thread::sleep(Duration::from_millis(1)); // Check if we're in grace period let elapsed = now.elapsed(); let in_grace = elapsed < grace_period; assert!(in_grace, "Should be in grace period after just 1ms"); // Simulate grace period expiration by creating an "old" instant // We can use tokio::time::Instant::now() - Duration (via the Sub trait) let old_instant = tokio::time::Instant::now() - Duration::from_secs(15); let elapsed = old_instant.elapsed(); let in_grace = elapsed < grace_period; assert!( !in_grace, "Should NOT be in grace period after 15s when grace is 10s" ); }