synctv/synctv-cluster/tests/k8s_lease_time_tests.rs

//! K8s Lease grace period time source tests
//!
//! These tests verify the behavior and document the limitations of grace period
//! calculations in the K8s Lease-based leader elector.
//!
//! # Background
//!
//! The K8sLeaderElector uses a grace period after losing leadership before
//! attempting re-acquisition. This prevents rapid flip-flopping during
//! transient network issues.
//!
//! # Clock Skew Consideration
//!
//! Unlike the Redis-based leader elector which uses Redis TIME for grace period
//! calculations, the K8s version uses local `tokio::time::Instant`. This is a
//! deliberate design choice because:
//!
//! 1. **Grace period is an optimization, not a safety guarantee**: The purpose
//!    is to reduce unnecessary API calls and log noise, not to prevent split-brain.
//!    Split-brain is already prevented by K8s Lease's `resourceVersion` optimistic
//!    locking.
//!
//! 2. **K8s API server time is not readily available**: Unlike Redis TIME, there's
//!    no lightweight way to get K8s API server time. The `renew_time` field in
//!    Lease is set by the client, not the server.
//!
//! 3. **Impact is limited**: Clock skew affects only the fairness of grace period
//!    expiration, not correctness. In the worst case, a node with a fast clock
//!    may exit grace period slightly early and attempt acquisition.
//!
//! # Monitoring Recommendation
//!
//! Deployments should monitor `synctv_cluster_leader_election_consecutive_failures`
//! metric. Rising values may indicate clock skew or network issues.

#![allow(clippy::unwrap_used)]

use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;

// Test 5: Document exponential backoff calculation

/// Test the exponential backoff formula for grace period.
///
/// Formula: grace = base * 2^(consecutive_losses - 1), capped at max
///
/// - consecutive_losses = 0: grace = base (5s)
/// - consecutive_losses = 1: grace = base * 2^0 = 5s
/// - consecutive_losses = 2: grace = base * 2^1 = 10s
/// - consecutive_losses = 3: grace = base * 2^2 = 20s
/// - consecutive_losses = 4: grace = base * 2^3 = 40s
/// - consecutive_losses = 5: grace = min(base * 2^4, max) = min(80, 60) = 60s
#[test]
fn test_exponential_backoff_formula() {
    let base_secs = 5u64;
    let max_secs = 60u64;

    let test_cases = [
        (0, 5),   // base
        (1, 5),   // base * 2^0 = 5
        (2, 10),  // base * 2^1 = 10
        (3, 20),  // base * 2^2 = 20
        (4, 40),  // base * 2^3 = 40
        (5, 60),  // base * 2^4 = 80, capped at 60
        (6, 60),  // capped at max
        (10, 60), // capped at max
    ];

    for (consecutive_losses, expected_secs) in test_cases {
        let multiplier = if consecutive_losses == 0 {
            1u64
        } else {
            1u64 << (consecutive_losses - 1).min(6)
        };
        let grace_secs = (base_secs * multiplier).min(max_secs);
        assert_eq!(
            grace_secs, expected_secs,
            "Failed for consecutive_losses = {consecutive_losses}"
        );
    }
}

// Test 6: Document that consecutive_losses is reset on leadership gain

/// Test that consecutive_losses counter is reset when leadership is gained.
///
/// This ensures that the exponential backoff doesn't persist across
/// successful leadership tenures.
#[test]
fn test_consecutive_losses_resets_on_leadership_gain() {
    let consecutive_losses = Arc::new(AtomicU64::new(0));

    // Simulate multiple losses
    consecutive_losses.fetch_add(1, Ordering::AcqRel); // 1
    consecutive_losses.fetch_add(1, Ordering::AcqRel); // 2
    consecutive_losses.fetch_add(1, Ordering::AcqRel); // 3

    assert_eq!(
        consecutive_losses.load(Ordering::Relaxed),
        3,
        "Should have 3 consecutive losses"
    );

    // Simulate leadership gain (reset)
    let previous = consecutive_losses.swap(0, Ordering::AcqRel);

    assert_eq!(previous, 3, "Previous value should be 3");
    assert_eq!(
        consecutive_losses.load(Ordering::Relaxed),
        0,
        "Should be reset to 0 after leadership gain"
    );
}

// Test 7: Document limitation and monitoring recommendation

// This test block documents the limitation of local time usage and the
// recommended monitoring approach.
// Limitation:
// The grace period uses local `tokio::time::Instant` which is monotonic
// but represents the node's local time. If the node's clock is significantly
// skewed, the grace period duration in wall-clock time may differ from
// other nodes' perspectives.
// Impact:
// - Minimal: The grace period is an optimization, not a correctness guarantee.
// - `tokio::time::Instant` is monotonic, so at least it's consistent within a node.
// - Split-brain is prevented by K8s resourceVersion, not grace period.
// Monitoring:
// Monitor the `synctv_cluster_leader_election_consecutive_failures` metric.
// Rising values may indicate network issues, RBAC problems, or K8s API overload.
// Test 8: Verify checked_sub for Instant arithmetic

/// Test that we can safely subtract from Instant for grace period calculations.
///
/// Note: `tokio::time::Instant` doesn't support `std::ops::Sub` directly.
/// Instead, we use `checked_sub` on Durations or compare elapsed time.
#[test]
fn test_instant_arithmetic_for_grace_period() {
    let now = tokio::time::Instant::now();

    // Simulate a past instant (5 seconds ago)
    // We can't directly subtract from Instant, but we can work with elapsed
    let grace_period = Duration::from_secs(10);

    std::thread::sleep(Duration::from_millis(1));

    // Check if we're in grace period
    let elapsed = now.elapsed();
    let in_grace = elapsed < grace_period;

    assert!(in_grace, "Should be in grace period after just 1ms");

    // Simulate grace period expiration by creating an "old" instant
    // We can use tokio::time::Instant::now() - Duration (via the Sub trait)
    let old_instant = tokio::time::Instant::now() - Duration::from_secs(15);

    let elapsed = old_instant.elapsed();
    let in_grace = elapsed < grace_period;

    assert!(
        !in_grace,
        "Should NOT be in grace period after 15s when grace is 10s"
    );
}