synctv/synctv-core/tests/database_deadlock_tests.rs

//! Database deadlock detection tests
//!
//! Tests verify that the application properly detects and handles deadlocks.
//!
//! Requires Docker for testcontainers.

use sqlx::PgPool;
use std::sync::Arc;
use synctv_core::{
    models::{
        MemberStatus, Room, RoomId, RoomMember, RoomRole, SignupMethod, User, UserId, UserStatus,
    },
    repository::{RoomMemberRepository, RoomRepository, UserRepository},
};
use synctv_core_testing::{create_test_database_with_options_and_label, ok, TestDatabase};
use tokio::sync::Barrier;

async fn create_test_pool() -> TestDatabase {
    create_test_database_with_options_and_label(
        "synctv_test",
        "database-deadlock",
        20,
        std::time::Duration::from_secs(30),
    )
    .await
}

/// Create a test user in the database (required for FK constraints)
async fn create_test_user(pool: &PgPool, user_id: &UserId) -> UserId {
    let username = format!("test_user_{user_id}");
    let user = User {
        id: *user_id,
        username,
        signup_method: SignupMethod::Email,
        role: synctv_core::models::UserRole::User,
        avatar_file_reference_id: None,
        status: UserStatus::Active,
        created_at: chrono::Utc::now(),
        updated_at: chrono::Utc::now(),
        version: 0,
        deleted_at: None,
        is_banned: false,
        banned_at: None,
        banned_by: None,
        banned_reason: None,
    };
    let user_repo = UserRepository::new(pool.clone());
    ok(user_repo.create(&user).await, "test user should be created").id
}

fn make_member(room_id: RoomId, user_id: UserId) -> RoomMember {
    RoomMember {
        room_id,
        user_id,
        role: RoomRole::Member,
        status: MemberStatus::Active,
        added_permissions: 0,
        removed_permissions: 0,
        admin_added_permissions: 0,
        admin_removed_permissions: 0,
        remark_name: String::new(),
        display_tag: String::new(),
        joined_at: chrono::Utc::now(),
        version: 0,
    }
}

fn make_room(creator_id: UserId) -> Room {
    Room {
        id: RoomId::new(),
        name: "Test Room".to_string(),
        description: "Test".to_string(),
        cover_file_reference_id: None,
        category: None,
        labels: Vec::new(),
        created_by: creator_id,
        status: synctv_core::models::RoomStatus::Active,
        is_banned: false,
        closed_at: None,
        created_at: chrono::Utc::now(),
        updated_at: chrono::Utc::now(),
        deleted_at: None,
        version: 0,
        last_activity_at: chrono::Utc::now(),
    }
}

#[tokio::test]
#[ignore = "Requires Docker"]
async fn test_deadlock_detection_opposite_lock_order() {
    let infra = create_test_pool().await;
    let pool = &infra.pool;

    let room_repo = RoomRepository::new(pool.clone());
    let member_repo = RoomMemberRepository::new(pool.clone());

    let creator_id = create_test_user(pool, &UserId::new()).await;
    let room = make_room(creator_id);
    let room = ok(room_repo.create(&room).await, "room should be created");

    let user1 = create_test_user(pool, &UserId::new()).await;
    let user2 = create_test_user(pool, &UserId::new()).await;

    let member1 = make_member(room.id, user1);
    let member2 = make_member(room.id, user2);
    ok(
        member_repo.add(&member1).await,
        "first member should be added",
    );
    ok(
        member_repo.add(&member2).await,
        "second member should be added",
    );

    // Try to cause deadlock with opposite lock order
    let barrier = Arc::new(Barrier::new(2));

    let pool1 = pool.clone();
    let room_id1 = room.id;
    let user1_clone = user1;
    let user2_clone1 = user2;
    let barrier1 = barrier.clone();

    let handle1 = tokio::spawn(async move {
        barrier1.wait().await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool1.begin().await?;

            // Lock user1 first
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                100_i64,
                room_id1.as_i64(),
                user1_clone.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;

            // Then lock user2
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                200_i64,
                room_id1.as_i64(),
                user2_clone1.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tx.commit().await?;
            Ok(())
        }
        .await;

        result
    });

    let pool2 = pool.clone();
    let room_id2 = room.id;
    let user1_clone2 = user1;
    let user2_clone2 = user2;
    let barrier2 = barrier.clone();

    let handle2 = tokio::spawn(async move {
        barrier2.wait().await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool2.begin().await?;

            // Lock user2 first (opposite order)
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                300_i64,
                room_id2.as_i64(),
                user2_clone2.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;

            // Then lock user1 (creates potential deadlock)
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                400_i64,
                room_id2.as_i64(),
                user1_clone2.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tx.commit().await?;
            Ok(())
        }
        .await;

        result
    });

    let result1 = ok(handle1.await, "first deadlock task should complete");
    let result2 = ok(handle2.await, "second deadlock task should complete");

    // At least one should fail with deadlock error, or both succeed
    let both_ok = result1.is_ok() && result2.is_ok();
    let has_deadlock = result1.is_err() || result2.is_err();

    if has_deadlock {
        let err1_str = format!("{result1:?}");
        let err2_str = format!("{result2:?}");
        let combined = format!("{err1_str} {err2_str}");

        // PostgreSQL deadlock error code is 40P01
        assert!(
            combined.contains("40P01") || combined.contains("deadlock"),
            "Expected deadlock error, got: {combined}"
        );
    } else {
        assert!(both_ok, "If no deadlock, both transactions should succeed");
    }
}

#[tokio::test]
#[ignore = "Requires Docker"]
async fn test_deadlock_with_for_update_nowait() {
    let infra = create_test_pool().await;
    let pool = &infra.pool;

    let room_repo = RoomRepository::new(pool.clone());
    let member_repo = RoomMemberRepository::new(pool.clone());

    let creator_id = create_test_user(pool, &UserId::new()).await;
    let room = make_room(creator_id);
    let room = ok(room_repo.create(&room).await, "room should be created");

    let user_id = create_test_user(pool, &UserId::new()).await;
    let member = make_member(room.id, user_id);
    ok(member_repo.add(&member).await, "member should be added");

    let barrier = Arc::new(Barrier::new(2));

    let pool1 = pool.clone();
    let room_id1 = room.id;
    let user_id1 = user_id;
    let barrier1 = barrier.clone();

    let handle1 = tokio::spawn(async move {
        barrier1.wait().await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool1.begin().await?;

            // Lock with FOR UPDATE
            sqlx::query!(
                r#"SELECT 1 AS "one!" FROM room_members
                 WHERE room_id = $1 AND user_id = $2
                 FOR UPDATE"#,
                room_id1.as_i64(),
                user_id1.as_i64()
            )
            .fetch_optional(&mut *tx)
            .await?;

            tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;

            tx.commit().await?;
            Ok(())
        }
        .await;

        result
    });

    let pool2 = pool.clone();
    let room_id2 = room.id;
    let user_id2 = user_id;
    let barrier2 = barrier.clone();

    let handle2 = tokio::spawn(async move {
        barrier2.wait().await;

        tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool2.begin().await?;

            // Try to lock with FOR UPDATE NOWAIT
            let result = sqlx::query!(
                r#"SELECT 1 AS "one!" FROM room_members
                 WHERE room_id = $1 AND user_id = $2
                 FOR UPDATE NOWAIT"#,
                room_id2.as_i64(),
                user_id2.as_i64()
            )
            .fetch_optional(&mut *tx)
            .await;

            match result {
                Ok(_) => {
                    tx.commit().await?;
                    Ok(())
                }
                Err(e) => Err(e),
            }
        }
        .await;

        result
    });

    let result1 = ok(handle1.await, "first nowait task should complete");
    let result2 = ok(handle2.await, "second nowait task should complete");

    assert!(result1.is_ok(), "First transaction should succeed");

    // Second transaction should fail immediately due to NOWAIT
    if result2.is_err() {
        let err_msg = format!("{result2:?}");
        // Lock not available error (55P03)
        assert!(
            err_msg.contains("55P03") || err_msg.contains("could not obtain lock"),
            "Expected lock timeout error, got: {err_msg}"
        );
    }
}

#[tokio::test]
#[ignore = "Requires Docker"]
async fn test_deadlock_avoidance_with_ordered_locks() {
    let infra = create_test_pool().await;
    let pool = &infra.pool;

    let room_repo = RoomRepository::new(pool.clone());
    let member_repo = RoomMemberRepository::new(pool.clone());

    let creator_id = create_test_user(pool, &UserId::new()).await;
    let room = make_room(creator_id);
    let room = ok(room_repo.create(&room).await, "room should be created");

    let user1 = create_test_user(pool, &UserId::new()).await;
    let user2 = create_test_user(pool, &UserId::new()).await;

    // Ensure consistent ordering
    let (first_user, second_user) = if user1 < user2 {
        (user1, user2)
    } else {
        (user2, user1)
    };

    let member1 = make_member(room.id, first_user);
    let member2 = make_member(room.id, second_user);
    ok(
        member_repo.add(&member1).await,
        "first member should be added",
    );
    ok(
        member_repo.add(&member2).await,
        "second member should be added",
    );

    let barrier = Arc::new(Barrier::new(2));

    // Both transactions lock in the same order
    let pool1 = pool.clone();
    let room_id1 = room.id;
    let first_user1 = first_user;
    let second_user1 = second_user;
    let barrier1 = barrier.clone();

    let handle1 = tokio::spawn(async move {
        barrier1.wait().await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool1.begin().await?;

            // Lock in consistent order
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                100_i64,
                room_id1.as_i64(),
                first_user1.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                200_i64,
                room_id1.as_i64(),
                second_user1.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tx.commit().await?;
            Ok(())
        }
        .await;

        result
    });

    let pool2 = pool.clone();
    let room_id2 = room.id;
    let first_user2 = first_user;
    let second_user2 = second_user;
    let barrier2 = barrier.clone();

    let handle2 = tokio::spawn(async move {
        barrier2.wait().await;

        let result: Result<(), sqlx::Error> = async {
            let mut tx = pool2.begin().await?;

            // Lock in same consistent order
            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                300_i64,
                room_id2.as_i64(),
                first_user2.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            sqlx::query!(
                "UPDATE room_members SET added_permissions = $1
                 WHERE room_id = $2 AND user_id = $3",
                400_i64,
                room_id2.as_i64(),
                second_user2.as_i64()
            )
            .execute(&mut *tx)
            .await?;

            tx.commit().await?;
            Ok(())
        }
        .await;

        result
    });

    let result1 = ok(handle1.await, "first ordered-lock task should complete");
    let result2 = ok(handle2.await, "second ordered-lock task should complete");

    // With consistent ordering, no deadlock should occur
    // One will complete, then the other
    assert!(
        result1.is_ok() || result2.is_ok(),
        "At least one should succeed"
    );
}

#[tokio::test]
#[ignore = "Requires Docker"]
async fn test_transaction_timeout_prevents_indefinite_wait() {
    let infra = create_test_pool().await;
    let pool = &infra.pool;

    let room_repo = RoomRepository::new(pool.clone());
    let member_repo = RoomMemberRepository::new(pool.clone());

    let creator_id = create_test_user(pool, &UserId::new()).await;
    let room = make_room(creator_id);
    let room = ok(room_repo.create(&room).await, "room should be created");

    let user_id = create_test_user(pool, &UserId::new()).await;
    let member = make_member(room.id, user_id);
    ok(member_repo.add(&member).await, "member should be added");

    let pool1 = pool.clone();
    let room_id1 = room.id;
    let user_id1 = user_id;

    let tx_handle = tokio::spawn(async move {
        let mut tx = pool1.begin().await?;

        // Lock the row
        sqlx::query!(
            r#"SELECT 1 AS "one!" FROM room_members
             WHERE room_id = $1 AND user_id = $2
             FOR UPDATE"#,
            room_id1.as_i64(),
            user_id1.as_i64()
        )
        .fetch_optional(&mut *tx)
        .await?;

        // Hold lock for 2 seconds
        tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;

        tx.commit().await?;
        Ok::<(), sqlx::Error>(())
    });

    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;

    let pool2 = pool.clone();
    let room_id2 = room.id;
    let user_id2 = user_id;

    let timeout_result = tokio::time::timeout(tokio::time::Duration::from_secs(1), async move {
        let mut tx = pool2.begin().await?;

        sqlx::query!(
            r#"SELECT 1 AS "one!" FROM room_members
                 WHERE room_id = $1 AND user_id = $2
                 FOR UPDATE"#,
            room_id2.as_i64(),
            user_id2.as_i64()
        )
        .fetch_optional(&mut *tx)
        .await?;

        tx.commit().await?;
        Ok::<(), sqlx::Error>(())
    })
    .await;

    // Should timeout waiting for lock
    assert!(timeout_result.is_err(), "Should timeout waiting for lock");

    let tx_result = ok(tx_handle.await, "first transaction task should complete");
    ok(tx_result, "first transaction should finish");
}