mirror of https://github.com/pixelfed/pixelfed
Merge 434adee993 into 8b6eee19cf
Shlee
2 days ago
committed by
GitHub
commit
efc2db67a0
@ -0,0 +1,174 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* The collation that correctly differentiates characters outside the
|
||||
* Basic Multilingual Plane (codepoints >= 0x10000). The historic default
|
||||
* of utf8mb4_unicode_ci treats all such characters as equal, which causes
|
||||
* distinct hashtags (e.g. Shavian vs cuneiform of the same length) to
|
||||
* collide on the unique name/slug indexes.
|
||||
*/
|
||||
private const TARGET_COLLATION = 'utf8mb4_unicode_520_ci';
|
||||
|
||||
/**
|
||||
* The collation to restore on rollback (the previous project default).
|
||||
*/
|
||||
private const PREVIOUS_COLLATION = 'utf8mb4_unicode_ci';
|
||||
|
||||
/**
|
||||
* Column definitions to keep intact while altering the collation. Both are
|
||||
* VARCHAR(255) NOT NULL with unique indexes; MODIFY preserves the index.
|
||||
*/
|
||||
private const COLUMNS = ['name', 'slug'];
|
||||
|
||||
/**
|
||||
* Tables that reference hashtags.id via a hashtag_id column. Rows in these
|
||||
* tables must be repointed from a duplicate hashtag to the surviving one
|
||||
* before the duplicate is deleted, so no references are orphaned.
|
||||
*
|
||||
* group_post_hashtags is intentionally excluded: it references the
|
||||
* separate group_hashtags table, not hashtags.
|
||||
*/
|
||||
private const REFERENCING_TABLES = [
|
||||
'status_hashtags',
|
||||
'hashtag_follows',
|
||||
'hashtag_related',
|
||||
'discover_category_hashtags',
|
||||
];
|
||||
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
if (! $this->isMysql()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The unique indexes on name/slug are enforced during the ALTER. Under
|
||||
// the stricter target collation, rows that were previously distinct may
|
||||
// now be considered equal, so merge those duplicates first to avoid a
|
||||
// "Duplicate entry" (1062) failure on the ALTER TABLE.
|
||||
$this->mergeDuplicates('name');
|
||||
$this->mergeDuplicates('slug');
|
||||
|
||||
$this->setCollation(self::TARGET_COLLATION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*
|
||||
* Note: merged duplicate rows are not restored on rollback (the data is
|
||||
* gone). Only the collation is reverted.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
if (! $this->isMysql()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->setCollation(self::PREVIOUS_COLLATION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge hashtags that collide on the given column under the target
|
||||
* collation. For each group of colliding rows, the lowest id is kept and
|
||||
* all references are repointed to it before the losing rows are deleted.
|
||||
*/
|
||||
private function mergeDuplicates(string $column): void
|
||||
{
|
||||
$collatedColumn = 'CONVERT(`'.$column.'` USING utf8mb4) COLLATE '.self::TARGET_COLLATION;
|
||||
|
||||
// Group by the value compared under the target collation. Any group
|
||||
// with more than one row would violate the unique index after the
|
||||
// collation change.
|
||||
$groups = DB::table('hashtags')
|
||||
->select(DB::raw('MIN(id) as keep_id'))
|
||||
->groupBy(DB::raw($collatedColumn))
|
||||
->havingRaw('COUNT(*) > 1')
|
||||
->get();
|
||||
|
||||
foreach ($groups as $group) {
|
||||
$keepId = (int) $group->keep_id;
|
||||
|
||||
// Find the losing rows: everything in the same collated group
|
||||
// except the surviving (lowest) id.
|
||||
$keepValue = DB::table('hashtags')->where('id', $keepId)->value($column);
|
||||
|
||||
$losers = DB::table('hashtags')
|
||||
->whereRaw($collatedColumn.' = CONVERT(? USING utf8mb4) COLLATE '.self::TARGET_COLLATION, [$keepValue])
|
||||
->where('id', '!=', $keepId)
|
||||
->pluck('id')
|
||||
->map(fn ($id) => (int) $id)
|
||||
->all();
|
||||
|
||||
if (empty($losers)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->repointReferences($losers, $keepId);
|
||||
|
||||
DB::table('hashtags')->whereIn('id', $losers)->delete();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repoint references from the losing hashtag ids to the surviving id.
|
||||
* UPDATE IGNORE avoids aborting on unique constraints in the referencing
|
||||
* tables; any rows that could not be repointed (because an equivalent
|
||||
* reference to keep_id already exists) are then deleted.
|
||||
*/
|
||||
private function repointReferences(array $loserIds, int $keepId): void
|
||||
{
|
||||
$placeholders = implode(',', array_fill(0, count($loserIds), '?'));
|
||||
|
||||
foreach (self::REFERENCING_TABLES as $table) {
|
||||
if (! DB::getSchemaBuilder()->hasColumn($table, 'hashtag_id')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DB::statement(
|
||||
'UPDATE IGNORE `'.$table.'` SET `hashtag_id` = ? WHERE `hashtag_id` IN ('.$placeholders.')',
|
||||
array_merge([$keepId], $loserIds)
|
||||
);
|
||||
|
||||
DB::statement(
|
||||
'DELETE FROM `'.$table.'` WHERE `hashtag_id` IN ('.$placeholders.')',
|
||||
$loserIds
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the given collation to the hashtags name and slug columns.
|
||||
*
|
||||
* Laravel's fluent ->change() does not reliably emit a collation-only
|
||||
* change on MySQL/MariaDB, so issue an explicit MODIFY per column.
|
||||
*/
|
||||
private function setCollation(string $collation): void
|
||||
{
|
||||
foreach (self::COLUMNS as $column) {
|
||||
DB::statement(
|
||||
'ALTER TABLE `hashtags` MODIFY `'.$column.'` '.
|
||||
'VARCHAR(255) CHARACTER SET utf8mb4 COLLATE '.$collation.' NOT NULL'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This migration only applies to MySQL/MariaDB. Postgres compares
|
||||
* hashtags with ILIKE (no collation quirk) and other drivers (e.g. the
|
||||
* sqlite test database) do not support these collations.
|
||||
*
|
||||
* Note: Laravel 11+ reports MariaDB as the distinct "mariadb" driver, so
|
||||
* both must be matched here.
|
||||
*/
|
||||
private function isMysql(): bool
|
||||
{
|
||||
return in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true);
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,153 @@
|
||||
<?php
|
||||
|
||||
use App\Models\Hashtag;
|
||||
use Illuminate\Foundation\Testing\LazilyRefreshDatabase;
|
||||
use Illuminate\Support\Facades\Artisan;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
uses(LazilyRefreshDatabase::class);
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Hashtag collation fix (PR #6098)
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Verifies that the hashtags collation migration runs safely on all drivers
|
||||
| and, on MySQL/MariaDB, prevents conflation of distinct hashtags that use
|
||||
| characters outside the Basic Multilingual Plane (codepoints >= 0x10000).
|
||||
|
|
||||
| It also verifies that pre-existing rows which become equal under the
|
||||
| stricter target collation are merged before the collation change, so the
|
||||
| migration no longer fails with a duplicate-entry (1062) error.
|
||||
|
|
||||
*/
|
||||
|
||||
it('migration runs without error regardless of database driver', function () {
|
||||
// The test suite uses sqlite by default. The migration detects the driver
|
||||
// and no-ops gracefully rather than attempting unsupported ALTER syntax.
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
// If we reach here without exception the no-op path worked.
|
||||
expect(true)->toBeTrue();
|
||||
});
|
||||
|
||||
it('migration rollback runs without error regardless of database driver', function () {
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
Artisan::call('migrate:rollback', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
expect(true)->toBeTrue();
|
||||
});
|
||||
|
||||
it('distinct BMP-outside hashtags do not collide on the unique index', function () {
|
||||
// This test is only meaningful on MySQL/MariaDB where the collation fix matters.
|
||||
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {
|
||||
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
|
||||
}
|
||||
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
// Two distinct 5-character hashtags using characters outside the BMP.
|
||||
// Under the old utf8mb4_unicode_ci collation these were considered equal.
|
||||
$shavian = '𐑖𐑱𐑝𐑾𐑯'; // Shavian script
|
||||
$cuneiform = '𒆳𒆍𒀭𒊏𒆠'; // Cuneiform script
|
||||
|
||||
$tag1 = Hashtag::create(['name' => $shavian, 'slug' => $shavian]);
|
||||
$tag2 = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]);
|
||||
|
||||
// Both must coexist as separate rows with distinct IDs.
|
||||
expect($tag1->id)->not->toBe($tag2->id);
|
||||
expect(Hashtag::where('slug', $shavian)->first()->id)->toBe($tag1->id);
|
||||
expect(Hashtag::where('slug', $cuneiform)->first()->id)->toBe($tag2->id);
|
||||
});
|
||||
|
||||
it('merges pre-existing colliding rows instead of failing the ALTER', function () {
|
||||
// Reproduces the original failure: rows that are distinct under the old
|
||||
// collation but equal under the target collation must be merged, and their
|
||||
// references repointed, before the unique index is re-enforced.
|
||||
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {
|
||||
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
|
||||
}
|
||||
|
||||
// Seed the table under the OLD collation so the two BMP-outside tags are
|
||||
// stored as separate rows (they only collide under the target collation).
|
||||
DB::statement('ALTER TABLE `hashtags` MODIFY `name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL');
|
||||
DB::statement('ALTER TABLE `hashtags` MODIFY `slug` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL');
|
||||
|
||||
$shavian = '𐑖𐑱𐑝𐑾𐑯';
|
||||
$cuneiform = '𒆳𒆍𒀭𒊏𒆠';
|
||||
|
||||
$keep = Hashtag::create(['name' => $shavian, 'slug' => $shavian]);
|
||||
$dup = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]);
|
||||
|
||||
// A status_hashtags reference pointing at the row that will be merged away.
|
||||
DB::table('status_hashtags')->insert([
|
||||
'status_id' => 1,
|
||||
'hashtag_id' => $dup->id,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]);
|
||||
|
||||
// Running the migration must not throw a duplicate-entry error.
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
// The duplicate row is gone, the surviving row remains.
|
||||
expect(Hashtag::find($dup->id))->toBeNull();
|
||||
expect(Hashtag::find($keep->id))->not->toBeNull();
|
||||
|
||||
// The reference was repointed to the surviving hashtag.
|
||||
expect(DB::table('status_hashtags')->where('hashtag_id', $dup->id)->count())->toBe(0);
|
||||
expect(DB::table('status_hashtags')->where('hashtag_id', $keep->id)->count())->toBe(1);
|
||||
});
|
||||
|
||||
it('same-script hashtags with the same slug still correctly deduplicate', function () {
|
||||
// Sanity check: two identical hashtags should NOT create duplicates.
|
||||
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {
|
||||
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
|
||||
}
|
||||
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
$tag = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']);
|
||||
$same = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']);
|
||||
|
||||
expect($tag->id)->toBe($same->id);
|
||||
expect(Hashtag::where('slug', 'hello')->count())->toBe(1);
|
||||
});
|
||||
|
||||
it('case-insensitivity is preserved after collation change', function () {
|
||||
// utf8mb4_unicode_520_ci is still case-insensitive, so #Hello == #hello.
|
||||
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {
|
||||
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
|
||||
}
|
||||
|
||||
Artisan::call('migrate', [
|
||||
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
|
||||
'--force' => true,
|
||||
]);
|
||||
|
||||
Hashtag::create(['name' => 'Pixelfed', 'slug' => 'pixelfed']);
|
||||
|
||||
// Case-insensitive lookup should find it with different casing.
|
||||
$found = Hashtag::where('slug', 'PIXELFED')->first();
|
||||
expect($found)->not->toBeNull();
|
||||
expect($found->slug)->toBe('pixelfed');
|
||||
});
|
||||
Loading…
Reference in New Issue