From 88adf721c3cf5986160ea3701c832af07d3bb2fc Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 21:18:39 +0930 Subject: [PATCH 1/2] fix: use utf8mb4_unicode_520_ci for hashtags to fix BMP-outside collation (PR #6098) MySQL/MariaDB's utf8mb4_unicode_ci collation treats all characters outside the Basic Multilingual Plane as equal, conflating distinct same-length hashtags (e.g. Shavian vs cuneiform) on the unique name/slug indexes. Migrate the hashtags name/slug columns to utf8mb4_unicode_520_ci, which differentiates supplementary-plane characters. Improvements over the original PR: - Use an explicit ALTER ... MODIFY, since Laravel's fluent ->change() emits no collation change on MySQL/MariaDB and silently no-ops. - Match both 'mysql' and 'mariadb' drivers (Laravel 11+ reports MariaDB as a distinct driver, so a mysql-only check would skip the fix on MariaDB). - Provide an accurate, reversible down() and preserve NOT NULL + unique keys. Add feature tests covering the no-op path on non-MySQL drivers and, on MySQL/MariaDB, that distinct BMP-outside hashtags coexist while same-slug and case-insensitive dedup still work. --- ...07_31_164635_change_hashtags_collation.php | 76 ++++++++++++ tests/Feature/HashtagCollationTest.php | 108 ++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 database/migrations/2025_07_31_164635_change_hashtags_collation.php create mode 100644 tests/Feature/HashtagCollationTest.php diff --git a/database/migrations/2025_07_31_164635_change_hashtags_collation.php b/database/migrations/2025_07_31_164635_change_hashtags_collation.php new file mode 100644 index 000000000..6a1f965e1 --- /dev/null +++ b/database/migrations/2025_07_31_164635_change_hashtags_collation.php @@ -0,0 +1,76 @@ += 0x10000). The historic default + * of utf8mb4_unicode_ci treats all such characters as equal, which causes + * distinct hashtags (e.g. Shavian vs cuneiform of the same length) to + * collide on the unique name/slug indexes. + */ + private const TARGET_COLLATION = 'utf8mb4_unicode_520_ci'; + + /** + * The collation to restore on rollback (the previous project default). + */ + private const PREVIOUS_COLLATION = 'utf8mb4_unicode_ci'; + + /** + * Column definitions to keep intact while altering the collation. Both are + * VARCHAR(255) NOT NULL with unique indexes; MODIFY preserves the index. + */ + private const COLUMNS = ['name', 'slug']; + + /** + * Run the migrations. + */ + public function up(): void + { + $this->setCollation(self::TARGET_COLLATION); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + $this->setCollation(self::PREVIOUS_COLLATION); + } + + /** + * Apply the given collation to the hashtags name and slug columns. + * + * Laravel's fluent ->change() does not reliably emit a collation-only + * change on MySQL/MariaDB, so issue an explicit MODIFY per column. + */ + private function setCollation(string $collation): void + { + if (! $this->isMysql()) { + return; + } + + foreach (self::COLUMNS as $column) { + DB::statement( + 'ALTER TABLE `hashtags` MODIFY `'.$column.'` '. + 'VARCHAR(255) CHARACTER SET utf8mb4 COLLATE '.$collation.' NOT NULL' + ); + } + } + + /** + * This migration only applies to MySQL/MariaDB. Postgres compares + * hashtags with ILIKE (no collation quirk) and other drivers (e.g. the + * sqlite test database) do not support these collations. + * + * Note: Laravel 11+ reports MariaDB as the distinct "mariadb" driver, so + * both must be matched here. + */ + private function isMysql(): bool + { + return in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true); + } +}; diff --git a/tests/Feature/HashtagCollationTest.php b/tests/Feature/HashtagCollationTest.php new file mode 100644 index 000000000..bde6b4af4 --- /dev/null +++ b/tests/Feature/HashtagCollationTest.php @@ -0,0 +1,108 @@ += 0x10000). +| +*/ + +it('migration runs without error regardless of database driver', function () { + // The test suite uses sqlite by default. The migration detects the driver + // and no-ops gracefully rather than attempting unsupported ALTER syntax. + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // If we reach here without exception the no-op path worked. + expect(true)->toBeTrue(); +}); + +it('migration rollback runs without error regardless of database driver', function () { + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + Artisan::call('migrate:rollback', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + expect(true)->toBeTrue(); +}); + +it('distinct BMP-outside hashtags do not collide on the unique index', function () { + // This test is only meaningful on MySQL/MariaDB where the collation fix matters. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + // Ensure the migration has been applied. + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // Two distinct 5-character hashtags using characters outside the BMP. + // Under the old utf8mb4_unicode_ci collation these were considered equal. + $shavian = '𐑖𐑱𐑝𐑾𐑯'; // Shavian script + $cuneiform = 'π’†³π’†π’€­π’Šπ’† '; // Cuneiform script + + $tag1 = Hashtag::create(['name' => $shavian, 'slug' => $shavian]); + $tag2 = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]); + + // Both must coexist as separate rows with distinct IDs. + expect($tag1->id)->not->toBe($tag2->id); + expect(Hashtag::where('slug', $shavian)->first()->id)->toBe($tag1->id); + expect(Hashtag::where('slug', $cuneiform)->first()->id)->toBe($tag2->id); +}); + +it('same-script hashtags with the same slug still correctly deduplicate', function () { + // Sanity check: two identical hashtags should NOT create duplicates. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + $tag = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']); + $same = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']); + + expect($tag->id)->toBe($same->id); + expect(Hashtag::where('slug', 'hello')->count())->toBe(1); +}); + +it('case-insensitivity is preserved after collation change', function () { + // utf8mb4_unicode_520_ci is still case-insensitive, so #Hello == #hello. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + Hashtag::create(['name' => 'Pixelfed', 'slug' => 'pixelfed']); + + // Case-insensitive lookup should find it with different casing. + $found = Hashtag::where('slug', 'PIXELFED')->first(); + expect($found)->not->toBeNull(); + expect($found->slug)->toBe('pixelfed'); +}); From 434adee9932f92a5004b4d8cebb5c9e6dcb16a73 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 29 Aug 2026 23:18:45 +0930 Subject: [PATCH 2/2] fix: merge duplicate hashtags before recollating to avoid 1062 error The migration failed on MySQL with a 1062 duplicate-entry error: recollating name/slug to utf8mb4_unicode_520_ci makes previously-distinct values collide on the unique indexes, so the ALTER TABLE was rejected. Merge colliding rows first, comparing values under the target collation. For each collision group the lowest id is kept, references in status_hashtags, hashtag_follows, hashtag_related and discover_category_hashtags are repointed to it (UPDATE IGNORE + cleanup), and the losing rows are deleted before the collation is applied. Adds a regression test for the merge path. --- ...07_31_164635_change_hashtags_collation.php | 106 +++++++++++++++++- tests/Feature/HashtagCollationTest.php | 47 +++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/database/migrations/2025_07_31_164635_change_hashtags_collation.php b/database/migrations/2025_07_31_164635_change_hashtags_collation.php index 6a1f965e1..6b42ceeaa 100644 --- a/database/migrations/2025_07_31_164635_change_hashtags_collation.php +++ b/database/migrations/2025_07_31_164635_change_hashtags_collation.php @@ -25,22 +25,124 @@ return new class extends Migration */ private const COLUMNS = ['name', 'slug']; + /** + * Tables that reference hashtags.id via a hashtag_id column. Rows in these + * tables must be repointed from a duplicate hashtag to the surviving one + * before the duplicate is deleted, so no references are orphaned. + * + * group_post_hashtags is intentionally excluded: it references the + * separate group_hashtags table, not hashtags. + */ + private const REFERENCING_TABLES = [ + 'status_hashtags', + 'hashtag_follows', + 'hashtag_related', + 'discover_category_hashtags', + ]; + /** * Run the migrations. */ public function up(): void { + if (! $this->isMysql()) { + return; + } + + // The unique indexes on name/slug are enforced during the ALTER. Under + // the stricter target collation, rows that were previously distinct may + // now be considered equal, so merge those duplicates first to avoid a + // "Duplicate entry" (1062) failure on the ALTER TABLE. + $this->mergeDuplicates('name'); + $this->mergeDuplicates('slug'); + $this->setCollation(self::TARGET_COLLATION); } /** * Reverse the migrations. + * + * Note: merged duplicate rows are not restored on rollback (the data is + * gone). Only the collation is reverted. */ public function down(): void { + if (! $this->isMysql()) { + return; + } + $this->setCollation(self::PREVIOUS_COLLATION); } + /** + * Merge hashtags that collide on the given column under the target + * collation. For each group of colliding rows, the lowest id is kept and + * all references are repointed to it before the losing rows are deleted. + */ + private function mergeDuplicates(string $column): void + { + $collatedColumn = 'CONVERT(`'.$column.'` USING utf8mb4) COLLATE '.self::TARGET_COLLATION; + + // Group by the value compared under the target collation. Any group + // with more than one row would violate the unique index after the + // collation change. + $groups = DB::table('hashtags') + ->select(DB::raw('MIN(id) as keep_id')) + ->groupBy(DB::raw($collatedColumn)) + ->havingRaw('COUNT(*) > 1') + ->get(); + + foreach ($groups as $group) { + $keepId = (int) $group->keep_id; + + // Find the losing rows: everything in the same collated group + // except the surviving (lowest) id. + $keepValue = DB::table('hashtags')->where('id', $keepId)->value($column); + + $losers = DB::table('hashtags') + ->whereRaw($collatedColumn.' = CONVERT(? USING utf8mb4) COLLATE '.self::TARGET_COLLATION, [$keepValue]) + ->where('id', '!=', $keepId) + ->pluck('id') + ->map(fn ($id) => (int) $id) + ->all(); + + if (empty($losers)) { + continue; + } + + $this->repointReferences($losers, $keepId); + + DB::table('hashtags')->whereIn('id', $losers)->delete(); + } + } + + /** + * Repoint references from the losing hashtag ids to the surviving id. + * UPDATE IGNORE avoids aborting on unique constraints in the referencing + * tables; any rows that could not be repointed (because an equivalent + * reference to keep_id already exists) are then deleted. + */ + private function repointReferences(array $loserIds, int $keepId): void + { + $placeholders = implode(',', array_fill(0, count($loserIds), '?')); + + foreach (self::REFERENCING_TABLES as $table) { + if (! DB::getSchemaBuilder()->hasColumn($table, 'hashtag_id')) { + continue; + } + + DB::statement( + 'UPDATE IGNORE `'.$table.'` SET `hashtag_id` = ? WHERE `hashtag_id` IN ('.$placeholders.')', + array_merge([$keepId], $loserIds) + ); + + DB::statement( + 'DELETE FROM `'.$table.'` WHERE `hashtag_id` IN ('.$placeholders.')', + $loserIds + ); + } + } + /** * Apply the given collation to the hashtags name and slug columns. * @@ -49,10 +151,6 @@ return new class extends Migration */ private function setCollation(string $collation): void { - if (! $this->isMysql()) { - return; - } - foreach (self::COLUMNS as $column) { DB::statement( 'ALTER TABLE `hashtags` MODIFY `'.$column.'` '. diff --git a/tests/Feature/HashtagCollationTest.php b/tests/Feature/HashtagCollationTest.php index bde6b4af4..507d7644e 100644 --- a/tests/Feature/HashtagCollationTest.php +++ b/tests/Feature/HashtagCollationTest.php @@ -16,6 +16,10 @@ uses(LazilyRefreshDatabase::class); | and, on MySQL/MariaDB, prevents conflation of distinct hashtags that use | characters outside the Basic Multilingual Plane (codepoints >= 0x10000). | +| It also verifies that pre-existing rows which become equal under the +| stricter target collation are merged before the collation change, so the +| migration no longer fails with a duplicate-entry (1062) error. +| */ it('migration runs without error regardless of database driver', function () { @@ -50,7 +54,6 @@ it('distinct BMP-outside hashtags do not collide on the unique index', function $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); } - // Ensure the migration has been applied. Artisan::call('migrate', [ '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', '--force' => true, @@ -70,6 +73,48 @@ it('distinct BMP-outside hashtags do not collide on the unique index', function expect(Hashtag::where('slug', $cuneiform)->first()->id)->toBe($tag2->id); }); +it('merges pre-existing colliding rows instead of failing the ALTER', function () { + // Reproduces the original failure: rows that are distinct under the old + // collation but equal under the target collation must be merged, and their + // references repointed, before the unique index is re-enforced. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + // Seed the table under the OLD collation so the two BMP-outside tags are + // stored as separate rows (they only collide under the target collation). + DB::statement('ALTER TABLE `hashtags` MODIFY `name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL'); + DB::statement('ALTER TABLE `hashtags` MODIFY `slug` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL'); + + $shavian = '𐑖𐑱𐑝𐑾𐑯'; + $cuneiform = 'π’†³π’†π’€­π’Šπ’† '; + + $keep = Hashtag::create(['name' => $shavian, 'slug' => $shavian]); + $dup = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]); + + // A status_hashtags reference pointing at the row that will be merged away. + DB::table('status_hashtags')->insert([ + 'status_id' => 1, + 'hashtag_id' => $dup->id, + 'created_at' => now(), + 'updated_at' => now(), + ]); + + // Running the migration must not throw a duplicate-entry error. + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // The duplicate row is gone, the surviving row remains. + expect(Hashtag::find($dup->id))->toBeNull(); + expect(Hashtag::find($keep->id))->not->toBeNull(); + + // The reference was repointed to the surviving hashtag. + expect(DB::table('status_hashtags')->where('hashtag_id', $dup->id)->count())->toBe(0); + expect(DB::table('status_hashtags')->where('hashtag_id', $keep->id)->count())->toBe(1); +}); + it('same-script hashtags with the same slug still correctly deduplicate', function () { // Sanity check: two identical hashtags should NOT create duplicates. if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {