diff --git a/database/migrations/2025_07_31_164635_change_hashtags_collation.php b/database/migrations/2025_07_31_164635_change_hashtags_collation.php new file mode 100644 index 000000000..6b42ceeaa --- /dev/null +++ b/database/migrations/2025_07_31_164635_change_hashtags_collation.php @@ -0,0 +1,174 @@ += 0x10000). The historic default + * of utf8mb4_unicode_ci treats all such characters as equal, which causes + * distinct hashtags (e.g. Shavian vs cuneiform of the same length) to + * collide on the unique name/slug indexes. + */ + private const TARGET_COLLATION = 'utf8mb4_unicode_520_ci'; + + /** + * The collation to restore on rollback (the previous project default). + */ + private const PREVIOUS_COLLATION = 'utf8mb4_unicode_ci'; + + /** + * Column definitions to keep intact while altering the collation. Both are + * VARCHAR(255) NOT NULL with unique indexes; MODIFY preserves the index. + */ + private const COLUMNS = ['name', 'slug']; + + /** + * Tables that reference hashtags.id via a hashtag_id column. Rows in these + * tables must be repointed from a duplicate hashtag to the surviving one + * before the duplicate is deleted, so no references are orphaned. + * + * group_post_hashtags is intentionally excluded: it references the + * separate group_hashtags table, not hashtags. + */ + private const REFERENCING_TABLES = [ + 'status_hashtags', + 'hashtag_follows', + 'hashtag_related', + 'discover_category_hashtags', + ]; + + /** + * Run the migrations. + */ + public function up(): void + { + if (! $this->isMysql()) { + return; + } + + // The unique indexes on name/slug are enforced during the ALTER. Under + // the stricter target collation, rows that were previously distinct may + // now be considered equal, so merge those duplicates first to avoid a + // "Duplicate entry" (1062) failure on the ALTER TABLE. + $this->mergeDuplicates('name'); + $this->mergeDuplicates('slug'); + + $this->setCollation(self::TARGET_COLLATION); + } + + /** + * Reverse the migrations. + * + * Note: merged duplicate rows are not restored on rollback (the data is + * gone). Only the collation is reverted. + */ + public function down(): void + { + if (! $this->isMysql()) { + return; + } + + $this->setCollation(self::PREVIOUS_COLLATION); + } + + /** + * Merge hashtags that collide on the given column under the target + * collation. For each group of colliding rows, the lowest id is kept and + * all references are repointed to it before the losing rows are deleted. + */ + private function mergeDuplicates(string $column): void + { + $collatedColumn = 'CONVERT(`'.$column.'` USING utf8mb4) COLLATE '.self::TARGET_COLLATION; + + // Group by the value compared under the target collation. Any group + // with more than one row would violate the unique index after the + // collation change. + $groups = DB::table('hashtags') + ->select(DB::raw('MIN(id) as keep_id')) + ->groupBy(DB::raw($collatedColumn)) + ->havingRaw('COUNT(*) > 1') + ->get(); + + foreach ($groups as $group) { + $keepId = (int) $group->keep_id; + + // Find the losing rows: everything in the same collated group + // except the surviving (lowest) id. + $keepValue = DB::table('hashtags')->where('id', $keepId)->value($column); + + $losers = DB::table('hashtags') + ->whereRaw($collatedColumn.' = CONVERT(? USING utf8mb4) COLLATE '.self::TARGET_COLLATION, [$keepValue]) + ->where('id', '!=', $keepId) + ->pluck('id') + ->map(fn ($id) => (int) $id) + ->all(); + + if (empty($losers)) { + continue; + } + + $this->repointReferences($losers, $keepId); + + DB::table('hashtags')->whereIn('id', $losers)->delete(); + } + } + + /** + * Repoint references from the losing hashtag ids to the surviving id. + * UPDATE IGNORE avoids aborting on unique constraints in the referencing + * tables; any rows that could not be repointed (because an equivalent + * reference to keep_id already exists) are then deleted. + */ + private function repointReferences(array $loserIds, int $keepId): void + { + $placeholders = implode(',', array_fill(0, count($loserIds), '?')); + + foreach (self::REFERENCING_TABLES as $table) { + if (! DB::getSchemaBuilder()->hasColumn($table, 'hashtag_id')) { + continue; + } + + DB::statement( + 'UPDATE IGNORE `'.$table.'` SET `hashtag_id` = ? WHERE `hashtag_id` IN ('.$placeholders.')', + array_merge([$keepId], $loserIds) + ); + + DB::statement( + 'DELETE FROM `'.$table.'` WHERE `hashtag_id` IN ('.$placeholders.')', + $loserIds + ); + } + } + + /** + * Apply the given collation to the hashtags name and slug columns. + * + * Laravel's fluent ->change() does not reliably emit a collation-only + * change on MySQL/MariaDB, so issue an explicit MODIFY per column. + */ + private function setCollation(string $collation): void + { + foreach (self::COLUMNS as $column) { + DB::statement( + 'ALTER TABLE `hashtags` MODIFY `'.$column.'` '. + 'VARCHAR(255) CHARACTER SET utf8mb4 COLLATE '.$collation.' NOT NULL' + ); + } + } + + /** + * This migration only applies to MySQL/MariaDB. Postgres compares + * hashtags with ILIKE (no collation quirk) and other drivers (e.g. the + * sqlite test database) do not support these collations. + * + * Note: Laravel 11+ reports MariaDB as the distinct "mariadb" driver, so + * both must be matched here. + */ + private function isMysql(): bool + { + return in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true); + } +}; diff --git a/tests/Feature/HashtagCollationTest.php b/tests/Feature/HashtagCollationTest.php new file mode 100644 index 000000000..507d7644e --- /dev/null +++ b/tests/Feature/HashtagCollationTest.php @@ -0,0 +1,153 @@ += 0x10000). +| +| It also verifies that pre-existing rows which become equal under the +| stricter target collation are merged before the collation change, so the +| migration no longer fails with a duplicate-entry (1062) error. +| +*/ + +it('migration runs without error regardless of database driver', function () { + // The test suite uses sqlite by default. The migration detects the driver + // and no-ops gracefully rather than attempting unsupported ALTER syntax. + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // If we reach here without exception the no-op path worked. + expect(true)->toBeTrue(); +}); + +it('migration rollback runs without error regardless of database driver', function () { + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + Artisan::call('migrate:rollback', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + expect(true)->toBeTrue(); +}); + +it('distinct BMP-outside hashtags do not collide on the unique index', function () { + // This test is only meaningful on MySQL/MariaDB where the collation fix matters. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // Two distinct 5-character hashtags using characters outside the BMP. + // Under the old utf8mb4_unicode_ci collation these were considered equal. + $shavian = '𐑖𐑱𐑝𐑾𐑯'; // Shavian script + $cuneiform = 'π’†³π’†π’€­π’Šπ’† '; // Cuneiform script + + $tag1 = Hashtag::create(['name' => $shavian, 'slug' => $shavian]); + $tag2 = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]); + + // Both must coexist as separate rows with distinct IDs. + expect($tag1->id)->not->toBe($tag2->id); + expect(Hashtag::where('slug', $shavian)->first()->id)->toBe($tag1->id); + expect(Hashtag::where('slug', $cuneiform)->first()->id)->toBe($tag2->id); +}); + +it('merges pre-existing colliding rows instead of failing the ALTER', function () { + // Reproduces the original failure: rows that are distinct under the old + // collation but equal under the target collation must be merged, and their + // references repointed, before the unique index is re-enforced. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + // Seed the table under the OLD collation so the two BMP-outside tags are + // stored as separate rows (they only collide under the target collation). + DB::statement('ALTER TABLE `hashtags` MODIFY `name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL'); + DB::statement('ALTER TABLE `hashtags` MODIFY `slug` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL'); + + $shavian = '𐑖𐑱𐑝𐑾𐑯'; + $cuneiform = 'π’†³π’†π’€­π’Šπ’† '; + + $keep = Hashtag::create(['name' => $shavian, 'slug' => $shavian]); + $dup = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]); + + // A status_hashtags reference pointing at the row that will be merged away. + DB::table('status_hashtags')->insert([ + 'status_id' => 1, + 'hashtag_id' => $dup->id, + 'created_at' => now(), + 'updated_at' => now(), + ]); + + // Running the migration must not throw a duplicate-entry error. + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + // The duplicate row is gone, the surviving row remains. + expect(Hashtag::find($dup->id))->toBeNull(); + expect(Hashtag::find($keep->id))->not->toBeNull(); + + // The reference was repointed to the surviving hashtag. + expect(DB::table('status_hashtags')->where('hashtag_id', $dup->id)->count())->toBe(0); + expect(DB::table('status_hashtags')->where('hashtag_id', $keep->id)->count())->toBe(1); +}); + +it('same-script hashtags with the same slug still correctly deduplicate', function () { + // Sanity check: two identical hashtags should NOT create duplicates. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + $tag = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']); + $same = Hashtag::firstOrCreate(['slug' => 'hello'], ['name' => 'hello']); + + expect($tag->id)->toBe($same->id); + expect(Hashtag::where('slug', 'hello')->count())->toBe(1); +}); + +it('case-insensitivity is preserved after collation change', function () { + // utf8mb4_unicode_520_ci is still case-insensitive, so #Hello == #hello. + if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) { + $this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.'); + } + + Artisan::call('migrate', [ + '--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php', + '--force' => true, + ]); + + Hashtag::create(['name' => 'Pixelfed', 'slug' => 'pixelfed']); + + // Case-insensitive lookup should find it with different casing. + $found = Hashtag::where('slug', 'PIXELFED')->first(); + expect($found)->not->toBeNull(); + expect($found->slug)->toBe('pixelfed'); +});