fix: merge duplicate hashtags before recollating to avoid 1062 error

The migration failed on MySQL with a 1062 duplicate-entry error: recollating
name/slug to utf8mb4_unicode_520_ci makes previously-distinct values collide
on the unique indexes, so the ALTER TABLE was rejected.

Merge colliding rows first, comparing values under the target collation. For
each collision group the lowest id is kept, references in status_hashtags,
hashtag_follows, hashtag_related and discover_category_hashtags are repointed
to it (UPDATE IGNORE + cleanup), and the losing rows are deleted before the
collation is applied. Adds a regression test for the merge path.
fix/hashtags-outside-bmp
Your Name 4 weeks ago
parent 88adf721c3
commit 434adee993

@ -25,22 +25,124 @@ return new class extends Migration
*/
private const COLUMNS = ['name', 'slug'];
/**
* Tables that reference hashtags.id via a hashtag_id column. Rows in these
* tables must be repointed from a duplicate hashtag to the surviving one
* before the duplicate is deleted, so no references are orphaned.
*
* group_post_hashtags is intentionally excluded: it references the
* separate group_hashtags table, not hashtags.
*/
private const REFERENCING_TABLES = [
'status_hashtags',
'hashtag_follows',
'hashtag_related',
'discover_category_hashtags',
];
/**
* Run the migrations.
*/
public function up(): void
{
if (! $this->isMysql()) {
return;
}
// The unique indexes on name/slug are enforced during the ALTER. Under
// the stricter target collation, rows that were previously distinct may
// now be considered equal, so merge those duplicates first to avoid a
// "Duplicate entry" (1062) failure on the ALTER TABLE.
$this->mergeDuplicates('name');
$this->mergeDuplicates('slug');
$this->setCollation(self::TARGET_COLLATION);
}
/**
* Reverse the migrations.
*
* Note: merged duplicate rows are not restored on rollback (the data is
* gone). Only the collation is reverted.
*/
public function down(): void
{
if (! $this->isMysql()) {
return;
}
$this->setCollation(self::PREVIOUS_COLLATION);
}
/**
* Merge hashtags that collide on the given column under the target
* collation. For each group of colliding rows, the lowest id is kept and
* all references are repointed to it before the losing rows are deleted.
*/
private function mergeDuplicates(string $column): void
{
$collatedColumn = 'CONVERT(`'.$column.'` USING utf8mb4) COLLATE '.self::TARGET_COLLATION;
// Group by the value compared under the target collation. Any group
// with more than one row would violate the unique index after the
// collation change.
$groups = DB::table('hashtags')
->select(DB::raw('MIN(id) as keep_id'))
->groupBy(DB::raw($collatedColumn))
->havingRaw('COUNT(*) > 1')
->get();
foreach ($groups as $group) {
$keepId = (int) $group->keep_id;
// Find the losing rows: everything in the same collated group
// except the surviving (lowest) id.
$keepValue = DB::table('hashtags')->where('id', $keepId)->value($column);
$losers = DB::table('hashtags')
->whereRaw($collatedColumn.' = CONVERT(? USING utf8mb4) COLLATE '.self::TARGET_COLLATION, [$keepValue])
->where('id', '!=', $keepId)
->pluck('id')
->map(fn ($id) => (int) $id)
->all();
if (empty($losers)) {
continue;
}
$this->repointReferences($losers, $keepId);
DB::table('hashtags')->whereIn('id', $losers)->delete();
}
}
/**
* Repoint references from the losing hashtag ids to the surviving id.
* UPDATE IGNORE avoids aborting on unique constraints in the referencing
* tables; any rows that could not be repointed (because an equivalent
* reference to keep_id already exists) are then deleted.
*/
private function repointReferences(array $loserIds, int $keepId): void
{
$placeholders = implode(',', array_fill(0, count($loserIds), '?'));
foreach (self::REFERENCING_TABLES as $table) {
if (! DB::getSchemaBuilder()->hasColumn($table, 'hashtag_id')) {
continue;
}
DB::statement(
'UPDATE IGNORE `'.$table.'` SET `hashtag_id` = ? WHERE `hashtag_id` IN ('.$placeholders.')',
array_merge([$keepId], $loserIds)
);
DB::statement(
'DELETE FROM `'.$table.'` WHERE `hashtag_id` IN ('.$placeholders.')',
$loserIds
);
}
}
/**
* Apply the given collation to the hashtags name and slug columns.
*
@ -49,10 +151,6 @@ return new class extends Migration
*/
private function setCollation(string $collation): void
{
if (! $this->isMysql()) {
return;
}
foreach (self::COLUMNS as $column) {
DB::statement(
'ALTER TABLE `hashtags` MODIFY `'.$column.'` '.

@ -16,6 +16,10 @@ uses(LazilyRefreshDatabase::class);
| and, on MySQL/MariaDB, prevents conflation of distinct hashtags that use
| characters outside the Basic Multilingual Plane (codepoints >= 0x10000).
|
| It also verifies that pre-existing rows which become equal under the
| stricter target collation are merged before the collation change, so the
| migration no longer fails with a duplicate-entry (1062) error.
|
*/
it('migration runs without error regardless of database driver', function () {
@ -50,7 +54,6 @@ it('distinct BMP-outside hashtags do not collide on the unique index', function
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
}
// Ensure the migration has been applied.
Artisan::call('migrate', [
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
'--force' => true,
@ -70,6 +73,48 @@ it('distinct BMP-outside hashtags do not collide on the unique index', function
expect(Hashtag::where('slug', $cuneiform)->first()->id)->toBe($tag2->id);
});
it('merges pre-existing colliding rows instead of failing the ALTER', function () {
// Reproduces the original failure: rows that are distinct under the old
// collation but equal under the target collation must be merged, and their
// references repointed, before the unique index is re-enforced.
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {
$this->markTestSkipped('Collation behavior is MySQL/MariaDB-specific.');
}
// Seed the table under the OLD collation so the two BMP-outside tags are
// stored as separate rows (they only collide under the target collation).
DB::statement('ALTER TABLE `hashtags` MODIFY `name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL');
DB::statement('ALTER TABLE `hashtags` MODIFY `slug` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL');
$shavian = '𐑖𐑱𐑝𐑾𐑯';
$cuneiform = '𒆳𒆍𒀭𒊏𒆠';
$keep = Hashtag::create(['name' => $shavian, 'slug' => $shavian]);
$dup = Hashtag::create(['name' => $cuneiform, 'slug' => $cuneiform]);
// A status_hashtags reference pointing at the row that will be merged away.
DB::table('status_hashtags')->insert([
'status_id' => 1,
'hashtag_id' => $dup->id,
'created_at' => now(),
'updated_at' => now(),
]);
// Running the migration must not throw a duplicate-entry error.
Artisan::call('migrate', [
'--path' => 'database/migrations/2025_07_31_164635_change_hashtags_collation.php',
'--force' => true,
]);
// The duplicate row is gone, the surviving row remains.
expect(Hashtag::find($dup->id))->toBeNull();
expect(Hashtag::find($keep->id))->not->toBeNull();
// The reference was repointed to the surviving hashtag.
expect(DB::table('status_hashtags')->where('hashtag_id', $dup->id)->count())->toBe(0);
expect(DB::table('status_hashtags')->where('hashtag_id', $keep->id)->count())->toBe(1);
});
it('same-script hashtags with the same slug still correctly deduplicate', function () {
// Sanity check: two identical hashtags should NOT create duplicates.
if (! in_array(DB::connection()->getDriverName(), ['mysql', 'mariadb'], true)) {

Loading…
Cancel
Save