From 95de5cc7004a179e84b1e2a1db841b2bd84ffdbe Mon Sep 17 00:00:00 2001 From: Steven Date: Mon, 20 Oct 2025 23:05:50 +0800 Subject: [PATCH] refactor: update migration history methods --- internal/version/version.go | 6 +- store/db/mysql/migration_history.go | 7 + store/db/postgres/migration_history.go | 4 + store/db/sqlite/migration_history.go | 4 + store/driver.go | 3 + store/migration_history.go | 8 + store/migrator.go | 225 +++++++++++++++++++------ 7 files changed, 208 insertions(+), 49 deletions(-) diff --git a/internal/version/version.go b/internal/version/version.go index ca64424eb..ba57f3a2f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -21,11 +21,15 @@ func GetCurrentVersion(mode string) string { return Version } +// GetMinorVersion extracts the minor version (e.g., "0.25") from a full version string (e.g., "0.25.1"). +// Returns the minor version string or empty string if the version format is invalid. +// Version format should be "major.minor.patch" (e.g., "0.25.1"). func GetMinorVersion(version string) string { versionList := strings.Split(version, ".") - if len(versionList) < 3 { + if len(versionList) < 2 { return "" } + // Return major.minor only (first two components) return versionList[0] + "." + versionList[1] } diff --git a/store/db/mysql/migration_history.go b/store/db/mysql/migration_history.go index bc6c89fc4..fbaf5a710 100644 --- a/store/db/mysql/migration_history.go +++ b/store/db/mysql/migration_history.go @@ -6,6 +6,8 @@ import ( "github.com/usememos/memos/store" ) +// FindMigrationHistoryList retrieves all migration history records. +// NOTE: This method is deprecated along with the migration_history table. func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) { query := "SELECT `version`, UNIX_TIMESTAMP(`created_ts`) FROM `migration_history` ORDER BY `created_ts` DESC" rows, err := d.db.QueryContext(ctx, query) @@ -34,6 +36,11 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio return list, nil } +// UpsertMigrationHistory inserts or updates a migration history record. +// NOTE: This method is deprecated along with the migration_history table. +// This uses separate INSERT and SELECT queries instead of INSERT...RETURNING because +// MySQL doesn't support RETURNING clause in the same way as PostgreSQL/SQLite. +// This could have race conditions but is acceptable for deprecated transition code. func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) { stmt := "INSERT INTO `migration_history` (`version`) VALUES (?) ON DUPLICATE KEY UPDATE `version` = ?" _, err := d.db.ExecContext(ctx, stmt, upsert.Version, upsert.Version) diff --git a/store/db/postgres/migration_history.go b/store/db/postgres/migration_history.go index 385b898b4..5464dfbca 100644 --- a/store/db/postgres/migration_history.go +++ b/store/db/postgres/migration_history.go @@ -6,6 +6,8 @@ import ( "github.com/usememos/memos/store" ) +// FindMigrationHistoryList retrieves all migration history records. +// NOTE: This method is deprecated along with the migration_history table. func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) { query := "SELECT version, created_ts FROM migration_history ORDER BY created_ts DESC" rows, err := d.db.QueryContext(ctx, query) @@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio return list, nil } +// UpsertMigrationHistory inserts or updates a migration history record. +// NOTE: This method is deprecated along with the migration_history table. func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) { stmt := ` INSERT INTO migration_history ( diff --git a/store/db/sqlite/migration_history.go b/store/db/sqlite/migration_history.go index b7ec01b24..3403bcfdb 100644 --- a/store/db/sqlite/migration_history.go +++ b/store/db/sqlite/migration_history.go @@ -6,6 +6,8 @@ import ( "github.com/usememos/memos/store" ) +// FindMigrationHistoryList retrieves all migration history records. +// NOTE: This method is deprecated along with the migration_history table. func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) { query := "SELECT `version`, `created_ts` FROM `migration_history` ORDER BY `created_ts` DESC" rows, err := d.db.QueryContext(ctx, query) @@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio return list, nil } +// UpsertMigrationHistory inserts or updates a migration history record. +// NOTE: This method is deprecated along with the migration_history table. func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) { stmt := ` INSERT INTO migration_history ( diff --git a/store/driver.go b/store/driver.go index bc35464f0..cb5d6eb9c 100644 --- a/store/driver.go +++ b/store/driver.go @@ -14,6 +14,9 @@ type Driver interface { IsInitialized(ctx context.Context) (bool, error) // MigrationHistory model related methods. + // NOTE: These methods are deprecated. The migration_history table is no longer used + // for tracking schema versions. Schema version is now stored in workspace_setting. + // These methods are kept for backward compatibility to migrate existing installations. FindMigrationHistoryList(ctx context.Context, find *FindMigrationHistory) ([]*MigrationHistory, error) UpsertMigrationHistory(ctx context.Context, upsert *UpsertMigrationHistory) (*MigrationHistory, error) diff --git a/store/migration_history.go b/store/migration_history.go index 693663bef..f25b31568 100644 --- a/store/migration_history.go +++ b/store/migration_history.go @@ -1,13 +1,21 @@ package store +// MigrationHistory represents a record in the migration_history table. +// NOTE: The migration_history table is deprecated in favor of storing schema version +// in workspace_setting (BASIC setting). This is kept for backward compatibility only. +// Migration from migration_history to workspace_setting happens automatically during startup. type MigrationHistory struct { Version string CreatedTs int64 } +// UpsertMigrationHistory is used to insert or update a migration history record. +// NOTE: This is deprecated along with the migration_history table. type UpsertMigrationHistory struct { Version string } +// FindMigrationHistory is used to query migration history records. +// NOTE: This is deprecated along with the migration_history table. type FindMigrationHistory struct { } diff --git a/store/migrator.go b/store/migrator.go index 3e7f10f1d..b2409aee5 100644 --- a/store/migrator.go +++ b/store/migrator.go @@ -18,6 +18,30 @@ import ( storepb "github.com/usememos/memos/proto/gen/store" ) +// Migration System Overview: +// +// The migration system handles database schema versioning and upgrades. +// Schema version is stored in workspace_setting (the new system). +// The old migration_history table is deprecated but still supported for backward compatibility. +// +// Migration Flow: +// 1. preMigrate: Check if DB is initialized. If not, apply LATEST.sql +// 2. normalizeMigrationHistoryList: Normalize old migration_history records (for pre-0.22 installations) +// 3. migrateSchemaVersionToSetting: Migrate version from migration_history to workspace_setting +// 4. Migrate (prod mode): Apply incremental migrations from current to target version +// 5. Migrate (demo mode): Seed database with demo data +// +// Version Tracking: +// - New installations: Schema version set in workspace_setting immediately +// - Old installations: Version migrated from migration_history to workspace_setting automatically +// - Empty version: Treated as 0.0.0 and all migrations applied +// +// Migration Files: +// - Location: store/migration/{driver}/{version}/NN__description.sql +// - Naming: NN is zero-padded patch number, description is human-readable +// - Ordering: Files sorted lexicographically and applied in order +// - LATEST.sql: Full schema for new installations (faster than incremental migrations) + //go:embed migration var migrationFS embed.FS @@ -31,8 +55,59 @@ const ( // LatestSchemaFileName is the name of the latest schema file. // This file is used to apply the latest schema when no migration history is found. LatestSchemaFileName = "LATEST.sql" + + // defaultSchemaVersion is used when schema version is empty or not set. + // This handles edge cases for old installations without version tracking. + defaultSchemaVersion = "0.0.0" + + // migrationHistoryNormalizedVersion is the version where migration_history normalization was completed. + // Before 0.22, migration history had inconsistent versioning that needed normalization. + migrationHistoryNormalizedVersion = "0.22" + + // Mode constants for profile mode + modeProd = "prod" + modeDemo = "demo" ) +// getSchemaVersionOrDefault returns the schema version or default if empty. +// This ensures safe version comparisons and handles old installations. +func getSchemaVersionOrDefault(schemaVersion string) string { + if schemaVersion == "" { + return defaultSchemaVersion + } + return schemaVersion +} + +// isVersionEmpty checks if the schema version is empty or the default value. +func isVersionEmpty(schemaVersion string) bool { + return schemaVersion == "" || schemaVersion == defaultSchemaVersion +} + +// shouldApplyMigration determines if a migration file should be applied. +// It checks if the file's version is between the current DB version and target version. +func shouldApplyMigration(fileVersion, currentDBVersion, targetVersion string) bool { + currentDBVersionSafe := getSchemaVersionOrDefault(currentDBVersion) + return version.IsVersionGreaterThan(fileVersion, currentDBVersionSafe) && + version.IsVersionGreaterOrEqualThan(targetVersion, fileVersion) +} + +// validateMigrationFileName checks if a migration file follows the expected naming convention. +// Expected format: "NN__description.sql" where NN is a zero-padded number. +func validateMigrationFileName(filename string) error { + if !strings.Contains(filename, MigrateFileNameSplit) { + return errors.Errorf("invalid migration filename format (missing %s): %s", MigrateFileNameSplit, filename) + } + parts := strings.Split(filename, MigrateFileNameSplit) + if len(parts) < 2 { + return errors.Errorf("invalid migration filename format: %s", filename) + } + // Check if first part is a number + if _, err := strconv.Atoi(parts[0]); err != nil { + return errors.Errorf("migration filename must start with a number: %s", filename) + } + return nil +} + // Migrate migrates the database schema to the latest version. // It checks the current schema version and applies any necessary migrations. // It also seeds the database with initial data if in demo mode. @@ -42,7 +117,7 @@ func (s *Store) Migrate(ctx context.Context) error { } switch s.profile.Mode { - case "prod": + case modeProd: workspaceBasicSetting, err := s.GetWorkspaceBasicSetting(ctx) if err != nil { return errors.Wrap(err, "failed to get workspace basic setting") @@ -51,53 +126,21 @@ func (s *Store) Migrate(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to get current schema version") } - if version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) { + // Check for downgrade (but skip if schema version is empty - that means fresh/old installation) + if !isVersionEmpty(workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) { slog.Error("cannot downgrade schema version", slog.String("databaseVersion", workspaceBasicSetting.SchemaVersion), slog.String("currentVersion", currentSchemaVersion), ) return errors.Errorf("cannot downgrade schema version from %s to %s", workspaceBasicSetting.SchemaVersion, currentSchemaVersion) } - if version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) { - filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath())) - if err != nil { - return errors.Wrap(err, "failed to read migration files") - } - sort.Strings(filePaths) - - // Start a transaction to apply the latest schema. - tx, err := s.driver.GetDB().Begin() - if err != nil { - return errors.Wrap(err, "failed to start transaction") - } - defer tx.Rollback() - - slog.Info("start migration", slog.String("currentSchemaVersion", workspaceBasicSetting.SchemaVersion), slog.String("targetSchemaVersion", currentSchemaVersion)) - for _, filePath := range filePaths { - fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath) - if err != nil { - return errors.Wrap(err, "failed to get schema version of migrate script") - } - if version.IsVersionGreaterThan(fileSchemaVersion, workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterOrEqualThan(currentSchemaVersion, fileSchemaVersion) { - bytes, err := migrationFS.ReadFile(filePath) - if err != nil { - return errors.Wrapf(err, "failed to read minor version migration file: %s", filePath) - } - stmt := string(bytes) - if err := s.execute(ctx, tx, stmt); err != nil { - return errors.Wrapf(err, "migrate error: %s", stmt) - } - } - } - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "failed to commit transaction") - } - slog.Info("end migrate") - if err := s.updateCurrentSchemaVersion(ctx, currentSchemaVersion); err != nil { - return errors.Wrap(err, "failed to update current schema version") + // Apply migrations if needed (including when schema version is empty) + if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) { + if err := s.applyMigrations(ctx, workspaceBasicSetting.SchemaVersion, currentSchemaVersion); err != nil { + return errors.Wrap(err, "failed to apply migrations") } } - case "demo": + case modeDemo: // In demo mode, we should seed the database. if err := s.seed(ctx); err != nil { return errors.Wrap(err, "failed to seed") @@ -108,6 +151,78 @@ func (s *Store) Migrate(ctx context.Context) error { return nil } +// applyMigrations applies all necessary migration files between current and target schema versions. +// It runs all migrations in a single transaction for atomicity. +func (s *Store) applyMigrations(ctx context.Context, currentSchemaVersion, targetSchemaVersion string) error { + filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath())) + if err != nil { + return errors.Wrap(err, "failed to read migration files") + } + sort.Strings(filePaths) + + // Start a transaction to apply migrations atomically + tx, err := s.driver.GetDB().Begin() + if err != nil { + return errors.Wrap(err, "failed to start transaction") + } + defer tx.Rollback() + + // Use safe version for comparison (handles empty version case) + schemaVersionForComparison := getSchemaVersionOrDefault(currentSchemaVersion) + if isVersionEmpty(currentSchemaVersion) { + slog.Warn("schema version is empty, treating as default for migration comparison", + slog.String("defaultVersion", defaultSchemaVersion)) + } + + slog.Info("start migration", + slog.String("currentSchemaVersion", schemaVersionForComparison), + slog.String("targetSchemaVersion", targetSchemaVersion)) + + migrationsApplied := 0 + for _, filePath := range filePaths { + fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath) + if err != nil { + return errors.Wrap(err, "failed to get schema version of migrate script") + } + + if shouldApplyMigration(fileSchemaVersion, currentSchemaVersion, targetSchemaVersion) { + // Validate migration filename before applying + filename := filepath.Base(filePath) + if err := validateMigrationFileName(filename); err != nil { + slog.Warn("migration file has invalid name but will be applied", slog.String("file", filePath), slog.String("error", err.Error())) + } + + slog.Info("applying migration", + slog.String("file", filePath), + slog.String("version", fileSchemaVersion)) + + bytes, err := migrationFS.ReadFile(filePath) + if err != nil { + return errors.Wrapf(err, "failed to read migration file: %s", filePath) + } + + stmt := string(bytes) + if err := s.execute(ctx, tx, stmt); err != nil { + return errors.Wrapf(err, "failed to execute migration %s: %s", filePath, err) + } + migrationsApplied++ + } + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "failed to commit migration transaction") + } + + slog.Info("migration completed", slog.Int("migrationsApplied", migrationsApplied)) + + // Update schema version after successful migration + if err := s.updateCurrentSchemaVersion(ctx, targetSchemaVersion); err != nil { + return errors.Wrap(err, "failed to update current schema version") + } + + return nil +} + // preMigrate checks if the database is initialized and applies the latest schema if not. func (s *Store) preMigrate(ctx context.Context) error { initialized, err := s.driver.IsInitialized(ctx) @@ -127,6 +242,7 @@ func (s *Store) preMigrate(ctx context.Context) error { return errors.Wrap(err, "failed to start transaction") } defer tx.Rollback() + slog.Info("initializing new database with latest schema", slog.String("file", filePath)) if err := s.execute(ctx, tx, string(bytes)); err != nil { return errors.Errorf("failed to execute SQL file %s, err %s", filePath, err) } @@ -139,12 +255,13 @@ func (s *Store) preMigrate(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to get current schema version") } + slog.Info("database initialized successfully", slog.String("schemaVersion", schemaVersion)) if err := s.updateCurrentSchemaVersion(ctx, schemaVersion); err != nil { return errors.Wrap(err, "failed to update current schema version") } } - if s.profile.Mode == "prod" { + if s.profile.Mode == modeProd { if err := s.normalizeMigrationHistoryList(ctx); err != nil { return errors.Wrap(err, "failed to normalize migration history list") } @@ -165,11 +282,11 @@ func (s *Store) getSeedBasePath() string { // seed seeds the database with initial data. // It reads all seed files from the embedded filesystem and executes them in order. -// This is only supported for SQLite databases. +// This is only supported for SQLite databases and is used in demo mode. func (s *Store) seed(ctx context.Context) error { - // Only seed for SQLite. + // Only seed for SQLite - other databases should use production data if s.profile.Driver != "sqlite" { - slog.Warn("seed is only supported for SQLite") + slog.Warn("seed is only supported for SQLite, skipping for other databases") return nil } @@ -265,6 +382,8 @@ func (s *Store) updateCurrentSchemaVersion(ctx context.Context, schemaVersion st // normalizeMigrationHistoryList normalizes the migration history list. // It checks the existing migration history and updates it to the latest schema version if necessary. +// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table. +// This ensures that old installations (< 0.22) have their migration_history normalized before migrating to workspace_setting. func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error { migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{}) if err != nil { @@ -281,9 +400,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error { latestVersion := versions[len(versions)-1] latestMinorVersion := version.GetMinorVersion(latestVersion) - // If the latest version is greater than 0.22, return. - // As of 0.22, the migration history is already normalized. - if version.IsVersionGreaterThan(latestMinorVersion, "0.22") { + // If the latest version is greater than migrationHistoryNormalizedVersion, return. + // As of that version, the migration history is already normalized. + if version.IsVersionGreaterThan(latestMinorVersion, migrationHistoryNormalizedVersion) { return nil } @@ -318,6 +437,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error { // migrateSchemaVersionToSetting migrates the schema version from the migration history to the workspace basic setting. // It retrieves the migration history, sorts the versions, and updates the workspace basic setting if necessary. +// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table. +// The migration_history table is deprecated in favor of storing schema version in workspace_setting. +// This handles upgrades from old installations that only have migration_history but no workspace_setting. func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error { migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{}) if err != nil { @@ -337,7 +459,14 @@ func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error { if err != nil { return errors.Wrap(err, "failed to get workspace basic setting") } - if version.IsVersionGreaterOrEqualThan(workspaceBasicSetting.SchemaVersion, latestVersion) { + + // If workspace_setting has no schema version (empty), or migration_history has a newer version, update workspace_setting. + // This handles upgrades from old installations where schema version was only tracked in migration_history. + if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(latestVersion, workspaceBasicSetting.SchemaVersion) { + slog.Info("migrating schema version from migration_history to workspace_setting", + slog.String("from", workspaceBasicSetting.SchemaVersion), + slog.String("to", latestVersion), + ) if err := s.updateCurrentSchemaVersion(ctx, latestVersion); err != nil { return errors.Wrap(err, "failed to update current schema version") }