From ca98367a0acd20f27888bed8f1410ac4528d3a03 Mon Sep 17 00:00:00 2001 From: Athurg Gooth Date: Wed, 27 Sep 2023 09:27:31 +0800 Subject: [PATCH] chore: store vacuum and clean (#2293) * Move all vacuum code into driver * Remove db from Store --- cmd/memos.go | 3 +- cmd/mvrss.go | 3 +- cmd/setup.go | 3 +- server/server.go | 2 +- store/driver.go | 4 ++ store/memo.go | 29 +---------- store/memo_organizer.go | 26 ---------- store/memo_relation.go | 11 ---- store/resource.go | 31 +----------- store/sqlite/memo.go | 23 +++++++++ store/sqlite/memo_organizer.go | 26 ++++++++++ store/sqlite/memo_relation.go | 11 ++++ store/sqlite/resource.go | 24 +++++++++ store/sqlite/sqlite.go | 92 +++++++++++++++++++++++++++++++++ store/sqlite/tag.go | 20 ++++++++ store/sqlite/user.go | 6 +++ store/sqlite/user_setting.go | 20 ++++++++ store/store.go | 93 ++-------------------------------- store/tag.go | 20 -------- store/user.go | 4 -- store/user_setting.go | 20 -------- test/server/server.go | 3 +- test/store/store.go | 3 +- 23 files changed, 239 insertions(+), 238 deletions(-) diff --git a/cmd/memos.go b/cmd/memos.go index 225470fef..0f0e06ad0 100644 --- a/cmd/memos.go +++ b/cmd/memos.go @@ -56,8 +56,7 @@ var ( } driver := sqlite.NewDriver(db.DBInstance) - - store := store.New(db.DBInstance, driver, profile) + store := store.New(driver, profile) s, err := server.NewServer(ctx, profile, store) if err != nil { cancel() diff --git a/cmd/mvrss.go b/cmd/mvrss.go index aec8385e7..310cc74ef 100644 --- a/cmd/mvrss.go +++ b/cmd/mvrss.go @@ -51,8 +51,7 @@ var ( } driver := sqlite.NewDriver(db.DBInstance) - - s := store.New(db.DBInstance, driver, profile) + s := store.New(driver, profile) resources, err := s.ListResources(ctx, &store.FindResource{}) if err != nil { fmt.Printf("failed to list resources, error: %+v\n", err) diff --git a/cmd/setup.go b/cmd/setup.go index c491974c7..c1a36bec9 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -48,8 +48,7 @@ var ( } driver := sqlite.NewDriver(db.DBInstance) - - store := store.New(db.DBInstance, driver, profile) + store := store.New(driver, profile) if err := ExecuteSetup(ctx, store, hostUsername, hostPassword); err != nil { fmt.Printf("failed to setup, error: %+v\n", err) return diff --git a/server/server.go b/server/server.go index f074a8efc..616860b4d 100644 --- a/server/server.go +++ b/server/server.go @@ -162,7 +162,7 @@ func (s *Server) Shutdown(ctx context.Context) { } // Close database connection - if err := s.Store.GetDB().Close(); err != nil { + if err := s.Store.Close(); err != nil { fmt.Printf("failed to close database, error: %v\n", err) } diff --git a/store/driver.go b/store/driver.go index e10ae63a7..6cde96b19 100644 --- a/store/driver.go +++ b/store/driver.go @@ -7,6 +7,10 @@ import ( ) type Driver interface { + Vacuum(ctx context.Context) error + BackupTo(ctx context.Context, filename string) error + Close() error + CreateActivity(ctx context.Context, create *Activity) (*Activity, error) CreateResource(ctx context.Context, create *Resource) (*Resource, error) diff --git a/store/memo.go b/store/memo.go index 3dab08027..a37d61b9e 100644 --- a/store/memo.go +++ b/store/memo.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" ) // Visibility is the type of a visibility. @@ -107,35 +106,9 @@ func (s *Store) UpdateMemo(ctx context.Context, update *UpdateMemo) error { } func (s *Store) DeleteMemo(ctx context.Context, delete *DeleteMemo) error { - if err := s.driver.DeleteMemo(ctx, delete); err != nil { - return err - } - if err := s.Vacuum(ctx); err != nil { - // Prevent linter warning. - return err - } - return nil + return s.driver.DeleteMemo(ctx, delete) } func (s *Store) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) ([]Visibility, error) { return s.driver.FindMemosVisibilityList(ctx, memoIDs) } - -func vacuumMemo(ctx context.Context, tx *sql.Tx) error { - stmt := ` - DELETE FROM - memo - WHERE - creator_id NOT IN ( - SELECT - id - FROM - user - )` - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - - return nil -} diff --git a/store/memo_organizer.go b/store/memo_organizer.go index 84a843c6c..5c153adae 100644 --- a/store/memo_organizer.go +++ b/store/memo_organizer.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" ) type MemoOrganizer struct { @@ -32,28 +31,3 @@ func (s *Store) GetMemoOrganizer(ctx context.Context, find *FindMemoOrganizer) ( func (s *Store) DeleteMemoOrganizer(ctx context.Context, delete *DeleteMemoOrganizer) error { return s.driver.DeleteMemoOrganizer(ctx, delete) } - -func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error { - stmt := ` - DELETE FROM - memo_organizer - WHERE - memo_id NOT IN ( - SELECT - id - FROM - memo - ) - OR user_id NOT IN ( - SELECT - id - FROM - user - )` - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - - return nil -} diff --git a/store/memo_relation.go b/store/memo_relation.go index 351062520..5555c26ca 100644 --- a/store/memo_relation.go +++ b/store/memo_relation.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" ) type MemoRelationType string @@ -54,13 +53,3 @@ func (s *Store) GetMemoRelation(ctx context.Context, find *FindMemoRelation) (*M func (s *Store) DeleteMemoRelation(ctx context.Context, delete *DeleteMemoRelation) error { return s.driver.DeleteMemoRelation(ctx, delete) } - -func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error { - if _, err := tx.ExecContext(ctx, ` - DELETE FROM memo_relation - WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo) - `); err != nil { - return err - } - return nil -} diff --git a/store/resource.go b/store/resource.go index 8e66df93f..87c149a16 100644 --- a/store/resource.go +++ b/store/resource.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" ) type Resource struct { @@ -75,33 +74,5 @@ func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) (*Re } func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) error { - err := s.driver.DeleteResource(ctx, delete) - if err != nil { - return err - } - - if err := s.Vacuum(ctx); err != nil { - // Prevent linter warning. - return err - } - return nil -} - -func vacuumResource(ctx context.Context, tx *sql.Tx) error { - stmt := ` - DELETE FROM - resource - WHERE - creator_id NOT IN ( - SELECT - id - FROM - user - )` - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - - return nil + return s.driver.DeleteResource(ctx, delete) } diff --git a/store/sqlite/memo.go b/store/sqlite/memo.go index f408811d8..9459c3e19 100644 --- a/store/sqlite/memo.go +++ b/store/sqlite/memo.go @@ -234,6 +234,10 @@ func (d *Driver) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error return err } + if err := d.Vacuum(ctx); err != nil { + // Prevent linter warning. + return err + } return nil } @@ -268,3 +272,22 @@ func (d *Driver) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) ( return visibilityList, nil } + +func vacuumMemo(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + memo + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + + return nil +} diff --git a/store/sqlite/memo_organizer.go b/store/sqlite/memo_organizer.go index f46421f9b..245e88a59 100644 --- a/store/sqlite/memo_organizer.go +++ b/store/sqlite/memo_organizer.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "database/sql" "fmt" "strings" @@ -80,3 +81,28 @@ func (d *Driver) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMe } return nil } + +func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + memo_organizer + WHERE + memo_id NOT IN ( + SELECT + id + FROM + memo + ) + OR user_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + + return nil +} diff --git a/store/sqlite/memo_relation.go b/store/sqlite/memo_relation.go index 86f6de018..c0fcdd2e2 100644 --- a/store/sqlite/memo_relation.go +++ b/store/sqlite/memo_relation.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "database/sql" "strings" "github.com/usememos/memos/store" @@ -104,3 +105,13 @@ func (d *Driver) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMem } return nil } + +func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, ` + DELETE FROM memo_relation + WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo) + `); err != nil { + return err + } + return nil +} diff --git a/store/sqlite/resource.go b/store/sqlite/resource.go index b71802a77..9095e3446 100644 --- a/store/sqlite/resource.go +++ b/store/sqlite/resource.go @@ -181,5 +181,29 @@ func (d *Driver) DeleteResource(ctx context.Context, delete *store.DeleteResourc return err } + if err := d.Vacuum(ctx); err != nil { + // Prevent linter warning. + return err + } + + return nil +} + +func vacuumResource(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + resource + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + return nil } diff --git a/store/sqlite/sqlite.go b/store/sqlite/sqlite.go index 69d5c3548..b9e53fdd0 100644 --- a/store/sqlite/sqlite.go +++ b/store/sqlite/sqlite.go @@ -1,8 +1,12 @@ package sqlite import ( + "context" "database/sql" + "github.com/pkg/errors" + "modernc.org/sqlite" + "github.com/usememos/memos/store" ) @@ -15,3 +19,91 @@ func NewDriver(db *sql.DB) store.Driver { db: db, } } + +func (d *Driver) Vacuum(ctx context.Context) error { + tx, err := d.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + if err := vacuumImpl(ctx, tx); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + // Vacuum sqlite database file size after deleting resource. + if _, err := d.db.Exec("VACUUM"); err != nil { + return err + } + + return nil +} + +func vacuumImpl(ctx context.Context, tx *sql.Tx) error { + if err := vacuumMemo(ctx, tx); err != nil { + return err + } + if err := vacuumResource(ctx, tx); err != nil { + return err + } + if err := vacuumUserSetting(ctx, tx); err != nil { + return err + } + if err := vacuumMemoOrganizer(ctx, tx); err != nil { + return err + } + if err := vacuumMemoRelations(ctx, tx); err != nil { + return err + } + if err := vacuumTag(ctx, tx); err != nil { + // Prevent revive warning. + return err + } + + return nil +} + +func (d *Driver) BackupTo(ctx context.Context, filename string) error { + conn, err := d.db.Conn(ctx) + if err != nil { + return errors.Errorf("fail to get conn %s", err) + } + defer conn.Close() + + err = conn.Raw(func(driverConn any) error { + type backuper interface { + NewBackup(string) (*sqlite.Backup, error) + } + backupConn, ok := driverConn.(backuper) + if !ok { + return errors.Errorf("db connection is not a sqlite backuper") + } + + bck, err := backupConn.NewBackup(filename) + if err != nil { + return errors.Errorf("fail to create sqlite backup %s", err) + } + + for more := true; more; { + more, err = bck.Step(-1) + if err != nil { + return errors.Errorf("fail to execute sqlite backup %s", err) + } + } + + return bck.Finish() + }) + if err != nil { + return errors.Errorf("fail to backup %s", err) + } + + return nil +} + +func (d *Driver) Close() error { + return d.db.Close() +} diff --git a/store/sqlite/tag.go b/store/sqlite/tag.go index 73400e91e..0132b3f41 100644 --- a/store/sqlite/tag.go +++ b/store/sqlite/tag.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "database/sql" "strings" "github.com/usememos/memos/store" @@ -73,3 +74,22 @@ func (d *Driver) DeleteTag(ctx context.Context, delete *store.DeleteTag) error { } return nil } + +func vacuumTag(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + tag + WHERE + creator_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + + return nil +} diff --git a/store/sqlite/user.go b/store/sqlite/user.go index e0d4c7b0f..1e97d9b50 100644 --- a/store/sqlite/user.go +++ b/store/sqlite/user.go @@ -168,5 +168,11 @@ func (d *Driver) DeleteUser(ctx context.Context, delete *store.DeleteUser) error if _, err := result.RowsAffected(); err != nil { return err } + + if err := d.Vacuum(ctx); err != nil { + // Prevent linter warning. + return err + } + return nil } diff --git a/store/sqlite/user_setting.go b/store/sqlite/user_setting.go index 75c7c3fec..f1916ec7f 100644 --- a/store/sqlite/user_setting.go +++ b/store/sqlite/user_setting.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "database/sql" "errors" "strings" @@ -153,3 +154,22 @@ func (d *Driver) ListUserSettingsV1(ctx context.Context, find *store.FindUserSet return userSettingList, nil } + +func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error { + stmt := ` + DELETE FROM + user_setting + WHERE + user_id NOT IN ( + SELECT + id + FROM + user + )` + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + + return nil +} diff --git a/store/store.go b/store/store.go index d4287b252..91ef61757 100644 --- a/store/store.go +++ b/store/store.go @@ -2,20 +2,14 @@ package store import ( "context" - "database/sql" "sync" - "modernc.org/sqlite" - - "github.com/pkg/errors" - "github.com/usememos/memos/server/profile" ) // Store provides database access to all raw objects. type Store struct { Profile *profile.Profile - db *sql.DB driver Driver systemSettingCache sync.Map // map[string]*SystemSetting userCache sync.Map // map[int]*User @@ -24,98 +18,21 @@ type Store struct { } // New creates a new instance of Store. -func New(db *sql.DB, driver Driver, profile *profile.Profile) *Store { +func New(driver Driver, profile *profile.Profile) *Store { return &Store{ Profile: profile, - db: db, driver: driver, } } -func (s *Store) GetDB() *sql.DB { - return s.db -} - func (s *Store) BackupTo(ctx context.Context, filename string) error { - conn, err := s.db.Conn(ctx) - if err != nil { - return errors.Errorf("fail to get conn %s", err) - } - defer conn.Close() - - err = conn.Raw(func(driverConn any) error { - type backuper interface { - NewBackup(string) (*sqlite.Backup, error) - } - backupConn, ok := driverConn.(backuper) - if !ok { - return errors.Errorf("db connection is not a sqlite backuper") - } - - bck, err := backupConn.NewBackup(filename) - if err != nil { - return errors.Errorf("fail to create sqlite backup %s", err) - } - - for more := true; more; { - more, err = bck.Step(-1) - if err != nil { - return errors.Errorf("fail to execute sqlite backup %s", err) - } - } - - return bck.Finish() - }) - if err != nil { - return errors.Errorf("fail to backup %s", err) - } - - return nil + return s.driver.BackupTo(ctx, filename) } func (s *Store) Vacuum(ctx context.Context) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() - - if err := s.vacuumImpl(ctx, tx); err != nil { - return err - } - - if err := tx.Commit(); err != nil { - return err - } - - // Vacuum sqlite database file size after deleting resource. - if _, err := s.db.Exec("VACUUM"); err != nil { - return err - } - - return nil + return s.driver.Vacuum(ctx) } -func (*Store) vacuumImpl(ctx context.Context, tx *sql.Tx) error { - if err := vacuumMemo(ctx, tx); err != nil { - return err - } - if err := vacuumResource(ctx, tx); err != nil { - return err - } - if err := vacuumUserSetting(ctx, tx); err != nil { - return err - } - if err := vacuumMemoOrganizer(ctx, tx); err != nil { - return err - } - if err := vacuumMemoRelations(ctx, tx); err != nil { - return err - } - if err := vacuumTag(ctx, tx); err != nil { - // Prevent revive warning. - return err - } - - return nil +func (s *Store) Close() error { + return s.driver.Close() } diff --git a/store/tag.go b/store/tag.go index afea07399..d5c38f6bf 100644 --- a/store/tag.go +++ b/store/tag.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" ) type Tag struct { @@ -30,22 +29,3 @@ func (s *Store) ListTags(ctx context.Context, find *FindTag) ([]*Tag, error) { func (s *Store) DeleteTag(ctx context.Context, delete *DeleteTag) error { return s.driver.DeleteTag(ctx, delete) } - -func vacuumTag(ctx context.Context, tx *sql.Tx) error { - stmt := ` - DELETE FROM - tag - WHERE - creator_id NOT IN ( - SELECT - id - FROM - user - )` - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - - return nil -} diff --git a/store/user.go b/store/user.go index 17948fa7a..6afd9dfb9 100644 --- a/store/user.go +++ b/store/user.go @@ -130,10 +130,6 @@ func (s *Store) DeleteUser(ctx context.Context, delete *DeleteUser) error { return err } - if err := s.Vacuum(ctx); err != nil { - // Prevent linter warning. - return err - } s.userCache.Delete(delete.ID) return nil } diff --git a/store/user_setting.go b/store/user_setting.go index 7726f7a83..98c4234c0 100644 --- a/store/user_setting.go +++ b/store/user_setting.go @@ -2,7 +2,6 @@ package store import ( "context" - "database/sql" storepb "github.com/usememos/memos/proto/gen/store" ) @@ -125,22 +124,3 @@ func (s *Store) GetUserAccessTokens(ctx context.Context, userID int32) ([]*store accessTokensUserSetting := userSetting.GetAccessTokens() return accessTokensUserSetting.AccessTokens, nil } - -func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error { - stmt := ` - DELETE FROM - user_setting - WHERE - user_id NOT IN ( - SELECT - id - FROM - user - )` - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - - return nil -} diff --git a/test/server/server.go b/test/server/server.go index 5bf4daeb9..1bf400d13 100644 --- a/test/server/server.go +++ b/test/server/server.go @@ -41,8 +41,7 @@ func NewTestingServer(ctx context.Context, t *testing.T) (*TestingServer, error) } driver := sqlite.NewDriver(db.DBInstance) - - store := store.New(db.DBInstance, driver, profile) + store := store.New(driver, profile) server, err := server.NewServer(ctx, profile, store) if err != nil { return nil, errors.Wrap(err, "failed to create server") diff --git a/test/store/store.go b/test/store/store.go index fdae9d888..d530fe685 100644 --- a/test/store/store.go +++ b/test/store/store.go @@ -25,7 +25,6 @@ func NewTestingStore(ctx context.Context, t *testing.T) *store.Store { } driver := sqlite.NewDriver(db.DBInstance) - - store := store.New(db.DBInstance, driver, profile) + store := store.New(driver, profile) return store }