Support storing how many plays media has and other database improvements

changelog/README.md

@@ -0,0 +1,5 @@
+## Changelog Data
+
+Changelog images, etc go in this folder.
+
+[View the real changelog for Watcharr.](../CHANGELOG.md)

doc/docs/server_config/backup.md

@@ -35,7 +35,7 @@ Backing up this way is discouraged because new important files could be added la
 
 If you don't care about the warning not to backup this way, here are the "important" files that you can single out for backup:
 
-- `watcharr.db` Your database, holds all users, their watchlists, etc.
+- `watcharr.db` & `watcharr.db-wal` Your database, holds all users, their watchlists, etc.
 - `watcharr.json` Server config.
 - `img/up` Profile picture uploads.
 - `img/games` Game posters (not exactly important, but scenarios in which this folder is not backed up have not been tested, only relevant for servers with game support enabled).

server/database/db.go

@@ -1,26 +1,40 @@
 package database
 
 import (
+	"log/slog"
 	"path"
+	"time"
 
 	"github.com/sbondCo/Watcharr/config"
 	"github.com/sbondCo/Watcharr/database/entity"
+	"github.com/sbondCo/Watcharr/database/migrate"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
 // Create a new database connection.
+// Also runs migrations, etc, before returning connection.
+// Any error returned from this func should always make our app Exit (caller
+// handled).
 func New() (*gorm.DB, error) {
+	slog.Info("New: Opening new database connection")
 	// Open the database.
 	db, err := gorm.Open(
 		sqlite.Open(path.Join(config.DataPath, "watcharr.db")),
 		&gorm.Config{TranslateError: true},
 	)
 	if err != nil {
+		slog.Error("New: Opening database failed.")
+		return nil, err
+	}
+	if err := configure(db); err != nil {
+		slog.Error("New: Configuring connection failed!", "error", err)
 		return nil, err
 	}
 	// Perform auto migration.
+	slog.Info("New: AutoMigrating")
 	err = db.AutoMigrate(
+		&migrate.MigrationRecord{},
 		&entity.User{},
 		&entity.UserServices{},
 		&entity.Content{},
@@ -36,7 +50,123 @@ func New() (*gorm.DB, error) {
 		&entity.Tag{},
 	)
 	if err != nil {
+		slog.Error("New: Auto migration failed.")
+		return nil, err
+	}
+	slog.Info("New: AutoMigrated")
+	// Perform our manual migrations.
+	if err := migrate.Now(db); err != nil {
+		slog.Error("New: Manual migrations failed.", "error", err)
+		return nil, err
+	}
+	// Optimize database.
+	if err := optimize(db); err != nil {
+		slog.Error("New: Optimizing database failed.", "error", err)
 		return nil, err
 	}
 	return db, nil
 }
+
+// Configure our SQLite database connection.
+// Some PRAGMAs need to be defined per-connection, so we do that here.
+func configure(db *gorm.DB) error {
+	slog.Info("configure: Configuring connection.")
+
+	// Synchronous: https://sqlite.org/pragma.html#pragma_synchronous
+	// Configured to `FULL` because I'm slightly confused.
+	// `NORMAL` is recommended for most apps with WAL, but you
+	// lose "durability", which doesn't sound like a good thing to me...
+	// personally I'm okay with less performance for the best durability.
+	if res := db.Exec("PRAGMA synchronous=2"); res.Error != nil {
+		slog.Error("configure: Configuring synchronous failed!")
+		return res.Error
+	}
+	slog.Info("configure: Configured synchronous.")
+
+	return nil
+}
+
+// Optimize the database.
+func optimize(db *gorm.DB) error {
+	slog.Info("optimize: Running optimizations.")
+
+	// WAL Checkpoint (aka commit anything in the WAL to the main db).
+	// Seems best to do this to make sure it has happened, especially since
+	// we are running vacuum next.
+	// https://sqlite.org/pragma.html#pragma_wal_checkpoint
+	timeBeforeQuery := time.Now()
+	if res := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); res.Error != nil {
+		slog.Error("optimize: Checkpoint failed!")
+		return res.Error
+	}
+	slog.Info("optimize: Checkpointed.", "took", time.Since(timeBeforeQuery))
+
+	// Optimize pragma.
+	// Running with recommended argument for our new long-living connection.
+	// https://sqlite.org/pragma.html#pragma_optimize
+	timeBeforeQuery = time.Now()
+	if res := db.Exec("PRAGMA optimize=0x10002"); res.Error != nil {
+		slog.Error("optimize: Optimize pragma failed!")
+		return res.Error
+	}
+	slog.Info("optimize: Optimize pragma succeeded.",
+		"took", time.Since(timeBeforeQuery))
+
+	// Vacuum.
+	// > VACUUM rebuilds the database file, repacking it into a minimal amount
+	// > of disk space.
+	// https://sqlite.org/lang_vacuum.html
+	timeBeforeQuery = time.Now()
+	if res := db.Exec("VACUUM"); res.Error != nil {
+		slog.Error("optimize: Vacuum failed!")
+		return res.Error
+	}
+	slog.Info("optimize: Vacuumed successfully.",
+		"took", time.Since(timeBeforeQuery))
+
+	// WAL Checkpoint (aka commit anything in the WAL to the main db).
+	// Do this after vacuum too to ensure we have a clean slate for this
+	// startup.
+	// https://sqlite.org/pragma.html#pragma_wal_checkpoint
+	timeBeforeQuery = time.Now()
+	if res := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); res.Error != nil {
+		slog.Error("optimize: Checkpoint failed!")
+		return res.Error
+	}
+	slog.Info("optimize: Checkpointed.", "took", time.Since(timeBeforeQuery))
+
+	slog.Info("optimize: Done.")
+	return nil
+}
+
+// Optimize task that is scheduled and ran every whenever.
+// So this func is for optimzations that we want to re-run every time the
+// task is scheduled for.
+func TaskOptimize(db *gorm.DB) error {
+	slog.Info("TaskOptimize: Running optimizations.")
+
+	// WAL Checkpoint (aka commit anything in the WAL to the main db).
+	// To avoid our WAL file becoming huge, we checkpoint regularly.
+	// https://sqlite.org/pragma.html#pragma_wal_checkpoint
+	timeBeforeQuery := time.Now()
+	if res := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); res.Error != nil {
+		slog.Error("TaskOptimize: Checkpoint failed!")
+		return res.Error
+	}
+	slog.Info("TaskOptimize: Checkpointed.",
+		"took", time.Since(timeBeforeQuery))
+
+	// Optimize pragma.
+	// No args for our task as recommended.
+	// https://sqlite.org/pragma.html#pragma_optimize
+	timeBeforeQuery = time.Now()
+	if res := db.Exec("PRAGMA optimize"); res.Error != nil {
+		slog.Error("TaskOptimize: Optimize pragma failed!")
+		return res.Error
+	}
+	slog.Info("TaskOptimize: Optimize pragma succeeded.",
+		"took", time.Since(timeBeforeQuery))
+
+	slog.Info("TaskOptimize: Done.")
+	return nil
+}

server/database/dbmodel/gorm_model.go

@@ -12,3 +12,11 @@ type GormModel struct {
 	UpdatedAt time.Time      `json:"updatedAt"`
 	DeletedAt gorm.DeletedAt `gorm:"index" json:"deletedAt"`
 }
+
+// Same as GormModel, but without the DeletedAt field,
+// so use this for tables where we don't need soft deletion.
+type GormModelNoDel struct {
+	ID        uint      `gorm:"primarykey" json:"id"`
+	CreatedAt time.Time `json:"createdAt"`
+	UpdatedAt time.Time `json:"updatedAt"`
+}

server/database/entity/activity.go

@@ -54,4 +54,18 @@ type Activity struct {
 	Data string `json:"data" gorm:"not null"`
 	// Custom date for the activity, that the user can define.
 	CustomDate *time.Time `json:"customDate,omitempty"`
+	// Count this Activity as a Play?
+	// Currently this was the best way I could see forward for implementing
+	// counting plays of media that doesn't involve inefficient querying of
+	// the actitivties table (or a whole new table, which would create extra
+	// complexities itself, ie, plays/activity showing different records).
+	// We write to this field when creating the activity to count is as a play
+	// or not (ie when we create STATUS CHANGE activities with status of
+	// FINISHED, imports, etc).
+	// We won't support the user (or the system) modifying this value after
+	// creation; if the user wants to delete a 'Play', they should delete the
+	// activity.
+	// Indexed (check migrations) to make search faster, since we frequently
+	// do it over the whole table for watched sorting at the moment.
+	CountAsPlay bool `json:"countAsPlay"`
 }

server/database/entity/watched.go

@@ -12,6 +12,18 @@ const (
 	DROPPED  WatchedStatus = "DROPPED"
 )
 
+func (r WatchedStatus) IsValid() bool {
+	switch r {
+	case FINISHED,
+		WATCHING,
+		PLANNED,
+		HOLD,
+		DROPPED:
+		return true
+	}
+	return false
+}
+
 type Watched struct {
 	dbmodel.GormModel
 	Status WatchedStatus `json:"status"`

server/database/entity/watched_episode.go

@@ -8,7 +8,7 @@ import "github.com/sbondCo/Watcharr/database/dbmodel"
 // since episodes can be removed and re-added. For this reason we store season and episodes nums instead
 // of just the episode id.
 type WatchedEpisode struct {
-	dbmodel.GormModel
+	dbmodel.GormModelNoDel
 	UserID        uint          `json:"-" gorm:"not null"`
 	User          User          `json:"-"`
 	WatchedID     uint          `json:"-" gorm:"uniqueIndex:we_watched_to_ens;not null"`

server/database/entity/watched_season.go

@@ -4,7 +4,7 @@ import "github.com/sbondCo/Watcharr/database/dbmodel"
 
 // UniqueIndex applied between WatchedID and SeasonNumber to avoid duplicates incase logic fails.
 type WatchedSeason struct {
-	dbmodel.GormModel
+	dbmodel.GormModelNoDel
 	UserID       uint          `json:"-" gorm:"not null"`
 	User         User          `json:"-"`
 	WatchedID    uint          `json:"-" gorm:"uniqueIndex:ws_watched_to_season_num;not null"`

server/database/migrate/entity.go

@@ -0,0 +1,11 @@
+package migrate
+
+import "time"
+
+// Record of all our applied migrations.
+type MigrationRecord struct {
+	// Migration ID
+	ID string `gorm:"primarykey"`
+	// When migration was applied on this db.
+	CreatedAt time.Time
+}

server/database/migrate/migrate.go

@@ -0,0 +1,100 @@
+package migrate
+
+import (
+	"log/slog"
+	"time"
+
+	"gorm.io/gorm"
+)
+
+type Migration struct {
+	// ID of migration, stick to YYYYMMDDHHMM.
+	ID string
+	// Apply migration func.
+	Up func(tx *gorm.DB) error
+	// When `true`, the migration is not run inside of a transaction.
+	// You should only use this when is required by sqlite that the command
+	// we need to run for eg cannot be ran from within a transaction!
+	// YOU SHOULD ENSURE YOU ONLY RUN ONE COMMAND PER MIGRATION WHEN USING
+	// THIS WITH `TRUE` TO AVOID BEING LEFT IN A BAD OR INCOMPLETE STATE!!!
+	//
+	// ALSO: All statements used with this should take into account that since,
+	// it isn't inside of a transaction, it's possible the migration succeeds,
+	// but creating the record of it doesn't. If a user starts the server again
+	// after we error in this case, the migration will run again, so it must
+	// not break data integrity or make any assumptions of it being the first
+	// time running!
+	UNSAFE bool
+}
+
+// Start our migrations.
+// NOTE: This is only to be ran after GORM's AutoMigrate.
+func Now(db *gorm.DB) error {
+	slog.Info("Starting migrations.")
+
+	for _, mig := range migrations {
+		slog.Debug("Processing migration.", "id", mig.ID)
+
+		migRecord := MigrationRecord{ID: mig.ID}
+
+		// First ensure that the migration hasn't already been applied.
+		var alreadyApplied int64
+		res := db.
+			Model(&MigrationRecord{}).
+			Where(&migRecord).
+			Count(&alreadyApplied)
+		if res.Error != nil {
+			slog.Error("already applied check failed!")
+			return res.Error
+		}
+		if alreadyApplied > 0 {
+			// If record exists in our table, then migration was applied
+			// already, so skip processing it.
+			slog.Debug("Migration already applied.", "id", mig.ID)
+			continue
+		}
+
+		// Timing the migration.
+		timeBeforeMig := time.Now()
+
+		// Apply the migration.
+		if mig.UNSAFE {
+			// Unsafe migrations are not ran inside of a transaction
+			// and are only used when required by sqlite engine.
+			if err := mig.Up(db); err != nil {
+				slog.Error("Migration failed!", "id", mig.ID, "error", err)
+				return err
+			}
+			// Record the migration record.
+			if res := db.Create(&migRecord); res.Error != nil {
+				slog.Error("Unsafe migration succeeded, but we failed to create the record of it!",
+					"id", mig.ID, "error", res.Error)
+				return res.Error
+			}
+		} else {
+			// Migrations go through a transaction wrapper.
+			err := db.Transaction(func(tx *gorm.DB) error {
+				if err := mig.Up(tx); err != nil {
+					// Errored.. rollback any changes made.
+					return err
+				}
+				// Migration succeeded.. record it.
+				// If the Create succeeds, all will be committed.
+				return tx.Create(&migRecord).Error
+			})
+			if err != nil {
+				// If any migration fails, we return here.
+				slog.Error("Migration failed!", "id", mig.ID, "error", err)
+				return err
+			}
+		}
+
+		slog.Debug("Migration applied successfully.",
+			"id", mig.ID,
+			"duration", time.Since(timeBeforeMig))
+	}
+
+	slog.Info("Done processing all migrations.")
+
+	return nil
+}