diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 4b28ad44..cc990026 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -145,11 +145,6 @@ If the environment variable is defined without **and** with the suffix at the sa - Description: A unix timestamp. If an imported listen has a timestamp before this, it will be discarded. -##### KOITO_FETCH_IMAGES_DURING_IMPORT - -- Default: `false` -- Description: When true, images will be downloaded and cached during imports. - ##### KOITO_CORS_ALLOWED_ORIGINS - Default: No CORS policy diff --git a/engine/engine.go b/engine/engine.go index fa02e021..ef344d7e 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -236,23 +236,33 @@ func Run( l.Info().Msg("Engine: Beginning startup tasks...") - l.Debug().Msg("Engine: Checking import configuration") - if !cfg.SkipImport() { - go func() { + go func() { + l.Info().Msg("Engine: Checking image cache migration status") + if err := catalog.MigrateImageCache(ctx, store); err != nil { + l.Err(err).Msg("Engine: Failed to migrate image cache") + } + l.Info().Msg("Engine: Pruning orphaned images") + if err := catalog.PruneOrphanedImages(ctx, store); err != nil { + l.Err(err).Msg("Engine: Failed to prune orphaned images") + } + + l.Debug().Msg("Engine: Checking import configuration") + if !cfg.SkipImport() { RunImporter(l, store, mbzC) - }() - } + } + + l.Info().Msg("Engine: Attempting to fetch missing artist images") + if err := catalog.FetchMissingArtistImages(ctx, store); err != nil { + l.Err(err).Msg("Engine: Failed to fetch missing artist images") + } + l.Info().Msg("Engine: Attempting to fetch missing album images") + if err := catalog.FetchMissingAlbumImages(ctx, store); err != nil { + l.Err(err).Msg("Engine: Failed to fetch missing album images") + } + }() - l.Info().Msg("Engine: Pruning orphaned images") - go catalog.PruneOrphanedImages(logger.NewContext(l), store) - l.Info().Msg("Engine: Checking image cache migration status") - go catalog.MigrateImageCache(logger.NewContext(l), store) l.Info().Msg("Engine: Running duration backfill task") go catalog.BackfillTrackDurationsFromMusicBrainz(ctx, store, mbzC) - l.Info().Msg("Engine: Attempting to fetch missing artist images") - go catalog.FetchMissingArtistImages(ctx, store) - l.Info().Msg("Engine: Attempting to fetch missing album images") - go catalog.FetchMissingAlbumImages(ctx, store) l.Info().Msg("Engine: Initialization finished") quit := make(chan os.Signal, 1) diff --git a/internal/catalog/associate_album.go b/internal/catalog/associate_album.go index c5651688..3de160c6 100644 --- a/internal/catalog/associate_album.go +++ b/internal/catalog/associate_album.go @@ -23,7 +23,7 @@ type AssociateAlbumOpts struct { ReleaseName string TrackName string // required Mbzc mbz.MusicBrainzCaller - SkipCacheImage bool + SkipImageLookup bool } func AssociateAlbum(ctx context.Context, d db.AlbumStore, opts AssociateAlbumOpts) (*models.Album, error) { @@ -121,29 +121,27 @@ func createOrUpdateAlbumWithMbzReleaseID(ctx context.Context, d db.AlbumStore, o } } - l.Debug().Msg("Searching for album images...") var imgid uuid.UUID - imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{ - Artists: utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))), - Album: release.Title, - ReleaseMbzID: &opts.ReleaseMbzID, - }) - - if err == nil && imgUrl != "" { - imgid = uuid.New() - if !opts.SkipCacheImage { + var imgUrl string + if !opts.SkipImageLookup { + l.Debug().Msg("Searching for album images...") + var imgErr error + imgUrl, imgErr = images.GetAlbumImage(ctx, images.AlbumImageOpts{ + Artists: utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))), + Album: release.Title, + ReleaseMbzID: &opts.ReleaseMbzID, + }) + if imgErr == nil && imgUrl != "" { + imgid = uuid.New() l.Debug().Msg("Downloading album image from source...") - err = imagecache.DownloadImage(imgid, imgUrl) - if err != nil { - l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image") + if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil { + l.Err(dlErr).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image") } + } else if imgErr != nil { + l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, imgErr.Error()) } } - if err != nil { - l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, err.Error()) - } - album, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{ Title: release.Title, MusicBrainzID: opts.ReleaseMbzID, @@ -210,24 +208,24 @@ func matchAlbumByTitle(ctx context.Context, d db.AlbumStore, opts AssociateAlbum return nil, fmt.Errorf("matchAlbumByTitle: %w", err) } else { var imgid uuid.UUID - imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{ - Artists: utils.FlattenArtistNames(opts.Artists), - Album: opts.ReleaseName, - ReleaseMbzID: &opts.ReleaseMbzID, - }) - if err == nil && imgUrl != "" { - imgid = uuid.New() - if !opts.SkipCacheImage { + var imgUrl string + if !opts.SkipImageLookup { + var imgErr error + imgUrl, imgErr = images.GetAlbumImage(ctx, images.AlbumImageOpts{ + Artists: utils.FlattenArtistNames(opts.Artists), + Album: opts.ReleaseName, + ReleaseMbzID: &opts.ReleaseMbzID, + }) + if imgErr == nil && imgUrl != "" { + imgid = uuid.New() l.Debug().Msg("Downloading album image from source...") - err = imagecache.DownloadImage(imgid, imgUrl) - if err != nil { - l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image") + if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil { + l.Err(dlErr).Msg("matchAlbumByTitle: failed to cache image") } + } else if imgErr != nil { + l.Debug().AnErr("error", imgErr).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName) } } - if err != nil { - l.Debug().AnErr("error", err).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName) - } a, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{ Title: releaseName, diff --git a/internal/catalog/associate_artists.go b/internal/catalog/associate_artists.go index c116e28b..61b99748 100644 --- a/internal/catalog/associate_artists.go +++ b/internal/catalog/associate_artists.go @@ -25,7 +25,7 @@ type AssociateArtistsOpts struct { TrackTitle string Mbzc mbz.MusicBrainzCaller - SkipCacheImage bool + SkipImageLookup bool } func AssociateArtists(ctx context.Context, d db.ArtistStore, opts AssociateArtistsOpts) ([]*models.Artist, error) { @@ -127,20 +127,21 @@ func matchArtistsByMBIDMappings(ctx context.Context, d db.ArtistStore, opts Asso l.Warn().AnErr("error", err).Msg("matchArtistsByMBIDMappings: MusicBrainz unreachable, creating new artist with provided MusicBrainz ID mapping") var imgid uuid.UUID - imgUrl, imgErr := images.GetArtistImage(ctx, images.ArtistImageOpts{ - Aliases: []string{a.Artist}, - }) - if imgErr == nil && imgUrl != "" { - imgid = uuid.New() - if !opts.SkipCacheImage { + var imgUrl string + if !opts.SkipImageLookup { + var imgErr error + imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{ + Aliases: []string{a.Artist}, + }) + if imgErr == nil && imgUrl != "" { + imgid = uuid.New() l.Debug().Msg("Downloading artist image from source...") - err = imagecache.DownloadImage(imgid, imgUrl) - if err != nil { - l.Err(err).Msg("Failed to cache image") + if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil { + l.Err(dlErr).Msg("Failed to cache image") } + } else { + l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist) } - } else { - l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist) } artist, err = d.SaveArtist(ctx, db.SaveArtistOpts{ @@ -240,20 +241,21 @@ func resolveAliasOrCreateArtist(ctx context.Context, mbzID uuid.UUID, names []st } var imgid uuid.UUID - imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{ - Aliases: aliases, - }) - if err == nil && imgUrl != "" { - imgid = uuid.New() - if !opts.SkipCacheImage { + var imgUrl string + if !opts.SkipImageLookup { + var imgErr error + imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{ + Aliases: aliases, + }) + if imgErr == nil && imgUrl != "" { + imgid = uuid.New() l.Debug().Msg("Downloading artist image from source...") - err = imagecache.DownloadImage(imgid, imgUrl) - if err != nil { - l.Err(err).Msg("Failed to cache image") + if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil { + l.Err(dlErr).Msg("Failed to cache image") } + } else if imgErr != nil { + l.Warn().AnErr("error", imgErr).Msg("Failed to get artist image from ImageSrc") } - } else if err != nil { - l.Warn().AnErr("error", err).Msg("Failed to get artist image from ImageSrc") } u, err := d.SaveArtist(ctx, db.SaveArtistOpts{ @@ -289,20 +291,21 @@ func matchArtistsByNames(ctx context.Context, names []string, existing []*models } if errors.Is(err, db.ErrNotFound) { var imgid uuid.UUID - imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{ - Aliases: []string{name}, - }) - if err == nil && imgUrl != "" { - imgid = uuid.New() - if !opts.SkipCacheImage { + var imgUrl string + if !opts.SkipImageLookup { + var imgErr error + imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{ + Aliases: []string{name}, + }) + if imgErr == nil && imgUrl != "" { + imgid = uuid.New() l.Debug().Msg("Downloading artist image from source...") - err = imagecache.DownloadImage(imgid, imgUrl) - if err != nil { - l.Err(err).Msg("Failed to cache image") + if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil { + l.Err(dlErr).Msg("Failed to cache image") } + } else if imgErr != nil { + l.Debug().AnErr("error", imgErr).Msgf("Failed to get artist images for %s", name) } - } else if err != nil { - l.Debug().AnErr("error", err).Msgf("Failed to get artist images for %s", name) } a, err = d.SaveArtist(ctx, db.SaveArtistOpts{Name: name, Image: imgid, ImageSrc: imgUrl}) if err != nil { diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 6ca8f038..dc3e4ed4 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -42,8 +42,8 @@ type SubmitListenOpts struct { // artist, release, release group, and track in DB SkipSaveListen bool - // When true, skips caching the images and only stores the image url in the db - SkipCacheImage bool + // When true, skips requests for album and artist images + SkipImageLookup bool MbzCaller mbz.MusicBrainzCaller ArtistNames []string @@ -88,13 +88,13 @@ func SubmitListen(ctx context.Context, store submitListenStore, opts SubmitListe ctx, store, AssociateArtistsOpts{ - ArtistMbzIDs: opts.ArtistMbzIDs, - ArtistNames: opts.ArtistNames, - ArtistName: opts.Artist, - ArtistMbidMap: opts.ArtistMbidMappings, - Mbzc: opts.MbzCaller, - TrackTitle: opts.TrackTitle, - SkipCacheImage: opts.SkipCacheImage, + ArtistMbzIDs: opts.ArtistMbzIDs, + ArtistNames: opts.ArtistNames, + ArtistName: opts.Artist, + ArtistMbidMap: opts.ArtistMbidMappings, + Mbzc: opts.MbzCaller, + TrackTitle: opts.TrackTitle, + SkipImageLookup: opts.SkipImageLookup, }) if err != nil { l.Err(err).Msg("Failed to associate artists to listen") @@ -116,7 +116,7 @@ func SubmitListen(ctx context.Context, store submitListenStore, opts SubmitListe TrackName: opts.TrackTitle, Mbzc: opts.MbzCaller, Artists: artists, - SkipCacheImage: opts.SkipCacheImage, + SkipImageLookup: opts.SkipImageLookup, }) if err != nil { l.Error().Err(err).Msg("Failed to associate release group to listen") diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 57353e96..b0a7ef12 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -57,38 +57,37 @@ type config struct { listenPort int configDir string // baseUrl string - sqliteEnabled bool - databaseUrl string - musicBrainzUrl string - musicBrainzRateLimit int - logLevel int - structuredLogging bool - lbzRelayEnabled bool - lbzRelayUrl string - lbzRelayToken string - defaultPw string - defaultUsername string - defaultTheme string - disableDeezer bool - disableCAA bool - disableMusicBrainz bool - subsonicUrl string - subsonicParams string - lastfmApiKey string - subsonicEnabled bool - skipImport bool - fetchImageDuringImport bool - allowedHosts []string - allowAllHosts bool - allowedOrigins []string - disableRateLimit bool - importThrottleMs int - userAgent string - importBefore time.Time - importAfter time.Time - artistSeparators []*regexp.Regexp - loginGate bool - forceTZ *time.Location + sqliteEnabled bool + databaseUrl string + musicBrainzUrl string + musicBrainzRateLimit int + logLevel int + structuredLogging bool + lbzRelayEnabled bool + lbzRelayUrl string + lbzRelayToken string + defaultPw string + defaultUsername string + defaultTheme string + disableDeezer bool + disableCAA bool + disableMusicBrainz bool + subsonicUrl string + subsonicParams string + lastfmApiKey string + subsonicEnabled bool + skipImport bool + allowedHosts []string + allowAllHosts bool + allowedOrigins []string + disableRateLimit bool + importThrottleMs int + userAgent string + importBefore time.Time + importAfter time.Time + artistSeparators []*regexp.Regexp + loginGate bool + forceTZ *time.Location } var ( @@ -163,7 +162,10 @@ func loadConfig(getenv func(string) string, version string) (*config, error) { cfg.disableRateLimit = parseBool(getenv(DISABLE_RATE_LIMIT_ENV)) cfg.structuredLogging = parseBool(getenv(ENABLE_STRUCTURED_LOGGING_ENV)) - cfg.fetchImageDuringImport = parseBool(getenv(FETCH_IMAGES_DURING_IMPORT_ENV)) + fetchImageDuringImport := getenv(FETCH_IMAGES_DURING_IMPORT_ENV) + if fetchImageDuringImport != "" { + return nil, fmt.Errorf(`loadConfig: %s has no effect. Images are always fetched after import`, FETCH_IMAGES_DURING_IMPORT_ENV) + } cfg.disableDeezer = parseBool(getenv(DISABLE_DEEZER_ENV)) cfg.disableCAA = parseBool(getenv(DISABLE_COVER_ART_ARCHIVE_ENV)) diff --git a/internal/cfg/getters.go b/internal/cfg/getters.go index b095ea68..1d911b48 100644 --- a/internal/cfg/getters.go +++ b/internal/cfg/getters.go @@ -181,12 +181,6 @@ func ImportWindow() (time.Time, time.Time) { return globalConfig.importBefore, globalConfig.importAfter } -func FetchImagesDuringImport() bool { - lock.RLock() - defer lock.RUnlock() - return globalConfig.fetchImageDuringImport -} - func ArtistSeparators() []*regexp.Regexp { lock.RLock() defer lock.RUnlock() diff --git a/internal/importer/lastfm.go b/internal/importer/lastfm.go index f5abcbd2..e1afb703 100644 --- a/internal/importer/lastfm.go +++ b/internal/importer/lastfm.go @@ -116,7 +116,7 @@ func ImportLastFMFile(ctx context.Context, store importStore, mbzc mbz.MusicBrai Client: "lastfm", Time: ts, UserID: 1, - SkipCacheImage: !cfg.FetchImagesDuringImport(), + SkipImageLookup: true, } err = catalog.SubmitListen(ctx, store, opts) if err != nil { diff --git a/internal/importer/listenbrainz.go b/internal/importer/listenbrainz.go index e089b34d..8aa91940 100644 --- a/internal/importer/listenbrainz.go +++ b/internal/importer/listenbrainz.go @@ -153,7 +153,7 @@ func ImportListenBrainzFile(ctx context.Context, store importStore, mbzc mbz.Mus Time: ts, UserID: 1, Client: client, - SkipCacheImage: !cfg.FetchImagesDuringImport(), + SkipImageLookup: true, } err = catalog.SubmitListen(ctx, store, opts) if err != nil { diff --git a/internal/importer/maloja.go b/internal/importer/maloja.go index 99122266..74c5fbf8 100644 --- a/internal/importer/maloja.go +++ b/internal/importer/maloja.go @@ -71,15 +71,15 @@ func ImportMalojaFile(ctx context.Context, store importStore, mbzc mbz.MusicBrai continue } opts := catalog.SubmitListenOpts{ - MbzCaller: mbzc, - Artist: item.Track.Artists[0], - ArtistNames: artists, - TrackTitle: item.Track.Title, - ReleaseTitle: item.Track.Album.Title, - Time: ts.Local(), - Client: "maloja", - UserID: 1, - SkipCacheImage: !cfg.FetchImagesDuringImport(), + MbzCaller: mbzc, + Artist: item.Track.Artists[0], + ArtistNames: artists, + TrackTitle: item.Track.Title, + ReleaseTitle: item.Track.Album.Title, + Time: ts.Local(), + Client: "maloja", + UserID: 1, + SkipImageLookup: true, } err = catalog.SubmitListen(ctx, store, opts) if err != nil { diff --git a/internal/importer/spotify.go b/internal/importer/spotify.go index 9aaa4f59..8f943652 100644 --- a/internal/importer/spotify.go +++ b/internal/importer/spotify.go @@ -58,15 +58,15 @@ func ImportSpotifyFile(ctx context.Context, store importStore, mbzc mbz.MusicBra continue } opts := catalog.SubmitListenOpts{ - MbzCaller: mbzc, - Artist: item.ArtistName, - TrackTitle: item.TrackName, - ReleaseTitle: item.AlbumName, - Duration: dur / 1000, - Time: item.Timestamp, - Client: "spotify", - UserID: 1, - SkipCacheImage: !cfg.FetchImagesDuringImport(), + MbzCaller: mbzc, + Artist: item.ArtistName, + TrackTitle: item.TrackName, + ReleaseTitle: item.AlbumName, + Duration: dur / 1000, + Time: item.Timestamp, + Client: "spotify", + UserID: 1, + SkipImageLookup: true, } err = catalog.SubmitListen(ctx, store, opts) if err != nil {