diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverFetcher.kt b/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverFetcher.kt
index 3fcf4321fc..923726ad1b 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverFetcher.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverFetcher.kt
@@ -43,13 +43,15 @@ class MangaCoverFetcher(
     private val manga: Manga,
     private val sourceLazy: Lazy<HttpSource?>,
     private val options: Options,
-    private val coverCache: CoverCache,
     private val callFactoryLazy: Lazy<Call.Factory>,
     private val diskCacheLazy: Lazy<DiskCache>,
+    private val diskCacheKeyLazy: Lazy<String>,
+    private val coverFileLazy: Lazy<File?>,
+    private val customCoverFileLazy: Lazy<File>,
 ) : Fetcher {
 
-    // For non-custom cover
-    private val diskCacheKey: String? by lazy { MangaCoverKeyer().key(manga, options) }
+    private val diskCacheKey: String
+        get() = diskCacheKeyLazy.value
     private lateinit var url: String
 
     val fileScope = CoroutineScope(Job() + Dispatchers.IO)
@@ -68,12 +70,13 @@ class MangaCoverFetcher(
         }
     }
 
-    @Suppress("RedundantSuspendModifier")
     private suspend fun tryCustomCover(): FetchResult? {
-        val customCoverFile by lazy { coverCache.getCustomCoverFile(manga) }
-        if (options.extras.getOrDefault(USE_CUSTOM_COVER_KEY) && customCoverFile.exists()) {
-            setRatioAndColorsInScope(manga, customCoverFile)
-            return fileLoader(customCoverFile)
+        if (options.extras.getOrDefault(USE_CUSTOM_COVER_KEY)) {
+            val customCoverFile = customCoverFileLazy.value
+            if (customCoverFile.exists()) {
+                setRatioAndColorsInScope(manga, customCoverFile)
+                return fileLoader(customCoverFile)
+            }
         }
         return null
     }
@@ -87,7 +90,7 @@ class MangaCoverFetcher(
             val customCoverLoader = tryCustomCover()
             if (customCoverLoader != null) return customCoverLoader
         }
-        val coverFile = coverCache.getCoverFile(manga.thumbnail_url, !manga.favorite)
+        val coverFile = coverFileLazy.value
         if (!shouldFetchRemotely && coverFile != null && coverFile.exists() && options.diskCachePolicy.readEnabled) {
             if (!manga.favorite) {
                 coverFile.setLastModified(Date().time)
@@ -343,8 +346,16 @@ class MangaCoverFetcher(
         private val sourceManager: SourceManager by injectLazy()
 
         override fun create(data: Manga, options: Options, imageLoader: ImageLoader): Fetcher {
-            val source = lazy { sourceManager.get(data.source) as? HttpSource }
-            return MangaCoverFetcher(data, source, options, coverCache, callFactoryLazy, diskCacheLazy)
+            return MangaCoverFetcher(
+                manga = data,
+                sourceLazy = lazy { sourceManager.get(data.source) as? HttpSource },
+                options = options,
+                callFactoryLazy = callFactoryLazy,
+                diskCacheLazy = diskCacheLazy,
+                diskCacheKeyLazy = lazy { MangaCoverKeyer().key(data, options) },
+                coverFileLazy = lazy { coverCache.getCoverFile(data.thumbnail_url, !data.favorite) },
+                customCoverFileLazy = lazy { coverCache.getCustomCoverFile(data) },
+            )
         }
     }
 
diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverKeyer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverKeyer.kt
index 3c79c5a814..fabc8af48d 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverKeyer.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/data/coil/MangaCoverKeyer.kt
@@ -7,17 +7,13 @@ import eu.kanade.tachiyomi.domain.manga.models.Manga
 import eu.kanade.tachiyomi.util.storage.DiskUtil
 
 class MangaCoverKeyer : Keyer<Manga> {
-    override fun key(data: Manga, options: Options): String? {
-        val hasCustomCover by lazy { data.hasCustomCover() }
-        val suffix by lazy { ";${data.cover_last_modified}" }
+    override fun key(data: Manga, options: Options): String {
+        val key = when {
+            data.hasCustomCover() -> data.id
+            data.favorite -> data.thumbnail_url?.let { DiskUtil.hashKeyForDisk(it) }
+            else -> data.thumbnail_url
+        }
 
-        if (data.thumbnail_url.isNullOrBlank() && !hasCustomCover) return null
-        if (hasCustomCover) return "${data.id}$suffix"
-
-        return if (!data.favorite) {
-            data.thumbnail_url!!
-        } else {
-            DiskUtil.hashKeyForDisk(data.thumbnail_url!!)
-        } + suffix
+        return "${key};${data.cover_last_modified}"
     }
 }
diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt
index 56b68adbc1..1eac65cab4 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt
@@ -235,11 +235,26 @@ fun Manga.hasCustomCover(coverCache: CoverCache = Injekt.get()): Boolean {
 /**
  * Call before updating [Manga.thumbnail_url] to ensure old cover can be cleared from cache
  */
-fun Manga.prepareCoverUpdate(coverCache: CoverCache = Injekt.get()) {
-    cover_last_modified = System.currentTimeMillis()
+fun Manga.prepareCoverUpdate(coverCache: CoverCache, remoteManga: SManga, refreshSameUrl: Boolean) {
+    // Never refresh covers if the new url is null, as the current url has possibly become invalid
+    val newUrl = remoteManga.thumbnail_url ?: return
 
-    if (!isLocal()) {
-        coverCache.deleteFromCache(this, true)
+    // Never refresh covers if the url is empty to avoid "losing" existing covers
+    if (newUrl.isEmpty()) return
+
+    if (!refreshSameUrl && thumbnail_url == newUrl) return
+
+    when {
+        isLocal() -> {
+            cover_last_modified = System.currentTimeMillis()
+        }
+        hasCustomCover(coverCache) -> {
+            coverCache.deleteFromCache(this, false)
+        }
+        else -> {
+            cover_last_modified = System.currentTimeMillis()
+            coverCache.deleteFromCache(this, false)
+        }
     }
 }
 
diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt
index ecb9a6a026..8ade774836 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt
@@ -268,10 +268,8 @@ class LibraryUpdateJob(private val context: Context, workerParams: WorkerParamet
                             null
                         }
                         if (networkManga != null) {
+                            manga.prepareCoverUpdate(coverCache, networkManga, false)
                             val thumbnailUrl = manga.thumbnail_url
-                            if (thumbnailUrl != networkManga.thumbnail_url) {
-                                manga.prepareCoverUpdate()
-                            }
                             manga.copyFrom(networkManga)
                             manga.initialized = true
                             val request: ImageRequest =
diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaDetailsPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaDetailsPresenter.kt
index e3761e5847..c4b6a61012 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaDetailsPresenter.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaDetailsPresenter.kt
@@ -389,10 +389,7 @@ class MangaDetailsPresenter(
 
             val networkManga = nManga.await()
             if (networkManga != null) {
-                if (manga.thumbnail_url != networkManga.thumbnail_url) {
-                    manga.prepareCoverUpdate()
-                }
-
+                manga.prepareCoverUpdate(coverCache, networkManga, false)
                 manga.copyFrom(networkManga)
                 manga.initialized = true