refactor: Async download

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt

@@ -1,22 +1,57 @@
 package eu.kanade.tachiyomi.data.download
 
+import android.app.Application
 import android.content.Context
+import android.net.Uri
+import co.touchlab.kermit.Logger
 import com.hippo.unifile.UniFile
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.extension.ExtensionManager
+import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.util.storage.DiskUtil
+import eu.kanade.tachiyomi.util.system.extension
+import eu.kanade.tachiyomi.util.system.launchNonCancellable
+import eu.kanade.tachiyomi.util.system.nameWithoutExtension
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.SharingStarted
+import kotlinx.coroutines.flow.debounce
+import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.flow.onStart
+import kotlinx.coroutines.flow.receiveAsFlow
+import kotlinx.coroutines.flow.shareIn
+import kotlinx.coroutines.flow.stateIn
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.withTimeoutOrNull
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.descriptors.PrimitiveKind
+import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.protobuf.ProtoBuf
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
+import yokai.domain.manga.interactor.GetManga
 import yokai.domain.storage.StorageManager
+import java.io.File
 import java.util.concurrent.*
+import kotlin.time.Duration.Companion.seconds
 
 /**
  * Cache where we dump the downloads directory from the filesystem. This class is needed because
@@ -34,7 +69,14 @@ class DownloadCache(
     private val provider: DownloadProvider,
     private val sourceManager: SourceManager,
     private val storageManager: StorageManager = Injekt.get(),
+    private val extensionManager: ExtensionManager = Injekt.get(),
 ) {
+    val scope = CoroutineScope(Dispatchers.IO)
+
+    private val _changes: Channel<Unit> = Channel(Channel.UNLIMITED)
+    val changes = _changes.receiveAsFlow()
+        .onStart { emit(Unit) }
+        .shareIn(scope, SharingStarted.Lazily, 1)
 
     /**
      * The interval after which this cache should be invalidated. 1 hour shouldn't cause major
@@ -46,14 +88,42 @@ class DownloadCache(
      * The last time the cache was refreshed.
      */
     private var lastRenew = 0L
+    private var renewalJob: Job? = null
+
+    private val _isInitializing = MutableStateFlow(false)
+    val isInitializing = _isInitializing
+        .debounce(1000L) // Don't notify if it finishes quickly enough
+        .stateIn(scope, SharingStarted.WhileSubscribed(), false)
 
     private var mangaFiles: MutableMap<Long, MutableSet<String>> = mutableMapOf()
 
-    val scope = CoroutineScope(Job() + Dispatchers.IO)
+    private val diskCacheFile: File
+        get() = File(context.cacheDir, "dl_index_cache_v3")
+
+    private val rootDownloadsDirLock = Mutex()
+    private var rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory())
 
     init {
+        // Attempt to read cache file
+        scope.launch {
+            rootDownloadsDirLock.withLock {
+                try {
+                    if (diskCacheFile.exists()) {
+                        val diskCache = diskCacheFile.inputStream().use {
+                            ProtoBuf.decodeFromByteArray<RootDirectory>(it.readBytes())
+                        }
+                        rootDownloadsDir = diskCache
+                        lastRenew = System.currentTimeMillis()
+                    }
+                } catch (e: Throwable) {
+                    Logger.e(e) { "Failed to initialize disk cache" }
+                    diskCacheFile.delete()
+                }
+            }
+        }
+
         storageManager.changes
-            .onEach { forceRenewCache() } // invalidate cache
+            .onEach { invalidateCache() }
             .launchIn(scope)
     }
 
@@ -64,18 +134,28 @@ class DownloadCache(
      * @param manga the manga of the chapter.
      * @param skipCache whether to skip the directory cache and check in the filesystem.
      */
-    fun isChapterDownloaded(chapter: Chapter, manga: Manga, skipCache: Boolean): Boolean {
+    fun isChapterDownloaded(
+        chapter: Chapter,
+        manga: Manga,
+        skipCache: Boolean,
+    ): Boolean {
+        val sourceId = manga.source
+
         if (skipCache) {
-            val source = sourceManager.get(manga.source) ?: return false
+            val source = sourceManager.get(sourceId) ?: return false
             return provider.findChapterDir(chapter, manga, source) != null
         }
 
-        checkRenew()
+        renewCache()
 
-        val files = mangaFiles[manga.id]?.toHashSet() ?: return false
+        val sourceDir = rootDownloadsDir.sourceDirs[sourceId]
-        return provider.getValidChapterDirNames(chapter).any { chapName ->
+        if (sourceDir != null) {
-            files.any { chapName.equals(it, true) || "$chapName.cbz".equals(it, true) }
+            val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)]
+            if (mangaDir != null) {
+                return provider.getValidChapterDirNames(chapter).any { it in mangaDir.files }
+            }
         }
+        return false
     }
 
     /**
@@ -84,85 +164,96 @@ class DownloadCache(
      * @param manga the manga to check.
      */
     fun getDownloadCount(manga: Manga, forceCheckFolder: Boolean = false): Int {
-        checkRenew()
+        renewCache()
-
-        if (forceCheckFolder) {
-            val source = sourceManager.get(manga.source) ?: return 0
-            val mangaDir = provider.findMangaDir(manga, source)
 
+        val sourceDir = rootDownloadsDir.sourceDirs[manga.source]
+        if (sourceDir != null) {
+            val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)]
             if (mangaDir != null) {
-                val listFiles =
+                return mangaDir.files.size
-                    mangaDir.listFiles { _, filename -> !filename.endsWith(Downloader.TMP_DIR_SUFFIX) }
-                if (!listFiles.isNullOrEmpty()) {
-                    return listFiles.size
-                }
             }
-            return 0
-        } else {
-            val files = mangaFiles[manga.id] ?: return 0
-            return files.filter { !it.endsWith(Downloader.TMP_DIR_SUFFIX) }.size
         }
+        return 0
     }
 
-    /**
+    fun invalidateCache() {
-     * Checks if the cache needs a renewal and performs it if needed.
+        lastRenew = 0L
-     */
+        renewalJob?.cancel()
-    @Synchronized
+        diskCacheFile.delete()
-    private fun checkRenew() {
+        renewCache()
-        if (lastRenew + renewInterval < System.currentTimeMillis()) {
-            renew()
-            lastRenew = System.currentTimeMillis()
-        }
-    }
-
-    fun forceRenewCache() {
-        renew()
-        lastRenew = System.currentTimeMillis()
     }
 
     /**
      * Renews the downloads cache.
      */
-    private fun renew() {
+    private fun renewCache() {
-        val onlineSources = sourceManager.getOnlineSources()
+        if (lastRenew + renewInterval >= System.currentTimeMillis() || renewalJob?.isActive == true) {
+            return
+        }
 
-        val sourceDirs = storageManager.getDownloadsDirectory()?.listFiles().orEmpty()
+        renewalJob = scope.launch {
-            .associate { it.name to SourceDirectory(it) }.mapNotNullKeys { entry ->
+            // Try to wait until extensions and sources have loaded
-                onlineSources.find { provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) }?.id
+            var sources = emptyList<Source>()
+            withTimeoutOrNull(30.seconds) {
+                extensionManager.isInitialized.first { it }
+                sourceManager.isInitialized.first { it }
+
+                sources = getSources()
             }
 
-        val db: DatabaseHelper by injectLazy()
+            val sourceMap = sources.associate { provider.getSourceDirName(it).lowercase() to it.id }
-        val mangas = db.getMangas().executeAsBlocking().groupBy { it.source }
 
-        sourceDirs.forEach { sourceValue ->
+            rootDownloadsDirLock.withLock {
-            val sourceMangaRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach
+                rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory())
-            val sourceMangaPair = sourceMangaRaw.partition { it.favorite }
 
-            val sourceDir = sourceValue.value
+                val sourceDirs = rootDownloadsDir.dir?.listFiles().orEmpty()
+                    .filter { it.isDirectory && !it.name.isNullOrBlank() }
+                    .mapNotNull { dir ->
+                        val sourceId = sourceMap[dir.name!!.lowercase()]
+                        sourceId?.let { it to SourceDirectory(dir) }
+                    }
+                    .toMap()
 
-            val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir ->
+                sourceDirs.values
-                val name = mangaDir.name ?: return@mapNotNull null
+                    .map { sourceDir ->
-                val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name?.substringBeforeLast(".cbz") }.toHashSet()
+                        async {
-                name to MangaDirectory(mangaDir, chapterDirs)
+                            sourceDir.mangaDirs.values.forEach { mangaDir ->
-            }.toMap()
+                                val chapterDirs = mangaDir.dir?.listFiles().orEmpty()
+                                    .mapNotNull {
+                                        when {
+                                            // Ignore incomplete downloads
+                                            it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
+                                            // Folder of images
+                                            it.isDirectory -> it.name
+                                            // CBZ files
+                                            it.isFile && it.extension == "cbz" -> it.nameWithoutExtension
+                                            // Anything else is irrelevant
+                                            else -> null
+                                        }
+                                    }.toHashSet()
+                                mangaDir.files = chapterDirs
+                            }
+                        }
+                    }.awaitAll()
+            }
 
-            val trueMangaDirs = mangaDirs.mapNotNull { mangaDir ->
+            _isInitializing.value = false
-                val manga = findManga(sourceMangaPair.first, mangaDir.key, sourceValue.key) ?: findManga(sourceMangaPair.second, mangaDir.key, sourceValue.key)
+        }.also {
-                val id = manga?.id ?: return@mapNotNull null
+            it.invokeOnCompletion(onCancelling = true) { exception ->
-                id to mangaDir.value.files
+                if (exception != null && exception !is CancellationException) {
-            }.toMap()
+                    Logger.e(exception) { "DownloadCache: failed to create cache" }
-
+                }
-            mangaFiles.putAll(trueMangaDirs)
+                lastRenew = System.currentTimeMillis()
+                notifyChanges()
+            }
         }
+
+        // Mainly to notify the indexing notifier UI
+        notifyChanges()
     }
 
-    /**
+    private fun getSources(): List<Source> {
-     * Searches a manga list and matches the given mangakey and source key
+        return sourceManager.getOnlineSources() + sourceManager.getStubSources()
-     */
-    private fun findManga(mangaList: List<Manga>, mangaKey: String, sourceKey: Long): Manga? {
-        return mangaList.find {
-            DiskUtil.buildValidFilename(it.originalTitle).equals(mangaKey, ignoreCase = true) && it.source == sourceKey
-        }
     }
 
     /**
@@ -172,15 +263,30 @@ class DownloadCache(
      * @param mangaUniFile the directory of the manga.
      * @param manga the manga of the chapter.
      */
-    @Synchronized
+    suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
-    fun addChapter(chapterDirName: String, manga: Manga) {
+        rootDownloadsDirLock.withLock {
-        val id = manga.id ?: return
+            // Retrieve the cached source directory or cache a new one
-        val files = mangaFiles[id]
+            var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
-        if (files == null) {
+            if (sourceDir == null) {
-            mangaFiles[id] = mutableSetOf(chapterDirName)
+                val source = sourceManager.get(manga.source) ?: return
-        } else {
+                val sourceUniFile = provider.findSourceDir(source) ?: return
-            mangaFiles[id]?.add(chapterDirName)
+                sourceDir = SourceDirectory(sourceUniFile)
+                rootDownloadsDir.sourceDirs += manga.source to sourceDir
+            }
+
+            // Retrieve the cached manga directory or cache a new one
+            val mangaDirName = provider.getMangaDirName(manga)
+            var mangaDir = sourceDir.mangaDirs[mangaDirName]
+            if (mangaDir == null) {
+                mangaDir = MangaDirectory(mangaUniFile)
+                sourceDir.mangaDirs += mangaDirName to mangaDir
+            }
+
+            // Save the chapter directory
+            mangaDir.files += chapterDirName
         }
+
+        notifyChanges()
     }
 
     /**
@@ -189,17 +295,18 @@ class DownloadCache(
      * @param chapters the list of chapter to remove.
      * @param manga the manga of the chapter.
      */
-    @Synchronized
+    suspend fun removeChapter(chapter: Chapter, manga: Manga) {
-    fun removeChapters(chapters: List<Chapter>, manga: Manga) {
+        rootDownloadsDirLock.withLock {
-        val id = manga.id ?: return
+            val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
-        for (chapter in chapters) {
+            val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
-            val list = provider.getValidChapterDirNames(chapter)
+            provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
-            list.forEach { fileName ->
+                if (it in mangaDir.chapterDirs) {
-                mangaFiles[id]?.firstOrNull { fileName.equals(it, true) }?.let { chapterFile ->
+                    mangaDir.chapterDirs -= it
-                    mangaFiles[id]?.remove(chapterFile)
                 }
             }
         }
+
+        notifyChanges()
     }
 
     fun removeFolders(folders: List<String>, manga: Manga) {
@@ -234,29 +341,11 @@ class DownloadCache(
         mangaFiles.remove(manga.id)
     }
 
-    /**
+    private fun notifyChanges() {
-     * Class to store the files under the root downloads directory.
+        scope.launchNonCancellable {
-     */
+            _changes.send(Unit)
-    private class RootDirectory(
+        }
-        val dir: UniFile,
+    }
-        var files: Map<Long, SourceDirectory> = hashMapOf(),
-    )
-
-    /**
-     * Class to store the files under a source directory.
-     */
-    private class SourceDirectory(
-        val dir: UniFile,
-        var files: Map<Long, MutableSet<String>> = hashMapOf(),
-    )
-
-    /**
-     * Class to store the files under a manga directory.
-     */
-    private class MangaDirectory(
-        val dir: UniFile,
-        var files: MutableSet<String> = hashSetOf(),
-    )
 
     /**
      * Returns a new map containing only the key entries of [transform] that are not null.
@@ -281,3 +370,53 @@ class DownloadCache(
         return destination
     }
 }
+
+/**
+ * Class to store the files under the root downloads directory.
+ */
+@Serializable
+private class RootDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
+    val dir: UniFile?,
+    var sourceDirs: Map<Long, SourceDirectory> = mapOf(),
+)
+
+/**
+ * Class to store the files under a source directory.
+ */
+@Serializable
+private class SourceDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
+    val dir: UniFile?,
+    var mangaDirs: Map<String, MangaDirectory> = mapOf(),
+)
+
+/**
+ * Class to store the files under a manga directory.
+ */
+@Serializable
+private class MangaDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
+    val dir: UniFile?,
+    var files: MutableSet<String> = mutableSetOf(),
+)
+
+private object UniFileAsStringSerializer : KSerializer<UniFile?> {
+    override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING)
+
+    override fun serialize(encoder: Encoder, value: UniFile?) {
+        return if (value == null) {
+            encoder.encodeNull()
+        } else {
+            encoder.encodeString(value.uri.toString())
+        }
+    }
+
+    override fun deserialize(decoder: Decoder): UniFile? {
+        return if (decoder.decodeNotNullMark()) {
+            UniFile.fromUri(Injekt.get<Application>(), Uri.parse(decoder.decodeString()))
+        } else {
+            decoder.decodeNull()
+        }
+    }
+}

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt

@@ -375,7 +375,7 @@ class DownloadManager(val context: Context) {
      * @param oldChapter the existing chapter with the old name.
      * @param newChapter the target chapter with the new name.
      */
-    fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
+    suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
         val oldNames = provider.getValidChapterDirNames(oldChapter).map { listOf(it, "$it.cbz") }.flatten()
         var newName = provider.getChapterDirName(newChapter, includeId = downloadPreferences.downloadWithId().get())
         val mangaDir = provider.getMangaDir(manga, source)
@@ -393,7 +393,7 @@ class DownloadManager(val context: Context) {
 
         if (oldDownload.renameTo(newName)) {
             cache.removeChapters(listOf(oldChapter), manga)
-            cache.addChapter(newName, manga)
+            cache.addChapter(newName, mangaDir, manga)
         } else {
             Logger.e { "Could not rename downloaded chapter: ${oldNames.joinToString()}" }
         }
@@ -401,7 +401,7 @@ class DownloadManager(val context: Context) {
 
     // forceRefresh the download cache
     fun refreshCache() {
-        cache.forceRenewCache()
+        cache.invalidateCache()
     }
 
     fun addListener(listener: DownloadQueue.DownloadListener) = queue.addListener(listener)

app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt

@@ -21,6 +21,7 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.async
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.parcelize.Parcelize
 import uy.kohesive.injekt.Injekt
@@ -45,6 +46,8 @@ class ExtensionManager(
     private val preferences: PreferencesHelper = Injekt.get(),
     private val trustExtension: TrustExtension = Injekt.get(),
 ) {
+    private val _isInitialized = MutableStateFlow(false)
+    val isInitialized: StateFlow<Boolean> = _isInitialized.asStateFlow()
 
     /**
      * API where all the available extensions can be found.
@@ -122,6 +125,8 @@ class ExtensionManager(
         _untrustedExtensionsFlow.value = extensions
             .filterIsInstance<LoadResult.Untrusted>()
             .map { it.extension }
+
+        _isInitialized.value = true
     }
 
     fun isInstalledByApp(extension: Extension.Available): Boolean {

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -17,6 +17,8 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.flow.collectLatest
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
@@ -27,6 +29,8 @@ class SourceManager(
     private val context: Context,
     private val extensionManager: ExtensionManager,
 ) {
+    private val _isInitialized = MutableStateFlow(false)
+    val isInitialized: StateFlow<Boolean> = _isInitialized.asStateFlow()
 
     private val scope = CoroutineScope(Job() + Dispatchers.IO)
 
@@ -73,6 +77,7 @@ class SourceManager(
                         }
                     }
                     sourcesMapFlow.value = mutableMap
+                    _isInitialized.value = true
                 }
         }
 
@@ -105,6 +110,11 @@ class SourceManager(
         return delegatedSources.values.find { it.urlName == urlName }?.delegatedHttpSource
     }
 
+    fun getStubSources(): List<StubSource> {
+        val onlineSourceIds = getOnlineSources().map { it.id }
+        return stubSourcesMap.values.filterNot { it.id in onlineSourceIds }
+    }
+
     fun getOnlineSources() = sourcesMapFlow.value.values.filterIsInstance<HttpSource>()
 
     fun getCatalogueSources() = sourcesMapFlow.value.values.filterIsInstance<CatalogueSource>()

app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt

@@ -445,7 +445,7 @@ class ReaderViewModel(
     private suspend fun preload(chapter: ReaderChapter) {
         if (chapter.pageLoader is HttpPageLoader) {
             val manga = manga ?: return
-            val isDownloaded = downloadManager.isChapterDownloaded(chapter.chapter, manga)
+            val isDownloaded = downloadManager.isChapterDownloaded(chapter.chapter, manga, true)
             if (isDownloaded) {
                 chapter.state = ReaderChapter.State.Wait
             }

app/src/main/java/eu/kanade/tachiyomi/ui/recents/RecentsPresenter.kt

@@ -496,7 +496,7 @@ class RecentsPresenter(
             item.downloadInfo = item.mch.extraChapters.map { chapter ->
                 val downloadInfo = RecentMangaItem.DownloadInfo()
                 downloadInfo.chapterId = chapter.id
-                if (downloadManager.isChapterDownloaded(chapter, item.mch.manga)) {
+                if (downloadManager.isChapterDownloaded(chapter, item.mch.manga, true)) {
                     downloadInfo.status = Download.State.DOWNLOADED
                 } else if (downloadManager.hasQueue()) {
                     downloadInfo.download = downloadManager.queue.find { it.chapter.id == chapter.id }