From b3fbc0bf399f74d6cdb54c1254763c3597419372 Mon Sep 17 00:00:00 2001 From: Ahmad Ansori Palembani Date: Sat, 7 Dec 2024 13:13:08 +0700 Subject: [PATCH] refactor: Port upstream's download cache system --- .../tachiyomi/data/download/DownloadCache.kt | 393 +++++++++++++----- .../data/download/DownloadManager.kt | 34 +- .../tachiyomi/data/download/Downloader.kt | 16 +- 3 files changed, 321 insertions(+), 122 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt index d237008307..3ff36e5a1e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt @@ -1,23 +1,53 @@ package eu.kanade.tachiyomi.data.download +import android.app.Application import android.content.Context +import android.net.Uri +import co.touchlab.kermit.Logger import com.hippo.unifile.UniFile import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.domain.manga.models.Manga +import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceManager -import eu.kanade.tachiyomi.util.storage.DiskUtil +import eu.kanade.tachiyomi.util.system.extension +import eu.kanade.tachiyomi.util.system.launchIO +import eu.kanade.tachiyomi.util.system.launchNonCancellableIO +import eu.kanade.tachiyomi.util.system.nameWithoutExtension +import java.io.File +import java.util.concurrent.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.debounce import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.flow.onStart +import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.flow.shareIn +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromByteArray +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encodeToByteArray +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.protobuf.ProtoBuf import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get -import uy.kohesive.injekt.injectLazy -import yokai.domain.manga.interactor.GetManga import yokai.domain.storage.StorageManager -import java.util.concurrent.* /** * Cache where we dump the downloads directory from the filesystem. This class is needed because @@ -37,6 +67,13 @@ class DownloadCache( private val storageManager: StorageManager = Injekt.get(), ) { + val scope = CoroutineScope(Dispatchers.IO) + + private val _changes: Channel = Channel(Channel.UNLIMITED) + val changes = _changes.receiveAsFlow() + .onStart { emit(Unit) } + .shareIn(scope, SharingStarted.Lazily, 1) + /** * The interval after which this cache should be invalidated. 1 hour shouldn't cause major * issues, as the cache is only used for UI feedback. @@ -47,12 +84,38 @@ class DownloadCache( * The last time the cache was refreshed. */ private var lastRenew = 0L + private var renewalJob: Job? = null - private var mangaFiles: MutableMap> = mutableMapOf() + private val _isInitializing = MutableStateFlow(false) + val isInitializing = _isInitializing + .debounce(1000L) // Don't notify if it finishes quickly enough + .stateIn(scope, SharingStarted.WhileSubscribed(), false) - val scope = CoroutineScope(Job() + Dispatchers.IO) + private val diskCacheFile: File + get() = File(context.cacheDir, "dl_index_cache_v3") + + private val rootDownloadsDirLock = Mutex() + private var rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory()) init { + // Attempt to read cache file + scope.launch { + rootDownloadsDirLock.withLock { + try { + if (diskCacheFile.exists()) { + val diskCache = diskCacheFile.inputStream().use { + ProtoBuf.decodeFromByteArray(it.readBytes()) + } + rootDownloadsDir = diskCache + lastRenew = System.currentTimeMillis() + } + } catch (e: Throwable) { + Logger.e(e) { "Failed to initialize disk cache" } + diskCacheFile.delete() + } + } + } + storageManager.changes .onEach { forceRenewCache() } // invalidate cache .launchIn(scope) @@ -71,12 +134,18 @@ class DownloadCache( return provider.findChapterDir(chapter, manga, source) != null } - checkRenew() + renewCache() - val files = mangaFiles[manga.id]?.toHashSet() ?: return false - return provider.getValidChapterDirNames(chapter).any { chapName -> - files.any { chapName.equals(it, true) || "$chapName.cbz".equals(it, true) } + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] + if (sourceDir != null) { + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)] + if (mangaDir != null) { + return provider.getValidChapterDirNames( + chapter, + ).any { it in mangaDir.chapterDirs } + } } + return false } /** @@ -85,84 +154,137 @@ class DownloadCache( * @param manga the manga to check. */ fun getDownloadCount(manga: Manga, forceCheckFolder: Boolean = false): Int { - checkRenew() + renewCache() + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] if (forceCheckFolder) { val source = sourceManager.get(manga.source) ?: return 0 val mangaDir = provider.findMangaDir(manga, source) if (mangaDir != null) { - val listFiles = - mangaDir.listFiles { _, filename -> !filename.endsWith(Downloader.TMP_DIR_SUFFIX) } + val listFiles = mangaDir.listFiles { _, filename -> !filename.endsWith(Downloader.TMP_DIR_SUFFIX) } if (!listFiles.isNullOrEmpty()) { return listFiles.size } } return 0 } else { - val files = mangaFiles[manga.id] ?: return 0 - return files.filter { !it.endsWith(Downloader.TMP_DIR_SUFFIX) }.size - } - } - - /** - * Checks if the cache needs a renewal and performs it if needed. - */ - @Synchronized - private fun checkRenew() { - if (lastRenew + renewInterval < System.currentTimeMillis()) { - renew() - lastRenew = System.currentTimeMillis() + if (sourceDir != null) { + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)] + if (mangaDir != null) { + return mangaDir.chapterDirs.size + } + } + return 0 } } fun forceRenewCache() { - renew() - lastRenew = System.currentTimeMillis() + lastRenew = 0L + renewalJob?.cancel() + diskCacheFile.delete() + renewCache() } /** * Renews the downloads cache. */ - private fun renew() { - val onlineSources = sourceManager.getOnlineSources() + private fun renewCache() { + if (lastRenew + renewInterval >= System.currentTimeMillis() || renewalJob?.isActive == true) { + return + } - val sourceDirs = storageManager.getDownloadsDirectory()?.listFiles().orEmpty() - .associate { it.name to SourceDirectory(it) }.mapNotNullKeys { entry -> - onlineSources.find { provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) }?.id + renewalJob = scope.launchIO { + if (lastRenew == 0L) { + _isInitializing.emit(true) } - val getManga: GetManga by injectLazy() - val mangas = runBlocking(Dispatchers.IO) { getManga.awaitAll().groupBy { it.source } } + val sources = getSources() - sourceDirs.forEach { sourceValue -> - val sourceMangaRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach - val sourceMangaPair = sourceMangaRaw.partition { it.favorite } + val sourceMap = sources.associate { provider.getSourceDirName(it).lowercase() to it.id } - val sourceDir = sourceValue.value + rootDownloadsDirLock.withLock { + rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory()) - val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir -> - val name = mangaDir.name ?: return@mapNotNull null - val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name?.substringBeforeLast(".cbz") }.toHashSet() - name to MangaDirectory(mangaDir, chapterDirs) - }.toMap() + val sourceDirs = rootDownloadsDir.dir?.listFiles().orEmpty() + .filter { it.isDirectory && !it.name.isNullOrBlank() } + .mapNotNull { dir -> + val sourceId = sourceMap[dir.name!!.lowercase()] + sourceId?.let { it to SourceDirectory(dir) } + } + .toMap() - val trueMangaDirs = mangaDirs.mapNotNull { mangaDir -> - val manga = findManga(sourceMangaPair.first, mangaDir.key, sourceValue.key) ?: findManga(sourceMangaPair.second, mangaDir.key, sourceValue.key) - val id = manga?.id ?: return@mapNotNull null - id to mangaDir.value.files - }.toMap() + rootDownloadsDir.sourceDirs = sourceDirs - mangaFiles.putAll(trueMangaDirs) + sourceDirs.values + .map { sourceDir -> + async { + sourceDir.mangaDirs = sourceDir.dir?.listFiles().orEmpty() + .filter { it.isDirectory && !it.name.isNullOrBlank() } + .associate { it.name!! to MangaDirectory(it) } + + sourceDir.mangaDirs.values.forEach { mangaDir -> + val chapterDirs = mangaDir.dir?.listFiles().orEmpty() + .mapNotNull { + when { + // Ignore incomplete downloads + it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null + // Folder of images + it.isDirectory -> it.name + // CBZ files + it.isFile && it.extension == "cbz" -> it.nameWithoutExtension + // Anything else is irrelevant + else -> null + } + } + .toMutableSet() + + mangaDir.chapterDirs = chapterDirs + } + } + } + .awaitAll() + + _isInitializing.emit(false) + } + }.also { + it.invokeOnCompletion(onCancelling = true) { exception -> + if (exception != null && exception !is CancellationException) { + Logger.e(exception) { "DownloadCache: failed to create cache" } + } + lastRenew = System.currentTimeMillis() + notifyChanges() + } } + + // Mainly to notify the indexing notifier UI + notifyChanges() } - /** - * Searches a manga list and matches the given mangakey and source key - */ - private fun findManga(mangaList: List, mangaKey: String, sourceKey: Long): Manga? { - return mangaList.find { - DiskUtil.buildValidFilename(it.originalTitle).equals(mangaKey, ignoreCase = true) && it.source == sourceKey + private fun getSources(): List { + return sourceManager.getOnlineSources() + } + + private fun notifyChanges() { + scope.launchNonCancellableIO { + _changes.send(Unit) + } + updateDiskCache() + } + + private var updateDiskCacheJob: Job? = null + private fun updateDiskCache() { + updateDiskCacheJob?.cancel() + updateDiskCacheJob = scope.launchIO { + delay(1000) + ensureActive() + val bytes = ProtoBuf.encodeToByteArray(rootDownloadsDir) + ensureActive() + try { + diskCacheFile.writeBytes(bytes) + } catch (e: Throwable) { + Logger.e(e) { "Failed to write disk cache file" } + } } } @@ -173,15 +295,30 @@ class DownloadCache( * @param mangaUniFile the directory of the manga. * @param manga the manga of the chapter. */ - @Synchronized - fun addChapter(chapterDirName: String, manga: Manga) { - val id = manga.id ?: return - val files = mangaFiles[id] - if (files == null) { - mangaFiles[id] = mutableSetOf(chapterDirName) - } else { - mangaFiles[id]?.add(chapterDirName) + suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile?, manga: Manga) { + rootDownloadsDirLock.withLock { + // Retrieve the cached source directory or cache a new one + var sourceDir = rootDownloadsDir.sourceDirs[manga.source] + if (sourceDir == null) { + val source = sourceManager.get(manga.source) ?: return + val sourceUniFile = provider.findSourceDir(source) ?: return + sourceDir = SourceDirectory(sourceUniFile) + rootDownloadsDir.sourceDirs += manga.source to sourceDir + } + + // Retrieve the cached manga directory or cache a new one + val mangaDirName = provider.getMangaDirName(manga) + var mangaDir = sourceDir.mangaDirs[mangaDirName] + if (mangaDir == null) { + mangaDir = MangaDirectory(mangaUniFile) + sourceDir.mangaDirs += mangaDirName to mangaDir + } + + // Save the chapter directory + mangaDir.chapterDirs += chapterDirName } + + notifyChanges() } /** @@ -190,26 +327,35 @@ class DownloadCache( * @param chapters the list of chapter to remove. * @param manga the manga of the chapter. */ - @Synchronized - fun removeChapters(chapters: List, manga: Manga) { - val id = manga.id ?: return - for (chapter in chapters) { - val list = provider.getValidChapterDirNames(chapter) - list.forEach { fileName -> - mangaFiles[id]?.firstOrNull { fileName.equals(it, true) }?.let { chapterFile -> - mangaFiles[id]?.remove(chapterFile) + suspend fun removeChapters(chapters: List, manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)] ?: return + chapters.forEach { chapter -> + provider.getValidChapterDirNames(chapter).forEach { + if (it in mangaDir.chapterDirs) { + mangaDir.chapterDirs -= it + } } } } + + notifyChanges() } - fun removeFolders(folders: List, manga: Manga) { - val id = manga.id ?: return - for (chapter in folders) { - if (mangaFiles[id] != null && chapter in mangaFiles[id]!!) { - mangaFiles[id]?.remove(chapter) + suspend fun removeChapterFolders(folders: List, manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)] ?: return + + folders.forEach { chapter -> + if (chapter in mangaDir.chapterDirs) { + mangaDir.chapterDirs -= chapter + } } } + + notifyChanges() } /*fun renameFolder(from: String, to: String, source: Long) { @@ -230,34 +376,25 @@ class DownloadCache( * * @param manga the manga to remove. */ - @Synchronized - fun removeManga(manga: Manga) { - mangaFiles.remove(manga.id) + suspend fun removeManga(manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDirName = provider.getMangaDirName(manga) + if (sourceDir.mangaDirs.containsKey(mangaDirName)) { + sourceDir.mangaDirs -= mangaDirName + } + } + + notifyChanges() } - /** - * Class to store the files under the root downloads directory. - */ - private class RootDirectory( - val dir: UniFile, - var files: Map = hashMapOf(), - ) + suspend fun removeSource(source: Source) { + rootDownloadsDirLock.withLock { + rootDownloadsDir.sourceDirs -= source.id + } - /** - * Class to store the files under a source directory. - */ - private class SourceDirectory( - val dir: UniFile, - var files: Map> = hashMapOf(), - ) - - /** - * Class to store the files under a manga directory. - */ - private class MangaDirectory( - val dir: UniFile, - var files: MutableSet = hashSetOf(), - ) + notifyChanges() + } /** * Returns a new map containing only the key entries of [transform] that are not null. @@ -282,3 +419,53 @@ class DownloadCache( return destination } } + +/** + * Class to store the files under the root downloads directory. + */ +@Serializable +private class RootDirectory( + @Serializable(with = UniFileAsStringSerializer::class) + val dir: UniFile?, + var sourceDirs: Map = hashMapOf(), +) + +/** + * Class to store the files under a source directory. + */ +@Serializable +private class SourceDirectory( + @Serializable(with = UniFileAsStringSerializer::class) + val dir: UniFile?, + var mangaDirs: Map = hashMapOf(), +) + +/** + * Class to store the files under a manga directory. + */ +@Serializable +private class MangaDirectory( + @Serializable(with = UniFileAsStringSerializer::class) + val dir: UniFile?, + var chapterDirs: MutableSet = hashSetOf(), +) + +private object UniFileAsStringSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: UniFile?) { + return if (value == null) { + encoder.encodeNull() + } else { + encoder.encodeString(value.uri.toString()) + } + } + + override fun deserialize(decoder: Decoder): UniFile? { + return if (decoder.decodeNotNullMark()) { + UniFile.fromUri(Injekt.get(), Uri.parse(decoder.decodeString())) + } else { + decoder.decodeNull() + } + } +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt index 619854bcde..ac0521a940 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt @@ -12,6 +12,7 @@ import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.util.system.ImageUtil +import eu.kanade.tachiyomi.util.system.launchIO import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope @@ -100,7 +101,7 @@ class DownloadManager(val context: Context) { */ fun clearQueue(isNotification: Boolean = false) { deletePendingDownloads(*downloader.queue.toTypedArray()) - downloader.clearQueue(isNotification) + downloader.removeFromQueue(isNotification) DownloadJob.callListeners(false, this) } @@ -298,7 +299,7 @@ class DownloadManager(val context: Context) { * @param manga the manga of the chapters. * @param source the source of the chapters. */ - fun cleanupChapters(allChapters: List, manga: Manga, source: Source, removeRead: Boolean, removeNonFavorite: Boolean): Int { + suspend fun cleanupChapters(allChapters: List, manga: Manga, source: Source, removeRead: Boolean, removeNonFavorite: Boolean): Int { var cleaned = 0 if (removeNonFavorite && !manga.favorite) { @@ -311,7 +312,7 @@ class DownloadManager(val context: Context) { val filesWithNoChapter = provider.findUnmatchedChapterDirs(allChapters, manga, source) cleaned += filesWithNoChapter.size - cache.removeFolders(filesWithNoChapter.mapNotNull { it.name }, manga) + cache.removeChapterFolders(filesWithNoChapter.mapNotNull { it.name }, manga) filesWithNoChapter.forEach { it.delete() } if (removeRead) { @@ -341,12 +342,23 @@ class DownloadManager(val context: Context) { * @param manga the manga to delete. * @param source the source of the manga. */ - fun deleteManga(manga: Manga, source: Source) { - downloader.clearQueue(manga, true) - queue.remove(manga) - provider.findMangaDir(manga, source)?.delete() - cache.removeManga(manga) - queue.updateListeners() + fun deleteManga(manga: Manga, source: Source, removeQueued: Boolean = true) { + launchIO { + if (removeQueued) { + downloader.removeFromQueue(manga, true) + queue.remove(manga) + queue.updateListeners() + } + provider.findMangaDir(manga, source)?.delete() + cache.removeManga(manga) + + // Delete source directory if empty + val sourceDir = provider.findSourceDir(source) + if (sourceDir?.listFiles()?.isEmpty() == true) { + sourceDir.delete() + cache.removeSource(source) + } + } } /** @@ -377,7 +389,7 @@ class DownloadManager(val context: Context) { * @param oldChapter the existing chapter with the old name. * @param newChapter the target chapter with the new name. */ - fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { + suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { val oldNames = provider.getValidChapterDirNames(oldChapter).map { listOf(it, "$it.cbz") }.flatten() var newName = provider.getChapterDirName(newChapter, includeId = downloadPreferences.downloadWithId().get()) val mangaDir = provider.getMangaDir(manga, source) @@ -395,7 +407,7 @@ class DownloadManager(val context: Context) { if (oldDownload.renameTo(newName)) { cache.removeChapters(listOf(oldChapter), manga) - cache.addChapter(newName, manga) + cache.addChapter(newName, mangaDir, manga) } else { Logger.e { "Could not rename downloaded chapter: ${oldNames.joinToString()}" } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt index be0db1d260..f66d79cfe3 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt @@ -28,6 +28,9 @@ import eu.kanade.tachiyomi.util.system.launchNow import eu.kanade.tachiyomi.util.system.withIOContext import eu.kanade.tachiyomi.util.system.withUIContext import eu.kanade.tachiyomi.util.system.writeText +import java.io.File +import java.util.* +import java.util.zip.* import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async @@ -50,13 +53,10 @@ import yokai.core.archive.ZipWriter import yokai.core.metadata.COMIC_INFO_FILE import yokai.core.metadata.ComicInfo import yokai.core.metadata.getComicInfo +import yokai.domain.category.interactor.GetCategories import yokai.domain.download.DownloadPreferences import yokai.i18n.MR import yokai.util.lang.getString -import java.io.File -import java.util.* -import java.util.zip.* -import yokai.domain.category.interactor.GetCategories /** * This class is the one in charge of downloading chapters. @@ -192,7 +192,7 @@ class Downloader( * * @param isNotification value that determines if status is set (needed for view updates) */ - fun clearQueue(isNotification: Boolean = false) { + fun removeFromQueue(isNotification: Boolean = false) { destroySubscription() // Needed to update the chapter view @@ -210,7 +210,7 @@ class Downloader( * * @param isNotification value that determines if status is set (needed for view updates) */ - fun clearQueue(manga: Manga, isNotification: Boolean = false) { + fun removeFromQueue(manga: Manga, isNotification: Boolean = false) { // Needed to update the chapter view if (isNotification) { queue.filter { it.status == Download.State.QUEUE && it.manga.id == manga.id } @@ -566,7 +566,7 @@ class Downloader( * @param tmpDir the directory where the download is currently stored. * @param dirname the real (non temporary) directory name of the download. */ - private fun ensureSuccessfulDownload( + private suspend fun ensureSuccessfulDownload( download: Download, mangaDir: UniFile, tmpDir: UniFile, @@ -602,7 +602,7 @@ class Downloader( } else { tmpDir.renameTo(dirname) } - cache.addChapter(dirname, download.manga) + cache.addChapter(dirname, mangaDir, download.manga) DiskUtil.createNoMediaFile(tmpDir, context)