null2264/yokai
1
0
Fork 0
mirror of https://github.com/null2264/yokai.git synced 2025-06-21 10:44:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: Async download

Browse source
This commit is contained in:
Ahmad Ansori Palembani 2024-06-19 15:02:12 +07:00
parent f6080cd5eb
commit 2268132bd6
Signed by: null2264
GPG key ID: BA64F8B60AF3EFB6
6 changed files with 264 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

345
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt
View file

@ -1,22 +1,57 @@
package eu.kanade.tachiyomi.data.download
import android.app.Application
import android.content.Context
import android.net.Uri
import co.touchlab.kermit.Logger
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.extension.ExtensionManager
import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.util.storage.DiskUtil
import eu.kanade.tachiyomi.util.system.extension
import eu.kanade.tachiyomi.util.system.launchNonCancellable
import eu.kanade.tachiyomi.util.system.nameWithoutExtension
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.debounce
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.onStart
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.shareIn
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.protobuf.ProtoBuf
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import uy.kohesive.injekt.injectLazy
import yokai.domain.manga.interactor.GetManga
import yokai.domain.storage.StorageManager
import java.io.File
import java.util.concurrent.*
import kotlin.time.Duration.Companion.seconds
/**
* Cache where we dump the downloads directory from the filesystem. This class is needed because
@ -34,7 +69,14 @@ class DownloadCache(
private val provider: DownloadProvider,
private val sourceManager: SourceManager,
private val storageManager: StorageManager = Injekt.get(),
private val extensionManager: ExtensionManager = Injekt.get(),
) {
val scope = CoroutineScope(Dispatchers.IO)
private val _changes: Channel<Unit> = Channel(Channel.UNLIMITED)
val changes = _changes.receiveAsFlow()
.onStart { emit(Unit) }
.shareIn(scope, SharingStarted.Lazily, 1)
/**
* The interval after which this cache should be invalidated. 1 hour shouldn't cause major
@ -46,14 +88,42 @@ class DownloadCache(
* The last time the cache was refreshed.
*/
private var lastRenew = 0L
private var renewalJob: Job? = null
private val _isInitializing = MutableStateFlow(false)
val isInitializing = _isInitializing
.debounce(1000L) // Don't notify if it finishes quickly enough
.stateIn(scope, SharingStarted.WhileSubscribed(), false)
private var mangaFiles: MutableMap<Long, MutableSet<String>> = mutableMapOf()
val scope = CoroutineScope(Job() + Dispatchers.IO)
private val diskCacheFile: File
get() = File(context.cacheDir, "dl_index_cache_v3")
private val rootDownloadsDirLock = Mutex()
private var rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory())
init {
// Attempt to read cache file
scope.launch {
rootDownloadsDirLock.withLock {
try {
if (diskCacheFile.exists()) {
val diskCache = diskCacheFile.inputStream().use {
ProtoBuf.decodeFromByteArray<RootDirectory>(it.readBytes())
}
rootDownloadsDir = diskCache
lastRenew = System.currentTimeMillis()
}
} catch (e: Throwable) {
Logger.e(e) { "Failed to initialize disk cache" }
diskCacheFile.delete()
}
}
}
storageManager.changes
.onEach { forceRenewCache() } // invalidate cache
.onEach { invalidateCache() }
.launchIn(scope)
}
@ -64,19 +134,29 @@ class DownloadCache(
* @param manga the manga of the chapter.
* @param skipCache whether to skip the directory cache and check in the filesystem.
*/
fun isChapterDownloaded(chapter: Chapter, manga: Manga, skipCache: Boolean): Boolean {
fun isChapterDownloaded(
chapter: Chapter,
manga: Manga,
skipCache: Boolean,
): Boolean {
val sourceId = manga.source
if (skipCache) {
val source = sourceManager.get(manga.source) ?: return false
val source = sourceManager.get(sourceId) ?: return false
return provider.findChapterDir(chapter, manga, source) != null
}
checkRenew()
renewCache()
val files = mangaFiles[manga.id]?.toHashSet() ?: return false
return provider.getValidChapterDirNames(chapter).any { chapName ->
files.any { chapName.equals(it, true) || "$chapName.cbz".equals(it, true) }
val sourceDir = rootDownloadsDir.sourceDirs[sourceId]
if (sourceDir != null) {
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)]
if (mangaDir != null) {
return provider.getValidChapterDirNames(chapter).any { it in mangaDir.files }
}
}
return false
}
/**
* Returns the amount of downloaded chapters for a manga.
@ -84,85 +164,96 @@ class DownloadCache(
* @param manga the manga to check.
*/
fun getDownloadCount(manga: Manga, forceCheckFolder: Boolean = false): Int {
checkRenew()
if (forceCheckFolder) {
val source = sourceManager.get(manga.source) ?: return 0
val mangaDir = provider.findMangaDir(manga, source)
renewCache()
val sourceDir = rootDownloadsDir.sourceDirs[manga.source]
if (sourceDir != null) {
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga)]
if (mangaDir != null) {
val listFiles =
mangaDir.listFiles { _, filename -> !filename.endsWith(Downloader.TMP_DIR_SUFFIX) }
if (!listFiles.isNullOrEmpty()) {
return listFiles.size
return mangaDir.files.size
}
}
return 0
} else {
val files = mangaFiles[manga.id] ?: return 0
return files.filter { !it.endsWith(Downloader.TMP_DIR_SUFFIX) }.size
}
}
/**
* Checks if the cache needs a renewal and performs it if needed.
*/
@Synchronized
private fun checkRenew() {
if (lastRenew + renewInterval < System.currentTimeMillis()) {
renew()
lastRenew = System.currentTimeMillis()
}
}
fun forceRenewCache() {
renew()
lastRenew = System.currentTimeMillis()
fun invalidateCache() {
lastRenew = 0L
renewalJob?.cancel()
diskCacheFile.delete()
renewCache()
}
/**
* Renews the downloads cache.
*/
private fun renew() {
val onlineSources = sourceManager.getOnlineSources()
val sourceDirs = storageManager.getDownloadsDirectory()?.listFiles().orEmpty()
.associate { it.name to SourceDirectory(it) }.mapNotNullKeys { entry ->
onlineSources.find { provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) }?.id
private fun renewCache() {
if (lastRenew + renewInterval >= System.currentTimeMillis() || renewalJob?.isActive == true) {
return
}
val db: DatabaseHelper by injectLazy()
val mangas = db.getMangas().executeAsBlocking().groupBy { it.source }
renewalJob = scope.launch {
// Try to wait until extensions and sources have loaded
var sources = emptyList<Source>()
withTimeoutOrNull(30.seconds) {
extensionManager.isInitialized.first { it }
sourceManager.isInitialized.first { it }
sourceDirs.forEach { sourceValue ->
val sourceMangaRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach
val sourceMangaPair = sourceMangaRaw.partition { it.favorite }
sources = getSources()
}
val sourceDir = sourceValue.value
val sourceMap = sources.associate { provider.getSourceDirName(it).lowercase() to it.id }
val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir ->
val name = mangaDir.name ?: return@mapNotNull null
val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name?.substringBeforeLast(".cbz") }.toHashSet()
name to MangaDirectory(mangaDir, chapterDirs)
}.toMap()
rootDownloadsDirLock.withLock {
rootDownloadsDir = RootDirectory(storageManager.getDownloadsDirectory())
val trueMangaDirs = mangaDirs.mapNotNull { mangaDir ->
val manga = findManga(sourceMangaPair.first, mangaDir.key, sourceValue.key) ?: findManga(sourceMangaPair.second, mangaDir.key, sourceValue.key)
val id = manga?.id ?: return@mapNotNull null
id to mangaDir.value.files
}.toMap()
val sourceDirs = rootDownloadsDir.dir?.listFiles().orEmpty()
.filter { it.isDirectory && !it.name.isNullOrBlank() }
.mapNotNull { dir ->
val sourceId = sourceMap[dir.name!!.lowercase()]
sourceId?.let { it to SourceDirectory(dir) }
}
.toMap()
mangaFiles.putAll(trueMangaDirs)
sourceDirs.values
.map { sourceDir ->
async {
sourceDir.mangaDirs.values.forEach { mangaDir ->
val chapterDirs = mangaDir.dir?.listFiles().orEmpty()
.mapNotNull {
when {
// Ignore incomplete downloads
it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
// Folder of images
it.isDirectory -> it.name
// CBZ files
it.isFile && it.extension == "cbz" -> it.nameWithoutExtension
// Anything else is irrelevant
else -> null
}
}.toHashSet()
mangaDir.files = chapterDirs
}
}
}.awaitAll()
}
_isInitializing.value = false
}.also {
it.invokeOnCompletion(onCancelling = true) { exception ->
if (exception != null && exception !is CancellationException) {
Logger.e(exception) { "DownloadCache: failed to create cache" }
}
lastRenew = System.currentTimeMillis()
notifyChanges()
}
}
/**
* Searches a manga list and matches the given mangakey and source key
*/
private fun findManga(mangaList: List<Manga>, mangaKey: String, sourceKey: Long): Manga? {
return mangaList.find {
DiskUtil.buildValidFilename(it.originalTitle).equals(mangaKey, ignoreCase = true) && it.source == sourceKey
// Mainly to notify the indexing notifier UI
notifyChanges()
}
private fun getSources(): List<Source> {
return sourceManager.getOnlineSources() + sourceManager.getStubSources()
}
/**
@ -172,15 +263,30 @@ class DownloadCache(
* @param mangaUniFile the directory of the manga.
* @param manga the manga of the chapter.
*/
@Synchronized
fun addChapter(chapterDirName: String, manga: Manga) {
val id = manga.id ?: return
val files = mangaFiles[id]
if (files == null) {
mangaFiles[id] = mutableSetOf(chapterDirName)
} else {
mangaFiles[id]?.add(chapterDirName)
suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
rootDownloadsDirLock.withLock {
// Retrieve the cached source directory or cache a new one
var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
if (sourceDir == null) {
val source = sourceManager.get(manga.source) ?: return
val sourceUniFile = provider.findSourceDir(source) ?: return
sourceDir = SourceDirectory(sourceUniFile)
rootDownloadsDir.sourceDirs += manga.source to sourceDir
}
// Retrieve the cached manga directory or cache a new one
val mangaDirName = provider.getMangaDirName(manga)
var mangaDir = sourceDir.mangaDirs[mangaDirName]
if (mangaDir == null) {
mangaDir = MangaDirectory(mangaUniFile)
sourceDir.mangaDirs += mangaDirName to mangaDir
}
// Save the chapter directory
mangaDir.files += chapterDirName
}
notifyChanges()
}
/**
@ -189,17 +295,18 @@ class DownloadCache(
* @param chapters the list of chapter to remove.
* @param manga the manga of the chapter.
*/
@Synchronized
fun removeChapters(chapters: List<Chapter>, manga: Manga) {
val id = manga.id ?: return
for (chapter in chapters) {
val list = provider.getValidChapterDirNames(chapter)
list.forEach { fileName ->
mangaFiles[id]?.firstOrNull { fileName.equals(it, true) }?.let { chapterFile ->
mangaFiles[id]?.remove(chapterFile)
suspend fun removeChapter(chapter: Chapter, manga: Manga) {
rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it
}
}
}
notifyChanges()
}
fun removeFolders(folders: List<String>, manga: Manga) {
@ -234,29 +341,11 @@ class DownloadCache(
mangaFiles.remove(manga.id)
}
/**
* Class to store the files under the root downloads directory.
*/
private class RootDirectory(
val dir: UniFile,
var files: Map<Long, SourceDirectory> = hashMapOf(),
)
/**
* Class to store the files under a source directory.
*/
private class SourceDirectory(
val dir: UniFile,
var files: Map<Long, MutableSet<String>> = hashMapOf(),
)
/**
* Class to store the files under a manga directory.
*/
private class MangaDirectory(
val dir: UniFile,
var files: MutableSet<String> = hashSetOf(),
)
private fun notifyChanges() {
scope.launchNonCancellable {
_changes.send(Unit)
}
}
/**
* Returns a new map containing only the key entries of [transform] that are not null.
@ -281,3 +370,53 @@ class DownloadCache(
return destination
}
}
/**
* Class to store the files under the root downloads directory.
*/
@Serializable
private class RootDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile?,
var sourceDirs: Map<Long, SourceDirectory> = mapOf(),
)
/**
* Class to store the files under a source directory.
*/
@Serializable
private class SourceDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile?,
var mangaDirs: Map<String, MangaDirectory> = mapOf(),
)
/**
* Class to store the files under a manga directory.
*/
@Serializable
private class MangaDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile?,
var files: MutableSet<String> = mutableSetOf(),
)
private object UniFileAsStringSerializer : KSerializer<UniFile?> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING)
override fun serialize(encoder: Encoder, value: UniFile?) {
return if (value == null) {
encoder.encodeNull()
} else {
encoder.encodeString(value.uri.toString())
}
}
override fun deserialize(decoder: Decoder): UniFile? {
return if (decoder.decodeNotNullMark()) {
UniFile.fromUri(Injekt.get<Application>(), Uri.parse(decoder.decodeString()))
} else {
decoder.decodeNull()
}
}
}

6
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt
View file

@ -375,7 +375,7 @@ class DownloadManager(val context: Context) {
* @param oldChapter the existing chapter with the old name.
* @param newChapter the target chapter with the new name.
*/
fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
val oldNames = provider.getValidChapterDirNames(oldChapter).map { listOf(it, "$it.cbz") }.flatten()
var newName = provider.getChapterDirName(newChapter, includeId = downloadPreferences.downloadWithId().get())
val mangaDir = provider.getMangaDir(manga, source)
@ -393,7 +393,7 @@ class DownloadManager(val context: Context) {
if (oldDownload.renameTo(newName)) {
cache.removeChapters(listOf(oldChapter), manga)
cache.addChapter(newName, manga)
cache.addChapter(newName, mangaDir, manga)
} else {
Logger.e { "Could not rename downloaded chapter: ${oldNames.joinToString()}" }
}
@ -401,7 +401,7 @@ class DownloadManager(val context: Context) {
// forceRefresh the download cache
fun refreshCache() {
cache.forceRenewCache()
cache.invalidateCache()
}
fun addListener(listener: DownloadQueue.DownloadListener) = queue.addListener(listener)

5
app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt
View file

@ -21,6 +21,7 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.async
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.parcelize.Parcelize
import uy.kohesive.injekt.Injekt
@ -45,6 +46,8 @@ class ExtensionManager(
private val preferences: PreferencesHelper = Injekt.get(),
private val trustExtension: TrustExtension = Injekt.get(),
) {
private val _isInitialized = MutableStateFlow(false)
val isInitialized: StateFlow<Boolean> = _isInitialized.asStateFlow()
/**
* API where all the available extensions can be found.
@ -122,6 +125,8 @@ class ExtensionManager(
_untrustedExtensionsFlow.value = extensions
.filterIsInstance<LoadResult.Untrusted>()
.map { it.extension }
_isInitialized.value = true
}
fun isInstalledByApp(extension: Extension.Available): Boolean {

10
app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt
View file

@ -17,6 +17,8 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.collectLatest
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.launch
@ -27,6 +29,8 @@ class SourceManager(
private val context: Context,
private val extensionManager: ExtensionManager,
) {
private val _isInitialized = MutableStateFlow(false)
val isInitialized: StateFlow<Boolean> = _isInitialized.asStateFlow()
private val scope = CoroutineScope(Job() + Dispatchers.IO)
@ -73,6 +77,7 @@ class SourceManager(
}
}
sourcesMapFlow.value = mutableMap
_isInitialized.value = true
}
}
@ -105,6 +110,11 @@ class SourceManager(
return delegatedSources.values.find { it.urlName == urlName }?.delegatedHttpSource
}
fun getStubSources(): List<StubSource> {
val onlineSourceIds = getOnlineSources().map { it.id }
return stubSourcesMap.values.filterNot { it.id in onlineSourceIds }
}
fun getOnlineSources() = sourcesMapFlow.value.values.filterIsInstance<HttpSource>()
fun getCatalogueSources() = sourcesMapFlow.value.values.filterIsInstance<CatalogueSource>()

2
app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderViewModel.kt
View file

@ -445,7 +445,7 @@ class ReaderViewModel(
private suspend fun preload(chapter: ReaderChapter) {
if (chapter.pageLoader is HttpPageLoader) {
val manga = manga ?: return
val isDownloaded = downloadManager.isChapterDownloaded(chapter.chapter, manga)
val isDownloaded = downloadManager.isChapterDownloaded(chapter.chapter, manga, true)
if (isDownloaded) {
chapter.state = ReaderChapter.State.Wait
}

2
app/src/main/java/eu/kanade/tachiyomi/ui/recents/RecentsPresenter.kt
View file

@ -496,7 +496,7 @@ class RecentsPresenter(
item.downloadInfo = item.mch.extraChapters.map { chapter ->
val downloadInfo = RecentMangaItem.DownloadInfo()
downloadInfo.chapterId = chapter.id
if (downloadManager.isChapterDownloaded(chapter, item.mch.manga)) {
if (downloadManager.isChapterDownloaded(chapter, item.mch.manga, true)) {
downloadInfo.status = Download.State.DOWNLOADED
} else if (downloadManager.hasQueue()) {
downloadInfo.download = downloadManager.queue.find { it.chapter.id == chapter.id }