compress to cbz after downloading (#1052)

* initial attempt

* remove test vars

* add logic to remove downloads

* download settings fixes

* remove cbz compression level

remove leftover cbz levels

* fix more cbz issues

fix issue that made cbz not be recognized as downloaded
fix issue where cleanup download would delete cbz

* remove redundant cbz funct

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt

@@ -85,7 +85,7 @@ class DownloadCache(
 
         val files = mangaFiles[manga.id]?.toHashSet() ?: return false
         return provider.getValidChapterDirNames(chapter).any {
-            it in files
+            it in files || "$it.cbz" in files
         }
     }
 
@@ -153,7 +153,7 @@ class DownloadCache(
 
             val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir ->
                 val name = mangaDir.name ?: return@mapNotNull null
-                val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name }.toHashSet()
+                val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name?.replace(".cbz", "") }.toHashSet()
                 name to MangaDirectory(mangaDir, chapterDirs)
             }.toMap()
 

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt

@@ -88,7 +88,7 @@ class DownloadProvider(private val context: Context) {
     fun findChapterDir(chapter: Chapter, manga: Manga, source: Source): UniFile? {
         val mangaDir = findMangaDir(manga, source)
         return getValidChapterDirNames(chapter).asSequence()
-            .mapNotNull { mangaDir?.findFile(it, true) }
+            .mapNotNull { mangaDir?.findFile(it, true) ?: mangaDir?.findFile("$it.cbz", true) }
             .firstOrNull()
     }
 
@@ -103,12 +103,16 @@ class DownloadProvider(private val context: Context) {
         val mangaDir = findMangaDir(manga, source) ?: return emptyList()
         val chapterNameHashSet = chapters.map { it.name }.toHashSet()
         val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet()
+        val scanalatorCbzNameHashSet = chapters.map { "${getChapterDirName(it)}.cbz" }.toHashSet()
 
         return mangaDir.listFiles()!!.asList().filter { file ->
             file.name?.let { fileName ->
                 if (scanalatorNameHashSet.contains(fileName)) {
                     return@filter true
                 }
+                if (scanalatorCbzNameHashSet.contains(fileName)) {
+                    return@filter true
+                }
                 val afterScanlatorCheck = fileName.substringAfter("_")
                 return@filter chapterNameHashSet.contains(fileName) || chapterNameHashSet.contains(afterScanlatorCheck)
             }
@@ -165,6 +169,7 @@ class DownloadProvider(private val context: Context) {
         val mangaDir = findMangaDir(manga, source) ?: return emptyList()
         val chapterNameHashSet = chapters.map { it.name }.toHashSet()
         val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet()
+        val scanalatorCbzNameHashSet = chapters.map { "${getChapterDirName(it)}.cbz" }.toHashSet()
 
         return mangaDir.listFiles()!!.asList().filter { file ->
             file.name?.let { fileName ->
@@ -175,6 +180,10 @@ class DownloadProvider(private val context: Context) {
                 if (scanalatorNameHashSet.contains(fileName)) {
                     return@filter false
                 }
+                if (scanalatorCbzNameHashSet.contains(fileName)) {
+                    return@filter false
+                }
+
                 val afterScanlatorCheck = fileName.substringAfter("_")
                 // check both these dont exist because who knows how a chapter name is and it might not trim scanlator correctly
                 return@filter !chapterNameHashSet.contains(fileName) && !chapterNameHashSet.contains(afterScanlatorCheck)

app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt

@@ -18,6 +18,7 @@ import eu.kanade.tachiyomi.data.download.model.Download
 import eu.kanade.tachiyomi.data.download.model.DownloadQueue
 import eu.kanade.tachiyomi.data.library.LibraryUpdateService
 import eu.kanade.tachiyomi.data.library.PER_SOURCE_QUEUE_WARNING_THRESHOLD
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.online.HttpSource
@@ -37,7 +38,11 @@ import rx.schedulers.Schedulers
 import rx.subscriptions.CompositeSubscription
 import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
+import java.io.BufferedOutputStream
 import java.io.File
+import java.util.zip.CRC32
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
 
 /**
  * This class is the one in charge of downloading chapters.
@@ -59,7 +64,7 @@ class Downloader(
     private val cache: DownloadCache,
     private val sourceManager: SourceManager
 ) {
-
+    private val preferences: PreferencesHelper by injectLazy()
     private val chapterCache: ChapterCache by injectLazy()
 
     /**
@@ -524,9 +529,32 @@ class Downloader(
 
         // Only rename the directory if it's downloaded.
         if (download.status == Download.State.DOWNLOADED) {
-            tmpDir.renameTo(dirname)
-            cache.addChapter(dirname, download.manga)
+            if (preferences.saveChaptersAsCBZ().get()) {
+                val zip = mangaDir.createFile("$dirname.cbz.tmp")
+                val zipOut = ZipOutputStream(BufferedOutputStream(zip.openOutputStream()))
+                zipOut.setMethod(ZipEntry.STORED)
 
+                tmpDir.listFiles()?.forEach { img ->
+                    val input = img.openInputStream()
+                    val data = input.readBytes()
+                    val entry = ZipEntry(img.name)
+                    val crc = CRC32()
+                    val size = img.length()
+                    crc.update(data)
+                    entry.crc = crc.value
+                    entry.compressedSize = size
+                    entry.size = size
+                    zipOut.putNextEntry(entry)
+                    zipOut.write(data)
+                    input.close()
+                }
+                zipOut.close()
+                zip.renameTo("$dirname.cbz")
+                tmpDir.delete()
+            } else {
+                tmpDir.renameTo(dirname)
+            }
+            cache.addChapter(dirname, download.manga)
             DiskUtil.createNoMediaFile(tmpDir, context)
         }
     }

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt

@@ -259,6 +259,8 @@ object PreferenceKeys {
 
     const val chaptersDescAsDefault = "chapters_desc_as_default"
 
+    const val saveChaptersAsCBZ = "save_chapter_as_cbz"
+
     fun trackUsername(syncId: Int) = "pref_mangasync_username_$syncId"
 
     fun trackPassword(syncId: Int) = "pref_mangasync_password_$syncId"

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt

@@ -299,6 +299,8 @@ class PreferencesHelper(val context: Context) {
 
     fun downloadNew() = flowPrefs.getBoolean(Keys.downloadNew, false)
 
+    fun saveChaptersAsCBZ() = flowPrefs.getBoolean(Keys.saveChaptersAsCBZ, false)
+
     fun downloadNewCategories() = flowPrefs.getStringSet(Keys.downloadNewCategories, emptySet())
     fun downloadNewCategoriesExclude() = flowPrefs.getStringSet(Keys.downloadNewCategoriesExclude, emptySet())
 

app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/DownloadPageLoader.kt

@@ -4,12 +4,17 @@ import android.app.Application
 import android.net.Uri
 import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.download.DownloadManager
+import eu.kanade.tachiyomi.data.download.DownloadProvider
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter
 import eu.kanade.tachiyomi.ui.reader.model.ReaderPage
+import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
+import eu.kanade.tachiyomi.util.system.ImageUtil
 import rx.Observable
 import uy.kohesive.injekt.injectLazy
+import java.io.File
+import java.util.zip.ZipFile
 
 /**
  * Loader used to load a chapter from the downloaded chapters.
@@ -25,26 +30,58 @@ class DownloadPageLoader(
      * The application context. Needed to open input streams.
      */
     private val context by injectLazy<Application>()
+    private val downloadProvider by lazy { DownloadProvider(context) }
 
     /**
      * Returns an observable containing the pages found on this downloaded chapter.
      */
     override fun getPages(): Observable<List<ReaderPage>> {
-        return downloadManager.buildPageList(source, manga, chapter.chapter)
-            .map { pages ->
-                pages.map { page ->
-                    ReaderPage(
-                        page.index,
-                        page.url,
-                        page.imageUrl,
-                        {
-                            context.contentResolver.openInputStream(page.uri ?: Uri.EMPTY)!!
+        val chapterPath = downloadProvider.findChapterDir(chapter.chapter, manga, source)
+        if (chapterPath?.isFile == true) {
+            val zip = if (!File(chapterPath.filePath!!).canRead()) {
+                val tmpFile = File.createTempFile(chapterPath.name!!.replace(".cbz", ""), ".cbz")
+                val buffer = ByteArray(1024)
+                chapterPath.openInputStream().use { input ->
+                    tmpFile.outputStream().use { fileOut ->
+                        while (true) {
+                            val length = input.read(buffer)
+                            if (length <= 0) break
+                            fileOut.write(buffer, 0, length)
                         }
-                    ).apply {
+                        fileOut.flush()
+                    }
+                }
+                ZipFile(tmpFile.absolutePath)
+            } else ZipFile(chapterPath.filePath)
+            return zip.entries().toList()
+                .filter { !it.isDirectory && ImageUtil.isImage(it.name) { zip.getInputStream(it) } }
+                .sortedWith { f1, f2 -> f1.name.compareToCaseInsensitiveNaturalOrder(f2.name) }
+                .mapIndexed { i, entry ->
+                    val streamFn = { zip.getInputStream(entry) }
+                    ReaderPage(i).apply {
+                        stream = streamFn
                         status = Page.READY
                     }
                 }
-            }
+                .let { Observable.just(it) }
+        } else {
+            return downloadManager.buildPageList(source, manga, chapter.chapter)
+                .map { pages ->
+                    pages.map { page ->
+                        ReaderPage(
+                            page.index,
+                            page.url,
+                            page.imageUrl,
+
+                            {
+                                context.contentResolver.openInputStream(page.uri ?: Uri.EMPTY)!!
+                            }
+                        ).apply {
+                            status = Page.READY
+                        }
+                    }
+                }
+        }
     }
 
     override fun getPage(page: ReaderPage): Observable<Int> {

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsDownloadController.kt

@@ -47,6 +47,11 @@ class SettingsDownloadController : SettingsController() {
             titleRes = R.string.only_download_over_wifi
             defaultValue = true
         }
+        switchPreference {
+            key = Keys.saveChaptersAsCBZ
+            titleRes = R.string.save_chapters_as_cbz
+            defaultValue = false
+        }
         preferenceCategory {
             titleRes = R.string.remove_after_read
 

app/src/main/res/values/strings.xml

@@ -899,6 +899,7 @@
     <string name="always_keep">Always keep</string>
     <string name="always_delete">Always delete</string>
     <string name="automatic_removal">Automatic removal</string>
+    <string name="save_chapters_as_cbz">Save chapters as CBZ</string>
 
     <!-- Time -->
     <string name="manual">Manual</string>