null2264/yokai
1
0
Fork 0
mirror of https://github.com/null2264/yokai.git synced 2025-06-21 10:44:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add option to automatically split tall downloaded images

Browse source 
minor additions:
Add download warning for a lot of downloads, regardless of multiple sources
Add manga title to download errors (closes #1251 )

minor changes as follows:
Reduce split errors strings

Co-Authored-By: S97 <39028181+saud-97@users.noreply.github.com>
Co-Authored-By: arkon <4098258+arkon@users.noreply.github.com>
This commit is contained in:
Jays2Kings 2022-05-19 14:24:26 -04:00
parent ae15cf3b1b
commit 5c4f9200c4
7 changed files with 217 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

5
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadNotifier.kt
View file

@ -256,11 +256,14 @@ internal class DownloadNotifier(private val context: Context) {
fun onError(
error: String? = null,
chapter: String? = null,
mangaTitle: String? = null,
customIntent: Intent? = null,
) {
// Create notification
with(notification) {
setContentTitle(chapter ?: context.getString(R.string.download_error))
setContentTitle(
mangaTitle?.plus(": $chapter") ?: context.getString(R.string.download_error),
)
setContentText(error ?: context.getString(R.string.could_not_download_unexpected_error))
setStyle(
NotificationCompat.BigTextStyle().bigText(

186
app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt
View file

@ -30,6 +30,7 @@ import eu.kanade.tachiyomi.util.storage.saveTo
import eu.kanade.tachiyomi.util.system.ImageUtil
import eu.kanade.tachiyomi.util.system.launchIO
import eu.kanade.tachiyomi.util.system.launchNow
import eu.kanade.tachiyomi.util.system.withUIContext
import kotlinx.coroutines.async
import okhttp3.Response
import rx.Observable
@ -290,12 +291,18 @@ class Downloader(
// Start downloader if needed
if (autoStart && wasEmpty) {
val largestSourceSize = queue
val queuedDownloads = queue.count { it.source !is UnmeteredSource }
val maxDownloadsFromSource = queue
.groupBy { it.source }
.filterKeys { it !is UnmeteredSource }
.maxOfOrNull { it.value.size } ?: 0
if (largestSourceSize > CHAPTERS_PER_SOURCE_QUEUE_WARNING_THRESHOLD) {
notifier.massDownloadWarning()
.maxOf { it.value.size }
if (
queuedDownloads > DOWNLOADS_QUEUED_WARNING_THRESHOLD ||
maxDownloadsFromSource > CHAPTERS_PER_SOURCE_QUEUE_WARNING_THRESHOLD
) {
withUIContext {
notifier.massDownloadWarning()
}
}
DownloadService.start(context)
} else if (!isRunning && !LibraryUpdateService.isRunning()) {
@ -329,6 +336,7 @@ class Downloader(
notifier.onError(
context.getString(R.string.external_storage_download_notice),
download.chapter.name,
download.manga.title,
intent,
)
return@defer Observable.just(download)
@ -338,12 +346,13 @@ class Downloader(
val pageListObservable = if (download.pages == null) {
// Pull page list from network and add them to download object
download.source.fetchPageList(download.chapter).doOnNext { pages ->
if (pages.isEmpty()) {
throw Exception(context.getString(R.string.no_pages_found))
download.source.fetchPageList(download.chapter)
.doOnNext { pages ->
if (pages.isEmpty()) {
throw Exception(context.getString(R.string.no_pages_found))
}
download.pages = pages
}
download.pages = pages
}
} else {
// Or if the page list already exists, start from the file
Observable.just(download.pages!!)
@ -364,6 +373,7 @@ class Downloader(
// Start downloading images, consider we can have downloaded images already
// Concurrently do 5 pages at a time
.flatMap({ page -> getOrDownloadImage(page, download, tmpDir) }, 5)
.onBackpressureLatest()
// Do when page is downloaded.
.doOnNext { notifier.onProgressChange(download) }
.toList()
@ -372,8 +382,9 @@ class Downloader(
.doOnNext { ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname) }
// If the page list threw, it will resume here
.onErrorReturn { error ->
Timber.e(error)
download.status = Download.State.ERROR
notifier.onError(error.message, download.chapter.name)
notifier.onError(error.message, download.chapter.name, download.manga.title)
download
}
}
@ -403,7 +414,7 @@ class Downloader(
tmpFile?.delete()
// Try to find the image file.
val imageFile = tmpDir.listFiles()!!.find { it.name!!.startsWith("$filename.") }
val imageFile = tmpDir.listFiles()!!.find { it.name!!.startsWith("$filename.") || it.name!!.contains("${filename}__001") }
// If the image is already downloaded, do nothing. Otherwise download from network
val pageObservable = when {
@ -413,8 +424,12 @@ class Downloader(
}
return pageObservable
// When the image is ready, set image path, progress (just in case) and status
// When the page is ready, set page path, progress (just in case) and status
.doOnNext { file ->
val success = splitTallImageIfNeeded(page, tmpDir)
if (success.not()) {
notifier.onError(context.getString(R.string.download_notifier_split_failed), download.chapter.name, download.manga.title)
}
page.uri = file.uri
page.progress = 100
download.downloadedImages++
@ -425,36 +440,11 @@ class Downloader(
.onErrorReturn {
page.progress = 0
page.status = Page.ERROR
notifier.onError(it.message, download.chapter.name, download.manga.title)
page
}
}
/**
* Return the observable which copies the image from cache.
*
* @param cacheFile the file from cache.
* @param tmpDir the temporary directory of the download.
* @param filename the filename of the image.
*/
private fun moveImageFromCache(
cacheFile: File,
tmpDir: UniFile,
filename: String,
): Observable<UniFile> {
return Observable.just(cacheFile).map {
val tmpFile = tmpDir.createFile("$filename.tmp")
cacheFile.inputStream().use { input ->
tmpFile.openOutputStream().use { output ->
input.copyTo(output)
}
}
val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return@map tmpFile
tmpFile.renameTo("$filename.${extension.extension}")
cacheFile.delete()
tmpFile
}
}
/**
* Returns the observable which downloads the image from network.
*
@ -489,6 +479,32 @@ class Downloader(
.retryWhen(RetryWithDelay(3, { (2 shl it - 1) * 1000 }, Schedulers.trampoline()))
}
/**
* Return the observable which copies the image from cache.
*
* @param cacheFile the file from cache.
* @param tmpDir the temporary directory of the download.
* @param filename the filename of the image.
*/
private fun moveImageFromCache(
cacheFile: File,
tmpDir: UniFile,
filename: String,
): Observable<UniFile> {
return Observable.just(cacheFile).map {
val tmpFile = tmpDir.createFile("$filename.tmp")
cacheFile.inputStream().use { input ->
tmpFile.openOutputStream().use { output ->
input.copyTo(output)
}
}
val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return@map tmpFile
tmpFile.renameTo("$filename.${extension.extension}")
cacheFile.delete()
tmpFile
}
}
/**
* Returns the extension of the downloaded image from the network response, or if it's null,
* analyze the file. If everything fails, assume it's a jpg.
@ -507,6 +523,21 @@ class Downloader(
return MimeTypeMap.getSingleton().getExtensionFromMimeType(mime) ?: "jpg"
}
private fun splitTallImageIfNeeded(page: Page, tmpDir: UniFile): Boolean {
if (!preferences.splitTallImages().get()) return true
val filename = String.format("%03d", page.number)
val imageFile = tmpDir.listFiles()?.find { it.name!!.startsWith(filename) }
?: throw Error(context.getString(R.string.download_notifier_split_page_not_found, page.number))
val imageFilePath = imageFile.filePath
?: throw Error(context.getString(R.string.download_notifier_split_page_not_found, page.number))
// check if the original page was previously split before then skip.
if (imageFile.name!!.contains("__")) return true
return ImageUtil.splitTallImage(imageFile, imageFilePath)
}
/**
* Checks if the download was successful.
*
@ -522,46 +553,59 @@ class Downloader(
dirname: String,
) {
// Ensure that the chapter folder has all the images.
val downloadedImages = tmpDir.listFiles().orEmpty().filterNot { it.name!!.endsWith(".tmp") }
val downloadedImages = tmpDir.listFiles().orEmpty().filterNot { it.name!!.endsWith(".tmp") || (it.name!!.contains("__") && !it.name!!.contains("__001.jpg")) }
download.status = if (downloadedImages.size == download.pages!!.size) {
Download.State.DOWNLOADED
} else {
Download.State.ERROR
}
// Only rename the directory if it's downloaded.
if (download.status == Download.State.DOWNLOADED) {
// Only rename the directory if it's downloaded.
if (preferences.saveChaptersAsCBZ().get()) {
val zip = mangaDir.createFile("$dirname.cbz.tmp")
val zipOut = ZipOutputStream(BufferedOutputStream(zip.openOutputStream()))
zipOut.setMethod(ZipEntry.STORED)
tmpDir.listFiles()?.forEach { img ->
val input = img.openInputStream()
val data = input.readBytes()
val entry = ZipEntry(img.name)
val crc = CRC32()
val size = img.length()
crc.update(data)
entry.crc = crc.value
entry.compressedSize = size
entry.size = size
zipOut.putNextEntry(entry)
zipOut.write(data)
input.close()
}
zipOut.close()
zip.renameTo("$dirname.cbz")
tmpDir.delete()
archiveChapter(mangaDir, dirname, tmpDir)
} else {
tmpDir.renameTo(dirname)
}
cache.addChapter(dirname, download.manga)
DiskUtil.createNoMediaFile(tmpDir, context)
Download.State.DOWNLOADED
} else {
Download.State.ERROR
}
}
/**
* Archive the chapter pages as a CBZ.
*/
private fun archiveChapter(
mangaDir: UniFile,
dirname: String,
tmpDir: UniFile,
) {
val zip = mangaDir.createFile("$dirname.cbz.tmp")
ZipOutputStream(BufferedOutputStream(zip.openOutputStream())).use { zipOut ->
zipOut.setMethod(ZipEntry.STORED)
tmpDir.listFiles()?.forEach { img ->
img.openInputStream().use { input ->
val data = input.readBytes()
val size = img.length()
val entry = ZipEntry(img.name).apply {
val crc = CRC32().apply {
update(data)
}
setCrc(crc.value)
compressedSize = size
setSize(size)
}
zipOut.putNextEntry(entry)
zipOut.write(data)
}
}
}
zip.renameTo("$dirname.cbz")
tmpDir.delete()
}
/**
* Completes a download. This method is called in the main thread.
*/
@ -589,10 +633,10 @@ class Downloader(
companion object {
const val TMP_DIR_SUFFIX = "_tmp"
const val CHAPTERS_PER_SOURCE_QUEUE_WARNING_THRESHOLD = 15
private const val DOWNLOADS_QUEUED_WARNING_THRESHOLD = 30
// Arbitrary minimum required space to start a download: 50 MB
const val MIN_DISK_SPACE = 50 * 1024 * 1024
// Arbitrary minimum required space to start a download: 200 MB
const val MIN_DISK_SPACE = 200 * 1024 * 1024
}
}
private const val CHAPTERS_PER_SOURCE_QUEUE_WARNING_THRESHOLD = 30

2
app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt
View file

@ -309,6 +309,8 @@ class PreferencesHelper(val context: Context) {
fun saveChaptersAsCBZ() = flowPrefs.getBoolean("save_chapter_as_cbz", true)
fun splitTallImages() = flowPrefs.getBoolean("split_tall_images", false)
fun downloadNewChapters() = flowPrefs.getBoolean(Keys.downloadNew, false)
fun downloadNewChaptersInCategories() = flowPrefs.getStringSet("download_new_categories", emptySet())

6
app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsDownloadController.kt
View file

@ -51,6 +51,12 @@ class SettingsDownloadController : SettingsController() {
bindTo(preferences.saveChaptersAsCBZ())
titleRes = R.string.save_chapters_as_cbz
}
switchPreference {
bindTo(preferences.splitTallImages())
titleRes = R.string.split_tall_images
summaryRes = R.string.split_tall_images_summary
}
preferenceCategory {
titleRes = R.string.remove_after_read

4
app/src/main/java/eu/kanade/tachiyomi/util/system/ContextExtensions.kt
View file

@ -44,6 +44,7 @@ import eu.kanade.tachiyomi.widget.CustomLayoutPickerActivity
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.io.File
import kotlin.math.max
private const val TABLET_UI_MIN_SCREEN_WIDTH_DP = 720
@ -166,6 +167,9 @@ val Resources.isLTR
fun Context.isTablet() = resources.configuration.smallestScreenWidthDp >= 600
val displayMaxHeightInPx: Int
get() = Resources.getSystem().displayMetrics.let { max(it.heightPixels, it.widthPixels) }
/** Gets the duration multiplier for general animations on the device
* @see Settings.Global.ANIMATOR_DURATION_SCALE
*/

83
app/src/main/java/eu/kanade/tachiyomi/util/system/ImageUtil.kt
View file

@ -5,6 +5,7 @@ import android.content.res.Configuration
import android.content.res.Resources
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.BitmapRegionDecoder
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Rect
@ -14,16 +15,21 @@ import android.graphics.drawable.Drawable
import android.graphics.drawable.GradientDrawable
import android.os.Build
import androidx.annotation.ColorInt
import com.hippo.unifile.UniFile
import eu.kanade.tachiyomi.R
import tachiyomi.decoder.Format
import tachiyomi.decoder.ImageDecoder
import timber.log.Timber
import java.io.BufferedInputStream
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.FileOutputStream
import java.io.InputStream
import java.net.URLConnection
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min
object ImageUtil {
@ -436,6 +442,77 @@ object ImageUtil {
return ByteArrayInputStream(output.toByteArray())
}
/**
* Check whether the image is considered a tall image.
*
* @return true if the height:width ratio is greater than 3.
*/
private fun isTallImage(imageStream: InputStream): Boolean {
val options = extractImageOptions(imageStream, false)
return (options.outHeight / options.outWidth) > 3
}
/**
* Splits tall images to improve performance of reader
*/
fun splitTallImage(imageFile: UniFile, imageFilePath: String): Boolean {
if (isAnimatedAndSupported(imageFile.openInputStream()) || !isTallImage(imageFile.openInputStream())) {
return true
}
val options = extractImageOptions(imageFile.openInputStream(), false).apply { inJustDecodeBounds = false }
// Values are stored as they get modified during split loop
val imageHeight = options.outHeight
val imageWidth = options.outWidth
val splitHeight = displayMaxHeightInPx
// -1 so it doesn't try to split when imageHeight = displayMaxHeightInPx
val partCount = (imageHeight - 1) / splitHeight + 1
Timber.d("Splitting ${imageHeight}px height image into $partCount part with estimated ${splitHeight}px per height")
val bitmapRegionDecoder = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
BitmapRegionDecoder.newInstance(imageFile.openInputStream())
} else {
@Suppress("DEPRECATION")
BitmapRegionDecoder.newInstance(imageFile.openInputStream(), false)
}
if (bitmapRegionDecoder == null) {
Timber.d("Failed to create new instance of BitmapRegionDecoder")
return false
}
try {
(0 until partCount).forEach { splitIndex ->
val splitPath = imageFilePath.substringBeforeLast(".") + "__${"%03d".format(splitIndex + 1)}.jpg"
val topOffset = splitIndex * splitHeight
val outputImageHeight = min(splitHeight, imageHeight - topOffset)
val bottomOffset = topOffset + outputImageHeight
Timber.d("Split #$splitIndex with topOffset=$topOffset height=$outputImageHeight bottomOffset=$bottomOffset")
val region = Rect(0, topOffset, imageWidth, bottomOffset)
FileOutputStream(splitPath).use { outputStream ->
val splitBitmap = bitmapRegionDecoder.decodeRegion(region, options)
splitBitmap.compress(Bitmap.CompressFormat.JPEG, 100, outputStream)
}
}
imageFile.delete()
return true
} catch (e: Exception) {
// Image splits were not successfully saved so delete them and keep the original image
(0 until partCount)
.map { imageFilePath.substringBeforeLast(".") + "__${"%03d".format(it + 1)}.jpg" }
.forEach { File(it).delete() }
Timber.e(e)
return false
} finally {
bitmapRegionDecoder.recycle()
}
}
private val Bitmap.rect: Rect
get() = Rect(0, 0, width, height)
@ -498,12 +575,16 @@ object ImageUtil {
/**
* Used to check an image's dimensions without loading it in the memory.
*/
private fun extractImageOptions(imageStream: InputStream): BitmapFactory.Options {
private fun extractImageOptions(
imageStream: InputStream,
resetAfterExtraction: Boolean = true,
): BitmapFactory.Options {
imageStream.mark(imageStream.available() + 1)
val imageBytes = imageStream.readBytes()
val options = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size, options)
if (resetAfterExtraction) imageStream.reset()
return options
}
}

4
app/src/main/res/values/strings.xml
View file

@ -922,6 +922,8 @@
<string name="visit_recents_for_download_queue">Visit the recents tab to access the download
queue. You can also double tap or press and hold for quicker access</string>
<string name="couldnt_download_low_space">Couldn\'t download chapters due to low disk space</string>
<string name="download_notifier_split_page_not_found">Page %d not found while splitting</string>
<string name="download_notifier_split_failed">Couldn\'t split downloaded image</string>
<!-- Download Notification -->
<string name="could_not_download_unexpected_error">Could not download chapter due to unexpected error</string>
@ -952,6 +954,8 @@
<string name="always_delete">Always delete</string>
<string name="automatic_removal">Automatic removal</string>
<string name="save_chapters_as_cbz">Save as CBZ archive</string>
<string name="split_tall_images">Auto split tall images</string>
<string name="split_tall_images_summary">Improves reader performance by splitting tall downloaded images.</string>
<!-- Time -->
<string name="manual">Manual</string>