refactor: Modularize the project (#97)

* refactor: Modularize the project

* chore: Move okhttp stuff back to androidMain

OkHttp decided to cancel multiplatform plan on 5.0

REF: https://square.github.io/okhttp/changelogs/changelog/#version-500-alpha13

* feat: Start using moko for i18n

* fix: Solve some errors

* chore: Remove manga from domain module

We'll do this later

* fix: Duplicate error

* fix: Conflict function name error

* fix: Target SManga

* fix: Breaking changes after the split

* fix: Not enough heap memory

* chore: Update proguard rules

Sorta similar to upstream

* refactor: Fix namespace

source-api/build.gradle.kts

@@ -0,0 +1,42 @@
+plugins {
+    kotlin("multiplatform")
+    kotlin("plugin.serialization")
+    id("com.android.library")
+}
+
+kotlin {
+    androidTarget()
+    sourceSets {
+        val commonMain by getting {
+            dependencies {
+                api(kotlinx.serialization.json)
+                api(libs.injekt.core)
+                api(libs.rxjava)
+                api(libs.jsoup)
+            }
+        }
+        val androidMain by getting {
+            dependencies {
+                implementation(projects.core)
+                api(androidx.preference)
+
+                // Workaround for https://youtrack.jetbrains.com/issue/KT-57605
+                implementation(kotlinx.coroutines.android)
+                implementation(project.dependencies.platform(kotlinx.coroutines.bom))
+            }
+        }
+    }
+}
+android {
+    namespace = "eu.kanade.tachiyomi.source"
+    defaultConfig {
+        consumerProguardFile("consumer-proguard.pro")
+    }
+}
+tasks {
+    withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
+        kotlinOptions.freeCompilerArgs += listOf(
+            "-Xexpect-actual-classes",
+        )
+    }
+}

source-api/consumer-proguard.pro

@@ -0,0 +1,5 @@
+-keep class eu.kanade.tachiyomi.source.model.** { public protected *; }
+-keep class eu.kanade.tachiyomi.source.online.** { public protected *; }
+-keep class eu.kanade.tachiyomi.source.** extends eu.kanade.tachiyomi.source.Source { public protected *; }
+
+-keep,allowoptimization class eu.kanade.tachiyomi.util.JsoupExtensionsKt { public protected *; }

source-api/src/androidMain/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest />

source-api/src/androidMain/kotlin/eu/kanade/tachiyomi/util/RxExtension.kt

@@ -0,0 +1,6 @@
+package eu.kanade.tachiyomi.util
+
+import rx.Observable
+import yokai.util.lang.awaitSingle
+
+actual suspend fun <T> Observable<T>.awaitSingle(): T = awaitSingle()

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/CatalogueSource.kt

@@ -0,0 +1,80 @@
+package eu.kanade.tachiyomi.source
+
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.util.awaitSingle
+import rx.Observable
+
+interface CatalogueSource : Source {
+
+    /**
+     * An ISO 639-1 compliant language code (two letters in lower case).
+     */
+    override val lang: String
+
+    /**
+     * Whether the source has support for latest updates.
+     */
+    val supportsLatest: Boolean
+
+    /**
+     * Get a page with a list of manga.
+     *
+     * @since extensions-lib 1.5
+     * @param page the page number to retrieve.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getPopularManga(page: Int): MangasPage {
+        return fetchPopularManga(page).awaitSingle()
+    }
+
+    /**
+     * Get a page with a list of manga.
+     *
+     * @since extensions-lib 1.5
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getSearchManga(page: Int, query: String, filters: FilterList): MangasPage {
+        return fetchSearchManga(page, query, filters).awaitSingle()
+    }
+
+    /**
+     * Get a page with a list of latest manga updates.
+     *
+     * @since extensions-lib 1.5
+     * @param page the page number to retrieve.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getLatestUpdates(page: Int): MangasPage {
+        return fetchLatestUpdates(page).awaitSingle()
+    }
+
+    /**
+     * Returns the list of filters for the source.
+     */
+    fun getFilterList(): FilterList
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getPopularManga"),
+    )
+    fun fetchPopularManga(page: Int): Observable<MangasPage> =
+        throw IllegalStateException("Not used")
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getSearchManga"),
+    )
+    fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> =
+        throw IllegalStateException("Not used")
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getLatestUpdates"),
+    )
+    fun fetchLatestUpdates(page: Int): Observable<MangasPage> =
+        throw IllegalStateException("Not used")
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/ConfigurableSource.kt

@@ -0,0 +1,28 @@
+package eu.kanade.tachiyomi.source
+
+import android.app.Application
+import android.content.Context
+import android.content.SharedPreferences
+import androidx.preference.PreferenceScreen
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+
+interface ConfigurableSource : Source {
+
+    /**
+     * Gets instance of [SharedPreferences] scoped to the specific source.
+     *
+     * @since extensions-lib 1.5
+     */
+    fun getSourcePreferences(): SharedPreferences =
+        Injekt.get<Application>().getSharedPreferences(preferenceKey(), Context.MODE_PRIVATE)
+
+    fun setupPreferenceScreen(screen: PreferenceScreen)
+}
+
+// TODO: use getSourcePreferences once all extensions are on ext-lib 1.5
+fun ConfigurableSource.sourcePreferences(): SharedPreferences =
+    Injekt.get<Application>().getSharedPreferences(preferenceKey(), Context.MODE_PRIVATE)
+
+fun sourcePreferences(key: String): SharedPreferences =
+    Injekt.get<Application>().getSharedPreferences(key, Context.MODE_PRIVATE)

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/Source.kt

@@ -0,0 +1,86 @@
+package eu.kanade.tachiyomi.source
+
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.awaitSingle
+import rx.Observable
+
+/**
+ * A basic interface for creating a source. It could be an online source, a local source, etc.
+ */
+interface Source {
+
+    /**
+     * ID for the source. Must be unique.
+     */
+    val id: Long
+
+    /**
+     * Name of the source.
+     */
+    val name: String
+
+    val lang: String
+        get() = ""
+
+    /**
+     * Get the updated details for a manga.
+     *
+     * @since extensions-lib 1.5
+     * @param manga the manga to update.
+     * @return the updated manga.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getMangaDetails(manga: SManga): SManga {
+        return fetchMangaDetails(manga).awaitSingle()
+    }
+
+    /**
+     * Get all the available chapters for a manga.
+     *
+     * @since extensions-lib 1.5
+     * @param manga the manga to update.
+     * @return the chapters for the manga.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getChapterList(manga: SManga): List<SChapter> {
+        return fetchChapterList(manga).awaitSingle()
+    }
+
+    /**
+     * Get the list of pages a chapter has. Pages should be returned
+     * in the expected order; the index is ignored.
+     *
+     * @since extensions-lib 1.5
+     * @param chapter the chapter.
+     * @return the pages for the chapter.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getPageList(chapter: SChapter): List<Page> {
+        return fetchPageList(chapter).awaitSingle()
+    }
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getMangaDetails"),
+    )
+    fun fetchMangaDetails(manga: SManga): Observable<SManga> =
+        throw IllegalStateException("Not used")
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getChapterList"),
+    )
+    fun fetchChapterList(manga: SManga): Observable<List<SChapter>> =
+        throw IllegalStateException("Not used")
+
+    @Deprecated(
+        "Use the non-RxJava API instead",
+        ReplaceWith("getPageList"),
+    )
+    fun fetchPageList(chapter: SChapter): Observable<List<Page>> =
+        throw IllegalStateException("Not used")
+}
+
+fun Source.preferenceKey(): String = "source_$id"

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/SourceFactory.kt

@@ -0,0 +1,12 @@
+package eu.kanade.tachiyomi.source
+
+/**
+ * A factory for creating sources at runtime.
+ */
+interface SourceFactory {
+    /**
+     * Create a new copy of the sources
+     * @return The created sources
+     */
+    fun createSources(): List<Source>
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/UnmeteredSource.kt

@@ -0,0 +1,8 @@
+package eu.kanade.tachiyomi.source
+
+/**
+ * A source that explicitly doesn't require traffic considerations.
+ *
+ * This typically applies for self-hosted sources.
+ */
+interface UnmeteredSource

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/Filter.kt

@@ -0,0 +1,39 @@
+package eu.kanade.tachiyomi.source.model
+
+sealed class Filter<T>(val name: String, var state: T) {
+    open class Header(name: String) : Filter<Any>(name, 0)
+    open class Separator(name: String = "") : Filter<Any>(name, 0)
+    abstract class Select<V>(name: String, val values: Array<V>, state: Int = 0) : Filter<Int>(name, state)
+    abstract class Text(name: String, state: String = "") : Filter<String>(name, state)
+    abstract class CheckBox(name: String, state: Boolean = false) : Filter<Boolean>(name, state)
+    abstract class TriState(name: String, state: Int = STATE_IGNORE) : Filter<Int>(name, state) {
+        fun isIgnored() = state == STATE_IGNORE
+        fun isIncluded() = state == STATE_INCLUDE
+        fun isExcluded() = state == STATE_EXCLUDE
+
+        companion object {
+            const val STATE_IGNORE = 0
+            const val STATE_INCLUDE = 1
+            const val STATE_EXCLUDE = 2
+        }
+    }
+    abstract class Group<V>(name: String, state: List<V>) : Filter<List<V>>(name, state)
+
+    abstract class Sort(name: String, val values: Array<String>, state: Selection? = null) :
+        Filter<Sort.Selection?>(name, state) {
+        data class Selection(val index: Int, val ascending: Boolean)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is Filter<*>) return false
+
+        return name == other.name && state == other.state
+    }
+
+    override fun hashCode(): Int {
+        var result = name.hashCode()
+        result = 31 * result + (state?.hashCode() ?: 0)
+        return result
+    }
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/FilterList.kt

@@ -0,0 +1,6 @@
+package eu.kanade.tachiyomi.source.model
+
+data class FilterList(val list: List<Filter<*>>) : List<Filter<*>> by list {
+
+    constructor(vararg fs: Filter<*>) : this(if (fs.isNotEmpty()) fs.asList() else emptyList())
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/MangasPage.kt

@@ -0,0 +1,3 @@
+package eu.kanade.tachiyomi.source.model
+
+data class MangasPage(val mangas: List<SManga>, val hasNextPage: Boolean)

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/Page.kt

@@ -0,0 +1,58 @@
+package eu.kanade.tachiyomi.source.model
+
+import android.net.Uri
+import eu.kanade.tachiyomi.network.ProgressListener
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
+
+@Serializable
+open class Page(
+    val index: Int,
+    val url: String = "",
+    var imageUrl: String? = null,
+    @Transient var uri: Uri? = null, // Deprecated but can't be deleted due to extensions
+) : ProgressListener {
+
+    val number: Int
+        get() = index + 1
+
+    @Transient
+    private val _statusFlow = MutableStateFlow(State.QUEUE)
+
+    @Transient
+    val statusFlow = _statusFlow.asStateFlow()
+    var status: State
+        get() = _statusFlow.value
+        set(value) {
+            _statusFlow.value = value
+        }
+
+    @Transient
+    private val _progressFlow = MutableStateFlow(0)
+
+    @Transient
+    val progressFlow = _progressFlow.asStateFlow()
+    var progress: Int
+        get() = _progressFlow.value
+        set(value) {
+            _progressFlow.value = value
+        }
+
+    override fun update(bytesRead: Long, contentLength: Long, done: Boolean) {
+        progress = if (contentLength > 0) {
+            (100 * bytesRead / contentLength).toInt()
+        } else {
+            -1
+        }
+    }
+
+    enum class State {
+        QUEUE,
+        LOAD_PAGE,
+        DOWNLOAD_IMAGE,
+        READY,
+        ERROR,
+    }
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/SChapter.kt

@@ -0,0 +1,30 @@
+package eu.kanade.tachiyomi.source.model
+
+import java.io.Serializable
+
+interface SChapter : Serializable {
+
+    var url: String
+
+    var name: String
+
+    var date_upload: Long
+
+    var chapter_number: Float
+
+    var scanlator: String?
+
+    fun copyFrom(other: SChapter) {
+        name = other.name
+        url = other.url
+        date_upload = other.date_upload
+        chapter_number = other.chapter_number
+        scanlator = other.scanlator
+    }
+
+    companion object {
+        fun create(): SChapter {
+            return SChapterImpl()
+        }
+    }
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/SChapterImpl.kt

@@ -0,0 +1,14 @@
+package eu.kanade.tachiyomi.source.model
+
+class SChapterImpl : SChapter {
+
+    override lateinit var url: String
+
+    override lateinit var name: String
+
+    override var date_upload: Long = 0
+
+    override var chapter_number: Float = -1f
+
+    override var scanlator: String? = null
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt

@@ -0,0 +1,52 @@
+package eu.kanade.tachiyomi.source.model
+
+import java.io.Serializable
+
+interface SManga : Serializable {
+
+    var url: String
+
+    var title: String
+
+    var artist: String?
+
+    var author: String?
+
+    var description: String?
+
+    var genre: String?
+
+    var status: Int
+
+    var thumbnail_url: String?
+
+    var update_strategy: UpdateStrategy
+
+    var initialized: Boolean
+
+    fun copy() = create().also {
+        it.url = url
+        it.title = title
+        it.artist = artist
+        it.author = author
+        it.description = description
+        it.genre = genre
+        it.status = status
+        it.thumbnail_url = thumbnail_url
+        it.initialized = initialized
+    }
+
+    companion object {
+        const val UNKNOWN = 0
+        const val ONGOING = 1
+        const val COMPLETED = 2
+        const val LICENSED = 3
+        const val PUBLISHING_FINISHED = 4
+        const val CANCELLED = 5
+        const val ON_HIATUS = 6
+
+        fun create(): SManga {
+            return SMangaImpl()
+        }
+    }
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt

@@ -0,0 +1,24 @@
+package eu.kanade.tachiyomi.source.model
+
+class SMangaImpl : SManga {
+
+    override lateinit var url: String
+
+    override lateinit var title: String
+
+    override var artist: String? = null
+
+    override var author: String? = null
+
+    override var description: String? = null
+
+    override var genre: String? = null
+
+    override var status: Int = 0
+
+    override var thumbnail_url: String? = null
+
+    override var update_strategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE
+
+    override var initialized: Boolean = false
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.source.model
+
+/**
+ * Define the update strategy for a single [SManga].
+ * The strategy used will only take effect on the library update.
+ *
+ * @since extensions-lib 1.4
+ */
+enum class UpdateStrategy {
+    /**
+     * Series marked as always update will be included in the library
+     * update if they aren't excluded by additional restrictions.
+     */
+    ALWAYS_UPDATE,
+
+    /**
+     * Series marked as only fetch once will be automatically skipped
+     * during library updates. Useful for cases where the series is previously
+     * known to be finished and have only a single chapter, for example.
+     */
+    ONLY_FETCH_ONCE,
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -0,0 +1,503 @@
+package eu.kanade.tachiyomi.source.online
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.awaitSuccess
+import eu.kanade.tachiyomi.network.newCachelessCallWithProgress
+import eu.kanade.tachiyomi.source.CatalogueSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.awaitSingle
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
+import rx.Observable
+import uy.kohesive.injekt.injectLazy
+import java.net.URI
+import java.net.URISyntaxException
+import java.security.MessageDigest
+
+/**
+ * A simple implementation for sources from a website.
+ */
+@Suppress("unused")
+abstract class HttpSource : CatalogueSource {
+
+    /**
+     * Network service.
+     */
+    protected val network: NetworkHelper by injectLazy()
+
+    /**
+     * Base url of the website without the trailing slash, like: http://mysite.com
+     */
+    abstract val baseUrl: String
+
+    /**
+     * Version id used to generate the source id. If the site completely changes and urls are
+     * incompatible, you may increase this value and it'll be considered as a new source.
+     */
+    open val versionId = 1
+
+    /**
+     * ID of the source. By default it uses a generated id using the first 16 characters (64 bits)
+     * of the MD5 of the string `"${name.lowercase()}/$lang/$versionId"`.
+     *
+     * The ID is generated by the [generateId] function, which can be reused if needed
+     * to generate outdated IDs for cases where the source name or language needs to
+     * be changed but migrations can be avoided.
+     *
+     * Note: the generated ID sets the sign bit to `0`.
+     */
+    override val id by lazy { generateId(name, lang, versionId) }
+
+    /**
+     * Headers used for requests.
+     */
+    val headers: Headers by lazy { headersBuilder().build() }
+
+    /**
+     * Default network client for doing requests.
+     */
+    open val client: OkHttpClient
+        get() = network.client
+
+    /**
+     * Generates a unique ID for the source based on the provided [name], [lang] and
+     * [versionId]. It will use the first 16 characters (64 bits) of the MD5 of the string
+     * `"${name.lowercase()}/$lang/$versionId"`.
+     *
+     * Note: the generated ID sets the sign bit to `0`.
+     *
+     * Can be used to generate outdated IDs, such as when the source name or language
+     * needs to be changed but migrations can be avoided.
+     *
+     * @since extensions-lib 1.5
+     * @param name [String] the name of the source
+     * @param lang [String] the language of the source
+     * @param versionId [Int] the version ID of the source
+     * @return a unique ID for the source
+     */
+    @Suppress("MemberVisibilityCanBePrivate")
+    protected fun generateId(name: String, lang: String, versionId: Int): Long {
+        val key = "${name.lowercase()}/$lang/$versionId"
+        val bytes = MessageDigest.getInstance("MD5").digest(key.toByteArray())
+        return (0..7).map { bytes[it].toLong() and 0xff shl 8 * (7 - it) }.reduce(Long::or) and Long.MAX_VALUE
+    }
+
+    /**
+     * Headers builder for requests. Implementations can override this method for custom headers.
+     */
+    protected open fun headersBuilder() = Headers.Builder().apply {
+        add("User-Agent", network.defaultUserAgent)
+    }
+
+    /**
+     * Visible name of the source.
+     */
+    override fun toString() = "$name (${lang.uppercase()})"
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page number to retrieve.
+     */
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+        return client.newCall(popularMangaRequest(page))
+            .asObservableSuccess()
+            .map { response ->
+                popularMangaParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for the popular manga given the page.
+     *
+     * @param page the page number to retrieve.
+     */
+    protected abstract fun popularMangaRequest(page: Int): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun popularMangaParse(response: Response): MangasPage
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    override fun fetchSearchManga(
+        page: Int,
+        query: String,
+        filters: FilterList,
+    ): Observable<MangasPage> {
+        return Observable.defer {
+            try {
+                client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess()
+            } catch (e: NoClassDefFoundError) {
+                // RxJava doesn't handle Errors, which tends to happen during global searches
+                // if an old extension using non-existent classes is still around
+                throw RuntimeException(e)
+            }
+        }
+            .map { response ->
+                searchMangaParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for the search manga given the page.
+     *
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    protected abstract fun searchMangaRequest(
+        page: Int,
+        query: String,
+        filters: FilterList,
+    ): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun searchMangaParse(response: Response): MangasPage
+
+    /**
+     * Returns an observable containing a page with a list of latest manga updates.
+     *
+     * @param page the page number to retrieve.
+     */
+    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
+        return client.newCall(latestUpdatesRequest(page))
+            .asObservableSuccess()
+            .map { response ->
+                latestUpdatesParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for latest manga given the page.
+     *
+     * @param page the page number to retrieve.
+     */
+    protected abstract fun latestUpdatesRequest(page: Int): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun latestUpdatesParse(response: Response): MangasPage
+
+    /**
+     * Get the updated details for a manga.
+     * Normally it's not needed to override this method.
+     *
+     * @param manga the manga to update.
+     * @return the updated manga.
+     */
+    @Suppress("DEPRECATION")
+    override suspend fun getMangaDetails(manga: SManga): SManga {
+        return fetchMangaDetails(manga).awaitSingle()
+    }
+
+    @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getMangaDetails"))
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return client.newCall(mangaDetailsRequest(manga))
+            .asObservableSuccess()
+            .map { response ->
+                mangaDetailsParse(response).apply { initialized = true }
+            }
+    }
+
+    /**
+     * Returns the request for the details of a manga. Override only if it's needed to change the
+     * url, send different headers or request method like POST.
+     *
+     * @param manga the manga to be updated.
+     */
+    open fun mangaDetailsRequest(manga: SManga): Request {
+        return GET(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns the details of a manga.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun mangaDetailsParse(response: Response): SManga
+
+    /**
+     * Get all the available chapters for a manga.
+     * Normally it's not needed to override this method.
+     *
+     * @param manga the manga to update.
+     * @return the chapters for the manga.
+     * @throws LicensedMangaChaptersException if a manga is licensed and therefore no chapters are available.
+     */
+    @Suppress("DEPRECATION")
+    override suspend fun getChapterList(manga: SManga): List<SChapter> {
+        if (manga.status == SManga.LICENSED) {
+            throw LicensedMangaChaptersException()
+        }
+
+        return fetchChapterList(manga).awaitSingle()
+    }
+
+    @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getChapterList"))
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        return if (manga.status != SManga.LICENSED) {
+            client.newCall(chapterListRequest(manga))
+                .asObservableSuccess()
+                .map { response ->
+                    chapterListParse(response)
+                }
+        } else {
+            Observable.error(LicensedMangaChaptersException())
+        }
+    }
+
+    /**
+     * Returns the request for updating the chapter list. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param manga the manga to look for chapters.
+     */
+    protected open fun chapterListRequest(manga: SManga): Request {
+        return GET(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns a list of chapters.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun chapterListParse(response: Response): List<SChapter>
+
+    /**
+     * Get the list of pages a chapter has. Pages should be returned
+     * in the expected order; the index is ignored.
+     *
+     * @param chapter the chapter.
+     * @return the pages for the chapter.
+     */
+    @Suppress("DEPRECATION")
+    override suspend fun getPageList(chapter: SChapter): List<Page> {
+        return fetchPageList(chapter).awaitSingle()
+    }
+
+    @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getPageList"))
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        return client.newCall(pageListRequest(chapter))
+            .asObservableSuccess()
+            .map { response ->
+                pageListParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for getting the page list. Override only if it's needed to override the
+     * url, send different headers or request method like POST.
+     *
+     * @param chapter the chapter whose page list has to be fetched.
+     */
+    protected open fun pageListRequest(chapter: SChapter): Request {
+        return GET(baseUrl + chapter.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns a list of pages.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun pageListParse(response: Response): List<Page>
+
+    /**
+     * Returns an observable with the page containing the source url of the image. If there's any
+     * error, it will return null instead of throwing an exception.
+     *
+     * @since extensions-lib 1.5
+     * @param page the page whose source image has to be fetched.
+     */
+    @Suppress("DEPRECATION")
+    open suspend fun getImageUrl(page: Page): String {
+        return fetchImageUrl(page).awaitSingle()
+    }
+
+    @Deprecated("Use the non-RxJava API instead", replaceWith = ReplaceWith("getImageUrl"))
+    open fun fetchImageUrl(page: Page): Observable<String> {
+        return client.newCall(imageUrlRequest(page))
+            .asObservableSuccess()
+            .map { imageUrlParse(it) }
+    }
+
+    /**
+     * Returns the request for getting the url to the source image. Override only if it's needed to
+     * override the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    protected open fun imageUrlRequest(page: Page): Request {
+        return GET(page.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun imageUrlParse(response: Response): String
+
+    /**
+     * Returns the response of the source image.
+     * Typically does not need to be overridden.
+     *
+     * @since extensions-lib 1.5
+     * @param page the page whose source image has to be downloaded.
+     */
+    open suspend fun getImage(page: Page): Response {
+        return client.newCachelessCallWithProgress(imageRequest(page), page)
+            .awaitSuccess()
+    }
+
+    /**
+     * Returns the request for getting the source image. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    protected open fun imageRequest(page: Page): Request {
+        return GET(page.imageUrl!!, headers)
+    }
+
+    /**
+     * Assigns the url of the chapter without the scheme and domain. It saves some redundancy from
+     * database and the urls could still work after a domain change.
+     *
+     * @param url the full url to the chapter.
+     */
+    fun SChapter.setUrlWithoutDomain(url: String) {
+        this.url = getUrlWithoutDomain(url)
+    }
+
+    /**
+     * Assigns the url of the manga without the scheme and domain. It saves some redundancy from
+     * database and the urls could still work after a domain change.
+     *
+     * @param url the full url to the manga.
+     */
+    fun SManga.setUrlWithoutDomain(url: String) {
+        this.url = getUrlWithoutDomain(url)
+    }
+
+    /**
+     * Returns the url of the given string without the scheme and domain.
+     *
+     * @param orig the full url.
+     */
+    private fun getUrlWithoutDomain(orig: String): String {
+        return try {
+            val uri = URI(orig.replace(" ", "%20"))
+            var out = uri.path
+            if (uri.query != null) {
+                out += "?" + uri.query
+            }
+            if (uri.fragment != null) {
+                out += "#" + uri.fragment
+            }
+            out
+        } catch (e: URISyntaxException) {
+            orig
+        }
+    }
+
+    /**
+     * Returns the url of the provided manga
+     *
+     * @since extensions-lib 1.4
+     * @param manga the manga
+     * @return url of the manga
+     */
+    open fun getMangaUrl(manga: SManga): String {
+        return mangaDetailsRequest(manga).url.toString()
+    }
+
+    /**
+     * Returns the url of the provided chapter, default is empty to use workaround when possible
+     *
+     * @since extensions-lib 1.4
+     * @param chapter the chapter
+     * @return url of the chapter
+     */
+    open fun getChapterUrl(chapter: SChapter): String {
+        return ""
+    }
+
+    fun getChapterUrl(manga: SManga?, chapter: SChapter): String? {
+        manga ?: return null
+
+        val chapterUrl = getUrlWithoutDomain(chapter.url)
+        val mangaUrl = getMangaUrl(manga)
+        return if (chapterUrl.isBlank()) {
+            mangaUrl
+        } else {
+            fullChapterUrl(mangaUrl, chapterUrl, chapter)
+        }
+    }
+
+    /** Helper method to handle guya-like sources */
+    private fun fullChapterUrl(mangaUrl: String, chapterUrl: String, chapter: SChapter): String {
+        val lowerUrl = baseUrl.lowercase()
+        return when {
+            chapter.url.startsWith("http") -> {
+                chapter.url
+            }
+            lowerUrl.contains("guya") || lowerUrl.contains("danke") ||
+                lowerUrl.contains("hachirumi") || lowerUrl.contains("mahoushoujobu") ||
+                (lowerUrl.contains("cubari") && !mangaUrl.contains("imgur")) -> {
+                // cubari links would have double / without the trim end
+                mangaUrl.trimEnd('/') + "/" + chapter.chapter_number.fmt().replace(".", "-")
+            }
+            else -> baseUrl + chapterUrl
+        }
+    }
+
+    private fun Float.fmt(): String {
+        return if (this == toLong().toFloat()) {
+            String.format("%d", toLong())
+        } else {
+            String.format("%s", this)
+        }
+    }
+
+    /**
+     * Called before inserting a new chapter into database. Use it if you need to override chapter
+     * fields, like the title or the chapter number. Do not change anything to [manga].
+     *
+     * @param chapter the chapter to be added.
+     * @param manga the manga of the chapter.
+     */
+    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {}
+
+    /**
+     * Returns the list of filters for the source.
+     */
+    override fun getFilterList() = FilterList()
+}
+
+class LicensedMangaChaptersException : Exception("Licensed - No chapters to show")

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt

@@ -0,0 +1,200 @@
+package eu.kanade.tachiyomi.source.online
+
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+/**
+ * A simple implementation for sources from a website using Jsoup, an HTML parser.
+ */
+abstract class ParsedHttpSource : HttpSource() {
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun popularMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(popularMangaSelector()).map { element ->
+            popularMangaFromElement(element)
+        }
+
+        val hasNextPage = popularMangaNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun popularMangaSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [popularMangaSelector].
+     */
+    protected abstract fun popularMangaFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun popularMangaNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun searchMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(searchMangaSelector()).map { element ->
+            searchMangaFromElement(element)
+        }
+
+        val hasNextPage = searchMangaNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun searchMangaSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [searchMangaSelector].
+     */
+    protected abstract fun searchMangaFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun searchMangaNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(latestUpdatesSelector()).map { element ->
+            latestUpdatesFromElement(element)
+        }
+
+        val hasNextPage = latestUpdatesNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun latestUpdatesSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [latestUpdatesSelector].
+     */
+    protected abstract fun latestUpdatesFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun latestUpdatesNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns the details of a manga.
+     *
+     * @param response the response from the site.
+     */
+    override fun mangaDetailsParse(response: Response): SManga {
+        return mangaDetailsParse(response.asJsoup())
+    }
+
+    /**
+     * Returns the details of the manga from the given [document].
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun mangaDetailsParse(document: Document): SManga
+
+    /**
+     * Parses the response from the site and returns a list of chapters.
+     *
+     * @param response the response from the site.
+     */
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val document = response.asJsoup()
+        return document.select(chapterListSelector()).map { chapterFromElement(it) }
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each chapter.
+     */
+    protected abstract fun chapterListSelector(): String
+
+    /**
+     * Returns a chapter from the given element.
+     *
+     * @param element an element obtained from [chapterListSelector].
+     */
+    protected abstract fun chapterFromElement(element: Element): SChapter
+
+    /**
+     * Parses the response from the site and returns the page list.
+     *
+     * @param response the response from the site.
+     */
+    override fun pageListParse(response: Response): List<Page> {
+        return pageListParse(response.asJsoup())
+    }
+
+    /**
+     * Returns a page list from the given document.
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun pageListParse(document: Document): List<Page>
+
+    /**
+     * Parse the response from the site and returns the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    override fun imageUrlParse(response: Response): String {
+        return imageUrlParse(response.asJsoup())
+    }
+
+    /**
+     * Returns the absolute url to the source image from the document.
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun imageUrlParse(document: Document): String
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/util/JsoupExtensions.kt

@@ -0,0 +1,26 @@
+package eu.kanade.tachiyomi.util
+
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+fun Element.selectText(css: String, defaultValue: String? = null): String? {
+    return select(css).first()?.text() ?: defaultValue
+}
+
+fun Element.selectInt(css: String, defaultValue: Int = 0): Int {
+    return select(css).first()?.text()?.toInt() ?: defaultValue
+}
+
+fun Element.attrOrText(css: String): String {
+    return if (css != "text") attr(css) else text()
+}
+
+/**
+ * Returns a Jsoup document for this response.
+ * @param html the body of the response. Use only if the body was read before calling this method.
+ */
+fun Response.asJsoup(html: String? = null): Document {
+    return Jsoup.parse(html ?: body.string(), request.url.toString())
+}

source-api/src/commonMain/kotlin/eu/kanade/tachiyomi/util/RxExtension.kt

@@ -0,0 +1,5 @@
+package eu.kanade.tachiyomi.util
+
+import rx.Observable
+
+expect suspend fun <T> Observable<T>.awaitSingle(): T