Extract source api from app module (#8014)

* Extract source api from app module

* Extract source online api from app module

source-api/.gitignore

@@ -0,0 +1 @@
+/build

source-api/build.gradle.kts

@@ -0,0 +1,39 @@
+plugins {
+    id("com.android.library")
+    kotlin("android")
+    kotlin("plugin.serialization")
+}
+
+android {
+    namespace = "eu.kanade.tachiyomi.source"
+    compileSdk = AndroidConfig.compileSdk
+
+    defaultConfig {
+        minSdk = AndroidConfig.minSdk
+        targetSdk = AndroidConfig.targetSdk
+    }
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+
+    kotlinOptions {
+        jvmTarget = JavaVersion.VERSION_1_8.toString()
+    }
+}
+
+dependencies {
+
+    implementation(project(":core"))
+
+    api(kotlinx.serialization.json)
+
+    api(libs.rxjava)
+
+    api(libs.preferencektx)
+
+    api(libs.jsoup)
+
+    implementation(androidx.corektx)
+}

source-api/src/main/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest />

source-api/src/main/java/eu/kanade/tachiyomi/source/CatalogueSource.kt

@@ -0,0 +1,46 @@
+package eu.kanade.tachiyomi.source
+
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import rx.Observable
+
+interface CatalogueSource : Source {
+
+    /**
+     * An ISO 639-1 compliant language code (two letters in lower case).
+     */
+    override val lang: String
+
+    /**
+     * Whether the source has support for latest updates.
+     */
+    val supportsLatest: Boolean
+
+    /**
+     * Returns an observable containing a page with a list of manga.
+     *
+     * @param page the page number to retrieve.
+     */
+    fun fetchPopularManga(page: Int): Observable<MangasPage>
+
+    /**
+     * Returns an observable containing a page with a list of manga.
+     *
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage>
+
+    /**
+     * Returns an observable containing a page with a list of latest manga updates.
+     *
+     * @param page the page number to retrieve.
+     */
+    fun fetchLatestUpdates(page: Int): Observable<MangasPage>
+
+    /**
+     * Returns the list of filters for the source.
+     */
+    fun getFilterList(): FilterList
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/ConfigurableSource.kt

@@ -0,0 +1,8 @@
+package eu.kanade.tachiyomi.source
+
+import androidx.preference.PreferenceScreen
+
+interface ConfigurableSource : Source {
+
+    fun setupPreferenceScreen(screen: PreferenceScreen)
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/Source.kt

@@ -0,0 +1,84 @@
+package eu.kanade.tachiyomi.source
+
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.lang.awaitSingle
+import rx.Observable
+
+/**
+ * A basic interface for creating a source. It could be an online source, a local source, etc...
+ */
+interface Source {
+
+    /**
+     * Id for the source. Must be unique.
+     */
+    val id: Long
+
+    /**
+     * Name of the source.
+     */
+    val name: String
+
+    val lang: String
+        get() = ""
+
+    /**
+     * Returns an observable with the updated details for a manga.
+     *
+     * @param manga the manga to update.
+     */
+    @Deprecated(
+        "Use the 1.x API instead",
+        ReplaceWith("getMangaDetails"),
+    )
+    fun fetchMangaDetails(manga: SManga): Observable<SManga> = throw IllegalStateException("Not used")
+
+    /**
+     * Returns an observable with all the available chapters for a manga.
+     *
+     * @param manga the manga to update.
+     */
+    @Deprecated(
+        "Use the 1.x API instead",
+        ReplaceWith("getChapterList"),
+    )
+    fun fetchChapterList(manga: SManga): Observable<List<SChapter>> = throw IllegalStateException("Not used")
+
+    // TODO: remove direct usages on this method
+    /**
+     * Returns an observable with the list of pages a chapter has.
+     *
+     * @param chapter the chapter.
+     */
+    @Deprecated(
+        "Use the 1.x API instead",
+        ReplaceWith("getPageList"),
+    )
+    fun fetchPageList(chapter: SChapter): Observable<List<Page>> = Observable.empty()
+
+    /**
+     * [1.x API] Get the updated details for a manga.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getMangaDetails(manga: SManga): SManga {
+        return fetchMangaDetails(manga).awaitSingle()
+    }
+
+    /**
+     * [1.x API] Get all the available chapters for a manga.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getChapterList(manga: SManga): List<SChapter> {
+        return fetchChapterList(manga).awaitSingle()
+    }
+
+    /**
+     * [1.x API] Get the list of pages a chapter has.
+     */
+    @Suppress("DEPRECATION")
+    suspend fun getPageList(chapter: SChapter): List<Page> {
+        return fetchPageList(chapter).awaitSingle()
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/SourceFactory.kt

@@ -0,0 +1,12 @@
+package eu.kanade.tachiyomi.source
+
+/**
+ * A factory for creating sources at runtime.
+ */
+interface SourceFactory {
+    /**
+     * Create a new copy of the sources
+     * @return The created sources
+     */
+    fun createSources(): List<Source>
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/UnmeteredSource.kt

@@ -0,0 +1,8 @@
+package eu.kanade.tachiyomi.source
+
+/**
+ * A source that explicitly doesn't require traffic considerations.
+ *
+ * This typically applies for self-hosted sources.
+ */
+interface UnmeteredSource

source-api/src/main/java/eu/kanade/tachiyomi/source/model/Filter.kt

@@ -0,0 +1,40 @@
+package eu.kanade.tachiyomi.source.model
+
+sealed class Filter<T>(val name: String, var state: T) {
+    open class Header(name: String) : Filter<Any>(name, 0)
+    open class Separator(name: String = "") : Filter<Any>(name, 0)
+    abstract class Select<V>(name: String, val values: Array<V>, state: Int = 0) : Filter<Int>(name, state)
+    abstract class Text(name: String, state: String = "") : Filter<String>(name, state)
+    abstract class CheckBox(name: String, state: Boolean = false) : Filter<Boolean>(name, state)
+    abstract class TriState(name: String, state: Int = STATE_IGNORE) : Filter<Int>(name, state) {
+        fun isIgnored() = state == STATE_IGNORE
+        fun isIncluded() = state == STATE_INCLUDE
+        fun isExcluded() = state == STATE_EXCLUDE
+
+        companion object {
+            const val STATE_IGNORE = 0
+            const val STATE_INCLUDE = 1
+            const val STATE_EXCLUDE = 2
+        }
+    }
+
+    abstract class Group<V>(name: String, state: List<V>) : Filter<List<V>>(name, state)
+
+    abstract class Sort(name: String, val values: Array<String>, state: Selection? = null) :
+        Filter<Sort.Selection?>(name, state) {
+        data class Selection(val index: Int, val ascending: Boolean)
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is Filter<*>) return false
+
+        return name == other.name && state == other.state
+    }
+
+    override fun hashCode(): Int {
+        var result = name.hashCode()
+        result = 31 * result + (state?.hashCode() ?: 0)
+        return result
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/FilterList.kt

@@ -0,0 +1,14 @@
+package eu.kanade.tachiyomi.source.model
+
+data class FilterList(val list: List<Filter<*>>) : List<Filter<*>> by list {
+
+    constructor(vararg fs: Filter<*>) : this(if (fs.isNotEmpty()) fs.asList() else emptyList())
+
+    override fun equals(other: Any?): Boolean {
+        return false
+    }
+
+    override fun hashCode(): Int {
+        return list.hashCode()
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/MangasPage.kt

@@ -0,0 +1,3 @@
+package eu.kanade.tachiyomi.source.model
+
+data class MangasPage(val mangas: List<SManga>, val hasNextPage: Boolean)

source-api/src/main/java/eu/kanade/tachiyomi/source/model/Page.kt

@@ -0,0 +1,66 @@
+package eu.kanade.tachiyomi.source.model
+
+import android.net.Uri
+import eu.kanade.tachiyomi.network.ProgressListener
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
+import rx.subjects.Subject
+
+@Serializable
+open class Page(
+    val index: Int,
+    val url: String = "",
+    var imageUrl: String? = null,
+    @Transient var uri: Uri? = null, // Deprecated but can't be deleted due to extensions
+) : ProgressListener {
+
+    val number: Int
+        get() = index + 1
+
+    @Transient
+    @Volatile
+    var status: Int = 0
+        set(value) {
+            field = value
+            statusSubject?.onNext(value)
+            statusCallback?.invoke(this)
+        }
+
+    @Transient
+    @Volatile
+    var progress: Int = 0
+        set(value) {
+            field = value
+            statusCallback?.invoke(this)
+        }
+
+    @Transient
+    private var statusSubject: Subject<Int, Int>? = null
+
+    @Transient
+    private var statusCallback: ((Page) -> Unit)? = null
+
+    override fun update(bytesRead: Long, contentLength: Long, done: Boolean) {
+        progress = if (contentLength > 0) {
+            (100 * bytesRead / contentLength).toInt()
+        } else {
+            -1
+        }
+    }
+
+    fun setStatusSubject(subject: Subject<Int, Int>?) {
+        this.statusSubject = subject
+    }
+
+    fun setStatusCallback(f: ((Page) -> Unit)?) {
+        statusCallback = f
+    }
+
+    companion object {
+        const val QUEUE = 0
+        const val LOAD_PAGE = 1
+        const val DOWNLOAD_IMAGE = 2
+        const val READY = 3
+        const val ERROR = 4
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/SChapter.kt

@@ -0,0 +1,30 @@
+package eu.kanade.tachiyomi.source.model
+
+import java.io.Serializable
+
+interface SChapter : Serializable {
+
+    var url: String
+
+    var name: String
+
+    var date_upload: Long
+
+    var chapter_number: Float
+
+    var scanlator: String?
+
+    fun copyFrom(other: SChapter) {
+        name = other.name
+        url = other.url
+        date_upload = other.date_upload
+        chapter_number = other.chapter_number
+        scanlator = other.scanlator
+    }
+
+    companion object {
+        fun create(): SChapter {
+            return SChapterImpl()
+        }
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/SChapterImpl.kt

@@ -0,0 +1,14 @@
+package eu.kanade.tachiyomi.source.model
+
+class SChapterImpl : SChapter {
+
+    override lateinit var url: String
+
+    override lateinit var name: String
+
+    override var date_upload: Long = 0
+
+    override var chapter_number: Float = -1f
+
+    override var scanlator: String? = null
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt

@@ -0,0 +1,83 @@
+package eu.kanade.tachiyomi.source.model
+
+import java.io.Serializable
+
+interface SManga : Serializable {
+
+    var url: String
+
+    var title: String
+
+    var artist: String?
+
+    var author: String?
+
+    var description: String?
+
+    var genre: String?
+
+    var status: Int
+
+    var thumbnail_url: String?
+
+    var initialized: Boolean
+
+    fun getGenres(): List<String>? {
+        if (genre.isNullOrBlank()) return null
+        return genre?.split(", ")?.map { it.trim() }?.filterNot { it.isBlank() }?.distinct()
+    }
+
+    fun copyFrom(other: SManga) {
+        if (other.author != null) {
+            author = other.author
+        }
+
+        if (other.artist != null) {
+            artist = other.artist
+        }
+
+        if (other.description != null) {
+            description = other.description
+        }
+
+        if (other.genre != null) {
+            genre = other.genre
+        }
+
+        if (other.thumbnail_url != null) {
+            thumbnail_url = other.thumbnail_url
+        }
+
+        status = other.status
+
+        if (!initialized) {
+            initialized = other.initialized
+        }
+    }
+
+    fun copy() = create().also {
+        it.url = url
+        it.title = title
+        it.artist = artist
+        it.author = author
+        it.description = description
+        it.genre = genre
+        it.status = status
+        it.thumbnail_url = thumbnail_url
+        it.initialized = initialized
+    }
+
+    companion object {
+        const val UNKNOWN = 0
+        const val ONGOING = 1
+        const val COMPLETED = 2
+        const val LICENSED = 3
+        const val PUBLISHING_FINISHED = 4
+        const val CANCELLED = 5
+        const val ON_HIATUS = 6
+
+        fun create(): SManga {
+            return SMangaImpl()
+        }
+    }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/model/SMangaImpl.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.source.model
+
+class SMangaImpl : SManga {
+
+    override lateinit var url: String
+
+    override lateinit var title: String
+
+    override var artist: String? = null
+
+    override var author: String? = null
+
+    override var description: String? = null
+
+    override var genre: String? = null
+
+    override var status: Int = 0
+
+    override var thumbnail_url: String? = null
+
+    override var initialized: Boolean = false
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -0,0 +1,376 @@
+package eu.kanade.tachiyomi.source.online
+
+import eu.kanade.tachiyomi.network.CACHE_CONTROL_NO_STORE
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.newCallWithProgress
+import eu.kanade.tachiyomi.source.CatalogueSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
+import rx.Observable
+import uy.kohesive.injekt.injectLazy
+import java.net.URI
+import java.net.URISyntaxException
+import java.security.MessageDigest
+
+/**
+ * A simple implementation for sources from a website.
+ */
+abstract class HttpSource : CatalogueSource {
+
+    /**
+     * Network service.
+     */
+    protected val network: NetworkHelper by injectLazy()
+
+    /**
+     * Base url of the website without the trailing slash, like: http://mysite.com
+     */
+    abstract val baseUrl: String
+
+    /**
+     * Version id used to generate the source id. If the site completely changes and urls are
+     * incompatible, you may increase this value and it'll be considered as a new source.
+     */
+    open val versionId = 1
+
+    /**
+     * Id of the source. By default it uses a generated id using the first 16 characters (64 bits)
+     * of the MD5 of the string: sourcename/language/versionId
+     * Note the generated id sets the sign bit to 0.
+     */
+    override val id by lazy {
+        val key = "${name.lowercase()}/$lang/$versionId"
+        val bytes = MessageDigest.getInstance("MD5").digest(key.toByteArray())
+        (0..7).map { bytes[it].toLong() and 0xff shl 8 * (7 - it) }.reduce(Long::or) and Long.MAX_VALUE
+    }
+
+    /**
+     * Headers used for requests.
+     */
+    val headers: Headers by lazy { headersBuilder().build() }
+
+    /**
+     * Default network client for doing requests.
+     */
+    open val client: OkHttpClient
+        get() = network.client
+
+    /**
+     * Headers builder for requests. Implementations can override this method for custom headers.
+     */
+    protected open fun headersBuilder() = Headers.Builder().apply {
+        add("User-Agent", network.defaultUserAgent)
+    }
+
+    /**
+     * Visible name of the source.
+     */
+    override fun toString() = "$name (${lang.uppercase()})"
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page number to retrieve.
+     */
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+        return client.newCall(popularMangaRequest(page))
+            .asObservableSuccess()
+            .map { response ->
+                popularMangaParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for the popular manga given the page.
+     *
+     * @param page the page number to retrieve.
+     */
+    protected abstract fun popularMangaRequest(page: Int): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun popularMangaParse(response: Response): MangasPage
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        return Observable.defer {
+            try {
+                client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess()
+            } catch (e: NoClassDefFoundError) {
+                // RxJava doesn't handle Errors, which tends to happen during global searches
+                // if an old extension using non-existent classes is still around
+                throw RuntimeException(e)
+            }
+        }
+            .map { response ->
+                searchMangaParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for the search manga given the page.
+     *
+     * @param page the page number to retrieve.
+     * @param query the search query.
+     * @param filters the list of filters to apply.
+     */
+    protected abstract fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun searchMangaParse(response: Response): MangasPage
+
+    /**
+     * Returns an observable containing a page with a list of latest manga updates.
+     *
+     * @param page the page number to retrieve.
+     */
+    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
+        return client.newCall(latestUpdatesRequest(page))
+            .asObservableSuccess()
+            .map { response ->
+                latestUpdatesParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for latest manga given the page.
+     *
+     * @param page the page number to retrieve.
+     */
+    protected abstract fun latestUpdatesRequest(page: Int): Request
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun latestUpdatesParse(response: Response): MangasPage
+
+    /**
+     * Returns an observable with the updated details for a manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param manga the manga to be updated.
+     */
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return client.newCall(mangaDetailsRequest(manga))
+            .asObservableSuccess()
+            .map { response ->
+                mangaDetailsParse(response).apply { initialized = true }
+            }
+    }
+
+    /**
+     * Returns the request for the details of a manga. Override only if it's needed to change the
+     * url, send different headers or request method like POST.
+     *
+     * @param manga the manga to be updated.
+     */
+    open fun mangaDetailsRequest(manga: SManga): Request {
+        return GET(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns the details of a manga.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun mangaDetailsParse(response: Response): SManga
+
+    /**
+     * Returns an observable with the updated chapter list for a manga. Normally it's not needed to
+     * override this method.  If a manga is licensed an empty chapter list observable is returned
+     *
+     * @param manga the manga to look for chapters.
+     */
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        return if (manga.status != SManga.LICENSED) {
+            client.newCall(chapterListRequest(manga))
+                .asObservableSuccess()
+                .map { response ->
+                    chapterListParse(response)
+                }
+        } else {
+            Observable.error(Exception("Licensed - No chapters to show"))
+        }
+    }
+
+    /**
+     * Returns the request for updating the chapter list. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param manga the manga to look for chapters.
+     */
+    protected open fun chapterListRequest(manga: SManga): Request {
+        return GET(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns a list of chapters.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun chapterListParse(response: Response): List<SChapter>
+
+    /**
+     * Returns an observable with the page list for a chapter.
+     *
+     * @param chapter the chapter whose page list has to be fetched.
+     */
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        return client.newCall(pageListRequest(chapter))
+            .asObservableSuccess()
+            .map { response ->
+                pageListParse(response)
+            }
+    }
+
+    /**
+     * Returns the request for getting the page list. Override only if it's needed to override the
+     * url, send different headers or request method like POST.
+     *
+     * @param chapter the chapter whose page list has to be fetched.
+     */
+    protected open fun pageListRequest(chapter: SChapter): Request {
+        return GET(baseUrl + chapter.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns a list of pages.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun pageListParse(response: Response): List<Page>
+
+    /**
+     * Returns an observable with the page containing the source url of the image. If there's any
+     * error, it will return null instead of throwing an exception.
+     *
+     * @param page the page whose source image has to be fetched.
+     */
+    open fun fetchImageUrl(page: Page): Observable<String> {
+        return client.newCall(imageUrlRequest(page))
+            .asObservableSuccess()
+            .map { imageUrlParse(it) }
+    }
+
+    /**
+     * Returns the request for getting the url to the source image. Override only if it's needed to
+     * override the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    protected open fun imageUrlRequest(page: Page): Request {
+        return GET(page.url, headers)
+    }
+
+    /**
+     * Parses the response from the site and returns the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    protected abstract fun imageUrlParse(response: Response): String
+
+    /**
+     * Returns an observable with the response of the source image.
+     *
+     * @param page the page whose source image has to be downloaded.
+     */
+    fun fetchImage(page: Page): Observable<Response> {
+        val request = imageRequest(page).newBuilder()
+            // images will be cached or saved manually, so don't take up network cache
+            .cacheControl(CACHE_CONTROL_NO_STORE)
+            .build()
+        return client.newCallWithProgress(request, page)
+            .asObservableSuccess()
+    }
+
+    /**
+     * Returns the request for getting the source image. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    protected open fun imageRequest(page: Page): Request {
+        return GET(page.imageUrl!!, headers)
+    }
+
+    /**
+     * Assigns the url of the chapter without the scheme and domain. It saves some redundancy from
+     * database and the urls could still work after a domain change.
+     *
+     * @param url the full url to the chapter.
+     */
+    fun SChapter.setUrlWithoutDomain(url: String) {
+        this.url = getUrlWithoutDomain(url)
+    }
+
+    /**
+     * Assigns the url of the manga without the scheme and domain. It saves some redundancy from
+     * database and the urls could still work after a domain change.
+     *
+     * @param url the full url to the manga.
+     */
+    fun SManga.setUrlWithoutDomain(url: String) {
+        this.url = getUrlWithoutDomain(url)
+    }
+
+    /**
+     * Returns the url of the given string without the scheme and domain.
+     *
+     * @param orig the full url.
+     */
+    private fun getUrlWithoutDomain(orig: String): String {
+        return try {
+            val uri = URI(orig.replace(" ", "%20"))
+            var out = uri.path
+            if (uri.query != null) {
+                out += "?" + uri.query
+            }
+            if (uri.fragment != null) {
+                out += "#" + uri.fragment
+            }
+            out
+        } catch (e: URISyntaxException) {
+            orig
+        }
+    }
+
+    /**
+     * Called before inserting a new chapter into database. Use it if you need to override chapter
+     * fields, like the title or the chapter number. Do not change anything to [manga].
+     *
+     * @param chapter the chapter to be added.
+     * @param manga the manga of the chapter.
+     */
+    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {}
+
+    /**
+     * Returns the list of filters for the source.
+     */
+    override fun getFilterList() = FilterList()
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/online/HttpSourceFetcher.kt

@@ -0,0 +1,25 @@
+package eu.kanade.tachiyomi.source.online
+
+import eu.kanade.tachiyomi.source.model.Page
+import rx.Observable
+
+fun HttpSource.getImageUrl(page: Page): Observable<Page> {
+    page.status = Page.LOAD_PAGE
+    return fetchImageUrl(page)
+        .doOnError { page.status = Page.ERROR }
+        .onErrorReturn { null }
+        .doOnNext { page.imageUrl = it }
+        .map { page }
+}
+
+fun HttpSource.fetchAllImageUrlsFromPageList(pages: List<Page>): Observable<Page> {
+    return Observable.from(pages)
+        .filter { !it.imageUrl.isNullOrEmpty() }
+        .mergeWith(fetchRemainingImageUrlsFromPageList(pages))
+}
+
+fun HttpSource.fetchRemainingImageUrlsFromPageList(pages: List<Page>): Observable<Page> {
+    return Observable.from(pages)
+        .filter { it.imageUrl.isNullOrEmpty() }
+        .concatMap { getImageUrl(it) }
+}

source-api/src/main/java/eu/kanade/tachiyomi/source/online/ParsedHttpSource.kt

@@ -0,0 +1,201 @@
+package eu.kanade.tachiyomi.source.online
+
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+/**
+ * A simple implementation for sources from a website using Jsoup, an HTML parser.
+ */
+@Suppress("unused")
+abstract class ParsedHttpSource : HttpSource() {
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun popularMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(popularMangaSelector()).map { element ->
+            popularMangaFromElement(element)
+        }
+
+        val hasNextPage = popularMangaNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun popularMangaSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [popularMangaSelector].
+     */
+    protected abstract fun popularMangaFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun popularMangaNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun searchMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(searchMangaSelector()).map { element ->
+            searchMangaFromElement(element)
+        }
+
+        val hasNextPage = searchMangaNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun searchMangaSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [searchMangaSelector].
+     */
+    protected abstract fun searchMangaFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun searchMangaNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns a [MangasPage] object.
+     *
+     * @param response the response from the site.
+     */
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.select(latestUpdatesSelector()).map { element ->
+            latestUpdatesFromElement(element)
+        }
+
+        val hasNextPage = latestUpdatesNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    protected abstract fun latestUpdatesSelector(): String
+
+    /**
+     * Returns a manga from the given [element]. Most sites only show the title and the url, it's
+     * totally fine to fill only those two values.
+     *
+     * @param element an element obtained from [latestUpdatesSelector].
+     */
+    protected abstract fun latestUpdatesFromElement(element: Element): SManga
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    protected abstract fun latestUpdatesNextPageSelector(): String?
+
+    /**
+     * Parses the response from the site and returns the details of a manga.
+     *
+     * @param response the response from the site.
+     */
+    override fun mangaDetailsParse(response: Response): SManga {
+        return mangaDetailsParse(response.asJsoup())
+    }
+
+    /**
+     * Returns the details of the manga from the given [document].
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun mangaDetailsParse(document: Document): SManga
+
+    /**
+     * Parses the response from the site and returns a list of chapters.
+     *
+     * @param response the response from the site.
+     */
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val document = response.asJsoup()
+        return document.select(chapterListSelector()).map { chapterFromElement(it) }
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each chapter.
+     */
+    protected abstract fun chapterListSelector(): String
+
+    /**
+     * Returns a chapter from the given element.
+     *
+     * @param element an element obtained from [chapterListSelector].
+     */
+    protected abstract fun chapterFromElement(element: Element): SChapter
+
+    /**
+     * Parses the response from the site and returns the page list.
+     *
+     * @param response the response from the site.
+     */
+    override fun pageListParse(response: Response): List<Page> {
+        return pageListParse(response.asJsoup())
+    }
+
+    /**
+     * Returns a page list from the given document.
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun pageListParse(document: Document): List<Page>
+
+    /**
+     * Parse the response from the site and returns the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    override fun imageUrlParse(response: Response): String {
+        return imageUrlParse(response.asJsoup())
+    }
+
+    /**
+     * Returns the absolute url to the source image from the document.
+     *
+     * @param document the parsed document.
+     */
+    protected abstract fun imageUrlParse(document: Document): String
+}

source-api/src/main/java/eu/kanade/tachiyomi/util/JsoupExtensions.kt

@@ -0,0 +1,26 @@
+package eu.kanade.tachiyomi.util
+
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+fun Element.selectText(css: String, defaultValue: String? = null): String? {
+    return select(css).first()?.text() ?: defaultValue
+}
+
+fun Element.selectInt(css: String, defaultValue: Int = 0): Int {
+    return select(css).first()?.text()?.toInt() ?: defaultValue
+}
+
+fun Element.attrOrText(css: String): String {
+    return if (css != "text") attr(css) else text()
+}
+
+/**
+ * Returns a Jsoup document for this response.
+ * @param html the body of the response. Use only if the body was read before calling this method.
+ */
+fun Response.asJsoup(html: String? = null): Document {
+    return Jsoup.parse(html ?: body.string(), request.url.toString())
+}