diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/base/MaterialFastScroll.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/base/MaterialFastScroll.kt
index fd01ea4881..3c59bc37fb 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/base/MaterialFastScroll.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/base/MaterialFastScroll.kt
@@ -9,6 +9,7 @@ import androidx.recyclerview.widget.RecyclerView
 import androidx.recyclerview.widget.StaggeredGridLayoutManager
 import eu.davidea.fastscroller.FastScroller
 import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.ui.base.controller.BaseController
 import eu.kanade.tachiyomi.util.system.dpToPx
 import eu.kanade.tachiyomi.util.system.dpToPxEnd
 import eu.kanade.tachiyomi.util.system.isLTR
@@ -20,6 +21,7 @@ class MaterialFastScroll @JvmOverloads constructor(context: Context, attrs: Attr
     var canScroll = false
     var startY = -1f
     var scrollOffset = 0
+    var controller: BaseController<*>? = null
     init {
         setViewsToUse(
             R.layout.material_fastscroll,
@@ -36,8 +38,10 @@ class MaterialFastScroll @JvmOverloads constructor(context: Context, attrs: Attr
 
     // Overriding to force a distance moved before scrolling
     override fun onTouchEvent(event: MotionEvent): Boolean {
-        if (recyclerView.computeVerticalScrollRange() <= recyclerView.computeVerticalScrollExtent()) {
-            return if (startY > -1f) {
+        if (controller?.isDragging == true ||
+            recyclerView.computeVerticalScrollRange() <= recyclerView.computeVerticalScrollExtent()
+        ) {
+            return if (startY > -1f || controller?.isDragging == true) {
                 dispatchTouchToRecycler(event)
                 false
             } else {
diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/download/DownloadBottomSheet.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/download/DownloadBottomSheet.kt
index a0f59d7bcf..7b6f053947 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/download/DownloadBottomSheet.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/download/DownloadBottomSheet.kt
@@ -8,7 +8,10 @@ import android.widget.LinearLayout
 import androidx.core.view.WindowInsetsCompat.Type.systemBars
 import androidx.core.view.isInvisible
 import androidx.core.view.updateLayoutParams
+import androidx.recyclerview.widget.ItemTouchHelper
+import androidx.recyclerview.widget.RecyclerView
 import com.google.android.material.bottomsheet.BottomSheetBehavior
+import eu.davidea.flexibleadapter.FlexibleAdapter
 import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.download.DownloadService
 import eu.kanade.tachiyomi.data.download.model.Download
@@ -32,8 +35,9 @@ class DownloadBottomSheet @JvmOverloads constructor(
     attrs: AttributeSet? =
         null
 ) : LinearLayout(context, attrs),
-    DownloadAdapter.DownloadItemListener {
-    lateinit var controller: RecentsController
+    DownloadAdapter.DownloadItemListener,
+    FlexibleAdapter.OnActionStateListener {
+    var controller: RecentsController? = null
     var sheetBehavior: BottomSheetBehavior<*>? = null
 
     /**
@@ -69,7 +73,9 @@ class DownloadBottomSheet @JvmOverloads constructor(
         binding.dlRecycler.adapter = adapter
         adapter?.isHandleDragEnabled = true
         adapter?.isSwipeEnabled = true
+        adapter?.isSwipeEnabled = true
         adapter?.fastScroller = binding.fastScroller
+        binding.fastScroller.controller = controller
         binding.dlRecycler.setHasFixedSize(true)
         binding.dlRecycler.addItemDecoration(ExtensionDividerItemDecoration(context))
         this.controller = controller
@@ -156,6 +162,12 @@ class DownloadBottomSheet @JvmOverloads constructor(
         setBottomSheet()
     }
 
+    override fun onActionStateChanged(viewHolder: RecyclerView.ViewHolder?, actionState: Int) {
+        if (actionState == ItemTouchHelper.ACTION_STATE_DRAG) {
+            controller?.isDragging = true
+        }
+    }
+
     /**
      * Called when the progress of a download changes.
      *
@@ -265,7 +277,7 @@ class DownloadBottomSheet @JvmOverloads constructor(
             sheetBehavior?.skipCollapsed = !hasQueue
             if (sheetBehavior.isCollapsed()) sheetBehavior?.hide()
         }
-        controller.setPadding(!hasQueue)
+        controller?.setPadding(!hasQueue)
     }
 
     /**
@@ -274,6 +286,7 @@ class DownloadBottomSheet @JvmOverloads constructor(
      * @param position The position of the released item.
      */
     override fun onItemReleased(position: Int) {
+        controller?.isDragging = false
         val adapter = adapter ?: return
         val downloads = adapter.headerItems.flatMap { header ->
             adapter.getSectionItems(header).map { item ->
@@ -301,7 +314,7 @@ class DownloadBottomSheet @JvmOverloads constructor(
             }
         }
         presenter.reorder(downloads)
-        controller.updateChapterDownload(download, false)
+        controller?.updateChapterDownload(download, false)
     }
 
     /**
@@ -354,5 +367,6 @@ class DownloadBottomSheet @JvmOverloads constructor(
 
     fun onDestroy() {
         presenter.onDestroy()
+        binding.fastScroller.controller = null
     }
 }
diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
index aa5dbcd023..76d1e7aec5 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
@@ -533,6 +533,7 @@ class LibraryController(
         binding.libraryGridRecycler.recycler.adapter = adapter
 
         adapter.fastScroller = binding.fastScroller
+        binding.fastScroller.controller = this
         binding.libraryGridRecycler.recycler.addOnScrollListener(scrollListener)
 
         binding.swipeRefresh.setStyle()
@@ -997,6 +998,7 @@ class LibraryController(
         destroyActionModeIfNeeded()
         if (isBindingInitialized) {
             binding.libraryGridRecycler.recycler.removeOnScrollListener(scrollListener)
+            binding.fastScroller.controller = null
         }
         displaySheet?.dismiss()
         displaySheet = null