Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] POC graphql filtering nested node lists #672

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[WIP] POC graphql filtering nested node lists #672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 213 additions & 2 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/ChapterDataLoader.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,32 @@
package suwayomi.tachidesk.graphql.dataLoaders

import com.expediagroup.graphql.dataloader.KotlinDataLoader
import mu.KotlinLogging
import org.dataloader.CacheKey
import org.dataloader.DataLoader
import org.dataloader.DataLoaderFactory
import org.dataloader.DataLoaderOptions
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.Slf4jSqlDebugLogger
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SortOrder.ASC
import org.jetbrains.exposed.sql.SortOrder.ASC_NULLS_FIRST
import org.jetbrains.exposed.sql.SortOrder.ASC_NULLS_LAST
import org.jetbrains.exposed.sql.SortOrder.DESC
import org.jetbrains.exposed.sql.SortOrder.DESC_NULLS_FIRST
import org.jetbrains.exposed.sql.SortOrder.DESC_NULLS_LAST
import org.jetbrains.exposed.sql.addLogger
import org.jetbrains.exposed.sql.orWhere
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import suwayomi.tachidesk.graphql.queries.ChapterQuery.BaseChapterCondition
import suwayomi.tachidesk.graphql.queries.ChapterQuery.ChapterFilter
import suwayomi.tachidesk.graphql.queries.ChapterQuery.ChapterOrderBy
import suwayomi.tachidesk.graphql.queries.filter.applyOps
import suwayomi.tachidesk.graphql.server.primitives.Cursor
import suwayomi.tachidesk.graphql.server.primitives.PageInfo
import suwayomi.tachidesk.graphql.server.primitives.QueryResults
import suwayomi.tachidesk.graphql.server.primitives.maybeSwap
import suwayomi.tachidesk.graphql.types.ChapterNodeList
import suwayomi.tachidesk.graphql.types.ChapterNodeList.Companion.toNodeList
import suwayomi.tachidesk.graphql.types.ChapterType
Expand All @@ -35,8 +55,8 @@ class ChapterDataLoader : KotlinDataLoader<Int, ChapterType?> {
}
}

class ChaptersForMangaDataLoader : KotlinDataLoader<Int, ChapterNodeList> {
override val dataLoaderName = "ChaptersForMangaDataLoader"
class ChaptersForMangaDataLoadesr : KotlinDataLoader<Int, ChapterNodeList> {
override val dataLoaderName = "ChaptersForMangaDataLoadesr"
override fun getDataLoader(): DataLoader<Int, ChapterNodeList> = DataLoaderFactory.newDataLoader<Int, ChapterNodeList> { ids ->
future {
transaction {
Expand All @@ -49,3 +69,194 @@ class ChaptersForMangaDataLoader : KotlinDataLoader<Int, ChapterNodeList> {
}
}
}

data class ChaptersContext<Condition : BaseChapterCondition>(
val condition: Condition? = null,
val filter: ChapterFilter? = null,
val orderBy: ChapterOrderBy? = null,
val orderByType: SortOrder? = null,
val before: Cursor? = null,
val after: Cursor? = null,
val first: Int? = null,
val last: Int? = null,
val offset: Int? = null
)

/**
* This data loader requires a context to be passed, if it is missing a NullPointerException will be thrown
*/
class ChaptersForMangaDataLoader : KotlinDataLoader<Int, ChapterNodeList> {
override val dataLoaderName = "ChaptersForMangaDataLoader"
override fun getDataLoader(): DataLoader<Int, ChapterNodeList> = DataLoaderFactory.newDataLoader<Int, ChapterNodeList> (
{ ids, env ->
future {
transaction {
addLogger(Slf4jSqlDebugLogger)

KotlinLogging.logger { }.info { "@Daniel chapters loader <-> ids $ids" }
KotlinLogging.logger { }.info { "@Daniel chapters loader <-> context ${env.getContext<Any>()}" }
KotlinLogging.logger { }.info { "@Daniel chapters loader <-> keyContexts ${env.keyContexts}" }
KotlinLogging.logger { }.info { "@Daniel chapters loader <-> keyContextsList ${env.keyContextsList}" }

// ids [1, 1, 2, 3, 4, 4, 1]
// idSets => [[(1, ctx), (2, ctx), (3, ctx), (4, ctx)], [(1, ctx), (4, ctx)], [(1, ctx)]]
//
// (id) 1 1 2 3 4 4 1
// id to list to list index [(list index, index in list)] => [(0, 0), (1, 0), (0, 1), (0, 2), (0, 3), (1, 2), (2, 0)]

// create sets of ids, which can be queried together - since the same id can have different contexts, these can not be queried together
val listOfIdCtxSets = mutableListOf<MutableList<Pair<Int, ChaptersContext<*>>>>()
val idToListToListIndexMap = mutableListOf<Pair<Int, Int>>()
ids.forEachIndexed { idIndex, id ->
var inserted = false
val idCtxPair = Pair(id, env.keyContextsList[idIndex] as ChaptersContext<*>)

for (idSet in listOfIdCtxSets) {
val idSetIndex = listOfIdCtxSets.indexOf(idSet)

val isDuplicate = idSet.any { (idInIdSet) -> idInIdSet == id }
if (!isDuplicate) {
idToListToListIndexMap.add(Pair(idSetIndex, idSet.size))

idSet.add(idCtxPair)
inserted = true
break
}
}

if (!inserted) {
idToListToListIndexMap.add(Pair(listOfIdCtxSets.size, 0))
listOfIdCtxSets.add(mutableListOf(idCtxPair))
}
}

KotlinLogging.logger { }.info { "@Daniel listOfIdSets $listOfIdCtxSets" }
KotlinLogging.logger { }.info { "@Daniel idToListToListIndexMap $idToListToListIndexMap" }

val result = listOfIdCtxSets.map { idCtxSet ->
val idSetIds = idCtxSet.map { it.first }

val query = ChapterTable.select { ChapterTable.manga inList idSetIds }
// filter chapters for each manga
idCtxSet.forEach { (id, ctx) ->
val (condition, filter) = ctx
query.orWhere { ChapterTable.manga eq id }.applyOps(condition, filter)
}

val mangaToChapterRowsMap = query
.groupBy { it[ChapterTable.manga].value }

idSetIds.map { mangaId ->
val chapterRows = mangaToChapterRowsMap[mangaId] ?: emptyList()
val (_, _, orderBy, orderByType, before, after, first, last, offset) = idCtxSet.find { it.first == mangaId }!!.second

val sortedChapterRows = if (orderBy != null || (last != null || before != null)) {
val orderByColumn = orderBy?.column ?: ChapterTable.id
val orderType = orderByType.maybeSwap(last ?: before)

if (orderBy == ChapterOrderBy.ID || orderBy == null) {
chapterRows.sortedBy { it[orderByColumn] as Comparable<Any> }
} else {
when (orderType) {
ASC -> chapterRows.sortedWith(compareBy({ it[orderByColumn] }, { it[ChapterTable.id] }))
DESC -> chapterRows.sortedWith(compareBy<ResultRow> { it[orderByColumn] }.reversed().thenBy { it[ChapterTable.id] })
ASC_NULLS_FIRST -> chapterRows.sortedWith(compareBy<ResultRow, Comparable<Any>>(nullsFirst()) { it[orderByColumn] as Comparable<Any> }.thenBy { it[ChapterTable.id] })
DESC_NULLS_FIRST -> chapterRows.sortedWith(compareBy<ResultRow, Comparable<Any>>(nullsFirst()) { it[orderByColumn] as Comparable<Any> }.reversed().thenBy { it[ChapterTable.id] })
ASC_NULLS_LAST -> chapterRows.sortedWith(compareBy<ResultRow, Comparable<Any>>(nullsLast()) { it[orderByColumn] as Comparable<Any> }.thenBy { it[ChapterTable.id] })
DESC_NULLS_LAST -> chapterRows.sortedWith(compareBy<ResultRow, Comparable<Any>>(nullsLast()) { it[orderByColumn] as Comparable<Any> }.reversed().thenBy { it[ChapterTable.id] })
}
}
} else {
chapterRows
}

val total = sortedChapterRows.size
val firstResult = sortedChapterRows.firstOrNull()?.get(ChapterTable.id)?.value
val lastResult = sortedChapterRows.lastOrNull()?.get(ChapterTable.id)?.value

var paginatedChapterRows = if (after != null) {
val afterIndex = sortedChapterRows.indexOfFirst { chapter -> chapter[ChapterTable.id].value == after.value.toInt() }

if (sortedChapterRows.size - 1 < afterIndex) {
emptyList()
} else {
sortedChapterRows.subList(afterIndex, -1)
}
} else if (before != null) {
val beforeIndex = sortedChapterRows.indexOfFirst { chapter -> chapter[ChapterTable.id].value == before.value.toInt() }
sortedChapterRows.subList(0, (sortedChapterRows.size - 1).coerceAtMost(beforeIndex))
} else {
sortedChapterRows
}

paginatedChapterRows = if (first != null) {
if (paginatedChapterRows.isEmpty()) {
emptyList()
} else {
paginatedChapterRows.subList(
(paginatedChapterRows.size - 1).coerceAtMost(offset ?: 0),
(paginatedChapterRows.size - 1).coerceAtMost(first)
)
}
} else if (last != null) {
paginatedChapterRows.takeLast(last)
} else {
paginatedChapterRows
}

val queryResults = QueryResults(total.toLong(), firstResult, lastResult, paginatedChapterRows)

val getAsCursor: (ChapterType) -> Cursor = (orderBy ?: ChapterOrderBy.ID)::asCursor

val resultsAsType = queryResults.results.map { ChapterType(it) }

ChapterNodeList(
resultsAsType,
if (resultsAsType.isEmpty()) {
emptyList()
} else {
listOfNotNull(
resultsAsType.firstOrNull()?.let {
ChapterNodeList.ChapterEdge(
getAsCursor(it),
it
)
},
resultsAsType.lastOrNull()?.let {
ChapterNodeList.ChapterEdge(
getAsCursor(it),
it
)
}
)
},
pageInfo = PageInfo(
hasNextPage = queryResults.lastKey != resultsAsType.lastOrNull()?.id,
hasPreviousPage = queryResults.firstKey != resultsAsType.firstOrNull()?.id,
startCursor = resultsAsType.firstOrNull()?.let { getAsCursor(it) },
endCursor = resultsAsType.lastOrNull()?.let { getAsCursor(it) }
),
totalCount = queryResults.total.toInt()
)
}
}

idToListToListIndexMap.map { (idSetIndex, idSetIdIndex) ->
result[idSetIndex][idSetIdIndex]
}
}
}
},
DataLoaderOptions.newOptions().setCacheKeyFunction(
object : CacheKey<Int> {
override fun getKey(input: Int): String {
return input.toString()
}

override fun getKeyWithContext(input: Int, context: Any): String {
return "${input}_$context"
}
}
)
)
}
87 changes: 66 additions & 21 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/ChapterQuery.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import graphql.schema.DataFetchingEnvironment
import org.jetbrains.exposed.sql.Column
import org.jetbrains.exposed.sql.Op
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.greater
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
import org.jetbrains.exposed.sql.andWhere
Expand Down Expand Up @@ -99,33 +100,30 @@ class ChapterQuery {
}
}

data class ChapterCondition(
val id: Int? = null,
val url: String? = null,
val name: String? = null,
val uploadDate: Long? = null,
val chapterNumber: Float? = null,
val scanlator: String? = null,
val mangaId: Int? = null,
val isRead: Boolean? = null,
val isBookmarked: Boolean? = null,
val lastPageRead: Int? = null,
val lastReadAt: Long? = null,
val sourceOrder: Int? = null,
val realUrl: String? = null,
val fetchedAt: Long? = null,
val isDownloaded: Boolean? = null,
val pageCount: Int? = null
) : HasGetOp {
override fun getOp(): Op<Boolean>? {
abstract class BaseChapterCondition : HasGetOp {
abstract val id: Int?
abstract val url: String?
abstract val name: String?
abstract val uploadDate: Long?
abstract val chapterNumber: Float?
abstract val scanlator: String?
abstract val isRead: Boolean?
abstract val isBookmarked: Boolean?
abstract val lastPageRead: Int?
abstract val lastReadAt: Long?
abstract val sourceOrder: Int?
abstract val realUrl: String?
abstract val fetchedAt: Long?
abstract val isDownloaded: Boolean?
abstract val pageCount: Int?
open fun buildOp(): OpAnd {
val opAnd = OpAnd()
opAnd.eq(id, ChapterTable.id)
opAnd.eq(url, ChapterTable.url)
opAnd.eq(name, ChapterTable.name)
opAnd.eq(uploadDate, ChapterTable.date_upload)
opAnd.eq(chapterNumber, ChapterTable.chapter_number)
opAnd.eq(scanlator, ChapterTable.scanlator)
opAnd.eq(mangaId, ChapterTable.manga)
opAnd.eq(isRead, ChapterTable.isRead)
opAnd.eq(isBookmarked, ChapterTable.isBookmarked)
opAnd.eq(lastPageRead, ChapterTable.lastPageRead)
Expand All @@ -136,7 +134,54 @@ class ChapterQuery {
opAnd.eq(isDownloaded, ChapterTable.isDownloaded)
opAnd.eq(pageCount, ChapterTable.pageCount)

return opAnd.op
return opAnd
}

override fun getOp(): Op<Boolean>? {
return buildOp().op
}
}

data class MangaChapterCondition(
override val id: Int? = null,
override val url: String? = null,
override val name: String? = null,
override val uploadDate: Long? = null,
override val chapterNumber: Float? = null,
override val scanlator: String? = null,
override val isRead: Boolean? = null,
override val isBookmarked: Boolean? = null,
override val lastPageRead: Int? = null,
override val lastReadAt: Long? = null,
override val sourceOrder: Int? = null,
override val realUrl: String? = null,
override val fetchedAt: Long? = null,
override val isDownloaded: Boolean? = null,
override val pageCount: Int? = null
) : BaseChapterCondition(), HasGetOp

data class ChapterCondition(
override val id: Int? = null,
override val url: String? = null,
override val name: String? = null,
override val uploadDate: Long? = null,
override val chapterNumber: Float? = null,
override val scanlator: String? = null,
override val isRead: Boolean? = null,
override val isBookmarked: Boolean? = null,
override val lastPageRead: Int? = null,
override val lastReadAt: Long? = null,
override val sourceOrder: Int? = null,
override val realUrl: String? = null,
override val fetchedAt: Long? = null,
override val isDownloaded: Boolean? = null,
override val pageCount: Int? = null,
val mangaId: Int? = null
) : BaseChapterCondition(), HasGetOp {
override fun buildOp(): OpAnd {
val opAnd = super.buildOp()
opAnd.eq(mangaId, ChapterTable.manga)
return opAnd
}
}

Expand Down
25 changes: 23 additions & 2 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@ package suwayomi.tachidesk.graphql.types

import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
import graphql.schema.DataFetchingEnvironment
import mu.KotlinLogging
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import suwayomi.tachidesk.graphql.dataLoaders.ChaptersContext
import suwayomi.tachidesk.graphql.queries.ChapterQuery.ChapterFilter
import suwayomi.tachidesk.graphql.queries.ChapterQuery.ChapterOrderBy
import suwayomi.tachidesk.graphql.queries.ChapterQuery.MangaChapterCondition
import suwayomi.tachidesk.graphql.server.primitives.Cursor
import suwayomi.tachidesk.graphql.server.primitives.Edge
import suwayomi.tachidesk.graphql.server.primitives.Node
Expand Down Expand Up @@ -79,8 +85,23 @@ class MangaType(
dataClass.chaptersLastFetchedAt
)

fun chapters(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<ChapterNodeList> {
return dataFetchingEnvironment.getValueFromDataLoader<Int, ChapterNodeList>("ChaptersForMangaDataLoader", id)
fun chapters(
dataFetchingEnvironment: DataFetchingEnvironment,
condition: MangaChapterCondition? = null,
filter: ChapterFilter? = null,
orderBy: ChapterOrderBy? = null,
orderByType: SortOrder? = null,
before: Cursor? = null,
after: Cursor? = null,
first: Int? = null,
last: Int? = null,
offset: Int? = null
): CompletableFuture<ChapterNodeList> {
val context = ChaptersContext(condition, filter, orderBy, orderByType, before, after, first, last, offset)
KotlinLogging.logger { }.info { "@Daniel $context" }
val dataLoader = dataFetchingEnvironment.getDataLoader<Int, ChapterNodeList>("ChaptersForMangaDataLoader")
return dataLoader.load(id, context)
// return dataFetchingEnvironment.getValueFromDataLoader<Int, ChapterNodeList>("ChaptersForMangaDataLoader", id)
}

fun age(): Long? {
Expand Down