TCGdex/cards-database
1
0
Fork 0
mirror of https://github.com/tcgdex/cards-database.git synced 2025-04-22 19:02:10 +00:00
Code Issues Releases Wiki Activity

feat: Add advanced filtering capabilities (#522)

Browse Source
This commit is contained in:
Florian Bouillon 2024-09-26 00:11:48 +02:00 committed by GitHub
parent 5899083e5d
commit 4700618047
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 887 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.bruno/cards/advanced-query.bru Normal file
View File

@ -0,0 +1,25 @@
meta {
name: Advanced Query
type: http
seq: 1
}
get {
url: {{BASE_URL}}/v2/en/cards?name=eq:Pikachu&hp=gte:60&hp=lt:70&localId=5&localId=not:tg&id=neq:cel25-5
body: none
auth: none
}
params:query {
name: eq:Pikachu
hp: gte:60
hp: lt:70
localId: 5
localId: not:tg
id: neq:cel25-5
}
assert {
res.status: eq 200
res.body: length 14
}

2
.bruno/environments/Developpement.bru
View File

@ -1,3 +1,3 @@
vars {
BASE_URL: http://localhost:3000
BASE_URL: http://127.0.0.1:3000
}

21
.bruno/sets/Advanced Query.bru Normal file
View File

@ -0,0 +1,21 @@
meta {
name: Advanced Query
type: http
seq: 4
}
get {
url: {{BASE_URL}}/v2/en/sets?cardCount.official=gt:64&id=swsh
body: none
auth: none
}
params:query {
cardCount.official:gt: 64
id: swsh
}
assert {
res.status: eq 200
res.body: length 17
}

9
.github/workflows/test.yml vendored
View File

@ -21,6 +21,7 @@ jobs:
- name: Install deps
run: |
bun install -g @usebruno/cli
bun install --frozen-lockfile
cd server
bun install --frozen-lockfile
@ -31,3 +32,11 @@ jobs:
bun run validate
cd server
bun run validate
- name: Validate some requests
run: |
cd server
bun run start &
sleep 10
cd ../.bruno
bru run --env Developpement

BIN
server/bun.lockb
View File

Binary file not shown.

70
server/src/V2/Components/Card.ts
View File

@ -1,48 +1,47 @@
import { objectLoop } from '@dzeio/object-util'
import { CardResume, Card as SDKCard, SupportedLanguages } from '@tcgdex/sdk'
import { Query } from '../../interfaces'
import { handlePagination, handleSort, handleValidation } from '../../util'
import Set from './Set'
import type { CardResume, Card as SDKCard, SupportedLanguages } from '@tcgdex/sdk'
import { executeQuery, type Query } from '../../libs/QueryEngine/filter'
import TCGSet from './Set'
import en from '../../../generated/en/cards.json'
import fr from '../../../generated/fr/cards.json'
import es from '../../../generated/es/cards.json'
import it from '../../../generated/it/cards.json'
import pt from '../../../generated/pt/cards.json'
import ptbr from '../../../generated/pt-br/cards.json'
import ptpt from '../../../generated/pt-pt/cards.json'
import de from '../../../generated/de/cards.json'
import nl from '../../../generated/nl/cards.json'
import pl from '../../../generated/pl/cards.json'
import ru from '../../../generated/ru/cards.json'
import en from '../../../generated/en/cards.json'
import es from '../../../generated/es/cards.json'
import fr from '../../../generated/fr/cards.json'
import id from '../../../generated/id/cards.json'
import it from '../../../generated/it/cards.json'
import ja from '../../../generated/ja/cards.json'
import ko from '../../../generated/ko/cards.json'
import zhtw from '../../../generated/zh-tw/cards.json'
import id from '../../../generated/id/cards.json'
import nl from '../../../generated/nl/cards.json'
import pl from '../../../generated/pl/cards.json'
import ptbr from '../../../generated/pt-br/cards.json'
import ptpt from '../../../generated/pt-pt/cards.json'
import pt from '../../../generated/pt/cards.json'
import ru from '../../../generated/ru/cards.json'
import th from '../../../generated/th/cards.json'
import zhcn from '../../../generated/zh-cn/cards.json'
import zhtw from '../../../generated/zh-tw/cards.json'
const cards = {
'en': en,
'fr': fr,
'es': es,
'it': it,
'pt': pt,
en: en,
fr: fr,
es: es,
it: it,
pt: pt,
'pt-br': ptbr,
'pt-pt': ptpt,
'de': de,
'nl': nl,
'pl': pl,
'ru': ru,
'ja': ja,
'ko': ko,
de: de,
nl: nl,
pl: pl,
ru: ru,
ja: ja,
ko: ko,
'zh-tw': zhtw,
'id': id,
'th': th,
id: id,
th: th,
'zh-cn': zhcn,
} as const
type LocalCard = Omit<SDKCard, 'set'> & {set: () => Set}
type LocalCard = Omit<SDKCard, 'set'> & {set: () => TCGSet}
interface variants {
normal?: boolean;
@ -93,8 +92,8 @@ export default class Card implements LocalCard {
})
}
public set(): Set {
return Set.findOne(this.lang, {filters: { id: this.card.set.id }}) as Set
public set(): TCGSet {
return TCGSet.findOne(this.lang, { id: this.card.set.id }) as TCGSet
}
public static getAll(lang: SupportedLanguages): Array<SDKCard> {
@ -102,16 +101,15 @@ export default class Card implements LocalCard {
}
public static find(lang: SupportedLanguages, query: Query<SDKCard>) {
return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query)
.map((it) => new Card(lang, it))
return executeQuery(Card.getAll(lang), query).data.map((it) => new Card(lang, it))
}
public static findOne(lang: SupportedLanguages, query: Query<SDKCard>) {
const res = handleSort(handleValidation(this.getAll(lang), query), query)
const res = Card.find(lang, query)
if (res.length === 0) {
return undefined
}
return new Card(lang, res[0])
return res[0]
}
public resume(): CardResume {

70
server/src/V2/Components/Serie.ts
View File

@ -1,49 +1,48 @@
import { objectLoop } from '@dzeio/object-util'
import { Serie as SDKSerie, SerieResume, SupportedLanguages } from '@tcgdex/sdk'
import { Query } from '../../interfaces'
import { handlePagination, handleSort, handleValidation } from '../../util'
import Set from './Set'
import type { Serie as SDKSerie, SerieResume, SupportedLanguages } from '@tcgdex/sdk'
import { executeQuery, type Query } from '../../libs/QueryEngine/filter'
import TCGSet from './Set'
import en from '../../../generated/en/series.json'
import fr from '../../../generated/fr/series.json'
import es from '../../../generated/es/series.json'
import it from '../../../generated/it/series.json'
import pt from '../../../generated/pt/series.json'
import ptbr from '../../../generated/pt-br/series.json'
import ptpt from '../../../generated/pt-pt/series.json'
import de from '../../../generated/de/series.json'
import nl from '../../../generated/nl/series.json'
import pl from '../../../generated/pl/series.json'
import ru from '../../../generated/ru/series.json'
import en from '../../../generated/en/series.json'
import es from '../../../generated/es/series.json'
import fr from '../../../generated/fr/series.json'
import id from '../../../generated/id/series.json'
import it from '../../../generated/it/series.json'
import ja from '../../../generated/ja/series.json'
import ko from '../../../generated/ko/series.json'
import zhtw from '../../../generated/zh-tw/series.json'
import id from '../../../generated/id/series.json'
import nl from '../../../generated/nl/series.json'
import pl from '../../../generated/pl/series.json'
import ptbr from '../../../generated/pt-br/series.json'
import ptpt from '../../../generated/pt-pt/series.json'
import pt from '../../../generated/pt/series.json'
import ru from '../../../generated/ru/series.json'
import th from '../../../generated/th/series.json'
import zhcn from '../../../generated/zh-cn/series.json'
import zhtw from '../../../generated/zh-tw/series.json'
const series = {
'en': en,
'fr': fr,
'es': es,
'it': it,
'pt': pt,
en: en,
fr: fr,
es: es,
it: it,
pt: pt,
'pt-br': ptbr,
'pt-pt': ptpt,
'de': de,
'nl': nl,
'pl': pl,
'ru': ru,
'ja': ja,
'ko': ko,
de: de,
nl: nl,
pl: pl,
ru: ru,
ja: ja,
ko: ko,
'zh-tw': zhtw,
'id': id,
'th': th,
id: id,
th: th,
'zh-cn': zhcn,
} as const
type LocalSerie = Omit<SDKSerie, 'sets'> & {sets: () => Array<Set>}
type LocalSerie = Omit<SDKSerie, 'sets'> & {sets: () => Array<TCGSet>}
export default class Serie implements LocalSerie {
@ -63,8 +62,8 @@ export default class Serie implements LocalSerie {
})
}
public sets(): Array<Set> {
return this.serie.sets.map((s) => Set.findOne(this.lang, {filters: { id: s.id }}) as Set)
public sets(): Array<TCGSet> {
return this.serie.sets.map((s) => TCGSet.findOne(this.lang, { id: s.id }) as TCGSet)
}
public static getAll(lang: SupportedLanguages): Array<SDKSerie> {
@ -72,16 +71,15 @@ export default class Serie implements LocalSerie {
}
public static find(lang: SupportedLanguages, query: Query<SDKSerie>) {
return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query)
.map((it) => new Serie(lang, it))
return executeQuery(Serie.getAll(lang), query).data.map((it) => new Serie(lang, it))
}
public static findOne(lang: SupportedLanguages, query: Query<SDKSerie>) {
const res = handleValidation(this.getAll(lang), query)
const res = Serie.find(lang, query)
if (res.length === 0) {
return undefined
}
return new Serie(lang, res[0])
return res[0]
}
public resume(): SerieResume {

66
server/src/V2/Components/Set.ts
View File

@ -1,45 +1,44 @@
import { objectLoop } from '@dzeio/object-util'
import { Set as SDKSet, SetResume, SupportedLanguages } from '@tcgdex/sdk'
import { Query } from '../../interfaces'
import { handlePagination, handleSort, handleValidation } from '../../util'
import type { Set as SDKSet, SetResume, SupportedLanguages } from '@tcgdex/sdk'
import { executeQuery, type Query } from '../../libs/QueryEngine/filter'
import Card from './Card'
import Serie from './Serie'
import en from '../../../generated/en/sets.json'
import fr from '../../../generated/fr/sets.json'
import es from '../../../generated/es/sets.json'
import it from '../../../generated/it/sets.json'
import pt from '../../../generated/pt/sets.json'
import ptbr from '../../../generated/pt-br/sets.json'
import ptpt from '../../../generated/pt-pt/sets.json'
import de from '../../../generated/de/sets.json'
import nl from '../../../generated/nl/sets.json'
import pl from '../../../generated/pl/sets.json'
import ru from '../../../generated/ru/sets.json'
import en from '../../../generated/en/sets.json'
import es from '../../../generated/es/sets.json'
import fr from '../../../generated/fr/sets.json'
import id from '../../../generated/id/sets.json'
import it from '../../../generated/it/sets.json'
import ja from '../../../generated/ja/sets.json'
import ko from '../../../generated/ko/sets.json'
import zhtw from '../../../generated/zh-tw/sets.json'
import id from '../../../generated/id/sets.json'
import nl from '../../../generated/nl/sets.json'
import pl from '../../../generated/pl/sets.json'
import ptbr from '../../../generated/pt-br/sets.json'
import ptpt from '../../../generated/pt-pt/sets.json'
import pt from '../../../generated/pt/sets.json'
import ru from '../../../generated/ru/sets.json'
import th from '../../../generated/th/sets.json'
import zhcn from '../../../generated/zh-cn/sets.json'
import zhtw from '../../../generated/zh-tw/sets.json'
const sets = {
'en': en,
'fr': fr,
'es': es,
'it': it,
'pt': pt,
en: en,
fr: fr,
es: es,
it: it,
pt: pt,
'pt-br': ptbr,
'pt-pt': ptpt,
'de': de,
'nl': nl,
'pl': pl,
'ru': ru,
'ja': ja,
'ko': ko,
de: de,
nl: nl,
pl: pl,
ru: ru,
ja: ja,
ko: ko,
'zh-tw': zhtw,
'id': id,
'th': th,
id: id,
th: th,
'zh-cn': zhcn,
} as const
@ -77,11 +76,11 @@ export default class Set implements LocalSet {
symbol?: string | undefined
public serie(): Serie {
return Serie.findOne(this.lang, {filters: { id: this.set.serie.id }}) as Serie
return Serie.findOne(this.lang, { id: this.set.serie.id }) as Serie
}
public cards(): Array<Card> {
return this.set.cards.map((s) => Card.findOne(this.lang, { filters: { id: s.id }}) as Card)
return this.set.cards.map((s) => Card.findOne(this.lang, { id: s.id }) as Card)
}
public static getAll(lang: SupportedLanguages): Array<SDKSet> {
@ -89,16 +88,15 @@ export default class Set implements LocalSet {
}
public static find(lang: SupportedLanguages, query: Query<SDKSet>) {
return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query)
.map((it) => new Set(lang, it))
return executeQuery(Set.getAll(lang), query).data.map((it) => new Set(lang, it))
}
public static findOne(lang: SupportedLanguages, query: Query<SDKSet>) {
const res = handleValidation(this.getAll(lang), query)
const res = Set.find(lang, query)
if (res.length === 0) {
return undefined
}
return new Set(lang, res[0])
return res[0]
}
public resume(): SetResume {

87
server/src/V2/endpoints/jsonEndpoints.ts
View File

@ -1,41 +1,42 @@
import { objectKeys, objectLoop } from '@dzeio/object-util'
import { Card as SDKCard } from '@tcgdex/sdk'
import { objectKeys } from '@dzeio/object-util'
import type { Card as SDKCard } from '@tcgdex/sdk'
import apicache from 'apicache'
import express, { Request } from 'express'
import { Query } from '../../interfaces'
import express, { type Request } from 'express'
import { Errors, sendError } from '../../libs/Errors'
import type { Query } from '../../libs/QueryEngine/filter'
import { recordToQuery } from '../../libs/QueryEngine/parsers'
import { betterSorter, checkLanguage, unique } from '../../util'
import Card from '../Components/Card'
import Serie from '../Components/Serie'
import Set from '../Components/Set'
import TCGSet from '../Components/Set'
type CustomRequest = Request & {
/**
* disable caching
*/
DO_NOT_CACHE?: boolean
advQuery?: Query<any>
advQuery?: Query
}
const server = express.Router()
const endpointToField: Record<string, keyof SDKCard> = {
"categories": 'category',
categories: 'category',
'energy-types': 'energyType',
"hp": 'hp',
'illustrators': 'illustrator',
"rarities": 'rarity',
hp: 'hp',
illustrators: 'illustrator',
rarities: 'rarity',
'regulation-marks': 'regulationMark',
"retreats": 'retreat',
"stages": "stage",
"suffixes": "suffix",
retreats: 'retreat',
stages: "stage",
suffixes: "suffix",
"trainer-types": "trainerType",
// fields that need special care
'dex-ids': 'dexId',
"sets": "set",
"types": "types",
"variants": "variants",
sets: "set",
types: "types",
variants: "variants",
}
server
@ -66,27 +67,7 @@ server
return
}
const items: Query = {
filters: undefined,
sort: undefined,
pagination: undefined
}
objectLoop(req.query as Record<string, string | Array<string>>, (value: string | Array<string>, key: string) => {
if (!key.includes(':')) {
key = 'filters:' + key
}
const [cat, item] = key.split(':', 2) as ['filters', string]
if (!items[cat]) {
items[cat] = {}
}
const finalValue = Array.isArray(value) ? value.map((it) => isNaN(parseInt(it)) ? it : parseInt(it)) : isNaN(parseInt(value)) ? value : parseInt(value)
// @ts-expect-error normal behavior
items[cat][item] = finalValue
})
req.advQuery = items
req.advQuery = recordToQuery(req.query as Record<string, string | Array<string>>)
next()
})
@ -102,15 +83,16 @@ server
return
}
// biome-ignore lint/style/noNonNullAssertion: <explanation>
const query: Query = req.advQuery!
let data: Array<Card | Set | Serie> = []
let data: Array<Card | TCGSet | Serie> = []
switch (what.toLowerCase()) {
case 'card':
data = Card.find(lang, query)
break
case 'set':
data = Set.find(lang, query)
data = TCGSet.find(lang, query)
break
case 'serie':
data = Serie.find(lang, query)
@ -132,7 +114,7 @@ server
.get('/:lang/:endpoint', (req: CustomRequest, res): void => {
let { lang, endpoint } = req.params
const query: Query = req.advQuery!
const query: Query = req.advQuery ?? {}
if (endpoint.endsWith('.json')) {
endpoint = endpoint.replace('.json', '')
@ -143,7 +125,7 @@ server
return
}
let result: any
let result: unknown
switch (endpoint) {
case 'cards':
@ -153,7 +135,7 @@ server
break
case 'sets':
result = Set
result = TCGSet
.find(lang, query)
.map((c) => c.resume())
break
@ -169,7 +151,6 @@ server
case "rarities":
case "regulation-marks":
case "retreats":
case "series":
case "stages":
case "suffixes":
case "trainer-types":
@ -224,26 +205,26 @@ server
return sendError(Errors.LANGUAGE_INVALID, res, { lang })
}
let result: any | undefined
let result: unknown
switch (endpoint) {
case 'cards':
result = Card.findOne(lang, { filters: { id }, strict: true })?.full()
result = Card.findOne(lang, { id })?.full()
if (!result) {
result = Card.findOne(lang, { filters: { name: id }, strict: true })?.full()
result = Card.findOne(lang, { name: id })?.full()
}
break
case 'sets':
result = Set.findOne(lang, { filters: { id }, strict: true })?.full()
result = TCGSet.findOne(lang, { id })?.full()
if (!result) {
result = Set.findOne(lang, {filters: { name: id }, strict: true })?.full()
result = TCGSet.findOne(lang, { name: id })?.full()
}
break
case 'series':
result = Serie.findOne(lang, { filters: { id }, strict: true })?.full()
result = Serie.findOne(lang, { id })?.full()
if (!result) {
result = Serie.findOne(lang, { filters: { name: id }, strict: true })?.full()
result = Serie.findOne(lang, { name: id })?.full()
}
break
default:
@ -282,12 +263,14 @@ server
return sendError(Errors.LANGUAGE_INVALID, res, { lang })
}
let result: any | undefined
let result: unknown
switch (endpoint) {
case 'sets':
// allow the dev to use a non prefixed value like `10` instead of `010` for newer sets
result = Card
.findOne(lang, { filters: { localId: subid, set: id }, strict: true})?.full()
// @ts-expect-error normal behavior until the filtering is more fiable
.findOne(lang, { localId: { $or: [subid.padStart(3, '0'), subid]}, 'set.id': id })?.full()
break
}
if (!result) {

4
server/src/V2/graphql/index.ts
View File

@ -1,6 +1,6 @@
import express from 'express'
import fs from 'fs'
import { buildSchema, GraphQLError } from 'graphql'
import fs from 'node:fs'
import { buildSchema, type GraphQLError } from 'graphql'
import { createHandler } from 'graphql-http/lib/use/express'
import { type ruruHTML as RuruHTML } from 'ruru/dist/server'
/** @ts-expect-error typing is not correctly mapped (real type at ruru/dist/server.d.ts) */

54
server/src/V2/graphql/resolver.ts
View File

@ -1,44 +1,54 @@
import { SupportedLanguages } from '@tcgdex/sdk'
import { Query } from '../../interfaces'
import type { SupportedLanguages } from '@tcgdex/sdk'
import { type Query, Sort } from '../../libs/QueryEngine/filter'
import { recordToQuery } from '../../libs/QueryEngine/parsers'
import { checkLanguage } from '../../util'
import Card from '../Components/Card'
import Serie from '../Components/Serie'
import Set from '../Components/Set'
const middleware = (fn: (lang: SupportedLanguages, query: Query) => any) => (
data: Query,
// TODO: make a better way to find the language
function getLang(e: any): SupportedLanguages {
// get the locale directive
const langArgument = e?.fieldNodes?.[0]?.directives?.[0]?.arguments?.[0]?.value
if (!langArgument) {
return 'en'
}
if (langArgument.kind === 'Variable') {
return e.variableValues[langArgument.name.value]
}
return langArgument.value
}
const middleware = (fn: (lang: SupportedLanguages, query: Query<object>) => any) => (
data: Record<string, any>,
_: any,
e: any
) => {
// get the locale directive
const langArgument = e?.fieldNodes?.[0]?.directives?.[0]?.arguments?.[0]?.value
const lang = getLang(e)
const query = recordToQuery(data.filters ?? {})
// Deprecated code handling
// @ts-expect-error count is deprectaed in the frontend
if (data.pagination?.count) {
// @ts-expect-error count is deprectaed in the frontend
data.pagination.itemsPerPage = data.pagination.count
if (data.pagination) {
query.$page = data.pagination.page ?? 1
query.$limit = data.pagination.itemsPerPage ?? 100
}
// if there is no locale directive
if (!langArgument) {
return fn('en', data)
if (data.sort) {
query.$sort = {
[data.sort.field]: data.sort.order === 'DESC' ? Sort.DESC : Sort.ASC
}
// set default locale directive value
let lang = 'en'
// handle variable for directive value
if (langArgument.kind === 'Variable') {
lang = e.variableValues[langArgument.name.value]
} else {
lang = langArgument.value
}
if (!checkLanguage(lang)) {
return undefined
}
return fn(lang, data)
return fn(lang, query)
}
export default {

463
server/src/libs/QueryEngine/filter.ts Normal file
View File

@ -0,0 +1,463 @@
import { objectGet, objectKeys, objectLoop, objectSize } from '@dzeio/object-util'
import { isNull } from '../../util'
interface QueryRootFilters<Obj extends object> {
/**
* one of the results should be true to be true
*/
$or?: Array<QueryList<Obj>>
/**
* every results should be false to be true
*/
$nor?: Array<QueryList<Obj>>
/**
* (default) make sure every sub queries return true
*/
$and?: Array<QueryList<Obj>>
/**
* at least one result must be false
*/
$nand?: Array<QueryList<Obj>>
/**
* invert the result from the following query
*/
$not?: QueryList<Obj>
/**************
* PAGINATION *
**************/
/**
* define a precise offset of the data you fetched
*/
$offset?: number
/**
* limit the number of elements returned from the dataset
*/
$limit?: number
/**
* instead of using a precise offset, use a page system
*/
$page?: number
/**********
* Sorting *
**********/
/**
* sort the data the way you want with each keys being priorized
*
* ex:
* {a: Sort.DESC, b: Sort.ASC}
*
* will sort first by a and if equal will sort by b
*/
$sort?: SortInterface<Obj>
}
/**
* Logical operators that can be used to filter data
*/
export type QueryLogicalOperator<Value> = {
/**
* one of the results should be true to be true
*/
$or: Array<QueryValues<Value>>
} | {
/**
* every results should be false to be true
*/
$nor: Array<QueryValues<Value>>
} | {
/**
* at least one result must be false
*/
$nand: Array<QueryValues<Value>>
} | {
/**
* (default) make sure every sub queries return true
*/
$and: Array<QueryValues<Value>>
} | {
/**
* invert the result from the following query
*/
$not: QueryValues<Value>
}
/**
* differents comparisons operators that can be used to filter data
*/
export type QueryComparisonOperator<Value> = {
/**
* the remote source value must be absolutelly equal to the proposed value
*/
$eq: Value | null
} | {
/**
* the remote source value must be greater than the proposed value
*/
$gt: number | Date
} | {
/**
* the remote source value must be lesser than the proposed value
*/
$lt: number | Date
} | {
/**
* the remote source value must be greater or equal than the proposed value
*/
$gte: number | Date
} | {
/**
* the remote source value must be lesser or equal than the proposed value
*/
$lte: number | Date
} | {
/**
* the remote source value must be one of the proposed values
*/
$in: Array<Value>
} | {
/**
* laxist validation of the remote value
*
* for strings: remote contains value while not following casing like ($lax) `pou` === `Pouet` (remote)
* for numbers: it does a string conversion first
*/
$lax: Value | null
} | {
/**
* (for arrays only) specify a needed length of the array
*/
$len: number | { $gt: number }
}
export type QueryList<Obj extends object> = {
[Key in keyof Obj]?: QueryValues<Obj[Key]>
}
/**
* Differents values the element can take
* if null it will check if it is NULL on the remote
* if array it will check oneOf
* if RegExp it will check if regexp match
*/
export type QueryValues<Value> = Value |
null |
Array<Value> |
RegExp |
QueryComparisonOperator<Value> |
QueryLogicalOperator<Value>
/**
* The query element that allows you to query different elements
*/
export type Query<Obj extends object = object> = QueryList<Obj> & QueryRootFilters<Obj>
// biome-ignore lint/style/useEnumInitializers: <explanation>
export enum Sort {
/**
* Sort the values from the lowest to the largest
*/
ASC,
/**
* Sort the values form the largest to the lowest
*/
DESC
}
/**
* sorting interface with priority
*/
export type SortInterface<Obj extends object> = {
[Key in keyof Obj]?: Sort
}
export declare type AllowedValues = string | number | bigint | boolean | null | undefined
interface FilterResult<T extends object> {
data: Array<T>
rows: number
pagination?: {
page: number
pageCount: number
totalRows: number
} | undefined
}
/**
*
* @param data the original data
* @param query the query to filter against
* @param options additionnal execution options
* @returns the filtered/ordered/paginated {@link data}
*/
export function executeQuery<T extends object = Record<string, unknown>>(data: Array<T>, query: Query<T>, options?: { debug?: boolean }): FilterResult<T> {
if (options?.debug) {
console.log('Query', query)
}
// filter
let filtered = data.filter((it) => {
const res = objectLoop(query, (value, key) => {
if (key === '$or') {
for (const sub of value as Array<QueryList<T>>) {
const final = filterEntry(sub, it)
// eslint-disable-next-line max-depth
if (final) {
return true
}
}
return false
}
if ((key as string).startsWith('$')) {
return true
}
return filterEntry(query, it)
})
return res
})
if (options?.debug) {
console.log('postFilters', filtered)
}
// sort
if (query.$sort && objectSize(query.$sort) >= 1) {
// temp until better solution is found
// get the first key
const firstKey = objectKeys(query.$sort)[0]
// biome-ignore lint/style/noNonNullAssertion: item is not null
const first = query.$sort[firstKey]!
// forst by the first key
filtered = filtered.sort((objA, objB) => {
const a = objA[firstKey]
const b = objB[firstKey]
const ascend = first !== Sort.DESC // it is Ascend by default, so compare against it
if (typeof a === 'number' && typeof b === 'number') {
if (ascend) {
return b - a
}
return a - b
}
if (a instanceof Date && b instanceof Date) {
if (ascend) {
return a.getTime() - b.getTime()
}
return b.getTime() - a.getTime()
}
if (typeof a === 'string' && typeof b === 'string') {
if (ascend) {
return a.localeCompare(b)
}
return b.localeCompare(a)
}
if (ascend) {
return a > b ? 1 : -1
}
return a > b ? -1 : 1
})
}
if (options?.debug) {
console.log('postSort', filtered)
}
// length of the query assuming a single page
const unpaginatedLength = filtered.length
let page: number | null = null
let pageCount: number | null = null
// limit
if (!isNull(query.$offset) || !isNull(query.$limit) || !isNull(query.$page)) {
let limit = query.$limit ?? -1
if (!isNull(query.$page) && isNull(query.$offset) && isNull(query.$limit)) {
console.warn('using $page NEED a $limit too, setting limit to `10`')
limit = 10
}
// when using $page, they start at 1 and not 0
const offset = query.$offset ?? (query.$page ? (query.$page - 1) * limit : 0)
filtered = filtered.slice(offset, limit >= 0 ? offset + limit : undefined)
page = Math.trunc(offset / limit)
pageCount = Math.ceil(unpaginatedLength / limit)
}
if (options?.debug) {
console.log('postLimit', filtered)
}
return {
data: filtered,
rows: filtered.length,
pagination: (!isNull(page) && !isNull(pageCount)) ? {
page: page,
pageCount: pageCount,
totalRows: unpaginatedLength
} : undefined
}
}
/**
*
* @param query the query of the entry
* @param item the implementation of the item
* @returns if it should be kept or not
*/
export function filterEntry<T extends object>(query: QueryList<T>, item: T): boolean {
// eslint-disable-next-line complexity
const res = objectLoop(query as any, (queryValue, key: keyof typeof query) => {
/**
* TODO: handle $keys
*/
if ((key as string).startsWith('$')) {
return true
}
let value: unknown = undefined
// handle deeply nested items
if ((key as string).includes('.')) {
value = objectGet(item, key as string)
}
// handle if nested item does not exists
if (typeof value === 'undefined') {
value = item[key]
}
return filterValue(value, queryValue)
})
return res
}
/**
* indicate if a value should be kept by an ENTIRE query
*
* @param value the value to filter
* @param query the full query
* @returns if the query should keep the value or not
*/
function filterValue<T extends AllowedValues>(value: unknown, query: QueryValues<T>) {
if (typeof query !== 'object' || query === null || query instanceof RegExp || Array.isArray(query)) {
return filterItem(value, query)
}
// loop through each keys of the query
// eslint-disable-next-line arrow-body-style
return objectLoop(query as any, (querySubValue: unknown, queryKey: string) => {
return filterItem(value, {[queryKey]: querySubValue } as QueryValues<T>)
})
}
/**
*
* @param value the value to check
* @param query a SINGLE query to check against
* @returns if the value should be kept or not
*/
// eslint-disable-next-line complexity
function filterItem(value: any, query: QueryValues<AllowedValues>): boolean {
/**
* check if the value is null
*/
if (query === null) {
return typeof value === 'undefined' || value === null
}
if (query instanceof RegExp) {
return query.test(typeof value === 'string' ? value : value.toString())
}
/**
* ?!?
*/
if (value === null || typeof value === 'undefined') {
return false
}
/**
* strict value check by default
*/
if (!(typeof query === 'object')) {
return query === value
}
/**
* Array checking and $in
*/
if (Array.isArray(query) || '$in' in query) {
const arr = Array.isArray(query) ? query : query.$in as Array<AllowedValues>
return arr.includes(value)
}
if ('$inc' in query) {
if (typeof value === 'number' && typeof query.$inc === 'number') {
return value === query.$inc
}
return (value.toString() as string).toLowerCase().includes(query.$inc!.toString()!.toLowerCase())
}
if ('$eq' in query) {
return query.$eq === value
}
/**
* numbers specific cases for numbers
*/
if ('$gt' in query) {
value = value instanceof Date ? value.getTime() : value
const comparedValue = query.$gt instanceof Date ? query.$gt.getTime() : query.$gt
return typeof value === 'number' && typeof comparedValue === 'number' && value > comparedValue
}
if ('$lt' in query) {
value = value instanceof Date ? value.getTime() : value
const comparedValue = query.$lt instanceof Date ? query.$lt.getTime() : query.$lt
return typeof value === 'number' && typeof comparedValue === 'number' && value < comparedValue
}
if ('$gte' in query) {
value = value instanceof Date ? value.getTime() : value
const comparedValue = query.$gte instanceof Date ? query.$gte.getTime() : query.$gte
return typeof value === 'number' && typeof comparedValue === 'number' && value >= comparedValue
}
if ('$lte' in query) {
value = value instanceof Date ? value.getTime() : value
const comparedValue = query.$lte instanceof Date ? query.$lte.getTime() : query.$lte
return typeof value === 'number' && typeof comparedValue === 'number' && value <= comparedValue
}
if ('$len' in query && Array.isArray(value)) {
return value.length === query.$len
}
/**
* Logical Operators
*/
if ('$or' in query && Array.isArray(query.$or)) {
return !!query.$or.find((it) => filterValue(value, it as QueryValues<any>))
}
if ('$and' in query && Array.isArray(query.$and)) {
return !query.$and.find((it) => !filterValue(value, it as QueryValues<any>))
}
if ('$not' in query) {
return !filterValue(value, query.$not as QueryValues<any>)
}
if ('$nor' in query && Array.isArray(query.$nor)) {
return !query.$nor.find((it) => filterValue(value, it as QueryValues<any>))
}
if ('$nand' in query && Array.isArray(query.$nand)) {
return !!query.$nand.find((it) => !filterValue(value, it as QueryValues<any>))
}
return false
}

195
server/src/libs/QueryEngine/parsers.ts Normal file
View File

@ -0,0 +1,195 @@
import { isObject, objectLoop } from '@dzeio/object-util'
import { Sort, type Query, type QueryValues } from './filter'
/**
* List of allowed prefixes
*/
const prefixes = [
'like',
'not',
'notlike',
'eq',
'neq',
'gte',
'gt',
'lt',
'lte',
'null',
'notnull'
] as const
type Prefix = typeof prefixes[number]
/**
* indicate if the string is a prefix or not
*
* @param str the string to check
* @returns {boolean} true if it's a prefix, else false
*/
function isPrefix(str: string): str is Prefix {
return prefixes.includes(str as Prefix)
}
/**
* Parse a {@link URL.searchParams} object into a {@link Query}
*
* @param searchParams the searchparams object to parse
* @param skip keys that are skipped by the transformer
*
* @returns the searchParams into a Query object
*/
export function parseSearchParams<T extends object = object>(searchParams: URLSearchParams, skip: Array<string> = []): Query<T> {
const query: Query<Record<string, unknown>> = {}
skip.push('sort:field', 'sort:order')
const sortField = searchParams.get('sort:field')
if (sortField) {
const order = searchParams.get('sort:order') ?? 'ASC'
query.$sort = { [sortField]: order === 'ASC' ? Sort.ASC : Sort.DESC }
}
for (const [key, value] of searchParams) {
if (key === 'pagination:page') {
query.$page = Number.parseInt(value)
continue
}
if (key === 'pagination:itemsPerPage') {
query.$limit = Number.parseInt(value)
continue
}
if (skip.includes(key)) {
continue
}
const params = parseParam(key, value)
if (!query[key]) {
query[key] = params
} else {
if (isObject(params)) {
objectLoop(params, (v, k) => {
(query[key] as any)[k] = v
return
})
} else {
query[key] = params
}
}
}
console.log(query)
return query as Query<T>
}
/**
* parse a simple {@link Record} object into a {@link Query}
*
* @param searchParams the searchparams object to parse
* @param skip keys that are skipped by the transformer
*
* @returns the searchParams into a Query object
*/
export function recordToQuery<T extends object = object>(input: Record<string, string | Array<string>>, skip: Array<string> = []): Query<T> {
const query: Query<Record<string, unknown>> = {}
skip.push('sort:field', 'sort:order')
const sortField = input['sort:field'] as string
if (sortField) {
const order = input['sort:order'] ?? 'ASC'
query.$sort = { [sortField]: order === 'ASC' ? Sort.ASC : Sort.DESC }
}
objectLoop(input, (value: string | Array<string>, key) => {
if (key === 'pagination:page') {
query.$page = Number.parseInt(value as string)
return
}
if (key === 'pagination:itemsPerPage') {
query.$limit = Number.parseInt(value as string)
return
}
if (skip.includes(key)) {
return
}
if (!Array.isArray(value)) {
value = [value]
}
for (const it of value) {
const params = parseParam(key, it)
if (!query[key]) {
query[key] = params
} else {
if (isObject(params)) {
objectLoop(params, (v, k) => {
(query[key] as any)[k] = v
return
})
} else {
query[key] = params
}
}
}
})
console.log(query)
return query as Query<T>
}
/**
* Parse a single element
*
* @param _key currently unused, kept for future compatibility
* @param value the value to parse
*
* @returns the parsed {@link Query} element to be added
*/
function parseParam(_key: string, value: string): QueryValues<unknown> {
const colonLocation = value.indexOf(':')
let filter: Prefix = 'like'
let compared: string | number = value
if (colonLocation >= 2) { // 2 because the smallest prefix is two characters long
const prefix = value.slice(0, colonLocation)
if (isPrefix(prefix)) {
filter = prefix
compared = value.slice(colonLocation + 1)
}
}
if (/^\d+\.?\d*$/g.test(compared)) {
compared = Number.parseFloat(compared)
}
switch (filter) {
case 'not':
case 'notlike':
return { $not: { $inc: compared }}
case 'eq':
return compared
case 'neq':
return { $not: compared }
case 'gte':
return { $gte: compared }
case 'gt':
return { $gt: compared }
case 'lt':
return { $lt: compared }
case 'lte':
return { $lte: compared }
case 'null':
return null
case 'notnull':
return { $not: null }
default:
return { $inc: compared }
}
}

4
server/src/status.ts
View File

@ -346,7 +346,7 @@ export default express.Router()
<table class="serie">
${objectMap(setsData, (serie, serieId) => {
// Loop through every series and name them
const name = Serie.findOne('en', { filters: { id: serieId }})?.name ?? Serie.findOne('ja' as any, { filters: { id: serieId }})?.name
const name = Serie.findOne('en', { id: serieId })?.name ?? Serie.findOne('ja' as any, { id: serieId })?.name
return `
<thead>
<tr><th class="notop" colspan="35"><h2>${name} (${serieId})</h2></th></tr>
@ -364,7 +364,7 @@ export default express.Router()
// loop through every sets
// find the set in the first available language (Should be English globally)
const setTotal = Set.findOne(data[0] as 'en', { filters: { id: setId }})
const setTotal = Set.findOne(data[0] as 'en', { id: setId })
let str = '<tr>' + `<td>${setTotal?.name} (${setId}) <br />${setTotal?.cardCount.total ?? 1} cards</td>`
// let str = '<tr>' + `<td>${setId})</td>`

3
server/tsconfig.json
View File

@ -1,7 +1,8 @@
{
"extends": "./node_modules/@dzeio/config/tsconfig.base.json",
"compilerOptions": {
"outDir": "dist"
"outDir": "dist",
"esModuleInterop": true
},
"include": ["src"]
}