fix: Ordering failing when one value is null (#490)

Co-authored-by: Avior <git@avior.me>

.bruno/fixes/489-crash-when-sorting-with-null-values.bru

@@ -0,0 +1,17 @@
+meta {
+  name: 489 - Crash When sorting with Null values
+  type: http
+  seq: 6
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/cards?name=pikachu&sort:field=hp&sort:order=DESC
+  body: none
+  auth: none
+}
+
+query {
+  name: pikachu
+  sort:field: hp
+  sort:order: DESC
+}

.bruno/random/Random Card.bru

@@ -0,0 +1,20 @@
+meta {
+  name: Random Card
+  type: http
+  seq: 1
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/card?name=furret
+  body: none
+  auth: none
+}
+
+query {
+  name: furret
+}
+
+assert {
+  res.status: eq 200
+  res.body.name: contains Furret
+}

.bruno/random/Random Serie.bru

@@ -0,0 +1,19 @@
+meta {
+  name: Random Serie
+  type: http
+  seq: 3
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/serie?name=p
+  body: none
+  auth: none
+}
+
+query {
+  name: p
+}
+
+assert {
+  res.status: eq 200
+}

.bruno/random/Random Set.bru

@@ -0,0 +1,20 @@
+meta {
+  name: Random Set
+  type: http
+  seq: 2
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/set?name=sword
+  body: none
+  auth: none
+}
+
+query {
+  name: sword
+}
+
+assert {
+  res.status: eq 200
+  res.body.name: contains Sword
+}

.github/workflows/build.yml

@@ -16,6 +16,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Docker meta
       id: meta

.github/workflows/test.yml

@@ -24,6 +24,7 @@ jobs:
         bun install --frozen-lockfile
         cd server
         bun install --frozen-lockfile
+        bun run compile
 
     - name: Validate the data & the server
       run: |

Dockerfile

@@ -3,6 +3,10 @@ FROM docker.io/oven/bun:1-alpine as BUILD_IMAGE
 # go to work folder
 WORKDIR /usr/src/app
 
+# Add git as it is used to fetch updated times
+RUN apk add git &&\
+git config --global safe.directory '*'
+
 ADD --chown=bun:bun package.json bun.lockb ./
 ADD --chown=bun:bun server/package.json server/bun.lockb ./server/
 

meta/codeshift.ts

@@ -56,6 +56,20 @@ function simplify(base: ObjectExpression): ObjectField {
 	}
 }
 
+function exists(path: ObjectExpression | ArrayExpression, key: string | number) {
+	if (path.type === 'ObjectExpression') {
+		path.properties.forEach((p) => {
+			const prop = p as Property
+			if ((prop.key as Identifier).name === (key + '')) {
+				return true
+			}
+		})
+		return false
+	} else {
+
+	}
+}
+
 function set(j: JSCodeshift, path: ObjectExpression | ArrayExpression, value: Possible, key: string | number, options?: {override?: boolean}) {
 
 	let exists = false

meta/definitions/api.d.ts

@@ -261,7 +261,9 @@ export interface Card extends CardResume {
 		 * Ability to play in expanded tournaments
 		 */
 		expanded: boolean;
-	};
+	}
+
+	updated: string
 }
 
 /**

meta/definitions/graphql.gql

@@ -27,7 +27,7 @@ type Query {
 
 	"""Find one card (using the id and set is deprecated)"""
 	card(
-		id: ID!,
+		id: ID,
 		set: String,
 		"""The new way to filter"""
 		filters: CardsFilters
@@ -35,14 +35,14 @@ type Query {
 
 	"""Find one set (using the id is deprecated)"""
 	set(
-		id: ID!,
+		id: ID,
 		"""The new way to filter"""
 		filters: SetFilters
 	): Set
 
 	"""Find one serie (using the id is deprecated)"""
 	serie(
-		id: ID!,
+		id: ID,
 		"""The new way to filter"""
 		filters: SerieFilters
 	): Serie

server/compiler/index.ts

@@ -1,9 +1,8 @@
 /* eslint-disable max-statements */
-import { FileFunction } from './compilerInterfaces'
 import { promises as fs } from 'fs'
-import { fetchRemoteFile } from './utils/util'
-import { objectValues } from '@dzeio/object-util'
 import { SupportedLanguages } from '../../interfaces'
+import { FileFunction } from './compilerInterfaces'
+import { fetchRemoteFile, loadLastEdits } from './utils/util'
 
 const LANGS: Array<SupportedLanguages> = ['en', 'fr', 'es', 'it', 'pt', 'de']
 
@@ -21,6 +20,8 @@ const DIST_FOLDER = './generated'
 		await fs.rm(DIST_FOLDER, {recursive: true})
 	} catch {}
 
+	console.log('Loading files last edit')
+	await loadLastEdits()
 
 	console.log('Let\'s GO !')
 

server/compiler/utils/cardUtil.ts

@@ -1,9 +1,10 @@
 /* eslint-disable sort-keys */
+import { exec } from 'child_process'
 import { Card, Set, SupportedLanguages, Types } from '../../../interfaces'
 import { CardResume, Card as CardSingle } from '../../../meta/definitions/api'
 import { setToSetSimple } from './setUtil'
 import translate from './translationUtil'
-import { DB_PATH, cardIsLegal, fetchRemoteFile, smartGlob } from './util'
+import { DB_PATH, cardIsLegal, fetchRemoteFile, getLastEdit, smartGlob } from './util'
 
 export async function getCardPictures(cardId: string, card: Card, lang: SupportedLanguages): Promise<string | undefined> {
 	try {
@@ -106,8 +107,8 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
 		legal: {
 			standard: cardIsLegal('standard', card, localId),
 			expanded: cardIsLegal('expanded', card, localId)
-		}
-
+		},
+		updated: await getCardLastEdit(localId, card)
 	}
 }
 
@@ -161,3 +162,8 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
 		return a >= b ? 1 : -1
 	})
 }
+
+async function getCardLastEdit(localId: string, card: Card): Promise<string> {
+	const path = `../data/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
+	return getLastEdit(path)
+}

server/compiler/utils/util.ts

@@ -1,6 +1,6 @@
+import { exec } from 'node:child_process'
+import { glob } from 'glob'
 import { Card, Set } from '../../../interfaces'
-import glob from 'glob'
-import fetch from 'node-fetch'
 import * as legals from '../../../meta/legals'
 
 interface fileCacheInterface {
@@ -18,9 +18,16 @@ const fileCache: fileCacheInterface = {}
  */
 export async function fetchRemoteFile<T = any>(url: string): Promise<T> {
 	if (!fileCache[url]) {
+		const signal = new AbortController()
+
+		const finished = setTimeout(() => {
+			signal.abort()
+		}, 60 * 1000);
+
 		const resp = await fetch(url, {
-			timeout: 60 * 1000
+			signal: signal.signal
 		})
+		clearTimeout(finished)
 		fileCache[url] = resp.json()
 	}
 	return fileCache[url]
@@ -30,9 +37,7 @@ const globCache: Record<string, Array<string>> = {}
 
 export async function smartGlob(query: string): Promise<Array<string>> {
 	if (!globCache[query]) {
-		globCache[query] = await new Promise((res) => {
-			glob(query, (_, matches) => res(matches))
-		})
+		globCache[query] = await glob(query)
 	}
 	return globCache[query]
 }
@@ -75,3 +80,45 @@ export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean {
 	}
 	return false
 }
+
+function runCommand(command: string): Promise<string> {
+	return new Promise<string>((res, rej) => {
+		exec(command, (err, out) => {
+			if (err) {
+				rej(err)
+				return
+			}
+			res(out)
+		})
+	})
+}
+
+let lastEditsCache: Record<string, string> = {}
+export async function loadLastEdits() {
+	const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
+	const files = (await runCommand(firstCommand)).split('\n')
+	console.log('Loaded files tree', files.length, 'files')
+	console.log('Loading their last edit time')
+	let processed = 0
+	for (let file of files) {
+		file = file.replace(/"/g, '').replace("\\303\\251", "é")
+		try {
+			lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`)
+		} catch {
+			console.warn('could not load file', file, 'hope it does not break everything else lol')
+		}
+		processed++
+		if (processed % 1000 === 0) {
+			console.log('loaded', processed, 'out of', files.length, 'files')
+		}
+	}
+	console.log('done loading files')
+}
+
+export function getLastEdit(path: string): string {
+	const date = lastEditsCache[path]
+	if (!date) {
+		throw new Error(`edit date not found for file ${path}`)
+	}
+	return date
+}

server/package.json

@@ -16,19 +16,15 @@
 		"@tcgdex/sdk": "^2",
 		"apicache": "^1",
 		"express": "^4",
-		"express-graphql": "^0.12.0",
-		"graphql": "^15"
+		"graphql": "^15",
+		"graphql-http": "^1.22.1",
+		"ruru": "^2.0.0-beta.11"
 	},
 	"devDependencies": {
 		"@types/apicache": "^1",
 		"@types/express": "^4",
-		"@types/glob": "^8",
-		"@types/node": "^18",
-		"@types/node-fetch": "^2",
-		"glob": "^8",
-		"node-fetch": "^2",
-		"ts-node": "^10",
-		"ts-node-dev": "^2",
-		"typescript": "^4"
+		"@types/node": "^20",
+		"glob": "^10",
+		"typescript": "^5"
 	}
 }

server/src/V2/endpoints/jsonEndpoints.ts

@@ -1,13 +1,21 @@
 import { objectKeys, objectLoop } from '@dzeio/object-util'
 import { Card as SDKCard } from '@tcgdex/sdk'
 import apicache from 'apicache'
-import express from 'express'
+import express, { Request } from 'express'
 import { Query } from '../../interfaces'
 import { betterSorter, checkLanguage, sendError, unique } from '../../util'
 import Card from '../Components/Card'
 import Serie from '../Components/Serie'
 import Set from '../Components/Set'
 
+type CustomRequest = Request & {
+	/**
+	 * disable caching
+	 */
+	DO_NOT_CACHE?: boolean
+	advQuery?: Query<any>
+}
+
 const server = express.Router()
 
 const endpointToField: Record<string, keyof SDKCard> = {
@@ -31,7 +39,7 @@ const endpointToField: Record<string, keyof SDKCard> = {
 
 server
 	// Midleware that handle caching only in production and on GET requests
-	.use(apicache.middleware('1 day', (req: Request, res: Response) => res.status < 400 && process.env.NODE_ENV === 'production' && req.method === 'GET', {}))
+	.use(apicache.middleware('1 day', (req: CustomRequest, res: Response) => !req.DO_NOT_CACHE && res.status < 400 && process.env.NODE_ENV === 'production' && req.method === 'GET', {}))
 
 	// .get('/cache/performance', (req, res) => {
 	// 	res.json(apicache.getPerformance())
@@ -43,14 +51,14 @@ server
 	// })
 
 	// Midleware that handle url transformation
-	.use((req, _, next) => {
+	.use((req: CustomRequest, _, next) => {
 		// this is ugly BUT it fix the problem with + not becoming spaces
 		req.url = req.url.replace(/\+/g, ' ')
 		next()
 	})
 
 	// handle Query builder
-	.use((req, _, next) => {
+	.use((req: CustomRequest, _, next) => {
 		// handle no query
 		if (!req.query) {
 			next()
@@ -77,22 +85,51 @@ server
 
 		})
 
-		// @ts-expect-error normal behavior
 		req.advQuery = items
 
 		next()
 	})
 
+	/**
+	 * Allows the user to fetch a random card/set/serie from the database
+	 */
+	.get('/:lang/random/:what', (req: CustomRequest, res): void => {
+		const { lang, what } = req.params
+
+		if (!checkLanguage(lang)) {
+			return sendError('LanguageNotFoundError', res, lang)
+		}
+
+		const query: Query = req.advQuery!
+
+		let data: Array<Card | Set | Serie> = []
+		switch (what.toLowerCase()) {
+			case 'card':
+				data = Card.find(lang, query)
+				break
+			case 'set':
+				data = Set.find(lang, query)
+				break
+			case 'serie':
+				data = Serie.find(lang, query)
+				break
+			default:
+				return sendError('EndpointNotFoundError', res, what)
+		}
+		const item = Math.min(data.length - 1, Math.max(0, Math.round(Math.random() * data.length)))
+		req.DO_NOT_CACHE = true
+		res.json(data[item])
+	})
+
 
 	/**
 	 * Listing Endpoint
 	 * ex: /v2/en/cards
 	 */
-	.get('/:lang/:endpoint', (req, res): void => {
+	.get('/:lang/:endpoint', (req: CustomRequest, res): void => {
 		let { lang, endpoint } = req.params
 
-		// @ts-expect-error normal behavior
-		const query: Query = req.advQuery
+		const query: Query = req.advQuery!
 
 		if (endpoint.endsWith('.json')) {
 			endpoint = endpoint.replace('.json', '')
@@ -170,7 +207,7 @@ server
 	 * Listing Endpoint
 	 * ex: /v2/en/cards/base1-1
 	 */
-	.get('/:lang/:endpoint/:id', (req, res) => {
+	.get('/:lang/:endpoint/:id', (req: CustomRequest, res) => {
 		let { id, lang, endpoint } = req.params
 
 		if (id.endsWith('.json')) {
@@ -223,7 +260,7 @@ server
 	 * sub id Endpoint (for the set endpoint only currently)
 	 * ex: /v2/en/sets/base1/1
 	 */
-	.get('/:lang/:endpoint/:id/:subid', (req, res) => {
+	.get('/:lang/:endpoint/:id/:subid', (req: CustomRequest, res) => {
 		let { id, lang, endpoint, subid } = req.params
 
 		if (subid.endsWith('.json')) {

server/src/V2/graphql/index.ts

@@ -1,9 +1,15 @@
 import express from 'express'
-import { graphqlHTTP } from 'express-graphql'
 import fs from 'fs'
-import { buildSchema, formatError, GraphQLError } from 'graphql'
+import { buildSchema, GraphQLError } from 'graphql'
+import { createHandler } from 'graphql-http/lib/use/express'
+import { type ruruHTML as RuruHTML } from 'ruru/dist/server'
+/** @ts-expect-error typing is not correctly mapped (real type at ruru/dist/server.d.ts) */
+import { makeHTMLParts, ruruHTML as tmp } from 'ruru/server'
 import resolver from './resolver'
 
+
+const ruruHTML: typeof RuruHTML = tmp
+
 // Init Express Router
 const router = express.Router()
 
@@ -14,11 +20,19 @@ const router = express.Router()
 const schema = buildSchema(fs.readFileSync('./public/v2/graphql.gql', 'utf-8'))
 
 // Error Logging for debugging
-function graphQLErrorHandle(error: GraphQLError) {
+function graphQLErrorHandle(error: Readonly<GraphQLError | Error>) {
 	if (process.env.NODE_ENV !== 'production') {
 		console.error(error)
 	}
-	if (error.source) {
+	if (!('source' in error)) {
+		const columns = (process?.stdout?.columns ?? 32) - 7
+		const dashes = ''.padEnd(columns / 2, '-')
+
+		console.error(`\x1b[91m${dashes} ERROR ${dashes}\x1b[0m`)
+		console.error('GraphQL Error')
+		console.error(error.message)
+		console.error(`\x1b[91m${dashes} ERROR ${dashes}\x1b[0m`)
+	} else if (error.source) {
 		const columns = (process?.stdout?.columns ?? 32) - 7
 		const dashes = ''.padEnd(columns / 2, '-')
 
@@ -28,18 +42,24 @@ function graphQLErrorHandle(error: GraphQLError) {
 		console.error(error.source?.body)
 		console.error(`\x1b[91m${dashes} ERROR ${dashes}\x1b[0m`)
 	}
-	return formatError(error)
+	return error
 }
 
-const graphql = graphqlHTTP({
-	schema,
+const graphql = createHandler({
+	schema: schema,
 	rootValue: resolver,
-	graphiql: true,
-	customFormatErrorFn: graphQLErrorHandle
+	formatError: graphQLErrorHandle
 })
 
 // Add graphql to the route
-router.get('/', graphql)
+router.get('/', (_, res) => {
+	res.type('html')
+
+	res.end(ruruHTML({ endpoint: '/v2/graphql' }, {
+		...makeHTMLParts(),
+		titleTag: '<title>GraphiQL - TCGdex API V2</title>'
+	}))
+})
 router.post('/', graphql)
 
 export default router

server/src/util.ts

@@ -150,9 +150,21 @@ export function handleSort(data: Array<any>, query: Query<any>) {
  * @param order the base ordering
  * @returns a function that is feed in the `sort` function
  */
-const advSort = (a: string | number, b: string | number, order: 'ASC' | 'DESC' = 'ASC') => {
-	a = tryParse(a) ?? a
-	b = tryParse(b) ?? b
+const advSort = (a?: string | number, b?: string | number, order: 'ASC' | 'DESC' = 'ASC') => {
+	const isANull = isNull(a)
+	const isBNull = isNull(b)
+	if (isANull && isBNull) {
+		return 0
+	}
+	if (isANull) {
+		return order === 'ASC' ? -1 : 1
+	}
+	if (isBNull) {
+		return order === 'ASC' ? 1 : -1
+	}
+
+	a = tryParse(a!) ?? a
+	b = tryParse(b!) ?? b
 
 	if (order === 'DESC') {
 		const tmp = a
@@ -164,7 +176,7 @@ const advSort = (a: string | number, b: string | number, order: 'ASC' | 'DESC' =
 		return a - b
 	}
 
-	return a.toString().localeCompare(b.toString())
+	return a!.toString().localeCompare(b!.toString())
 }
 
 function tryParse(value: string | number): number | null {