fix: Ordering failing when one value is null (#490)

Co-authored-by: Avior <git@avior.me>

.bruno/fixes/489-crash-when-sorting-with-null-values.bru

@@ -0,0 +1,17 @@
+meta {
+  name: 489 - Crash When sorting with Null values
+  type: http
+  seq: 6
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/cards?name=pikachu&sort:field=hp&sort:order=DESC
+  body: none
+  auth: none
+}
+
+query {
+  name: pikachu
+  sort:field: hp
+  sort:order: DESC
+}

.bruno/random/Random Card.bru

@@ -0,0 +1,20 @@
+meta {
+  name: Random Card
+  type: http
+  seq: 1
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/card?name=furret
+  body: none
+  auth: none
+}
+
+query {
+  name: furret
+}
+
+assert {
+  res.status: eq 200
+  res.body.name: contains Furret
+}

.bruno/random/Random Serie.bru

@@ -0,0 +1,19 @@
+meta {
+  name: Random Serie
+  type: http
+  seq: 3
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/serie?name=p
+  body: none
+  auth: none
+}
+
+query {
+  name: p
+}
+
+assert {
+  res.status: eq 200
+}

.bruno/random/Random Set.bru

@@ -0,0 +1,20 @@
+meta {
+  name: Random Set
+  type: http
+  seq: 2
+}
+
+get {
+  url: {{BASE_URL}}/v2/en/random/set?name=sword
+  body: none
+  auth: none
+}
+
+query {
+  name: sword
+}
+
+assert {
+  res.status: eq 200
+  res.body.name: contains Sword
+}

.github/workflows/build.yml

@@ -16,6 +16,8 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Docker meta
       id: meta

Dockerfile

@@ -3,6 +3,10 @@ FROM docker.io/oven/bun:1-alpine as BUILD_IMAGE
 # go to work folder
 WORKDIR /usr/src/app
 
+# Add git as it is used to fetch updated times
+RUN apk add git &&\
+git config --global safe.directory '*'
+
 ADD --chown=bun:bun package.json bun.lockb ./
 ADD --chown=bun:bun server/package.json server/bun.lockb ./server/
 

meta/codeshift.ts

@@ -56,6 +56,20 @@ function simplify(base: ObjectExpression): ObjectField {
 	}
 }
 
+function exists(path: ObjectExpression | ArrayExpression, key: string | number) {
+	if (path.type === 'ObjectExpression') {
+		path.properties.forEach((p) => {
+			const prop = p as Property
+			if ((prop.key as Identifier).name === (key + '')) {
+				return true
+			}
+		})
+		return false
+	} else {
+
+	}
+}
+
 function set(j: JSCodeshift, path: ObjectExpression | ArrayExpression, value: Possible, key: string | number, options?: {override?: boolean}) {
 
 	let exists = false

meta/definitions/api.d.ts

@@ -261,7 +261,9 @@ export interface Card extends CardResume {
 		 * Ability to play in expanded tournaments
 		 */
 		expanded: boolean;
-	};
+	}
+
+	updated: string
 }
 
 /**

server/compiler/index.ts

@@ -2,7 +2,7 @@
 import { promises as fs } from 'fs'
 import { SupportedLanguages } from '../../interfaces'
 import { FileFunction } from './compilerInterfaces'
-import { fetchRemoteFile } from './utils/util'
+import { fetchRemoteFile, loadLastEdits } from './utils/util'
 
 const LANGS: Array<SupportedLanguages> = ['en', 'fr', 'es', 'it', 'pt', 'de']
 
@@ -20,6 +20,8 @@ const DIST_FOLDER = './generated'
 		await fs.rm(DIST_FOLDER, {recursive: true})
 	} catch {}
 
+	console.log('Loading files last edit')
+	await loadLastEdits()
 
 	console.log('Let\'s GO !')
 

server/compiler/utils/cardUtil.ts

@@ -1,9 +1,10 @@
 /* eslint-disable sort-keys */
+import { exec } from 'child_process'
 import { Card, Set, SupportedLanguages, Types } from '../../../interfaces'
 import { CardResume, Card as CardSingle } from '../../../meta/definitions/api'
 import { setToSetSimple } from './setUtil'
 import translate from './translationUtil'
-import { DB_PATH, cardIsLegal, fetchRemoteFile, smartGlob } from './util'
+import { DB_PATH, cardIsLegal, fetchRemoteFile, getLastEdit, smartGlob } from './util'
 
 export async function getCardPictures(cardId: string, card: Card, lang: SupportedLanguages): Promise<string | undefined> {
 	try {
@@ -106,8 +107,8 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
 		legal: {
 			standard: cardIsLegal('standard', card, localId),
 			expanded: cardIsLegal('expanded', card, localId)
-		}
-
+		},
+		updated: await getCardLastEdit(localId, card)
 	}
 }
 
@@ -161,3 +162,8 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
 		return a >= b ? 1 : -1
 	})
 }
+
+async function getCardLastEdit(localId: string, card: Card): Promise<string> {
+	const path = `../data/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
+	return getLastEdit(path)
+}

server/compiler/utils/util.ts

@@ -1,3 +1,4 @@
+import { exec } from 'node:child_process'
 import { glob } from 'glob'
 import { Card, Set } from '../../../interfaces'
 import * as legals from '../../../meta/legals'
@@ -79,3 +80,45 @@ export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean {
 	}
 	return false
 }
+
+function runCommand(command: string): Promise<string> {
+	return new Promise<string>((res, rej) => {
+		exec(command, (err, out) => {
+			if (err) {
+				rej(err)
+				return
+			}
+			res(out)
+		})
+	})
+}
+
+let lastEditsCache: Record<string, string> = {}
+export async function loadLastEdits() {
+	const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
+	const files = (await runCommand(firstCommand)).split('\n')
+	console.log('Loaded files tree', files.length, 'files')
+	console.log('Loading their last edit time')
+	let processed = 0
+	for (let file of files) {
+		file = file.replace(/"/g, '').replace("\\303\\251", "é")
+		try {
+			lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`)
+		} catch {
+			console.warn('could not load file', file, 'hope it does not break everything else lol')
+		}
+		processed++
+		if (processed % 1000 === 0) {
+			console.log('loaded', processed, 'out of', files.length, 'files')
+		}
+	}
+	console.log('done loading files')
+}
+
+export function getLastEdit(path: string): string {
+	const date = lastEditsCache[path]
+	if (!date) {
+		throw new Error(`edit date not found for file ${path}`)
+	}
+	return date
+}

server/src/V2/endpoints/jsonEndpoints.ts

@@ -1,13 +1,21 @@
 import { objectKeys, objectLoop } from '@dzeio/object-util'
 import { Card as SDKCard } from '@tcgdex/sdk'
 import apicache from 'apicache'
-import express from 'express'
+import express, { Request } from 'express'
 import { Query } from '../../interfaces'
 import { betterSorter, checkLanguage, sendError, unique } from '../../util'
 import Card from '../Components/Card'
 import Serie from '../Components/Serie'
 import Set from '../Components/Set'
 
+type CustomRequest = Request & {
+	/**
+	 * disable caching
+	 */
+	DO_NOT_CACHE?: boolean
+	advQuery?: Query<any>
+}
+
 const server = express.Router()
 
 const endpointToField: Record<string, keyof SDKCard> = {
@@ -31,7 +39,7 @@ const endpointToField: Record<string, keyof SDKCard> = {
 
 server
 	// Midleware that handle caching only in production and on GET requests
-	.use(apicache.middleware('1 day', (req: Request, res: Response) => res.status < 400 && process.env.NODE_ENV === 'production' && req.method === 'GET', {}))
+	.use(apicache.middleware('1 day', (req: CustomRequest, res: Response) => !req.DO_NOT_CACHE && res.status < 400 && process.env.NODE_ENV === 'production' && req.method === 'GET', {}))
 
 	// .get('/cache/performance', (req, res) => {
 	// 	res.json(apicache.getPerformance())
@@ -43,14 +51,14 @@ server
 	// })
 
 	// Midleware that handle url transformation
-	.use((req, _, next) => {
+	.use((req: CustomRequest, _, next) => {
 		// this is ugly BUT it fix the problem with + not becoming spaces
 		req.url = req.url.replace(/\+/g, ' ')
 		next()
 	})
 
 	// handle Query builder
-	.use((req, _, next) => {
+	.use((req: CustomRequest, _, next) => {
 		// handle no query
 		if (!req.query) {
 			next()
@@ -77,22 +85,51 @@ server
 
 		})
 
-		// @ts-expect-error normal behavior
 		req.advQuery = items
 
 		next()
 	})
 
+	/**
+	 * Allows the user to fetch a random card/set/serie from the database
+	 */
+	.get('/:lang/random/:what', (req: CustomRequest, res): void => {
+		const { lang, what } = req.params
+
+		if (!checkLanguage(lang)) {
+			return sendError('LanguageNotFoundError', res, lang)
+		}
+
+		const query: Query = req.advQuery!
+
+		let data: Array<Card | Set | Serie> = []
+		switch (what.toLowerCase()) {
+			case 'card':
+				data = Card.find(lang, query)
+				break
+			case 'set':
+				data = Set.find(lang, query)
+				break
+			case 'serie':
+				data = Serie.find(lang, query)
+				break
+			default:
+				return sendError('EndpointNotFoundError', res, what)
+		}
+		const item = Math.min(data.length - 1, Math.max(0, Math.round(Math.random() * data.length)))
+		req.DO_NOT_CACHE = true
+		res.json(data[item])
+	})
+
 
 	/**
 	 * Listing Endpoint
 	 * ex: /v2/en/cards
 	 */
-	.get('/:lang/:endpoint', (req, res): void => {
+	.get('/:lang/:endpoint', (req: CustomRequest, res): void => {
 		let { lang, endpoint } = req.params
 
-		// @ts-expect-error normal behavior
-		const query: Query = req.advQuery
+		const query: Query = req.advQuery!
 
 		if (endpoint.endsWith('.json')) {
 			endpoint = endpoint.replace('.json', '')
@@ -170,7 +207,7 @@ server
 	 * Listing Endpoint
 	 * ex: /v2/en/cards/base1-1
 	 */
-	.get('/:lang/:endpoint/:id', (req, res) => {
+	.get('/:lang/:endpoint/:id', (req: CustomRequest, res) => {
 		let { id, lang, endpoint } = req.params
 
 		if (id.endsWith('.json')) {
@@ -223,7 +260,7 @@ server
 	 * sub id Endpoint (for the set endpoint only currently)
 	 * ex: /v2/en/sets/base1/1
 	 */
-	.get('/:lang/:endpoint/:id/:subid', (req, res) => {
+	.get('/:lang/:endpoint/:id/:subid', (req: CustomRequest, res) => {
 		let { id, lang, endpoint, subid } = req.params
 
 		if (subid.endsWith('.json')) {

server/src/util.ts

@@ -150,9 +150,21 @@ export function handleSort(data: Array<any>, query: Query<any>) {
  * @param order the base ordering
  * @returns a function that is feed in the `sort` function
  */
-const advSort = (a: string | number, b: string | number, order: 'ASC' | 'DESC' = 'ASC') => {
-	a = tryParse(a) ?? a
-	b = tryParse(b) ?? b
+const advSort = (a?: string | number, b?: string | number, order: 'ASC' | 'DESC' = 'ASC') => {
+	const isANull = isNull(a)
+	const isBNull = isNull(b)
+	if (isANull && isBNull) {
+		return 0
+	}
+	if (isANull) {
+		return order === 'ASC' ? -1 : 1
+	}
+	if (isBNull) {
+		return order === 'ASC' ? 1 : -1
+	}
+
+	a = tryParse(a!) ?? a
+	b = tryParse(b!) ?? b
 
 	if (order === 'DESC') {
 		const tmp = a
@@ -164,7 +176,7 @@ const advSort = (a: string | number, b: string | number, order: 'ASC' | 'DESC' =
 		return a - b
 	}
 
-	return a.toString().localeCompare(b.toString())
+	return a!.toString().localeCompare(b!.toString())
 }
 
 function tryParse(value: string | number): number | null {