TCGdex/cards-database
1
0
Fork 0
mirror of https://github.com/tcgdex/cards-database.git synced 2025-06-15 00:49:18 +00:00
Code Issues Releases Wiki Activity

feat: Add support for Asians Pokémon Cards (#481)

Browse Source
This commit is contained in:
Florian Bouillon
2024-06-07 12:53:08 +02:00
committed by GitHub
parent a35fadd50c
commit a26ef0e5eb
8069 changed files with 379200 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
server/compiler/utils/cardUtil.ts
View File

@ -2,9 +2,9 @@
import { exec } from 'child_process'
import { Card, Set, SupportedLanguages, Types } from '../../../interfaces'
import { CardResume, Card as CardSingle } from '../../../meta/definitions/api'
import { setToSetSimple } from './setUtil'
import { getSet, setToSetSimple } from './setUtil'
import translate from './translationUtil'
import { DB_PATH, cardIsLegal, fetchRemoteFile, getLastEdit, smartGlob } from './util'
import { DB_PATH, cardIsLegal, fetchRemoteFile, getDataFolder, getLastEdit, smartGlob } from './util'
export async function getCardPictures(cardId: string, card: Card, lang: SupportedLanguages): Promise<string | undefined> {
try {
@ -108,7 +108,7 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
standard: cardIsLegal('standard', card, localId),
expanded: cardIsLegal('expanded', card, localId)
},
updated: await getCardLastEdit(localId, card)
updated: await getCardLastEdit(localId, card, lang)
}
}
@ -118,8 +118,12 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
* @param id the local id of the card
* @returns [the local id, the Card object]
*/
export async function getCard(serie: string, setName: string, id: string): Promise<Card> {
return (await import(`../../${DB_PATH}/data/${serie}/${setName}/${id}.ts`)).default
export async function getCard(set: Set, id: string, lang: SupportedLanguages): Promise<Card> {
try {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${set.serie.name.en ?? set.serie.name[lang]}/${set.name.en ?? set.name[lang]}/${id}.ts`)).default
} catch {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${set.serie.id}/${set.id}/${id}.ts`)).default
}
}
/**
@ -129,7 +133,10 @@ export async function getCard(serie: string, setName: string, id: string): Promi
* @returns An array with the 0 = localId, 1 = Card Object
*/
export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Array<[string, Card]>> {
const cards = await smartGlob(`${DB_PATH}/data/${(set && set.serie.name.en) ?? '*'}/${(set && set.name.en) ?? '*'}/*.ts`)
let cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && (set.serie.name.en ?? set.serie.name[lang])) ?? '*'}/${(set && (set.name.en ?? set.name[lang])) ?? '*'}/*.ts`)
if (cards.length === 0) {
cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && set.serie.id) ?? '*'}/${(set && set.id) ?? '*'}/*.ts`)
}
const list: Array<[string, Card]> = []
for (const path of cards) {
let items = path.split('/')
@ -140,12 +147,19 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
id = id.substring(0, id.lastIndexOf('.'))
// get it's set name
const setName = (set && set.name.en) ?? items[1]
const setName = items[1]
// get it's serie name
const serieName = (set && set.serie.name.en) ?? items[0]
// console.log(path, id, setName)
const c = await getCard(serieName, setName, id)
const serieName = items[0]
const set = await getSet(setName, serieName, lang)
if (!(lang in set.name)) {
continue
}
// console.log(path, id, set, lang)
const c = await getCard(set, id, lang)
if (!c.name[lang]) {
continue
}
@ -163,7 +177,18 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
})
}
async function getCardLastEdit(localId: string, card: Card): Promise<string> {
const path = `../data/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
return getLastEdit(path)
export async function getCardLastEdit(localId: string, card: Card, lang: SupportedLanguages): Promise<string> {
try {
const path = `../${getDataFolder(lang)}/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
return getLastEdit(path)
} catch (e) {
try {
const path = `../${getDataFolder(lang)}/${card.set.serie.id}/${card.set.id}/${localId}.ts`
return getLastEdit(path)
} catch (e2) {
console.error(card)
console.error(e)
throw e2
}
}
}

10
server/compiler/utils/serieUtil.ts
View File

@ -1,10 +1,10 @@
import { Serie, Set, SupportedLanguages } from '../../../interfaces'
import { SerieResume, Serie as SerieSingle } from '../../../meta/definitions/api'
import { getSets, setToSetSimple } from './setUtil'
import { DB_PATH, smartGlob } from './util'
import { DB_PATH, getDataFolder, smartGlob } from './util'
export async function getSerie(name: string): Promise<Serie> {
return (await import(`../../${DB_PATH}/data/${name}.ts`)).default
export async function getSerie(name: string, lang: SupportedLanguages): Promise<Serie> {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${name}.ts`)).default
}
export async function isSerieAvailable(serie: Serie, lang: SupportedLanguages): Promise<boolean> {
@ -16,11 +16,11 @@ export async function isSerieAvailable(serie: Serie, lang: SupportedLanguages):
}
export async function getSeries(lang: SupportedLanguages): Promise<Array<Serie>> {
let series: Array<Serie> = (await Promise.all((await smartGlob(`${DB_PATH}/data/*.ts`))
let series: Array<Serie> = (await Promise.all((await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/*.ts`))
// Find Serie's name
.map((it) => it.substring(it.lastIndexOf('/') + 1, it.length - 3))
// Fetch the Serie
.map((it) => getSerie(it))))
.map((it) => getSerie(it, lang))))
// Filter the serie if no name's exists in the selected lang
.filter((serie) => Boolean(serie.name[lang]))

17
server/compiler/utils/setUtil.ts
View File

@ -1,7 +1,8 @@
import { objectKeys } from '@dzeio/object-util'
import { Set, SupportedLanguages } from '../../../interfaces'
import { SetResume, Set as SetSingle } from '../../../meta/definitions/api'
import { cardToCardSimple, getCards } from './cardUtil'
import { DB_PATH, fetchRemoteFile, setIsLegal, smartGlob } from './util'
import { DB_PATH, fetchRemoteFile, getDataFolder, setIsLegal, smartGlob } from './util'
interface t {
[key: string]: Set
@ -17,14 +18,16 @@ export function isSetAvailable(set: Set, lang: SupportedLanguages): boolean {
* Return the set
* @param name the name of the set
*/
export async function getSet(name: string, serie = '*'): Promise<Set> {
export async function getSet(name: string, serie = '*', lang: SupportedLanguages): Promise<Set> {
if (!setCache[name]) {
const file = `${DB_PATH}/${getDataFolder(lang)}/${serie}/${name}.ts`
try {
const [path] = await smartGlob(`${DB_PATH}/data/${serie}/${name}.ts`)
const [path] = await smartGlob(file)
// console.log(`${DB_PATH}/${getDataFolder(lang)}/${serie}/${name}.ts`)
setCache[name] = (await import(`../../${path}`)).default
} catch (error) {
console.error(error)
console.error(`Error trying to import importing (${`db/data/${serie}/${name}.ts`})`)
console.error(`Error trying to import importing (${file})`)
process.exit(1)
}
}
@ -34,9 +37,9 @@ export async function getSet(name: string, serie = '*'): Promise<Set> {
// Dont use cache as it wont necessary have them all
export async function getSets(serie = '*', lang: SupportedLanguages): Promise<Array<Set>> {
// list sets names
const rawSets = (await smartGlob(`${DB_PATH}/data/${serie}/*.ts`)).map((set) => set.substring(set.lastIndexOf('/') + 1, set.lastIndexOf('.')))
const rawSets = (await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${serie}/*.ts`)).map((set) => set.substring(set.lastIndexOf('/') + 1, set.lastIndexOf('.')))
// Fetch sets
const sets = (await Promise.all(rawSets.map((set) => getSet(set, serie))))
const sets = (await Promise.all(rawSets.map((set) => getSet(set, serie, lang))))
// Filter sets
.filter((set) => isSetAvailable(set, lang))
// Sort sets by release date
@ -93,7 +96,7 @@ export async function setToSetSingle(set: Set, lang: SupportedLanguages): Promis
},
logo: pics[0],
name: set.name[lang] as string,
releaseDate: set.releaseDate,
releaseDate: typeof set.releaseDate === 'object' ? set.releaseDate[lang] ?? set.releaseDate[objectKeys(set.releaseDate)[0]]! : set.releaseDate,
serie: {
id: set.serie.id,
name: set.serie.name[lang] as string

55
server/compiler/utils/util.ts
View File

@ -1,6 +1,8 @@
import { exec } from 'node:child_process'
import { objectSize } from '@dzeio/object-util'
import { glob } from 'glob'
import { Card, Set } from '../../../interfaces'
import { exec, spawn } from 'node:child_process'
import { writeFileSync } from 'node:fs'
import { Card, Set, SupportedLanguages } from '../../../interfaces'
import * as legals from '../../../meta/legals'
interface fileCacheInterface {
@ -81,11 +83,42 @@ export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean {
return false
}
function runCommand(command: string): Promise<string> {
export function getDataFolder(lang: SupportedLanguages) {
return ['ja', 'ko', 'zh-tw', 'id', 'th', 'zh-cn'].includes(lang) ? 'data-asia' : 'data'
}
/**
* run a command on the OS, it uses Spawn by default because exec seems to have a bug linked to the Buffer
*
* @param command the command to run
* @param useSpawn select the method to use to run the command
* @returns a string with the stdout
*/
function runCommand(command: string, useSpawn = true): Promise<string> {
if (!useSpawn) {
return new Promise<string>((res, rej) => {
exec(command, (err, out) => {
if (err) {
rej(err)
}
res(out)
})
})
}
const splitted = command.split(' ')
command = splitted.shift()!
return new Promise<string>((res, rej) => {
exec(command, (err, out) => {
if (err) {
rej(err)
const cmd = spawn(command, splitted)
let out: string = ''
cmd.stdout.on('data', (data) => {
out += data.toString()
})
cmd.on('close', (code) => {
if (code !== 0) {
console.log(`command exited with code ${code}`);
rej(code)
return
}
res(out)
@ -95,24 +128,28 @@ function runCommand(command: string): Promise<string> {
let lastEditsCache: Record<string, string> = {}
export async function loadLastEdits() {
console.log('Loading Git File Tree...')
const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
const files = (await runCommand(firstCommand)).split('\n')
const secondCommand = 'git ls-tree -r --name-only HEAD ../data-asia'
files.push(...(await runCommand(secondCommand)).split('\n'))
console.log('Loaded files tree', files.length, 'files')
console.log('Loading their last edit time')
let processed = 0
for (let file of files) {
file = file.replace(/"/g, '').replace("\\303\\251", "é")
try {
lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`)
// don't really know why but it does not correctly execute the command when using Spawn
lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false)
} catch {
console.warn('could not load file', file, 'hope it does not break everything else lol')
}
processed++
if (processed % 1000 === 0) {
console.log('loaded', processed, 'out of', files.length, 'files')
console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
}
}
console.log('done loading files')
console.log('done loading files', objectSize(lastEditsCache))
}
export function getLastEdit(path: string): string {