TCGdex/cards-database
1
0
Fork 0
mirror of https://github.com/tcgdex/cards-database.git synced 2025-07-29 03:09:49 +00:00
Code Issues Releases Wiki Activity

feat: Add support for Asians Pokémon Cards (#481)

Browse Source
This commit is contained in:
Florian Bouillon
2024-06-07 12:53:08 +02:00
committed by GitHub
parent a35fadd50c
commit a26ef0e5eb
8069 changed files with 379200 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
server/compiler/endpoints/cards.ts
View File

@ -4,7 +4,10 @@ import { cardToCardSingle, getCards } from '../utils/cardUtil'
const fn: FileFunction = async (lang: SupportedLanguages) => {
const common = await getCards(lang)
return await Promise.all(common.map((card) => cardToCardSingle(card[0], card[1], lang)))
return await Promise.all(common.map((card) => cardToCardSingle(card[0], card[1], lang).catch((e) => {
console.log('error compiling card', `${card[1].set.id}-${card[0]}`)
throw e
})))
}
export default fn

17
server/compiler/endpoints/stats.ts
View File

@ -1,8 +1,9 @@
import { getSets, setToSetSingle } from '../utils/setUtil'
import { SupportedLanguages } from '../../../interfaces'
import { Set } from '../../../meta/definitions/api'
import { FileFunction } from '../compilerInterfaces'
import { getCards } from '../utils/cardUtil'
import { getSeries } from '../utils/serieUtil'
import { getSets, setToSetSingle } from '../utils/setUtil'
interface Stats {
count: number
@ -14,10 +15,20 @@ interface Stats {
const fn: FileFunction = async (lang: SupportedLanguages) => {
const stats: Partial<Stats> = {}
stats.count = (await getCards(lang)).length
// temporary fix until a better solution is found
const referenceLang = ['ja', 'ko', 'zh-tw', 'id', 'th', 'zh-cn'].includes(lang) ? 'ja' : 'en'
const langSets = await Promise.all(await getSets(undefined, lang).then((sets) => sets.map(async (set) => await setToSetSingle(set, lang))))
const englishSets = await Promise.all(await getSets(undefined, 'en').then((sets) => sets.map(async (set) => await setToSetSingle(set, 'en'))))
stats.total = langSets.reduce((p, set) => p + (englishSets.find((s) => set.id === s.id)?.cardCount?.total ?? 0), 0)
const englishSets = await Promise.all(await getSets(undefined, referenceLang).then((sets) => sets.map(async (set) => await setToSetSingle(set, referenceLang))))
function max(lSet?: Set, refSet?: Set) {
if (!lSet) return refSet!.cardCount.total
if (!refSet) return lSet.cardCount.total
if (lSet.cardCount.total > refSet.cardCount.total) {
return lSet.cardCount.total
}
return refSet.cardCount.total
}
stats.total = langSets.reduce((p, set) => p + max(set, englishSets.find((s) => set.id === s.id)), 0)
stats.images = langSets.reduce((p1, set) => p1 + (set.cards.reduce((p2, card) => p2 + (card.image ? 1 : 0), 0)), 0)
stats.sets = {}

18
server/compiler/index.ts
View File

@ -4,7 +4,10 @@ import { SupportedLanguages } from '../../interfaces'
import { FileFunction } from './compilerInterfaces'
import { fetchRemoteFile, loadLastEdits } from './utils/util'
const LANGS: Array<SupportedLanguages> = ['en', 'fr', 'es', 'it', 'pt', 'de']
const LANGS: Array<SupportedLanguages> = [
'en', 'fr', 'es', 'it', 'pt', 'pt-br', 'pt-pt', 'de', 'nl', 'pl', 'ru',
'ja', 'ko', 'zh-tw', 'id', 'th', 'zh-cn'
]
const DIST_FOLDER = './generated'
@ -12,7 +15,7 @@ const DIST_FOLDER = './generated'
const paths = (await fs.readdir('./compiler/endpoints')).filter((p) => p.endsWith('.ts'))
// Prefetch the pictures at the start as it can bug because of bad connection
console.log('Prefetching pictures')
console.log('1. Prefetching pictures datas')
await fetchRemoteFile('https://assets.tcgdex.net/datas.json')
// Delete dist folder to be sure to have a clean base
@ -20,14 +23,14 @@ const DIST_FOLDER = './generated'
await fs.rm(DIST_FOLDER, {recursive: true})
} catch {}
console.log('Loading files last edit')
console.log('\n2. Loading informations from GIT')
await loadLastEdits()
console.log('Let\'s GO !')
console.log('\n3. Compiling Files')
// Process each languages
let progressIndex = 0
for await (const lang of LANGS) {
console.log('Processing', lang)
// loop through """endpoints"""
for await (const file of paths) {
@ -41,7 +44,7 @@ const DIST_FOLDER = './generated'
const fn = (await import(`./endpoints/${file}`)).default as FileFunction
// Run the function
console.log(file, 'Running...')
console.log(' ', 'Compiling', lang, file)
const item = await fn(lang)
// Write to file
@ -49,10 +52,11 @@ const DIST_FOLDER = './generated'
item
))
console.log(file, 'Finished!')
console.log(`${(++progressIndex / (LANGS.length * paths.length) * 100).toFixed(2).padStart(5, '0')}%`, 'Compiled ', lang, file)
}
}
console.log('4. Copying static files to public folder')
// Finally copy definitions files to the public folder :D
for await (const file of await fs.readdir('../meta/definitions')) {
await fs.copyFile('../meta/definitions/' + file, './public/v2/' + file)

51
server/compiler/utils/cardUtil.ts
View File

@ -2,9 +2,9 @@
import { exec } from 'child_process'
import { Card, Set, SupportedLanguages, Types } from '../../../interfaces'
import { CardResume, Card as CardSingle } from '../../../meta/definitions/api'
import { setToSetSimple } from './setUtil'
import { getSet, setToSetSimple } from './setUtil'
import translate from './translationUtil'
import { DB_PATH, cardIsLegal, fetchRemoteFile, getLastEdit, smartGlob } from './util'
import { DB_PATH, cardIsLegal, fetchRemoteFile, getDataFolder, getLastEdit, smartGlob } from './util'
export async function getCardPictures(cardId: string, card: Card, lang: SupportedLanguages): Promise<string | undefined> {
try {
@ -108,7 +108,7 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
standard: cardIsLegal('standard', card, localId),
expanded: cardIsLegal('expanded', card, localId)
},
updated: await getCardLastEdit(localId, card)
updated: await getCardLastEdit(localId, card, lang)
}
}
@ -118,8 +118,12 @@ export async function cardToCardSingle(localId: string, card: Card, lang: Suppor
* @param id the local id of the card
* @returns [the local id, the Card object]
*/
export async function getCard(serie: string, setName: string, id: string): Promise<Card> {
return (await import(`../../${DB_PATH}/data/${serie}/${setName}/${id}.ts`)).default
export async function getCard(set: Set, id: string, lang: SupportedLanguages): Promise<Card> {
try {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${set.serie.name.en ?? set.serie.name[lang]}/${set.name.en ?? set.name[lang]}/${id}.ts`)).default
} catch {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${set.serie.id}/${set.id}/${id}.ts`)).default
}
}
/**
@ -129,7 +133,10 @@ export async function getCard(serie: string, setName: string, id: string): Promi
* @returns An array with the 0 = localId, 1 = Card Object
*/
export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Array<[string, Card]>> {
const cards = await smartGlob(`${DB_PATH}/data/${(set && set.serie.name.en) ?? '*'}/${(set && set.name.en) ?? '*'}/*.ts`)
let cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && (set.serie.name.en ?? set.serie.name[lang])) ?? '*'}/${(set && (set.name.en ?? set.name[lang])) ?? '*'}/*.ts`)
if (cards.length === 0) {
cards = await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${(set && set.serie.id) ?? '*'}/${(set && set.id) ?? '*'}/*.ts`)
}
const list: Array<[string, Card]> = []
for (const path of cards) {
let items = path.split('/')
@ -140,12 +147,19 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
id = id.substring(0, id.lastIndexOf('.'))
// get it's set name
const setName = (set && set.name.en) ?? items[1]
const setName = items[1]
// get it's serie name
const serieName = (set && set.serie.name.en) ?? items[0]
// console.log(path, id, setName)
const c = await getCard(serieName, setName, id)
const serieName = items[0]
const set = await getSet(setName, serieName, lang)
if (!(lang in set.name)) {
continue
}
// console.log(path, id, set, lang)
const c = await getCard(set, id, lang)
if (!c.name[lang]) {
continue
}
@ -163,7 +177,18 @@ export async function getCards(lang: SupportedLanguages, set?: Set): Promise<Arr
})
}
async function getCardLastEdit(localId: string, card: Card): Promise<string> {
const path = `../data/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
return getLastEdit(path)
export async function getCardLastEdit(localId: string, card: Card, lang: SupportedLanguages): Promise<string> {
try {
const path = `../${getDataFolder(lang)}/${card.set.serie.name.en}/${card.set.name.en ?? card.set.name.fr}/${localId}.ts`
return getLastEdit(path)
} catch (e) {
try {
const path = `../${getDataFolder(lang)}/${card.set.serie.id}/${card.set.id}/${localId}.ts`
return getLastEdit(path)
} catch (e2) {
console.error(card)
console.error(e)
throw e2
}
}
}

10
server/compiler/utils/serieUtil.ts
View File

@ -1,10 +1,10 @@
import { Serie, Set, SupportedLanguages } from '../../../interfaces'
import { SerieResume, Serie as SerieSingle } from '../../../meta/definitions/api'
import { getSets, setToSetSimple } from './setUtil'
import { DB_PATH, smartGlob } from './util'
import { DB_PATH, getDataFolder, smartGlob } from './util'
export async function getSerie(name: string): Promise<Serie> {
return (await import(`../../${DB_PATH}/data/${name}.ts`)).default
export async function getSerie(name: string, lang: SupportedLanguages): Promise<Serie> {
return (await import(`../../${DB_PATH}/${getDataFolder(lang)}/${name}.ts`)).default
}
export async function isSerieAvailable(serie: Serie, lang: SupportedLanguages): Promise<boolean> {
@ -16,11 +16,11 @@ export async function isSerieAvailable(serie: Serie, lang: SupportedLanguages):
}
export async function getSeries(lang: SupportedLanguages): Promise<Array<Serie>> {
let series: Array<Serie> = (await Promise.all((await smartGlob(`${DB_PATH}/data/*.ts`))
let series: Array<Serie> = (await Promise.all((await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/*.ts`))
// Find Serie's name
.map((it) => it.substring(it.lastIndexOf('/') + 1, it.length - 3))
// Fetch the Serie
.map((it) => getSerie(it))))
.map((it) => getSerie(it, lang))))
// Filter the serie if no name's exists in the selected lang
.filter((serie) => Boolean(serie.name[lang]))

17
server/compiler/utils/setUtil.ts
View File

@ -1,7 +1,8 @@
import { objectKeys } from '@dzeio/object-util'
import { Set, SupportedLanguages } from '../../../interfaces'
import { SetResume, Set as SetSingle } from '../../../meta/definitions/api'
import { cardToCardSimple, getCards } from './cardUtil'
import { DB_PATH, fetchRemoteFile, setIsLegal, smartGlob } from './util'
import { DB_PATH, fetchRemoteFile, getDataFolder, setIsLegal, smartGlob } from './util'
interface t {
[key: string]: Set
@ -17,14 +18,16 @@ export function isSetAvailable(set: Set, lang: SupportedLanguages): boolean {
* Return the set
* @param name the name of the set
*/
export async function getSet(name: string, serie = '*'): Promise<Set> {
export async function getSet(name: string, serie = '*', lang: SupportedLanguages): Promise<Set> {
if (!setCache[name]) {
const file = `${DB_PATH}/${getDataFolder(lang)}/${serie}/${name}.ts`
try {
const [path] = await smartGlob(`${DB_PATH}/data/${serie}/${name}.ts`)
const [path] = await smartGlob(file)
// console.log(`${DB_PATH}/${getDataFolder(lang)}/${serie}/${name}.ts`)
setCache[name] = (await import(`../../${path}`)).default
} catch (error) {
console.error(error)
console.error(`Error trying to import importing (${`db/data/${serie}/${name}.ts`})`)
console.error(`Error trying to import importing (${file})`)
process.exit(1)
}
}
@ -34,9 +37,9 @@ export async function getSet(name: string, serie = '*'): Promise<Set> {
// Dont use cache as it wont necessary have them all
export async function getSets(serie = '*', lang: SupportedLanguages): Promise<Array<Set>> {
// list sets names
const rawSets = (await smartGlob(`${DB_PATH}/data/${serie}/*.ts`)).map((set) => set.substring(set.lastIndexOf('/') + 1, set.lastIndexOf('.')))
const rawSets = (await smartGlob(`${DB_PATH}/${getDataFolder(lang)}/${serie}/*.ts`)).map((set) => set.substring(set.lastIndexOf('/') + 1, set.lastIndexOf('.')))
// Fetch sets
const sets = (await Promise.all(rawSets.map((set) => getSet(set, serie))))
const sets = (await Promise.all(rawSets.map((set) => getSet(set, serie, lang))))
// Filter sets
.filter((set) => isSetAvailable(set, lang))
// Sort sets by release date
@ -93,7 +96,7 @@ export async function setToSetSingle(set: Set, lang: SupportedLanguages): Promis
},
logo: pics[0],
name: set.name[lang] as string,
releaseDate: set.releaseDate,
releaseDate: typeof set.releaseDate === 'object' ? set.releaseDate[lang] ?? set.releaseDate[objectKeys(set.releaseDate)[0]]! : set.releaseDate,
serie: {
id: set.serie.id,
name: set.serie.name[lang] as string

55
server/compiler/utils/util.ts
View File

@ -1,6 +1,8 @@
import { exec } from 'node:child_process'
import { objectSize } from '@dzeio/object-util'
import { glob } from 'glob'
import { Card, Set } from '../../../interfaces'
import { exec, spawn } from 'node:child_process'
import { writeFileSync } from 'node:fs'
import { Card, Set, SupportedLanguages } from '../../../interfaces'
import * as legals from '../../../meta/legals'
interface fileCacheInterface {
@ -81,11 +83,42 @@ export function setIsLegal(type: 'standard' | 'expanded', set: Set): boolean {
return false
}
function runCommand(command: string): Promise<string> {
export function getDataFolder(lang: SupportedLanguages) {
return ['ja', 'ko', 'zh-tw', 'id', 'th', 'zh-cn'].includes(lang) ? 'data-asia' : 'data'
}
/**
* run a command on the OS, it uses Spawn by default because exec seems to have a bug linked to the Buffer
*
* @param command the command to run
* @param useSpawn select the method to use to run the command
* @returns a string with the stdout
*/
function runCommand(command: string, useSpawn = true): Promise<string> {
if (!useSpawn) {
return new Promise<string>((res, rej) => {
exec(command, (err, out) => {
if (err) {
rej(err)
}
res(out)
})
})
}
const splitted = command.split(' ')
command = splitted.shift()!
return new Promise<string>((res, rej) => {
exec(command, (err, out) => {
if (err) {
rej(err)
const cmd = spawn(command, splitted)
let out: string = ''
cmd.stdout.on('data', (data) => {
out += data.toString()
})
cmd.on('close', (code) => {
if (code !== 0) {
console.log(`command exited with code ${code}`);
rej(code)
return
}
res(out)
@ -95,24 +128,28 @@ function runCommand(command: string): Promise<string> {
let lastEditsCache: Record<string, string> = {}
export async function loadLastEdits() {
console.log('Loading Git File Tree...')
const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
const files = (await runCommand(firstCommand)).split('\n')
const secondCommand = 'git ls-tree -r --name-only HEAD ../data-asia'
files.push(...(await runCommand(secondCommand)).split('\n'))
console.log('Loaded files tree', files.length, 'files')
console.log('Loading their last edit time')
let processed = 0
for (let file of files) {
file = file.replace(/"/g, '').replace("\\303\\251", "é")
try {
lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`)
// don't really know why but it does not correctly execute the command when using Spawn
lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false)
} catch {
console.warn('could not load file', file, 'hope it does not break everything else lol')
}
processed++
if (processed % 1000 === 0) {
console.log('loaded', processed, 'out of', files.length, 'files')
console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
}
}
console.log('done loading files')
console.log('done loading files', objectSize(lastEditsCache))
}
export function getLastEdit(path: string): string {