From 7809913d18c3a6edb7d065f92851056a758d77d9 Mon Sep 17 00:00:00 2001 From: Florian BOUILLON Date: Wed, 3 Feb 2021 14:02:44 +0100 Subject: [PATCH] Updated Signed-off-by: Florian BOUILLON --- .drone.yml | 55 ------------------------- .gitignore | 2 + all.ts | 4 +- endpoints/cardUtil.ts | 14 +++---- endpoints/cards/index.ts | 18 +++++---- endpoints/cards/item.ts | 17 ++++---- endpoints/categories/index.ts | 8 ++-- endpoints/categories/item.ts | 15 ++++--- endpoints/expansionUtil.ts | 38 +++++++----------- endpoints/expansions/index.ts | 12 +++--- endpoints/expansions/item.ts | 14 ++++--- endpoints/hp/index.ts | 15 ++++--- endpoints/hp/item.ts | 14 ++++--- endpoints/illustrators/index.ts | 7 +++- endpoints/illustrators/item.ts | 14 ++++--- endpoints/illustrators/updateDB.ts | 7 ++-- endpoints/rarities/index.ts | 10 ++--- endpoints/rarities/item.ts | 15 +++---- endpoints/retreat/index.ts | 18 ++++----- endpoints/retreat/item.ts | 18 ++++----- endpoints/setUtil.ts | 21 +++++----- endpoints/sets/index.ts | 16 ++++---- endpoints/sets/item.ts | 17 ++++---- endpoints/sets/subitem.ts | 32 ++++++++------- endpoints/tags/index.ts | 4 +- endpoints/tags/item.ts | 11 +++-- endpoints/types/index.ts | 9 +++-- endpoints/types/item.ts | 51 ++++++++++++------------ endpoints/util.ts | 64 +++++++----------------------- 29 files changed, 242 insertions(+), 298 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 968fe9b..0000000 --- a/.drone.yml +++ /dev/null @@ -1,55 +0,0 @@ -kind: pipeline -name: default -type: docker - -steps: - - name: Submodules - image: alpine/git - commands: - - git submodule update --init --recursive - - git submodule foreach git checkout master - - git submodule foreach git pull origin master - - - name: Prepare - image: node:alpine - commands: - - apk -q add yarn - - yarn - - yarn db:compile - - - name: Compile - image: node:alpine - commands: - - apk -q add yarn - - rm -r dist/* - - yarn gen - - CARDLANG=fr yarn gen - - - name: Push - image: alpine/git - commands: - - cd dist - - git config --global user.email "$PUSH_EMAIL" - - git config --global user.name "$PUSH_NAME" - - git add . - - git commit -m "Updated API" - - git push -u origin master - environment: - PUSH_EMAIL: - from_secret: PUSH_EMAIL - PUSH_NAME: - from_secret: PUSH_NAME - - - name: Upload - image: node:alpine - commands: - - node upload.js - environment: - UPLOAD_USERNAME: - from_secret: UPLOAD_USERNAME - UPLOAD_PASSWORD: - from_secret: UPLOAD_PASSWORD - UPLOAD_DIST: - from_secret: UPLOAD_DIST - UPLOAD_REMOTE: - from_secret: UPLOAD_REMOTE diff --git a/.gitignore b/.gitignore index 3c3629e..4c96ead 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules +db +dist \ No newline at end of file diff --git a/all.ts b/all.ts index c238833..e81be62 100644 --- a/all.ts +++ b/all.ts @@ -45,7 +45,7 @@ import { fetchRemoteFile } from './endpoints/util' fetchRemoteFile('https://assets.tcgdex.net/data-fr.json'), fetchRemoteFile('https://assets.tcgdex.net/data-univ.json') ]) - logger.log('UPDATING...') + logger.log('Updating...') await Promise.all([ cardIndex(), cardItem(), @@ -61,12 +61,10 @@ import { fetchRemoteFile } from './endpoints/util' illustratorsIndex(), illustratorsItem(), - illustratorsDB(), raritiesIndex(), raritiesItem(), - retreatIndex(), retreatItem(), diff --git a/endpoints/cardUtil.ts b/endpoints/cardUtil.ts index 3282a5f..bdb78f2 100644 --- a/endpoints/cardUtil.ts +++ b/endpoints/cardUtil.ts @@ -22,7 +22,7 @@ type RemoteData = ObjectList>> export async function cardToCardSimple(card: Card, lang: Langs): Promise { let image: string = undefined const file: RemoteData = await fetchRemoteFile(`https://assets.tcgdex.net/data-${lang}.json`) - const expansion = getExpansionFromSetName(card.set.code) + const expansion = await getExpansionFromSetName(card.set.code) if (file[expansion.code] && file[expansion.code][card.set.code] && file[expansion.code][card.set.code][card.localId]) { const basePath = `https://assets.tcgdex.net/${lang}/${expansion.code}/${card.set.code}/${card.localId}` image = `${basePath}/low` @@ -36,10 +36,6 @@ export async function cardToCardSimple(card: Card, lang: Langs): Promise { let images: { @@ -48,7 +44,7 @@ export async function cardToCardSingle(card: Card, lang: Langs): Promise el[lang]), - tags: card.tags.map((el) => tagToTagSimple(el, lang)), + tags: card.tags?.map((el) => tagToTagSimple(el, lang)), illustrator: card.illustrator && { id: fetchIllustratorsSync().indexOf(card.illustrator), name: card.illustrator, @@ -101,9 +97,9 @@ export async function cardToCardSingle(card: Card, lang: Langs): Promise { if (!(lang in card.name)) return false - const set = getSet(card) + const set = await getSet(card) if ("availability" in set && (lang in set.availability)) { return set.availability[lang] } diff --git a/endpoints/cards/index.ts b/endpoints/cards/index.ts index 023b238..bc2c023 100644 --- a/endpoints/cards/index.ts +++ b/endpoints/cards/index.ts @@ -3,21 +3,24 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' import { CardList, CardSimple } from "@tcgdex/sdk/interfaces/Card" import { cardToCardSimple, isCardAvailable } from "../cardUtil" -import { getBaseFolder, getAllCards2 } from "../util" +import { getBaseFolder, getAllCards2, getAllCards } from "../util" + +import Logger from '@dzeio/logger' +const logger = new Logger('cards/index') + const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "cards") export default async () => { - console.log(endpoint) - const list = await getAllCards2() + logger.log('Fetching Cards') + const list = await getAllCards() const items: Array = [] for (let el of list) { - el = el.replace("./", "../../") - const card: Card = require(el).default + const card: Card = (await import(el)).default - if (!isCardAvailable(card, lang)) continue + if (!(await isCardAvailable(card, lang))) continue items.push( await cardToCardSimple(card, lang) ) @@ -29,9 +32,10 @@ export default async () => { count: items.length, list: items } + logger.log('Writing') await fs.mkdir(`${endpoint}`, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(cardList)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/cards/item.ts b/endpoints/cards/item.ts index 96e5575..8118d44 100644 --- a/endpoints/cards/item.ts +++ b/endpoints/cards/item.ts @@ -1,21 +1,24 @@ -import { getAllCards2, getBaseFolder } from "..//util" +import { getAllCards, getAllCards2, getBaseFolder } from "..//util" import Card from "@tcgdex/sdk/interfaces/Card" import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' import { cardToCardSingle, isCardAvailable } from "../cardUtil" +import Logger from '@dzeio/logger' +const logger = new Logger('cards/item') + + const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "cards") export default async () => { - console.log(endpoint) - const list = await getAllCards2() + logger.log('Fetching Cards') + const list = await getAllCards() for (let el of list) { - el = el.replace("./", "../../") - const card: Card = require(el).default + const card: Card = (await import(el)).default - if (!isCardAvailable(card, lang)) continue + if (!(await isCardAvailable(card, lang))) continue try { await fs.mkdir(`${endpoint}/${encodeURI(card.id)}/`, {recursive: true}) @@ -26,5 +29,5 @@ export default async () => { // if (if (typeof card.set.availability === "undefined")) } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/categories/index.ts b/endpoints/categories/index.ts index 8c9e091..a932a20 100644 --- a/endpoints/categories/index.ts +++ b/endpoints/categories/index.ts @@ -3,15 +3,16 @@ import Category, { CategorySimple, CategoryList } from '@tcgdex/sdk/interfaces/C import TranslationUtil from "@tcgdex/sdk/TranslationUtil" import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' + import Logger from '@dzeio/logger' +const logger = new Logger('category/index') const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "categories") -const logger = new Logger('Category/Index') export default async () => { - logger.log(endpoint) + logger.log('Fetching Categories') const list: Array = [] for (const cat of Object.values(Category)) { @@ -26,8 +27,9 @@ export default async () => { count: list.length, list: list } + logger.log('Writing') await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res)) - logger.log('ended') + logger.log('Finished') } diff --git a/endpoints/categories/item.ts b/endpoints/categories/item.ts index 9a679af..e590158 100644 --- a/endpoints/categories/item.ts +++ b/endpoints/categories/item.ts @@ -6,6 +6,9 @@ import TranslationUtil from "@tcgdex/sdk/TranslationUtil" import { promises } from "fs" import Category, { CategorySingle } from "@tcgdex/sdk/interfaces/Category" +import Logger from '@dzeio/logger' +const logger = new Logger('category/item') + type categoryCards = { [key in Category]?: Array } @@ -14,14 +17,15 @@ const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "categories") + export default async () => { - console.log(endpoint) - const list = getAllCards() + logger.log('Fetching Cards') + const list = await getAllCards() const arr: categoryCards = {} for (const i of list) { - const card = await fetchCard(i) + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang)) continue + if (!(await isCardAvailable(card, lang))) continue const c = card.category @@ -34,6 +38,7 @@ export default async () => { if (arr.hasOwnProperty(cat)) { const cards: Array = arr[cat]; const rCat: Category = parseInt(cat) + logger.log('Processing Category', TranslationUtil.translate("category", rCat, lang)) const toSave: CategorySingle = { id: rCat, name: TranslationUtil.translate("category", rCat, lang), @@ -50,5 +55,5 @@ export default async () => { await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave)) } } - console.log('ended ' + endpoint) + logger.log('ended ' + endpoint) } diff --git a/endpoints/expansionUtil.ts b/endpoints/expansionUtil.ts index 1820436..4617032 100644 --- a/endpoints/expansionUtil.ts +++ b/endpoints/expansionUtil.ts @@ -3,41 +3,33 @@ import Set from "@tcgdex/sdk/interfaces/Set" import * as glob from 'glob' import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { ExpansionSingle } from "@tcgdex/sdk/interfaces/Expansion" -import { getAllSets } from "./util" +import { getAllSets, smartGlob } from "./util" import { setToSetSimple, fetchSet } from "./setUtil" +import Logger from "@dzeio/logger" + +const logger = new Logger('ExpansionUtils') export function getExpansion(set: Set): Expansion { if ("expansion" in set) return set.expansion return require(`../../db/expansions/${set.expansionCode}`) } -const setExpansionLink: Record = {} - -export function getExpansionFromSetName(setName: string): Expansion { +export async function getExpansionFromSetName(setName: string): Promise { try { - if (!setExpansionLink[setName]) { - setExpansionLink[setName] = glob.sync(`./db/sets/**/${setName}.ts`)[0].split('/')[3] - } - const expansionName = setExpansionLink[setName] + const expansionName = (await smartGlob(`./db/sets/**/${setName}.js`))[0].split('/')[3].replace('.js', '') return fetchExpansion(expansionName) } catch (e) { - console.error(glob.sync(`./db/sets/**/${setName}`)) + logger.error(e, setName) throw new Error(setName) } } -export function getAllExpansions(): Array { - return glob.sync("./db/expansions/*.ts").map(el => el.split('/')[3].substr(0, el.length-1-3)) // -15 = start -1 = 0 index -3 = .ts +export async function getAllExpansions(): Promise> { + return (await smartGlob('./db/expansions/*.js')).map((it) => it.substring(it.lastIndexOf('/') + 1, it.length - 3)) // -15 = start -1 = 0 index -3 = .ts } -const expansionCache: Record = {} - -export function fetchExpansion(name: string): Expansion { - name = name.replace('.ts', '') - if (!expansionCache[name]) { - expansionCache[name] = require(`../db/expansions/${name}.js`).default - } - return expansionCache[name] +export async function fetchExpansion(name: string): Promise { + return (await import(`../db/expansions/${name}.js`)).default } export function expansionToExpansionSimple(expansion: Expansion, lang: Langs) { @@ -47,10 +39,10 @@ export function expansionToExpansionSimple(expansion: Expansion, lang: Langs) { } } -export function expansionToExpansionSingle(expansion: Expansion, lang: Langs): ExpansionSingle { - const sets = getAllSets(expansion.code, true) - .map(el => fetchSet(expansion.code, el)) - .sort((a, b) => { +export async function expansionToExpansionSingle(expansion: Expansion, lang: Langs): Promise { + const setsTmp = await Promise.all((await getAllSets(expansion.code)) + .map(el => fetchSet(expansion.code, el))) + const sets = setsTmp.sort((a, b) => { return a.releaseDate > b.releaseDate ? 1 : -1 }) .map(el => setToSetSimple(el, lang)) diff --git a/endpoints/expansions/index.ts b/endpoints/expansions/index.ts index 591029c..8106c93 100644 --- a/endpoints/expansions/index.ts +++ b/endpoints/expansions/index.ts @@ -9,21 +9,23 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "expansions") +import Logger from '@dzeio/logger' +const logger = new Logger('expansions/index') export default async () => { - console.log(endpoint) - const expansions = getAllExpansions() + logger.log('Fetching Expansions') + const expansions = await getAllExpansions() let list: Array<{ release: string, expansion: Expansion }> = [] for (const i of expansions) { const expansion: Expansion = require(`../../db/expansions/${i}`).default - const sets = getAllSets(expansion.code, true) + const sets = await getAllSets(expansion.code) expansion.sets = sets let oldestRelease = "9999-99-99" for (const j of sets) { - const set = fetchSet(expansion.code, j) + const set = await fetchSet(expansion.code, j) oldestRelease = set.releaseDate < oldestRelease ? set.releaseDate : oldestRelease } list.push({ @@ -41,5 +43,5 @@ export default async () => { await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/expansions/item.ts b/endpoints/expansions/item.ts index 7bb7de8..ecd064a 100644 --- a/endpoints/expansions/item.ts +++ b/endpoints/expansions/item.ts @@ -3,20 +3,24 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { getBaseFolder } from "../util" import { promises as fs } from 'fs' +import Logger from '@dzeio/logger' +const logger = new Logger('expansions/index') + const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "expansions") export default async () => { - console.log(endpoint) - const list = getAllExpansions() + logger.log('Fetching Expansions') + const list = await getAllExpansions() for (const i of list) { - const expansion = fetchExpansion(i) + logger.log('Processing Expansion', i) + const expansion = await fetchExpansion(i) await fs.mkdir(`${endpoint}/${expansion.code}/`, {recursive: true}) - await fs.writeFile(`${endpoint}/${expansion.code}/index.json`, JSON.stringify(expansionToExpansionSingle(expansion, lang))) + await fs.writeFile(`${endpoint}/${expansion.code}/index.json`, JSON.stringify(await expansionToExpansionSingle(expansion, lang))) } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/hp/index.ts b/endpoints/hp/index.ts index b7aa276..f641e45 100644 --- a/endpoints/hp/index.ts +++ b/endpoints/hp/index.ts @@ -3,20 +3,24 @@ import { fetchCard, isCardAvailable } from "../cardUtil" import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { HpList } from "@tcgdex/sdk/interfaces/Hp" import { promises as fs } from 'fs' +import Card from "@tcgdex/sdk/interfaces/Card" + +import Logger from '@dzeio/logger' +const logger = new Logger('hp/index') const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "hp") export default async () => { - console.log(endpoint) - const cards = getAllCards() + logger.log('Fetching Cards') + const cards = await getAllCards() const hps: Array = [] for (const i of cards) { - const card = fetchCard(i) - if (!isCardAvailable(card, lang) || !card.hp) continue + const card: Card = (await import(i)).default + if (!(await isCardAvailable(card, lang)) || !card.hp) continue if (hps.includes(card.hp)) continue hps.push(card.hp) @@ -26,9 +30,10 @@ export default async () => { count: hps.length, list: hps.sort((a, b) => a > b ? 1 : -1) } + logger.log('Writing to files') await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(hpList)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/hp/item.ts b/endpoints/hp/item.ts index 8b96485..9575aee 100644 --- a/endpoints/hp/item.ts +++ b/endpoints/hp/item.ts @@ -5,6 +5,9 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { HpSingle } from "@tcgdex/sdk/interfaces/Hp" import { promises as fs } from 'fs' +import Logger from '@dzeio/logger' +const logger = new Logger('hp/item') + interface t { [key: number]: Array } @@ -13,13 +16,13 @@ const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "hp") export default async () => { - console.log(endpoint) - const files = getAllCards() + logger.log('Fetching Cards') + const files = await getAllCards() const pools: t = {} for (const file of files) { - const card = fetchCard(file) + const card: Card = (await import(file)).default - if (!isCardAvailable(card, lang) || !card.hp) continue + if (!(await isCardAvailable(card, lang)) || !card.hp) continue if (!(card.hp in pools)) pools[card.hp] = [] pools[card.hp].push(card) @@ -27,6 +30,7 @@ export default async () => { for (const hp in pools) { if (pools.hasOwnProperty(hp)) { + logger.log('Processing HP', hp) const cards = pools[hp]; const toSave: HpSingle = { @@ -38,5 +42,5 @@ export default async () => { await fs.writeFile(`${endpoint}/${toSave.hp}/index.json`, JSON.stringify(toSave)) } } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/illustrators/index.ts b/endpoints/illustrators/index.ts index 949f2a4..4c4dd18 100644 --- a/endpoints/illustrators/index.ts +++ b/endpoints/illustrators/index.ts @@ -3,11 +3,14 @@ import { IllustratorsList } from "@tcgdex/sdk/interfaces/Illustrator" import { getBaseFolder } from "../util" import { promises as fs} from "fs" +import Logger from '@dzeio/logger' +const logger = new Logger('illustrators/index') + const lang = process.env.CARDLANG || "en" const endpoint = getBaseFolder(lang, "illustrators") export default async () => { - console.log(endpoint) + logger.log('Fetching illustrators') const db = await fetchIllustrators() @@ -18,5 +21,5 @@ export default async () => { await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/illustrators/item.ts b/endpoints/illustrators/item.ts index 4853793..10738d7 100644 --- a/endpoints/illustrators/item.ts +++ b/endpoints/illustrators/item.ts @@ -6,6 +6,9 @@ import Card from "@tcgdex/sdk/interfaces/Card" import { isCardAvailable, cardToCardSimple } from "../cardUtil" import { Langs } from "@tcgdex/sdk/interfaces/LangList" +import Logger from '@dzeio/logger' +const logger = new Logger('illustrators/item') + const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "illustrators") @@ -14,18 +17,18 @@ interface t { } export default async () => { - console.log(endpoint) + logger.log('Fetching Illustrators/Cards') const db = await fetchIllustrators() - const cards = getAllCards() + const cards = await getAllCards() const tmp: t = {} for (const i of cards) { - const card: Card = require(`../../db/cards/${i}`).default + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang) || !card.illustrator) continue + if (!(await isCardAvailable(card, lang)) || !card.illustrator) continue if (!(card.illustrator in tmp)) tmp[card.illustrator] = [] tmp[card.illustrator].push(card) @@ -33,6 +36,7 @@ export default async () => { for (const illustrator in tmp) { if (tmp.hasOwnProperty(illustrator)) { + logger.log('Processing illustrator', illustrator) const list = tmp[illustrator]; const toSave: IllustratorSingle = { @@ -45,5 +49,5 @@ export default async () => { await fs.writeFile(`${endpoint}/${toSave.id}/index.json`, JSON.stringify(toSave)) } } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/illustrators/updateDB.ts b/endpoints/illustrators/updateDB.ts index 10b71b2..6f115f3 100644 --- a/endpoints/illustrators/updateDB.ts +++ b/endpoints/illustrators/updateDB.ts @@ -1,4 +1,4 @@ -import { getAllCards2 } from "../util" +import { getAllCards, getAllCards2 } from "../util" import Card from "@tcgdex/sdk/interfaces/Card" import { promises as fs} from "fs" @@ -10,10 +10,9 @@ export default async () => { const db = await fetchIllustrators() - const list = getAllCards2() + const list = await getAllCards() for (let i of list) { - i = i.replace("./", "../../") - const card: Card = require(i).default + const card: Card = (await import(i)).default if (!card.illustrator) continue diff --git a/endpoints/rarities/index.ts b/endpoints/rarities/index.ts index 29c0bb9..da2b2fa 100644 --- a/endpoints/rarities/index.ts +++ b/endpoints/rarities/index.ts @@ -4,15 +4,15 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' import Rarity, { RaritySimple, RarityList } from "@tcgdex/sdk/interfaces/Rarity" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Rarities/Index' +import Logger from '@dzeio/logger' +const logger = new Logger('rarities/index') const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "rarities") export default async () => { - console.log(endpoint) + logger.log('Fetching Rarities') const list: Array = [] for (const cat of Object.values(Rarity)) { @@ -27,8 +27,8 @@ export default async () => { count: list.length, list: list } - + logger.log('Writing to file') await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/rarities/item.ts b/endpoints/rarities/item.ts index 786f22e..9dc835e 100644 --- a/endpoints/rarities/item.ts +++ b/endpoints/rarities/item.ts @@ -6,8 +6,8 @@ import TranslationUtil from "@tcgdex/sdk/TranslationUtil" import { promises } from "fs" import Rarity, { RaritySingle } from "@tcgdex/sdk/interfaces/Rarity" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Rarities/Item' +import Logger from '@dzeio/logger' +const logger = new Logger('rarities/item') type rarityCards = { @@ -19,13 +19,13 @@ const endpoint = getBaseFolder(lang, "rarities") export default async () => { - console.log(endpoint) - const list = getAllCards() + logger.log('Fetching Cards') + const list = await getAllCards() const arr: rarityCards = {} for (const i of list) { - const card = await fetchCardAsync(i) + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang)) continue + if (!(await isCardAvailable(card, lang))) continue const c = card.rarity @@ -38,6 +38,7 @@ export default async () => { if (arr.hasOwnProperty(cat)) { const cards: Array = arr[cat]; const rCat: Rarity = parseInt(cat) + logger.log('Processing Rarity', TranslationUtil.translate("rarity", rCat, lang)) const toSave: RaritySingle = { id: rCat, name: TranslationUtil.translate("rarity", rCat, lang), @@ -54,5 +55,5 @@ export default async () => { await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave)) } } - console.log('ended ' + endpoint) + logger.log('ended ' + endpoint) } diff --git a/endpoints/retreat/index.ts b/endpoints/retreat/index.ts index 4608c65..5f68b1b 100644 --- a/endpoints/retreat/index.ts +++ b/endpoints/retreat/index.ts @@ -1,26 +1,25 @@ import Card from "@tcgdex/sdk/interfaces/Card" -import { getAllCards2, getBaseFolder } from "../util" +import { getAllCards, getAllCards2, getBaseFolder } from "../util" import { promises as fs } from 'fs' import { isCardAvailable } from "../cardUtil" import { RetreatList } from '@tcgdex/sdk/interfaces/Retreat' import { Langs } from "@tcgdex/sdk/interfaces/LangList" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Retreat/Index' +import Logger from '@dzeio/logger' +const logger = new Logger('retreat/index') const lang = (process.env.CARDLANG || "en") as Langs const endpoint = getBaseFolder(lang, "retreat") export default async () => { - console.log(endpoint) - const files = await getAllCards2() + logger.log('Fetching Cards') + const files = await getAllCards() const count: Array = [] for (let file of files) { - file = file.replace("./", "../../") - const card: Card = await require(file).default + const card: Card = (await import(file)).default if ( - !isCardAvailable(card, lang) || + !(await isCardAvailable(card, lang)) || !card.retreat || count.includes(card.retreat) ) continue @@ -31,8 +30,9 @@ export default async () => { count: count.length, list: count } + logger.log('Writingto file') await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(list)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/retreat/item.ts b/endpoints/retreat/item.ts index 8ea41fd..a3c55f2 100644 --- a/endpoints/retreat/item.ts +++ b/endpoints/retreat/item.ts @@ -1,26 +1,25 @@ import Card from "@tcgdex/sdk/interfaces/Card" -import { getAllCards2, getBaseFolder } from "../util" +import { getAllCards, getAllCards2, getBaseFolder } from "../util" import { promises as fs } from 'fs' import { isCardAvailable, cardToCardSimple } from "../cardUtil" import { RetreatSingle } from '@tcgdex/sdk/interfaces/Retreat' import { Langs } from "@tcgdex/sdk/interfaces/LangList" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Retreat/Item' +import Logger from '@dzeio/logger' +const logger = new Logger('retreat/item') const lang = (process.env.CARDLANG || "en") as Langs const endpoint = getBaseFolder(lang, "retreat") export default async () => { - console.log(endpoint) - const files = await getAllCards2() + logger.log('Fetching cards') + const files = await getAllCards() const count: Array> = [] for (let file of files) { - file = file.replace("./", "../../") - const card: Card = await require(file).default + const card: Card = (await import(file)).default if ( - !isCardAvailable(card, lang) || + !(await isCardAvailable(card, lang)) || !card.retreat ) continue if (!(card.retreat in count)) count[card.retreat] = [] @@ -29,6 +28,7 @@ export default async () => { for (const retreat in count) { if (count.hasOwnProperty(retreat)) { + logger.log('Processing Retreat', retreat) const cardArr = count[retreat]; const item: RetreatSingle = { @@ -40,5 +40,5 @@ export default async () => { await fs.writeFile(`${endpoint}/${item.id}/index.json`, JSON.stringify(item)) } } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/setUtil.ts b/endpoints/setUtil.ts index aff547b..ac46a9f 100644 --- a/endpoints/setUtil.ts +++ b/endpoints/setUtil.ts @@ -5,7 +5,7 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { SetSimple, SetSingle } from "@tcgdex/sdk/interfaces/Set" import { cardToCardSimple } from "./cardUtil" import { CardSimple } from "@tcgdex/sdk/interfaces/Card" -import { getAllCards2 } from "./util" +import { getAllCards, getAllCards2, smartGlob } from "./util" interface t { [key: string]: T @@ -17,18 +17,18 @@ export function isSet(set: Set | {name: string, code: string}): set is Set { return "releaseDate" in set } -export function getSet(card: Card): Set { +export async function getSet(card: Card): Promise { if (!(card.set.code in setCache)) { if (isSet(card.set)) setCache[card.set.code] = card.set - let setPath = glob.sync(`./db/sets/**/${card.set.code}.js`)[0] + let setPath = (await smartGlob(`./db/sets/**/${card.set.code}.js`))[0] setPath = setPath.replace('./', '../') setCache[card.set.code] = require(setPath).default } return setCache[card.set.code] } -export function fetchSet(expansion: string, set: string): Set { - return require(`../db/sets/${expansion}/${set}.js`).default +export async function fetchSet(expansion: string, set: string): Promise { + return (await import(`../db/sets/${expansion}/${set}.js`)).default } export function isSetAvailable(set: Set, lang: Langs) { @@ -47,18 +47,21 @@ export function setToSetSimple(set: Set, lang: Langs): SetSimple { } export async function getSetCards(set: Set, lang: Langs): Promise> { - const cardes = getAllCards2(set.code) + const cardes = await getAllCards(set.code, set.expansionCode ?? set.expansion.code) const cards: Array = [] for (let el of cardes) { - el = el.replace("./", "../") - const card: Card = require(el).default + el = el.replace("../../", "../") + const card: Card = (await import(el)).default cards.push( card ) } return await Promise.all(cards.sort((a, b) => { - if (!isNaN(parseInt(a.localId + "")) && !isNaN(parseInt(b.localId + ""))) { + if ( + !isNaN(parseInt(a.localId + "")) && + !isNaN(parseInt(b.localId + "")) + ) { return parseInt(a.localId + "") - parseInt(b.localId + "") } return a.localId > b.localId ? 1 : -1 diff --git a/endpoints/sets/index.ts b/endpoints/sets/index.ts index 38f155c..f0ff8ae 100644 --- a/endpoints/sets/index.ts +++ b/endpoints/sets/index.ts @@ -3,10 +3,11 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' import { SetSimple, SetList } from "@tcgdex/sdk/interfaces/Set" import { getAllSets, getBaseFolder } from "../util" -import { isSetAvailable, setToSetSimple } from "../setUtil" +import { fetchSet, isSetAvailable, setToSetSimple } from "../setUtil" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Sets/Index' +import Logger from '@dzeio/logger' +import { getExpansionFromSetName } from "../expansionUtil" +const logger = new Logger('sets/index') const lang = process.env.CARDLANG as Langs || "en" @@ -14,19 +15,20 @@ const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "sets") export default async () => { - console.log(endpoint) + logger.log('Fetching sets') const list = await getAllSets() let items: Array = [] for (let el of list) { - el = el.replace("./", "../../") - const set: Set = require(el).default + const expansion = (await getExpansionFromSetName(el)) + const set: Set = await fetchSet(expansion.code, el) if (!isSetAvailable(set, lang)) continue items.push( set ) } + logger.log('Procesing Sets') items = items.sort((a, b) => a.releaseDate > b.releaseDate ? 1 : -1) @@ -40,5 +42,5 @@ export default async () => { await fs.mkdir(`${endpoint}`, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(cardList)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/sets/item.ts b/endpoints/sets/item.ts index 3eb861e..d06812a 100644 --- a/endpoints/sets/item.ts +++ b/endpoints/sets/item.ts @@ -2,21 +2,24 @@ import { getBaseFolder, getAllSets } from "../util" import Set from "@tcgdex/sdk/interfaces/Set" import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' -import { isSetAvailable, setToSetSingle } from "../setUtil" +import { fetchSet, isSetAvailable, setToSetSingle } from "../setUtil" +import { getExpansionFromSetName } from "../expansionUtil" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Sets/Item' +import Logger from '@dzeio/logger' +const logger = new Logger('sets/item') const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "sets") export default async () => { - console.log(endpoint) + logger.log('Fetching Sets') const list = await getAllSets() + logger.log(list) for (let el of list) { - el = el.replace("./", "../../") - const set: Set = require(el).default + logger.log('Processing set', el) + const expansion = (await getExpansionFromSetName(el)) + const set: Set = await fetchSet(expansion.code, el) if (!isSetAvailable(set, lang)) continue @@ -25,5 +28,5 @@ export default async () => { } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/sets/subitem.ts b/endpoints/sets/subitem.ts index bab7f29..4a06517 100644 --- a/endpoints/sets/subitem.ts +++ b/endpoints/sets/subitem.ts @@ -1,14 +1,15 @@ -import { getBaseFolder, getAllSets } from "../util" +import { getBaseFolder, getAllSets, getAllCards } from "../util" import Set from "@tcgdex/sdk/interfaces/Set" import { Langs } from "@tcgdex/sdk/interfaces/LangList" import { promises as fs } from 'fs' -import { isSetAvailable } from "../setUtil" +import { fetchSet, isSetAvailable } from "../setUtil" import { getAllCards2 } from "../util" import Card from "@tcgdex/sdk/interfaces/Card" import { cardToCardSingle, isCardAvailable } from "../cardUtil" +import { getExpansionFromSetName } from "../expansionUtil" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Sets/SubItem' +import Logger from '@dzeio/logger' +const logger = new Logger('sets/subitem') const lang = process.env.CARDLANG as Langs || "en" @@ -16,27 +17,28 @@ const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "sets") export default async () => { - console.log(endpoint) + logger.log('Fetching Sets') const list = await getAllSets() for (let el of list) { - el = el.replace("./", "../../") - const set: Set = require(el).default + const expansion = (await getExpansionFromSetName(el)) + const set: Set = await fetchSet(expansion.code, el) if (!isSetAvailable(set, lang)) continue - const lit = await getAllCards2(set.code) + const lit = await getAllCards(set.code, set?.expansionCode ?? set.expansion.code) + logger.log('Fetching/Writing Cards for set', el) for (let i of lit) { - i = i.replace("./", "../../") - const card: Card = require(i).default + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang)) continue + if (!(await isCardAvailable(card, lang))) continue - await fs.mkdir(`${endpoint}/${set.code}/${card.localId}`, {recursive: true}) - await fs.writeFile(`${endpoint}/${set.code}/${card.localId}/index.json`, JSON.stringify(await cardToCardSingle(card, lang))) + const localId = card.localId === '?' ? '%3F' : card.localId + + await fs.mkdir(`${endpoint}/${set.code}/${localId}`, {recursive: true}) + await fs.writeFile(`${endpoint}/${set.code}/${localId}/index.json`, JSON.stringify(await cardToCardSingle(card, lang))) } } - - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/tags/index.ts b/endpoints/tags/index.ts index 060d39b..18cb87a 100644 --- a/endpoints/tags/index.ts +++ b/endpoints/tags/index.ts @@ -11,7 +11,7 @@ const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "tags") export default async () => { - logger.log(endpoint) + logger.log('Fetching Tags') const list: Array = [] for (const cat of Object.values(Tag)) { @@ -29,5 +29,5 @@ export default async () => { await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res)) - logger.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/tags/item.ts b/endpoints/tags/item.ts index fc17740..298e28b 100644 --- a/endpoints/tags/item.ts +++ b/endpoints/tags/item.ts @@ -9,7 +9,7 @@ import Tag, { TagSingle } from "@tcgdex/sdk/interfaces/Tag" import Logger from '@dzeio/logger' import { objectKeys, objectSize } from '@dzeio/object-util' -const logger = new Logger('Tags/Item') +const logger = new Logger('tags/item') type tagCards = { [key in Tag]?: Array @@ -20,20 +20,19 @@ const endpoint = getBaseFolder(lang, "tags") export default async () => { - logger.log('part 1') - const list = getAllCards() + logger.log('Fetching cards') + const list = await getAllCards() const arr: tagCards = {} for (const i of list) { - const card = await fetchCardAsync(i) + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang) || !card.tags) continue + if (!(await isCardAvailable(card, lang)) || !card.tags) continue for (const tag of card.tags) { if (!(tag in arr)) arr[tag] = [] arr[tag].push(card) } } - logger.log('part 2') for (const type in arr) { if (arr.hasOwnProperty(type)) { const cards: Array = arr[type]; diff --git a/endpoints/types/index.ts b/endpoints/types/index.ts index 7c65689..ed5df11 100644 --- a/endpoints/types/index.ts +++ b/endpoints/types/index.ts @@ -8,11 +8,11 @@ import { List } from "@tcgdex/sdk/interfaces/General" const lang = process.env.CARDLANG as Langs || "en" const endpoint = getBaseFolder(lang, "types") -import { logger as console } from '@dzeio/logger' -console.prefix = 'Types/Index' +import Logger from '@dzeio/logger' +const logger = new Logger('types/index') export default async () => { - console.log(endpoint) + logger.log('Fetching types') const typeArr: Array = [] for (const i of Object.values(Type)) { if (typeof i !== "number") continue @@ -27,7 +27,8 @@ export default async () => { list: typeArr } + logger.log('Writing types') await fs.mkdir(endpoint, {recursive: true}) await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(typeList)) - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/types/item.ts b/endpoints/types/item.ts index effd7ee..0b409be 100644 --- a/endpoints/types/item.ts +++ b/endpoints/types/item.ts @@ -6,9 +6,8 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList" import TranslationUtil from "@tcgdex/sdk/TranslationUtil" import { promises } from "fs" -import { logger as console } from '@dzeio/logger' -console.prefix = 'Types/Item' - +import Logger from '@dzeio/logger' +const logger = new Logger('types/item') type typeCards = { [key in Type]?: Array } @@ -18,13 +17,13 @@ const endpoint = getBaseFolder(lang, "types") export default async () => { - console.log(endpoint) - const list = getAllCards() + logger.log('Fetching cards') + const list = await getAllCards() const arr: typeCards = {} for (const i of list) { - const card = await fetchCard(i) + const card: Card = (await import(i)).default - if (!isCardAvailable(card, lang) || !card.type) continue + if (!(await isCardAvailable(card, lang)) || !card.type) continue for (const type of card.type) { if (!(type in arr)) arr[type] = [] @@ -33,24 +32,26 @@ export default async () => { } for (const type in arr) { - if (arr.hasOwnProperty(type)) { - const cards: Array = arr[type]; - const rType: Type = parseInt(type) - const toSave: TypeSingle = { - id: rType, - name: TranslationUtil.translate("type", rType, lang), - cards: await Promise.all(cards.map(el => cardToCardSimple(el, lang))) - } - - const index = `${endpoint}/${toSave.id}` - const name = `${endpoint}/${urlize(toSave.name)}` - - await promises.mkdir(index, {recursive: true}) - await promises.mkdir(name, {recursive: true}) - - await promises.writeFile(`${index}/index.json`, JSON.stringify(toSave)) - await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave)) + if (!Object.prototype.hasOwnProperty.call(arr, type)) { + continue } + const cards: Array = arr[type]; + const rType: Type = parseInt(type) + logger.log('Processing type', TranslationUtil.translate("type", rType, lang)) + const toSave: TypeSingle = { + id: rType, + name: TranslationUtil.translate("type", rType, lang), + cards: await Promise.all(cards.map(el => cardToCardSimple(el, lang))) + } + + const index = `${endpoint}/${toSave.id}` + const name = `${endpoint}/${urlize(toSave.name)}` + + await promises.mkdir(index, {recursive: true}) + await promises.mkdir(name, {recursive: true}) + + await promises.writeFile(`${index}/index.json`, JSON.stringify(toSave)) + await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave)) } - console.log('ended ' + endpoint) + logger.log('Finished') } diff --git a/endpoints/util.ts b/endpoints/util.ts index 1b10e3a..b5d63a5 100644 --- a/endpoints/util.ts +++ b/endpoints/util.ts @@ -1,70 +1,25 @@ -import { promises as fs, promises } from 'fs' import * as glob from 'glob' import fetch from 'node-fetch' const VERSION = 'v1' -export function getAllCards(set = "**", expansion = "**") { - return glob.sync(`./db/cards/${expansion}/${set}/*.js`).map(el => { - return el.substr(11, el.length-10-1-3) - }) +export async function getAllCards(set = "**", expansion = "**") { + return (await smartGlob(`./db/cards/${expansion}/${set}/*.js`)).map((it) => it.replace('./', '../../')) } export function getAllCards2(set = "**", expansion = "**") { return glob.sync(`./db/cards/${expansion}/${set}/*.js`) } -export function getAllSets(expansion = "**", nameOnly = false) { - if (nameOnly) return glob.sync(`./db/sets/${expansion}/*.js`).map(el => el.substr(11+expansion.length, el.length-(10+expansion.length)-1-3)) - return glob.sync(`./db/sets/${expansion}/*.js`) -} - -export function getAllCardsJSON() { - return glob.sync("./db/cards/**/**/*.json") -} - -async function listFolder(folder: string): Promise> { - const files = await fs.readdir(folder) - const res = [] - for (const file of files) { - if (file.endsWith(".json")) res.push(`${folder}/${file}`) - if ((await fs.stat(`${folder}/${file}`)).isDirectory()) { - res.push(...await listFolder(`${folder}/${file}`)) - } - } - return res +export async function getAllSets(expansion = "**") { + return (await smartGlob(`./db/sets/${expansion}/*.js`)) + .map(el => el.substring(el.lastIndexOf('/') + 1, el.lastIndexOf('.'))) } export function getBaseFolder(lang: string, endpoint: string) { return `./dist/${VERSION}/${lang}/${endpoint}` } -export async function del(path: string) { - let files = [] - const promiseArr = [] - try { - files = await promises.readdir(path) - } catch { - return - } - if (files.length > 0) { - for (const file of files) { - const fPath = `${path}/${file}` - if ((await promises.stat(fPath)).isDirectory()) { - promiseArr.push( - del(fPath) - ) - } else { - promiseArr.push( - promises.unlink(fPath) - ) - } - } - } - await Promise.all(promiseArr) - await promises.rmdir(path) -} - export function urlize(str: string): string { str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "") return str.replace(/ /g, "-").toLowerCase() @@ -84,3 +39,12 @@ export async function fetchRemoteFile(url: string): Promise { } return fileCache[url] } + +const globCache: Record> = {} + +export async function smartGlob(query: string) { + if (!globCache[query]) { + globCache[query] = await new Promise((res) => glob(query, (err, matches) => res(matches))) + } + return globCache[query] +}