Updated

Signed-off-by: Florian BOUILLON <florian.bouillon@delta-wings.net>

.drone.yml

@@ -1,55 +0,0 @@
-kind: pipeline
-name: default
-type: docker
-
-steps:
-  - name: Submodules
-    image: alpine/git
-    commands:
-    - git submodule update --init --recursive
-    - git submodule foreach git checkout master
-    - git submodule foreach git pull origin master
-
-  - name: Prepare
-    image: node:alpine
-    commands:
-    - apk -q add yarn
-    - yarn
-    - yarn db:compile
-
-  - name: Compile
-    image: node:alpine
-    commands:
-    - apk -q add yarn
-    - rm -r dist/*
-    - yarn gen
-    - CARDLANG=fr yarn gen
-
-  - name: Push
-    image: alpine/git
-    commands:
-    - cd dist
-    - git config --global user.email "$PUSH_EMAIL"
-    - git config --global user.name "$PUSH_NAME"
-    - git add .
-    - git commit -m "Updated API"
-    - git push -u origin master
-    environment:
-      PUSH_EMAIL:
-        from_secret: PUSH_EMAIL
-      PUSH_NAME:
-        from_secret: PUSH_NAME
-
-  - name: Upload
-    image: node:alpine
-    commands:
-      - node upload.js
-    environment:
-      UPLOAD_USERNAME:
-        from_secret: UPLOAD_USERNAME
-      UPLOAD_PASSWORD:
-        from_secret: UPLOAD_PASSWORD
-      UPLOAD_DIST:
-        from_secret: UPLOAD_DIST
-      UPLOAD_REMOTE:
-        from_secret: UPLOAD_REMOTE

.gitignore

@@ -1 +1,3 @@
 node_modules
+db
+dist

all.ts

@@ -45,7 +45,7 @@ import { fetchRemoteFile } from './endpoints/util'
 		fetchRemoteFile('https://assets.tcgdex.net/data-fr.json'),
 		fetchRemoteFile('https://assets.tcgdex.net/data-univ.json')
 	])
-	logger.log('UPDATING...')
+	logger.log('Updating...')
 	await Promise.all([
 		cardIndex(),
 		cardItem(),
@@ -61,12 +61,10 @@ import { fetchRemoteFile } from './endpoints/util'
 
 		illustratorsIndex(),
 		illustratorsItem(),
-		illustratorsDB(),
 
 		raritiesIndex(),
 		raritiesItem(),
 
-
 		retreatIndex(),
 		retreatItem(),
 

endpoints/cardUtil.ts

@@ -22,7 +22,7 @@ type RemoteData = ObjectList<ObjectList<ObjectList<boolean>>>
 export async function cardToCardSimple(card: Card, lang: Langs): Promise<CardSimple> {
 	let image: string = undefined
 	const file: RemoteData = await fetchRemoteFile(`https://assets.tcgdex.net/data-${lang}.json`)
-	const expansion = getExpansionFromSetName(card.set.code)
+	const expansion = await getExpansionFromSetName(card.set.code)
 	if (file[expansion.code] && file[expansion.code][card.set.code] && file[expansion.code][card.set.code][card.localId]) {
 		const basePath = `https://assets.tcgdex.net/${lang}/${expansion.code}/${card.set.code}/${card.localId}`
 		image = `${basePath}/low`
@@ -36,10 +36,6 @@ export async function cardToCardSimple(card: Card, lang: Langs): Promise<CardSim
 	}
 }
 
-export function getCardExpansion(card: Card): Expansion {
-	return getExpansion(getSet(card))
-}
-
 export async function cardToCardSingle(card: Card, lang: Langs): Promise<CardSingle> {
 
 	let images: {
@@ -48,7 +44,7 @@ export async function cardToCardSingle(card: Card, lang: Langs): Promise<CardSin
 	} = undefined
 
 	const file: RemoteData = await fetchRemoteFile(`https://assets.tcgdex.net/data-${lang}.json`)
-	const expansion = getExpansionFromSetName(card.set.code)
+	const expansion = await getExpansionFromSetName(card.set.code)
 	if (file[expansion.code] && file[expansion.code][card.set.code] && file[expansion.code][card.set.code][card.localId]) {
 		const basePath = `https://assets.tcgdex.net/${lang}/${expansion.code}/${card.set.code}/${card.localId}`
 		images = {
@@ -70,7 +66,7 @@ export async function cardToCardSingle(card: Card, lang: Langs): Promise<CardSin
 
 		evolveFrom: card.evolveFrom && card.evolveFrom[lang],
 		evolveTo: card.evolveTo && card.evolveTo.map((el) => el[lang]),
-		tags: card.tags.map((el) => tagToTagSimple(el, lang)),
+		tags: card.tags?.map((el) => tagToTagSimple(el, lang)),
 		illustrator: card.illustrator && {
 			id: fetchIllustratorsSync().indexOf(card.illustrator),
 			name: card.illustrator,
@@ -101,9 +97,9 @@ export async function cardToCardSingle(card: Card, lang: Langs): Promise<CardSin
 	}
 }
 
-export function isCardAvailable(card: Card, lang: Langs): boolean {
+export async function isCardAvailable(card: Card, lang: Langs): Promise<boolean> {
 	if (!(lang in card.name)) return false
-	const set = getSet(card)
+	const set = await getSet(card)
 	if ("availability" in set && (lang in set.availability)) {
 		return set.availability[lang]
 	}

endpoints/cards/index.ts

@@ -3,21 +3,24 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
 import { CardList, CardSimple } from "@tcgdex/sdk/interfaces/Card"
 import { cardToCardSimple, isCardAvailable } from "../cardUtil"
-import { getBaseFolder, getAllCards2 } from "../util"
+import { getBaseFolder, getAllCards2, getAllCards } from "../util"
+
+import Logger from '@dzeio/logger'
+const logger = new Logger('cards/index')
+
 
 const lang = process.env.CARDLANG as Langs || "en"
 
 const endpoint = getBaseFolder(lang, "cards")
 
 export default async () => {
-	console.log(endpoint)
-	const list = await getAllCards2()
+	logger.log('Fetching Cards')
+	const list = await getAllCards()
 	const items: Array<CardSimple> = []
 	for (let el of list) {
-		el = el.replace("./", "../../")
-		const card: Card = require(el).default
+		const card: Card = (await import(el)).default
 
-		if (!isCardAvailable(card, lang)) continue
+		if (!(await isCardAvailable(card, lang))) continue
 		items.push(
 			await cardToCardSimple(card, lang)
 		)
@@ -29,9 +32,10 @@ export default async () => {
 		count: items.length,
 		list: items
 	}
+	logger.log('Writing')
 
 	await fs.mkdir(`${endpoint}`, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(cardList))
 
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/cards/item.ts

@@ -1,21 +1,24 @@
-import { getAllCards2, getBaseFolder } from "..//util"
+import { getAllCards, getAllCards2, getBaseFolder } from "..//util"
 import Card from "@tcgdex/sdk/interfaces/Card"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
 import { cardToCardSingle, isCardAvailable } from "../cardUtil"
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('cards/item')
+
+
 const lang = process.env.CARDLANG as Langs || "en"
 
 const endpoint = getBaseFolder(lang, "cards")
 
 export default async () => {
-	console.log(endpoint)
-	const list = await getAllCards2()
+	logger.log('Fetching Cards')
+	const list = await getAllCards()
 	for (let el of list) {
-		el = el.replace("./", "../../")
-		const card: Card = require(el).default
+		const card: Card = (await import(el)).default
 
-		if (!isCardAvailable(card, lang)) continue
+		if (!(await isCardAvailable(card, lang))) continue
 
 		try {
 			await fs.mkdir(`${endpoint}/${encodeURI(card.id)}/`, {recursive: true})
@@ -26,5 +29,5 @@ export default async () => {
 
 		// if (if (typeof card.set.availability === "undefined"))
 	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/categories/index.ts

@@ -3,15 +3,16 @@ import Category, { CategorySimple, CategoryList } from '@tcgdex/sdk/interfaces/C
 import TranslationUtil from "@tcgdex/sdk/TranslationUtil"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
+
 import Logger from '@dzeio/logger'
+const logger = new Logger('category/index')
 
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "categories")
 
-const logger = new Logger('Category/Index')
 
 export default async () => {
-	logger.log(endpoint)
+	logger.log('Fetching Categories')
 
 	const list: Array<CategorySimple> = []
 	for (const cat of Object.values(Category)) {
@@ -26,8 +27,9 @@ export default async () => {
 		count: list.length,
 		list: list
 	}
+	logger.log('Writing')
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res))
-	logger.log('ended')
+	logger.log('Finished')
 }

endpoints/categories/item.ts

@@ -6,6 +6,9 @@ import TranslationUtil from "@tcgdex/sdk/TranslationUtil"
 import { promises } from "fs"
 import Category, { CategorySingle } from "@tcgdex/sdk/interfaces/Category"
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('category/item')
+
 type categoryCards = {
 	[key in Category]?: Array<Card>
 }
@@ -14,14 +17,15 @@ const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "categories")
 
 
+
 export default async () => {
-	console.log(endpoint)
-	const list = getAllCards()
+	logger.log('Fetching Cards')
+	const list = await getAllCards()
 	const arr: categoryCards = {}
 	for (const i of list) {
-		const card = await fetchCard(i)
+		const card: Card = (await import(i)).default
 
-		if (!isCardAvailable(card, lang)) continue
+		if (!(await isCardAvailable(card, lang))) continue
 
 		const c = card.category
 
@@ -34,6 +38,7 @@ export default async () => {
 		if (arr.hasOwnProperty(cat)) {
 			const cards: Array<Card> = arr[cat];
 			const rCat: Category = parseInt(cat)
+			logger.log('Processing Category', TranslationUtil.translate("category", rCat, lang))
 			const toSave: CategorySingle = {
 				id: rCat,
 				name: TranslationUtil.translate("category", rCat, lang),
@@ -50,5 +55,5 @@ export default async () => {
 			await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave))
 		}
 	}
-	console.log('ended ' + endpoint)
+	logger.log('ended ' + endpoint)
 }

endpoints/expansionUtil.ts

@@ -3,41 +3,33 @@ import Set from "@tcgdex/sdk/interfaces/Set"
 import * as glob from 'glob'
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { ExpansionSingle } from "@tcgdex/sdk/interfaces/Expansion"
-import { getAllSets } from "./util"
+import { getAllSets, smartGlob } from "./util"
 import { setToSetSimple, fetchSet } from "./setUtil"
+import Logger from "@dzeio/logger"
+
+const logger = new Logger('ExpansionUtils')
 
 export function getExpansion(set: Set): Expansion {
 	if ("expansion" in set) return set.expansion
 	return require(`../../db/expansions/${set.expansionCode}`)
 }
 
-const setExpansionLink: Record<string, string> = {}
-
-export function getExpansionFromSetName(setName: string): Expansion {
+export async function getExpansionFromSetName(setName: string): Promise<Expansion> {
 	try {
-		if (!setExpansionLink[setName]) {
-			setExpansionLink[setName] = glob.sync(`./db/sets/**/${setName}.ts`)[0].split('/')[3]
-		}
-		const expansionName = setExpansionLink[setName]
+		const expansionName = (await smartGlob(`./db/sets/**/${setName}.js`))[0].split('/')[3].replace('.js', '')
 		return fetchExpansion(expansionName)
 	} catch (e) {
-		console.error(glob.sync(`./db/sets/**/${setName}`))
+		logger.error(e, setName)
 		throw new Error(setName)
 	}
 }
 
-export function getAllExpansions(): Array<string> {
-	return glob.sync("./db/expansions/*.ts").map(el => el.split('/')[3].substr(0, el.length-1-3)) // -15 = start -1 = 0 index -3 = .ts
+export async function getAllExpansions(): Promise<Array<string>> {
+	return (await smartGlob('./db/expansions/*.js')).map((it) => it.substring(it.lastIndexOf('/') + 1, it.length - 3)) // -15 = start -1 = 0 index -3 = .ts
 }
 
-const expansionCache: Record<string, Expansion> = {}
-
-export function fetchExpansion(name: string): Expansion {
-	name = name.replace('.ts', '')
-	if (!expansionCache[name]) {
-		expansionCache[name] = require(`../db/expansions/${name}.js`).default
-	}
-	return expansionCache[name]
+export async function fetchExpansion(name: string): Promise<Expansion> {
+	return (await import(`../db/expansions/${name}.js`)).default
 }
 
 export function expansionToExpansionSimple(expansion: Expansion, lang: Langs) {
@@ -47,10 +39,10 @@ export function expansionToExpansionSimple(expansion: Expansion, lang: Langs) {
 	}
 }
 
-export function expansionToExpansionSingle(expansion: Expansion, lang: Langs): ExpansionSingle {
-	const sets = getAllSets(expansion.code, true)
-		.map(el => fetchSet(expansion.code, el))
-		.sort((a, b) => {
+export async function expansionToExpansionSingle(expansion: Expansion, lang: Langs): Promise<ExpansionSingle> {
+	const setsTmp = await Promise.all((await getAllSets(expansion.code))
+		.map(el => fetchSet(expansion.code, el)))
+	const sets = setsTmp.sort((a, b) => {
 			return a.releaseDate > b.releaseDate ? 1 : -1
 		})
 		.map(el => setToSetSimple(el, lang))

endpoints/expansions/index.ts

@@ -9,21 +9,23 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "expansions")
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('expansions/index')
 
 export default async () => {
-	console.log(endpoint)
-	const expansions = getAllExpansions()
+	logger.log('Fetching Expansions')
+	const expansions = await getAllExpansions()
 	let list: Array<{
 		release: string,
 		expansion: Expansion
 	}> = []
 	for (const i of expansions) {
 		const expansion: Expansion = require(`../../db/expansions/${i}`).default
-		const sets = getAllSets(expansion.code, true)
+		const sets = await getAllSets(expansion.code)
 		expansion.sets = sets
 		let oldestRelease = "9999-99-99"
 		for (const j of sets) {
-			const set = fetchSet(expansion.code, j)
+			const set = await fetchSet(expansion.code, j)
 			oldestRelease = set.releaseDate < oldestRelease ? set.releaseDate : oldestRelease
 		}
 		list.push({
@@ -41,5 +43,5 @@ export default async () => {
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res))
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/expansions/item.ts

@@ -3,20 +3,24 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { getBaseFolder } from "../util"
 import { promises as fs } from 'fs'
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('expansions/index')
+
 const lang = process.env.CARDLANG as Langs || "en"
 
 const endpoint = getBaseFolder(lang, "expansions")
 
 
 export default async () => {
-	console.log(endpoint)
-	const list = getAllExpansions()
+	logger.log('Fetching Expansions')
+	const list = await getAllExpansions()
 	for (const i of list) {
-		const expansion = fetchExpansion(i)
+		logger.log('Processing Expansion', i)
+		const expansion = await fetchExpansion(i)
 
 
 		await fs.mkdir(`${endpoint}/${expansion.code}/`, {recursive: true})
-		await fs.writeFile(`${endpoint}/${expansion.code}/index.json`, JSON.stringify(expansionToExpansionSingle(expansion, lang)))
+		await fs.writeFile(`${endpoint}/${expansion.code}/index.json`, JSON.stringify(await expansionToExpansionSingle(expansion, lang)))
 	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/hp/index.ts

@@ -3,20 +3,24 @@ import { fetchCard, isCardAvailable } from "../cardUtil"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { HpList } from "@tcgdex/sdk/interfaces/Hp"
 import { promises as fs } from 'fs'
+import Card from "@tcgdex/sdk/interfaces/Card"
+
+import Logger from '@dzeio/logger'
+const logger = new Logger('hp/index')
 
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "hp")
 
 
 export default async () => {
-	console.log(endpoint)
-	const cards = getAllCards()
+	logger.log('Fetching Cards')
+	const cards = await getAllCards()
 
 	const hps: Array<number> = []
 
 	for (const i of cards) {
-		const card = fetchCard(i)
-		if (!isCardAvailable(card, lang) || !card.hp) continue
+		const card: Card = (await import(i)).default
+		if (!(await isCardAvailable(card, lang)) || !card.hp) continue
 
 		if (hps.includes(card.hp)) continue
 		hps.push(card.hp)
@@ -26,9 +30,10 @@ export default async () => {
 		count: hps.length,
 		list: hps.sort((a, b) => a > b ? 1 : -1)
 	}
+	logger.log('Writing to files')
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(hpList))
 
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/hp/item.ts

@@ -5,6 +5,9 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { HpSingle } from "@tcgdex/sdk/interfaces/Hp"
 import { promises as fs } from 'fs'
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('hp/item')
+
 interface t {
 	[key: number]: Array<Card>
 }
@@ -13,13 +16,13 @@ const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "hp")
 
 export default async () => {
-	console.log(endpoint)
-	const files = getAllCards()
+	logger.log('Fetching Cards')
+	const files = await getAllCards()
 	const pools: t = {}
 	for (const file of files) {
-		const card = fetchCard(file)
+		const card: Card = (await import(file)).default
 
-		if (!isCardAvailable(card, lang) || !card.hp) continue
+		if (!(await isCardAvailable(card, lang)) || !card.hp) continue
 
 		if (!(card.hp in pools)) pools[card.hp] = []
 		pools[card.hp].push(card)
@@ -27,6 +30,7 @@ export default async () => {
 
 	for (const hp in pools) {
 		if (pools.hasOwnProperty(hp)) {
+			logger.log('Processing HP', hp)
 			const cards = pools[hp];
 
 			const toSave: HpSingle = {
@@ -38,5 +42,5 @@ export default async () => {
 			await fs.writeFile(`${endpoint}/${toSave.hp}/index.json`, JSON.stringify(toSave))
 		}
 	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/illustrators/index.ts

@@ -3,11 +3,14 @@ import { IllustratorsList } from "@tcgdex/sdk/interfaces/Illustrator"
 import { getBaseFolder } from "../util"
 import { promises as fs} from "fs"
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('illustrators/index')
+
 const lang = process.env.CARDLANG || "en"
 const endpoint = getBaseFolder(lang, "illustrators")
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching illustrators')
 
 	const db = await fetchIllustrators()
 
@@ -18,5 +21,5 @@ export default async () => {
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res))
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/illustrators/item.ts

@@ -6,6 +6,9 @@ import Card from "@tcgdex/sdk/interfaces/Card"
 import { isCardAvailable, cardToCardSimple } from "../cardUtil"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 
+import Logger from '@dzeio/logger'
+const logger = new Logger('illustrators/item')
+
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "illustrators")
 
@@ -14,18 +17,18 @@ interface t {
 }
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching Illustrators/Cards')
 
 	const db = await fetchIllustrators()
-	const cards = getAllCards()
+	const cards = await getAllCards()
 
 	const tmp: t = {}
 
 
 	for (const i of cards) {
-		const card: Card = require(`../../db/cards/${i}`).default
+		const card: Card = (await import(i)).default
 
-		if (!isCardAvailable(card, lang) || !card.illustrator) continue
+		if (!(await isCardAvailable(card, lang)) || !card.illustrator) continue
 
 		if (!(card.illustrator in tmp)) tmp[card.illustrator] = []
 		tmp[card.illustrator].push(card)
@@ -33,6 +36,7 @@ export default async () => {
 
 	for (const illustrator in tmp) {
 		if (tmp.hasOwnProperty(illustrator)) {
+			logger.log('Processing illustrator', illustrator)
 			const list = tmp[illustrator];
 
 			const toSave: IllustratorSingle = {
@@ -45,5 +49,5 @@ export default async () => {
 			await fs.writeFile(`${endpoint}/${toSave.id}/index.json`, JSON.stringify(toSave))
 		}
 	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/illustrators/updateDB.ts

@@ -1,4 +1,4 @@
-import { getAllCards2 } from "../util"
+import { getAllCards, getAllCards2 } from "../util"
 import Card from "@tcgdex/sdk/interfaces/Card"
 
 import { promises as fs} from "fs"
@@ -10,10 +10,9 @@ export default async () => {
 	const db = await fetchIllustrators()
 
 
-	const list = getAllCards2()
+	const list = await getAllCards()
 	for (let i of list) {
-		i = i.replace("./", "../../")
-		const card: Card = require(i).default
+		const card: Card = (await import(i)).default
 
 		if (!card.illustrator) continue
 

endpoints/rarities/index.ts

@@ -4,15 +4,15 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
 import Rarity, { RaritySimple, RarityList } from "@tcgdex/sdk/interfaces/Rarity"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Rarities/Index'
+import Logger from '@dzeio/logger'
+const logger = new Logger('rarities/index')
 
 
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "rarities")
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching Rarities')
 
 	const list: Array<RaritySimple> = []
 	for (const cat of Object.values(Rarity)) {
@@ -27,8 +27,8 @@ export default async () => {
 		count: list.length,
 		list: list
 	}
-
+	logger.log('Writing to file')
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res))
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/rarities/item.ts

@@ -6,8 +6,8 @@ import TranslationUtil from "@tcgdex/sdk/TranslationUtil"
 import { promises } from "fs"
 import Rarity, { RaritySingle } from "@tcgdex/sdk/interfaces/Rarity"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Rarities/Item'
+import Logger from '@dzeio/logger'
+const logger = new Logger('rarities/item')
 
 
 type rarityCards = {
@@ -19,13 +19,13 @@ const endpoint = getBaseFolder(lang, "rarities")
 
 
 export default async () => {
-	console.log(endpoint)
-	const list = getAllCards()
+	logger.log('Fetching Cards')
+	const list = await getAllCards()
 	const arr: rarityCards = {}
 	for (const i of list) {
-		const card = await fetchCardAsync(i)
+		const card: Card = (await import(i)).default
 
-		if (!isCardAvailable(card, lang)) continue
+		if (!(await isCardAvailable(card, lang))) continue
 
 		const c = card.rarity
 
@@ -38,6 +38,7 @@ export default async () => {
 		if (arr.hasOwnProperty(cat)) {
 			const cards: Array<Card> = arr[cat];
 			const rCat: Rarity = parseInt(cat)
+			logger.log('Processing Rarity', TranslationUtil.translate("rarity", rCat, lang))
 			const toSave: RaritySingle = {
 				id: rCat,
 				name: TranslationUtil.translate("rarity", rCat, lang),
@@ -54,5 +55,5 @@ export default async () => {
 			await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave))
 		}
 	}
-	console.log('ended ' + endpoint)
+	logger.log('ended ' + endpoint)
 }

endpoints/retreat/index.ts

@@ -1,26 +1,25 @@
 import Card from "@tcgdex/sdk/interfaces/Card"
-import { getAllCards2, getBaseFolder } from "../util"
+import { getAllCards, getAllCards2, getBaseFolder } from "../util"
 import { promises as fs } from 'fs'
 import { isCardAvailable } from "../cardUtil"
 import { RetreatList } from '@tcgdex/sdk/interfaces/Retreat'
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Retreat/Index'
+import Logger from '@dzeio/logger'
+const logger = new Logger('retreat/index')
 
 const lang = (process.env.CARDLANG || "en") as Langs
 const endpoint = getBaseFolder(lang, "retreat")
 
 export default async () => {
-	console.log(endpoint)
-	const files = await getAllCards2()
+	logger.log('Fetching Cards')
+	const files = await getAllCards()
 	const count: Array<number> = []
 	for (let file of files) {
-		file = file.replace("./", "../../")
-		const card: Card = await require(file).default
+		const card: Card = (await import(file)).default
 
 		if (
-			!isCardAvailable(card, lang) ||
+			!(await isCardAvailable(card, lang)) ||
 			!card.retreat ||
 			count.includes(card.retreat)
 		) continue
@@ -31,8 +30,9 @@ export default async () => {
 		count: count.length,
 		list: count
 	}
+	logger.log('Writingto file')
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(list))
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/retreat/item.ts

@@ -1,26 +1,25 @@
 import Card from "@tcgdex/sdk/interfaces/Card"
-import { getAllCards2, getBaseFolder } from "../util"
+import { getAllCards, getAllCards2, getBaseFolder } from "../util"
 import { promises as fs } from 'fs'
 import { isCardAvailable, cardToCardSimple } from "../cardUtil"
 import { RetreatSingle } from '@tcgdex/sdk/interfaces/Retreat'
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Retreat/Item'
+import Logger from '@dzeio/logger'
+const logger = new Logger('retreat/item')
 
 const lang = (process.env.CARDLANG || "en") as Langs
 const endpoint = getBaseFolder(lang, "retreat")
 
 export default async () => {
-	console.log(endpoint)
-	const files = await getAllCards2()
+	logger.log('Fetching cards')
+	const files = await getAllCards()
 	const count: Array<Array<Card>> = []
 	for (let file of files) {
-		file = file.replace("./", "../../")
-		const card: Card = await require(file).default
+		const card: Card = (await import(file)).default
 
 		if (
-			!isCardAvailable(card, lang) ||
+			!(await isCardAvailable(card, lang)) ||
 			!card.retreat
 		) continue
 		if (!(card.retreat in count)) count[card.retreat] = []
@@ -29,6 +28,7 @@ export default async () => {
 
 	for (const retreat in count) {
 		if (count.hasOwnProperty(retreat)) {
+			logger.log('Processing Retreat', retreat)
 			const cardArr = count[retreat];
 
 			const item: RetreatSingle = {
@@ -40,5 +40,5 @@ export default async () => {
 			await fs.writeFile(`${endpoint}/${item.id}/index.json`, JSON.stringify(item))
 		}
 	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/setUtil.ts

@@ -5,7 +5,7 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { SetSimple, SetSingle } from "@tcgdex/sdk/interfaces/Set"
 import { cardToCardSimple } from "./cardUtil"
 import { CardSimple } from "@tcgdex/sdk/interfaces/Card"
-import { getAllCards2 } from "./util"
+import { getAllCards, getAllCards2, smartGlob } from "./util"
 
 interface t<T = Set> {
 	[key: string]: T
@@ -17,18 +17,18 @@ export function isSet(set: Set | {name: string, code: string}): set is Set {
 	return "releaseDate" in set
 }
 
-export function getSet(card: Card): Set {
+export async function getSet(card: Card): Promise<Set> {
 	if (!(card.set.code in setCache)) {
 		if (isSet(card.set)) setCache[card.set.code] = card.set
-		let setPath = glob.sync(`./db/sets/**/${card.set.code}.js`)[0]
+		let setPath = (await smartGlob(`./db/sets/**/${card.set.code}.js`))[0]
 		setPath = setPath.replace('./', '../')
 		setCache[card.set.code] = require(setPath).default
 	}
 	return setCache[card.set.code]
 }
 
-export function fetchSet(expansion: string, set: string): Set {
-	return require(`../db/sets/${expansion}/${set}.js`).default
+export async function fetchSet(expansion: string, set: string): Promise<Set> {
+	return (await import(`../db/sets/${expansion}/${set}.js`)).default
 }
 
 export function isSetAvailable(set: Set, lang: Langs) {
@@ -47,18 +47,21 @@ export function setToSetSimple(set: Set, lang: Langs): SetSimple {
 }
 
 export async function getSetCards(set: Set, lang: Langs): Promise<Array<CardSimple>> {
-	const cardes = getAllCards2(set.code)
+	const cardes = await getAllCards(set.code, set.expansionCode ?? set.expansion.code)
 	const cards: Array<Card> = []
 	for (let el of cardes) {
-		el = el.replace("./", "../")
-		const card: Card = require(el).default
+		el = el.replace("../../", "../")
+		const card: Card = (await import(el)).default
 
 		cards.push(
 			card
 		)
 	}
 	return await Promise.all(cards.sort((a, b) => {
-		if (!isNaN(parseInt(a.localId + "")) && !isNaN(parseInt(b.localId + ""))) {
+		if (
+			!isNaN(parseInt(a.localId + "")) &&
+			!isNaN(parseInt(b.localId + ""))
+		) {
 			return parseInt(a.localId + "") - parseInt(b.localId + "")
 		}
 		return a.localId > b.localId ? 1 : -1

endpoints/sets/index.ts

@@ -3,10 +3,11 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
 import { SetSimple, SetList } from "@tcgdex/sdk/interfaces/Set"
 import { getAllSets, getBaseFolder } from "../util"
-import { isSetAvailable, setToSetSimple } from "../setUtil"
+import { fetchSet, isSetAvailable, setToSetSimple } from "../setUtil"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Sets/Index'
+import Logger from '@dzeio/logger'
+import { getExpansionFromSetName } from "../expansionUtil"
+const logger = new Logger('sets/index')
 
 
 const lang = process.env.CARDLANG as Langs || "en"
@@ -14,19 +15,20 @@ const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "sets")
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching sets')
 
 	const list = await getAllSets()
 	let items: Array<Set> = []
 	for (let el of list) {
-		el = el.replace("./", "../../")
-		const set: Set = require(el).default
+		const expansion = (await getExpansionFromSetName(el))
+		const set: Set = await fetchSet(expansion.code, el)
 
 		if (!isSetAvailable(set, lang)) continue
 		items.push(
 			set
 		)
 	}
+	logger.log('Procesing Sets')
 
 	items = items.sort((a, b) => a.releaseDate > b.releaseDate ? 1 : -1)
 
@@ -40,5 +42,5 @@ export default async () => {
 	await fs.mkdir(`${endpoint}`, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(cardList))
 
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/sets/item.ts

@@ -2,21 +2,24 @@ import { getBaseFolder, getAllSets } from "../util"
 import Set from "@tcgdex/sdk/interfaces/Set"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
-import { isSetAvailable, setToSetSingle } from "../setUtil"
+import { fetchSet, isSetAvailable, setToSetSingle } from "../setUtil"
+import { getExpansionFromSetName } from "../expansionUtil"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Sets/Item'
+import Logger from '@dzeio/logger'
+const logger = new Logger('sets/item')
 
 const lang = process.env.CARDLANG as Langs || "en"
 
 const endpoint = getBaseFolder(lang, "sets")
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching Sets')
 	const list = await getAllSets()
+	logger.log(list)
 	for (let el of list) {
-		el = el.replace("./", "../../")
-		const set: Set = require(el).default
+		logger.log('Processing set', el)
+		const expansion = (await getExpansionFromSetName(el))
+		const set: Set = await fetchSet(expansion.code, el)
 
 		if (!isSetAvailable(set, lang)) continue
 
@@ -25,5 +28,5 @@ export default async () => {
 	}
 
 
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/sets/subitem.ts

@@ -1,14 +1,15 @@
-import { getBaseFolder, getAllSets } from "../util"
+import { getBaseFolder, getAllSets, getAllCards } from "../util"
 import Set from "@tcgdex/sdk/interfaces/Set"
 import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import { promises as fs } from 'fs'
-import { isSetAvailable } from "../setUtil"
+import { fetchSet, isSetAvailable } from "../setUtil"
 import { getAllCards2 } from "../util"
 import Card from "@tcgdex/sdk/interfaces/Card"
 import { cardToCardSingle, isCardAvailable } from "../cardUtil"
+import { getExpansionFromSetName } from "../expansionUtil"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Sets/SubItem'
+import Logger from '@dzeio/logger'
+const logger = new Logger('sets/subitem')
 
 
 const lang = process.env.CARDLANG as Langs || "en"
@@ -16,27 +17,28 @@ const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "sets")
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching Sets')
 	const list = await getAllSets()
 	for (let el of list) {
-		el = el.replace("./", "../../")
-		const set: Set = require(el).default
+		const expansion = (await getExpansionFromSetName(el))
+		const set: Set = await fetchSet(expansion.code, el)
 
 		if (!isSetAvailable(set, lang)) continue
 
-		const lit = await getAllCards2(set.code)
+		const lit = await getAllCards(set.code, set?.expansionCode ?? set.expansion.code)
+		logger.log('Fetching/Writing Cards for set', el)
 		for (let i of lit) {
-			i = i.replace("./", "../../")
-			const card: Card = require(i).default
+			const card: Card = (await import(i)).default
 
-			if (!isCardAvailable(card, lang)) continue
+			if (!(await isCardAvailable(card, lang))) continue
 
-			await fs.mkdir(`${endpoint}/${set.code}/${card.localId}`, {recursive: true})
-			await fs.writeFile(`${endpoint}/${set.code}/${card.localId}/index.json`, JSON.stringify(await cardToCardSingle(card, lang)))
+			const localId = card.localId === '?' ? '%3F' : card.localId
+
+			await fs.mkdir(`${endpoint}/${set.code}/${localId}`, {recursive: true})
+			await fs.writeFile(`${endpoint}/${set.code}/${localId}/index.json`, JSON.stringify(await cardToCardSingle(card, lang)))
 		}
 
 	}
 
-
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/tags/index.ts

@@ -11,7 +11,7 @@ const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "tags")
 
 export default async () => {
-	logger.log(endpoint)
+	logger.log('Fetching Tags')
 
 	const list: Array<TagSimple> = []
 	for (const cat of Object.values(Tag)) {
@@ -29,5 +29,5 @@ export default async () => {
 
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(res))
-	logger.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/tags/item.ts

@@ -9,7 +9,7 @@ import Tag, { TagSingle } from "@tcgdex/sdk/interfaces/Tag"
 
 import Logger from '@dzeio/logger'
 import { objectKeys, objectSize } from '@dzeio/object-util'
-const logger = new Logger('Tags/Item')
+const logger = new Logger('tags/item')
 
 type tagCards = {
 	[key in Tag]?: Array<Card>
@@ -20,20 +20,19 @@ const endpoint = getBaseFolder(lang, "tags")
 
 
 export default async () => {
-	logger.log('part 1')
-	const list = getAllCards()
+	logger.log('Fetching cards')
+	const list = await getAllCards()
 	const arr: tagCards = {}
 	for (const i of list) {
-		const card = await fetchCardAsync(i)
+		const card: Card = (await import(i)).default
 
-		if (!isCardAvailable(card, lang) || !card.tags) continue
+		if (!(await isCardAvailable(card, lang)) || !card.tags) continue
 
 		for (const tag of card.tags) {
 			if (!(tag in arr)) arr[tag] = []
 			arr[tag].push(card)
 		}
 	}
-	logger.log('part 2')
 	for (const type in arr) {
 		if (arr.hasOwnProperty(type)) {
 			const cards: Array<Card> = arr[type];

endpoints/types/index.ts

@@ -8,11 +8,11 @@ import { List } from "@tcgdex/sdk/interfaces/General"
 const lang = process.env.CARDLANG as Langs || "en"
 const endpoint = getBaseFolder(lang, "types")
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Types/Index'
+import Logger from '@dzeio/logger'
+const logger = new Logger('types/index')
 
 export default async () => {
-	console.log(endpoint)
+	logger.log('Fetching types')
 	const typeArr: Array<TypeSimple> = []
 	for (const i of Object.values(Type)) {
 		if (typeof i !== "number") continue
@@ -27,7 +27,8 @@ export default async () => {
 		list: typeArr
 	}
 
+	logger.log('Writing types')
 	await fs.mkdir(endpoint, {recursive: true})
 	await fs.writeFile(`${endpoint}/index.json`, JSON.stringify(typeList))
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/types/item.ts

@@ -6,9 +6,8 @@ import { Langs } from "@tcgdex/sdk/interfaces/LangList"
 import TranslationUtil from "@tcgdex/sdk/TranslationUtil"
 import { promises } from "fs"
 
-import { logger as console } from '@dzeio/logger'
-console.prefix = 'Types/Item'
-
+import Logger from '@dzeio/logger'
+const logger = new Logger('types/item')
 type typeCards = {
 	[key in Type]?: Array<Card>
 }
@@ -18,13 +17,13 @@ const endpoint = getBaseFolder(lang, "types")
 
 
 export default async () => {
-	console.log(endpoint)
-	const list = getAllCards()
+	logger.log('Fetching cards')
+	const list = await getAllCards()
 	const arr: typeCards = {}
 	for (const i of list) {
-		const card = await fetchCard(i)
+		const card: Card = (await import(i)).default
 
-		if (!isCardAvailable(card, lang) || !card.type) continue
+		if (!(await isCardAvailable(card, lang)) || !card.type) continue
 
 		for (const type of card.type) {
 			if (!(type in arr)) arr[type] = []
@@ -33,9 +32,12 @@ export default async () => {
 	}
 
 	for (const type in arr) {
-		if (arr.hasOwnProperty(type)) {
+		if (!Object.prototype.hasOwnProperty.call(arr, type)) {
+			continue
+		}
 		const cards: Array<Card> = arr[type];
 		const rType: Type = parseInt(type)
+		logger.log('Processing type', TranslationUtil.translate("type", rType, lang))
 		const toSave: TypeSingle = {
 			id: rType,
 			name: TranslationUtil.translate("type", rType, lang),
@@ -51,6 +53,5 @@ export default async () => {
 		await promises.writeFile(`${index}/index.json`, JSON.stringify(toSave))
 		await promises.writeFile(`${name}/index.json`, JSON.stringify(toSave))
 	}
-	}
-	console.log('ended ' + endpoint)
+	logger.log('Finished')
 }

endpoints/util.ts

@@ -1,70 +1,25 @@
-import { promises as fs, promises } from 'fs'
 import * as glob from 'glob'
 import fetch from 'node-fetch'
 
 const VERSION = 'v1'
 
-export function getAllCards(set = "**", expansion = "**") {
-	return glob.sync(`./db/cards/${expansion}/${set}/*.js`).map(el => {
-		return el.substr(11, el.length-10-1-3)
-	})
+export async function getAllCards(set = "**", expansion = "**") {
+	return (await smartGlob(`./db/cards/${expansion}/${set}/*.js`)).map((it) => it.replace('./', '../../'))
 }
 
 export function getAllCards2(set = "**", expansion = "**") {
 	return glob.sync(`./db/cards/${expansion}/${set}/*.js`)
 }
 
-export function getAllSets(expansion = "**", nameOnly = false) {
-	if (nameOnly) return glob.sync(`./db/sets/${expansion}/*.js`).map(el => el.substr(11+expansion.length, el.length-(10+expansion.length)-1-3))
-	return glob.sync(`./db/sets/${expansion}/*.js`)
-}
-
-export function getAllCardsJSON() {
-	return glob.sync("./db/cards/**/**/*.json")
-}
-
-async function listFolder(folder: string): Promise<Array<string>> {
-	const files = await fs.readdir(folder)
-	const res = []
-	for (const file of files) {
-		if (file.endsWith(".json")) res.push(`${folder}/${file}`)
-		if ((await fs.stat(`${folder}/${file}`)).isDirectory()) {
-			res.push(...await listFolder(`${folder}/${file}`))
-		}
-	}
-	return res
+export async function getAllSets(expansion = "**") {
+	return (await smartGlob(`./db/sets/${expansion}/*.js`))
+		.map(el => el.substring(el.lastIndexOf('/') + 1, el.lastIndexOf('.')))
 }
 
 export function getBaseFolder(lang: string, endpoint: string) {
 	return `./dist/${VERSION}/${lang}/${endpoint}`
 }
 
-export async function del(path: string) {
-	let files = []
-	const promiseArr = []
-	try {
-		files = await promises.readdir(path)
-	} catch {
-		return
-	}
-	if (files.length > 0) {
-		for (const file of files) {
-			const fPath = `${path}/${file}`
-			if ((await promises.stat(fPath)).isDirectory()) {
-				promiseArr.push(
-					del(fPath)
-				)
-			} else {
-				promiseArr.push(
-					promises.unlink(fPath)
-				)
-			}
-		}
-	}
-	await Promise.all(promiseArr)
-	await promises.rmdir(path)
-}
-
 export function urlize(str: string): string {
 	str = str.normalize('NFD').replace(/[\u0300-\u036f]/g, "")
 	return str.replace(/ /g, "-").toLowerCase()
@@ -84,3 +39,12 @@ export async function fetchRemoteFile<T = any>(url: string): Promise<T> {
 	}
 	return fileCache[url]
 }
+
+const globCache: Record<string, Array<string>> = {}
+
+export async function smartGlob(query: string) {
+	if (!globCache[query]) {
+		globCache[query] = await new Promise((res) => glob(query, (err, matches) => res(matches)))
+	}
+	return globCache[query]
+}