diff --git a/bun.lockb b/bun.lockb index 70c1e4ffc..9226ec6dc 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/server/bun.lockb b/server/bun.lockb index cb672bee5..e2df89fa4 100755 Binary files a/server/bun.lockb and b/server/bun.lockb differ diff --git a/server/compiler/utils/util.ts b/server/compiler/utils/util.ts index c45c4a9f4..d3d550d7e 100644 --- a/server/compiler/utils/util.ts +++ b/server/compiler/utils/util.ts @@ -1,4 +1,5 @@ import { objectSize } from '@dzeio/object-util' +import Queue from '@dzeio/queue' import { glob } from 'glob' import { exec, spawn } from 'node:child_process' import { writeFileSync } from 'node:fs' @@ -126,7 +127,7 @@ function runCommand(command: string, useSpawn = true): Promise { }) } -let lastEditsCache: Record = {} +const lastEditsCache: Record = {} export async function loadLastEdits() { console.log('Loading Git File Tree...') const firstCommand = 'git ls-tree -r --name-only HEAD ../data' @@ -136,19 +137,35 @@ export async function loadLastEdits() { console.log('Loaded files tree', files.length, 'files') console.log('Loading their last edit time') let processed = 0 - for (let file of files) { + const queue = new Queue(1000, 10) + queue.start() + + for await (let file of files) { file = file.replace(/"/g, '').replace("\\303\\251", "é") - try { - // don't really know why but it does not correctly execute the command when using Spawn - lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false) - } catch { + await queue.add(runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false).then((res) => { + lastEditsCache[file] = res + }) + .catch(() => { console.warn('could not load file', file, 'hope it does not break everything else lol') - } - processed++ - if (processed % 1000 === 0) { - console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) - } + }) + .finally(() => { + processed++ + if (processed % 1000 === 0) { + console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) + } + })) + // try { + // // don't really know why but it does not correctly execute the command when using Spawn + // lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false) + // } catch { + // console.warn('could not load file', file, 'hope it does not break everything else lol') + // } + // processed++ + // if (processed % 1000 === 0) { + // console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`) + // } } + await queue.waitEnd() console.log('done loading files', objectSize(lastEditsCache)) } diff --git a/server/package.json b/server/package.json index f802fc940..df78e30cd 100644 --- a/server/package.json +++ b/server/package.json @@ -12,11 +12,12 @@ "dependencies": { "@dzeio/config": "^1", "@dzeio/object-util": "^1", + "@dzeio/queue": "^1", "@tcgdex/sdk": "^2", "apicache": "^1", "express": "^4", "graphql": "^15", - "graphql-http": "^1.22.1", + "graphql-http": "^1", "ruru": "^2.0.0-beta.14", "swagger-ui-express": "^5.0.1", "yaml": "^2.5.0"