import { ConnectConfig, Client, SFTPWrapper } from "ssh2"; import { Stats, InputAttributes } from 'ssh2-streams' import { promises as fs } from 'fs' import { posix as pathJS } from 'path' import Queue from '@dzeio/queue' import Logger from '@dzeio/logger' const logger = new Logger('SFTPPromise') export default class SFTPPromise { private sftp?: SFTPWrapper private conn: Client = new Client() public debug = false public tmpFolder?: string public constructor(public config: ConnectConfig) {} public connect() { return new Promise((res, rej) => { this.conn.on('ready', () => { this.conn.sftp((err, sftpLocal) => { this.sftp = sftpLocal res() }) }).connect(this.config) }) } public async makeTemporaryFolder() { this.l('Making temporary Folder') this.tmpFolder = await fs.mkdtemp('tcgdex-generator') } public stat(path: string): Promise { const sftp = this.getSFTP() return new Promise((res, rej) => { sftp.stat(path, (err, stats) => { if (err) { rej(err) } res(stats) }) }) } public async exists(path: string): Promise { try { await this.stat(path) return true } catch { return false } } public async mkdir(path: string, recursive = false, attributes?: InputAttributes, ) { this.l('Creating remote folder', path) if (recursive) { this.l('Making temporary Folder') const folders = path.split('/') let current = '' for (const item of folders) { current += '/' + item if (!(await this.exists(current))) { await this.mkdir(current) } } return } if (await this.exists(path)) { return } const sftp = this.getSFTP() return new Promise((res, rej) => { const result = (err?: any) => { if (err) { rej(err) } res() } if (attributes) { sftp.mkdir(path, attributes, result) } else { sftp.mkdir(path, result) } }) } public async upload(localFile: Buffer|string, path: string) { const sftp = this.getSFTP() if (typeof localFile !== 'string') { if (!this.tmpFolder) { await this.makeTemporaryFolder() } const tmpFile = pathJS.join(this.tmpFolder as string, Math.random().toString().replace('.', '')) // this.l('Writing to temporary Folder') await fs.writeFile(tmpFile, localFile) localFile = tmpFile } const remoteFolder = pathJS.dirname(path).replace(/\\/g, '/') if (!(await this.exists(remoteFolder))) { await this.mkdir(remoteFolder, true) } return new Promise((result, rej) => { this.l('Sending file', path) sftp.fastPut(localFile as string, path, { concurrency: 1, step: (uploaded, u, total) => { this.l(path.substr(path.lastIndexOf('/')), Math.round(uploaded*100/total),'%', '/', 100, '%') } }, (err) => { if (err) { this.l('Error fastPutting file', localFile, 'to', path) rej(err) } this.l('Done') result() return }) }) } private queue = new Queue(50, 10) private filesToUpload = 0 private filesUploaded = 0 public async listDir(path: string, exclude?: RegExp): Promise> { const files = await fs.readdir(path) logger.log('Reading', path) let res: Array = [] for (const file of files) { if (exclude?.test(file)) {continue} const filePath = `${path}/${file}` const stat = await fs.stat(filePath) if (stat.isDirectory()) { res = res.concat(await this.listDir(filePath)) } else { res.push(filePath) } } return res } private lastTimeDiff: Array = Array.from(new Array(900), () => 0) public async uploadDir(localPath: string, remotePath: string, exclude?: RegExp, root = true) { if (root) { this.filesToUpload = 0 this.filesUploaded = 0 } logger.log('Listing files...') const files = await this.listDir(localPath, exclude) console.log(files) logger.log('Running !') this.filesToUpload += files.length this.queue.start() for (const file of files) { // console.log('t1') if (exclude?.test(file)) {continue} // console.log('t1') const filePath = file const remoteFilePath = `${remotePath}/${file.replace(localPath, '')}` // console.log('t1') const now = new Date().getTime() await this.queue.add( this.upload(filePath, remoteFilePath).then(() => { this.filesUploaded++ this.lastTimeDiff.push(new Date().getTime() - now) this.lastTimeDiff.shift() const time = ((this.filesToUpload-this.filesUploaded)*(this.lastTimeDiff.reduce((p, c) => p + c, 0)/this.lastTimeDiff.length)/10000) console.log(`Files uploaded ${(this.filesUploaded * 100 / this.filesToUpload).toFixed(0)}% ${time > 60 ? `${(time/60).toFixed(0)}m` : `${time.toFixed(0)}s`} ${this.filesUploaded}/${this.filesToUpload}`) }).catch((err) => logger.log(err, 'Error uploading', filePath, 'to', remoteFilePath)) ) } if (root) { await this.queue.waitEnd() console.log('DONE !') } } private getSFTP(): SFTPWrapper { if (!this.sftp) { throw new Error('please use SFTPPromise.connect() before') } return this.sftp } private l(...messages: Array) { if (this.debug) { logger.log(...messages) } } }