Cleaned-up (#16)

* Cleaned-up

Signed-off-by: Avior <florian.bouillon@delta-wings.net>

* Update rule

Signed-off-by: Avior <florian.bouillon@delta-wings.net>

* Fix interface

Signed-off-by: Avior <florian.bouillon@delta-wings.net>
This commit is contained in:
Florian Bouillon 2021-08-12 22:55:27 +02:00 committed by GitHub
parent 95d88f081a
commit dd41d619d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 3444 additions and 1832 deletions

View File

@ -1,6 +1 @@
UPLOAD_REMOTE=sftp.server.com
UPLOAD_DIST=/path/to/remote
UPLOAD_USERNAME=username
UPLOAD_PASSWORD=password
TCGDEX_COMPILER_LANG=en TCGDEX_COMPILER_LANG=en

View File

@ -1,322 +0,0 @@
/**
* ESLint custom configuration v1.0.0
* packages needed:
* eslint
* for Typescript
* @typescript-eslint/parser
* @typescript-eslint/eslint-plugin
*/
module.exports = {
env: {
browser: true,
es6: true,
node: true
},
extends: [
"eslint:all",
"plugin:@typescript-eslint/recommended",
],
globals: {
Atomics: "readonly",
SharedArrayBuffer: "readonly"
},
parser: "@typescript-eslint/parser",
parserOptions: {
project: "tsconfig.json"
},
plugins: [
"@typescript-eslint"
],
root: true,
rules: {
// Tab indent and force switch to have one too
indent: [
"error",
"tab",
{SwitchCase: 1}
],
// \n linebreak
"linebreak-style": [
"error",
"unix"
],
// Disable eslint default quote because Typescript replace it
quotes: "off",
"@typescript-eslint/quotes": [
"error",
"single",
{ avoidEscape: true }
],
// Simply diallow using ';' unless mandatory
semi: "off",
"@typescript-eslint/semi": [
"error",
"never",
{ "beforeStatementContinuationChars": "always"}
],
// Disallow things that do nothing in the end
"no-unused-expressions": "off",
"@typescript-eslint/no-unused-expressions": [
"error",
{ "enforceForJSX": true,}
],
// force overloads to be next to one another
"@typescript-eslint/adjacent-overload-signatures": "error",
// Force to use `Array<thing>
"@typescript-eslint/array-type": [
"error",
{ default: 'generic' }
],
// Warn when no return type is specified
"@typescript-eslint/explicit-module-boundary-types": "warn",
// disallow certain types not safe
"@typescript-eslint/ban-types": [
"error",
{
"types": {
"{}": false
}
}
],
"@typescript-eslint/consistent-type-assertions": "error",
"@typescript-eslint/consistent-type-definitions": "error",
"@typescript-eslint/explicit-member-accessibility": [
"error",
{
accessibility: "explicit"
}
],
"@typescript-eslint/interface-name-prefix": "off",
"@typescript-eslint/member-delimiter-style": [
"error",
{
multiline: {
delimiter: "none",
requireLast: true
},
singleline: {
delimiter: "comma",
requireLast: false
}
}
],
"@typescript-eslint/member-ordering": "error",
"@typescript-eslint/no-empty-function": "error",
"@typescript-eslint/no-empty-interface": "error",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-misused-new": "error",
"@typescript-eslint/no-namespace": "error",
"@typescript-eslint/no-parameter-properties": "off",
"@typescript-eslint/no-use-before-define": "off",
"@typescript-eslint/prefer-for-of": "error",
"@typescript-eslint/prefer-function-type": "error",
"@typescript-eslint/prefer-namespace-keyword": "error",
"@typescript-eslint/triple-slash-reference": "error",
"@typescript-eslint/type-annotation-spacing": "error",
"@typescript-eslint/unified-signatures": "error",
"arrow-body-style": "error",
"arrow-parens": [
"error",
"always"
],
camelcase: "error",
complexity: "off",
"constructor-super": "error",
curly: "error",
"dot-notation": "error",
"eol-last": "error",
eqeqeq: [
"error",
"smart"
],
"guard-for-in": "warn",
"id-blacklist": [
"error",
"any",
"Number",
"number",
"String",
"string",
"Boolean",
"boolean",
"Undefined"
],
"id-match": "error",
"max-classes-per-file": [
"error",
1
],
"max-len": [
"warn",
{
code: 200
}
],
"@typescript-eslint/no-inferrable-types": "off",
"new-parens": "error",
"no-bitwise": "error",
"no-caller": "error",
"no-cond-assign": "error",
"no-debugger": "error",
"no-empty": "error",
"no-eval": "error",
"no-fallthrough": "off",
"no-invalid-this": "off",
"no-multiple-empty-lines": "error",
"no-new-wrappers": "error",
"no-shadow": [
"error",
{
hoist: "all"
}
],
"no-throw-literal": "error",
"no-trailing-spaces": "error",
"no-undef-init": "error",
"no-underscore-dangle": "error",
"no-unsafe-finally": "error",
"no-unused-labels": "error",
"no-unused-vars": "off",
"no-var": "error",
"object-shorthand": "error",
"one-var": [
"error",
"never"
],
"prefer-const": "error",
"quote-props": [
"error",
"consistent-as-needed"
],
"radix": "error",
"space-before-function-paren": "off",
"@typescript-eslint/space-before-function-paren": ["error", {
asyncArrow: "always",
anonymous: "never",
named: "never"
}],
"spaced-comment": "error",
"use-isnan": "error",
"valid-typeof": "off",
// some tests from eslint:all
"no-tabs": "off",
"padded-blocks": [
"error",
{
"blocks": "never",
"classes": "always",
"switches": "never"
}
],
"sort-imports": "off",
"no-console": "off",
"function-call-argument-newline": [
"error",
"consistent"
],
"dot-location": [
"error",
"property"
],
"object-curly-spacing": [
"error",
"always"
],
"array-element-newline": [
"error",
"consistent"
],
"function-paren-newline": [
"error",
"consistent"
],
"no-use-before-define": "off",
"@typescript-eslint/no-use-before-define": "error",
"capitalized-comments": "off",
"multiline-comment-style": "off",
"no-extra-parens": "off",
"@typescript-eslint/no-extra-parens": [
"error",
"all",
{ "ignoreJSX": "multi-line" }
],
"func-style": [
"error",
"declaration",
{ "allowArrowFunctions": true }
],
"no-ternary": "off",
"multiline-ternary": "off",
"no-magic-numbers": "off",
"max-lines-per-function": [
"warn",
{
"skipBlankLines": true,
"skipComments": true
}
],
"prefer-promise-reject-errors": "warn",
"object-property-newline": [
"error",
{ "allowAllPropertiesOnSameLine": true }
],
"no-await-in-loop": "warn",
"no-undefined": "off",
"id-length": "warn",
"class-methods-use-this": "off",
"array-bracket-newline": [
"error",
"consistent"
],
"no-confusing-arrow": "off",
"no-nested-ternary": "off",
"no-mixed-operators": "off",
"max-statements": [
"warn",
15
],
"semi-style": [
"error",
"first"
],
"no-useless-constructor": "off",
"@typescript-eslint/no-useless-constructor": "error",
"lines-between-class-members": "off",
"@typescript-eslint/lines-between-class-members": "error",
"max-lines": [
"warn",
{
"max": 500,
"skipBlankLines": true,
"skipComments": true
}
],
"no-plusplus": "off",
"id-length": [
"warn",
{ "exceptions": ["_"] }
],
"default-param-last": "off",
// "@typescript-eslint/default-param-last": "error",
// Temporary OFF
"@typescript-eslint/default-param-last": "off",
"no-continue": "off",
"require-atomic-updates": "off",
"require-await": "off",
"prefer-destructuring": "off",
"max-params": ["warn", 5]
}
};

6
.eslintrc.json Normal file
View File

@ -0,0 +1,6 @@
{
"extends": "./node_modules/@dzeio/config/eslint/typescript",
"rules": {
"require-atomic-updates": "off"
}
}

View File

@ -97,12 +97,3 @@ jobs:
github_token: ${{ secrets.PUSH_TOKEN }} github_token: ${{ secrets.PUSH_TOKEN }}
directory: ./dist directory: ./dist
repository: tcgdex/distribution repository: tcgdex/distribution
# - name: Upload
# run: yarn upload
# if: ${{ steps.commit.outcome == 'success' && (github.event_name == 'push' || github.event_name == 'repository_dispatch') }}
# env:
# UPLOAD_REMOTE: ${{ secrets.UPLOAD_REMOTE }}
# UPLOAD_DIST: ${{ secrets.UPLOAD_DIST }}
# UPLOAD_USERNAME: ${{ secrets.UPLOAD_USERNAME }}
# UPLOAD_PASSWORD: ${{ secrets.UPLOAD_PASSWORD }}

View File

@ -1,211 +0,0 @@
/* eslint-disable max-statements */
import { ConnectConfig, Client, SFTPWrapper } from 'ssh2'
import { Stats, InputAttributes } from 'ssh2-streams'
import { promises as fs } from 'fs'
import { posix as pathJS } from 'path'
import Queue from '@dzeio/queue'
import Logger from '@dzeio/logger'
const logger = new Logger('SFTPPromise')
export default class SFTPPromise {
public debug = false
public tmpFolder?: string
private sftp?: SFTPWrapper
private conn: Client = new Client()
private queue = new Queue(50, 10)
private filesToUpload = 0
private filesUploaded = 0
private lastTimeDiff: Array<number> = Array.from(new Array(900), () => 0)
public constructor(public config: ConnectConfig) {}
public connect(): Promise<void> {
return new Promise<void>((res) => {
this.conn.on('ready', () => {
this.conn.sftp((err, sftpLocal) => {
this.sftp = sftpLocal
res()
})
}).connect(this.config)
})
}
public async makeTemporaryFolder(): Promise<void> {
this.l('Making temporary Folder')
this.tmpFolder = await fs.mkdtemp('tcgdex-generator')
}
public stat(path: string): Promise<Stats> {
const sftp = this.getSFTP()
return new Promise((res, rej) => {
sftp.stat(path, (err, stats) => {
if (err) {
rej(err)
}
res(stats)
})
})
}
public async exists(path: string): Promise<boolean> {
try {
await this.stat(path)
return true
} catch {
return false
}
}
public async mkdir(path: string, recursive = false, attributes?: InputAttributes): Promise<void> {
this.l('Creating remote folder', path)
if (recursive) {
this.l('Making temporary Folder')
const folders = path.split('/')
let current = ''
await Promise.all(folders.map(async (item) => {
current += `/${item}`
if (!await this.exists(current)) {
await this.mkdir(current)
}
}))
return
}
if (await this.exists(path)) {
return
}
const sftp = this.getSFTP()
await new Promise<void>((res, rej) => {
const result = (err?: any) => {
if (err) {
rej(err)
}
res()
}
if (attributes) {
sftp.mkdir(path, attributes, result)
} else {
sftp.mkdir(path, result)
}
})
}
public async upload(localFile: Buffer|string, path: string): Promise<void> {
const sftp = this.getSFTP()
let file = localFile
if (typeof file !== 'string') {
if (!this.tmpFolder) {
await this.makeTemporaryFolder()
}
const tmpFile = pathJS.join(
this.tmpFolder as string,
Math.random()
.toString()
.replace('.', '')
)
await fs.writeFile(tmpFile, file)
// IDK there is no race condition
// eslint-disable-next-line require-atomic-updates
file = tmpFile
}
const remoteFolder = pathJS.dirname(path).replace(/\\/gu, '/')
if (!await this.exists(remoteFolder)) {
await this.mkdir(remoteFolder, true)
}
await new Promise<void>((result, rej) => {
this.l('Sending file', path)
sftp.fastPut(file as string, path, {
concurrency: 1,
step: (uploaded, _, total) => {
this.l(path.substr(path.lastIndexOf('/')), Math.round(uploaded * 100 / total), '%', '/', 100, '%')
}
}, (err) => {
if (err) {
this.l('Error fastPutting file', file, 'to', path)
rej(err)
}
this.l('Done')
result()
})
})
}
public async listDir(path: string, exclude?: RegExp): Promise<Array<string>> {
const files = await fs.readdir(path)
logger.log('Reading', path)
const res: Array<string> = []
await Promise.all(files.map(async (file) => {
if (exclude?.test(file)) {
return
}
const filePath = `${path}/${file}`
const stat = await fs.stat(filePath)
if (stat.isDirectory()) {
res.push(...await this.listDir(filePath))
} else {
res.push(filePath)
}
}))
return res
}
public async uploadDir(localPath: string, remotePath: string, exclude?: RegExp, root = true): Promise<void> {
if (root) {
this.filesToUpload = 0
this.filesUploaded = 0
}
logger.log('Listing files...')
const files = await this.listDir(localPath, exclude)
console.log(files)
logger.log('Running !')
this.filesToUpload += files.length
this.queue.start()
for await (const file of files) {
// console.log('t1')
if (exclude?.test(file)) {
continue
}
// console.log('t1')
const filePath = file
const remoteFilePath = `${remotePath}/${file.replace(localPath, '')}`
// console.log('t1')
const now = new Date().getTime()
await this.queue.add(
this.upload(filePath, remoteFilePath)
.then(() => {
this.filesUploaded++
this.lastTimeDiff.push(new Date().getTime() - now)
this.lastTimeDiff.shift()
const time = (this.filesToUpload - this.filesUploaded) * (this.lastTimeDiff.reduce((p, c) => p + c, 0) / this.lastTimeDiff.length) / 10000
console.log(`Files uploaded ${(this.filesUploaded * 100 / this.filesToUpload).toFixed(0)}% ${time > 60 ? `${(time / 60).toFixed(0)}m` : `${time.toFixed(0)}s`} ${this.filesUploaded}/${this.filesToUpload}`)
})
.catch((err) => logger.log(err, 'Error uploading', filePath, 'to', remoteFilePath))
)
}
if (root) {
await this.queue.waitEnd()
console.log('DONE !')
}
}
private getSFTP(): SFTPWrapper {
if (!this.sftp) {
throw new Error('please use SFTPPromise.connect() before')
}
return this.sftp
}
// eslint-disable-next-line id-length
private l(...messages: Array<any>) {
if (this.debug) {
logger.log(...messages)
}
}
}

1
interfaces.d.ts vendored
View File

@ -1,3 +1,4 @@
// eslint-disable-next-line @typescript-eslint/ban-types
export interface Endpoint<Index extends {} = {}, Item extends {} = {}, SubItem extends {} = {}, C = undefined> { export interface Endpoint<Index extends {} = {}, Item extends {} = {}, SubItem extends {} = {}, C = undefined> {
index(common: C): Promise<Index | undefined> index(common: C): Promise<Index | undefined>
item(common: C): Promise<Record<string, Item> | undefined> item(common: C): Promise<Record<string, Item> | undefined>

3421
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -2,6 +2,7 @@
"name": "@tcgdex/compiler", "name": "@tcgdex/compiler",
"version": "2.0.0", "version": "2.0.0",
"repository": "https://github.com/tcgdex/compiler.git", "repository": "https://github.com/tcgdex/compiler.git",
"homepage": "https://github.com/tcgdex/compiler",
"author": "Avior <github@avior.me>", "author": "Avior <github@avior.me>",
"license": "MIT", "license": "MIT",
"private": false, "private": false,
@ -9,28 +10,24 @@
"db:compile": "cd db && tsc --project tsconfig.json", "db:compile": "cd db && tsc --project tsconfig.json",
"db:test": "cd db && tsc --noEmit --project tsconfig.json", "db:test": "cd db && tsc --noEmit --project tsconfig.json",
"start": "node --require ts-node/register --unhandled-rejections=strict main.ts", "start": "node --require ts-node/register --unhandled-rejections=strict main.ts",
"upload": "node --require ts-node/register --unhandled-rejections=strict upload.ts",
"lint": "eslint ./utils ./endpoints ./*.ts --ext .ts" "lint": "eslint ./utils ./endpoints ./*.ts --ext .ts"
}, },
"dependencies": { "dependencies": {
"@dzeio/logger": "^2.0.0-alpha.0", "@dzeio/object-util": "^1.2.0",
"@dzeio/object-util": "^1.0.4", "@tcgdex/sdk": "^2.4.4",
"@dzeio/queue": "^1.2.0", "@types/glob": "^7.1.4",
"@tcgdex/sdk": "^2.2.0-beta", "@types/node": "^16.6.0",
"@types/glob": "^7.1.1", "@types/node-fetch": "^2.5.12",
"@types/node": "^15.0.1", "dotenv": "^10.0.0",
"@types/node-fetch": "^2.5.7", "glob": "^7.1.7",
"@types/ssh2": "^0.5.46", "node-fetch": "^2.6.1",
"dotenv": "^8.2.0", "ts-node": "^10.2.0",
"glob": "^7.1.6", "typescript": "^4.3.5"
"node-fetch": "^2.6.0",
"ssh2": "^0.8.9",
"ts-node": "^9.1.1",
"typescript": "^4.2.3"
}, },
"devDependencies": { "devDependencies": {
"@typescript-eslint/eslint-plugin": "^4.26.0", "@dzeio/config": "^1.1.1",
"@typescript-eslint/parser": "^4.26.0", "@typescript-eslint/eslint-plugin": "^4.29.1",
"eslint": "^7.27.0" "@typescript-eslint/parser": "^4.29.1",
"eslint": "^7.32.0"
} }
} }

View File

@ -2,14 +2,8 @@
"ts-node": { "ts-node": {
"transpileOnly": true "transpileOnly": true
}, },
"extends": "./node_modules/@dzeio/config/tsconfig.base",
"compilerOptions": { "compilerOptions": {
"baseUrl": "./", "baseUrl": "./",
"target": "es5",
"module": "commonjs",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true
} }
} }

View File

@ -1,22 +0,0 @@
import { config } from 'dotenv'
import { ConnectConfig } from 'ssh2'
import SFTPPromise from './SFTPPromise'
config()
const sshConfig: ConnectConfig = {
host: process.env.UPLOAD_REMOTE,
password: process.env.UPLOAD_PASSWORD,
port: 22,
username: process.env.UPLOAD_USERNAME
}
;(async () => {
const client = new SFTPPromise(sshConfig)
// client.debug = true
await client.connect()
const src = `${__dirname}/dist`
const dst = process.env.UPLOAD_DIST as string
await client.uploadDir(src, dst, /\.git/gu)
process.exit(0)
})()

1238
yarn.lock

File diff suppressed because it is too large Load Diff