From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 2606576A80 for ; Fri, 16 Jul 2021 16:18:10 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1903111A81 for ; Fri, 16 Jul 2021 16:18:10 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id B7CB011A73 for ; Fri, 16 Jul 2021 16:18:08 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7DBFB42116 for ; Fri, 16 Jul 2021 16:18:08 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com Date: Fri, 16 Jul 2021 16:18:07 +0200 Message-Id: <20210716141807.1182069-1-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.174 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_ASCII_DIVIDERS 0.8 Spam that uses ascii formatting tricks KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH pve-eslint] use worker_threads for linting X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 16 Jul 2021 14:18:10 -0000 instead linting all files in the main thread, use worker threads for that (4 by default) and add the '-t' switch to able to control that since nodejs always wants a module/script to load for a thread, give a small script that load the file itself a basic benchmark of eslint of pve-manager showed some performance gains: Benchmark #1: Current Time (mean ± σ): 6.449 s ± 0.207 s [User: 9.818 s, System: 0.362 s] Range (min … max): 6.190 s … 6.773 s 10 runs Benchmark #2: 2Threads Time (mean ± σ): 4.525 s ± 0.143 s [User: 12.646 s, System: 0.584 s] Range (min … max): 4.324 s … 4.799 s 10 runs Benchmark #3: 4Threads Time (mean ± σ): 3.443 s ± 0.041 s [User: 16.393 s, System: 0.672 s] Range (min … max): 3.354 s … 3.508 s 10 runs Benchmark #4: 8Threads Time (mean ± σ): 2.835 s ± 0.052 s [User: 22.343 s, System: 1.023 s] Range (min … max): 2.764 s … 2.934 s 10 runs Summary '8Threads' ran 1.21 ± 0.03 times faster than '4Threads' 1.60 ± 0.06 times faster than '2Threads' 2.28 ± 0.08 times faster than 'Current' after 8 threads, there were no real performance benefits since the overhead to load the eslint js file seems to be the biggest factor. Signed-off-by: Dominik Csapak --- i recently looked how we could do that, but did not find the docs for the worker_threads. i stumbled upon it today, and quickly threw this together. the self loading inline script is a bit of a hack, but the only way to do it better would be to ship eslint and the worker code as module, but i did not look into that for now... src/app.js | 65 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/src/app.js b/src/app.js index 9226234..71a88bc 100644 --- a/src/app.js +++ b/src/app.js @@ -1,9 +1,18 @@ -(function() { +(async function() { 'use strict'; const path = require('path'); const color = require('colors'); const program = require('commander'); +const worker = require('worker_threads'); + +if (!worker.isMainThread) { + const data = worker.workerData; + const cli = new eslint.CLIEngine(data.cliOptions); + const report = cli.executeOnFiles(data.files); + worker.parentPort.postMessage(report); + process.exit(0); +} program .usage('[options] []') @@ -11,6 +20,7 @@ program .option('-e, --extend ', 'uses ontop of default eslint config.') .option('-f, --fix', 'if set, fixes will be applied.') .option('-s, --strict', 'if set, also exit uncleanly on warnings') + .option('-t, --threads ', 'how many worker_threads should be used (default=4)') .option('--output-config', 'if set, only output the config as JSON and exit.') ; @@ -39,6 +49,11 @@ if (!paths.length) { paths = [process.cwd()]; } +let threadCount = 4; +if (program.threads) { + threadCount = program.threads; +} + const defaultConfig = { parserOptions: { ecmaVersion: 2020, @@ -280,20 +295,56 @@ if (program.outputConfig) { process.exit(0); } -const cli = new eslint.CLIEngine({ +const cliOptions = { baseConfig: config, useEslintrc: true, fix: !!program.fix, cwd: process.cwd(), -}); +}; + +let lintFiles = async function(files) { + return new Promise((resolve, reject) => { + const child = new worker.Worker( + ` + const worker = require('worker_threads'); + let file = worker.workerData.__filename; + delete worker.workerData.__filename; + require(file); + `, + { + eval: true, + workerData: { + __filename, + cliOptions, + files, + } + } + ); + child.on('message', resolve); + child.on('error', reject); + child.on('exit', (code) => { + if (code !== 0) + reject(new Error(`Worker stopped with exit code ${code}`)); + }); + }); +}; + +let promises = []; +let filesPerThread = Math.round(paths.length / threadCount); +for (let i = 0; i < (threadCount - 1); i++) { + let files = paths.splice(0, filesPerThread); + promises.push(lintFiles(files)); +} +// the remaining paths +promises.push(lintFiles(paths)); -const report = cli.executeOnFiles(paths); +let results = (await Promise.all(promises)).map(res => res.results).flat(1); let exitcode = 0; let files_err = [], files_warn = [], files_ok = []; let fixes = 0; console.log('------------------------------------------------------------'); -report.results.forEach(function(result) { +results.forEach(function(result) { let filename = path.relative(process.cwd(), result.filePath); let msgs = result.messages; let max_sev = 0; @@ -345,7 +396,7 @@ report.results.forEach(function(result) { console.log('------------------------------------------------------------'); }); -if (report.results.length > 1) { +if (results.length > 1) { console.log(`${color.bold(files_ok.length + files_err.length)} files:`); if (files_err.length > 0) { console.log(color.red(` ${color.bold(files_err.length)} files have Errors`)); @@ -364,7 +415,7 @@ console.log('------------------------------------------------------------'); if (program.fix) { if (fixes > 0) { console.log(`Writing ${color.bold(fixes)} fixed files...`); - eslint.CLIEngine.outputFixes(report); + eslint.CLIEngine.outputFixes({ results }); console.log('Done'); } else { console.log("No fixable Errors/Warnings found."); -- 2.30.2