From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 7A7DB76F11 for ; Mon, 19 Jul 2021 12:31:57 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 786ED24F30 for ; Mon, 19 Jul 2021 12:31:57 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5624724F0F for ; Mon, 19 Jul 2021 12:31:56 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id D8535422E3 for ; Mon, 19 Jul 2021 12:31:50 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com Date: Mon, 19 Jul 2021 12:31:49 +0200 Message-Id: <20210719103149.3430829-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210719103149.3430829-1-d.csapak@proxmox.com> References: <20210719103149.3430829-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.164 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_ASCII_DIVIDERS 0.8 Spam that uses ascii formatting tricks KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v2 3/3] use worker_threads for linting X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 19 Jul 2021 10:31:57 -0000 instead linting all files in the main thread, use worker threads for that (4 by default) and add the '-t' switch to able to control that a basic benchmark of eslint of pve-manager showed some performance gains: Benchmark #1: Current Time (mean ± σ): 6.468 s ± 0.116 s [User: 9.803 s, System: 0.333 s] Range (min … max): 6.264 s … 6.647 s 10 runs Benchmark #2: 2Threads Time (mean ± σ): 4.509 s ± 0.106 s [User: 12.706 s, System: 0.530 s] Range (min … max): 4.335 s … 4.674 s 10 runs Benchmark #3: 4Threads Time (mean ± σ): 3.471 s ± 0.033 s [User: 16.390 s, System: 0.630 s] Range (min … max): 3.431 s … 3.542 s 10 runs Benchmark #4: 8Threads Time (mean ± σ): 2.880 s ± 0.044 s [User: 22.454 s, System: 0.938 s] Range (min … max): 2.813 s … 2.964 s 10 runs Summary '8Threads' ran 1.21 ± 0.02 times faster than '4Threads' 1.57 ± 0.04 times faster than '2Threads' 2.25 ± 0.05 times faster than 'Current' after 8 threads, there were no real performance benefits since the overhead to load the module seems to be the biggest factor. Signed-off-by: Dominik Csapak --- src/bin/app.js | 35 +++++++++++++++++++++++++++++------ src/index.js | 2 ++ src/lib/worker.js | 27 +++++++++++++++++++++++++++ src/package.json | 3 ++- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 src/lib/worker.js diff --git a/src/bin/app.js b/src/bin/app.js index 8a28923..10e7e6a 100644 --- a/src/bin/app.js +++ b/src/bin/app.js @@ -6,6 +6,7 @@ const path = require('path'); const color = require('colors'); const program = require('commander'); +const worker = require('worker_threads'); const eslint = require('pve-eslint'); program @@ -14,6 +15,7 @@ program .option('-e, --extend ', 'uses ontop of default eslint config.') .option('-f, --fix', 'if set, fixes will be applied.') .option('-s, --strict', 'if set, also exit uncleanly on warnings') + .option('-t, --threads ', 'how many worker_threads should be used (default=4)') .option('--output-config', 'if set, only output the config as JSON and exit.') ; @@ -42,6 +44,11 @@ if (!paths.length) { paths = [process.cwd()]; } +let threadCount = 4; +if (program.threads) { + threadCount = program.threads; +} + const defaultConfig = { parserOptions: { ecmaVersion: 2020, @@ -283,20 +290,36 @@ if (program.outputConfig) { process.exit(0); } -const cli = new eslint.CLIEngine({ +const cliOptions = { baseConfig: config, useEslintrc: true, fix: !!program.fix, cwd: process.cwd(), -}); +}; + +let promises = []; +let filesPerThread = Math.round(paths.length / threadCount); +for (let i = 0; i < (threadCount - 1); i++) { + let files = paths.splice(0, filesPerThread); + promises.push(eslint.createWorker({ + cliOptions, + files + })); +} + +// the remaining paths +promises.push(eslint.createWorker({ + cliOptions, + files: paths +})); -const report = cli.executeOnFiles(paths); +let results = (await Promise.all(promises)).map(res => res.results).flat(1); let exitcode = 0; let files_err = [], files_warn = [], files_ok = []; let fixes = 0; console.log('------------------------------------------------------------'); -report.results.forEach(function(result) { +results.forEach(function(result) { let filename = path.relative(process.cwd(), result.filePath); let msgs = result.messages; let max_sev = 0; @@ -348,7 +371,7 @@ report.results.forEach(function(result) { console.log('------------------------------------------------------------'); }); -if (report.results.length > 1) { +if (results.length > 1) { console.log(`${color.bold(files_ok.length + files_err.length)} files:`); if (files_err.length > 0) { console.log(color.red(` ${color.bold(files_err.length)} files have Errors`)); @@ -367,7 +390,7 @@ console.log('------------------------------------------------------------'); if (program.fix) { if (fixes > 0) { console.log(`Writing ${color.bold(fixes)} fixed files...`); - eslint.CLIEngine.outputFixes(report); + eslint.CLIEngine.outputFixes({ results }); console.log('Done'); } else { console.log("No fixable Errors/Warnings found."); diff --git a/src/index.js b/src/index.js index 01b9a1d..311ae38 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,5 @@ const eslint = require('./lib/eslint.js'); +const createWorker = require('./lib/worker.js'); module.exports = eslint; +module.exports.createWorker = createWorker; diff --git a/src/lib/worker.js b/src/lib/worker.js new file mode 100644 index 0000000..9a8c955 --- /dev/null +++ b/src/lib/worker.js @@ -0,0 +1,27 @@ +'use strict'; + +const worker = require('worker_threads'); + +if (!worker.isMainThread) { + const eslint = require('pve-eslint'); + const data = worker.workerData; + const cli = new eslint.CLIEngine(data.cliOptions); + const report = cli.executeOnFiles(data.files); + worker.parentPort.postMessage(report); +} else { + module.exports = async function createWorker(workerData) { + return new Promise((resolve, reject) => { + const child = new worker.Worker(__filename, + { + workerData, + }, + ); + child.on('message', resolve); + child.on('error', reject); + child.on('exit', (code) => { + if (code !== 0) {reject(new Error(`Worker stopped with exit code ${code}`));} + }); + }); + } +} + diff --git a/src/package.json b/src/package.json index b08184b..e912069 100644 --- a/src/package.json +++ b/src/package.json @@ -4,6 +4,7 @@ "files": [ "index.js", "bin/app.js", - "lib/eslint.js" + "lib/eslint.js", + "lib/worker.js" ] } -- 2.30.2