From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 2606576A80
for <pve-devel@lists.proxmox.com>; Fri, 16 Jul 2021 16:18:10 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 1903111A81
for <pve-devel@lists.proxmox.com>; Fri, 16 Jul 2021 16:18:10 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id B7CB011A73
for <pve-devel@lists.proxmox.com>; Fri, 16 Jul 2021 16:18:08 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7DBFB42116
for <pve-devel@lists.proxmox.com>; Fri, 16 Jul 2021 16:18:08 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pve-devel@lists.proxmox.com
Date: Fri, 16 Jul 2021 16:18:07 +0200
Message-Id: <20210716141807.1182069-1-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.174 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
KAM_ASCII_DIVIDERS 0.8 Spam that uses ascii formatting tricks
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pve-devel] [PATCH pve-eslint] use worker_threads for linting
X-BeenThere: pve-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox VE development discussion <pve-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pve-devel/>
List-Post: <mailto:pve-devel@lists.proxmox.com>
List-Help: <mailto:pve-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Fri, 16 Jul 2021 14:18:10 -0000
instead linting all files in the main thread, use worker threads
for that (4 by default) and add the '-t' switch to able to control that
since nodejs always wants a module/script to load for a thread,
give a small script that load the file itself
a basic benchmark of eslint of pve-manager showed some performance
gains:
Benchmark #1: Current
Time (mean ± σ): 6.449 s ± 0.207 s [User: 9.818 s, System: 0.362 s]
Range (min … max): 6.190 s … 6.773 s 10 runs
Benchmark #2: 2Threads
Time (mean ± σ): 4.525 s ± 0.143 s [User: 12.646 s, System: 0.584 s]
Range (min … max): 4.324 s … 4.799 s 10 runs
Benchmark #3: 4Threads
Time (mean ± σ): 3.443 s ± 0.041 s [User: 16.393 s, System: 0.672 s]
Range (min … max): 3.354 s … 3.508 s 10 runs
Benchmark #4: 8Threads
Time (mean ± σ): 2.835 s ± 0.052 s [User: 22.343 s, System: 1.023 s]
Range (min … max): 2.764 s … 2.934 s 10 runs
Summary
'8Threads' ran
1.21 ± 0.03 times faster than '4Threads'
1.60 ± 0.06 times faster than '2Threads'
2.28 ± 0.08 times faster than 'Current'
after 8 threads, there were no real performance benefits since the
overhead to load the eslint js file seems to be the biggest factor.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
i recently looked how we could do that, but did not find the docs for
the worker_threads. i stumbled upon it today, and quickly threw this
together. the self loading inline script is a bit of a hack, but the
only way to do it better would be to ship eslint and the worker code as
module, but i did not look into that for now...
src/app.js | 65 ++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 58 insertions(+), 7 deletions(-)
diff --git a/src/app.js b/src/app.js
index 9226234..71a88bc 100644
--- a/src/app.js
+++ b/src/app.js
@@ -1,9 +1,18 @@
-(function() {
+(async function() {
'use strict';
const path = require('path');
const color = require('colors');
const program = require('commander');
+const worker = require('worker_threads');
+
+if (!worker.isMainThread) {
+ const data = worker.workerData;
+ const cli = new eslint.CLIEngine(data.cliOptions);
+ const report = cli.executeOnFiles(data.files);
+ worker.parentPort.postMessage(report);
+ process.exit(0);
+}
program
.usage('[options] [<file(s) ...>]')
@@ -11,6 +20,7 @@ program
.option('-e, --extend <configfile>', 'uses <configfile> ontop of default eslint config.')
.option('-f, --fix', 'if set, fixes will be applied.')
.option('-s, --strict', 'if set, also exit uncleanly on warnings')
+ .option('-t, --threads <threads>', 'how many worker_threads should be used (default=4)')
.option('--output-config', 'if set, only output the config as JSON and exit.')
;
@@ -39,6 +49,11 @@ if (!paths.length) {
paths = [process.cwd()];
}
+let threadCount = 4;
+if (program.threads) {
+ threadCount = program.threads;
+}
+
const defaultConfig = {
parserOptions: {
ecmaVersion: 2020,
@@ -280,20 +295,56 @@ if (program.outputConfig) {
process.exit(0);
}
-const cli = new eslint.CLIEngine({
+const cliOptions = {
baseConfig: config,
useEslintrc: true,
fix: !!program.fix,
cwd: process.cwd(),
-});
+};
+
+let lintFiles = async function(files) {
+ return new Promise((resolve, reject) => {
+ const child = new worker.Worker(
+ `
+ const worker = require('worker_threads');
+ let file = worker.workerData.__filename;
+ delete worker.workerData.__filename;
+ require(file);
+ `,
+ {
+ eval: true,
+ workerData: {
+ __filename,
+ cliOptions,
+ files,
+ }
+ }
+ );
+ child.on('message', resolve);
+ child.on('error', reject);
+ child.on('exit', (code) => {
+ if (code !== 0)
+ reject(new Error(`Worker stopped with exit code ${code}`));
+ });
+ });
+};
+
+let promises = [];
+let filesPerThread = Math.round(paths.length / threadCount);
+for (let i = 0; i < (threadCount - 1); i++) {
+ let files = paths.splice(0, filesPerThread);
+ promises.push(lintFiles(files));
+}
+// the remaining paths
+promises.push(lintFiles(paths));
-const report = cli.executeOnFiles(paths);
+let results = (await Promise.all(promises)).map(res => res.results).flat(1);
let exitcode = 0;
let files_err = [], files_warn = [], files_ok = [];
let fixes = 0;
console.log('------------------------------------------------------------');
-report.results.forEach(function(result) {
+results.forEach(function(result) {
let filename = path.relative(process.cwd(), result.filePath);
let msgs = result.messages;
let max_sev = 0;
@@ -345,7 +396,7 @@ report.results.forEach(function(result) {
console.log('------------------------------------------------------------');
});
-if (report.results.length > 1) {
+if (results.length > 1) {
console.log(`${color.bold(files_ok.length + files_err.length)} files:`);
if (files_err.length > 0) {
console.log(color.red(` ${color.bold(files_err.length)} files have Errors`));
@@ -364,7 +415,7 @@ console.log('------------------------------------------------------------');
if (program.fix) {
if (fixes > 0) {
console.log(`Writing ${color.bold(fixes)} fixed files...`);
- eslint.CLIEngine.outputFixes(report);
+ eslint.CLIEngine.outputFixes({ results });
console.log('Done');
} else {
console.log("No fixable Errors/Warnings found.");
--
2.30.2