[pve-devel] [PATCH pve-eslint] use worker_threads for linting

[pve-devel] [PATCH pve-eslint] use worker_threads for linting

[Dominik Csapak]

instead linting all files in the main thread, use worker threads
for that (4 by default) and add the '-t' switch to able to control that

since nodejs always wants a module/script to load for a thread,
give a small script that load the file itself

a basic benchmark of eslint of pve-manager showed some performance
gains:

Benchmark #1: Current
  Time (mean ± σ):      6.449 s ±  0.207 s    [User: 9.818 s, System: 0.362 s]
  Range (min … max):    6.190 s …  6.773 s    10 runs

Benchmark #2: 2Threads
  Time (mean ± σ):      4.525 s ±  0.143 s    [User: 12.646 s, System: 0.584 s]
  Range (min … max):    4.324 s …  4.799 s    10 runs

Benchmark #3: 4Threads
  Time (mean ± σ):      3.443 s ±  0.041 s    [User: 16.393 s, System: 0.672 s]
  Range (min … max):    3.354 s …  3.508 s    10 runs

Benchmark #4: 8Threads
  Time (mean ± σ):      2.835 s ±  0.052 s    [User: 22.343 s, System: 1.023 s]
  Range (min … max):    2.764 s …  2.934 s    10 runs

Summary
  '8Threads' ran
    1.21 ± 0.03 times faster than '4Threads'
    1.60 ± 0.06 times faster than '2Threads'
    2.28 ± 0.08 times faster than 'Current'

after 8 threads, there were no real performance benefits since the
overhead to load the eslint js file seems to be the biggest factor.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
i recently looked how we could do that, but did not find the docs for
the worker_threads. i stumbled upon it today, and quickly threw this
together. the self loading inline script is a bit of a hack, but the
only way to do it better would be to ship eslint and the worker code as
module, but i did not look into that for now...

 src/app.js | 65 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 58 insertions(+), 7 deletions(-)

diff --git a/src/app.js b/src/app.js
index 9226234..71a88bc 100644
--- a/src/app.js
+++ b/src/app.js
@@ -1,9 +1,18 @@
-(function() {
+(async function() {
 'use strict';
 
 const path = require('path');
 const color = require('colors');
 const program = require('commander');
+const worker = require('worker_threads');
+
+if (!worker.isMainThread) {
+    const data = worker.workerData;
+    const cli = new eslint.CLIEngine(data.cliOptions);
+    const report = cli.executeOnFiles(data.files);
+    worker.parentPort.postMessage(report);
+    process.exit(0);
+}
 
 program
     .usage('[options] [<file(s) ...>]')
@@ -11,6 +20,7 @@ program
     .option('-e, --extend <configfile>', 'uses <configfile> ontop of default eslint config.')
     .option('-f, --fix', 'if set, fixes will be applied.')
     .option('-s, --strict', 'if set, also exit uncleanly on warnings')
+    .option('-t, --threads <threads>', 'how many worker_threads should be used (default=4)')
     .option('--output-config', 'if set, only output the config as JSON and exit.')
     ;
 
@@ -39,6 +49,11 @@ if (!paths.length) {
     paths = [process.cwd()];
 }
 
+let threadCount = 4;
+if (program.threads) {
+    threadCount = program.threads;
+}
+
 const defaultConfig = {
     parserOptions: {
 	ecmaVersion: 2020,
@@ -280,20 +295,56 @@ if (program.outputConfig) {
     process.exit(0);
 }
 
-const cli = new eslint.CLIEngine({
+const cliOptions = {
     baseConfig: config,
     useEslintrc: true,
     fix: !!program.fix,
     cwd: process.cwd(),
-});
+};
+
+let lintFiles = async function(files) {
+    return new Promise((resolve, reject) => {
+	const child = new worker.Worker(
+	    `
+		const worker = require('worker_threads');
+		let file = worker.workerData.__filename;
+		delete worker.workerData.__filename;
+		require(file);
+	    `,
+	    {
+		eval: true,
+		workerData: {
+		    __filename,
+		    cliOptions,
+		    files,
+		}
+	    }
+	);
+	child.on('message', resolve);
+	child.on('error', reject);
+	child.on('exit', (code) => {
+	    if (code !== 0)
+		reject(new Error(`Worker stopped with exit code ${code}`));
+	});
+    });
+};
+
+let promises = [];
+let filesPerThread = Math.round(paths.length / threadCount);
+for (let i = 0; i < (threadCount - 1); i++) {
+    let files = paths.splice(0, filesPerThread);
+    promises.push(lintFiles(files));
+}
+// the remaining paths
+promises.push(lintFiles(paths));
 
-const report = cli.executeOnFiles(paths);
+let results = (await Promise.all(promises)).map(res => res.results).flat(1);
 
 let exitcode = 0;
 let files_err = [], files_warn = [], files_ok = [];
 let fixes = 0;
 console.log('------------------------------------------------------------');
-report.results.forEach(function(result) {
+results.forEach(function(result) {
     let filename = path.relative(process.cwd(), result.filePath);
     let msgs = result.messages;
     let max_sev = 0;
@@ -345,7 +396,7 @@ report.results.forEach(function(result) {
     console.log('------------------------------------------------------------');
 });
 
-if (report.results.length > 1) {
+if (results.length > 1) {
     console.log(`${color.bold(files_ok.length + files_err.length)} files:`);
     if (files_err.length > 0) {
 	console.log(color.red(` ${color.bold(files_err.length)} files have Errors`));
@@ -364,7 +415,7 @@ console.log('------------------------------------------------------------');
 if (program.fix) {
     if (fixes > 0) {
 	console.log(`Writing ${color.bold(fixes)} fixed files...`);
-	eslint.CLIEngine.outputFixes(report);
+	eslint.CLIEngine.outputFixes({ results });
 	console.log('Done');
     } else {
 	console.log("No fixable Errors/Warnings found.");
-- 
2.30.2