Make lighthouse reports concurrent (#53)

Co-authored-by: emersonthis <emerson@cloudfour.com>

Author: calebeby

.changeset/polite-foxes-compete.md

@@ -0,0 +1,5 @@
+---
+'lighthouse-parade': minor
+---
+
+Run lighthouse instances concurrently, and change CLI output

.gitignore

@@ -1,4 +1,4 @@
 .DS_Store
-data
+lighthouse-parade-data
 node_modules
 /dist

README.md

@@ -37,10 +37,11 @@ Runs a crawler on the provided URL. Discovers all URLs and runs a lighthouse rep
 ### Options
 
 ```
---ignore-robots         Crawl pages even if they are listed in the site's robots.txt  (default false)
---crawler-user-agent    Pass a user agent string to be used by the crawler (not by Lighthouse)
--v, --version           Displays current version
--h, --help              Displays help text
+--ignore-robots             Crawl pages even if they are listed in the site's robots.txt  (default: false)
+--crawler-user-agent        Pass a user agent string to be used by the crawler (not by Lighthouse)
+--lighthouse-concurrency    Control the maximum number of ligthhouse reports to run concurrently  (default: number of CPU cores minus one)
+-v, --version               Displays current version
+-h, --help                  Displays help text
 ```
 
 ## Versioning Notice

cli.ts

@@ -1,6 +1,9 @@
 #!/usr/bin/env node
 
 import * as fs from 'fs';
+import * as os from 'os';
+import * as kleur from 'kleur/colors';
+import logUpdate from 'log-update';
 import * as path from 'path';
 import sade from 'sade';
 import { scan } from './scan-task';
@@ -16,6 +19,11 @@ It may in the future make sense to use a bundler to combine all the dist/ files
 // eslint-disable-next-line @cloudfour/typescript-eslint/no-var-requires
 const { version } = require('../package.json');
 
+const symbols = {
+  error: kleur.red('✖'),
+  success: kleur.green('✔'),
+};
+
 sade('lighthouse-parade <url> [dataDirectory]', true)
   .version(version)
   .describe(
@@ -30,6 +38,11 @@ sade('lighthouse-parade <url> [dataDirectory]', true)
     '--crawler-user-agent',
     'Pass a user agent string to be used by the crawler (not by Lighthouse)'
   )
+  .option(
+    '--lighthouse-concurrency',
+    'Control the maximum number of ligthhouse reports to run concurrently',
+    os.cpus().length - 1
+  )
   .action(
     (
       url,
@@ -54,14 +67,96 @@ sade('lighthouse-parade <url> [dataDirectory]', true)
         throw new Error('--crawler-user-agent flag must be a string');
       }
 
-      const scanner = scan(url, { ignoreRobotsTxt, dataDirectory });
+      const lighthouseConcurrency = opts['lighthouse-concurrency'];
+
+      const scanner = scan(url, {
+        ignoreRobotsTxt,
+        dataDirectory,
+        lighthouseConcurrency,
+      });
+
+      const enum State {
+        Pending,
+        ReportInProgress,
+        ReportComplete,
+      }
+      const urlStates = new Map<
+        string,
+        { state: State; error?: Error | string }
+      >();
+
+      const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+      let i = 0;
+
+      const printLine = (url: string, state: State, error?: Error | string) => {
+        const frame = kleur.blue(frames[i]);
+        const statusIcon = error
+          ? symbols.error
+          : state === State.Pending
+          ? ' '
+          : state === State.ReportInProgress
+          ? frame
+          : symbols.success;
+        let output = `${statusIcon} ${url}`;
+        if (error) {
+          output += `\n  ${kleur.gray(error.toString())}`;
+        }
+
+        return output;
+      };
+
+      const render = () => {
+        const pendingUrls: string[] = [];
+        const currentUrls: string[] = [];
+        urlStates.forEach(({ state, error }, url) => {
+          if (state === State.ReportComplete) return;
+          const line = `${printLine(url, state, error)}\n`;
+          if (state === State.Pending) pendingUrls.push(line);
+          else currentUrls.push(line);
+        });
+        const numPendingToDisplay = Math.min(
+          Math.max(process.stdout.rows - currentUrls.length - 3, 1),
+          pendingUrls.length
+        );
+        const numHiddenUrls =
+          numPendingToDisplay === pendingUrls.length
+            ? ''
+            : kleur.dim(
+                `\n...And ${
+                  pendingUrls.length - numPendingToDisplay
+                } more pending`
+              );
+        logUpdate(
+          currentUrls.join('') +
+            pendingUrls.slice(0, numPendingToDisplay).join('') +
+            numHiddenUrls
+        );
+      };
+
+      const intervalId = setInterval(() => {
+        i = (i + 1) % frames.length;
+        render();
+      }, 80);
+
+      /**
+       * Allows you to run a console.log that will output _above_ the persistent logUpdate log
+       * Pass a callback where you run your console.log or console.error
+       */
+      const printAboveLogUpdate = (cb: () => void) => {
+        logUpdate.clear();
+        cb();
+        render();
+      };
+
+      const log = (...messages: string[]) =>
+        printAboveLogUpdate(() => console.log(...messages));
 
       const urlsFile = path.join(dataDirectory, 'urls.csv');
       fs.writeFileSync(urlsFile, 'URL,content_type,bytes,response\n');
       const urlsStream = fs.createWriteStream(urlsFile, { flags: 'a' });
 
       scanner.on('urlFound', (url, contentType, bytes, statusCode) => {
-        console.log('Crawled %s [%s] (%d bytes)', url, contentType, bytes);
+        urlStates.set(url, { state: State.Pending });
         const csvLine = [
           JSON.stringify(url),
           contentType,
@@ -70,15 +165,28 @@ sade('lighthouse-parade <url> [dataDirectory]', true)
         ].join(',');
         urlsStream.write(`${csvLine}\n`);
       });
+      scanner.on('reportBegin', (url) => {
+        urlStates.set(url, { state: State.ReportInProgress });
+      });
+      scanner.on('reportFail', (url, error) => {
+        urlStates.set(url, { state: State.ReportComplete, error });
+        log(printLine(url, State.ReportComplete, error));
+      });
       scanner.on('reportComplete', (url, reportData) => {
-        console.log('Report is done for', url);
+        urlStates.set(url, { state: State.ReportComplete });
+        log(printLine(url, State.ReportComplete));
         const reportFileName = makeFileNameFromUrl(url, 'csv');
 
         fs.writeFileSync(path.join(reportsDirPath, reportFileName), reportData);
       });
 
       scanner.on('info', (message) => {
-        console.log(message);
+        log(message);
+      });
+
+      scanner.promise.then(() => {
+        clearInterval(intervalId);
+        console.log('DONE!');
       });
     }
   )

emitter.ts

@@ -20,6 +20,7 @@ export const createEmitter = <Events extends EventMap, Resolve = never>() => {
 
   const on = <E extends keyof Events>(eventName: E, handler: Events[E]) => {
     (eventHandlers[eventName] ??= [] as Events[E][]).push(handler);
+    return emitter; // Allow chaining
   };
 
   const promise = new Promise<Resolve>((resolve, reject) => {
@@ -28,13 +29,14 @@ export const createEmitter = <Events extends EventMap, Resolve = never>() => {
   });
   const eventHandlers: { [E in keyof Events]?: Events[E][] } = {};
   interface Emit {
-    (eventName: 'resolve', value?: Resolve): void;
-    (eventName: 'reject', value?: unknown): void;
     <E extends keyof Events>(
       eventName: E,
       ...args: Parameters<Events[E]>
     ): void;
+    (eventName: 'resolve', value?: Resolve): void;
+    (eventName: 'reject', value?: unknown): void;
   }
 
-  return { promise, on, emit };
+  const emitter = { promise, on, emit };
+  return emitter;
 };

lighthouse.ts

@@ -1,25 +1,76 @@
-import { spawnSync } from 'child_process';
+import { spawn } from 'child_process';
+import { createEmitter } from './emitter';
 
 const lighthouseCli = require.resolve('lighthouse/lighthouse-cli');
 
+let lighthouseLimit = 2;
+let currentLighthouseInstances = 0;
+const lighthouseQueue: (() => void)[] = [];
+
+const runLighthouseQueue = () => {
+  while (
+    lighthouseQueue.length > 0 &&
+    currentLighthouseInstances < lighthouseLimit
+  ) {
+    const run = lighthouseQueue.shift() as () => void;
+    currentLighthouseInstances++;
+    run();
+  }
+};
+
+type LighthouseEvents = {
+  begin: () => void;
+  complete: (reportData: string) => void;
+  error: (message: Error) => void;
+};
+
 // This function is marked as async to wrap it with a promise to make error handling easier
 // Even though the underlying process is spawned synchronously
 // eslint-disable-next-line @cloudfour/typescript-eslint/require-await
-export const runLighthouseReport = async (url: string) => {
-  const { status = -1, stdout } = spawnSync('node', [
-    lighthouseCli,
-    url,
-    '--output=csv',
-    '--output-path=stdout',
-    '--emulated-form-factor=mobile',
-    '--only-categories=performance',
-    '--chrome-flags="--headless"',
-    '--max-wait-for-load=45000',
-  ]);
-
-  if (status !== 0) {
-    throw new Error(`Lighthouse report failed for: ${url}`);
-  }
+export const runLighthouseReport = (url: string, maxConcurrency?: number) => {
+  if (maxConcurrency) lighthouseLimit = maxConcurrency;
+  const { on, emit } = createEmitter<LighthouseEvents>();
+  const run = () => {
+    emit('begin');
+    const lighthouseProcess = spawn('node', [
+      lighthouseCli,
+      url,
+      '--output=csv',
+      '--output-path=stdout',
+      '--emulated-form-factor=mobile',
+      '--only-categories=performance',
+      '--chrome-flags="--headless"',
+      '--max-wait-for-load=45000',
+    ]);
+
+    let stdout = '';
+    let stderr = '';
+
+    lighthouseProcess.stdout.on('data', (d) => {
+      stdout += d;
+    });
+
+    lighthouseProcess.stderr.on('data', (d) => {
+      if (/runtime error encountered/i.test(d)) stderr += d;
+    });
+
+    lighthouseProcess.on('close', (status) => {
+      if (status === 0) {
+        emit('complete', String(stdout).replace(/\r\n/g, '\n'));
+      } else {
+        emit(
+          'error',
+          new Error(stderr.trim() || `Lighthouse report failed for: ${url}`)
+        );
+      }
+
+      currentLighthouseInstances--;
+      runLighthouseQueue();
+    });
+  };
+
+  lighthouseQueue.push(run);
+  runLighthouseQueue();
 
-  return String(stdout).replace(/\r\n/g, '\n');
+  return { on };
 };