Merge pull request #391 from MatrixAI/feature-pkg_integration_tests

Packaged executable integration tests - initial integration tests with docker

Author: CMCDragonkai

.env.example

@@ -31,3 +31,9 @@ AWS_SECRET_ACCESS_KEY=
 
 # Authenticate to GitHub with `gh`
 # GITHUB_TOKEN=
+
+# To allow testing different executables in the bin tests
+# Both PK_TEST_COMMAND and PK_TEST_PLATFORM must be set at the same time
+# PK_TEST_COMMAND= #Specify the shell command we want to test against
+# PK_TEST_PLATFORM=docker #Overrides the auto set `testPlatform` variable used for enabling platform specific tests
+# PK_TEST_TMPDIR= #Sets the `global.tmpDir` variable to allow overriding the temp directory used for tests

.gitlab-ci.yml

@@ -330,15 +330,20 @@ integration:docker:
     - integration:builds
     - job: integration:deployment
       optional: true
-  image: docker:20.10.11
   services:
-    - docker:20.10.11-dind
+    - docker:20.10.16-dind
   variables:
     DOCKER_TLS_CERTDIR: "/certs"
+    FF_NETWORK_PER_BUILD: "true"
   script:
     - docker info
-    - image="$(docker load --input ./builds/*docker* | cut -d' ' -f3)"
-    - docker run "$image"
+    - >
+      nix-shell --run $'
+        PK_TEST_COMMAND="docker run \${DOCKER_OPTIONS} $(docker load --input ./builds/*docker* | cut -d\' \' -f3) polykey" \
+        PK_TEST_PLATFORM=docker \
+        PK_TEST_TMPDIR=/builds/$CI_PROJECT_PATH/tmp \
+        exec npm run test -- tests/bin
+      '
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/

jest.config.js

@@ -15,6 +15,7 @@ moduleNameMapper['^jose/(.*)$'] = "<rootDir>/node_modules/jose/dist/node/cjs/$1"
 
 // Global variables that are shared across the jest worker pool
 // These variables must be static and serializable
+if ((process.env.PK_TEST_PLATFORM != null) !== (process.env.PK_TEST_COMMAND != null)) throw Error('Both PK_TEST_PLATFORM and PK_TEST_COMMAND must be set together.')
 const globals = {
   // Absolute directory to the project root
   projectDir: __dirname,
@@ -31,7 +32,8 @@ const globals = {
   // Timeouts rely on setTimeout which takes 32 bit numbers
   maxTimeout: Math.pow(2, 31) - 1,
   testCmd: process.env.PK_TEST_COMMAND,
-  testPlatform: process.env.PK_TEST_COMMAND_DOCKER,
+  testPlatform: process.env.PK_TEST_PLATFORM,
+  tmpDir: process.env.PK_TEST_TMPDIR ?? os.tmpdir(),
 };
 
 // The `globalSetup` and `globalTeardown` cannot access the `globals`

scripts/docker-run.sh

@@ -2,7 +2,7 @@ import type { FileSystem } from './types';
 import type { PolykeyWorkerManagerInterface } from './workers/types';
 import type { ConnectionData, Host, Port } from './network/types';
 import type { SeedNodes } from './nodes/types';
-import type { KeyManagerChangeData } from './keys/types';
+import type { KeyManagerChangeData, PrivateKeyPem } from './keys/types';
 import path from 'path';
 import process from 'process';
 import Logger from '@matrixai/logger';
@@ -108,6 +108,7 @@ class PolykeyAgent {
       rootCertDuration?: number;
       dbKeyBits?: number;
       recoveryCode?: string;
+      privateKeyPemOverride?: PrivateKeyPem;
     };
     proxyConfig?: {
       authToken?: string;

src/PolykeyAgent.ts

@@ -37,6 +37,7 @@ class CommandStart extends CommandPolykey {
     this.addOption(binOptions.backgroundOutFile);
     this.addOption(binOptions.backgroundErrFile);
     this.addOption(binOptions.fresh);
+    this.addOption(binOptions.rootKeyFile);
     this.action(async (options) => {
       options.clientHost =
         options.clientHost ?? config.defaults.networkConfig.clientHost;
@@ -88,12 +89,16 @@ class CommandStart extends CommandPolykey {
       const [seedNodes, defaults] = options.seedNodes;
       let seedNodes_ = seedNodes;
       if (defaults) seedNodes_ = { ...options.network, ...seedNodes };
+      const privateKeyPem = await binProcessors.processRootKey(
+        options.rootKeyFile,
+      );
       const agentConfig = {
         password,
         nodePath: options.nodePath,
         keysConfig: {
           rootKeyPairBits: options.rootKeyPairBits,
           recoveryCode: recoveryCodeIn,
+          privateKeyPemOverride: privateKeyPem,
         },
         proxyConfig: {
           connConnectTime: options.connectionTimeout,

src/bin/agent/CommandStart.ts

@@ -11,6 +11,7 @@ class CommandBootstrap extends CommandPolykey {
     this.addOption(binOptions.recoveryCodeFile);
     this.addOption(binOptions.rootKeyPairBits);
     this.addOption(binOptions.fresh);
+    this.addOption(binOptions.rootKeyFile);
     this.action(async (options) => {
       const bootstrapUtils = await import('../../bootstrap/utils');
       const password = await binProcessors.processNewPassword(
@@ -21,19 +22,23 @@ class CommandBootstrap extends CommandPolykey {
         options.recoveryCodeFile,
         this.fs,
       );
+      const privateKeyPem = await binProcessors.processRootKey(
+        options.rootKeyFile,
+      );
       const recoveryCodeOut = await bootstrapUtils.bootstrapState({
         password,
         nodePath: options.nodePath,
         keysConfig: {
           rootKeyPairBits: options.rootKeyPairBits,
           recoveryCode: recoveryCodeIn,
+          privateKeyPemOverride: privateKeyPem,
         },
         fresh: options.fresh,
         fs: this.fs,
         logger: this.logger,
       });
       this.logger.info(`Bootstrapped ${options.nodePath}`);
-      process.stdout.write(recoveryCodeOut + '\n');
+      if (recoveryCodeOut != null) process.stdout.write(recoveryCodeOut + '\n');
     });
   }
 }

src/bin/bootstrap/CommandBootstrap.ts

@@ -29,6 +29,11 @@ class ErrorCLIRecoveryCodeFileRead<T> extends ErrorCLI<T> {
   exitCode = sysexits.NOINPUT;
 }
 
+class ErrorCLIPrivateKeyFileRead<T> extends ErrorCLI<T> {
+  static description = 'Failed to read private key Pem file';
+  exitCode = sysexits.NOINPUT;
+}
+
 class ErrorCLIFileRead<T> extends ErrorCLI<T> {
   static description = 'Failed to read file';
   exitCode = sysexits.NOINPUT;
@@ -61,6 +66,7 @@ export {
   ErrorCLIPasswordMissing,
   ErrorCLIPasswordFileRead,
   ErrorCLIRecoveryCodeFileRead,
+  ErrorCLIPrivateKeyFileRead,
   ErrorCLIFileRead,
   ErrorCLIPolykeyAgentStatus,
   ErrorCLIPolykeyAgentProcess,

src/bin/errors.ts

@@ -163,6 +163,11 @@ const noPing = new commander.Option('--no-ping', 'Skip ping step').default(
   true,
 );
 
+const rootKeyFile = new commander.Option(
+  '--root-key-file <rootKeyFile>',
+  'Override key generation with a private key Pem from a file.',
+);
+
 export {
   nodePath,
   format,
@@ -187,4 +192,5 @@ export {
   pullVault,
   forceNodeAdd,
   noPing,
+  rootKeyFile,
 };

src/bin/utils/options.ts

@@ -1,5 +1,5 @@
 import type { FileSystem } from '../../types';
-import type { RecoveryCode } from '../../keys/types';
+import type { RecoveryCode, PrivateKeyPem } from '../../keys/types';
 import type { NodeId } from '../../nodes/types';
 import type { Host, Port } from '../../network/types';
 import type {
@@ -403,6 +403,29 @@ async function processAuthentication(
   return meta;
 }
 
+async function processRootKey(
+  privateKeyFile: string | undefined,
+  fs: FileSystem = require('fs'),
+): Promise<PrivateKeyPem | undefined> {
+  if (privateKeyFile != null) {
+    try {
+      return (await fs.promises.readFile(privateKeyFile, 'utf-8')).trim();
+    } catch (e) {
+      throw new binErrors.ErrorCLIPrivateKeyFileRead(e.message, {
+        data: {
+          errno: e.errno,
+          syscall: e.syscall,
+          code: e.code,
+          path: e.path,
+        },
+        cause: e,
+      });
+    }
+  } else if (typeof process.env['PK_ROOT_KEY'] === 'string') {
+    return process.env['PK_ROOT_KEY'];
+  }
+}
+
 export {
   promptPassword,
   promptNewPassword,
@@ -412,4 +435,5 @@ export {
   processClientOptions,
   processClientStatus,
   processAuthentication,
+  processRootKey,
 };