enable external partner testing

add names to external partner tests

use shallow cloning, use bower if necessary, add more external partners

nice names

smoke tests

prettier and storefront

use execa

fix bower command

use node 10 in travis

Author: runspired

.travis.yml

@@ -3,7 +3,7 @@ language: node_js
 sudo: false
 dist: trusty
 node_js:
-  - "6"
+  - "10"
 
 addons:
   chrome: stable
@@ -24,7 +24,8 @@ branches:
 stages:
   - test
   - additional tests
-  - older version tests
+  - ember version tests
+  - external partner tests
   - name: deploy
     if: type = push AND (branch IN (master, beta, release) OR tag IS present)
 
@@ -34,18 +35,18 @@ jobs:
   include:
     # runs tests with current locked deps and linting
     - stage: test
-      env: NAME=test                  # used only to make Travis UI show description
+      name: "Basic Tests"
       script:
         - ./bin/lint-features
         - yarn lint:js
         - yarn test
 
     - stage: additional tests
-      env: NAME=optional-features     # used only to make Travis UI show description
+      name: "Optional Features"
       install: yarn install
       script: yarn test:optional-features
 
-    - env: NAME=floating dependencies # used only to make Travis UI show description
+    - name: "Floating Dependencies"
       install: yarn install --no-lockfile --non-interactive
       script: yarn test
 
@@ -54,24 +55,48 @@ jobs:
       env: TARGET_IE11=true
       script: yarn test
 
-    - env: NAME=production            # used only to make Travis UI show description
+    - name: "Production"
       install: yarn install
       script: yarn test:production
 
-    - env: NAME=node-tests            # used only to make Travis UI show description
+    - name: "Node Tests"
       install: yarn install
       script: yarn test:node
 
     # runs tests against each supported Ember version
-    - stage: older version tests
+    - stage: ember version tests
+      name: "Ember LTS 2.16"
       env: EMBER_TRY_SCENARIO=ember-lts-2.16
-    - env: EMBER_TRY_SCENARIO=ember-lts-2.18
-    - env: EMBER_TRY_SCENARIO=ember-release
-    - env: EMBER_TRY_SCENARIO=ember-beta
-    - env: EMBER_TRY_SCENARIO=ember-canary
+    - name: "Ember LTS 2.18"
+      env: EMBER_TRY_SCENARIO=ember-lts-2.18
+    - name: "Ember Release"
+      env: EMBER_TRY_SCENARIO=ember-release
+    - name: "Ember Beta"
+      env: EMBER_TRY_SCENARIO=ember-beta
+    - name: "Ember Canary"
+      env: EMBER_TRY_SCENARIO=ember-canary
+
+    # runs tests against various open-source projects for early-warning regression analysis
+    - stage: external partner tests
+      name: "Ember Data Model Fragments"
+      script: yarn test-external:model-fragments
+    - name: "Ember Observer"
+      script: yarn test-external:ember-observer
+    - name: "Travis Web"
+      script: yarn test-external:travis-web
+    - name: "emberaddons.com"
+      script: yarn test-external:emberaddons.com
+    - name: "Ember Data Change Tracker"
+      script: yarn test-external:ember-data-change-tracker
+    - name: "ember-m3"
+      script: yarn test-external:ember-m3
+    - name: "Ember Data Storefront"
+      script: yarn test-external:storefront
+    - name: "Ember Data Factory Guy"
+      script: yarn test-external:factory-guy
 
     - stage: deploy
-      env: NAME=publish                # used only to make Travis UI show description
+      name: "Publish"
       install: yarn install
       script:
         - node_modules/.bin/ember try:reset
@@ -85,6 +110,7 @@ before_install:
 
 install:
   - yarn install
+  - yarn add global bower
 
 script:
   - node_modules/.bin/ember try:one $EMBER_TRY_SCENARIO --skip-cleanup

lib/scripts/test-external.js

@@ -0,0 +1,116 @@
+'use strict';
+/* eslint-disable no-console, node/no-extraneous-require, node/no-unpublished-require */
+const fs = require('fs');
+const path = require('path');
+const { shellSync } = require('execa');
+// apparently violates no-extraneous require? /shrug
+const debug = require('debug')('test-external');
+const rimraf = require('rimraf');
+
+const projectRoot = path.resolve(__dirname, '../../');
+const externalProjectName = process.argv[2];
+const gitUrl = process.argv[3];
+const tempDir = path.join(projectRoot, '../__external-test-cache');
+const projectTempDir = path.join(tempDir, externalProjectName);
+
+if (!gitUrl) {
+  throw new Error(
+    'No git url provided to `node ./lib/scripts/test-external`. An https git url should be the first argument.'
+  );
+} else if (gitUrl.indexOf('https') !== 0) {
+  throw new Error(
+    `The git url provided to \`node ./lib/scripts/test-external\` should use https. Received '${gitUrl}'`
+  );
+}
+
+console.log(
+  `Preparing to test external project ${externalProjectName} located at ${gitUrl} against this ember-data commit.`
+);
+
+function execWithLog(command, force) {
+  if (debug.enabled || force) {
+    return shellSync(command, { stdio: [0, 1, 2] });
+  }
+
+  return shellSync(command);
+}
+
+if (!fs.existsSync(tempDir)) {
+  fs.mkdirSync(tempDir);
+}
+
+if (fs.existsSync(projectTempDir)) {
+  rimraf.sync(projectTempDir);
+}
+
+// install the project
+try {
+  execWithLog(
+    `cd ../__external-test-cache && git clone --depth=1 ${gitUrl} ./${externalProjectName}`
+  );
+} catch (e) {
+  debug(e);
+  throw new Error(
+    `Install of ${gitUrl} for external project ${externalProjectName} testing failed.`
+  );
+}
+
+const useYarn = fs.existsSync(path.join(projectTempDir, 'yarn.lock'));
+const useBower = fs.existsSync(path.join(projectTempDir, 'bower.json'));
+
+// install project dependencies and link our local version of ember-data
+try {
+  execWithLog(
+    `${
+      useYarn ? 'yarn link' : 'npm link'
+    } && cd ../__external-test-cache/${externalProjectName} && ${useYarn ? 'yarn' : 'npm install'}${
+      useBower ? ' && bower install' : ''
+    }`
+  );
+} catch (e) {
+  debug(e);
+  throw new Error(
+    `Unable to complete install of dependencies for external project ${externalProjectName}`
+  );
+}
+
+// run project tests
+console.log(`Running tests for ${externalProjectName}`);
+
+let smokeTestPassed = true;
+let commitTestPassed = true;
+
+try {
+  debug('Running Smoke Test');
+  execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`);
+} catch (e) {
+  smokeTestPassed = false;
+}
+
+try {
+  execWithLog(`${useYarn ? 'yarn link ember-data' : 'npm link ember-data'}`);
+} catch (e) {
+  debug(e);
+  throw new Error(
+    `Unable to \`${useYarn ? 'yarn' : 'npm'} link ember-data\` for ${externalProjectName}`
+  );
+}
+
+try {
+  debug('Re-running tests against EmberData commit');
+  execWithLog(`cd ../__external-test-cache/${externalProjectName} && ember test`, true);
+} catch (e) {
+  commitTestPassed = false;
+}
+
+if (!smokeTestPassed && !commitTestPassed) {
+  throw new Error(
+    `Commit may result in a regression, but the smoke test for ${externalProjectName} also failed.`
+  );
+} else if (smokeTestPassed && !commitTestPassed) {
+  throw new Error(`Commit results in a regression in ${externalProjectName}`);
+} else if (!smokeTestPassed) {
+  console.log(`Commit may resolve issues present in the smoke test for ${externalProjectName}`);
+} else {
+  console.log(`Commit does not regress ${externalProjectName}`);
+}

package.json

@@ -16,7 +16,15 @@
     "test:all": "ember try:each",
     "test:node": "node node-tests/nodetest-runner.js",
     "test:production": "ember test -e production && ember test -e production --record-data-rfc-build",
-    "test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build"
+    "test:optional-features": "ember test -e test-optional-features && ember test -e test-optional-features --record-data-rfc-build",
+    "test-external:ember-m3": "node ./lib/scripts/test-external ember-m3 https://github.com/hjdivad/ember-m3.git",
+    "test-external:ember-data-change-tracker": "node ./lib/scripts/test-external ember-data-change-tracker https://github.com/danielspaniel/ember-data-change-tracker.git",
+    "test-external:emberaddons.com": "node ./lib/scripts/test-external ember-cli-addon-search https://github.com/gcollazo/ember-cli-addon-search.git",
+    "test-external:model-fragments": "node ./lib/scripts/test-external ember-data-model-fragments https://github.com/lytics/ember-data-model-fragments.git",
+    "test-external:ember-observer": "node ./lib/scripts/test-external ember-observer https://github.com/emberobserver/client.git",
+    "test-external:travis-web": "node ./lib/scripts/test-external travis-web https://github.com/travis-ci/travis-web.git",
+    "test-external:storefront": "node ./lib/scripts/test-external storefront https://github.com/embermap/ember-data-storefront.git",
+    "test-external:factory-guy": "node ./lib/scripts/test-external factory-guy https://github.com/danielspaniel/ember-data-factory-guy.git"
   },
   "author": "",
   "license": "MIT",
@@ -92,6 +100,7 @@
     "eslint-config-prettier": "^2.9.0",
     "eslint-plugin-node": "^7.0.1",
     "eslint-plugin-prettier": "2.6.2",
+    "execa": "^0.10.0",
     "fs-extra": "^7.0.0",
     "github": "^1.1.1",
     "glob": "^7.1.2",

yarn.lock

@@ -3157,7 +3157,7 @@ exec-sh@^0.2.0:
 
 execa@^0.10.0:
   version "0.10.0"
-  resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
+  resolved "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
   dependencies:
     cross-spawn "^6.0.0"
     get-stream "^3.0.0"