DLP sample: Add autoPopulateTimespan option for creating job triggers. (#64)

Author: micheldavid

dlp/.gitignore

@@ -0,0 +1,3 @@
+# Test outputs
+*.actual.png
+*.actual.csv

dlp/package.json

@@ -21,6 +21,8 @@
   "devDependencies": {
     "@google-cloud/nodejs-repo-tools": "^2.3.0",
     "ava": "^0.25.0",
+    "pixelmatch": "^4.0.2",
+    "pngjs": "^3.3.3",
     "uuid": "^3.3.2"
   }
 }

dlp/system-test/deid.test.js

@@ -123,7 +123,7 @@ test(`should handle FPE decryption errors`, async t => {
 
 // deidentify_date_shift
 test(`should date-shift a CSV file`, async t => {
-  const outputCsvFile = path.join(__dirname, 'dates.result.csv');
+  const outputCsvFile = 'dates.actual.csv';
   const output = await tools.runAsync(
     `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields}`,
     cwd
@@ -138,9 +138,9 @@ test(`should date-shift a CSV file`, async t => {
 });
 
 test(`should date-shift a CSV file using a context field`, async t => {
-  const outputCsvFile = path.join(__dirname, 'dates-context.result.csv');
-  const correctResultFile =
-    'system-test/resources/date-shift-context.correct.csv';
+  const outputCsvFile = 'dates-context.actual.csv';
+  const expectedCsvFile =
+    'system-test/resources/date-shift-context.expected.csv';
   const output = await tools.runAsync(
     `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}`,
     cwd
@@ -150,7 +150,7 @@ test(`should date-shift a CSV file using a context field`, async t => {
   );
   t.is(
     fs.readFileSync(outputCsvFile).toString(),
-    fs.readFileSync(correctResultFile).toString()
+    fs.readFileSync(expectedCsvFile).toString()
   );
 });
 

dlp/system-test/redact.test.js

@@ -19,6 +19,8 @@ const path = require('path');
 const test = require('ava');
 const fs = require('fs');
 const tools = require('@google-cloud/nodejs-repo-tools');
+const PNG = require('pngjs').PNG;
+const pixelmatch = require('pixelmatch');
 
 const cmd = 'node redact.js';
 const cwd = path.join(__dirname, `..`);
@@ -28,6 +30,33 @@ const testResourcePath = 'system-test/resources';
 
 test.before(tools.checkCredentials);
 
+function readImage(filePath) {
+  return new Promise((resolve, reject) => {
+    fs.createReadStream(filePath)
+      .pipe(new PNG())
+      .on('error', reject)
+      .on('parsed', function() {
+        resolve(this);
+      });
+  });
+}
+
+async function getImageDiffPercentage(image1Path, image2Path) {
+  let image1 = await readImage(image1Path);
+  let image2 = await readImage(image2Path);
+  let diff = new PNG({width: image1.width, height: image1.height});
+
+  const diffPixels = pixelmatch(
+    image1.data,
+    image2.data,
+    diff.data,
+    image1.width,
+    image1.height
+  );
+  return diffPixels / (diff.width * diff.height);
+}
+
+// redact_text
 test(`should redact a single sensitive data type from a string`, async t => {
   const output = await tools.runAsync(
     `${cmd} string "My email is jenny@example.com" -t EMAIL_ADDRESS`,
@@ -56,33 +85,33 @@ test(`should handle string with no sensitive data`, async t => {
 test(`should redact a single sensitive data type from an image`, async t => {
   const testName = `redact-single-type`;
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER`,
+    `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER`,
     cwd
   );
 
   t.regex(output, /Saved image redaction results to path/);
 
-  const correct = fs.readFileSync(
-    `${testResourcePath}/${testName}.correct.png`
+  const difference = await getImageDiffPercentage(
+    `${testName}.actual.png`,
+    `${testResourcePath}/${testName}.expected.png`
   );
-  const result = fs.readFileSync(`${testName}.result.png`);
-  t.deepEqual(correct, result);
+  t.true(difference < 0.03);
 });
 
 test(`should redact multiple sensitive data types from an image`, async t => {
   const testName = `redact-multiple-types`;
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER EMAIL_ADDRESS`,
+    `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER EMAIL_ADDRESS`,
     cwd
   );
 
   t.regex(output, /Saved image redaction results to path/);
 
-  const correct = fs.readFileSync(
-    `${testResourcePath}/${testName}.correct.png`
+  const difference = await getImageDiffPercentage(
+    `${testName}.actual.png`,
+    `${testResourcePath}/${testName}.expected.png`
   );
-  const result = fs.readFileSync(`${testName}.result.png`);
-  t.deepEqual(correct, result);
+  t.true(difference < 0.03);
 });
 
 test(`should report info type errors`, async t => {
@@ -95,7 +124,7 @@ test(`should report info type errors`, async t => {
 
 test(`should report image redaction handling errors`, async t => {
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} nonexistent.result.png -t BAD_TYPE`,
+    `${cmd} image ${testImage} output.png -t BAD_TYPE`,
     cwd
   );
   t.regex(output, /Error in redactImage/);

…resources/date-shift-context.correct.csv …esources/date-shift-context.expected.csvdlp/system-test/resources/date-shift-context.correct.csv renamed to dlp/system-test/resources/date-shift-context.expected.csv dlp/system-test/resources/date-shift-context.correct.csv renamed to dlp/system-test/resources/date-shift-context.expected.csv

@@ -29,7 +29,7 @@ const uniqueField = 'Name';
 const repeatedField = 'Mystery';
 const numericField = 'Age';
 const stringBooleanField = 'Gender';
-const testProjectId = 'nodejs-docs-samples';
+const testProjectId = process.env.GCLOUD_PROJECT;
 
 test.before(tools.checkCredentials);
 

…ources/redact-multiple-types.correct.png …urces/redact-multiple-types.expected.pngdlp/system-test/resources/redact-multiple-types.correct.png renamed to dlp/system-test/resources/redact-multiple-types.expected.png dlp/system-test/resources/redact-multiple-types.correct.png renamed to dlp/system-test/resources/redact-multiple-types.expected.png

@@ -42,7 +42,7 @@ test.serial(`should create template`, async t => {
 });
 
 test(`should handle template creation errors`, async t => {
-  const output = await tools.runAsync(`${cmd} create -t BAD_INFOTYPE`);
+  const output = await tools.runAsync(`${cmd} create -i invalid_template#id`);
   t.regex(output, /Error in createInspectTemplate/);
 });