Add an integration test for uploading SARIF when the run fails

Author: henrymercer

.github/workflows/submit-sarif-failure.yml

@@ -0,0 +1,34 @@
+name: Submit SARIF after failure
+on: push
+
+env:
+  # Internal-only environment variable used to indicate that the post-init Action
+  # should expect to upload a SARIF file for the failed run.
+  CODEQL_ACTION_EXPECT_UPLOAD_FAILED_SARIF: true
+  # Make sure the uploading SARIF files feature is enabled.
+  CODEQL_ACTION_UPLOAD_FAILED_SARIF: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        language: [javascript]
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ./init
+      with:
+        languages: ${{ matrix.language }}
+    - name: Fail
+      # We want this job to pass if the Action correctly uploads the SARIF file for
+      # the failed run.
+      # Setting this step to continue on error means that it is marked as completing
+      # successfully, so will not fail the job.
+      continue-on-error: true
+      run: exit 1
+    - uses: ./analyze
+      # In a real workflow, this step wouldn't run. Since we used `continue-on-error`
+      # above, we manually disable it with an `if` condition.
+      if: false
+      with:
+        category: "/language:${{ matrix.language }}"

.github/workflows/test-report-failed-run.yml

@@ -87,14 +87,28 @@ export async function run(
     return;
   }
 
+  // Environment variable used to integration test uploading a SARIF file for failed runs
+  const expectFailedSarifUpload =
+    process.env["CODEQL_ACTION_EXPECT_UPLOAD_FAILED_SARIF"] === "true";
+
   if (process.env[CODEQL_ACTION_ANALYZE_DID_UPLOAD_SARIF] !== "true") {
     try {
       await uploadFailedSarif(config, repositoryNwo, featureEnablement, logger);
     } catch (e) {
+      if (expectFailedSarifUpload) {
+        throw new Error(
+          "Expected to upload a SARIF file for the failed run, but encountered " +
+            `the following error: ${e}`
+        );
+      }
       logger.warning(
         `Failed to upload a SARIF file for the failed run. Error: ${e}`
       );
     }
+  } else if (expectFailedSarifUpload) {
+    throw new Error(
+      "Expected to upload a SARIF file for the failed run, but didn't."
+    );
   }
 
   // Upload appropriate Actions artifacts for debugging