Retry all http calls for artifact upload and download

packages/artifact/__tests__/download.test.ts

@@ -71,7 +71,7 @@ describe('Download Tests', () => {
     setupFailedResponse()
     const downloadHttpClient = new DownloadHttpClient()
     expect(downloadHttpClient.listArtifacts()).rejects.toThrow(
-      'Unable to list artifacts for the run'
+      'List Artifacts failed: Artifact service responded with 500'
     )
   })
 
@@ -113,7 +113,7 @@ describe('Download Tests', () => {
         configVariables.getRuntimeUrl()
       )
     ).rejects.toThrow(
-      `Unable to get ContainersItems from ${configVariables.getRuntimeUrl()}`
+      `Get Container Items failed: Artifact service responded with 500`
     )
   })
 

packages/artifact/__tests__/retry.test.ts

@@ -0,0 +1,116 @@
+import {retry} from '../src/internal/requestUtils'
+import * as core from '@actions/core'
+
+interface ITestResponse {
+  statusCode: number
+  result: string | null
+  error: Error | null
+}
+
+function TestResponse(
+  action: number | Error,
+  result: string | null = null
+): ITestResponse {
+  if (action instanceof Error) {
+    return {
+      statusCode: -1,
+      result,
+      error: action
+    }
+  } else {
+    return {
+      statusCode: action,
+      result,
+      error: null
+    }
+  }
+}
+
+async function handleResponse(
+  response: ITestResponse | undefined
+): Promise<ITestResponse> {
+  if (!response) {
+    // eslint-disable-next-line no-undef
+    fail('Retry method called too many times')
+  }
+
+  if (response.error) {
+    throw response.error
+  } else {
+    return Promise.resolve(response)
+  }
+}
+
+async function testRetryExpectingResult(
+  responses: ITestResponse[],
+  expectedResult: string | null
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  const actualResult = await retry(
+    'test',
+    async () => handleResponse(responses.pop()),
+    (response: ITestResponse) => response.statusCode,
+    new Map(), // extra error message for any particular http codes
+    2, // maxAttempts
+    0 // delay
+  )
+
+  expect(actualResult.result).toEqual(expectedResult)
+}
+
+async function testRetryExpectingError(
+  responses: ITestResponse[]
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  expect(
+    retry(
+      'test',
+      async () => handleResponse(responses.pop()),
+      (response: ITestResponse) => response.statusCode,
+      new Map(), // extra error message for any particular http codes
+      2, // maxAttempts,
+      0 // delay
+    )
+  ).rejects.toBeInstanceOf(Error)
+}
+
+beforeAll(async () => {
+  // mock all output so that there is less noise when running tests
+  jest.spyOn(console, 'log').mockImplementation(() => {})
+  jest.spyOn(core, 'debug').mockImplementation(() => {})
+  jest.spyOn(core, 'info').mockImplementation(() => {})
+  jest.spyOn(core, 'warning').mockImplementation(() => {})
+  jest.spyOn(core, 'error').mockImplementation(() => {})
+})
+
+test('retry works on successful response', async () => {
+  await testRetryExpectingResult([TestResponse(200, 'Ok')], 'Ok')
+})
+
+test('retry works after retryable status code', async () => {
+  await testRetryExpectingResult(
+    [TestResponse(503), TestResponse(200, 'Ok')],
+    'Ok'
+  )
+})
+
+test('retry fails after exhausting retries', async () => {
+  await testRetryExpectingError([
+    TestResponse(503),
+    TestResponse(503),
+    TestResponse(200, 'Ok')
+  ])
+})
+
+test('retry fails after non-retryable status code', async () => {
+  await testRetryExpectingError([TestResponse(500), TestResponse(200, 'Ok')])
+})
+
+test('retry works after error', async () => {
+  await testRetryExpectingResult(
+    [TestResponse(new Error('Test error')), TestResponse(200, 'Ok')],
+    'Ok'
+  )
+})

packages/artifact/__tests__/upload.test.ts

@@ -102,7 +102,7 @@ describe('Upload Tests', () => {
       uploadHttpClient.createArtifactInFileContainer(artifactName)
     ).rejects.toEqual(
       new Error(
-        `Unable to create a container for the artifact invalid-artifact-name at ${getArtifactUrl()}`
+        `Create Artifact Container failed: Artifact service responded with 400 : The artifact name invalid-artifact-name is not valid. Request URL ${getArtifactUrl()}`
       )
     )
   })
@@ -125,7 +125,7 @@ describe('Upload Tests', () => {
       uploadHttpClient.createArtifactInFileContainer(artifactName)
     ).rejects.toEqual(
       new Error(
-        'Artifact storage quota has been hit. Unable to upload any new artifacts'
+        'Create Artifact Container failed: Artifact service responded with 403 : Artifact storage quota has been hit. Unable to upload any new artifacts'
       )
     )
   })
@@ -362,7 +362,9 @@ describe('Upload Tests', () => {
     const uploadHttpClient = new UploadHttpClient()
     expect(
       uploadHttpClient.patchArtifactSize(-2, 'my-artifact')
-    ).rejects.toThrow('Unable to finish uploading artifact my-artifact')
+    ).rejects.toThrow(
+      'Patch Artifact Size failed: Artifact service responded with 400'
+    )
   })
 
   /**

packages/artifact/src/internal/download-http-client.ts

@@ -11,7 +11,8 @@ import {
   tryGetRetryAfterValueTimeInMilliseconds,
   displayHttpDiagnostics,
   getFileSize,
-  rmFile
+  rmFile,
+  sleep
 } from './utils'
 import {URL} from 'url'
 import {StatusReporter} from './status-reporter'
@@ -22,6 +23,7 @@ import {HttpManager} from './http-manager'
 import {DownloadItem} from './download-specification'
 import {getDownloadFileConcurrency, getRetryLimit} from './config-variables'
 import {IncomingHttpHeaders} from 'http'
+import {retryHttpClientResponse} from './requestUtils'
 
 export class DownloadHttpClient {
   // http manager is used for concurrent connections when downloading multiple files at once
@@ -46,16 +48,11 @@ export class DownloadHttpClient {
     // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
     const client = this.downloadHttpManager.getClient(0)
     const headers = getDownloadHeaders('application/json')
-    const response = await client.get(artifactUrl, headers)
-    const body: string = await response.readBody()
-
-    if (isSuccessStatusCode(response.message.statusCode) && body) {
-      return JSON.parse(body)
-    }
-    displayHttpDiagnostics(response)
-    throw new Error(
-      `Unable to list artifacts for the run. Resource Url ${artifactUrl}`
+    const response = await retryHttpClientResponse('List Artifacts', async () =>
+      client.get(artifactUrl, headers)
     )
+    const body: string = await response.readBody()
+    return JSON.parse(body)
   }
 
   /**
@@ -74,14 +71,12 @@ export class DownloadHttpClient {
     // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
     const client = this.downloadHttpManager.getClient(0)
     const headers = getDownloadHeaders('application/json')
-    const response = await client.get(resourceUrl.toString(), headers)
+    const response = await retryHttpClientResponse(
+      'Get Container Items',
+      async () => client.get(resourceUrl.toString(), headers)
+    )
     const body: string = await response.readBody()
-
-    if (isSuccessStatusCode(response.message.statusCode) && body) {
-      return JSON.parse(body)
-    }
-    displayHttpDiagnostics(response)
-    throw new Error(`Unable to get ContainersItems from ${resourceUrl}`)
+    return JSON.parse(body)
   }
 
   /**
@@ -188,14 +183,14 @@ export class DownloadHttpClient {
           core.info(
             `Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the download`
           )
-          await new Promise(resolve => setTimeout(resolve, retryAfterValue))
+          await sleep(retryAfterValue)
         } else {
           // Back off using an exponential value that depends on the retry count
           const backoffTime = getExponentialRetryTimeInMilliseconds(retryCount)
           core.info(
             `Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the download`
           )
-          await new Promise(resolve => setTimeout(resolve, backoffTime))
+          await sleep(backoffTime)
         }
         core.info(
           `Finished backoff for retry #${retryCount}, continuing with download`

packages/artifact/src/internal/requestUtils.ts

@@ -0,0 +1,74 @@
+import {IHttpClientResponse} from '@actions/http-client/interfaces'
+import {isRetryableStatusCode, isSuccessStatusCode, sleep} from './utils'
+import * as core from '@actions/core'
+
+export async function retry<T>(
+  name: string,
+  method: () => Promise<T>,
+  getStatusCode: (response: T) => number | undefined,
+  errorMessages: Map<number, string>,
+  maxAttempts: number,
+  delay: number
+): Promise<T> {
+  let response: T | undefined = undefined
+  let statusCode: number | undefined = undefined
+  let isRetryable = false
+  let errorMessage = ''
+  let extraErrorInformation: string | undefined = undefined
+  let attempt = 1
+
+  while (attempt <= maxAttempts) {
+    try {
+      response = await method()
+      statusCode = getStatusCode(response)
+
+      if (isSuccessStatusCode(statusCode)) {
+        return response
+      }
+
+      // Extra error information that we want to display if a particular response code is hit
+      if (statusCode) {
+        extraErrorInformation = errorMessages.get(statusCode)
+      }
+
+      isRetryable = isRetryableStatusCode(statusCode)
+      errorMessage = `Artifact service responded with ${statusCode}`
+    } catch (error) {
+      isRetryable = true
+      errorMessage = error.message
+    }
+
+    if (!isRetryable) {
+      core.info(`${name} - Error is not retryable`)
+      break
+    }
+
+    core.info(
+      `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`
+    )
+
+    await sleep(delay)
+    attempt++
+  }
+
+  if (extraErrorInformation) {
+    throw Error(`${name} failed: ${errorMessage} : ${extraErrorInformation}`)
+  }
+  throw Error(`${name} failed: ${errorMessage}`)
+}
+
+export async function retryHttpClientResponse<T>(
+  name: string,
+  method: () => Promise<IHttpClientResponse>,
+  errorMessages: Map<number, string> = new Map(),
+  maxAttempts = 3
+): Promise<IHttpClientResponse> {
+  return await retry(
+    name,
+    method,
+    (response: IHttpClientResponse) => response.message.statusCode,
+    errorMessages,
+    maxAttempts,
+    5000
+  )
+}