gh-actions-api-client.ts

import * as unzipper from 'unzipper';
import * as path from 'path';
import * as fs from 'fs-extra';
import { showAndLogErrorMessage, showAndLogWarningMessage, tmpDir } from '../helpers';
import { Credentials } from '../authentication';
import { logger } from '../logging';
import { RemoteQueryWorkflowResult } from './remote-query-workflow-result';
import { DownloadLink, createDownloadPath } from './download-link';
import { RemoteQuery } from './remote-query';
import { RemoteQueryFailureIndexItem, RemoteQueryResultIndex, RemoteQuerySuccessIndexItem } from './remote-query-result-index';
import { getErrorMessage } from '../pure/helpers-pure';

interface ApiSuccessIndexItem {
  nwo: string;
  id: string;
  sha?: string;
  results_count: number;
  bqrs_file_size: number;
  sarif_file_size?: number;
}

interface ApiFailureIndexItem {
  nwo: string;
  id: string;
  error: string;
}

interface ApiResultIndex {
  successes: ApiSuccessIndexItem[];
  failures: ApiFailureIndexItem[];
}

export async function getRemoteQueryIndex(
  credentials: Credentials,
  remoteQuery: RemoteQuery
): Promise<RemoteQueryResultIndex | undefined> {
  const controllerRepo = remoteQuery.controllerRepository;
  const owner = controllerRepo.owner;
  const repoName = controllerRepo.name;
  const workflowRunId = remoteQuery.actionsWorkflowRunId;

  const workflowUri = `https://github.com/${owner}/${repoName}/actions/runs/${workflowRunId}`;
  const artifactsUrlPath = `/repos/${owner}/${repoName}/actions/artifacts`;

  const artifactList = await listWorkflowRunArtifacts(credentials, owner, repoName, workflowRunId);
  const resultIndexArtifactId = tryGetArtifactIDfromName('result-index', artifactList);
  if (!resultIndexArtifactId) {
    return undefined;
  }
  const resultIndex = await getResultIndex(credentials, owner, repoName, resultIndexArtifactId);

  const successes = resultIndex?.successes.map(item => {
    const artifactId = getArtifactIDfromName(item.id, workflowUri, artifactList);

    return {
      id: item.id.toString(),
      artifactId: artifactId,
      nwo: item.nwo,
      sha: item.sha,
      resultCount: item.results_count,
      bqrsFileSize: item.bqrs_file_size,
      sarifFileSize: item.sarif_file_size
    } as RemoteQuerySuccessIndexItem;
  });

  const failures = resultIndex?.failures.map(item => {
    return {
      id: item.id.toString(),
      nwo: item.nwo,
      error: item.error
    } as RemoteQueryFailureIndexItem;
  });

  return {
    artifactsUrlPath,
    successes: successes || [],
    failures: failures || []
  };
}

export async function cancelRemoteQuery(
  credentials: Credentials,
  remoteQuery: RemoteQuery
): Promise<void> {
  const octokit = await credentials.getOctokit();
  const { actionsWorkflowRunId, controllerRepository: { owner, name } } = remoteQuery;
  const response = await octokit.request(`POST /repos/${owner}/${name}/actions/runs/${actionsWorkflowRunId}/cancel`);
  if (response.status >= 300) {
    throw new Error(`Error cancelling variant analysis: ${response.status} ${response?.data?.message || ''}`);
  }
}

export async function downloadArtifactFromLink(
  credentials: Credentials,
  storagePath: string,
  downloadLink: DownloadLink
): Promise<string> {

  const octokit = await credentials.getOctokit();

  const extractedPath = createDownloadPath(storagePath, downloadLink);

  // first check if we already have the artifact
  if (!(await fs.pathExists(extractedPath))) {
    // Download the zipped artifact.
    const response = await octokit.request(`GET ${downloadLink.urlPath}/zip`, {});

    const zipFilePath = createDownloadPath(storagePath, downloadLink, 'zip');
    await saveFile(`${zipFilePath}`, response.data as ArrayBuffer);

    // Extract the zipped artifact.
    await unzipFile(zipFilePath, extractedPath);
  }
  return path.join(extractedPath, downloadLink.innerFilePath || '');
}

/**
 * Downloads the result index artifact and extracts the result index items.
 * @param credentials Credentials for authenticating to the GitHub API.
 * @param owner
 * @param repo
 * @param workflowRunId The ID of the workflow run to get the result index for.
 * @returns An object containing the result index.
 */
async function getResultIndex(
  credentials: Credentials,
  owner: string,
  repo: string,
  artifactId: number
): Promise<ApiResultIndex | undefined> {
  const artifactPath = await downloadArtifact(credentials, owner, repo, artifactId);
  const indexFilePath = path.join(artifactPath, 'index.json');
  if (!(await fs.pathExists(indexFilePath))) {
    void showAndLogWarningMessage('Could not find an `index.json` file in the result artifact.');
    return undefined;
  }
  const resultIndex = await fs.readFile(path.join(artifactPath, 'index.json'), 'utf8');

  try {
    return JSON.parse(resultIndex);
  } catch (error) {
    throw new Error(`Invalid result index file: ${error}`);
  }
}

/**
 * Gets the status of a workflow run.
 * @param credentials Credentials for authenticating to the GitHub API.
 * @param owner
 * @param repo
 * @param workflowRunId The ID of the workflow run to get the result index for.
 * @returns The workflow run status.
 */
export async function getWorkflowStatus(
  credentials: Credentials,
  owner: string,
  repo: string,
  workflowRunId: number): Promise<RemoteQueryWorkflowResult> {
  const octokit = await credentials.getOctokit();

  const workflowRun = await octokit.rest.actions.getWorkflowRun({
    owner,
    repo,
    run_id: workflowRunId
  });

  if (workflowRun.data.status === 'completed') {
    if (workflowRun.data.conclusion === 'success') {
      return { status: 'CompletedSuccessfully' };
    } else {
      const error = getWorkflowError(workflowRun.data.conclusion);
      return { status: 'CompletedUnsuccessfully', error };
    }
  }

  return { status: 'InProgress' };
}

/**
 * Lists the workflow run artifacts for the given workflow run ID.
 * @param credentials Credentials for authenticating to the GitHub API.
 * @param owner
 * @param repo
 * @param workflowRunId The ID of the workflow run to list artifacts for.
 * @returns An array of artifact details (including artifact name and ID).
 */
async function listWorkflowRunArtifacts(
  credentials: Credentials,
  owner: string,
  repo: string,
  workflowRunId: number
) {
  const octokit = await credentials.getOctokit();

  // There are limits on the number of artifacts that are returned by the API
  // so we use paging to make sure we retrieve all of them.
  let morePages = true;
  let pageNum = 1;
  const allArtifacts = [];

  while (morePages) {
    const response = await octokit.rest.actions.listWorkflowRunArtifacts({
      owner,
      repo,
      run_id: workflowRunId,
      per_page: 100,
      page: pageNum
    });

    allArtifacts.push(...response.data.artifacts);
    pageNum++;
    if (response.data.artifacts.length < 100) {
      morePages = false;
    }
  }

  return allArtifacts;
}

/**
 * @param artifactName The artifact name, as a string.
 * @param artifacts An array of artifact details (from the "list workflow run artifacts" API response).
 * @returns The artifact ID corresponding to the given artifact name.
 */
function getArtifactIDfromName(
  artifactName: string,
  workflowUri: string,
  artifacts: Array<{ id: number, name: string }>
): number {
  const artifactId = tryGetArtifactIDfromName(artifactName, artifacts);

  if (!artifactId) {
    const errorMessage =
      `Could not find artifact with name ${artifactName} in workflow ${workflowUri}.
      Please check whether the workflow run has successfully completed.`;
    throw Error(errorMessage);
  }

  return artifactId;
}

/**
 * @param artifactName The artifact name, as a string.
 * @param artifacts An array of artifact details (from the "list workflow run artifacts" API response).
 * @returns The artifact ID corresponding to the given artifact name, if it exists.
 */
function tryGetArtifactIDfromName(
  artifactName: string,
  artifacts: Array<{ id: number, name: string }>
): number | undefined {
  const artifact = artifacts.find(a => a.name === artifactName);

  return artifact?.id;
}

/**
 * Downloads an artifact from a workflow run.
 * @param credentials Credentials for authenticating to the GitHub API.
 * @param owner
 * @param repo
 * @param artifactId The ID of the artifact to download.
 * @returns The path to the enclosing directory of the unzipped artifact.
 */
async function downloadArtifact(
  credentials: Credentials,
  owner: string,
  repo: string,
  artifactId: number
): Promise<string> {
  const octokit = await credentials.getOctokit();
  const response = await octokit.rest.actions.downloadArtifact({
    owner,
    repo,
    artifact_id: artifactId,
    archive_format: 'zip',
  });
  const artifactPath = path.join(tmpDir.name, `${artifactId}`);
  await saveFile(`${artifactPath}.zip`, response.data as ArrayBuffer);
  await unzipFile(`${artifactPath}.zip`, artifactPath);
  return artifactPath;
}

async function saveFile(filePath: string, data: ArrayBuffer): Promise<void> {
  void logger.log(`Saving file to ${filePath}`);
  await fs.writeFile(filePath, Buffer.from(data));
}

async function unzipFile(sourcePath: string, destinationPath: string) {
  void logger.log(`Unzipping file to ${destinationPath}`);
  const file = await unzipper.Open.file(sourcePath);
  await file.extract({ path: destinationPath });
}

function getWorkflowError(conclusion: string | null): string {
  if (!conclusion) {
    return 'Workflow finished without a conclusion';
  }

  if (conclusion === 'cancelled') {
    return 'Variant analysis execution was cancelled.';
  }

  if (conclusion === 'timed_out') {
    return 'Variant analysis execution timed out.';
  }

  if (conclusion === 'failure') {
    // TODO: Get the actual error from the workflow or potentially
    // from an artifact from the action itself.
    return 'Variant analysis execution has failed.';
  }

  return `Unexpected variant analysis execution conclusion: ${conclusion}`;
}

/**
 * Creates a gist with the given description and files.
 * Returns the URL of the created gist.
 */
export async function createGist(
  credentials: Credentials,
  description: string,
  files: { [key: string]: { content: string } }
): Promise<string | undefined> {
  const octokit = await credentials.getOctokit();
  const response = await octokit.request('POST /gists', {
    description,
    files,
    public: false,
  });
  if (response.status >= 300) {
    throw new Error(`Error exporting variant analysis results: ${response.status} ${response?.data || ''}`);
  }
  return response.data.html_url;
}

const repositoriesMetadataQuery = `query Stars($repos: String!, $pageSize: Int!, $cursor: String) {
  search(
    query: $repos
    type: REPOSITORY
    first: $pageSize
    after: $cursor
  ) {
    edges {
      node {
        ... on Repository {
          name
          owner {
            login
          }
          stargazerCount
          updatedAt
        }
      }
      cursor
    }
  }
}`;

type RepositoriesMetadataQueryResponse = {
  search: {
    edges: {
      cursor: string;
      node: {
        name: string;
        owner: {
          login: string;
        };
        stargazerCount: number;
        updatedAt: string; // Actually a ISO Date string
      }
    }[]
  }
};

export type RepositoriesMetadata = Record<string, { starCount: number, lastUpdated: number }>

export async function getRepositoriesMetadata(credentials: Credentials, nwos: string[], pageSize = 100): Promise<RepositoriesMetadata> {
  const octokit = await credentials.getOctokit();
  const repos = `repo:${nwos.join(' repo:')} fork:true`;
  let cursor = null;
  const metadata: RepositoriesMetadata = {};
  try {
    do {
      const response: RepositoriesMetadataQueryResponse = await octokit.graphql({
        query: repositoriesMetadataQuery,
        repos,
        pageSize,
        cursor
      });
      cursor = response.search.edges.length === pageSize ? response.search.edges[pageSize - 1].cursor : null;

      for (const edge of response.search.edges) {
        const node = edge.node;
        const owner = node.owner.login;
        const name = node.name;
        const starCount = node.stargazerCount;
        // lastUpdated is always negative since it happened in the past.
        const lastUpdated = new Date(node.updatedAt).getTime() - Date.now();
        metadata[`${owner}/${name}`] = {
          starCount, lastUpdated
        };
      }

    } while (cursor);
  } catch (e) {
    void showAndLogErrorMessage(`Error retrieving repository metadata for variant analysis: ${getErrorMessage(e)}`);
  }

  return metadata;
}