Skip to content

Commit a188ebf

Browse files
dagooddagood
authored
Make SDL/Guardian scripts and templates flexible (#7611)
1 parent 1b053ba commit a188ebf

6 files changed

Lines changed: 395 additions & 57 deletions

File tree

‎eng/common/sdl/configure-sdl-tool.ps1‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
Param(
2+
[string] $GuardianCliLocation,
3+
[string] $WorkingDirectory,
4+
[string] $TargetDirectory,
5+
[string] $GdnFolder,
6+
# The list of Guardian tools to configure. For each object in the array:
7+
# - If the item is a [hashtable], it must contain these entries:
8+
# - Name = The tool name as Guardian knows it.
9+
# - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
10+
# among all tool entries with the same Name.
11+
# - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
12+
# - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
13+
[object[]] $ToolsList,
14+
[string] $GuardianLoggerLevel='Standard',
15+
# Optional: Additional params to add to any tool using CredScan.
16+
[string[]] $CrScanAdditionalRunConfigParams,
17+
# Optional: Additional params to add to any tool using PoliCheck.
18+
[string[]] $PoliCheckAdditionalRunConfigParams
19+
)
20+
21+
$ErrorActionPreference = 'Stop'
22+
Set-StrictMode -Version 2.0
23+
$disableConfigureToolsetImport = $true
24+
$global:LASTEXITCODE = 0
25+
26+
try {
27+
# `tools.ps1` checks $ci to perform some actions. Since the SDL
28+
# scripts don't necessarily execute in the same agent that run the
29+
# build.ps1/sh script this variable isn't automatically set.
30+
$ci = $true
31+
. $PSScriptRoot\..\tools.ps1
32+
33+
# Normalize tools list: all in [hashtable] form with defined values for each key.
34+
$ToolsList = $ToolsList |
35+
ForEach-Object {
36+
if ($_ -is [string]) {
37+
$_ = @{ Name = $_ }
38+
}
39+
40+
if (-not ($_['Scenario'])) { $_.Scenario = "" }
41+
if (-not ($_['Args'])) { $_.Args = @() }
42+
$_
43+
}
44+
45+
Write-Host "List of tools to configure:"
46+
$ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
47+
48+
# We store config files in the r directory of .gdn
49+
$gdnConfigPath = Join-Path $GdnFolder 'r'
50+
$ValidPath = Test-Path $GuardianCliLocation
51+
52+
if ($ValidPath -eq $False)
53+
{
54+
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
55+
ExitWithExitCode 1
56+
}
57+
58+
foreach ($tool in $ToolsList) {
59+
# Put together the name and scenario to make a unique key.
60+
$toolConfigName = $tool.Name
61+
if ($tool.Scenario) {
62+
$toolConfigName += "_" + $tool.Scenario
63+
}
64+
65+
Write-Host "=== Configuring $toolConfigName..."
66+
67+
$gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
68+
69+
# For some tools, add default and automatic args.
70+
if ($tool.Name -eq 'credscan') {
71+
if ($targetDirectory) {
72+
$tool.Args += "TargetDirectory < $TargetDirectory"
73+
}
74+
$tool.Args += "OutputType < pre"
75+
$tool.Args += $CrScanAdditionalRunConfigParams
76+
} elseif ($tool.Name -eq 'policheck') {
77+
if ($targetDirectory) {
78+
$tool.Args += "Target < $TargetDirectory"
79+
}
80+
$tool.Args += $PoliCheckAdditionalRunConfigParams
81+
}
82+
83+
# Create variable pointing to the args array directly so we can use splat syntax later.
84+
$toolArgs = $tool.Args
85+
86+
# Configure the tool. If args array is provided or the current tool has some default arguments
87+
# defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
88+
# one per parameter. Doc page for "guardian configure":
89+
# https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
90+
Exec-BlockVerbosely {
91+
& $GuardianCliLocation configure `
92+
--working-directory $WorkingDirectory `
93+
--tool $tool.Name `
94+
--output-path $gdnConfigFile `
95+
--logger-level $GuardianLoggerLevel `
96+
--noninteractive `
97+
--force `
98+
$(if ($toolArgs) { "--args" }) @toolArgs
99+
Exit-IfNZEC "Sdl"
100+
}
101+
102+
Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
103+
}
104+
}
105+
catch {
106+
Write-Host $_.ScriptStackTrace
107+
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
108+
ExitWithExitCode 1
109+
}

‎eng/common/sdl/execute-all-sdl-tools.ps1‎

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,17 @@ Param(
77
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
88
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
99
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
10-
[string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
11-
[string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
10+
11+
# Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
12+
# format.
13+
[object[]] $SourceToolsList,
14+
# Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
15+
# list format.
16+
[object[]] $ArtifactToolsList,
17+
# Optional: list of SDL tools to run without automatically specifying a target directory. See
18+
# 'configure-sdl-tool.ps1' for tools list format.
19+
[object[]] $CustomToolsList,
20+
1221
[bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
1322
[string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
1423
[string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
@@ -63,13 +72,16 @@ try {
6372
ExitWithExitCode 1
6473
}
6574

66-
& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
75+
Exec-BlockVerbosely {
76+
& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
77+
}
6778
$gdnFolder = Join-Path $workingDirectory '.gdn'
6879

6980
if ($TsaOnboard) {
7081
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
71-
Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
72-
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
82+
Exec-BlockVerbosely {
83+
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
84+
}
7385
if ($LASTEXITCODE -ne 0) {
7486
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
7587
ExitWithExitCode $LASTEXITCODE
@@ -80,20 +92,51 @@ try {
8092
}
8193
}
8294

83-
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
84-
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
95+
# Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
96+
function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
97+
if ($tools -and $tools.Count -gt 0) {
98+
Exec-BlockVerbosely {
99+
& $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
100+
-GuardianCliLocation $guardianCliLocation `
101+
-WorkingDirectory $workingDirectory `
102+
-TargetDirectory $targetDirectory `
103+
-GdnFolder $gdnFolder `
104+
-ToolsList $tools `
105+
-AzureDevOpsAccessToken $AzureDevOpsAccessToken `
106+
-GuardianLoggerLevel $GuardianLoggerLevel `
107+
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
108+
-PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
109+
if ($BreakOnFailure) {
110+
Exit-IfNZEC "Sdl"
111+
}
112+
}
113+
}
85114
}
86-
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
87-
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
115+
116+
# Configure Artifact and Source tools with default Target directories.
117+
Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
118+
Configure-ToolsList $SourceToolsList $SourceDirectory
119+
# Configure custom tools with no default Target directory.
120+
Configure-ToolsList $CustomToolsList $null
121+
122+
# At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
123+
# (If we used "run" multiple times, each run would overwrite data from earlier runs.)
124+
Exec-BlockVerbosely {
125+
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
126+
-GuardianCliLocation $guardianCliLocation `
127+
-WorkingDirectory $workingDirectory `
128+
-UpdateBaseline $UpdateBaseline `
129+
-GdnFolder $gdnFolder
88130
}
89131

90132
if ($TsaPublish) {
91133
if ($TsaBranchName -and $BuildNumber) {
92134
if (-not $TsaRepositoryName) {
93135
$TsaRepositoryName = "$($Repository)-$($BranchName)"
94136
}
95-
Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
96-
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
137+
Exec-BlockVerbosely {
138+
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
139+
}
97140
if ($LASTEXITCODE -ne 0) {
98141
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
99142
ExitWithExitCode $LASTEXITCODE
@@ -106,7 +149,11 @@ try {
106149

107150
if ($BreakOnFailure) {
108151
Write-Host "Failing the build in case of breaking results..."
109-
& $guardianCliLocation break
152+
Exec-BlockVerbosely {
153+
& $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
154+
}
155+
} else {
156+
Write-Host "Letting the build pass even if there were breaking results..."
110157
}
111158
}
112159
catch {

‎eng/common/sdl/extract-artifact-archives.ps1‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# This script looks for each archive file in a directory and extracts it into the target directory.
2+
# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
3+
# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
4+
param(
5+
# Full path to directory where archives are stored.
6+
[Parameter(Mandatory=$true)][string] $InputPath,
7+
# Full path to directory to extract archives into. May be the same as $InputPath.
8+
[Parameter(Mandatory=$true)][string] $ExtractPath
9+
)
10+
11+
$ErrorActionPreference = 'Stop'
12+
Set-StrictMode -Version 2.0
13+
14+
$disableConfigureToolsetImport = $true
15+
16+
try {
17+
# `tools.ps1` checks $ci to perform some actions. Since the SDL
18+
# scripts don't necessarily execute in the same agent that run the
19+
# build.ps1/sh script this variable isn't automatically set.
20+
$ci = $true
21+
. $PSScriptRoot\..\tools.ps1
22+
23+
Measure-Command {
24+
$jobs = @()
25+
26+
# Find archive files for non-Windows and Windows builds.
27+
$archiveFiles = @(
28+
Get-ChildItem (Join-Path $InputPath "*.tar.gz")
29+
Get-ChildItem (Join-Path $InputPath "*.zip")
30+
)
31+
32+
foreach ($targzFile in $archiveFiles) {
33+
$jobs += Start-Job -ScriptBlock {
34+
$file = $using:targzFile
35+
$fileName = [System.IO.Path]::GetFileName($file)
36+
$extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
37+
38+
New-Item $extractDir -ItemType Directory -Force | Out-Null
39+
40+
Write-Host "Extracting '$file' to '$extractDir'..."
41+
42+
# Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
43+
# This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
44+
# error. Save output so it can be stored in the exception string along with context.
45+
$output = tar -xf $file -C $extractDir 2>&1
46+
# Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
47+
# don't have access to the outer scope.
48+
if ($LASTEXITCODE -ne 0) {
49+
throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
50+
}
51+
52+
Write-Host "Extracted to $extractDir"
53+
}
54+
}
55+
56+
Receive-Job $jobs -Wait
57+
}
58+
}
59+
catch {
60+
Write-Host $_
61+
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
62+
ExitWithExitCode 1
63+
}

‎eng/common/sdl/run-sdl.ps1‎

Lines changed: 13 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
Param(
22
[string] $GuardianCliLocation,
33
[string] $WorkingDirectory,
4-
[string] $TargetDirectory,
54
[string] $GdnFolder,
6-
[string[]] $ToolsList,
75
[string] $UpdateBaseline,
8-
[string] $GuardianLoggerLevel='Standard',
9-
[string[]] $CrScanAdditionalRunConfigParams,
10-
[string[]] $PoliCheckAdditionalRunConfigParams
6+
[string] $GuardianLoggerLevel='Standard'
117
)
128

139
$ErrorActionPreference = 'Stop'
@@ -23,7 +19,6 @@ try {
2319
. $PSScriptRoot\..\tools.ps1
2420

2521
# We store config files in the r directory of .gdn
26-
Write-Host $ToolsList
2722
$gdnConfigPath = Join-Path $GdnFolder 'r'
2823
$ValidPath = Test-Path $GuardianCliLocation
2924

@@ -33,37 +28,18 @@ try {
3328
ExitWithExitCode 1
3429
}
3530

36-
$configParam = @('--config')
37-
38-
foreach ($tool in $ToolsList) {
39-
$gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
40-
Write-Host $tool
41-
# We have to manually configure tools that run on source to look at the source directory only
42-
if ($tool -eq 'credscan') {
43-
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
44-
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
45-
if ($LASTEXITCODE -ne 0) {
46-
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
47-
ExitWithExitCode $LASTEXITCODE
48-
}
49-
}
50-
if ($tool -eq 'policheck') {
51-
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
52-
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
53-
if ($LASTEXITCODE -ne 0) {
54-
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
55-
ExitWithExitCode $LASTEXITCODE
56-
}
57-
}
58-
59-
$configParam+=$gdnConfigFile
60-
}
61-
62-
Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
63-
& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
64-
if ($LASTEXITCODE -ne 0) {
65-
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
66-
ExitWithExitCode $LASTEXITCODE
31+
$gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
32+
Write-Host "Discovered Guardian config files:"
33+
$gdnConfigFiles | Out-String | Write-Host
34+
35+
Exec-BlockVerbosely {
36+
& $GuardianCliLocation run `
37+
--working-directory $WorkingDirectory `
38+
--baseline mainbaseline `
39+
--update-baseline $UpdateBaseline `
40+
--logger-level $GuardianLoggerLevel `
41+
--config @gdnConfigFiles
42+
Exit-IfNZEC "Sdl"
6743
}
6844
}
6945
catch {

0 commit comments

Comments
 (0)