Skip to content

Commit 247e21a

Browse files
mrlockstarmrlockstar
committed
making hub terraform
1 parent c8ec2f5 commit 247e21a

34 files changed

Lines changed: 1574 additions & 104 deletions

‎.azuredevops/pipelines/deploy.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
trigger: none
2+
pr: none
3+
4+
parameters:
5+
- name: commitSHA
6+
displayName: Commit SHA
7+
type: string
8+
- name: environment
9+
displayName: Environment
10+
type: string
11+
- name: prNumber
12+
displayName: Pull request number
13+
type: string
14+
default: ''
15+
- name: pool
16+
displayName: ADO management pool
17+
type: string
18+
19+
stages:
20+
- stage: ${{ parameters.environment }}
21+
displayName: Deploy to ${{ parameters.environment }} environment
22+
pool:
23+
name: ${{ parameters.pool }}
24+
lockBehavior: sequential
25+
isSkippable: false
26+
27+
jobs:
28+
- deployment: DeployApp
29+
displayName: Deploy application
30+
environment: ${{ parameters.environment }}
31+
strategy:
32+
runOnce:
33+
deploy:
34+
steps:
35+
- checkout: self
36+
37+
- task: UsePythonVersion@0
38+
inputs:
39+
versionSpec: '3.x'
40+
architecture: 'x64'
41+
42+
- task: TerraformInstaller@1
43+
displayName: Install terraform
44+
inputs:
45+
terraformVersion: 1.7.0
46+
47+
- task: AzureCLI@2
48+
displayName: Run terraform
49+
inputs:
50+
azureSubscription: lung-${{ parameters.environment }}
51+
scriptType: bash
52+
scriptLocation: inlineScript
53+
addSpnToEnvironment: true
54+
inlineScript: |
55+
export ARM_TENANT_ID="$tenantId"
56+
export ARM_CLIENT_ID="$servicePrincipalId"
57+
export ARM_OIDC_TOKEN="$idToken"
58+
export ARM_USE_OIDC=true
59+
make ci ${{ parameters.environment }} terraform-apply DOCKER_IMAGE_TAG=git-sha-${{ parameters.commitSHA }} PR_NUMBER=${{ parameters.prNumber }}
60+
61+
# - task: AzureCLI@2
62+
# displayName: Run database setup
63+
# inputs:
64+
# azureSubscription: lungcs-${{ parameters.environment }}
65+
# scriptType: bash
66+
# scriptLocation: inlineScript
67+
# addSpnToEnvironment: true
68+
# inlineScript: make ci ${{ parameters.environment }} db-setup PR_NUMBER=${{ parameters.prNumber }}
69+
70+
# - task: AzureCLI@2
71+
# displayName: Run notifications smoke test
72+
# inputs:
73+
# azureSubscription: lungcs-${{ parameters.environment }}
74+
# scriptType: bash
75+
# scriptLocation: inlineScript
76+
# addSpnToEnvironment: true
77+
# inlineScript: make ci ${{ parameters.environment }} notifications-smoke-test PR_NUMBER=${{ parameters.prNumber }}

‎.azuredevops/pipelines/hub-infrastructure-dev.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
---
2+
name: $(Build.SourceBranchName)-$(Date:yyyyMMdd)_$(Rev:r)
3+
trigger: none
4+
pr: none
5+
6+
pool:
7+
name: private-pool-hub-nonlive-uks
8+
# vmImage: ubuntu-latest
9+
10+
resources:
11+
repositories:
12+
- repository: dtos-devops-templates
13+
type: github
14+
name: NHSDigital/dtos-devops-templates
15+
ref: main
16+
endpoint: NHSDigital
17+
18+
variables:
19+
- group: NON_LIVE_hub_backend
20+
- name: TF_DIRECTORY
21+
value: $(System.DefaultWorkingDirectory)/lung_cancer_screening/infrastructure/terraform/hub
22+
- name: TF_VERSION
23+
value: 1.14.3
24+
- name: TF_PLAN_ARTIFACT
25+
value: tf_plan_hub_art_NONLIVE_dev
26+
- name: ENVIRONMENT
27+
value: nonlive-hub
28+
29+
stages:
30+
- stage: terraform_plan
31+
displayName: Terraform Plan
32+
condition: eq(variables['Build.Reason'], 'Manual')
33+
variables:
34+
tfVarsFile: ../../environments/$(ENVIRONMENT)/variables.tfvars
35+
jobs:
36+
- job: init_and_plan
37+
displayName: Init, plan, store artifact
38+
steps:
39+
- checkout: self
40+
- checkout: dtos-devops-templates
41+
- task: Bash@3
42+
displayName: 'Debug Terraform directory'
43+
inputs:
44+
targetType: 'inline'
45+
script: |
46+
find . -type d | grep dtos-devops-templates
47+
pwd
48+
ls -la
49+
echo "TF_DIRECTORY=$(TF_DIRECTORY)"
50+
# cd $(TF_DIRECTORY)
51+
ls -ltr
52+
find .
53+
terraform --version || true
54+
- template: .azuredevops/templates/steps/tf_plan.yaml@dtos-devops-templates
55+
56+
- stage: terraform_apply
57+
displayName: Terraform Apply
58+
dependsOn: [terraform_plan]
59+
condition: and(eq(dependencies.terraform_plan.outputs['init_and_plan.TerraformPlan.changesPresent'], 'true'), eq(variables['Build.Reason'], 'Manual'))
60+
jobs:
61+
- deployment: terraform_apply
62+
displayName: Init, get plan artifact, apply
63+
environment: $(ENVIRONMENT)
64+
strategy:
65+
runOnce:
66+
deploy:
67+
steps:
68+
- checkout: self
69+
- checkout: dtos-devops-templates
70+
- template: .azuredevops/templates/steps/tf_apply.yaml@dtos-devops-templates
71+
# parameters:
72+
# # The Application Gateway Config null resource needs an authenticated PowerShell context, hence our shell is pscore
73+
# # AzureCLI@2 task is used rather than AzurePowerShell@5 because Terraform is unable to use a PowerShell authentication context directly
74+
# tfApplyShell: pscore
75+
# tfApplyScript: |
76+
# if (-not (Get-Module -ListAvailable -Name Az)) { Install-Module -Name Az -AllowClobber -Scope CurrentUser -Force -SkipPublisherCheck }
77+
# $token = az account get-access-token --resource https://management.azure.com/ --query accessToken --output tsv
78+
# Connect-AzAccount -AccessToken $token -AccountId $env:servicePrincipalId -Tenant $env:tenantId -Subscription $(TF_VAR_TARGET_SUBSCRIPTION_ID) -ErrorAction Stop

‎.gitleaksignore‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
# SEE: https://github.com/gitleaks/gitleaks/blob/master/README.md#gitleaksignore
22

33
cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:generic-api-key:37
4-
infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:10
5-
infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:11
6-
infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:12
7-
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:29
8-
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:30
9-
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:31
10-
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:32
11-
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:33
12-
infrastructure/terraform/resource_group_init/storage.bicep:generic-api-key:59
13-
infrastructure/terraform/resource_group_init/keyVault.bicep:generic-api-key:10
4+
infrastructure/terraform/spoke/resource_group_init/core.bicep:generic-api-key:10
5+
infrastructure/terraform/spoke/resource_group_init/core.bicep:generic-api-key:11
6+
infrastructure/terraform/spoke/resource_group_init/core.bicep:generic-api-key:12
7+
infrastructure/terraform/spoke/resource_group_init/main.bicep:generic-api-key:29
8+
infrastructure/terraform/spoke/resource_group_init/main.bicep:generic-api-key:30
9+
infrastructure/terraform/spoke/resource_group_init/main.bicep:generic-api-key:31
10+
infrastructure/terraform/spoke/resource_group_init/main.bicep:generic-api-key:32
11+
infrastructure/terraform/spoke/resource_group_init/main.bicep:generic-api-key:33
12+
infrastructure/terraform/spoke/resource_group_init/storage.bicep:generic-api-key:59
13+
infrastructure/terraform/spoke/resource_group_init/keyVault.bicep:generic-api-key:10
1414
infrastructure/bootstrap/core.bicep:generic-api-key:10
1515
infrastructure/bootstrap/core.bicep:generic-api-key:11
1616
infrastructure/bootstrap/core.bicep:generic-api-key:12
@@ -21,10 +21,17 @@ infrastructure/bootstrap/hub.bicep:generic-api-key:56
2121
infrastructure/bootstrap/hub.bicep:generic-api-key:57
2222
infrastructure/bootstrap/hub.bicep:generic-api-key:58
2323
infrastructure/bootstrap/hub.bicep:generic-api-key:59
24+
infrastructure/bootstrap/hub.bicep:generic-api-key:60
25+
infrastructure/bootstrap/hub.bicep:generic-api-key:61
2426
infrastructure/bootstrap/main.bicep:generic-api-key:29
2527
infrastructure/bootstrap/main.bicep:generic-api-key:30
2628
infrastructure/bootstrap/main.bicep:generic-api-key:31
2729
infrastructure/bootstrap/main.bicep:generic-api-key:32
2830
infrastructure/bootstrap/modules/storage.bicep:generic-api-key:59
2931
infrastructure/bootstrap/modules/keyVault.bicep:generic-api-key:10
3032
infrastructure/bootstrap/modules/storage.bicep:generic-api-key:59
33+
infrastructure/terraform/hub/data.tf:generic-api-key:18
34+
infrastructure/terraform/hub/data.tf:generic-api-key:22
35+
infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:11
36+
infrastructure/terraform/resource_group_init/keyVault.bicep:generic-api-key:10
37+
infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:30

‎docs/infrastructure/bootstrap.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
make hub-nonlive bootstrap

‎docs/infrastructure/infra-faq.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Below is an example of how to do it.
2222
export ARM_USE_AZUREAD=true
2323
export MSYS_NO_PATHCONV=true
2424

25-
terraform -chdir=infrastructure/terraform import -var-file ../environments/${ENV_CONFIG}/variables.tfvars module.infra[0].module.postgres_subnet.azurerm_subnet.subnet /subscriptions/xxx/resourceGroups/rg-lungrc-review-uks/providers/Microsoft.Network/virtualNetworks/vnet-review-uks-lungrc/subnets/snet-postgres
25+
terraform -chdir=infrastructure/terraform/spoke import -var-file ../environments/${ENV_CONFIG}/variables.tfvars module.infra[0].module.postgres_subnet.azurerm_subnet.subnet /subscriptions/xxx/resourceGroups/rg-lungcs-review-uks/providers/Microsoft.Network/virtualNetworks/vnet-review-uks-lungcs/subnets/snet-postgres
2626
```
2727

2828
### Error: Failed to load state
@@ -65,7 +65,7 @@ Example:
6565
Running Azure CLI Login.
6666
...
6767
Attempting Azure CLI login by using OIDC...
68-
Error: AADSTS70025: The client '***'(mi-lungrc-ado-review-temp) has no configured federated identity credentials. Trace ID: xxx Correlation ID: xxx Timestamp: xxx
68+
Error: AADSTS70025: The client '***'(mi-lungcs-ado-review-temp) has no configured federated identity credentials. Trace ID: xxx Correlation ID: xxx Timestamp: xxx
6969
7070
Error: Interactive authentication is needed. Please run:
7171
az login
@@ -122,10 +122,10 @@ The ADO group must have the "View project-level information" permission.
122122
Example:
123123
124124
```shell
125-
The pipeline is not valid. Job DeployApp: Step input azureSubscription references service connection lungrc-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz. Job DeployApp: Step input azureSubscription references service connection lungrc-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz.
125+
The pipeline is not valid. Job DeployApp: Step input azureSubscription references service connection lungcs-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz. Job DeployApp: Step input azureSubscription references service connection lungcs-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz.
126126
```
127127
128-
The Azure service connection lungrc-[environment] is missing
128+
The Azure service connection lungcs-[environment] is missing
129129
130130
## Bicep errors
131131
@@ -149,7 +149,7 @@ If you can't find the right scope, follow this process:
149149
150150
```shell
151151
 ~ Microsoft.Authorization/roleAssignments/abcd-123 [2022-04-01]
152-
    ~ properties.principalId: "xxx" => "[reference('/subscriptions/xxx/resourceGroups/rg-mi-review-uks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mi-lungrc-ado-review-uks', '2024-11-30').principalId]"
152+
    ~ properties.principalId: "xxx" => "[reference('/subscriptions/xxx/resourceGroups/rg-mi-review-uks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mi-lungcs-ado-review-uks', '2024-11-30').principalId]"
153153
```
154154
155155
- Get the subscription id
@@ -207,20 +207,20 @@ When initially creating the terraform; the pipeline will try to create a state f
207207
Example:
208208
209209
```shell
210-
Failed to get existing workspaces: containers.Client#ListBlobs: Failure sending request: StatusCode=0 -- Original Error: Get "https://salungrcpreprodtfstate.blob.core.windows.net/terraform-state?comp=list&prefix=preprod.tfstateenv%3A&restype=container": dial tcp: lookup salungrcpreprodtfstate.blob.core.windows.net on *.*.*.*:53: no such host
210+
Failed to get existing workspaces: containers.Client#ListBlobs: Failure sending request: StatusCode=0 -- Original Error: Get "https://salungcspreprodtfstate.blob.core.windows.net/terraform-state?comp=list&prefix=preprod.tfstateenv%3A&restype=container": dial tcp: lookup salungcspreprodtfstate.blob.core.windows.net on *.*.*.*:53: no such host
211211
```
212212
213213
You can check to see if the blobstorage is accessible via logging into the VDI machine and trying to do an nslookup on the blob storage account: -
214214
215215
```shell
216-
$ nslookup salungrcpreprodtfstate.blob.core.windows.net
216+
$ nslookup salungcspreprodtfstate.blob.core.windows.net
217217
Server: UnKnown
218218
Address: _._._._
219219
220220
Non-authoritative answer:
221-
Name: salungrcpreprodtfstate.privatelink.blob.core.windows.net
221+
Name: salungcspreprodtfstate.privatelink.blob.core.windows.net
222222
Address: _._._._
223-
Aliases: salungrcpreprodtfstate.blob.core.windows.net
223+
Aliases: salungcspreprodtfstate.blob.core.windows.net
224224
```
225225
226226
In the above example it was discoverd that the pipeline pool was on the wrong ADO management pool, i.e on the private-pool-dev-uks instead of the private-pool-prod-uks.

‎docs/infrastructure/new-subscription-setup.md‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,37 @@ New subscription that are created within Azure often have limitations on them, t
77
Resource providers components need to be enabled:-
88

99
- Microsoft.Authorization
10-
- Microsoft.AzureTerraform
10+
- Microsoft.AzureTerraform +
1111
- Microsoft.Billing
1212
- Microsoft.ChangeSafety
1313
- Microsoft.ClassicSubscription
1414
- Microsoft.Commerce
15-
- Microsoft.Compute
16-
- Microsoft.ComputeSchedule
15+
- Microsoft.Compute +
16+
- Microsoft.ComputeSchedule +
1717
- Microsoft.Consumption
18-
- Microsoft.ContainerService
18+
- Microsoft.ContainerService +
1919
- Microsoft.CostManagement
20-
- Microsoft.DesktopVirtualization
21-
- Microsoft.DevCenter
22-
- Microsoft.DevOpsInfrastructure
20+
- Microsoft.DesktopVirtualization +
21+
- Microsoft.DevCenter +
22+
- Microsoft.DevOpsInfrastructure +
2323
- Microsoft.Diagnostics
2424
- Microsoft.Features
2525
- Microsoft.GuestConfiguration
2626
- Microsoft.Insights
27-
- Microsoft.KeyVault
27+
- Microsoft.KeyVault +
2828
- Microsoft.ManagedIdentity
2929
- Microsoft.MarketplaceOrdering
30-
- Microsoft.Network
30+
- Microsoft.Network +
3131
- Microsoft.PolicyInsights
3232
- Microsoft.Portal
33-
- Microsoft.Quota
33+
- Microsoft.Quota +
3434
- Microsoft.ResourceGraph
3535
- Microsoft.ResourceIntelligence
3636
- Microsoft.ResourceNotifications
3737
- Microsoft.Resources
3838
- Microsoft.Security
3939
- Microsoft.SerialConsole
40-
- Microsoft.Storage
40+
- Microsoft.Storage +
4141
- Microsoft.Support
4242

4343
## Step 2

‎infrastructure/bootstrap/environments/nonlive/variables.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
AZURE_SUBSCRIPTION="Lung Cancer Risk Check - Non-live hub"
2+
# AZURE_SUBSCRIPTION="Lung Cancer Screening - Dev"
23
BOOTSTRAP=hub
34
HUB_TYPE=nonlive

‎infrastructure/bootstrap/hub.bicep‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,9 @@ param vnetAddressPrefixes array
2828
param enableSoftDelete bool
2929

3030

31-
// var keyVaultName = 'kv-lungcs-${envConfig}-inf'
32-
3331
// removed when generalised
34-
var appShortName = 'lungrc'
32+
var appShortName = 'lungcs'
33+
//var appShortName = 'lungal'
3534

3635
var devCenterSuffix = substring(uniqueString(subscription().id), 0, 3)
3736
var devCenterName = 'devc-hub-${hubType}-${regionShortName}-${devCenterSuffix}'
@@ -50,13 +49,16 @@ var storageAccountName = 'sa${appShortName}${regionShortName}state'
5049
var miADOtoAZname = 'mi-${appShortName}-${hubType}-adotoaz-${regionShortName}'
5150
var miGHtoADOname = 'mi-${appShortName}-${hubType}-ghtoado-${regionShortName}'
5251

52+
5353
// See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
5454
var roleID = {
5555
CDNContributor: 'ec156ff8-a8d1-4d15-830c-5b80698ca432'
5656
kvSecretsUser: '4633458b-17de-408a-b874-0445c86b69e6'
5757
networkContributor: '4d97b98b-1d4f-4787-a291-c67834d212e7'
5858
rbacAdmin: 'f58310d9-a9f6-439a-9e8d-f62e7b41a168'
5959
reader: 'acdd72a7-3385-48ef-bd42-f606fba81ae7'
60+
contributor: 'b24988ac-6180-42a0-ab88-20f7382dd24c'
61+
storageBlobDataContributor: 'ba92f5b4-2d11-453d-a403-e96b0029c9fe'
6062
}
6163

6264

@@ -78,6 +80,7 @@ module virtualNetwork 'modules/virtualNetwork.bicep' = {
7880
module managedDevopsPool 'modules/managedDevopsPool.bicep' = {
7981
scope: bootstrapRG
8082
params: {
83+
//adoOrg: 'nhse-pps-1'
8184
adoOrg: 'nhse-dtos'
8285
agentProfileMaxAgentLifetime: '00.04:00:00'
8386
devCenterName: devCenterName
@@ -179,6 +182,26 @@ resource CDNContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-
179182
}
180183
}
181184

185+
@description('Let the managed identity deploy terraform on the subscription')
186+
resource TerraformContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
187+
name: guid(subscription().subscriptionId, hubType, 'TerraformContributor')
188+
properties: {
189+
roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.contributor)
190+
principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID
191+
description: '${miADOtoAZname} Terraform Contributor access to subscription'
192+
}
193+
}
194+
195+
@description('Let the managed identity strore blobs in storage account')
196+
resource StorageAccountBlobContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
197+
name: guid(subscription().subscriptionId, hubType, 'StorageAccountBlobContributorAssignment')
198+
properties: {
199+
roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.storageBlobDataContributor)
200+
principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID
201+
description: '${miADOtoAZname} Storage Account Blob Contributor access to subscription'
202+
}
203+
}
204+
182205
@description('Create the managed identity assumed by Github actions to trigger Azure devops pipelines')
183206
module managedIdentiyGHtoADO 'modules/managedIdentity.bicep' = {
184207
scope: managedIdentityRG
@@ -193,7 +216,6 @@ module managedIdentiyGHtoADO 'modules/managedIdentity.bicep' = {
193216
}
194217
}
195218

196-
197219
@description('Let the GHtoADO managed identity access a subscription')
198220
resource readerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
199221
name: guid(subscription().subscriptionId, hubType, 'reader')

0 commit comments

Comments
 (0)