Feat/enable autoresume for byoc (#2470)

adds a route from the edge proxies to call the API to resume sandboxes when autoresume is on
implements the oidc auth for the API so we can use the JWTs from edge <> our control plane
new scope: sandboxes:lifecycle

Author: matthewlouisbrockman

.env.gcp.template

@@ -123,6 +123,9 @@ DEFAULT_PERSISTENT_VOLUME_TYPE=
 # Persistent volume types (default: {})
 PERSISTENT_VOLUME_TYPES=
 
+# Client proxy OIDC issuer for edge gRPC autoresume auth.
+CLIENT_PROXY_OIDC_ISSUER_URL=
+
 # Sandbox firewall: comma-separated CIDRs to allow through the private-range deny list
 # ALLOW_SANDBOX_INTERNAL_CIDRS=
 

.github/actions/start-services/action.yml

@@ -104,7 +104,7 @@ runs:
         SHARED_CHUNK_CACHE_PATH: "./.e2b-chunk-cache"
         REDIS_URL: "localhost:6379"
         # Client-proxy config
-        API_GRPC_ADDRESS: "localhost:5009"
+        API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
         DEFAULT_PERSISTENT_VOLUME_TYPE: "test-volume-type"
         SANDBOX_STORAGE_BACKEND: "redis"
       run: |

.github/workflows/integration_tests.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Run Integration Tests
         env:
           TESTS_API_SERVER_URL: "http://localhost:3000"
-          TESTS_API_GRPC_ADDRESS: "localhost:5009"
+          TESTS_API_INTERNAL_GRPC_ADDRESS: "localhost:5009"
           TESTS_ORCHESTRATOR_HOST: "localhost:5008"
           TESTS_ENVD_PROXY: "http://localhost:3002"
           TESTS_CLIENT_PROXY: "http://localhost:3002"

iac/modules/job-api/jobs/api.hcl

@@ -19,10 +19,12 @@ job "api" {
         static = "${port_number}"
       }
 
-      port "grpc" {
-        static = "${api_grpc_port}"
+      port "api_internal_grpc" {
+        static = "${api_internal_grpc_port}"
       }
 
+      port "grpc_api" {}
+
       %{ if prevent_colocation }
       port "scheduling-block" {
         // This port is used to block scheduling of jobs with the same block on the same node.
@@ -61,16 +63,39 @@ job "api" {
     }
 
     service {
-      name = "api-grpc"
-      port = "grpc"
+      name = "api-internal-grpc"
+      port = "api_internal_grpc"
       task = "start"
 
       check {
         type     = "tcp"
-        name     = "grpc"
+        name     = "api-internal-grpc"
+        interval = "3s"
+        timeout  = "3s"
+        port     = "api_internal_grpc"
+      }
+    }
+
+    service {
+      name = "grpc-api"
+      port = "grpc_api"
+      task = "start"
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.grpc-api.rule=HostRegexp(`grpc-api.{domain:.+}`)",
+        "traefik.http.routers.grpc-api.ruleSyntax=v2",
+        "traefik.http.routers.grpc-api.priority=500",
+        "traefik.http.routers.grpc-api.service=grpc-api",
+        "traefik.http.services.grpc-api.loadbalancer.server.scheme=h2c"
+      ]
+
+      check {
+        type     = "tcp"
+        name     = "grpc-api"
         interval = "3s"
         timeout  = "3s"
-        port     = "grpc"
+        port     = "grpc_api"
       }
     }
 
@@ -114,7 +139,8 @@ job "api" {
         NODE_ID                        = "$${node.unique.id}"
         NOMAD_TOKEN                    = "${nomad_acl_token}"
         ORCHESTRATOR_PORT              = "${orchestrator_port}"
-        API_GRPC_PORT                  = "${api_grpc_port}"
+        API_INTERNAL_GRPC_PORT         = "${api_internal_grpc_port}"
+        API_EDGE_GRPC_PORT             = "$${NOMAD_PORT_grpc_api}"
         ADMIN_TOKEN                    = "${admin_token}"
         SANDBOX_ACCESS_TOKEN_HASH_SEED = "${sandbox_access_token_hash_seed}"
 
@@ -164,7 +190,7 @@ job "api" {
       config {
         network_mode = "host"
         image        = "${api_docker_image}"
-        ports        = ["${port_name}"]
+        ports        = ["${port_name}", "grpc_api"]
         args         = [
           "--port", "${port_number}",
         ]

iac/modules/job-api/main.tf

@@ -23,7 +23,7 @@ resource "nomad_job" "api" {
     logs_collector_address                  = var.logs_collector_address
     port_name                               = var.port_name
     port_number                             = var.port_number
-    api_grpc_port                           = var.api_grpc_port
+    api_internal_grpc_port                  = var.api_internal_grpc_port
     api_docker_image                        = var.api_docker_image
     postgres_connection_string              = var.postgres_connection_string
     postgres_read_replica_connection_string = var.postgres_read_replica_connection_string

iac/modules/job-api/variables.tf

@@ -31,7 +31,7 @@ variable "port_number" {
   type = number
 }
 
-variable "api_grpc_port" {
+variable "api_internal_grpc_port" {
   type    = number
   default = 5009
 }

iac/modules/job-client-proxy/jobs/client-proxy.hcl

@@ -113,9 +113,19 @@ job "client-proxy" {
         REDIS_TLS_CA_BASE64      = "${redis_tls_ca_base64}"
         REDIS_URL                = "${redis_url}"
 
-        %{ if api_grpc_address != "" }
-        # used only when client-proxy is deployed directly in the cluster next to the API
-        API_GRPC_ADDRESS = "${api_grpc_address}"
+        %{ if api_internal_grpc_address != "" }
+        # used by in-cluster client-proxy to call API ResumeSandbox over gRPC
+        API_INTERNAL_GRPC_ADDRESS = "${api_internal_grpc_address}"
+        %{ endif }
+
+        %{ if api_edge_grpc_address != "" }
+        # used by external client-proxy to call edge API ResumeSandbox over gRPC
+        API_EDGE_GRPC_ADDRESS             = "${api_edge_grpc_address}"
+        %{ if api_edge_grpc_oauth_client_id != "" }
+        API_EDGE_GRPC_OAUTH_CLIENT_ID     = "${api_edge_grpc_oauth_client_id}"
+        API_EDGE_GRPC_OAUTH_CLIENT_SECRET = "${api_edge_grpc_oauth_client_secret}"
+        API_EDGE_GRPC_OAUTH_TOKEN_URL     = "${api_edge_grpc_oauth_token_url}"
+        %{ endif }
         %{ endif }
 
         %{ if launch_darkly_api_key != "" }

iac/modules/job-client-proxy/main.tf

@@ -1,3 +1,8 @@
+locals {
+  api_internal_grpc_address = trimspace(var.api_internal_grpc_address)
+  api_edge_grpc_address     = trimspace(var.api_edge_grpc_address)
+}
+
 resource "nomad_job" "client_proxy" {
   jobspec = templatefile("${path.module}/jobs/client-proxy.hcl", {
     update_stanza       = var.update_stanza
@@ -17,8 +22,13 @@ resource "nomad_job" "client_proxy" {
     redis_tls_ca_base64 = var.redis_tls_ca_base64
     redis_pool_size     = var.redis_pool_size
 
-    image            = var.image
-    api_grpc_address = trimspace(var.api_grpc_address)
+    image                     = var.image
+    api_internal_grpc_address = local.api_internal_grpc_address
+    api_edge_grpc_address     = local.api_edge_grpc_address
+
+    api_edge_grpc_oauth_client_id     = trimspace(var.api_edge_grpc_oauth_client_id)
+    api_edge_grpc_oauth_client_secret = trimspace(var.api_edge_grpc_oauth_client_secret)
+    api_edge_grpc_oauth_token_url     = trimspace(var.api_edge_grpc_oauth_token_url)
 
     otel_collector_grpc_endpoint = var.otel_collector_grpc_endpoint
     logs_collector_address       = var.logs_collector_address

iac/modules/job-client-proxy/variables.tf

@@ -63,11 +63,34 @@ variable "image" {
   type = string
 }
 
-variable "api_grpc_address" {
+variable "api_internal_grpc_address" {
   type    = string
   default = ""
 }
 
+variable "api_edge_grpc_address" {
+  type    = string
+  default = ""
+}
+
+variable "api_edge_grpc_oauth_client_id" {
+  type      = string
+  default   = ""
+  sensitive = true
+}
+
+variable "api_edge_grpc_oauth_client_secret" {
+  type      = string
+  default   = ""
+  sensitive = true
+}
+
+variable "api_edge_grpc_oauth_token_url" {
+  type      = string
+  default   = ""
+  sensitive = true
+}
+
 variable "otel_collector_grpc_endpoint" {
   type = string
 }