Validate DB connection in internal healthcheck

- Try to initialize a new connection
  - ActiveRecord persists connections after credential change
  - New connections fails after credential change
- This means that while old connections exist we will identify that the
  credentials have changed and trigger a redeployment of containers
  - This avoids any disturbance to user traffic and avoids any `500`
    error messages
- Additionally increase the frequency of ELB healthchecks to limit
  inpact to users in any potential future situations when a container
  becomes unhealthy

Author: TheOneFromNorway

Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /rails
 
 # Install base packages
 RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl libjemalloc2 libvips libicu-dev postgresql-client && \
+    apt-get install --no-install-recommends -y curl libjemalloc2 libvips libicu-dev postgresql-client jq && \
     rm -rf /var/lib/apt/lists /var/cache/apt/archives
 
 # Set production environment

bin/internal_healthcheck

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+export PGPASSWORD="$(echo $DB_CREDENTIALS | jq -r .password)"
+psql -h "$DB_HOST" -d "$DB_NAME" -U "$(echo $DB_CREDENTIALS | jq -r .username)" -c "select 1" || {
+  echo "DB connection could not be established: Internal healthcheck failed."; exit 1;
+}
+curl -f "$1" || {
+  echo "DB connection could not be established, but $1 did not return a 200 response."; exit 2;
+}

terraform/app/ecs.tf

@@ -31,7 +31,7 @@ module "web_service" {
     task_role_arn        = aws_iam_role.ecs_task_role.arn
     log_group_name       = aws_cloudwatch_log_group.ecs_log_group.name
     region               = var.region
-    health_check_command = ["CMD-SHELL", "curl -f http://localhost:4000/health/database || exit 1"]
+    health_check_command = ["CMD-SHELL", "./bin/internal_healthcheck http://localhost:4000/health/database"]
   }
   network_params = {
     subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]
@@ -70,7 +70,7 @@ module "good_job_service" {
     task_role_arn        = aws_iam_role.ecs_task_role.arn
     log_group_name       = aws_cloudwatch_log_group.ecs_log_group.name
     region               = var.region
-    health_check_command = ["CMD-SHELL", "curl -f http://localhost:4000/status/connected || exit 1"]
+    health_check_command = ["CMD-SHELL", "./bin/internal_healthcheck http://localhost:4000/status/connected"]
   }
   network_params = {
     subnets = [aws_subnet.private_subnet_a.id, aws_subnet.private_subnet_b.id]

terraform/app/loadbalancer.tf

@@ -82,8 +82,8 @@ resource "aws_lb_target_group" "blue" {
     protocol            = "HTTP"
     port                = "traffic-port"
     matcher             = "200"
-    interval            = 10
-    timeout             = 5
+    interval            = 5
+    timeout             = 4
     healthy_threshold   = 2
     unhealthy_threshold = 2
   }
@@ -100,8 +100,8 @@ resource "aws_lb_target_group" "green" {
     protocol            = "HTTP"
     port                = "traffic-port"
     matcher             = "200"
-    interval            = 10
-    timeout             = 5
+    interval            = 5
+    timeout             = 4
     healthy_threshold   = 2
     unhealthy_threshold = 2
   }