cdc-mpg-mkf.tf

# Infrastructure for change data capture from Yandex Managed Service for PostgreSQL and delivery to Yandex Managed Service for Apache Kafka® via Data Transfer
#
# RU: https://yandex.cloud/ru/docs/data-transfer/tutorials/cdc-mpg
# EN: https://yandex.cloud/en/docs/data-transfer/tutorials/cdc-mpg
#
# Specify the following settings:

locals {
  # The following settings are to be specified by the user. Change them as you wish.
  
  # Settings for the Network infrastructure
  network_name = "" # Name of the network for the Managed Service for PostgreSQL and Managed Service for Apache Kafka® clusters
  subnet_name  = "" # Name of the subnet for the Managed Service for PostgreSQL and Managed Service for Apache Kafka® clusters
  pg_sg_name   = "" # Name of the security group for the Managed Service for PostgreSQL cluster
  kf_sg_name   = "" # Name of the security group for the Managed Service for Apache Kafka® cluster

  # Settings for the Managed Service for PostgreSQL cluster
  pg_cluster_version = "" # Version of the Managed Service for PostgreSQL cluster. For the list of supported versions, see https://yandex.cloud/en/docs/managed-postgresql/.
  pg_cluster_name    = "" # Name of the Managed Service for PostgreSQL cluster
  pg_password        = "" # Password of the Managed Service for PostgreSQL user
  
  # Settings for the Managed Service for Apache Kafka®
  kf_cluster_version = "" # Version of the Managed Service for Apache Kafka® cluster. For the list of supported versions, see https://yandex.cloud/en/docs/managed-kafka/.
  kf_cluster_name    = "" # Name of the Managed Service for Apache Kafka® cluster
  kf_password        = "" # Password of the Managed Service for Apache Kafka® user

  # Settings for the Yandex Data Transfer
  source_endpoint_name = "" # Name of the source endpoint for the Managed Service for PostgreSQL
  target_endpoint_name = "" # Name of the target endpoint for the Managed Service for Apache Kafka®
  transfer_name        = "" # Name of the transfer from the Managed Service for PostgreSQL to the Managed Service for Apache Kafka®

  # Specify this setting ONLY AFTER the clusters are created. Then run "terraform apply" command again.
  transfer_enabled = 0  # Set to 1 to create Data Transfer endpoints and the transfer

  # The following settings are predefined. Change them only if necessary.

  # Settings for the Managed Service for PostgreSQL cluster
  pg_db_name  = "db1"     # Name of the Managed Service for PostgreSQL database
  pg_username = "pg-user" # Name of the Managed Service for PostgreSQL user

  # Settings for the Managed Service for Apache Kafka® cluster
  kf_topic    = "cdc.public.measurements" # Name of the Managed Service for Apache Kafka® topic
  kf_username = "kafka-user"              # Name of the Managed Service for Apache Kafka® user

  # Settings for the Network infrastructure
  zone_a_v4_cidr_blocks = "10.1.0.0/16" # CIDR block for the subnet in the ru-central1-a availability zone
}

# Network infrastructure

resource "yandex_vpc_network" "network" {
  description = "Network for the Managed Service for PostgreSQL and Managed Service for Apache Kafka® clusters"
  name        = local.network_name
}

resource "yandex_vpc_subnet" "subnet-a" {
  description    = "Subnet in the ru-central1-a availability zone"
  name           = local.subnet_name
  zone           = "ru-central1-a"
  network_id     = yandex_vpc_network.network.id
  v4_cidr_blocks = [local.zone_a_v4_cidr_blocks]
}

resource "yandex_vpc_security_group" "pg_sg" {
  description = "Security group for the Managed Service for PostgreSQL"
  network_id  = yandex_vpc_network.network.id
  name        = local.pg_sg_name

  ingress {
    description    = "Allow incoming traffic from the Internet"
    protocol       = "TCP"
    port           = 6432
    v4_cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    description    = "Allow outgoing traffic to the Internet"
    protocol       = "ANY"
    from_port      = 0
    to_port        = 65535
    v4_cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "yandex_vpc_security_group" "kf_sg" {
  description = "Security group for the Managed Service for Apache Kafka®"
  network_id  = yandex_vpc_network.network.id
  name        = local.kf_sg_name

  ingress {
    description    = "Allow incoming traffic from the port 9091"
    protocol       = "TCP"
    port           = 9091
    v4_cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    description    = "Allow outgoing traffic to the Internet"
    protocol       = "ANY"
    from_port      = 0
    to_port        = 65535
    v4_cidr_blocks = ["0.0.0.0/0"]
  }
}

# Infrastructure for the Managed Service for PostgreSQL cluster

resource "yandex_mdb_postgresql_cluster" "pg_cluster" {
  description        = "Managed Service for PostgreSQL cluster"
  name               = local.pg_cluster_name
  environment        = "PRODUCTION"
  network_id         = yandex_vpc_network.network.id
  security_group_ids = [yandex_vpc_security_group.pg_sg.id]

  config {
    version = local.pg_cluster_version
    
    resources {
      resource_preset_id = "s2.micro" # 2 vCPU, 8 GB RAM
      disk_type_id       = "network-ssd"
      disk_size          = "20" # GB
    }
  }

  host {
    zone             = "ru-central1-a"
    subnet_id        = yandex_vpc_subnet.subnet-a.id
    assign_public_ip = true # Required for connection from the Internet
  }
}

# User of the Managed Service for PostgreSQL cluster
resource "yandex_mdb_postgresql_user" "pg_user" {
  cluster_id = yandex_mdb_postgresql_cluster.pg_cluster.id
  name       = local.pg_username
  password   = local.pg_password
  grants     = ["mdb_replication"] # Grants permissions required for logical replication
}

# Database of the Managed Service for PostgreSQL cluster
resource "yandex_mdb_postgresql_database" "pg_db" {
  cluster_id = yandex_mdb_postgresql_cluster.pg_cluster.id
  name       = local.pg_db_name
  owner      = yandex_mdb_postgresql_user.pg_user.name
  depends_on = [
    yandex_mdb_postgresql_user.pg_user
  ]
}

# Infrastructure for the Managed Service for Apache Kafka® cluster

resource "yandex_mdb_kafka_cluster" "kf_cluster" {
  description        = "Managed Service for Apache Kafka® cluster"
  environment        = "PRODUCTION"
  name               = local.kf_cluster_name
  network_id         = yandex_vpc_network.network.id
  security_group_ids = [yandex_vpc_security_group.kf_sg.id]

  config {
    assign_public_ip = true # Required for connection from the Internet
    brokers_count    = 1
    version          = local.kf_cluster_version
    
    kafka {
      resources {
        disk_size          = 10 # GB
        disk_type_id       = "network-ssd"
        resource_preset_id = "s2.micro" # 2 vCPU, 8 GB RAM
      }

      kafka_config {}
    }

    zones = ["ru-central1-a"]
  }

  depends_on = [
    yandex_vpc_subnet.subnet-a
  ]
}

# Topic of the Managed Service for Apache Kafka® cluster
resource "yandex_mdb_kafka_topic" "kf_topic" {
  cluster_id         = yandex_mdb_kafka_cluster.kf_cluster.id
  name               = local.kf_topic
  partitions         = 1
  replication_factor = 1
}

# User of the Managed Service for Apache Kafka® cluster
resource "yandex_mdb_kafka_user" "kf_user" {
  cluster_id = yandex_mdb_kafka_cluster.kf_cluster.id
  name       = local.kf_username
  password   = local.kf_password
  
  permission {
    topic_name = yandex_mdb_kafka_topic.kf_topic.name
    role       = "ACCESS_ROLE_CONSUMER"
  }
  
  permission {
    topic_name = yandex_mdb_kafka_topic.kf_topic.name
    role       = "ACCESS_ROLE_PRODUCER"
  }
}

# Data Transfer infrastructure

resource "yandex_datatransfer_endpoint" "pg_source" {
  description = "Source endpoint for the Managed Service for PostgreSQL cluster"
  count       = local.transfer_enabled
  name        = local.source_endpoint_name
  
  settings {
    postgres_source {
      connection {
        mdb_cluster_id = yandex_mdb_postgresql_cluster.pg_cluster.id
      }

      database       = yandex_mdb_postgresql_database.pg_db.name
      user           = yandex_mdb_postgresql_user.pg_user.name
      include_tables = ["public.measurements"]
      
      password {
        raw = local.pg_password
      }
    }
  }
}

resource "yandex_datatransfer_endpoint" "kf_target" {
  description = "Target endpoint for the Managed Service for Apache Kafka® cluster"
  name        = local.target_endpoint_name
  count       = local.transfer_enabled

  settings {
    kafka_target {
      connection {
        cluster_id = yandex_mdb_kafka_cluster.kf_cluster.id
      }

      auth {
        sasl {
          user = yandex_mdb_kafka_user.kf_user.name

          password {
            raw = local.kf_password
          }
        }
      }

      topic_settings {
        topic {
          topic_name = local.kf_topic
        }
      }

      serializer {
        serializer_auto {}
      }
    }
  }
}

resource "yandex_datatransfer_transfer" "mpg-mkf-transfer" {
  description = "Transfer from the Managed Service for PostgreSQL to the Managed Service for Apache Kafka®"
  count       = local.transfer_enabled
  name        = local.transfer_name
  source_id   = yandex_datatransfer_endpoint.pg_source[0].id
  target_id   = yandex_datatransfer_endpoint.kf_target[0].id
  type        = "INCREMENT_ONLY" # Data replication from the source Managed Service for PostgreSQL cluster to the target Managed Service for Apache Kafka® topic 
}