Terraform – Kubernetes cluster on AWS EC2

Artem26.02.201926.02.2019Terraform

Terraform configuration example that creates Kubernetes cluster (Bare Metal) on AWS EC2. Creates Ingress with NodePort. IP addresses Ingress nodes.

This template creates the following EC2 instances:

1 manager
2 workers
2 ingresses

variables.tf

variable "REGION" {
  default = "us-east-1"
}

variable "PROJECT_NAME" {
  default = "artem_k8s"
}

variable "SSH_USER" { 
  default = "ubuntu"
}

variable "SSH_KEY_NAME" { 
  default = "artem.gatchenko"
}

variable "SSH_KEY_PATH" { 
  default = "/home/artem/.ssh/id_rsa"
}

variable "VPC_SUBNET" { 
  default = "192.168.1.0/24"
}

variable "INSTANCE_TYPE" {
  default = "t2.micro"
}

variable "WORKER_NUMBER" {
  default = "2"
}

main.tf

provider "aws" {
  region = "us-east-1"
}

provider "aws" {
  alias = "vpc"
  region = "${var.REGION}"
}

vpc.tf

// CREATE VPC
resource "aws_vpc" "vpc" {
  cidr_block = "${var.VPC_SUBNET}"
  enable_dns_hostnames = "true"
  enable_dns_support = "true"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE GATEWAY
resource "aws_internet_gateway" "vpc" {
  vpc_id = "${aws_vpc.vpc.id}"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE ROUTE TABLE
resource "aws_route_table" "vpc" {
  vpc_id = "${aws_vpc.vpc.id}"
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.vpc.id}"
  }

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE SUBNET
resource "aws_subnet" "vpc" {
  vpc_id     = "${aws_vpc.vpc.id}"
  cidr_block = "${var.VPC_SUBNET}"

  map_public_ip_on_launch = "true"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

resource "aws_route_table_association" "vpc" {
  subnet_id      = "${aws_subnet.vpc.id}"
  route_table_id = "${aws_route_table.vpc.id}"
}

// CREATE SECURITY GROUP
resource "aws_security_group" "vpc" {
  vpc_id      = "${aws_vpc.vpc.id}"

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input SSH"
  }

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input HTTP"
  }

  ingress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input HTTPS"
  }

  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    self = true
    description = "Allow triffic between instances in SG"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow all ouput traffic"
  }

  tags {
    Name = "${var.PROJECT_NAME}"
    Description = "${var.PROJECT_NAME}"
  }

}

instance_master.tf

// CREATE INSTANCE FOR KUBERNETES MASTER
resource "aws_instance" "master" {
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "ingress-nginx.yml"
    destination = "/tmp/ingress-nginx.yml"
  }

  provisioner "remote-exec" {
    inline = [
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo kubeadm init --apiserver-advertise-address=${aws_instance.master.private_ip} --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors SystemVerification --ignore-preflight-errors NumCPU",
      "mkdir -p $HOME/.kube",
      "sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config",
      "sudo chown $(id -u):$(id -g) $HOME/.kube/config",
      "echo 'source &lt;(kubectl completion bash)' &gt;&gt; $HOME/.bashrc",
      "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"
    ]
  }

  provisioner "local-exec" {
    command = "ssh -o StrictHostKeyChecking=no ubuntu@${aws_instance.master.public_ip} -i ${var.SSH_KEY_PATH} 'kubeadm token create --print-join-command' &gt; token"
  }

  provisioner "local-exec" {
    command = "if [ -e ip_ingresses ]; then rm -rf ip_ingresses; fi"
  }

  tags {
    Name = "${var.PROJECT_NAME}_master"
  }

}

instance_workers.tf

// CREATE INSTANCE FOR KUBERNETES WORKERS
resource "aws_instance" "worker" {
  depends_on = ["aws_instance.master"]
  count = "${var.WORKER_NUMBER}"
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "token"
    destination = "/tmp/token"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo `cat /tmp/token` --ignore-preflight-errors=SystemVerification"
    ]
  }

  tags {
    Name = "${var.PROJECT_NAME}_worker"
  }
}

instance_ingresses.tf

// CREATE INSTANCE FOR KUBERNETES WORKERS
resource "aws_instance" "ingress" {
  depends_on = ["aws_instance.master"]
  # depends_on = [
  #   "aws_instance.master",
  #   "aws_instance.ingress",
  # ]
  count = "2"
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "local-exec" {
    command = "echo ${self.private_ip} &gt;&gt; ip_ingresses"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "token"
    destination = "/tmp/token"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo `cat /tmp/token` --ignore-preflight-errors=SystemVerification"
    ]
  }

  tags {
    Name = "${var.PROJECT_NAME}_ingress"
  }
}

ingress.tf

resource "null_resource" "ingress" {

  triggers = {
    instance_id = "${aws_instance.master.id}"
  }

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    host = "${aws_instance.master.public_ip}"
    agent = false
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "ip_ingresses"
    destination = "/tmp/ip_ingresses"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml",
      "IP_INGRESS_1=`sed -n 1p /tmp/ip_ingresses`",
      "IP_INGRESS_2=`sed -n 2p /tmp/ip_ingresses`",
      "sed -i \"s/IP_INGRESS_1/$IP_INGRESS_1/g\" /tmp/ingress-nginx.yml",
      "sed -i \"s/IP_INGRESS_2/$IP_INGRESS_2/g\" /tmp/ingress-nginx.yml",
      "INGRESS_NAME_1=ip-$(cat /tmp/ip_ingresses | sed -n 1p | sed 's/\\./-/g')",
      "INGRESS_NAME_2=ip-$(cat /tmp/ip_ingresses | sed -n 2p | sed 's/\\./-/g')",
      "kubectl label node $INGRESS_NAME_1 node-role.kubernetes.io/ingress=ingress",
      "kubectl label node $INGRESS_NAME_2 node-role.kubernetes.io/ingress=ingress",
      "kubectl apply -f /tmp/ingress-nginx.yml"
    ]
  }
  depends_on = [
    "aws_instance.master",
    "aws_instance.ingress",
  ]
}

ingress-nginx.yml

apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
  externalIPs:
    - IP_INGRESS_1
    - IP_INGRESS_2
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

k8s.sh

#!/bin/bash

# THIS SCRIPT FOR INSTALL K8S. SCRIPT FOR UBUNTU

# INSTALL DOCKER
apt update
apt install -y apt-transport-https ca-certificates curl software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -

add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

apt update
apt install -y docker-ce

systemctl enable docker
systemctl start docker

# INSTALL KUBERNETES

curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 
cat &lt;&lt;EOF &gt;/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
 
apt update
apt install -y kubelet kubeadm kubectl

systemctl enable kubelet
systemctl start kubelet

output.tf

output "aws-k8s-ingresses-ip" {
  value = "${aws_instance.ingress.*.public_ip}"
}

Download all one archive can be here.

How to run the Terraform template:

terraform init
terraform plan
terraform apply