Terraform — Kubernetes кластер на AWS EC2

Artem26.02.201926.02.2019Terraform

Пример конфигурации Terraform, который создает Kubernetes кластер (Bare Metal) на AWS EC2. Создает Ingress с NodePort. И в конце выполнения вывод публичные IP адреса Ingress нод.

Данный темплейт создает следующие EC2 инстансы:

1 manager
2 workers
2 ingresses

variables.tf

variable "REGION" {
  default = "us-east-1"
}

variable "PROJECT_NAME" {
  default = "artem_k8s"
}

variable "SSH_USER" { 
  default = "ubuntu"
}

variable "SSH_KEY_NAME" { 
  default = "artem.gatchenko"
}

variable "SSH_KEY_PATH" { 
  default = "/home/artem/.ssh/id_rsa"
}

variable "VPC_SUBNET" { 
  default = "192.168.1.0/24"
}

variable "INSTANCE_TYPE" {
  default = "t2.micro"
}

variable "WORKER_NUMBER" {
  default = "2"
}

main.tf

provider "aws" {
  region = "us-east-1"
}

provider "aws" {
  alias = "vpc"
  region = "${var.REGION}"
}

vpc.tf

// CREATE VPC
resource "aws_vpc" "vpc" {
  cidr_block = "${var.VPC_SUBNET}"
  enable_dns_hostnames = "true"
  enable_dns_support = "true"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE GATEWAY
resource "aws_internet_gateway" "vpc" {
  vpc_id = "${aws_vpc.vpc.id}"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE ROUTE TABLE
resource "aws_route_table" "vpc" {
  vpc_id = "${aws_vpc.vpc.id}"
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.vpc.id}"
  }

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

// CREATE SUBNET
resource "aws_subnet" "vpc" {
  vpc_id     = "${aws_vpc.vpc.id}"
  cidr_block = "${var.VPC_SUBNET}"

  map_public_ip_on_launch = "true"

  tags {
    Name = "${var.PROJECT_NAME}"
  }
}

resource "aws_route_table_association" "vpc" {
  subnet_id      = "${aws_subnet.vpc.id}"
  route_table_id = "${aws_route_table.vpc.id}"
}

// CREATE SECURITY GROUP
resource "aws_security_group" "vpc" {
  vpc_id      = "${aws_vpc.vpc.id}"

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input SSH"
  }

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input HTTP"
  }

  ingress {
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow input HTTPS"
  }

  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    self = true
    description = "Allow triffic between instances in SG"
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
    description = "Allow all ouput traffic"
  }

  tags {
    Name = "${var.PROJECT_NAME}"
    Description = "${var.PROJECT_NAME}"
  }

}

instance_master.tf

// CREATE INSTANCE FOR KUBERNETES MASTER
resource "aws_instance" "master" {
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "ingress-nginx.yml"
    destination = "/tmp/ingress-nginx.yml"
  }

  provisioner "remote-exec" {
    inline = [
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo kubeadm init --apiserver-advertise-address=${aws_instance.master.private_ip} --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors SystemVerification --ignore-preflight-errors NumCPU",
      "mkdir -p $HOME/.kube",
      "sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config",
      "sudo chown $(id -u):$(id -g) $HOME/.kube/config",
      "echo 'source <(kubectl completion bash)' >> $HOME/.bashrc",
      "kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"
    ]
  }

  provisioner "local-exec" {
    command = "ssh -o StrictHostKeyChecking=no ubuntu@${aws_instance.master.public_ip} -i ${var.SSH_KEY_PATH} 'kubeadm token create --print-join-command' > token"
  }

  provisioner "local-exec" {
    command = "if [ -e ip_ingresses ]; then rm -rf ip_ingresses; fi"
  }

  tags {
    Name = "${var.PROJECT_NAME}_master"
  }

}

instance_workers.tf

// CREATE INSTANCE FOR KUBERNETES WORKERS
resource "aws_instance" "worker" {
  depends_on = ["aws_instance.master"]
  count = "${var.WORKER_NUMBER}"
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "token"
    destination = "/tmp/token"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo `cat /tmp/token` --ignore-preflight-errors=SystemVerification"
    ]
  }

  tags {
    Name = "${var.PROJECT_NAME}_worker"
  }
}

instance_ingresses.tf

// CREATE INSTANCE FOR KUBERNETES WORKERS
resource "aws_instance" "ingress" {
  depends_on = ["aws_instance.master"]
  # depends_on = [
  #   "aws_instance.master",
  #   "aws_instance.ingress",
  # ]
  count = "2"
  ami = "ami-0ac019f4fcb7cb7e6"
  instance_type = "${var.INSTANCE_TYPE}"
  key_name      = "${var.SSH_KEY_NAME}"
  vpc_security_group_ids = ["${aws_security_group.vpc.id}"]
  subnet_id = "${aws_subnet.vpc.id}"
  associate_public_ip_address = true
  source_dest_check = false

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "local-exec" {
    command = "echo ${self.private_ip} >> ip_ingresses"
  }

  provisioner "file" {
    source      = "k8s.sh"
    destination = "/tmp/k8s.sh"
  }

  provisioner "file" {
    source      = "token"
    destination = "/tmp/token"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "sudo chmod +x /tmp/k8s.sh",
      "sudo /tmp/k8s.sh",
      "sudo `cat /tmp/token` --ignore-preflight-errors=SystemVerification"
    ]
  }

  tags {
    Name = "${var.PROJECT_NAME}_ingress"
  }
}

ingress.tf

resource "null_resource" "ingress" {

  triggers = {
    instance_id = "${aws_instance.master.id}"
  }

  connection {
    type     = "ssh"
    user     = "${var.SSH_USER}"
    host = "${aws_instance.master.public_ip}"
    agent = false
    private_key = "${file("${var.SSH_KEY_PATH}")}"
  }

  provisioner "file" {
    source      = "ip_ingresses"
    destination = "/tmp/ip_ingresses"
  }

  provisioner "remote-exec" {
    inline = [
      "sleep 15",
      "kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml",
      "IP_INGRESS_1=`sed -n 1p /tmp/ip_ingresses`",
      "IP_INGRESS_2=`sed -n 2p /tmp/ip_ingresses`",
      "sed -i \"s/IP_INGRESS_1/$IP_INGRESS_1/g\" /tmp/ingress-nginx.yml",
      "sed -i \"s/IP_INGRESS_2/$IP_INGRESS_2/g\" /tmp/ingress-nginx.yml",
      "INGRESS_NAME_1=ip-$(cat /tmp/ip_ingresses | sed -n 1p | sed 's/\\./-/g')",
      "INGRESS_NAME_2=ip-$(cat /tmp/ip_ingresses | sed -n 2p | sed 's/\\./-/g')",
      "kubectl label node $INGRESS_NAME_1 node-role.kubernetes.io/ingress=ingress",
      "kubectl label node $INGRESS_NAME_2 node-role.kubernetes.io/ingress=ingress",
      "kubectl apply -f /tmp/ingress-nginx.yml"
    ]
  }
  depends_on = [
    "aws_instance.master",
    "aws_instance.ingress",
  ]
}

ingress-nginx.yml

apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
  externalIPs:
    - IP_INGRESS_1
    - IP_INGRESS_2
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

k8s.sh

#!/bin/bash

# THIS SCRIPT FOR INSTALL K8S. SCRIPT FOR UBUNTU

# INSTALL DOCKER
apt update
apt install -y apt-transport-https ca-certificates curl software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -

add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

apt update
apt install -y docker-ce

systemctl enable docker
systemctl start docker

# INSTALL KUBERNETES

curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
 
apt update
apt install -y kubelet kubeadm kubectl

systemctl enable kubelet
systemctl start kubelet

output.tf

output "aws-k8s-ingresses-ip" {
  value = "${aws_instance.ingress.*.public_ip}"
}

Скачать все одним архивом можно тут.

Как запустить Terraform темплейт:

terraform init
terraform plan
terraform apply