DEV Community: MohammedBanabila

Stacked-Topology

MohammedBanabila — Fri, 18 Apr 2025 01:53:24 +0000

Setup Stacked Topology High availability clusters using kubeadm

steps: to High availability clusters

deploy infrastructure at aws for self hosted kubernetes cluster
vpc1=77.132.0.0/16 and integrate with internet gateway
3 public subnet for each Azs cidrblock will be at range 77.132.100.0/24, 77.132.101.0/24,77.132.102.0/24
3 private subnet for each Azs cidrblock will be at range 77.132.103.0/24 , 77.132.104.0/24 , 77.132.105.0/24 5 . associate routetable with public subnets
associate per routetable that have route to outside by Nat gateways with each private subnet
deploy network load balancer which have target registered with public subnets
deploy 3 security groups for load balancer , control plane nodes , worker nodes
use session manager for access the all nodes by adding a iam role and have permission .
deploy ec2 autoscaling for control plane and worker nodes

to deploy high availability clusters :

require to have 3 control plane nodes which give beneficial performance and role decision to whom be master
2 . following formula of Quarom= N/2+1 which N=No.of control plane node
ex 3/2+1 = 2 meaning the minimum to consider it Healthy at 2 control plane node are ready for role decision and preemption
to setup High availability control plane :
disable swap
2, update kernel params
install container runtime ,containerd
install runc
install cni pluging
install kubeadm ,kubelet , kubectl
optional check version of kubeadm , kubelet,kubectl
initialize the control plane with kubeadm

sudo kubeadm --control-plane-endpoint “Load-Balancer-DNS:6443” --pod-network-cidr 192.168.0.0/16 --upload-certs optional[--node-name master1 ]

install calico cni pluging 3.29.3
when you join other control plane node , use

sudo kubeadm join mylb-b12f4818ac02ac54.elb.us-east-1.amazonaws.com:6443 --token wme2w0.ft4sn3x0tc2035bu \
--discovery-token-ca-cert-hash sha256:a4e516c840d20093ef2262c57b3afb171be07cf0ec0514de8198b91f46418b36 \
--control-plane --certificate-key 403d1068384eebe16c1a50fc25e81b24ac7ccef26423244af9e1966e80531f9b --node-name

Note: give name for node using --node-name

setup worker nodes
disable swap 2, update kernel params
install container runtime ,containerd
install runc
install cni pluging
install kubeadm ,kubelet , kubectl
optional check version of kubeadm , kubelet,kubectl
copy kubeconfig from controlplane node to worker node kubeconfig

if forget or missing token output when kubeadm bootstraps finish

sudo kubeadm token --print-join-command

master.sh

` #!/bin/bash 


`#disable swap 
sudo swapoff -a
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# update kernel params 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf 
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system

# install container runtime ,containerd 

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo mkdir -p /usr/local/lib/systemd/system/
sudo mv containerd.service /usr/local/lib/systemd/system/
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml

sudo systemctl daemon-reload
sudo systemctl enable containerd --now


# install runc 

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc 

# install cni 

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

# install kubeadm, kubelet, kubectl

sudo apt-get update -y
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update -y
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

kubeadm version 
kubelet --version
kubectl version --client

# initial kubeadm for Ha cluster

sudo kubeadm init --control-plane-endpoint mylb-030c3a1c6c4fae6a.elb.us-east-1.amazonaws.com:6443 --pod-network-cidr 192.168.0.0/16 --upload-certs --node-name master1

# configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chown $(id -u):$(id -g) /var/run/containerd


mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# This command changes the ownership of the Kubernetes admin configuration file
# - $(id -u) gets the current user's ID number
# - $(id -g) gets the current user's group ID number
# - /etc/kubernetes/admin.conf is the Kubernetes admin config file containing cluster credentials
# - chown changes the owner and group of the file to match the current user
# This allows the current user to access the Kubernetes cluster without needing sudo
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf


# install calico cni plugin 3.29.3 

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/tigera-operator.yaml

curl -o custom-resources.yaml https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/custom-resources.yaml

kubectl create -f custom-resources.yaml


# alias kubectl utility 

alias k=kubectl > .bashrc &&  source .bashrc


master-ha1.sh  


#!/bin/bash 
# disable swap 
sudo swapoff -a
sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# update kernel params 
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf 
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system

# install container runtime ,containerd 

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo mkdir -p /usr/local/lib/systemd/system/
sudo mv containerd.service /usr/local/lib/systemd/system/
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml

sudo systemctl daemon-reload
sudo systemctl enable containerd --now


# install runc 

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc 

# install cni 

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

# install kubeadm, kubelet, kubectl

sudo apt-get update -y
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update -y
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

kubeadm version 
kubelet --version
kubectl version --client

# initial kubeadm for Ha cluster

sudo kubeadm join mylb-030c3a1c6c4fae6a.elb.us-east-1.amazonaws.com:6443 --token 236pfl.0f9p8r06gcmfmj7p \
--discovery-token-ca-cert-hash sha256:992eb054da8690fbfc3ccbf2b0b30e5ce323c6b3444b3f11724dafa072e5f45a \
--control-plane --certificate-key 3fb4811956c257fd5407fb6aafd099077eb66a2da357683447b1b186d4dd890c --node-name master2

# sudo kubeadm join mylb-b12f4818ac02ac54.elb.us-east-1.amazonaws.com:6443 --token wme2w0.ft4sn3x0tc2035bu \
# --discovery-token-ca-cert-hash sha256:a4e516c840d20093ef2262c57b3afb171be07cf0ec0514de8198b91f46418b36 \
# --control-plane --certificate-key 403d1068384eebe16c1a50fc25e81b24ac7ccef26423244af9e1966e80531f9b --node-name master2

# configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chown $(id -u):$(id -g) /var/run/containerd


mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf

Note:
   when   join  other  control plane  node    , no  need  to  install calico  cni  2.29.3 , automatically  installed 

# alias kubectl utility 

alias k=kubectl > .bashrc &&  source .bashrc

master-ha2.sh

#!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system

install container runtime ,containerd

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo mkdir -p /usr/local/lib/systemd/system/
sudo mv containerd.service /usr/local/lib/systemd/system/
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

sudo apt-get update -y
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update -y
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

kubeadm version
kubelet --version
kubectl version --client

initial kubeadm for Ha cluster

sudo kubeadm join mylb-030c3a1c6c4fae6a.elb.us-east-1.amazonaws.com:6443 --token 236pfl.0f9p8r06gcmfmj7p \
--discovery-token-ca-cert-hash sha256:992eb054da8690fbfc3ccbf2b0b30e5ce323c6b3444b3f11724dafa072e5f45a \
--control-plane --certificate-key 3fb4811956c257fd5407fb6aafd099077eb66a2da357683447b1b186d4dd890c --node-name master3

sudo kubeadm join mylb-b12f4818ac02ac54.elb.us-east-1.amazonaws.com:6443 --token wme2w0.ft4sn3x0tc2035bu \

--discovery-token-ca-cert-hash sha256:a4e516c840d20093ef2262c57b3afb171be07cf0ec0514de8198b91f46418b36 \

--control-plane --certificate-key 403d1068384eebe16c1a50fc25e81b24ac7ccef26423244af9e1966e80531f9b --node-name master3

configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chown $(id -u):$(id -g) /var/run/containerd

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf

alias kubectl utility

alias k=kubectl > .bashrc && source .bashrc

for worker nodes

worker1.sh

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

install container runtime ,containerd

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

initial kubeadm for Ha cluster

sudo kubeadm token create --print-join-command

sudo kubeadm join mylb-7ede46b278348924.elb.us-east-1.amazonaws.com:6443 --token s1pzb2.e49unzo701usj6er \

--discovery-token-ca-cert-hash sha256:3d5d6836796b31f8d638feef97711973cbb62388cd549fb7676016e4d11d3689 --node-name worker1

configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chown $(id -u):$(id -g) /run/containerd/containerd.sock

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf

copy kubeconfig from control plane to worker nodes kubeconfig

alias kubectl utility

alias k=kubectl > .bashrc && source .bashrc

worker2.sh

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

install container runtime ,containerd

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

initial kubeadm for Ha cluster

sudo kubeadm token create --print-join-command

sudo kubeadm join mylb-7ede46b278348924.elb.us-east-1.amazonaws.com:6443 --token s1pzb2.e49unzo701usj6er \

--discovery-token-ca-cert-hash sha256:3d5d6836796b31f8d638feef97711973cbb62388cd549fb7676016e4d11d3689 --node-name worker2

sudo kubeadm join mylb-7966c35b34cfe6a2.elb.us-east-1.amazonaws.com:6443 --token wedeoe.3dktwv0s0a1tfudi \
--discovery-token-ca-cert-hash sha256:1104fe6342e316036524c87a994e47035f091f6869a4f84292cecc8dd4f279bd --node-name worker2

configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chwon $(id -u):$(id -g) /run/containerd/containerd.sock

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf

alias kubectl utility

alias k=kubectl > .bashrc && source .bashrc

worker3.sh

#!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

install container runtime ,containerd

curl -LO https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-2.0.4-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

initial kubeadm for Ha cluster

sudo kubeadm token create --print-join-command

sudo kubeadm join mylb-7ede46b278348924.elb.us-east-1.amazonaws.com:6443 --token s1pzb2.e49unzo701usj6er \

--discovery-token-ca-cert-hash sha256:3d5d6836796b31f8d638feef97711973cbb62388cd549fb7676016e4d11d3689 --node-name worker2

configure crictl to work with containerd

sudo crictl config runtime-endpoint: unix:///run/containerd/containerd.sock
sudo chwon $(id -u):$(id -g) /run/containerd/containerd.sock

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo chown $(id -u):$(id -g) /etc/kubernetes/admin.conf

export KUBECONFIG=/etc/kubernetes/admin.conf

alias kubectl utility

alias k=kubectl > .bashrc && source .bashrc

Note: remember to copy kubeconfig from control plane to worker nodes kubeconfig

to deploy infrastructure using by pulumi python and using pulumi esc to store key/value

"""An AWS Python Pulumi program"""
import pulumi , pulumi_aws as aws , json
import pulumi

cfg1=pulumi.Config()

vpc1=aws.ec2.Vpc(
"vpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require(key="block1"),
tags={
"Name" : "vpc1"
}
)
)

intgw1=aws.ec2.InternetGateway(
"intgw1",
aws.ec2.InternetGatewayArgs(
vpc_id=vpc1.id,
tags={
"Name" : "intgw1"
}
)
)

zones=["us-east-1a", "us-east-1b" , "us-east-1c"]
cidr1=cfg1.require(key="cidr1")
cidr2=cfg1.require(key="cidr2")
cidr3=cfg1.require(key="cidr3")
cidr4=cfg1.require(key="cidr4")
cidr5=cfg1.require(key="cidr5")
cidr6=cfg1.require(key="cidr6")
pubsubnets=[cidr1,cidr2,cidr3]
privsubnets=[cidr4,cidr5,cidr6]

pubnames=["pub1" , "pub2" , "pub3" ]
privnames=[ "priv1" , "priv2" , "priv3" ]

for allpub in range(len(pubnames)):
pubnames[allpub]=aws.ec2.Subnet(
pubnames[allpub],
aws.ec2.SubnetArgs(
vpc_id=vpc1.id,
cidr_block=pubsubnets[allpub],
availability_zone=zones[allpub],
map_public_ip_on_launch=True,
tags={
"Name" : pubnames[allpub]
}

)
)

table1=aws.ec2.RouteTable(
"table1",
aws.ec2.RouteTableArgs(
vpc_id=vpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require(key="any_ipv4_traffic"),
gateway_id=intgw1.id
)
],
tags={
"Name" : "table1"
}

)
)

pub_associate1=aws.ec2.RouteTableAssociation(
"pub_associate1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pubnames[0].id,
route_table_id=table1.id
)
)

pub_associate2=aws.ec2.RouteTableAssociation(
"pub_associate2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pubnames[1].id,
route_table_id=table1.id
)
)

pub_associate3=aws.ec2.RouteTableAssociation(
"pub_associate3",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pubnames[2].id,
route_table_id=table1.id
)
)

for allpriv in range(len(privnames)):
privnames[allpriv]=aws.ec2.Subnet(
privnames[allpriv],
aws.ec2.SubnetArgs(
vpc_id=vpc1.id,
cidr_block=privsubnets[allpriv],
availability_zone=zones[allpriv],
tags={
"Name" : privnames[allpriv]
}

)
)

eips=[ "eip1" , "eip2" , "eip3" ]

for alleips in range(len(eips)):
eips[alleips]=aws.ec2.Eip(
eips[alleips],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name" : eips[alleips]
}
)
)

nats=[ "natgw1" , "natgw2" , "natgw3" ]

mynats= {

"nat1" : aws.ec2.NatGateway(
"nat1",
connectivity_type="public",
subnet_id=pubnames[allpub].id,
allocation_id=eips[0].allocation_id,
tags={
"Name" : "nat1"
}
),

"nat2" : aws.ec2.NatGateway(
"nat2",
connectivity_type="public",
subnet_id=pubnames[1].id,
allocation_id=eips[1].allocation_id,
tags={
"Name" : "nat2"
}
),

"nat3" : aws.ec2.NatGateway(
"nat3",
connectivity_type="public",
subnet_id=pubnames[2].id,
allocation_id=eips[2].allocation_id,
tags={
"Name" : "nat3"
}
),

}

priv2table2=aws.ec2.RouteTable(
"priv2table2",
aws.ec2.RouteTableArgs(
vpc_id=vpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require(key="any_ipv4_traffic"),
nat_gateway_id=mynats["nat1"]
)
],
tags={

"Name" : "priv2table2"
}
)
)

priv3table3=aws.ec2.RouteTable(
"priv3table3",
aws.ec2.RouteTableArgs(
vpc_id=vpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require(key="any_ipv4_traffic"),
nat_gateway_id=mynats["nat2"]
)
],
tags={

"Name" : "priv3table3"
}
)
)

priv4table4=aws.ec2.RouteTable(
"priv4table4",
aws.ec2.RouteTableArgs(
vpc_id=vpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require(key="any_ipv4_traffic"),
nat_gateway_id=mynats["nat3"]
)
],
tags={

"Name" : "priv4table4"
}
)
)

privassociate1=aws.ec2.RouteTableAssociation(
"privassociate1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=privnames[0].id,
route_table_id=priv2table2.id
)
)

privassociate2=aws.ec2.RouteTableAssociation(
"privassociate2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=privnames[1].id,
route_table_id=priv3table3.id
)
)

privassociate3=aws.ec2.RouteTableAssociation(
"privassociate3",
aws.ec2.RouteTableAssociationArgs(
subnet_id=privnames[2].id,
route_table_id=priv4table4.id
)
)

nacl_ingress_trafic=[
aws.ec2.NetworkAclIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="myips"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=100
),
aws.ec2.NetworkAclIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="deny",
rule_no=101
),
aws.ec2.NetworkAclIngressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200

),
aws.ec2.NetworkAclIngressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=300
),
aws.ec2.NetworkAclIngressArgs(
from_port=1024,
to_port=65535,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=400
),
aws.ec2.NetworkAclIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=500
)
]

nacl_egress_trafic=[
aws.ec2.NetworkAclEgressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="myips"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=100
),
aws.ec2.NetworkAclEgressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="deny",
rule_no=101
),
aws.ec2.NetworkAclEgressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200

),
aws.ec2.NetworkAclEgressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=300
),

aws.ec2.NetworkAclEgressArgs(
from_port=1024,
to_port=65535,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=400
),
aws.ec2.NetworkAclEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=500
)
]

mynacls=aws.ec2.NetworkAcl(
"mynacls",
aws.ec2.NetworkAclArgs(
vpc_id=vpc1.id,
ingress=nacl_ingress_trafic,
egress=nacl_egress_trafic,
tags={
"Name": "mynacls"
}
)
)

nacl_associate1=aws.ec2.NetworkAclAssociation(
"nacl_associate1",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=pubnames[0].id
),
)

nacl1=aws.ec2.NetworkAclAssociation(
"nacl1",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=pubnames[0].id
)
)
nacl2=aws.ec2.NetworkAclAssociation(
"nacl2",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=pubnames[1].id
)
),

nacl3=aws.ec2.NetworkAclAssociation(
"nacl3",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=pubnames[2].id
)
),

nacl4=aws.ec2.NetworkAclAssociation(
"nacl4",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=privnames[0].id
)
),

nacl5=aws.ec2.NetworkAclAssociation(
"nacl5",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=privnames[1].id
)
),
nacl6=aws.ec2.NetworkAclAssociation(
"nacl6",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=mynacls.id,
subnet_id=privnames[2].id
)
),

master_ingress_rule={
"rule2":aws.ec2.SecurityGroupIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[ cfg1.require(key="any_ipv4_traffic") ]
),
}

master_egress_rule={
"rule5": aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[ cfg1.require(key="any_ipv4_traffic") ]
)
}

mastersecurity=aws.ec2.SecurityGroup(
"mastersecurity",
aws.ec2.SecurityGroupArgs(
name="mastersecurity",
vpc_id=vpc1.id,
tags={
"Name" : "mastersecurity"
},
ingress=[
master_ingress_rule["rule2"],
],
egress=[master_egress_rule["rule5"]]
)
)

worker_ingress_rule={
"rule2": aws.ec2.SecurityGroupIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
),
}

worker_egress_rule={
"rule5": aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[ cfg1.require(key="any_ipv4_traffic") ]
)
}

workersecurity=aws.ec2.SecurityGroup(
"workersecurity",
aws.ec2.SecurityGroupArgs(
name="workersecurity",
vpc_id=vpc1.id,
tags={
"Name" : "workersecurity"
},
ingress=[
worker_ingress_rule["rule2"]
],
egress=[worker_egress_rule["rule5"]]
)
)

lbsecurity=aws.ec2.SecurityGroup(
"lbsecurity",
aws.ec2.SecurityGroupArgs(
name="lbsecurity",
vpc_id=vpc1.id,
tags={
"Name" : "lbsecurity"
},
ingress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=6443,
to_port=6443,
protocol="tcp",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")] # for kube-apiserver
),
],
egress=[
aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
)
]
)
)

mylb=aws.lb.LoadBalancer(
"mylb",
aws.lb.LoadBalancerArgs(
name="mylb",
load_balancer_type="network",
idle_timeout=300,
ip_address_type="ipv4",
security_groups=[lbsecurity.id],
subnets=[
pubnames[0].id,
pubnames[1].id,
pubnames[2].id
],
tags={
"Name" : "mylb"
},
internal=False,
enable_cross_zone_load_balancing=True
)
)

mytargets2=aws.lb.TargetGroup(
"mytargets2",
aws.lb.TargetGroupArgs(
name="mytargets2",
port=6443,
protocol="TCP",
target_type="instance",
vpc_id=vpc1.id,
tags={
"Name" : "mytargets2"
},
health_check=aws.lb.TargetGroupHealthCheckArgs(
enabled=True,
interval=60,
port="traffic-port",
healthy_threshold=3,
unhealthy_threshold=3,
timeout=30,
matcher="200-599"
)
)
)

listener1=aws.lb.Listener(
"listener1",
aws.lb.ListenerArgs(
load_balancer_arn=mylb.arn,
port=6443,
protocol="TCP",
default_actions=[
aws.lb.ListenerDefaultActionArgs(
type="forward",
target_group_arn=mytargets2.arn
),
]
)
)

clusters_role=aws.iam.Role(
"clusters_role",
aws.iam.RoleArgs(
name="clustersroles",
assume_role_policy=json.dumps({
"Version" : "2012-10-17",
"Statement" : [{
"Effect" : "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"Service" : "ec2.amazonaws.com"
}
}]

})
)
)

ssmattach1=aws.iam.RolePolicyAttachment(
"ssmattach1",
aws.iam.RolePolicyAttachmentArgs(
role=clusters_role.name,
policy_arn=aws.iam.ManagedPolicy.AMAZON_SSM_MANAGED_EC2_INSTANCE_DEFAULT_POLICY
)
)

myprofile=aws.iam.InstanceProfile(
"myprofile",
aws.iam.InstanceProfileArgs(
name="myprofile",
role=clusters_role.name,
tags={
"Name" : "myprofile"
}

)
)

mastertemps=aws.ec2.LaunchTemplate(
"mastertemps",
aws.ec2.LaunchTemplateArgs(
image_id=cfg1.require(key="ami"),
name="mastertemps",
instance_type=cfg1.require(key="instance-type"),
vpc_security_group_ids=[mastersecurity.id],
block_device_mappings=[
aws.ec2.LaunchTemplateBlockDeviceMappingArgs(
device_name="/dev/sdb",
ebs=aws.ec2.LaunchTemplateBlockDeviceMappingEbsArgs(
volume_size=8,
volume_type="gp3",
delete_on_termination=True,
encrypted="false"
)
)
],
tags={
"Name" : "mastertemps"
},
iam_instance_profile= aws.ec2.LaunchTemplateIamInstanceProfileArgs(
arn=myprofile.arn
)
)
)

controlplane=aws.autoscaling.Group(
"controlplane",
aws.autoscaling.GroupArgs(
name="controlplane",
vpc_zone_identifiers=[
pubnames[0].id,
pubnames[1].id,
pubnames[2].id
],
desired_capacity=3,
max_size=9,
min_size=1,
launch_template=aws.autoscaling.GroupLaunchTemplateArgs(
id=mastertemps.id,
version="$Latest"
),
default_cooldown=600,
tags=[
aws.autoscaling.GroupTagArgs(
key="Name",
value="controlplane",
propagate_at_launch=True
)
],
health_check_grace_period=600,
health_check_type="EC2",
)
)

lbattach=aws.autoscaling.Attachment(
"lbattach",
aws.autoscaling.AttachmentArgs(
autoscaling_group_name=controlplane.name ,
lb_target_group_arn=mytargets2.arn
)
)

workertemps=aws.ec2.LaunchTemplate(
"workertemps",
aws.ec2.LaunchTemplateArgs(
image_id=cfg1.require(key="ami"),
name="workertemps",
instance_type=cfg1.require(key="instance-type"),
vpc_security_group_ids=[mastersecurity.id],
block_device_mappings=[
aws.ec2.LaunchTemplateBlockDeviceMappingArgs(
device_name="/dev/sdr",
ebs=aws.ec2.LaunchTemplateBlockDeviceMappingEbsArgs(
volume_size=8,
volume_type="gp3",
delete_on_termination=True,
encrypted="false"
)
)
],
tags={
"Name" : "workertemps"
},
iam_instance_profile=aws.ec2.LaunchTemplateIamInstanceProfileArgs(
arn=myprofile.arn
)
)
)

worker=aws.autoscaling.Group(
"worker",
aws.autoscaling.GroupArgs(
name="worker",
vpc_zone_identifiers=[
privnames[0].id,
privnames[1].id,
privnames[2].id
],
desired_capacity=3,
max_size=9,
min_size=1,
launch_template=aws.autoscaling.GroupLaunchTemplateArgs(
id=workertemps.id,
version="$Latest"
),
default_cooldown=600,
tags=[
aws.autoscaling.GroupTagArgs(
key="Name",
value="worker",
propagate_at_launch=True
)
],
health_check_grace_period=600,
health_check_type="EC2",
)
)

myssmdoc=aws.ssm.Document(
"myssmdoc",
aws.ssm.DocumentArgs(
name="mydoc",
document_format="YAML",
document_type="Command",
version_name="1.1.0",
content="""schemaVersion: '1.2'
description: Check ip configuration of a Linux instance.
parameters: {}
runtimeConfig:
'aws:runShellScript':
properties:

id: '0.aws:runShellScript' runCommand:
sudo su
sudo apt update -y && sudo apt upgrade -y
sudo apt install -y net-tools
sudo systemctl enable amazon-ssm-agent --now

"""
)
)

myssmassociates=aws.ssm.Association(
"myssmassociates",
aws.ssm.AssociationArgs(
name=myssmdoc.name,
association_name="myssmassociates",
targets=[
aws.ssm.AssociationTargetArgs(
key="InstanceIds",
values=["*"]
)
],
)
)

controlplanes=aws.ec2.get_instances(
instance_tags={
"Name": "controlplane"
},
instance_state_names=["running"],
)

workers=aws.ec2.get_instances(
instance_tags={
"Name": "worker"
},
instance_state_names=["running"],
)

get_controlplanepubips=controlplanes.public_ips
get_controlplaneprivips=controlplanes.private_ips
get_workerprivips=workers.private_ips
get_loadbalancer=mylb.dns_name
pulumi.export("get_controlplanepubips", get_controlplanepubips)
pulumi.export("get_controlplaneprivips", get_controlplaneprivips)
pulumi.export("get_workerprivips", get_workerprivips)
pulumi.export("get_loadbalancer", get_loadbalancer)

Note:

for control plane security group

   allow   all  traffic   in  inbound and  outbound  rules

for worker secuiity group

  allow  all  traffic   in  inbound   and outbound  rules

for network load balancer security group

 allow tcp 6443 for apiserver  which  allow  registered  target control plane nodes

outcomes : from control plane and worker nodes

references:

https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/

https://www.youtube.com/watch?v=FHXucrzqcZM&t=1944s

backup and restore etcd

MohammedBanabila — Sun, 30 Mar 2025 01:41:05 +0000

.setup self managed k8s cluster using kubeadm and those steps are :

create vpc and subnet and in this lab we create vpc with cidr block=77.132.0.0/16 and 3 public subnets with block size=24
deploy 3 ec2 instance for all node and define which be control plane , others be worker nodes
create 2 security groups for control plane node and worker nodes 4 . setup network acceess control list 5 . associate elastic ip with those instances
integrate internet gateway with vpc 7 . create routetable and associate it with subnets
after the infrastructure are up and running example ex2 instances

note:
we use control plane node ports which need it by its components

apiserver port 6443
ectd port 2379-2380
scheduler port 10257
controller-manager 10259
access to control plane node port 22 only to my ip address /32

and for workers node port

kubelet    port 10250 
kube-proxy   port 10256 
nodePort      30000-32767 
access to worker node    port 22   only to  my ip  address /32

we add those ports to control plane security group and worker security groups

before start setup , first update and upgrade packages using sudo apt update -y && sudo apt upgrade -y
step for setup control plane node as master node :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
initialize control plane node using kubeadm and print output token which let you to join worker nodes 8 . install cni plugin calico 9 . use kubeadm join to master optional: you can change hostname at nodes change hostname to be worker1 , worker2 for worker nodes

step for setup worker nodes :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
use kubeadm join to master

Note: copy kubeconfig from control plane to workers node at .kube/config

at control plane node
cd .kube/
cat config from master node and copy it to worker node

Note:
you can deploy it at console or as infrastructure as code
in this lab , I used pulumi python for provisioning the resources.

"""An AWS Python Pulumi program"""

import pulumi , pulumi_aws as aws , json

cfg1=pulumi.Config()

vpc1=aws.ec2.Vpc(
"vpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require(key='block1'),
tags={
"Name": "vpc1"
}
)
)

intgw1=aws.ec2.InternetGateway(
"intgw1",
aws.ec2.InternetGatewayArgs(
vpc_id=vpc1.id,
tags={
"Name": "intgw1"
}
)
)

zones="us-east-1a"
publicsubnets=["subnet1","subnet2","subnet3"]
cidr1=cfg1.require(key="cidr1")
cidr2=cfg1.require(key="cidr2")
cidr3=cfg1.require(key="cidr3")

cidrs=[ cidr1 , cidr2 , cidr3 ]

for allsubnets in range(len(publicsubnets)):
publicsubnets[allsubnets]=aws.ec2.Subnet(
publicsubnets[allsubnets],
aws.ec2.SubnetArgs(
vpc_id=vpc1.id,
cidr_block=cidrs[allsubnets],
map_public_ip_on_launch=False,
availability_zone=zones,
tags={
"Name" : publicsubnets[allsubnets]
}
)
)

associate1=aws.ec2.RouteTableAssociation(
"associate1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=publicsubnets[0].id,
route_table_id=table1.id
)
)
associate2=aws.ec2.RouteTableAssociation(
"associate2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=publicsubnets[1].id,
route_table_id=table1.id
)

)

associate3=aws.ec2.RouteTableAssociation(
"associate3",
aws.ec2.RouteTableAssociationArgs(
subnet_id=publicsubnets[2].id,
route_table_id=table1.id
)

)

ingress_traffic=[
aws.ec2.NetworkAclIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="myips"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=100
),
aws.ec2.NetworkAclIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="deny",
rule_no=101
),
aws.ec2.NetworkAclIngressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200
),
aws.ec2.NetworkAclIngressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=300
),
aws.ec2.NetworkAclIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=400
)

]

egress_traffic=[
aws.ec2.NetworkAclEgressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="myips"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=100
),
aws.ec2.NetworkAclEgressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="deny",
rule_no=101
),
aws.ec2.NetworkAclEgressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200
),
aws.ec2.NetworkAclEgressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=300
),
aws.ec2.NetworkAclEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=400
)
]

nacls1=aws.ec2.NetworkAcl(
"nacls1",
aws.ec2.NetworkAclArgs(
vpc_id=vpc1.id,
ingress=ingress_traffic,
egress=egress_traffic,
tags={
"Name" : "nacls1"
},
)
)

nacllink1=aws.ec2.NetworkAclAssociation(
"nacllink1",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[0].id
)
)

nacllink2=aws.ec2.NetworkAclAssociation(
"nacllink2",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[1].id
)
)

nacllink3=aws.ec2.NetworkAclAssociation(
"nacllink3",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[2].id
)
)

masteringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")],

),
aws.ec2.SecurityGroupIngressArgs(
from_port=6443,
to_port=6443,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=2379,
to_port=2380,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]

aws.ec2.SecurityGroupIngressArgs(
from_port=10249,
to_port=10260,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
)
]
masteregress=[
aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
),
]

mastersecurity=aws.ec2.SecurityGroup(
"mastersecurity",
aws.ec2.SecurityGroupArgs(
vpc_id=vpc1.id,
name="mastersecurity",
ingress=masteringress,
egress=masteregress,
tags={
"Name" : "mastersecurity"
}
)
)

workeringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")]
),

aws.ec2.SecurityGroupIngressArgs(
from_port=10250,
to_port=10250,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=10256,
to_port=10256,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=30000,
to_port=32767,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
)

]
workeregress=[
aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]

),
]

workersecurity=aws.ec2.SecurityGroup(
"workersecurity",
aws.ec2.SecurityGroupArgs(
vpc_id=vpc1.id,
name="workersecurity",
ingress=workeringress,
egress=workeregress,
tags={
"Name" : "workersecurity"
}
)
)

master=aws.ec2.Instance(
"master",
aws.ec2.InstanceArgs(
ami=cfg1.require(key='ami'),
instance_type=cfg1.require(key='instance-type'),
vpc_security_group_ids=[mastersecurity.id],
subnet_id=publicsubnets[0].id,
availability_zone=zones,
key_name="mykey1",
tags={
"Name" : "master"
},
ebs_block_devices=[
aws.ec2.InstanceEbsBlockDeviceArgs(
device_name="/dev/sdm",
volume_size=8,
volume_type="gp3"
)
]
)
)

worker1=aws.ec2.Instance(
"worker1",
aws.ec2.InstanceArgs(
ami=cfg1.require(key='ami'),
instance_type=cfg1.require(key='instance-type'),
vpc_security_group_ids=[workersecurity.id],
subnet_id=publicsubnets[1].id,
availability_zone=zones,
key_name="mykey2",
tags={
"Name" : "worker1"
},
ebs_block_devices=[
aws.ec2.InstanceEbsBlockDeviceArgs(
device_name="/dev/sdb",
volume_size=8,
volume_type="gp3"
)
],
)
)

worker2=aws.ec2.Instance(
"worker2",
aws.ec2.InstanceArgs(
ami=cfg1.require(key='ami'),
instance_type=cfg1.require(key='instance-type'),
vpc_security_group_ids=[workersecurity.id],
subnet_id=publicsubnets[2].id,
key_name="mykey2",
tags={
"Name" : "worker2"
},
ebs_block_devices=[
aws.ec2.InstanceEbsBlockDeviceArgs(
device_name="/dev/sdc",
volume_size=8,
volume_type="gp3"
)
],
)
)

eips=[ "eip1" , "eip2" , "eip3" ]
for alleips in range(len(eips)):
eips[alleips]=aws.ec2.Eip(
eips[alleips],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name" : eips[alleips]
}
)
)

eiplink1=aws.ec2.EipAssociation(
"eiplink1",
aws.ec2.EipAssociationArgs(
allocation_id=eips[0].id,
instance_id=master.id
)
)

eiplink2=aws.ec2.EipAssociation(
"eiplink2",
aws.ec2.EipAssociationArgs(
allocation_id=eips[1].id,
instance_id=worker1.id
)
)

eiplink3=aws.ec2.EipAssociation(
"eiplink3",
aws.ec2.EipAssociationArgs(
allocation_id=eips[2].id,
instance_id=worker2.id
)
)

pulumi.export("master_eip" , value=eips[0].public_ip )

pulumi.export("worker1_eip", value=eips[1].public_ip )

pulumi.export("worker2_eip", value=eips[2].public_ip )

pulumi.export("master_private_ip", value=master.private_ip)

pulumi.export("worker1_private_ip" , value=worker1.private_ip)

pulumi.export( "worker2_private_ip" , value=worker2.private_ip )

master.sh script : for master node

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

initialize the control plane node using kubeadm

sudo kubeadm init --pod-network-cidr=192.168.0.0/16 --apiserver-advertise-address=77.132.100.111 --node-name master

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

export KUBECONFIG=/etc/kubernetes/admin.conf

install cni calico

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/tigera-operator.yaml

curl -o custom-resources.yaml https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/custom-resources.yaml

kubectl apply -f custom-resources.yaml

Notice:

sudo kubeadm join 77.132.100.111:6443 --token uwic8i.4btsato9aj46v7pz \
--discovery-token-ca-cert-hash sha256:084b7c1384239e35a6d18d5b6034cfc621193a2d4dd206fa3dfc405dd6976335

output from kubadm after finished installation so can use to access worker node

if you missing the token , you can initiate new token that worker node can be ue it

sudo kubeadm token create --print-join-command >> join-master.sh

sudo chmod +x join-master.sh

worker1.sh script for worker node1

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

sudo hostname worker1

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.111:6443 --token uwic8i.4btsato9aj46v7pz \
--discovery-token-ca-cert-hash sha256:084b7c1384239e35a6d18d5b6034cfc621193a2d4dd206fa3dfc405dd6976335

worker2 script for worker node 2

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

sudo hostname worker2

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.111:6443 --token uwic8i.4btsato9aj46v7pz \
--discovery-token-ca-cert-hash sha256:084b7c1384239e35a6d18d5b6034cfc621193a2d4dd206fa3dfc405dd6976335

after finishiing the setup for kuberbernetes cluster and worker node:

create alias for kubectll command

Exmaple

alias k=kubectl > .bashrc for all nodes

create deployment nginx1 with image nginx:1.27.3 , replicas=4 --port=8000

expose deployment nginx1 with type=NodePort –port=80, --target-port=8000 –name=nginx-svc

k create deployment/nginx1 –image=nginx:1.27.3 –replicas=4 –port=8000
k expose deployment/nginx1 --port=80 --target-port=8000 --name=nginx-svc
k get pods

k get svc
k get deploy
k descrinbe pod pod-name
k describe deploy deployment-name
k describe svc service-name
Install etcdctl on master node
sudo apt install -y etcd-client

ECTDCTL_API=3 so you can use command etcdctl

export ECTDCTL_API=3

type etcdctl

cd /etc/kubernetes/manifest

cat etcd.yaml

     notice     --data-dir  |  --cacert=  |--cert=  |  --trust-ca-file  | --key

default directory for etcd /var/lib/etcd

1.backup all datta into /opt/etcd-backup.db
sudo etcdctl --endpoints= \ --cacert= \ --cert= \ --key= snapshot save /opt/etcd-backup.db

restore all data sudo etcdctl --endpoints= \ --cacert= \ --cert= \ --key= snapshot restore /opt/etcd-backup.db --data-dir=/var/lib/etcd-backup-and-restore

use nano or vim editor to change new path of --data-dir=/var/lib/etcd-backup-and-restore on etcd.yaml also with volume mountpath

restart all components by :

kube-apiserver.yaml 
kube-scheduler.yaml 
kube-controller-manager.yaml  
etcd.yaml

mv .yaml /tmp

sudo mv /tmp/.yaml .

also recommeded to restart the kubelet and all daemon

sudo systemctl restart kubelet && sudo systemctl daemon-reload

check on pod etcd-master using k describe pod etcd-master

Example for etcdctl

etcdctl snapshot save

sudo etcdctl --endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot save /opt/etcd-backup.db

etcdctl snapshot restore
sudo etcdctl --endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot restore /opt/etcd-backup.db \
--data-dir /var/lib/etcd-backup-and-restore
output table of etcd-backup.db for etcdctll

sudo etcdctl --write-out=table --endpoints=https://127.0.0.1:2379 \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/server.crt \
--key=/etc/kubernetes/pki/etcd/server.key \
snapshot status /opt/etcd-backup.dd

References:

Options for Highly Available Topology(etcd) https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/
1. Operating etcd clusters for Kubernetes https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/
2. cka series 2024-2025 ,backup and restore etcd https://www.youtube.com/watch?v=R2wuFCYgnm4&list=PLl4APkPHzsUUOkOv3i62UidrLmSB8DcGC&index=36

Setup and upgrade kubernetes version for control plane and worker nodes

MohammedBanabila — Sat, 22 Mar 2025 02:23:43 +0000

in this lab , has two stages:

setup kubernetes version 1.31 for control plane and worker nodes using kubeadm

Note:

we deploy control plane and worker nodes with any cloud provider example Aws

using ec2 instance with run on ubuntu

Steps:

create vpc and subnet and in this lab we create vpc with cidr block=77.132.0.0/16 and 3 public subnets with block size=24
deploy 3 ec2 instance for all node and define which be control plane , others be worker nodes
create 2 security groups for control plane node and worker nodes 4 . setup network acceess control list 5 . associate elastic ip with those instances
integrate internet gateway with vpc 7 . create route table and associate it with subnets
after the infrastructure are up and running example ec2 instances

note:
we use control plane node ports which need it by its components

apiserver port 6443
ectd port 2379-2380
scheduler port 10257
controller-manager 10259
access to control plane node port 22 only to my ip address /32

and for workers node port

kubelet    port 10250 
kube-proxy   port 10256 
nodePort      30000-32767 
access to worker node    port 22   only to  my ip  address /32

we add those ports to control plane security group and worker security groups

before start setup , first update and upgrade packages using sudo apt update -y && sudo apt upgrade -y
step for setup control plane node as master node :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
initialize control plane node using kubeadm and print output token which let you to join worker nodes 8 . install cni plugin cilium 9 . use kubeadm join to master optional: you can change hostname at nodes change hostname to be worker1 , worker2 for worker nodes

step for setup worker nodes :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
use kubeadm join to master

Note: copy kubeconfig from control plane to workers node at .kube/config

at control plane node
cd .kube/
cat config and copy it all

Note:
you can deploy it at console or as infrastructure as code
in this lab ,I used pulumi python for provisioning the resources.

"""An AWS Python Pulumi program"""
import pulumi , pulumi_aws as aws , json

cfg1=pulumi.Config()

vpc1=aws.ec2.Vpc(
"vpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require(key='block1'),
tags={
"Name": "vpc1"
}
)
)

intgw1=aws.ec2.InternetGateway(
"intgw1",
aws.ec2.InternetGatewayArgs(
vpc_id=vpc1.id,
tags={
"Name": "intgw1"
}
)
)

zones="us-east-1a"
publicsubnets=["subnet1","subnet2","subnet3"]
cidr1=cfg1.require(key="cidr1")
cidr2=cfg1.require(key="cidr2")
cidr3=cfg1.require(key="cidr3")

cidrs=[ cidr1 , cidr2 , cidr3 ]

)

associate3=aws.ec2.RouteTableAssociation(
"associate3",
aws.ec2.RouteTableAssociationArgs(
subnet_id=publicsubnets[2].id,
route_table_id=table1.id
)

)

aws.ec2.NetworkAclIngressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200
)

]

aws.ec2.NetworkAclEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_block=cfg1.require(key="any_ipv4_traffic"),
icmp_code=0,
icmp_type=0,
action="allow",
rule_no=200
)
]

nacls1=aws.ec2.NetworkAcl(
"nacls1",
aws.ec2.NetworkAclArgs(
vpc_id=vpc1.id,
ingress=ingress_traffic,
egress=egress_traffic,
tags={
"Name" : "nacls1"
},
)
)

nacllink1=aws.ec2.NetworkAclAssociation(
"nacllink1",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[0].id
)
)

nacllink2=aws.ec2.NetworkAclAssociation(
"nacllink2",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[1].id
)
)

nacllink3=aws.ec2.NetworkAclAssociation(
"nacllink3",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[2].id
)
)

masteringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")],

workeringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")]
),

aws.ec2.SecurityGroupIngressArgs(
from_port=80,
to_port=80,
protocol="tcp",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=443,
to_port=443,
protocol="tcp",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]
),

]
workeregress=[
aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]

),
]

eips=[ "eip1" , "eip2" , "eip3" ]
for alleips in range(len(eips)):
eips[alleips]=aws.ec2.Eip(
eips[alleips],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name" : eips[alleips]
}
)
)

eiplink1=aws.ec2.EipAssociation(
"eiplink1",
aws.ec2.EipAssociationArgs(
allocation_id=eips[0].id,
instance_id=master.id
)
)

eiplink2=aws.ec2.EipAssociation(
"eiplink2",
aws.ec2.EipAssociationArgs(
allocation_id=eips[1].id,
instance_id=worker1.id
)
)

eiplink3=aws.ec2.EipAssociation(
"eiplink3",
aws.ec2.EipAssociationArgs(
allocation_id=eips[2].id,
instance_id=worker2.id
)
)

pulumi.export("master_eip" , value=eips[0].public_ip )

pulumi.export("worker1_eip", value=eips[1].public_ip )

pulumi.export("worker2_eip", value=eips[2].public_ip )

pulumi.export("master_private_ip", value=master.private_ip)

pulumi.export("worker1_private_ip" , value=worker1.private_ip)

pulumi.export( "worker2_private_ip" , value=worker2.private_ip )

Notes:

remember not all cni plugins support network policies
you can use calico or cilium plugins
for kubeadm , kubetlet , kubectl should same version package
in this lab I used v1.31 to have 1.31.7
references:
https://kubernetes.io/docs/reference/networking/ports-and-protocols/
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
https://github.com/opencontainers/runc/releases/
https://github.com/containerd/containerd/releases
https://github.com/opencontainers/runc

Cka 2024-2025 series:
https://www.youtube.com/watch?v=6_gMoe7Ik8k&list=PLl4APkPHzsUUOkOv3i62UidrLmSB8DcGC

shellscript for control plane ,worker1 node , worker2 node :

Under control plane use master.sh

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

sudo apt-get update -y
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update -y
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

initialize the control plane node using kubeadm

sudo kubeadm init --pod-network-cidr=192.168.0.0/16 --apiserver-advertise-address=77.132.100.19 --node-name master

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

export KUBECONFIG=/etc/kubernetes/admin.conf

install helm v3

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
sudo chmod 777 get_helm.sh
./get_helm.sh

install cni cilium cli

CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}

cilium status --wait

install cni cilium using helm

helm repo add cilium https://helm.cilium.io/

helm install cilium cilium/cilium --version 1.17.2 --namespace kube-system

Under worker1 node use worker1.sh

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

initialize the control plane node using kubeadm

sudo hostname worker1

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.19:6443 --token h0oyqy.f26d28ikx6acev65 \
--discovery-token-ca-cert-hash sha256:3551b3a9363969229557eabb5a51a9d3851e335b49f0f2f2ededc535c2fdf77d

Under worker2 node use worker2.sh

!/bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

initialize the control plane node using kubeadm

sudo hostname worker2

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.19:6443 --token h0oyqy.f26d28ikx6acev65 \
--discovery-token-ca-cert-hash sha256:3551b3a9363969229557eabb5a51a9d3851e335b49f0f2f2ededc535c2fdf77d

Second stage:

after setup control plane and worker nodes with kubernetes v1.31 are all ready. will upgrade to kubernetes v1.32 which lead to use version 1.32.3

  use    kubectl get  nodes   to   see  version  1.31.7   for kubectl

upgrade control plane cluster and its components:

check for kubernetes package

pager /etc/apt/sources.list.d/kubernetes.list

output:

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /

change permission for /etc/apt/sources.list.d/kubernetes.list

sudo chmod 777 /etc/apt/sources.list.d/kubernetes.list

change v1.31 to v1.32 by using nano /etc/apt/sources.list.d/kubernetes.list

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ / change pkg 1.32

determine to which version you want to upgrade

sudo apt update
sudo apt-cache madison kubeadm

sudo apt-mark unhold kubeadm && \
sudo apt-get update && sudo apt-get install -y kubeadm='1.32.3-1.1' && \
sudo apt-mark hold kubeadm

sudo kubeadm upgrade plan

sudo kubeadm upgrade apply 1.32.3-1.1

kubectl drain master --ignore-daemonsets

sudo apt-mark unhold kubelet kubectl && \
sudo apt-get update && sudo apt-get install -y kubelet=''1.32.3-1.1'' kubectl=''1.32.3-1.1'' && \
sudo apt-mark hold kubelet kubectl

sudo systemctl daemon-reload
sudo systemctl restart kubelet

kubectl uncordon master

Upgrade for worker1 node

pager /etc/apt/sources.list.d/kubernetes.list

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ / change pkg 1.32

sudo chmod 777 /etc/apt/sources.list.d/kubernetes.list

sudo apt update
sudo apt-cache madison kubeadm

sudo apt-mark unhold kubeadm && \
sudo apt-get update && sudo apt-get install -y kubeadm='1.32.3-1.1' && \
sudo apt-mark hold kubeadm

sudo kubeadm upgrade node

kubectl drain worker1 --ignore-daemonsets

sudo apt-mark unhold kubelet kubectl && \
sudo apt-get update && sudo apt-get install -y kubelet=''1.32.3-1.1'' kubectl=''1.32.3-1.1'' && \
sudo apt-mark hold kubelet kubectl

sudo systemctl daemon-reload
sudo systemctl restart kubelet

kubectl uncordon worker1

upgrade for worker2 node

pager /etc/apt/sources.list.d/kubernetes.list

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /

sudo chmod 777 /etc/apt/sources.list.d/kubernetes.list

deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ / change pkg 1.32

sudo apt update
sudo apt-cache madison kubeadm

sudo apt-mark unhold kubeadm && \
sudo apt-get update && sudo apt-get install -y kubeadm='1.32.3-1.1' && \
sudo apt-mark hold kubeadm

sudo kubeadm upgrade node

kubectl drain worker2 --ignore-daemonsets

sudo apt-mark unhold kubelet kubectl && \
sudo apt-get update && sudo apt-get install -y kubelet=''1.32.3-1.1'' kubectl=''1.32.3-1.1'' && \
sudo apt-mark hold kubelet kubectl

sudo systemctl daemon-reload
sudo systemctl restart kubelet

kubectl uncordon worker2

use a command kubectl get nodes to see changes version to 1.32.3
also check :
kubeadm version
kubelet --version
kubectl -- version

Note :

kubectl drain [ node] it evicted pod and be unschedule
kubectl uncordon [node] it will be schedule to pod at a node

References:

https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/

https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/change-package-repository/

https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/upgrading-linux-nodes/

https://docs.cilium.io/en/stable/installation/k8s-install-kubeadm/#installation-using-kubeadm

Cka series 2024-2025:

https://www.youtube.com/watch?v=NtX75Ze47EU&list=PLl4APkPHzsUUOkOv3i62UidrLmSB8DcGC&index=35

Setup multi node kubernetes cluster using kubeadm

MohammedBanabila — Wed, 19 Mar 2025 17:13:02 +0000

in this lab we deploy control plane and worker nodes with any cloud provider example Aws

using ec2 instance with run on ubuntu .

Steps:

create vpc and subnet and in this lab we create vpc with cidr block=77.132.0.0/16 and 3 public subnets with block size=24
deploy 3 ec2 instance for all node and define which be control plane , others be worker nodes
create 2 security groups for control plane node and worker nodes 4 . setup network acceess control list 5 . associate elastic ip with those instances
integrate internet gateway with vpc 7 . create routetable and associate it with subnets
after the infrastructure are up and running example ex2 instances

note:
we use control plane node ports which need it by its components

apiserver port 6443
ectd port 2379-2380
scheduler port 10257
controller-manager 10259
access to control plane node port 22 only to my ip address /32

and for workers node port

kubelet    port 10250 
kube-proxy   port 10256 
nodePort      30000-32767 
access to worker node    port 22   only to  my ip  address /32

we add those ports to control plane security group and worker security groups

before start setup , first update and upgrade packages using sudo apt update -y && sudo apt upgrade -y
step for setup control plane node as master node :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
initialize control plane node using kubeadm and print output token which let you to join worker nodes 8 . install cni plugin calico 9 . use kubeadm join to master optional: you can change hostname at nodes change hostname to be worker1 , worker2 for worker nodes

step for setup worker nodes :

disable swap
update kernel params 3 install container runtime
install runc
install cni plugin
install kubeadm , kubelet , kubectl and check version
use kubeadm join to master

Note: copy kubeconfig from control plane to workers node at .kube/config

at control plane node
cd .kube/
cat config and copy it all

Note:
you can deploy it at console or as infrastructure as code
in this lab , I used pulumi python for provisioning the resources.

"""An AWS Python Pulumi program"""
import pulumi , pulumi_aws as aws , json

cfg1=pulumi.Config()

vpc1=aws.ec2.Vpc(
"vpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require(key='block1'),
tags={
"Name": "vpc1"
}
)
)

intgw1=aws.ec2.InternetGateway(
"intgw1",
aws.ec2.InternetGatewayArgs(
vpc_id=vpc1.id,
tags={
"Name": "intgw1"
}
)
)

zones="us-east-1a"
publicsubnets=["subnet1","subnet2","subnet3"]
cidr1=cfg1.require(key="cidr1")
cidr2=cfg1.require(key="cidr2")
cidr3=cfg1.require(key="cidr3")

cidrs=[ cidr1 , cidr2 , cidr3 ]

)

associate3=aws.ec2.RouteTableAssociation(
"associate3",
aws.ec2.RouteTableAssociationArgs(
subnet_id=publicsubnets[2].id,
route_table_id=table1.id
)

)

aws.ec2.NetworkAclIngressArgs(
  from_port=0,
  to_port=0,
  protocol="-1",
  cidr_block=cfg1.require(key="any_ipv4_traffic"),
  icmp_code=0,
  icmp_type=0,
  action="allow",
  rule_no=200
)

]

aws.ec2.NetworkAclEgressArgs(
  from_port=0,
  to_port=0,
  protocol="-1",
  cidr_block=cfg1.require(key="any_ipv4_traffic"),
  icmp_code=0,
  icmp_type=0,
  action="allow",
  rule_no=200
)

]

nacls1=aws.ec2.NetworkAcl(
"nacls1",
aws.ec2.NetworkAclArgs(
vpc_id=vpc1.id,
ingress=ingress_traffic,
egress=egress_traffic,
tags={
"Name" : "nacls1"
},
)
)

nacllink1=aws.ec2.NetworkAclAssociation(
"nacllink1",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[0].id
)
)

nacllink2=aws.ec2.NetworkAclAssociation(
"nacllink2",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[1].id
)
)

nacllink3=aws.ec2.NetworkAclAssociation(
"nacllink3",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacls1.id,
subnet_id=publicsubnets[2].id
)
)

masteringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")],

workeringress=[
aws.ec2.SecurityGroupIngressArgs(
from_port=22,
to_port=22,
protocol="tcp",
cidr_blocks=[cfg1.require(key="myips")]
),

aws.ec2.SecurityGroupIngressArgs(
from_port=10250,
to_port=10250,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=10256,
to_port=10256,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
aws.ec2.SecurityGroupIngressArgs(
from_port=30000,
to_port=32767,
protocol="tcp",
cidr_blocks=[vpc1.cidr_block]
),
]
workeregress=[
aws.ec2.SecurityGroupEgressArgs(
from_port=0,
to_port=0,
protocol="-1",
cidr_blocks=[cfg1.require(key="any_ipv4_traffic")]

),
]

eips=[ "eip1" , "eip2" , "eip3" ]
for alleips in range(len(eips)):
eips[alleips]=aws.ec2.Eip(
eips[alleips],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name" : eips[alleips]
}
)
)

eiplink1=aws.ec2.EipAssociation(
"eiplink1",
aws.ec2.EipAssociationArgs(
allocation_id=eips[0].id,
instance_id=master.id
)
)

eiplink2=aws.ec2.EipAssociation(
"eiplink2",
aws.ec2.EipAssociationArgs(
allocation_id=eips[1].id,
instance_id=worker1.id
)
)

eiplink3=aws.ec2.EipAssociation(
"eiplink3",
aws.ec2.EipAssociationArgs(
allocation_id=eips[2].id,
instance_id=worker2.id
)
)

pulumi.export("master_eip" , value=eips[0].public_ip )

pulumi.export("worker1_eip", value=eips[1].public_ip )

pulumi.export("worker2_eip", value=eips[2].public_ip )

pulumi.export("master_private_ip", value=master.private_ip)

pulumi.export("worker1_private_ip" , value=worker1.private_ip)

pulumi.export( "worker2_private_ip" , value=worker2.private_ip )

================================================================

Notes:
create 3 script shell for control plane and worker nodes

example:
master.sh for control plane node
worker1.sh for worker node 1
worker2.sh for worker node 2

give permission to execute those script

sudo chmod +x master.sh

sudo chmod +x worker1.sh
sudo chmod +x worker2.sh

Under master.sh :

!bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

initialize the control plane node using kubeadm

sudo kubeadm init --pod-network-cidr=192.168.0.0/16 --apiserver-advertise-address=77.132.100.6 --node-name master

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

export KUBECONFIG=/etc/kubernetes/admin.conf

install cni calico

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/tigera-operator.yaml

curl -o custom-resources.yaml https://raw.githubusercontent.com/projectcalico/calico/v3.29.2/manifests/custom-resources.yaml

kubectl apply -f custom-resources.yaml

On control plane, get join command:

sudo kubeadm token create --print-join-command >> join-master.sh
sudo chmod +x join-master.sh

sudo kubeadm join 77.132.100.6:6443 --token 26qu35.wtps7hdhutk22nt8 \
--discovery-token-ca-cert-hash sha256:4c0cf2ce4c52d2b1e767056ffd7e2196358ef95c508f1a1abbe83e706a273538

Under worker1.sh :

!bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

sudo hostname worker1

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.6:6443 --token 26qu35.wtps7hdhutk22nt8 \
--discovery-token-ca-cert-hash sha256:4c0cf2ce4c52d2b1e767056ffd7e2196358ef95c508f1a1abbe83e706a273538

=======================
Under worker2.sh

!bin/bash

disable swap

sudo swapoff -a
sudo sed -i '/ swap / s/^(.*)$/#\1/g' /etc/fstab

update kernel params

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo sysctl --system

install container runtime

install containerd 1.7.27

curl -LO https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz

sudo tar Cxzvf /usr/local containerd-1.7.27-linux-amd64.tar.gz

curl -LO https://raw.githubusercontent.com/containerd/containerd/main/containerd.service

sudo systemctl daemon-reload
sudo systemctl enable containerd --now

install runc

curl -LO https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64

sudo install -m 755 runc.amd64 /usr/local/sbin/runc

install cni plugins

curl -LO "https://github.com/containernetworking/plugins/releases/download/v1.6.2/cni-plugins-linux-amd64-v1.6.2.tgz"

sudo mkdir -p /opt/cni/bin

sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.6.2.tgz

install kubeadm, kubelet, kubectl

kubeadm version
kubelet --version
kubectl version --client

configure crictl to work with containerd

sudo crictl config runtime-endpoint unix:///run/containerd/containerd.sock
sudo chmod 777 -R /var/run/containerd/

sudo hostname worker2

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

sudo chmod 777 .kube/
sudo chmod 777 -R /etc/kubernetes/

sudo kubeadm join 77.132.100.6:6443 --token 26qu35.wtps7hdhutk22nt8 \
--discovery-token-ca-cert-hash sha256:4c0cf2ce4c52d2b1e767056ffd7e2196358ef95c508f1a1abbe83e706a273538

Notes:

remember not all cni plugins support network policies
you can use calico or cilium plugins
for kubeadm , kubetlet , kubectl should same version package
in this lab I used v1.31 to have 1.31.7
references:
https://kubernetes.io/docs/reference/networking/ports-and-protocols/
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
https://github.com/opencontainers/runc/releases/
https://github.com/containerd/containerd/releases
https://github.com/opencontainers/runc

Cka 2024-2025 series:
https://www.youtube.com/watch?v=6_gMoe7Ik8k&list=PLl4APkPHzsUUOkOv3i62UidrLmSB8DcGC

Note: --apiserver-advertise-address= [ use private ip address of control plane node ] example 77.132.100.6

pulumi stack reference and Kubernetes provider to be centralized stack cluster for Eks auto mode and use Kubernetes provider

MohammedBanabila — Tue, 28 Jan 2025 15:14:41 +0000

                                                                Eks auto mode

Before this feature, we require to manage the compute, networking, storage, observability, authentication and authorization, with eks cluster. to deploy and access workload.
In this feature, it handles the compute, networking, storage, observability, authentication and authorization, and ease you only to deploy workloads
And manage them depend on your requirements.
To enable auto mode with cluster:

Enable access with authentication mode either API or API_AND_CONFIGMAP
Enable compute which manage and operate nodepools
Enable storage which let to use Ebs or file system to store the data
Enable vpc configuration to manage subnet, endpoint public access, endpoint private access,
Enable Kubernetes configuration which let you to deploy elastic load balancing
Without need to install aws load balancer controller, optional to add network cidr of Kubernetes either ipv4 or ipv6 or on behalf will be create it either 10.100.0.0/16 or 172.31.0.0/16

Note: You can deploy eks auto mode to exiting cluster by enable all components
You can deploy eks auto mode with:
Aws cli
Eksctl
Infrastructure as code example terraform, pulumi and cloudformation
Aws management console

My studies references:
1. AWS re:Invent 2024 - Automate your entire Kubernetes cluster with Amazon EKS Auto Mode (KUB204-NEW)

     https://www.youtube.com/watch?v=a_aDPo9oTMo&t=919s
    2. Dive into Amazon EKS Auto Mode
    https://www.youtube.com/watch?v=qxPP6zb_3mM&t=2092s
   3. AWS re:Invent 2024 - Simplify Kubernetes workloads with Karpenter & Amazon EKS Auto Mode (KUB312)
   https://www.youtube.com/watch?v=JwzP8I8tdaY  
   4. Simplify Amazon EKS Cluster Access Management using EKS Access Entry API

       https://www.youtube.com/watch?v=NvfOulAqy8w&t=48s

EKS Auto - Theory and Live Demo and Q/A (From Principal SA at AWS)

https://www.youtube.com/watch?v=qxVUMyW3a_g
6. Simplified Amazon EKS Access - NEW Cluster Access Management Controls
https://www.youtube.com/watch?v=ae25cbV5Lxo

Automate cluster infrastructure with EKS Auto Mode

https://docs.aws.amazon.com/eks/latest/userguide/automode.html

A deep dive into simplified Amazon EKS access management controls

https://aws.amazon.com/blogs/containers/a-deep-dive-into-simplified-amazon-eks-access-management-controls/

             example of deploy eks auto mode with pulumi at python

a"""An AWS Python Pulumi program"""
import pulumi , pulumi_aws as aws ,json

cfg1=pulumi.Config()

eksvpc1=aws.ec2.Vpc(
"eksvpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require_secret(key="block1"),
tags={
"Name": "eksvpc1",
},
enable_dns_hostnames=True,
enable_dns_support=True
)
)
intgw1=aws.ec2.InternetGateway(
"intgw1" ,
aws.ec2.InternetGatewayArgs(
vpc_id=eksvpc1.id,
tags={
"Name": "intgw1",
},
)
)

pbsubs=["public1","public2"]
zones=["us-east-1a","us-east-1b"]
pbcidr1=cfg1.require_secret("cidr1")
pbcidr2=cfg1.require_secret("cidr2")
pbcidrs=[pbcidr1,pbcidr2]
for allpbsub in range(len(pbsubs)):
pbsubs[allpbsub]=aws.ec2.Subnet(
pbsubs[allpbsub],
aws.ec2.SubnetArgs(
vpc_id=eksvpc1.id,
cidr_block=pbcidrs[allpbsub],
availability_zone=zones[allpbsub],
map_public_ip_on_launch=True,
tags={
"Name" : pbsubs[allpbsub],
"kubernetes.io/role/elb": "1",
}
)
)

ndsubs=["node1","node2"]
ndcidr1=cfg1.require_secret("cidr3")
ndcidr2=cfg1.require_secret("cidr4")
ndcidrs=[ndcidr1,ndcidr2]
for allndsub in range(len(ndsubs)):
ndsubs[allndsub]=aws.ec2.Subnet(
ndsubs[allndsub],
aws.ec2.SubnetArgs(
vpc_id=eksvpc1.id,
cidr_block=ndcidrs[allndsub],
availability_zone=zones[allndsub],
tags={
"Name" : ndsubs[allndsub],
"kubernetes.io/role/internal-elb": "1",
}
)
)

publictable=aws.ec2.RouteTable(
"publictable",
aws.ec2.RouteTableArgs(
vpc_id=eksvpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
gateway_id=intgw1.id,
),
],
tags={
"Name": "publictable",
},
)
)

tblink1=aws.ec2.RouteTableAssociation(
"tblink1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pbsubs[0].id,
route_table_id=publictable.id,
)
)

tblink2=aws.ec2.RouteTableAssociation(
"tblink2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pbsubs[1].id,
route_table_id=publictable.id,
)
)

eips=["eip1", "eip2"]
for alleip in range(len(eips)):
eips[alleip]=aws.ec2.Eip(
eips[alleip],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name": eips[alleip],
},
)
)

natgws=["natgw1", "natgw2"]
allocates=[eips[0].id , eips[1].id]
for allnat in range(len(natgws)):
natgws[allnat]=aws.ec2.NatGateway(
natgws[allnat],
aws.ec2.NatGatewayArgs(
subnet_id=pbsubs[allpbsub].id,
allocation_id=allocates[allnat],
tags={
"Name": natgws[allnat],
},
)
)

privatetables=["privatetable1" , "privatetable2"]
for allprivtable in range(len(privatetables)):
privatetables[allprivtable]=aws.ec2.RouteTable(
privatetables[allprivtable],
aws.ec2.RouteTableArgs(
vpc_id=eksvpc1.id,
routes=[
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
nat_gateway_id=natgws[0].id
),
aws.ec2.RouteTableRouteArgs(
cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
nat_gateway_id=natgws[1].id
),
],
tags={
"Name": privatetables[allprivtable],
},
)
)

privatetablelink1=aws.ec2.RouteTableAssociation(
"privatetablelink1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=ndsubs[0].id,
route_table_id=privatetables[0].id,
)
)

privatetablelink2=aws.ec2.RouteTableAssociation(
"privatetablelink2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=ndsubs[1].id,
route_table_id=privatetables[1].id,
)
)

inbound_traffic=[
aws.ec2.NetworkAclIngressArgs(
from_port=0,
to_port=0,
rule_no=100,
action="allow",
protocol="-1",
cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
icmp_code=0,
icmp_type=0
),
]
outbound_traffic=[
aws.ec2.NetworkAclEgressArgs(
from_port=0,
to_port=0,
rule_no=100,
action="allow",
protocol="-1",
cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
icmp_code=0,
icmp_type=0
),
]

nacllists=["mynacls1" , "mynacls2"]
for allnacls in range(len(nacllists)):
nacllists[allnacls]=aws.ec2.NetworkAcl(
nacllists[allnacls],
aws.ec2.NetworkAclArgs(
vpc_id=eksvpc1.id,
ingress=inbound_traffic,
egress=outbound_traffic,
tags={
"Name": nacllists[allnacls],
}
)
)
nacls30=aws.ec2.NetworkAclAssociation(
"nacls30",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[0].id,
subnet_id=pbsubs[0].id
)
)

nacls31=aws.ec2.NetworkAclAssociation(
"nacls31",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[0].id,
subnet_id=pbsubs[1].id
)
)

nacls10=aws.ec2.NetworkAclAssociation(
"nacls10",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[1].id,
subnet_id=ndsubs[0].id
)
)

nacls11=aws.ec2.NetworkAclAssociation(
"nacls11",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[1].id,
subnet_id=ndsubs[1].id
)
)

eksrole=aws.iam.Role(
"eksrole",
aws.iam.RoleArgs(
assume_role_policy=json.dumps({
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{
"Service":"eks.amazonaws.com"
},
"Action": [
"sts:AssumeRole",
"sts:TagSession",
]
}
] })

))

nodesrole=aws.iam.Role(
"nodesrole",
aws.iam.RoleArgs(
assume_role_policy=json.dumps({
"Version": "2012-10-17",
"Statement":[
{
"Effect": "Allow",
"Principal":{
"Service":"ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}

]
})
))

clusterattach1=aws.iam.RolePolicyAttachment(
"clusterattach1",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
)
)

clusterattach2=aws.iam.RolePolicyAttachment(
"clusterattach2",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSComputePolicy",
)
)

clusterattach3=aws.iam.RolePolicyAttachment(
"clusterattach3",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy",
)
)

clusterattach4=aws.iam.RolePolicyAttachment(
"clusterattach4",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy",
)
)

clusterattach5=aws.iam.RolePolicyAttachment(
"clusterattach5",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy",
)
)

nodesattach1=aws.iam.RolePolicyAttachment(
"nodesattach1",
aws.iam.RolePolicyAttachmentArgs(
role=nodesrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy",
)
)

nodesattach2=aws.iam.RolePolicyAttachment(
"nodesattach2",
aws.iam.RolePolicyAttachmentArgs(
role=nodesrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly",
)
)

automode=aws.eks.Cluster(
"automode",
aws.eks.ClusterArgs(
name="automode",
bootstrap_self_managed_addons=False,
role_arn=eksrole.arn,
version="1.31",
compute_config={
"enabled": True,
"node_pools": ["general-purpose"],
"node_role_arn": nodesrole.arn,
},
access_config={
"authentication_mode": "API",
},
storage_config={
"block_storage": {
"enabled": True,
},

},
kubernetes_network_config={
"elastic_load_balancing": {
"enabled": True,
},
},

    tags={
      "Name" : "automode"    
    },
    vpc_config={
    "endpoint_private_access": True,
    "endpoint_public_access": True,
    "public_access_cidrs": [
        cfg1.require_secret(key="myips"),
    ],
    "subnet_ids": [
        ndsubs[0].id,
        ndsubs[1].id,
    ],
    }
),
opts=pulumi.ResourceOptions(
        depends_on=[
          clusterattach1,
          clusterattach2,
          clusterattach3,
          clusterattach4,
          clusterattach5,
        ]
    )

)

myentry1=aws.eks.AccessEntry(
"myentry1",
aws.eks.AccessEntryArgs(
cluster_name=automode.name,
principal_arn=cfg1.require_secret(key="principal"),
type="STANDARD"
),
opts=pulumi.ResourceOptions(
depends_on=[
automode
]
)
)

entrypolicy1=aws.eks.AccessPolicyAssociation(
"entrypolicy1",
aws.eks.AccessPolicyAssociationArgs(
cluster_name=automode.name,
principal_arn=myentry1.principal_arn,
policy_arn="arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy",
access_scope={
"type" : "cluster",
}
),
opts=pulumi.ResourceOptions(
depends_on=[
automode
]
)
)

Note:
In case you can’t access eks cluster from local pc or labtop
Follow this video , Fixing 'The Server Has Asked for Credentials' in Kubernetes Cluster API - EKS AWS Production

https://www.youtube.com/watch?v=4aLwQASVHAE
create deploy.yaml , for deployment and service NodePort
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deploy
namespace: default
labels:
app: nginx
spec:
selector:
matchLabels:
app: nginx
replicas: 4
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: nginx
spec:
# initContainers:
# Init containers are exactly like regular containers, except:
# - Init containers always run to completion.
# - Each init container must complete successfully before the next one starts.

  containers:
  - name:  nginx
    image:  nginx:1.27.3
    imagePullPolicy: IfNotPresent
    resources:
      requests:
        cpu: 100m
        memory: 100Mi
      limits:
        cpu: 100m
        memory: 100Mi
    livenessProbe:
      tcpSocket:
        port: 80
      initialDelaySeconds: 5
      timeoutSeconds: 5
      successThreshold: 1
      failureThreshold: 3
      periodSeconds: 10
    readinessProbe:
      httpGet:
        path: /
        port: 80
      initialDelaySeconds: 5
      timeoutSeconds: 2
      successThreshold: 1
      failureThreshold: 3
      periodSeconds: 10
    ports:
    - containerPort:  80
      name:  http

apiVersion: v1
kind: Service
metadata:
name: nginx-svc
namespace: default
spec:
selector:
app: nginx
type: NodePort
ports:

name: nginx-svc protocol: TCP port: 80 targetPort: 80 # If you set the spec.type field to NodePort and you want a specific port number, # you can specify a value in the spec.ports[*].nodePort field.

Create ingress

apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
namespace: default
labels:
app.kubernetes.io/name: myingress
name: myingress
spec:
controller: eks.amazonaws.com/alb

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: default
name: myalblab
annotations:

alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/target-type: ip

spec:
ingressClassName: myingress
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: nginx-svc
port:
number: 80

create Nlb with service loadbalancer

apiVersion: v1
kind: Service
metadata:
name: lbsvc
namespace: default
annotations:

service.beta.kubernetes.io/aws-load-balancer-type: nlb
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip

spec:
selector:
app: lbsvc
type: LoadBalancer
ports:

name: lbsvc protocol: TCP port: 80 targetPort: 80 # If you set the spec.type field to NodePort and you want a specific port number, # you can specify a value in the spec.ports[*].nodePort field.

Other options: using stack reference and pulumi Kubernetes registry
Main goals:
To have centralized stack which other stacks able to use and deploy from it their resources. Meaning the outputs from stack A be reference value to the stack B as variable. To ease manage and control the provisioning resources
Using stack reference we would require to have:
Organization/project name/stack name

Note: Organization = pulumi account name

Example:
Stack name : eksdev which deploy eks auto mode
Stack name : kubedev which deploy pod ,service, ingress with pulumi Kubernetes registry

Stack name kubedev

 """A k8s Python Pulumi program"""

import pulumi , pulumi_kubernetes as k8s

cluster_stack="MohammedBanabila/lab-eks/eksdev"

stackref1=pulumi.StackReference(name="stackref1", stack_name=cluster_stack)

cfg1=pulumi.Config()

mycluster=pulumi.export("mycluster", value=stackref1.get_output("cluster"))

provider=k8s.Provider("provider", cluster=mycluster)

deploy=k8s.apps.v1.Deployment(
"deploy",
metadata=k8s.meta.v1.ObjectMetaArgs(
name="nginx-deploy",
namespace="default",
labels={
"app":"nginx"
}
),
spec=k8s.apps.v1.DeploymentSpecArgs(
replicas=4,
selector=k8s.meta.v1.LabelSelectorArgs(
match_labels={
"app":"nginx"
}
),
template=k8s.core.v1.PodTemplateSpecArgs(
metadata=k8s.meta.v1.ObjectMetaArgs(
labels={
"app":"nginx"
}
),
spec=k8s.core.v1.PodSpecArgs(
containers=[
k8s.core.v1.ContainerArgs(
name="nginx",
image="nginx",
ports=[
k8s.core.v1.ContainerPortArgs(
container_port=80
)
]
)
]
)
)
) ,

opts=pulumi.ResourceOptions(provider=provider)

)

svc1=k8s.core.v1.Service(
"svc1",
metadata=k8s.meta.v1.ObjectMetaArgs(
name="nginx-svc",
namespace="default"
),
spec=k8s.core.v1.ServiceSpecArgs(
selector={
"app":"nginx"
},
ports=[
k8s.core.v1.ServicePortArgs(
port=80,
target_port=80
)
],
type="NodePort"
) ,

opts=pulumi.ResourceOptions(provider=provider)

)

ingressclaass=k8s.networking.v1.IngressClass(
"ingressclass",
metadata=k8s.meta.v1.ObjectMetaArgs(
name="nginx-ingress",
namespace="default",
labels={
"app.kubernetes.io/name":"nginx-ingress"
}
),
spec=k8s.networking.v1.IngressClassSpecArgs(
controller="eks.amazonaws.com/alb"
),

opts=pulumi.ResourceOptions(provider=provider)

)

myingress=k8s.networking.v1.Ingress(
"myingress",
metadata=k8s.meta.v1.ObjectMetaArgs(
name="nginx-ingress",
namespace="default",
labels={
"app" : "nginx"
},
annotations={
"alb.ingress.kubernetes.io/scheme":"internet-facing",
"alb.ingress.kubernetes.io/target-type":"ip"
}
),
spec=k8s.networking.v1.IngressSpecArgs(
ingress_class_name="nginx-ingress",
rules=[
k8s.networking.v1.IngressRuleArgs(
http=k8s.networking.v1.HTTPIngressRuleValueArgs(
paths=[
k8s.networking.v1.HTTPIngressPathArgs(
path="/",
path_type="Prefix",
backend=k8s.networking.v1.IngressBackendArgs(
service=k8s.networking.v1.IngressServiceBackendArgs(
name="nginx-svc",
port=k8s.networking.v1.ServiceBackendPortArgs(
number=80
)
)
)
)
]
)
)
]
) ,

opts=pulumi.ResourceOptions(provider=provider)

)

Eks auto mode using pulumi

MohammedBanabila — Fri, 24 Jan 2025 15:34:41 +0000

Enable access with authentication mode either API or API_AND_CONFIGMAP
Enable compute which manage and operate nodepools
Enable storage which let to use Ebs or file system to store the data
Enable vpc configuration to manage subnet, endpoint public access, endpoint private access,
Enable Kubernetes configuration which let you to deploy elastic load balancing
Without need to install aws load balancer controller, optional to add network cidr of Kubernetes either ipv4 or ipv6 or on behalf will be create it either 10.100.0.0/16 or 172.31.0.0/16

Note: You can deploy eks auto mode to exiting cluster by enable all components
You can deploy eks auto mode with:
Aws cli
Eksctl
Infrastructure as code example terraform, pulumi and cloudformation
Aws management console

` """An AWS Python Pulumi program"""
import pulumi , pulumi_aws as aws ,json

cfg1=pulumi.Config()

eksvpc1=aws.ec2.Vpc(
"eksvpc1",
aws.ec2.VpcArgs(
cidr_block=cfg1.require_secret(key="block1"),
tags={
"Name": "eksvpc1",
},
enable_dns_hostnames=True,
enable_dns_support=True
)
)

intgw1=aws.ec2.InternetGateway(
"intgw1" ,
aws.ec2.InternetGatewayArgs(
vpc_id=eksvpc1.id,
tags={
"Name": "intgw1",
},
)
)

tblink1=aws.ec2.RouteTableAssociation(
"tblink1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pbsubs[0].id,
route_table_id=publictable.id,
)
)

tblink2=aws.ec2.RouteTableAssociation(
"tblink2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=pbsubs[1].id,
route_table_id=publictable.id,
)
)

eips=["eip1", "eip2"]
for alleip in range(len(eips)):
eips[alleip]=aws.ec2.Eip(
eips[alleip],
aws.ec2.EipArgs(
domain="vpc",
tags={
"Name": eips[alleip],
},
)
)

        ],
        tags={
            "Name": privatetables[allprivtable],
        },
    )
)

privatetablelink1=aws.ec2.RouteTableAssociation(
"privatetablelink1",
aws.ec2.RouteTableAssociationArgs(
subnet_id=ndsubs[0].id,
route_table_id=privatetables[0].id,
)
)

privatetablelink2=aws.ec2.RouteTableAssociation(
"privatetablelink2",
aws.ec2.RouteTableAssociationArgs(
subnet_id=ndsubs[1].id,
route_table_id=privatetables[1].id,
)
)

inbound_traffic=[

aws.ec2.NetworkAclIngressArgs(
    from_port=22,
    to_port=22,
    rule_no=100,
    action="deny",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),
aws.ec2.NetworkAclIngressArgs(
    from_port=22,
    to_port=22,
    rule_no=101,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="myips"),
    icmp_code=0,
    icmp_type=0
    ),
aws.ec2.NetworkAclIngressArgs(
    from_port=80,
    to_port=80,
    rule_no=200,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),
aws.ec2.NetworkAclIngressArgs(
    from_port=443,
    to_port=443,
    rule_no=300,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),
aws.ec2.NetworkAclIngressArgs(
    from_port=30000,
    to_port=65535,
    rule_no=400,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),
aws.ec2.NetworkAclIngressArgs(
    from_port=0,
    to_port=0,
    rule_no=500,
    action="allow",
    protocol="-1",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    )

]

outbound_traffic=[

aws.ec2.NetworkAclEgressArgs(
    from_port=22,
    to_port=22,
    rule_no=100,
    action="deny",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),
 aws.ec2.NetworkAclEgressArgs(
    from_port=22,
    to_port=22,
    rule_no=101,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="myips"),
    icmp_code=0,
    icmp_type=0
    ),

 aws.ec2.NetworkAclEgressArgs(
    from_port=80,
    to_port=80,
    rule_no=200,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),

 aws.ec2.NetworkAclEgressArgs(
    from_port=443,
    to_port=443,
    rule_no=300,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),


aws.ec2.NetworkAclEgressArgs(
    from_port=30000,
    to_port=65535,
    rule_no=400,
    action="allow",
    protocol="tcp",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),

aws.ec2.NetworkAclEgressArgs(
    from_port=0,
    to_port=0,
    rule_no=500,
    action="allow",
    protocol="-1",
    cidr_block=cfg1.require_secret(key="any-traffic-ipv4"),
    icmp_code=0,
    icmp_type=0
    ),

]

nacls30=aws.ec2.NetworkAclAssociation(
"nacls30",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[0].id,
subnet_id=pbsubs[0].id
)
)
nacls31=aws.ec2.NetworkAclAssociation(
"nacls31",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[0].id,
subnet_id=pbsubs[1].id
)
)

nacls10=aws.ec2.NetworkAclAssociation(
"nacls10",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[1].id,
subnet_id=ndsubs[0].id
)
)
nacls11=aws.ec2.NetworkAclAssociation(
"nacls11",
aws.ec2.NetworkAclAssociationArgs(
network_acl_id=nacllists[1].id,
subnet_id=ndsubs[1].id
)
)

))

nodesrole=aws.iam.Role(
"nodesrole",
aws.iam.RoleArgs(
assume_role_policy=json.dumps({
"Version": "2012-10-17",
"Statement":[
{
"Effect": "Allow",
"Principal":{
"Service":"ec2.amazonaws.com"
},

                "Action": "sts:AssumeRole"
            }   
        ]
    })

))

clusterattach1=aws.iam.RolePolicyAttachment(
"clusterattach1",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",

)
)

clusterattach2=aws.iam.RolePolicyAttachment(
"clusterattach2",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSComputePolicy",
)
)

clusterattach3=aws.iam.RolePolicyAttachment(
"clusterattach3",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy",
)
)

clusterattach4=aws.iam.RolePolicyAttachment(
"clusterattach4",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy",
)
)

clusterattach5=aws.iam.RolePolicyAttachment(
"clusterattach5",
aws.iam.RolePolicyAttachmentArgs(
role=eksrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy",
)
)

nodesattach1=aws.iam.RolePolicyAttachment(
"nodesattach1",
aws.iam.RolePolicyAttachmentArgs(
role=nodesrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy",
)
)

nodesattach2=aws.iam.RolePolicyAttachment(
"nodesattach2",
aws.iam.RolePolicyAttachmentArgs(
role=nodesrole.name,
policy_arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly",
)
)

    },
    access_config={
     "authentication_mode": "API",
    },
    storage_config={
       "block_storage": {
        "enabled": True,
    },     
      },
    kubernetes_network_config={
        "elastic_load_balancing": {
        "enabled": True,
        },
    },
    tags={
      "Name" : "automode"    
    },
    vpc_config={
    "endpoint_private_access": True,
    "endpoint_public_access": True,
    "public_access_cidrs": [
        cfg1.require_secret(key="myips"),
    ],
    "subnet_ids": [
        ndsubs[0].id,
        ndsubs[1].id,
    ],
    }
),
opts=pulumi.ResourceOptions(
        depends_on=[
          clusterattach1,
          clusterattach2,
          clusterattach3,
          clusterattach4,
          clusterattach5,
        ]
    )

)

Note:
In case you can’t access eks cluster from local pc or labtop
Follow this video , Fixing 'The Server Has Asked for Credentials' in Kubernetes Cluster API - EKS AWS Production

https://www.youtube.com/watch?v=4aLwQASVHAE

`apiVersion: apps/v1
kind: Deployment
metadata:
name: httpd-deploy
namespace: default
labels:
app: httpd
spec:
selector:
matchLabels:
app: httpd
replicas: 4
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: httpd
spec:
# initContainers:
# Init containers are exactly like regular containers, except:
# - Init containers always run to completion.
# - Each init container must complete successfully before the next one starts.

  containers:
  - name:  httpd
    image:  httpd:2.4.41-alpine
    imagePullPolicy: IfNotPresent
    resources:
      requests:
        cpu: 100m
        memory: 100Mi
      limits:
        cpu: 100m
        memory: 100Mi
    livenessProbe:
      tcpSocket:
        port: 80
      initialDelaySeconds: 5
      timeoutSeconds: 5
      successThreshold: 1
      failureThreshold: 3
      periodSeconds: 10
    readinessProbe:
      httpGet:
        path: /
        port: 80
      initialDelaySeconds: 5
      timeoutSeconds: 2
      successThreshold: 1
      failureThreshold: 3
      periodSeconds: 10
    ports:
    - containerPort:  80
      name:  http

apiVersion: v1
kind: Service
metadata:
name: httpd-svc
namespace: default
spec:
selector:
app: httpd
type: NodePort
ports:

name: httpd-svc protocol: TCP port: 80 targetPort: 80 # If you set the spec.type field to NodePort and you want a specific port number, # you can specify a value in the spec.ports[*].nodePort field.

`---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
namespace: default
labels:
app.kubernetes.io/name: myingress
name: myingress
spec:
controller: eks.amazonaws.com/alb

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: default
name: myalblab
annotations:

alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/target-type: ip

spec:
ingressClassName: myingress
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: httpd-svc
port:
number: 80

                              `                                                             `