From 3faaed294dab1ed46646622748c2364e55b804ab Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 19 Dec 2017 22:55:10 -0700 Subject: [PATCH 1/4] Initial commit for kubeadm learning environment Add initial set of files for a kubeadm-based learning environment. Environment is completely non-functional. Signed-off-by: Scott Lowe --- kubernetes/kubeadm-vagrant/README.md | 41 +++++++++++++++ kubernetes/kubeadm-vagrant/Vagrantfile | 69 +++++++++++++++++++++++++ kubernetes/kubeadm-vagrant/machines.yml | 46 +++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 kubernetes/kubeadm-vagrant/README.md create mode 100644 kubernetes/kubeadm-vagrant/Vagrantfile create mode 100644 kubernetes/kubeadm-vagrant/machines.yml diff --git a/kubernetes/kubeadm-vagrant/README.md b/kubernetes/kubeadm-vagrant/README.md new file mode 100644 index 0000000..6087d66 --- /dev/null +++ b/kubernetes/kubeadm-vagrant/README.md @@ -0,0 +1,41 @@ +# Docker Swarm Mode with CoreOS + +These files were created to allow users to use Vagrant ([http://www.vagrantup.com](http://www.vagrantup.com)) quickly and relatively easily test Docker "Swarm Mode" using CentOS Atomic Host. This environment was tested using Vagrant 1.9.1 with VirtualBox 5.1.14, but should work with Vagrant's VMware provider as well. + +## Contents + +* **machines.yml**: This YAML file contains a list of VM definitions. It is referenced by `Vagrantfile` when Vagrant instantiates the VMs. You will need to edit this file to provide the correct Vagrant box names. Other edits should not be necessary. + +* **README.md**: This file you're currently reading. + +* **Vagrantfile**: This file is used by Vagrant to spin up the virtual machines. This file is fairly extensively commented to help explain what's happening. You should be able to use this file unchanged; all the VM configuration options are stored outside this file. + +## Instructions + +These instructions assume you've already installed your virtualization provider (VMware Fusion/Workstation or VirtualBox), Vagrant, and any necessary plugins (such as the Vagrant VMware plugin). Please refer to the documentation for those products for more information on installation or configuration. + +1. Use `vagrant box add` to install the CentOS Atomic Host box. For VirtualBox, you can use `vagrant box add centos/atomic-host`. Users of a VMware virtualization provider will likely need to create their own Vagrant box. + +2. Place the files from the `docker/atomic-swarm-mode` directory of this GitHub repository into a directory on your local system. You can clone the entire "learning-tools" repository (using `git clone`) or just download the specific files from the the `docker/atomic-swarm-mode` folder. + +3. If the name of your CentOS Atomic Host Vagrant box is _not_ "centos/atomic-host", edit `machines.yml` to supply the correct box name. This file has separate lines for the VirtualBox and VMware providers; be sure to edit the correct line for your provider. + +4. Run `vagrant up` to instantiate 4 VMs---one manager and three workers. All the VMs will be running CentOS Atomic Host. + +5. Once Vagrant is finished, use `vagrant ssh manager` to log into the manager VM. + +6. Edit the `/etc/docker/daemon.json` file to specify "live-restore" is false (change from true). Restart the Docker daemon with `sudo systemctl restart docker` in order for that change to take effect. (This setting is incompatible with Docker's Swarm mode.) + +7. While logged into the manager VM, run `docker swarm init --advertise-addr 192.168.100.100` to create a new swarm. If you have edited the IP addresses supplied in `machines.yml`, then you'll need to adjust the command accordingly to use the IP address supplied for the manager VM. + + The output of that command will provide a command-line to use to join worker nodes to the swarm. Copy and paste this output; you'll need it shortly. + +8. Log out of the VM and use `vagrant ssh worker-01` to log in to the first worker VM. Repeat step #6 to make the Docker daemon ready to join the Swarm. + +9. Paste the output of step #7 to have the worker node join the swarm. + +10. Repeat steps #8 and #9 for the remaining worker nodes. + +11. Use `vagrant ssh manager` to log back into the manager VM. Run `docker node ls` to show the Docker Engine instances participating in the swarm. + +Enjoy! diff --git a/kubernetes/kubeadm-vagrant/Vagrantfile b/kubernetes/kubeadm-vagrant/Vagrantfile new file mode 100644 index 0000000..ed0d192 --- /dev/null +++ b/kubernetes/kubeadm-vagrant/Vagrantfile @@ -0,0 +1,69 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Specify minimum Vagrant version and Vagrant API version +Vagrant.require_version '>= 1.6.0' +VAGRANTFILE_API_VERSION = '2' + +# Require 'yaml' module +require 'yaml' + +# Read YAML file with VM details (box, CPU, RAM, IP addresses) +# Edit machines.yml to change VM configuration details +machines = YAML.load_file(File.join(File.dirname(__FILE__), 'machines.yml')) + +# Create and configure the VMs +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + # Always use Vagrant's default insecure key + config.ssh.insert_key = false + + # Iterate through entries in YAML file to create VMs + machines.each do |machine| + + # Configure the VMs per details in machines.yml + config.vm.define machine['name'] do |srv| + + # Don't check for box updates + srv.vm.box_check_update = false + + # Specify the hostname of the VM + srv.vm.hostname = machine['name'] + + # Specify the Vagrant box to use (use VMware box by default) + srv.vm.box = machine['box']['vmw'] + + # Configure default synced folder (disable by default) + if machine['sync_disabled'] != nil + srv.vm.synced_folder '.', '/vagrant', disabled: machine['sync_disabled'] + else + srv.vm.synced_folder '.', '/vagrant', disabled: true + end #if machine['sync_disabled'] + + # Iterate through networks as per settings in machines.yml + machine['nics'].each do |net| + if net['ip_addr'] == 'dhcp' + srv.vm.network net['type'], type: net['ip_addr'] + else + srv.vm.network net['type'], ip: net['ip_addr'] + end # if net['ip_addr'] + end # machine['nics'].each + + # Configure CPU & RAM per settings in machines.yml (Fusion) + srv.vm.provider 'vmware_fusion' do |vmw| + vmw.vmx['memsize'] = machine['ram'] + vmw.vmx['numvcpus'] = machine['vcpu'] + if machine['nested'] == true + vmw.vmx['vhv.enable'] = 'TRUE' + end #if machine['nested'] + end # srv.vm.provider 'vmware_fusion' + + # Configure CPU & RAM per settings in machines.yml (VirtualBox) + srv.vm.provider 'virtualbox' do |vb, override| + vb.memory = machine['ram'] + vb.cpus = machine['vcpu'] + override.vm.box = machine['box']['vb'] + end # srv.vm.provider 'virtualbox' + end # config.vm.define + end # machines.each +end # Vagrant.configure diff --git a/kubernetes/kubeadm-vagrant/machines.yml b/kubernetes/kubeadm-vagrant/machines.yml new file mode 100644 index 0000000..9a4c460 --- /dev/null +++ b/kubernetes/kubeadm-vagrant/machines.yml @@ -0,0 +1,46 @@ +--- +- box: + vmw: "centos/atomic-host" + vb: "centos/atomic-host" + name: "manager" + nics: + - type: "private_network" + ip_addr: "192.168.100.100" + ram: "512" + vcpu: "1" +- box: + vmw: "centos/atomic-host" + vb: "centos/atomic-host" + name: "worker-01" + nics: + - type: "private_network" + ip_addr: "192.168.100.101" + ram: "512" + vcpu: "1" +- box: + vmw: "centos/atomic-host" + vb: "centos/atomic-host" + name: "worker-02" + nics: + - type: "private_network" + ip_addr: "192.168.100.102" + ram: "512" + vcpu: "1" +- box: + vmw: "centos/atomic-host" + vb: "centos/atomic-host" + name: "worker-03" + nics: + - type: "private_network" + ip_addr: "192.168.100.103" + ram: "512" + vcpu: "1" +- box: + vmw: "centos/atomic-host" + vb: "centos/atomic-host" + name: "worker-04" + nics: + - type: "private_network" + ip_addr: "192.168.100.104" + ram: "512" + vcpu: "1" From 2b49f918c929d2c675cbaa035dd3db2df27c76e9 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 7 Mar 2018 04:01:44 -0700 Subject: [PATCH 2/4] Fix Vagrant data file Update machines.yml with correct information. Update README.md with preliminary instructions. Signed-off-by: Scott Lowe --- kubernetes/kubeadm-vagrant/README.md | 28 +++++------------ kubernetes/kubeadm-vagrant/machines.yml | 42 +++++++------------------ 2 files changed, 20 insertions(+), 50 deletions(-) diff --git a/kubernetes/kubeadm-vagrant/README.md b/kubernetes/kubeadm-vagrant/README.md index 6087d66..d9964d0 100644 --- a/kubernetes/kubeadm-vagrant/README.md +++ b/kubernetes/kubeadm-vagrant/README.md @@ -1,6 +1,6 @@ -# Docker Swarm Mode with CoreOS +# Local Kubernetes Cluster with Kubeadm -These files were created to allow users to use Vagrant ([http://www.vagrantup.com](http://www.vagrantup.com)) quickly and relatively easily test Docker "Swarm Mode" using CentOS Atomic Host. This environment was tested using Vagrant 1.9.1 with VirtualBox 5.1.14, but should work with Vagrant's VMware provider as well. +These files were created to allow users to use Vagrant ([http://www.vagrantup.com](http://www.vagrantup.com)) quickly and relatively easily spin up a local Kubernetes cluster that is bootstrapped using `kubeadm`. ## Contents @@ -14,28 +14,16 @@ These files were created to allow users to use Vagrant ([http://www.vagrantup.co These instructions assume you've already installed your virtualization provider (VMware Fusion/Workstation or VirtualBox), Vagrant, and any necessary plugins (such as the Vagrant VMware plugin). Please refer to the documentation for those products for more information on installation or configuration. -1. Use `vagrant box add` to install the CentOS Atomic Host box. For VirtualBox, you can use `vagrant box add centos/atomic-host`. Users of a VMware virtualization provider will likely need to create their own Vagrant box. +1. Use `vagrant box add` to install an Ubuntu 16.04 base box. Review `machines.yml` for some suggested boxes for both VirtualBox and VMware virtualization platforms. -2. Place the files from the `docker/atomic-swarm-mode` directory of this GitHub repository into a directory on your local system. You can clone the entire "learning-tools" repository (using `git clone`) or just download the specific files from the the `docker/atomic-swarm-mode` folder. +2. Place the files from the `kubernetes/kubeadm-vagrant` directory of this GitHub repository into a directory on your local system. You can clone the entire "learning-tools" repository (using `git clone`) or just download the specific files from the the `kubernetes/kubeadm-vagrant` folder. -3. If the name of your CentOS Atomic Host Vagrant box is _not_ "centos/atomic-host", edit `machines.yml` to supply the correct box name. This file has separate lines for the VirtualBox and VMware providers; be sure to edit the correct line for your provider. +3. If the name of your Ubuntu 16.04 base box is _not_ one of the ones listed in `machines.yml`, edit `machines.yml` to supply the correct box name(s). This file has separate lines for the VirtualBox and VMware providers; be sure to edit the correct line for your provider. -4. Run `vagrant up` to instantiate 4 VMs---one manager and three workers. All the VMs will be running CentOS Atomic Host. +4. Run `vagrant up` to instantiate 3 VMs---one manager and two nodes. All the VMs will be running Ubuntu 16.04. -5. Once Vagrant is finished, use `vagrant ssh manager` to log into the manager VM. +5. Once Vagrant is finished, use `vagrant ssh master` to log into the manager VM. -6. Edit the `/etc/docker/daemon.json` file to specify "live-restore" is false (change from true). Restart the Docker daemon with `sudo systemctl restart docker` in order for that change to take effect. (This setting is incompatible with Docker's Swarm mode.) - -7. While logged into the manager VM, run `docker swarm init --advertise-addr 192.168.100.100` to create a new swarm. If you have edited the IP addresses supplied in `machines.yml`, then you'll need to adjust the command accordingly to use the IP address supplied for the manager VM. - - The output of that command will provide a command-line to use to join worker nodes to the swarm. Copy and paste this output; you'll need it shortly. - -8. Log out of the VM and use `vagrant ssh worker-01` to log in to the first worker VM. Repeat step #6 to make the Docker daemon ready to join the Swarm. - -9. Paste the output of step #7 to have the worker node join the swarm. - -10. Repeat steps #8 and #9 for the remaining worker nodes. - -11. Use `vagrant ssh manager` to log back into the manager VM. Run `docker node ls` to show the Docker Engine instances participating in the swarm. +6. Enjoy! diff --git a/kubernetes/kubeadm-vagrant/machines.yml b/kubernetes/kubeadm-vagrant/machines.yml index 9a4c460..1952257 100644 --- a/kubernetes/kubeadm-vagrant/machines.yml +++ b/kubernetes/kubeadm-vagrant/machines.yml @@ -1,46 +1,28 @@ --- - box: - vmw: "centos/atomic-host" - vb: "centos/atomic-host" - name: "manager" + vmw: "generic/ubuntu1604" + vb: "ubuntu/xenial64" + name: "master" nics: - type: "private_network" ip_addr: "192.168.100.100" - ram: "512" + ram: "2048" vcpu: "1" - box: - vmw: "centos/atomic-host" - vb: "centos/atomic-host" - name: "worker-01" + vmw: "generic/ubuntu1604" + vb: "ubuntu/xenial64" + name: "node-01" nics: - type: "private_network" ip_addr: "192.168.100.101" - ram: "512" + ram: "2048" vcpu: "1" - box: - vmw: "centos/atomic-host" - vb: "centos/atomic-host" - name: "worker-02" + vmw: "generic/ubuntu1604" + vb: "ubuntu/xenial64" + name: "node-02" nics: - type: "private_network" ip_addr: "192.168.100.102" - ram: "512" - vcpu: "1" -- box: - vmw: "centos/atomic-host" - vb: "centos/atomic-host" - name: "worker-03" - nics: - - type: "private_network" - ip_addr: "192.168.100.103" - ram: "512" - vcpu: "1" -- box: - vmw: "centos/atomic-host" - vb: "centos/atomic-host" - name: "worker-04" - nics: - - type: "private_network" - ip_addr: "192.168.100.104" - ram: "512" + ram: "2048" vcpu: "1" From 99f1a3d73a2cad8092a0221967fb5250272a4cc5 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Sun, 11 Mar 2018 16:38:59 -0600 Subject: [PATCH 3/4] Add Ansible support Add support for running Ansible playbook to configure VMs with packages necessary to turn up Kubernetes using kubeadm Signed-off-by: Scott Lowe --- kubernetes/kubeadm-vagrant/Vagrantfile | 5 +++ kubernetes/kubeadm-vagrant/ansible.cfg | 5 +++ kubernetes/kubeadm-vagrant/ansible.yml | 42 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 kubernetes/kubeadm-vagrant/ansible.cfg create mode 100644 kubernetes/kubeadm-vagrant/ansible.yml diff --git a/kubernetes/kubeadm-vagrant/Vagrantfile b/kubernetes/kubeadm-vagrant/Vagrantfile index ed0d192..06d02ca 100644 --- a/kubernetes/kubeadm-vagrant/Vagrantfile +++ b/kubernetes/kubeadm-vagrant/Vagrantfile @@ -66,4 +66,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end # srv.vm.provider 'virtualbox' end # config.vm.define end # machines.each + + # Configure the VM using Ansible + config.vm.provision 'ansible' do |ansible| + ansible.playbook = 'ansible.yml' + end # config.vm.provision end # Vagrant.configure diff --git a/kubernetes/kubeadm-vagrant/ansible.cfg b/kubernetes/kubeadm-vagrant/ansible.cfg new file mode 100644 index 0000000..e7c5f46 --- /dev/null +++ b/kubernetes/kubeadm-vagrant/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +inventory = .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory +private_key_file = ~/.vagrant.d/insecure_private_key +remote_user = vagrant +host_key_checking = False \ No newline at end of file diff --git a/kubernetes/kubeadm-vagrant/ansible.yml b/kubernetes/kubeadm-vagrant/ansible.yml new file mode 100644 index 0000000..f35e45d --- /dev/null +++ b/kubernetes/kubeadm-vagrant/ansible.yml @@ -0,0 +1,42 @@ +--- +- hosts: "all" + become: true + remote_user: "vagrant" + gather_facts: false + + tasks: + - name: "Install Python" + raw: "apt-get -y -q install python" + + - name: "Gather facts" + setup: + + - name: "Install Kubernetes APT Key" + apt_key: + url: "https://packages.cloud.google.com/apt/doc/apt-key.gpg" + state: "present" + tags: + - manager + - worker + + - name: "Add Kubernetes repository for Ubuntu Xenial" + apt_repository: + repo: "deb http://apt.kubernetes.io/ kubernetes-xenial main" + state: "present" + update_cache: "yes" + tags: + - manager + - worker + + - name: "Install packages for Docker and Kubernetes" + apt: + name: "{{ item }}" + state: "present" + with_items: + - "docker.io" + - "kubectl" + - "kubelet" + - "kubeadm" + tags: + - manager + - node From 224ea4e35c297a57917c252c0556b9a9cc2ef387 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Wed, 18 Apr 2018 19:18:08 -0600 Subject: [PATCH 4/4] Update kubeadm-Vagrant environment Add Libvirt support to Vagrant data file. Update Vagrantfile to support Libvirt as a provider. Update and expand instructions in README.md. Signed-off-by: Scott Lowe --- kubernetes/kubeadm-vagrant/README.md | 28 +++++++++++++++++++++---- kubernetes/kubeadm-vagrant/Vagrantfile | 10 +++++++++ kubernetes/kubeadm-vagrant/machines.yml | 3 +++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/kubernetes/kubeadm-vagrant/README.md b/kubernetes/kubeadm-vagrant/README.md index d9964d0..2564453 100644 --- a/kubernetes/kubeadm-vagrant/README.md +++ b/kubernetes/kubeadm-vagrant/README.md @@ -4,8 +4,12 @@ These files were created to allow users to use Vagrant ([http://www.vagrantup.co ## Contents +* **ansible.cfg**: This Ansible configuration file configures Ansible for use with this Vagrant environment. No changes to this file should be necessary. + * **machines.yml**: This YAML file contains a list of VM definitions. It is referenced by `Vagrantfile` when Vagrant instantiates the VMs. You will need to edit this file to provide the correct Vagrant box names. Other edits should not be necessary. +* **provision.yml**: This Ansible playbook configures the VMs with the necessary package repositories, and installs the prerequisite packages. No changes to this file should be necessary. + * **README.md**: This file you're currently reading. * **Vagrantfile**: This file is used by Vagrant to spin up the virtual machines. This file is fairly extensively commented to help explain what's happening. You should be able to use this file unchanged; all the VM configuration options are stored outside this file. @@ -14,16 +18,32 @@ These files were created to allow users to use Vagrant ([http://www.vagrantup.co These instructions assume you've already installed your virtualization provider (VMware Fusion/Workstation or VirtualBox), Vagrant, and any necessary plugins (such as the Vagrant VMware plugin). Please refer to the documentation for those products for more information on installation or configuration. -1. Use `vagrant box add` to install an Ubuntu 16.04 base box. Review `machines.yml` for some suggested boxes for both VirtualBox and VMware virtualization platforms. +1. Use `vagrant box add` to install an Ubuntu 16.04 base box. Review `machines.yml` for some suggested boxes for Libvirt, VirtualBox, and the VMware virtualization platforms. -2. Place the files from the `kubernetes/kubeadm-vagrant` directory of this GitHub repository into a directory on your local system. You can clone the entire "learning-tools" repository (using `git clone`) or just download the specific files from the the `kubernetes/kubeadm-vagrant` folder. +2. If the name of your Ubuntu 16.04 base box is _not_ one of the ones listed in `machines.yml`, edit `machines.yml` to supply the correct box name(s). This file has separate lines for each provider; be sure to edit the correct line for your provider ("lv" is for Libvirt, "vb" is for VirtualBox, and "vmw" is for VMware). -3. If the name of your Ubuntu 16.04 base box is _not_ one of the ones listed in `machines.yml`, edit `machines.yml` to supply the correct box name(s). This file has separate lines for the VirtualBox and VMware providers; be sure to edit the correct line for your provider. +3. Place the files from the `kubernetes/kubeadm-vagrant` directory of this GitHub repository into a directory on your local system. You can clone the entire "learning-tools" repository (using `git clone`) or just download the specific files from the the `kubernetes/kubeadm-vagrant` folder. 4. Run `vagrant up` to instantiate 3 VMs---one manager and two nodes. All the VMs will be running Ubuntu 16.04. 5. Once Vagrant is finished, use `vagrant ssh master` to log into the manager VM. -6. +6. While logged into "master", run the following command to start bootstrapping your Kubernetes cluster: + + kubeadm init --apiserver-advertise-address 192.168.100.100 \ + --feature-gates CoreDNS=true DynamicKubeletConfig=true \ + SelfHosting=true --pod-network-cidr 172.24.0.0/16 + +7. When the command completes, follow the steps listed to install a configuration file for `kubelet`. + +8. Copy the final output of the `kubeadm init` command, which includes a token, to the clipboard. You'll need it later. + +9. Install a CNI plugin, such as Calico: + + kubectl apply -f https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/kubeadm/1.7/calico.yaml + +10. Log out and run `vagrant ssh node-01` to log into "node-01". Run the `kubeadm join` command you copied from the output on "master". + +11. Repeat step 10, but on "node-02". Enjoy! diff --git a/kubernetes/kubeadm-vagrant/Vagrantfile b/kubernetes/kubeadm-vagrant/Vagrantfile index 06d02ca..239646b 100644 --- a/kubernetes/kubeadm-vagrant/Vagrantfile +++ b/kubernetes/kubeadm-vagrant/Vagrantfile @@ -64,6 +64,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.cpus = machine['vcpu'] override.vm.box = machine['box']['vb'] end # srv.vm.provider 'virtualbox' + + # Configure CPU & RAM per settings in machines.yml (Libvirt) + srv.vm.provider 'libvirt' do |lv,override| + lv.memory = machine['ram'] + lv.cpus = machine['vcpu'] + override.vm.box = machine['box']['lv'] + if machine['nested'] == true + lv.nested = true + end # if machine['nested'] + end # srv.vm.provider 'libvirt' end # config.vm.define end # machines.each diff --git a/kubernetes/kubeadm-vagrant/machines.yml b/kubernetes/kubeadm-vagrant/machines.yml index 1952257..bfab79f 100644 --- a/kubernetes/kubeadm-vagrant/machines.yml +++ b/kubernetes/kubeadm-vagrant/machines.yml @@ -2,6 +2,7 @@ - box: vmw: "generic/ubuntu1604" vb: "ubuntu/xenial64" + lv: "generic/ubuntu1604" name: "master" nics: - type: "private_network" @@ -11,6 +12,7 @@ - box: vmw: "generic/ubuntu1604" vb: "ubuntu/xenial64" + lv: "generic/ubuntu1604" name: "node-01" nics: - type: "private_network" @@ -20,6 +22,7 @@ - box: vmw: "generic/ubuntu1604" vb: "ubuntu/xenial64" + lv: "generic/ubuntu1604" name: "node-02" nics: - type: "private_network"