From f77e9928b5172d51531891b82e01a2a0e5a23749 Mon Sep 17 00:00:00 2001 From: "Scott S. Lowe" Date: Fri, 21 Oct 2016 12:45:13 -0600 Subject: [PATCH] Update environment Update etcd-backed Docker Swarm environment with Ubuntu and Photon. Switch from shell provisioning to Ansible. Update Vagrantfile to use new techniques pioneered in other environments. Switch away from meta-data/user-data for Photon customization, use Ansible instead. Signed-off-by: Scott S. Lowe --- swarm-etcd2-photon/Vagrantfile | 126 ++++++++++++--------------- swarm-etcd2-photon/ansible.cfg | 5 ++ swarm-etcd2-photon/cfg-docker.yml | 60 +++++++++++++ swarm-etcd2-photon/cfg-etcd.yml | 56 ++++++++++++ swarm-etcd2-photon/docker-tcp.socket | 10 +++ swarm-etcd2-photon/docker.service | 11 +++ swarm-etcd2-photon/docker.socket | 12 +++ swarm-etcd2-photon/etcd.defaults.erb | 11 +-- swarm-etcd2-photon/machines.yml | 55 ++++++++++++ swarm-etcd2-photon/meta-data | 2 - swarm-etcd2-photon/provision.sh | 35 -------- swarm-etcd2-photon/servers.yml | 37 -------- swarm-etcd2-photon/user-data | 57 ------------ 13 files changed, 270 insertions(+), 207 deletions(-) create mode 100644 swarm-etcd2-photon/ansible.cfg create mode 100644 swarm-etcd2-photon/cfg-docker.yml create mode 100644 swarm-etcd2-photon/cfg-etcd.yml create mode 100644 swarm-etcd2-photon/docker-tcp.socket create mode 100644 swarm-etcd2-photon/docker.service create mode 100644 swarm-etcd2-photon/docker.socket create mode 100644 swarm-etcd2-photon/machines.yml delete mode 100644 swarm-etcd2-photon/meta-data delete mode 100644 swarm-etcd2-photon/provision.sh delete mode 100644 swarm-etcd2-photon/servers.yml delete mode 100644 swarm-etcd2-photon/user-data diff --git a/swarm-etcd2-photon/Vagrantfile b/swarm-etcd2-photon/Vagrantfile index fc08168..51fd3e7 100644 --- a/swarm-etcd2-photon/Vagrantfile +++ b/swarm-etcd2-photon/Vagrantfile @@ -1,42 +1,34 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# Specify Vagrant version, Vagrant API version, and Vagrant clone location -# Adjust commented lines to specify provider -Vagrant.require_version ">= 1.6.0" -VAGRANTFILE_API_VERSION = "2" -#ENV['VAGRANT_DEFAULT_PROVIDER'] = 'vmware_fusion' -ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox' -#ENV['VAGRANT_VMWARE_CLONE_DIRECTORY'] = '~/.vagrant' +# Specify minimum Vagrant version and Vagrant API version +Vagrant.require_version '>= 1.6.0' +VAGRANTFILE_API_VERSION = '2' # Require 'yaml', 'fileutils', and 'erb' modules require 'yaml' require 'fileutils' require 'erb' -# Look for user-data file to configure/customize Photon VMs -# No changes should need to be made to this file -USER_DATA = File.join(File.dirname(__FILE__), "user-data") -META_DATA = File.join(File.dirname(__FILE__), "meta-data") - # Read YAML file with VM details (box, CPU, RAM, IP addresses) -# Be sure to edit servers.yml to provide correct IP addresses -servers = YAML.load_file(File.join(File.dirname(__FILE__), 'servers.yml')) +# Be sure to edit machines.yml to provide correct IP addresses +machines = YAML.load_file(File.join(File.dirname(__FILE__), 'machines.yml')) # Use config from YAML file to write out templates for etcd overrides template = File.join(File.dirname(__FILE__), 'etcd.defaults.erb') content = ERB.new File.new(template).read +# Build etcd override files from ERB template etcd_initial_cluster = [] -servers.each do |servers| - if servers["etcd"] == "active" - etcd_initial_cluster << "#{servers['name']}=http://#{servers['priv_ip']}:2380" +machines.each do |machine| + if machine['etcd'] == true + etcd_initial_cluster << "#{machine['name']}=http://#{machine['ip_addr']}:2380" end end -servers.each do |servers| - if servers["etcd"] == "active" - ip = servers['priv_ip'] - target = File.join(File.dirname(__FILE__), "#{servers['name']}.defaults") +machines.each do |machine| + if machine['etcd'] == true + ip = machine['ip_addr'] + target = File.join(File.dirname(__FILE__), "#{machine['name']}.defaults") File.open(target, 'w') { |f| f.write(content.result(binding)) } end end @@ -48,64 +40,56 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.ssh.insert_key = false # Iterate through entries in YAML file to create VMs - servers.each do |servers| - config.vm.define servers["name"] do |srv| + machines.each do |machine| + + # Configure the VMs per details in machines.yml + config.vm.define machine['name'] do |srv| + # Don't check for box updates srv.vm.box_check_update = false - # Set VM hostname and box - srv.vm.hostname = servers["name"] - srv.vm.box = servers["box"] + # Specify the hostname of the VM + srv.vm.hostname = machine['name'] - # Set the VM name for use later - etcdfilename = servers["name"] + ".defaults" + # Specify the Vagrant box to use (use VMware box by default) + srv.vm.box = machine['box']['vmw'] - # Assign an additional static private network - srv.vm.network "private_network", ip: servers["priv_ip"] + # Configure default synced folder (disable by default) + if machine['sync_disabled'] != nil + srv.vm.synced_folder '.', '/vagrant', disabled: machine['sync_disabled'] + else + srv.vm.synced_folder '.', '/vagrant', disabled: true + end #if machine['sync_disabled'] - # Disable default synced folder - srv.vm.synced_folder ".", "/vagrant", disabled: true + # Assign additional private network + if machine['ip_addr'] != nil + srv.vm.network 'private_network', ip: machine['ip_addr'] + end # if machine['ip_addr'] - # Configure VMs based on Photon box - if srv.vm.box == "vmware/photon" - # Make directory on Photon VMs for cloud-init files - srv.vm.provision "shell", inline: "mkdir -p /var/lib/cloud/seed/nocloud", privileged: true + # Configure CPU & RAM per settings in machines.yml (Fusion) + srv.vm.provider 'vmware_fusion' do |vmw| + vmw.vmx['memsize'] = machine['ram'] + vmw.vmx['numvcpus'] = machine['vcpu'] + if machine['nested'] == true + vmw.vmx['vhv.enable'] = 'TRUE' + end #if machine['nested'] + end # srv.vm.provider 'vmware_fusion' - # Copy user_data file into Photon VMs and more to correct location - srv.vm.provision "file", source: "#{USER_DATA}", destination: "/home/vagrant/user-data" - srv.vm.provision "shell", inline: "mv /home/vagrant/user-data /var/lib/cloud/seed/nocloud/", privileged: true + # Configure CPU & RAM per settings in machines.yml (VirtualBox) + srv.vm.provider 'virtualbox' do |vb, override| + vb.memory = machine['ram'] + vb.cpus = machine['vcpu'] + override.vm.box = machine['box']['vb'] + end # srv.vm.provider 'virtualbox' - # Copy meta_data file into Photon VMs and more to correct location - srv.vm.provision "file", source: "#{USER_DATA}", destination: "/home/vagrant/meta-data" - srv.vm.provision "shell", inline: "mv /home/vagrant/meta-data /var/lib/cloud/seed/nocloud/", privileged: true - end # srv.vm.box is vmware/photon - - # Configure VMs based on Ubuntu box - if srv.vm.box == "ubuntu/trusty64" - # Disable default synced folder - srv.vm.synced_folder ".", "/vagrant", disabled: true - - # Copy files into the VMs - srv.vm.provision "file", source: "#{etcdfilename}", destination: "/home/vagrant/#{etcdfilename}" - srv.vm.provision "shell", inline: "mv /home/vagrant/#{etcdfilename} /etc/default/etcd", privileged: true - srv.vm.provision "file", source: "etcd.conf", destination: "/home/vagrant/etcd.conf" - srv.vm.provision "shell", inline: "mv /home/vagrant/etcd.conf /etc/init/etcd.conf", privileged: true - - # Run final provisioning script - srv.vm.provision "shell", path: "provision.sh", privileged: true - end # srv.vm.box is ubuntu/trusty64 - - # Configure VMs with RAM and CPUs per settings in servers.yml - srv.vm.provider :vmware_fusion do |vmw| - vmw.vmx["memsize"] = servers["ram"] - vmw.vmx["numvcpus"] = servers["vcpu"] - end # srv.vm.provider vmware_fusion - - # Configure VirtualBox VMs CPU & RAM per settings in servers.yml - srv.vm.provider :virtualbox do |vb| - vb.memory = servers["ram"] - vb.cpus = servers["vcpu"] - end # srv.vm.provider virtualbox + # Provision VMs with Ansible + srv.vm.provision 'ansible' do |ansible| + if srv.vm.box.include?('ubuntu') + ansible.playbook = 'cfg-etcd.yml' + elsif srv.vm.box.include?('photon') + ansible.playbook = 'cfg-docker.yml' + end # if srv.vm.box + end # srv.vm.provision end # config.vm.define - end # servers each + end # machines.each end # Vagrant.configure diff --git a/swarm-etcd2-photon/ansible.cfg b/swarm-etcd2-photon/ansible.cfg new file mode 100644 index 0000000..b3a4567 --- /dev/null +++ b/swarm-etcd2-photon/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +inventory = ./.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory +private_key_file = ~/.vagrant.d/insecure_private_key +remote_user = vagrant +host_key_checking = False \ No newline at end of file diff --git a/swarm-etcd2-photon/cfg-docker.yml b/swarm-etcd2-photon/cfg-docker.yml new file mode 100644 index 0000000..30d403e --- /dev/null +++ b/swarm-etcd2-photon/cfg-docker.yml @@ -0,0 +1,60 @@ +--- +- hosts: "all" + become: "yes" + remote_user: "vagrant" + + tasks: + - name: "Copy Docker TCP socket file" + copy: + src: "docker-tcp.socket" + dest: "/etc/systemd/system/docker-tcp.socket" + owner: "root" + group: "root" + mode: "0644" + + - name: "Copy Docker socket file" + copy: + src: "docker.socket" + dest: "/lib/systemd/system/docker.socket" + owner: "root" + group: "root" + mode: "0644" + + - name: "Copy Docker TCP socket file" + copy: + src: "docker.service" + dest: "/lib/systemd/system/docker.service" + owner: "root" + group: "root" + mode: "0644" + + - name: "Enable Docker TCP socket" + service: + name: "docker-tcp.socket" + enabled: "yes" + state: "started" + + - name: "Enable Docker socket" + service: + name: "docker.socket" + enabled: "yes" + state: "started" + + - name: "Configure iptables to allow ICMP" + iptables: + chain: "INPUT" + source: "192.168.100.0/24" + protocol: "icmp" + jump: "ACCEPT" + + - name: "Configure iptables to allow Docker traffic" + iptables: + chain: "INPUT" + source: "192.168.100.0/24" + protocol: "tcp" + match: "tcp" + destination_port: "{{ item }}" + jump: "ACCEPT" + with_items: + - "2375" + - "4000" diff --git a/swarm-etcd2-photon/cfg-etcd.yml b/swarm-etcd2-photon/cfg-etcd.yml new file mode 100644 index 0000000..0d98621 --- /dev/null +++ b/swarm-etcd2-photon/cfg-etcd.yml @@ -0,0 +1,56 @@ +--- +- hosts: "all" + become: "yes" + remote_user: "vagrant" + + vars: + etcd_ver: "v2.3.7" + etcd_os: "linux-amd64" + etcd_dir: "etcd-{{ etcd_ver }}-{{ etcd_os }}" + + tasks: + - name: "Create etcd-related directories" + file: + path: "{{ item }}" + mode: 0755 + state: "directory" + with_items: + - "/var/etcd" + - "/var/etcd/dl" + + - name: "Get etcd from GitHub and unarchive" + unarchive: + src: "https://github.com/coreos/etcd/releases/download/{{ etcd_ver }}/{{ etcd_dir }}.tar.gz" + dest: "/var/etcd/dl" + copy: "no" + + - name: "Move files into place" + copy: + src: "/var/etcd/dl/{{ etcd_dir }}/{{ item }}" + dest: "/usr/local/bin/" + remote_src: true + mode: 0755 + with_items: + - "etcd" + - "etcdctl" + + - name: "Copy etcd Upstart configuration file" + copy: + src: "etcd.conf" + dest: "/etc/init/etcd.conf" + owner: "root" + group: "root" + mode: 0644 + + - name: "Copy etcd override file" + copy: + src: "{{ ansible_hostname }}.defaults" + dest: "/etc/default/etcd" + owner: "root" + group: "root" + mode: 0644 + + - name: "Ensure etcd service is running" + service: + name: "etcd" + state: "started" diff --git a/swarm-etcd2-photon/docker-tcp.socket b/swarm-etcd2-photon/docker-tcp.socket new file mode 100644 index 0000000..2d29cca --- /dev/null +++ b/swarm-etcd2-photon/docker-tcp.socket @@ -0,0 +1,10 @@ +[Unit] +Description=Docker TCP Socket for the API + +[Socket] +ListenStream=2375 +BindIPv6Only=both +Service=docker.service + +[Install] +WantedBy=sockets.target diff --git a/swarm-etcd2-photon/docker.service b/swarm-etcd2-photon/docker.service new file mode 100644 index 0000000..de04e7e --- /dev/null +++ b/swarm-etcd2-photon/docker.service @@ -0,0 +1,11 @@ +[Unit] +Description=Docker Daemon + +[Service] +ExecStart=/usr/bin/docker daemon -H fd:// -s overlay +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/swarm-etcd2-photon/docker.socket b/swarm-etcd2-photon/docker.socket new file mode 100644 index 0000000..7dd9509 --- /dev/null +++ b/swarm-etcd2-photon/docker.socket @@ -0,0 +1,12 @@ +[Unit] +Description=Docker Socket for the API +PartOf=docker.service + +[Socket] +ListenStream=/var/run/docker.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker + +[Install] +WantedBy=sockets.target diff --git a/swarm-etcd2-photon/etcd.defaults.erb b/swarm-etcd2-photon/etcd.defaults.erb index 6cc2f89..105a713 100644 --- a/swarm-etcd2-photon/etcd.defaults.erb +++ b/swarm-etcd2-photon/etcd.defaults.erb @@ -1,12 +1,13 @@ # Override file for etcd Upstart script # Generated from etcd.override.erb by Vagrant + # Environment variables export ETCD_INITIAL_CLUSTER="<%= etcd_initial_cluster.join(',') %>" export ETCD_INITIAL_CLUSTER_STATE="new" export ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-01" -export ETCD_INITIAL_ADVERTISE_PEER_URLS="http://<%= servers['priv_ip'] %>:2380" +export ETCD_INITIAL_ADVERTISE_PEER_URLS="http://<%= machine['ip_addr'] %>:2380" export ETCD_DATA_DIR="/var/etcd" -export ETCD_LISTEN_PEER_URLS="http://<%= servers['priv_ip'] %>:2380" -export ETCD_LISTEN_CLIENT_URLS="http://<%= servers['priv_ip'] %>:2379,http://0.0.0.0:4001" -export ETCD_ADVERTISE_CLIENT_URLS="http://<%= servers['priv_ip'] %>:2379" -export ETCD_NAME="<%= servers['name'] %>" +export ETCD_LISTEN_PEER_URLS="http://<%= machine['ip_addr'] %>:2380" +export ETCD_LISTEN_CLIENT_URLS="http://<%= machine['ip_addr'] %>:2379,http://127.0.0.1:2379" +export ETCD_ADVERTISE_CLIENT_URLS="http://<%= machine['ip_addr'] %>:2379" +export ETCD_NAME="<%= machine['name'] %>" diff --git a/swarm-etcd2-photon/machines.yml b/swarm-etcd2-photon/machines.yml new file mode 100644 index 0000000..6c32ebd --- /dev/null +++ b/swarm-etcd2-photon/machines.yml @@ -0,0 +1,55 @@ +--- +- name: "etcd-01" + box: + vmw: "slowe/ubuntu-trusty-x64" + vb: "ubuntu/trusty64" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.101" + etcd: true + playbook: "cfg-etcd.yml" +- name: "etcd-02" + box: + vmw: "slowe/ubuntu-trusty-x64" + vb: "ubuntu/trusty64" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.102" + etcd: true + playbook: "cfg-etcd.yml" +- name: "etcd-03" + box: + vmw: "slowe/ubuntu-trusty-x64" + vb: "ubuntu/trusty64" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.103" + etcd: true + playbook: "cfg-etcd.yml" +- name: "photon-01" + box: + vmw: "vmware/photon" + vb: "vmware/photon" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.111" + etcd: false + playbook: "cfg-docker.yml" +- name: "photon-02" + box: + vmw: "vmware/photon" + vb: "vmware/photon" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.112" + etcd: false + playbook: "cfg-docker.yml" +- name: "photon-03" + box: + vmw: "vmware/photon" + vb: "vmware/photon" + ram: "512" + vcpu: "1" + ip_addr: "192.168.100.113" + etcd: false + playbook: "cfg-docker.yml" diff --git a/swarm-etcd2-photon/meta-data b/swarm-etcd2-photon/meta-data deleted file mode 100644 index 77da86d..0000000 --- a/swarm-etcd2-photon/meta-data +++ /dev/null @@ -1,2 +0,0 @@ -instance-id: iid-local01 -local-hostname: cloudimg diff --git a/swarm-etcd2-photon/provision.sh b/swarm-etcd2-photon/provision.sh deleted file mode 100644 index 14dc2f5..0000000 --- a/swarm-etcd2-photon/provision.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# install curl if needed -if [[ ! -e /usr/bin/curl ]]; then - apt-get update - apt-get -yqq install curl -fi - -if [[ ! -e /usr/local/bin/etcd ]]; then - # Get etcd download - curl -sL https://github.com/coreos/etcd/releases/download/v2.1.2/etcd-v2.1.2-linux-amd64.tar.gz -o etcd-v2.1.2-linux-amd64.tar.gz - - # Expand the download - tar xzvf etcd-v2.1.2-linux-amd64.tar.gz - - # Move etcd and etcdctl to /usr/local/bin - cd etcd-v2.1.2-linux-amd64 - sudo mv etcd /usr/local/bin/ - sudo mv etcdctl /usr/local/bin/ - cd .. - - # Remove etcd download and directory - rm etcd-v2.1.2-linux-amd64.tar.gz - rm -rf etcd-v2.1.2-linux-amd64 - - # Create directories needed by etcd - sudo mkdir -p /var/etcd -fi - -# Copy files into the correct locations; requires shared folders -#sudo cp /vagrant/etcd.conf /etc/init/etcd.conf -#sudo cp /vagrant/$HOSTNAME.defaults /etc/default/etcd - -# restart if already running, otherwise start. -initctl status etcd && initctl restart etcd || initctl start etcd diff --git a/swarm-etcd2-photon/servers.yml b/swarm-etcd2-photon/servers.yml deleted file mode 100644 index 132e958..0000000 --- a/swarm-etcd2-photon/servers.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: etcd-01 - box: ubuntu/trusty64 - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.101 - etcd: active -- name: etcd-02 - box: ubuntu/trusty64 - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.102 - etcd: active -- name: etcd-03 - box: ubuntu/trusty64 - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.103 - etcd: active -- name: photon-01 - box: vmware/photon - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.104 - etcd: inactive -- name: photon-02 - box: vmware/photon - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.105 - etcd: inactive -- name: photon-03 - box: vmware/photon - ram: "512" - vcpu: "1" - priv_ip: 192.168.100.106 - etcd: inactive diff --git a/swarm-etcd2-photon/user-data b/swarm-etcd2-photon/user-data deleted file mode 100644 index 7dc4dbe..0000000 --- a/swarm-etcd2-photon/user-data +++ /dev/null @@ -1,57 +0,0 @@ -#cloud-config - -groups: - - docker: [root,vagrant] -write_files: - - content: | - [Unit] - Description=Docker TCP Socket for the API - - [Socket] - ListenStream=2375 - BindIPv6Only=both - Service=docker.service - - [Install] - WantedBy=sockets.target - path: /etc/systemd/system/docker-tcp.socket - owner: root:root - permissions: '0644' - - content: | - [Unit] - Description=Docker Socket for the API - PartOf=docker.service - - [Socket] - ListenStream=/var/run/docker.sock - SocketMode=0660 - SocketUser=root - SocketGroup=docker - - [Install] - WantedBy=sockets.target - path: /lib/systemd/system/docker.socket - owner: root:root - permissions: '0644' - - content: | - [Unit] - Description=Docker Daemon - - [Service] - ExecStart=/bin/docker -d -H fd:// -s overlay - ExecReload=/bin/kill -HUP $MAINPID - KillMode=process - Restart=always - - [Install] - WantedBy=multi-user.target - path: /lib/systemd/system/docker.service - owner: root:root - permissions: '0644' -runcmd: - - systemctl daemon-reload - - systemctl enable docker.socket - - systemctl enable docker-tcp.socket - - systemctl stop docker.service - - systemctl start docker-tcp.socket - - systemctl start docker.socket