Update environment

Update etcd-backed Docker Swarm environment with Ubuntu and Photon. Switch from shell provisioning to Ansible. Update Vagrantfile to use new techniques pioneered in other environments. Switch away from meta-data/user-data for Photon customization, use Ansible instead.

Signed-off-by: Scott S. Lowe <scott.lowe@scottlowe.org>

swarm-etcd2-photon/Vagrantfile

@@ -1,42 +1,34 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-# Specify Vagrant version, Vagrant API version, and Vagrant clone location
-# Adjust commented lines to specify provider
-Vagrant.require_version ">= 1.6.0"
-VAGRANTFILE_API_VERSION = "2"
-#ENV['VAGRANT_DEFAULT_PROVIDER'] = 'vmware_fusion'
-ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox'
-#ENV['VAGRANT_VMWARE_CLONE_DIRECTORY'] = '~/.vagrant'
+# Specify minimum Vagrant version and Vagrant API version
+Vagrant.require_version '>= 1.6.0'
+VAGRANTFILE_API_VERSION = '2'
 
 # Require 'yaml', 'fileutils', and 'erb' modules
 require 'yaml'
 require 'fileutils'
 require 'erb'
 
-# Look for user-data file to configure/customize Photon VMs
-# No changes should need to be made to this file
-USER_DATA = File.join(File.dirname(__FILE__), "user-data")
-META_DATA = File.join(File.dirname(__FILE__), "meta-data")
-
 # Read YAML file with VM details (box, CPU, RAM, IP addresses)
-# Be sure to edit servers.yml to provide correct IP addresses
-servers = YAML.load_file(File.join(File.dirname(__FILE__), 'servers.yml'))
+# Be sure to edit machines.yml to provide correct IP addresses
+machines = YAML.load_file(File.join(File.dirname(__FILE__), 'machines.yml'))
 
 # Use config from YAML file to write out templates for etcd overrides
 template = File.join(File.dirname(__FILE__), 'etcd.defaults.erb')
 content = ERB.new File.new(template).read
 
+# Build etcd override files from ERB template
 etcd_initial_cluster = []
-servers.each do |servers|
-  if servers["etcd"] == "active"
-    etcd_initial_cluster << "#{servers['name']}=http://#{servers['priv_ip']}:2380"
+machines.each do |machine|
+  if machine['etcd'] == true
+    etcd_initial_cluster << "#{machine['name']}=http://#{machine['ip_addr']}:2380"
   end
 end
-servers.each do |servers|
-  if servers["etcd"] == "active"
-    ip = servers['priv_ip']
-    target = File.join(File.dirname(__FILE__), "#{servers['name']}.defaults")
+machines.each do |machine|
+  if machine['etcd'] == true
+    ip = machine['ip_addr']
+    target = File.join(File.dirname(__FILE__), "#{machine['name']}.defaults")
     File.open(target, 'w') { |f| f.write(content.result(binding)) }
   end
 end
@@ -48,64 +40,56 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.ssh.insert_key = false
 
   # Iterate through entries in YAML file to create VMs
-  servers.each do |servers|
-    config.vm.define servers["name"] do |srv|
+  machines.each do |machine|
+
+    # Configure the VMs per details in machines.yml
+    config.vm.define machine['name'] do |srv|
+
       # Don't check for box updates
       srv.vm.box_check_update = false
 
-      # Set VM hostname and box
-      srv.vm.hostname = servers["name"]
-      srv.vm.box = servers["box"]
+      # Specify the hostname of the VM
+      srv.vm.hostname = machine['name']
 
-      # Set the VM name for use later
-      etcdfilename = servers["name"] + ".defaults"
+      # Specify the Vagrant box to use (use VMware box by default)
+      srv.vm.box = machine['box']['vmw']
 
-      # Assign an additional static private network
-      srv.vm.network "private_network", ip: servers["priv_ip"]
+      # Configure default synced folder (disable by default)
+      if machine['sync_disabled'] != nil
+        srv.vm.synced_folder '.', '/vagrant', disabled: machine['sync_disabled']
+      else
+        srv.vm.synced_folder '.', '/vagrant', disabled: true
+      end #if machine['sync_disabled']
 
-      # Disable default synced folder
-      srv.vm.synced_folder ".", "/vagrant", disabled: true
+      # Assign additional private network
+      if machine['ip_addr'] != nil
+        srv.vm.network 'private_network', ip: machine['ip_addr']
+      end # if machine['ip_addr']
 
-      # Configure VMs based on Photon box
-      if srv.vm.box == "vmware/photon"
-        # Make directory on Photon VMs for cloud-init files
-        srv.vm.provision "shell", inline: "mkdir -p /var/lib/cloud/seed/nocloud", privileged: true
+      # Configure CPU & RAM per settings in machines.yml (Fusion)
+      srv.vm.provider 'vmware_fusion' do |vmw|
+        vmw.vmx['memsize'] = machine['ram']
+        vmw.vmx['numvcpus'] = machine['vcpu']
+        if machine['nested'] == true
+          vmw.vmx['vhv.enable'] = 'TRUE'
+        end #if machine['nested']
+      end # srv.vm.provider 'vmware_fusion'
 
-        # Copy user_data file into Photon VMs and more to correct location
-        srv.vm.provision "file", source: "#{USER_DATA}", destination: "/home/vagrant/user-data"
-        srv.vm.provision "shell", inline: "mv /home/vagrant/user-data /var/lib/cloud/seed/nocloud/", privileged: true
+      # Configure CPU & RAM per settings in machines.yml (VirtualBox)
+      srv.vm.provider 'virtualbox' do |vb, override|
+        vb.memory = machine['ram']
+        vb.cpus = machine['vcpu']
+        override.vm.box = machine['box']['vb']
+      end # srv.vm.provider 'virtualbox'
 
-        # Copy meta_data file into Photon VMs and more to correct location
-        srv.vm.provision "file", source: "#{USER_DATA}", destination: "/home/vagrant/meta-data"
-        srv.vm.provision "shell", inline: "mv /home/vagrant/meta-data /var/lib/cloud/seed/nocloud/", privileged: true
-      end # srv.vm.box is vmware/photon
-
-      # Configure VMs based on Ubuntu box
-      if srv.vm.box == "ubuntu/trusty64"
-        # Disable default synced folder
-        srv.vm.synced_folder ".", "/vagrant", disabled: true
-
-        # Copy files into the VMs
-        srv.vm.provision "file", source: "#{etcdfilename}", destination: "/home/vagrant/#{etcdfilename}"
-        srv.vm.provision "shell", inline: "mv /home/vagrant/#{etcdfilename} /etc/default/etcd", privileged: true
-        srv.vm.provision "file", source: "etcd.conf", destination: "/home/vagrant/etcd.conf"
-        srv.vm.provision "shell", inline: "mv /home/vagrant/etcd.conf /etc/init/etcd.conf", privileged: true
-
-        # Run final provisioning script
-        srv.vm.provision "shell", path: "provision.sh", privileged: true
-      end # srv.vm.box is ubuntu/trusty64
-
-      # Configure VMs with RAM and CPUs per settings in servers.yml
-      srv.vm.provider :vmware_fusion do |vmw|
-        vmw.vmx["memsize"] = servers["ram"]
-        vmw.vmx["numvcpus"] = servers["vcpu"]
-      end # srv.vm.provider vmware_fusion
-
-      # Configure VirtualBox VMs CPU & RAM per settings in servers.yml
-      srv.vm.provider :virtualbox do |vb|
-        vb.memory = servers["ram"]
-        vb.cpus = servers["vcpu"]
-      end # srv.vm.provider virtualbox
+      # Provision VMs with Ansible
+      srv.vm.provision 'ansible' do |ansible|
+        if srv.vm.box.include?('ubuntu')
+          ansible.playbook = 'cfg-etcd.yml'
+        elsif srv.vm.box.include?('photon')
+          ansible.playbook = 'cfg-docker.yml'
+        end # if srv.vm.box
+      end # srv.vm.provision
     end # config.vm.define
-  end # servers each
+  end # machines.each
 end # Vagrant.configure

swarm-etcd2-photon/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+inventory = ./.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory
+private_key_file = ~/.vagrant.d/insecure_private_key
+remote_user = vagrant
+host_key_checking = False

swarm-etcd2-photon/cfg-docker.yml

@@ -0,0 +1,60 @@
+---
+- hosts: "all"
+  become: "yes"
+  remote_user: "vagrant"
+
+  tasks:
+  - name: "Copy Docker TCP socket file"
+    copy:
+      src: "docker-tcp.socket"
+      dest: "/etc/systemd/system/docker-tcp.socket"
+      owner: "root"
+      group: "root"
+      mode: "0644"
+
+  - name: "Copy Docker socket file"
+    copy:
+      src: "docker.socket"
+      dest: "/lib/systemd/system/docker.socket"
+      owner: "root"
+      group: "root"
+      mode: "0644"
+
+  - name: "Copy Docker TCP socket file"
+    copy:
+      src: "docker.service"
+      dest: "/lib/systemd/system/docker.service"
+      owner: "root"
+      group: "root"
+      mode: "0644"
+
+  - name: "Enable Docker TCP socket"
+    service:
+      name: "docker-tcp.socket"
+      enabled: "yes"
+      state: "started"
+
+  - name: "Enable Docker socket"
+    service:
+      name: "docker.socket"
+      enabled: "yes"
+      state: "started"
+
+  - name: "Configure iptables to allow ICMP"
+    iptables:
+      chain: "INPUT"
+      source: "192.168.100.0/24"
+      protocol: "icmp"
+      jump: "ACCEPT"
+
+  - name: "Configure iptables to allow Docker traffic"
+    iptables:
+      chain: "INPUT"
+      source: "192.168.100.0/24"
+      protocol: "tcp"
+      match: "tcp"
+      destination_port: "{{ item }}"
+      jump: "ACCEPT"
+    with_items:
+      - "2375"
+      - "4000"

swarm-etcd2-photon/cfg-etcd.yml

@@ -0,0 +1,56 @@
+---
+- hosts: "all"
+  become: "yes"
+  remote_user: "vagrant"
+
+  vars:
+    etcd_ver: "v2.3.7"
+    etcd_os: "linux-amd64"
+    etcd_dir: "etcd-{{ etcd_ver }}-{{ etcd_os }}"
+
+  tasks:
+  - name: "Create etcd-related directories"
+    file:
+      path: "{{ item }}"
+      mode: 0755
+      state: "directory"
+    with_items:
+      - "/var/etcd"
+      - "/var/etcd/dl"
+
+  - name: "Get etcd from GitHub and unarchive"
+    unarchive:
+      src: "https://github.com/coreos/etcd/releases/download/{{ etcd_ver }}/{{ etcd_dir }}.tar.gz"
+      dest: "/var/etcd/dl"
+      copy: "no"
+
+  - name: "Move files into place"
+    copy:
+      src: "/var/etcd/dl/{{ etcd_dir }}/{{ item }}"
+      dest: "/usr/local/bin/"
+      remote_src: true
+      mode: 0755
+    with_items:
+      - "etcd"
+      - "etcdctl"
+
+  - name: "Copy etcd Upstart configuration file"
+    copy:
+      src: "etcd.conf"
+      dest: "/etc/init/etcd.conf"
+      owner: "root"
+      group: "root"
+      mode: 0644
+
+  - name: "Copy etcd override file"
+    copy:
+      src: "{{ ansible_hostname }}.defaults"
+      dest: "/etc/default/etcd"
+      owner: "root"
+      group: "root"
+      mode: 0644
+
+  - name: "Ensure etcd service is running"
+    service:
+      name: "etcd"
+      state: "started"

swarm-etcd2-photon/docker-tcp.socket

@@ -0,0 +1,10 @@
+[Unit]
+Description=Docker TCP Socket for the API
+
+[Socket]
+ListenStream=2375
+BindIPv6Only=both
+Service=docker.service
+
+[Install]
+WantedBy=sockets.target

swarm-etcd2-photon/docker.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Docker Daemon
+
+[Service]
+ExecStart=/usr/bin/docker daemon -H fd:// -s overlay
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

swarm-etcd2-photon/docker.socket

@@ -0,0 +1,12 @@
+[Unit]
+Description=Docker Socket for the API
+PartOf=docker.service
+
+[Socket]
+ListenStream=/var/run/docker.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target

swarm-etcd2-photon/etcd.defaults.erb

@@ -1,12 +1,13 @@
 # Override file for etcd Upstart script
 # Generated from etcd.override.erb by Vagrant
+
 # Environment variables
 export ETCD_INITIAL_CLUSTER="<%= etcd_initial_cluster.join(',') %>"
 export ETCD_INITIAL_CLUSTER_STATE="new"
 export ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-01"
-export ETCD_INITIAL_ADVERTISE_PEER_URLS="http://<%= servers['priv_ip'] %>:2380"
+export ETCD_INITIAL_ADVERTISE_PEER_URLS="http://<%= machine['ip_addr'] %>:2380"
 export ETCD_DATA_DIR="/var/etcd"
-export ETCD_LISTEN_PEER_URLS="http://<%= servers['priv_ip'] %>:2380"
-export ETCD_LISTEN_CLIENT_URLS="http://<%= servers['priv_ip'] %>:2379,http://0.0.0.0:4001"
-export ETCD_ADVERTISE_CLIENT_URLS="http://<%= servers['priv_ip'] %>:2379"
-export ETCD_NAME="<%= servers['name'] %>"
+export ETCD_LISTEN_PEER_URLS="http://<%= machine['ip_addr'] %>:2380"
+export ETCD_LISTEN_CLIENT_URLS="http://<%= machine['ip_addr'] %>:2379,http://127.0.0.1:2379"
+export ETCD_ADVERTISE_CLIENT_URLS="http://<%= machine['ip_addr'] %>:2379"
+export ETCD_NAME="<%= machine['name'] %>"

swarm-etcd2-photon/machines.yml

@@ -0,0 +1,55 @@
+---
+- name: "etcd-01"
+  box:
+    vmw: "slowe/ubuntu-trusty-x64"
+    vb: "ubuntu/trusty64"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.101"
+  etcd: true
+  playbook: "cfg-etcd.yml"
+- name: "etcd-02"
+  box:
+    vmw: "slowe/ubuntu-trusty-x64"
+    vb: "ubuntu/trusty64"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.102"
+  etcd: true
+  playbook: "cfg-etcd.yml"
+- name: "etcd-03"
+  box:
+    vmw: "slowe/ubuntu-trusty-x64"
+    vb: "ubuntu/trusty64"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.103"
+  etcd: true
+  playbook: "cfg-etcd.yml"
+- name: "photon-01"
+  box:
+    vmw: "vmware/photon"
+    vb: "vmware/photon"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.111"
+  etcd: false
+  playbook: "cfg-docker.yml"
+- name: "photon-02"
+  box:
+    vmw: "vmware/photon"
+    vb: "vmware/photon"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.112"
+  etcd: false
+  playbook: "cfg-docker.yml"
+- name: "photon-03"
+  box:
+    vmw: "vmware/photon"
+    vb: "vmware/photon"
+  ram: "512"
+  vcpu: "1"
+  ip_addr: "192.168.100.113"
+  etcd: false
+  playbook: "cfg-docker.yml"

swarm-etcd2-photon/meta-data

@@ -1,2 +0,0 @@
-instance-id: iid-local01
-local-hostname: cloudimg

swarm-etcd2-photon/provision.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# install curl if needed
-if [[ ! -e /usr/bin/curl ]]; then
-  apt-get update
-  apt-get -yqq install curl
-fi
-
-if [[ ! -e /usr/local/bin/etcd ]]; then
-  # Get etcd download
-  curl -sL  https://github.com/coreos/etcd/releases/download/v2.1.2/etcd-v2.1.2-linux-amd64.tar.gz -o etcd-v2.1.2-linux-amd64.tar.gz
-
-  # Expand the download
-  tar xzvf etcd-v2.1.2-linux-amd64.tar.gz
-
-  # Move etcd and etcdctl to /usr/local/bin
-  cd etcd-v2.1.2-linux-amd64
-  sudo mv etcd /usr/local/bin/
-  sudo mv etcdctl /usr/local/bin/
-  cd ..
-
-  # Remove etcd download and directory
-  rm etcd-v2.1.2-linux-amd64.tar.gz
-  rm -rf etcd-v2.1.2-linux-amd64
-
-  # Create directories needed by etcd
-  sudo mkdir -p /var/etcd
-fi
-
-# Copy files into the correct locations; requires shared folders
-#sudo cp /vagrant/etcd.conf /etc/init/etcd.conf
-#sudo cp /vagrant/$HOSTNAME.defaults /etc/default/etcd
-
-# restart if already running, otherwise start.
-initctl status etcd && initctl restart etcd || initctl start etcd

swarm-etcd2-photon/servers.yml

@@ -1,37 +0,0 @@
----
-- name: etcd-01
-  box: ubuntu/trusty64
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.101
-  etcd: active
-- name: etcd-02
-  box: ubuntu/trusty64
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.102
-  etcd: active
-- name: etcd-03
-  box: ubuntu/trusty64
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.103
-  etcd: active
-- name: photon-01
-  box: vmware/photon
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.104
-  etcd: inactive
-- name: photon-02
-  box: vmware/photon
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.105
-  etcd: inactive
-- name: photon-03
-  box: vmware/photon
-  ram: "512"
-  vcpu: "1"
-  priv_ip: 192.168.100.106
-  etcd: inactive

swarm-etcd2-photon/user-data

@@ -1,57 +0,0 @@
-#cloud-config
-
-groups:
-  - docker: [root,vagrant]
-write_files:
-  - content: |
-      [Unit]
-      Description=Docker TCP Socket for the API
-
-      [Socket]
-      ListenStream=2375
-      BindIPv6Only=both
-      Service=docker.service
-
-      [Install]
-      WantedBy=sockets.target
-    path: /etc/systemd/system/docker-tcp.socket
-    owner: root:root
-    permissions: '0644'
-  - content: |
-      [Unit]
-      Description=Docker Socket for the API
-      PartOf=docker.service
-
-      [Socket]
-      ListenStream=/var/run/docker.sock
-      SocketMode=0660
-      SocketUser=root
-      SocketGroup=docker
-
-      [Install]
-      WantedBy=sockets.target
-    path: /lib/systemd/system/docker.socket
-    owner: root:root
-    permissions: '0644'
-  - content: |
-      [Unit]
-      Description=Docker Daemon
-
-      [Service]
-      ExecStart=/bin/docker -d -H fd:// -s overlay
-      ExecReload=/bin/kill -HUP $MAINPID
-      KillMode=process
-      Restart=always
-
-      [Install]
-      WantedBy=multi-user.target
-    path: /lib/systemd/system/docker.service
-    owner: root:root
-    permissions: '0644'
-runcmd:
-  - systemctl daemon-reload
-  - systemctl enable docker.socket
-  - systemctl enable docker-tcp.socket
-  - systemctl stop docker.service
-  - systemctl start docker-tcp.socket
-  - systemctl start docker.socket