From c817ef7e6f2cd08a68f24f06f60383e811efa7f3 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Fri, 29 Jan 2016 14:30:44 -0700 Subject: [PATCH 1/3] Initial commit for OVS-Geneve environment Add files for a OVS-Geneve learning environment, including YAML data file, Ansible configuration and playbook, Libvirt network definition in XML, and Vagrantfile. --- ovs-geneve/README.md | 31 ++++++++++++++++ ovs-geneve/Vagrantfile | 76 ++++++++++++++++++++++++++++++++++++++++ ovs-geneve/ansible.cfg | 5 +++ ovs-geneve/machines.yml | 13 +++++++ ovs-geneve/provision.yml | 18 ++++++++++ ovs-geneve/vxlan.xml | 6 ++++ 6 files changed, 149 insertions(+) create mode 100644 ovs-geneve/README.md create mode 100644 ovs-geneve/Vagrantfile create mode 100644 ovs-geneve/ansible.cfg create mode 100644 ovs-geneve/machines.yml create mode 100644 ovs-geneve/provision.yml create mode 100644 ovs-geneve/vxlan.xml diff --git a/ovs-geneve/README.md b/ovs-geneve/README.md new file mode 100644 index 0000000..abe944a --- /dev/null +++ b/ovs-geneve/README.md @@ -0,0 +1,31 @@ +# Virtualization with KVM + +These files were created to allow users to use Vagrant ([http://www.vagrantup.com](http://www.vagrantup.com)) and Ansible ([http://www.ansible.com](http://www.ansible.com)) to quickly and relatively easily experiment with KVM on Ubuntu 14.04 LTS. This configuration was tested using Vagrant 1.8.1, VMware Fusion 8.1.0, the Vagrant VMware plugin, and Ansible 1.9.1. Other versions of these products are likely to work, but haven't been tested. + +## Contents + +* **ansible.cfg**: This Ansible configuration file instructs Ansible to use an inventory file named `hosts` in the current directory, to use the default vagrant user (`vagrant`) and default insecure Vagrant SSH private key, and to use the Vagrant-generated Ansible inventory file (found, by default, in `./.vagrant/provisioners/inventory/vagrant_ansible_inventory`). + +* **machines.yml**: This YAML file contains a list of VM definitions and associated configuration data. It is referenced by `Vagrantfile` when Vagrant instantiates the VMs. Generally the only change this file needs is to ensure the "box" value correctly references the Vagrant box you're using. The "ip_addr" value is also required in order to build the Ansible inventory file; you may want to edit this to a value appropriate for your environment. + +* **provision.yml**: This YAML file is an Ansible playbook that configures the VMs created by Vagrant when they are first provisioned. Ansible is called automatically by Vagrant; you do not need to invoke Ansible separately. No changes are needed to this file. + +* **README.md**: The file you're currently reading. + +* **Vagrantfile**: This file is used by Vagrant to spin up the virtual machines. This file is fairly extensively commented to help explain what's happening. You should be able to use this file unchanged; all the VM configuration options are stored in `machines.yml`. + +## Instructions + +These instructions assume you've already installed VMware Fusion, Vagrant, the Vagrant VMware plugin, and Ansible. Please refer to the documentation for those products for more information on installation or configuration. + +1. Use `vagrant box add` to install an Ubuntu 14.04 x64 box for the "vmware_fusion" provider. I have a base box you can use for this purpose; to use my Ubuntu 14.04 x64 base box, add the box with `vagrant box add slowe/ubuntu-trusty-x64`. (In theory you should be able to use this Vagrant environment with VMware Workstation as well, but only VMware Fusion was tested.) + +2. Copy the files from the `kvm` directory of this repository (the "learning-tools" repository) to a directory on your system. You can clone the entire "learning-tools" repository (using `git clone`), or just download the specific files from the `kvm` directory. + +3. Edit `machines.yml` to ensure that the box specified in that file matches the Ubuntu 14.04 x64 base box you just installed and will be using. _I recommend that you do not change any other values in this file unless you know it is necessary._ + +4. From a terminal window, change into the directory where the files from this directory are stored and run `vagrant up` to bring up the VMs according to the instructions in `machines.yml` and `Vagrantfile`. (By default, it will create and power on only a single VM.) + +5. Once Vagrant has finished creating, booting, and provisioning the VM (note you'll need Internet access for this step), log into the VM (named "kvm-01" by default) using `vagrant ssh`. + +You can now use this VM to do any testing or experimentation with KVM and Libvirt. diff --git a/ovs-geneve/Vagrantfile b/ovs-geneve/Vagrantfile new file mode 100644 index 0000000..6f689c1 --- /dev/null +++ b/ovs-geneve/Vagrantfile @@ -0,0 +1,76 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Specify Vagrant version, Vagrant API version, and desired clone location +Vagrant.require_version '>= 1.6.0' +VAGRANTFILE_API_VERSION = '2' +ENV['VAGRANT_DEFAULT_PROVIDER'] = 'vmware_fusion' +ENV['VAGRANT_VMWARE_CLONE_DIRECTORY'] = '~/.vagrant' +#ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox' +#ENV['VAGRANT_DEFAULT_PROVIDER'] = 'vmware_appcatalyst' + +# Require 'yaml' module +require 'yaml' + +# Read YAML file with VM details (box, CPU, and RAM) +machines = YAML.load_file(File.join(File.dirname(__FILE__), 'machines.yml')) + +# Create and configure the VMs +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + # Always use Vagrant's default insecure key + config.ssh.insert_key = false + + # Iterate through entries in YAML file to create VMs + machines.each do |machine| + config.vm.define machine['name'] do |srv| + + # Don't check for box updates + srv.vm.box_check_update = false + srv.vm.hostname = machine['name'] + srv.vm.box = machine['box'] + + # Configure default synced folder (disable by default) + if machine['sync_disabled'] != nil + srv.vm.synced_folder '.', '/vagrant', disabled: machine['sync_disabled'] + else + srv.vm.synced_folder '.', '/vagrant', disabled: true + end #if machine['sync_disabled'] + + # Assign additional private network + if machine['ip_addr'] != nil + srv.vm.network 'private_network', ip: machine['ip_addr'] + end # if machine['ip_addr'] + + # Configure CPU & RAM per settings in machines.yml (Fusion) + srv.vm.provider :vmware_fusion do |vmw| + vmw.vmx['memsize'] = machine['ram'] + vmw.vmx['numvcpus'] = machine['vcpu'] + if machine['nested'] == true + vmw.vmx['vhv.enable'] = 'TRUE' + end #if machine['nested'] + end # srv.vm.provider vmware_fusion + + # Configure CPU & RAM per settings in machines.yml (AppCatalyst) + srv.vm.provider :vmware_appcatalyst do |vmw| + vmw.vmx['memsize'] = machine['ram'] + vmw.vmx['numvcpus'] = machine['vcpu'] + #vmw.vmx['guestos'] = 'other3xlinux-64' + if machine['nested'] == true + vmw.vmx['vhv.enable'] = 'TRUE' + end #if machine['nested'] + end # srv.vm.provider vmware_appcatalyst + + # Configure CPU & RAM per settings in machines.yml (VirtualBox) + srv.vm.provider :virtualbox do |vb| + vb.memory = machine['ram'] + vb.cpus = machine['vcpu'] + end # srv.vm.provider virtualbox + end # config.vm.define + + # Provision the VM with Ansible + config.vm.provision 'ansible' do |ansible| + ansible.playbook = 'provision.yml' + end # config.vm.provision + end # machines.each +end # Vagrant.configure diff --git a/ovs-geneve/ansible.cfg b/ovs-geneve/ansible.cfg new file mode 100644 index 0000000..b3a4567 --- /dev/null +++ b/ovs-geneve/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +inventory = ./.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory +private_key_file = ~/.vagrant.d/insecure_private_key +remote_user = vagrant +host_key_checking = False \ No newline at end of file diff --git a/ovs-geneve/machines.yml b/ovs-geneve/machines.yml new file mode 100644 index 0000000..d683597 --- /dev/null +++ b/ovs-geneve/machines.yml @@ -0,0 +1,13 @@ +--- +- name: kvm-01 + box: slowe/ubuntu-trusty-x64 + ram: 1024 + vcpu: 1 + ip_addr: 192.168.100.100 + nested: true +- name: kvm-02 + box: slowe/ubuntu-trusty-x64 + ram: 1024 + vcpu: 1 + ip_addr: 192.168.100.101 + nested: true diff --git a/ovs-geneve/provision.yml b/ovs-geneve/provision.yml new file mode 100644 index 0000000..88239d7 --- /dev/null +++ b/ovs-geneve/provision.yml @@ -0,0 +1,18 @@ +--- +- hosts: "all" + sudo: "yes" + remote_user: "vagrant" + + tasks: + - name: "Install KVM and Libvirt" + apt: + state: "present" + update_cache: "yes" + name: "{{ item }}" + with_items: + - qemu-kvm + - libvirt-bin + - git + - autoconf + - libtool + - virtinst diff --git a/ovs-geneve/vxlan.xml b/ovs-geneve/vxlan.xml new file mode 100644 index 0000000..826f3d3 --- /dev/null +++ b/ovs-geneve/vxlan.xml @@ -0,0 +1,6 @@ + + vxlan-net + + + + From 793bd7889f148b2053934a2c1c24f03a67ad8be3 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Tue, 9 Feb 2016 14:41:46 -0700 Subject: [PATCH 2/3] Continue work on OVS+Geneve tunneling environment Update Ansible playbook to install newer kernel (required for Geneve support). Rename vxlan.xml to geneve-net.xml and update contents for Libvirt network definition. --- ovs-geneve/{vxlan.xml => geneve-net.xml} | 2 +- ovs-geneve/provision.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) rename ovs-geneve/{vxlan.xml => geneve-net.xml} (80%) diff --git a/ovs-geneve/vxlan.xml b/ovs-geneve/geneve-net.xml similarity index 80% rename from ovs-geneve/vxlan.xml rename to ovs-geneve/geneve-net.xml index 826f3d3..c06f652 100644 --- a/ovs-geneve/vxlan.xml +++ b/ovs-geneve/geneve-net.xml @@ -1,5 +1,5 @@ - vxlan-net + geneve-net diff --git a/ovs-geneve/provision.yml b/ovs-geneve/provision.yml index 88239d7..b3e3a2d 100644 --- a/ovs-geneve/provision.yml +++ b/ovs-geneve/provision.yml @@ -16,3 +16,6 @@ - autoconf - libtool - virtinst + - linux-headers-3.19.0-49-generic + - linux-image-3.19.0-49-generic + - linux-image-extra-3.19.0-49-generic From 6be177233124a4ba6d90e95762ef2cd0d70c5ac2 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Fri, 12 Feb 2016 19:35:51 -0700 Subject: [PATCH 3/3] Update environment for Geneve tunneling with OVS Add Bash shell script to help with namespace setup. Add sections to Ansible playbook for cloning Git repo and inserting Bash shell script for namespace setup. Update README.md with final instructions. --- ovs-geneve/README.md | 46 ++++++++++++++++++++++++++++++++++----- ovs-geneve/Vagrantfile | 10 ++++----- ovs-geneve/geneve-net.xml | 6 ----- ovs-geneve/provision.yml | 18 ++++++++++++++- ovs-geneve/setup.sh | 46 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 17 deletions(-) delete mode 100644 ovs-geneve/geneve-net.xml create mode 100644 ovs-geneve/setup.sh diff --git a/ovs-geneve/README.md b/ovs-geneve/README.md index abe944a..52186ca 100644 --- a/ovs-geneve/README.md +++ b/ovs-geneve/README.md @@ -1,4 +1,4 @@ -# Virtualization with KVM +# Using Geneve Tunnels with OVS These files were created to allow users to use Vagrant ([http://www.vagrantup.com](http://www.vagrantup.com)) and Ansible ([http://www.ansible.com](http://www.ansible.com)) to quickly and relatively easily experiment with KVM on Ubuntu 14.04 LTS. This configuration was tested using Vagrant 1.8.1, VMware Fusion 8.1.0, the Vagrant VMware plugin, and Ansible 1.9.1. Other versions of these products are likely to work, but haven't been tested. @@ -6,10 +6,14 @@ These files were created to allow users to use Vagrant ([http://www.vagrantup.co * **ansible.cfg**: This Ansible configuration file instructs Ansible to use an inventory file named `hosts` in the current directory, to use the default vagrant user (`vagrant`) and default insecure Vagrant SSH private key, and to use the Vagrant-generated Ansible inventory file (found, by default, in `./.vagrant/provisioners/inventory/vagrant_ansible_inventory`). +* **geneve-net.xml**: This is a snippet of Libvirt network XML that is used by Ansible when provisioning the VMs. No changes are needed to this file. + * **machines.yml**: This YAML file contains a list of VM definitions and associated configuration data. It is referenced by `Vagrantfile` when Vagrant instantiates the VMs. Generally the only change this file needs is to ensure the "box" value correctly references the Vagrant box you're using. The "ip_addr" value is also required in order to build the Ansible inventory file; you may want to edit this to a value appropriate for your environment. * **provision.yml**: This YAML file is an Ansible playbook that configures the VMs created by Vagrant when they are first provisioned. Ansible is called automatically by Vagrant; you do not need to invoke Ansible separately. No changes are needed to this file. +* **setup.sh**: This Bash shell script prepares the VMs to test connectivity between a network namespace on each host over the Geneve tunnel. No changes should be necessary to this file; Ansible copies the file into the VMs during provisioning. + * **README.md**: The file you're currently reading. * **Vagrantfile**: This file is used by Vagrant to spin up the virtual machines. This file is fairly extensively commented to help explain what's happening. You should be able to use this file unchanged; all the VM configuration options are stored in `machines.yml`. @@ -20,12 +24,44 @@ These instructions assume you've already installed VMware Fusion, Vagrant, the V 1. Use `vagrant box add` to install an Ubuntu 14.04 x64 box for the "vmware_fusion" provider. I have a base box you can use for this purpose; to use my Ubuntu 14.04 x64 base box, add the box with `vagrant box add slowe/ubuntu-trusty-x64`. (In theory you should be able to use this Vagrant environment with VMware Workstation as well, but only VMware Fusion was tested.) -2. Copy the files from the `kvm` directory of this repository (the "learning-tools" repository) to a directory on your system. You can clone the entire "learning-tools" repository (using `git clone`), or just download the specific files from the `kvm` directory. +2. Copy the files from the `ovs-geneve` directory of this repository (the "learning-tools" repository) to a directory on your system. You can clone the entire "learning-tools" repository (using `git clone`), or just download the specific files from the `ovs-geneve` directory. 3. Edit `machines.yml` to ensure that the box specified in that file matches the Ubuntu 14.04 x64 base box you just installed and will be using. _I recommend that you do not change any other values in this file unless you know it is necessary._ -4. From a terminal window, change into the directory where the files from this directory are stored and run `vagrant up` to bring up the VMs according to the instructions in `machines.yml` and `Vagrantfile`. (By default, it will create and power on only a single VM.) +4. From a terminal window, change into the directory where the files from this directory are stored and run `vagrant up` to bring up the VMs according to the instructions in `machines.yml` and `Vagrantfile`. (By default, it will create and power on two VMs.) Internet access is required for this step, as Ansible will download updates to the VMs as part of the provisioning process. -5. Once Vagrant has finished creating, booting, and provisioning the VM (note you'll need Internet access for this step), log into the VM (named "kvm-01" by default) using `vagrant ssh`. +5. Once Vagrant has finished creating, booting, and provisioning the VM (note you'll need Internet access for this step), run `vagrant reload` to reboot the VMs, thus ensuring that the latest kernel (required for support of Geneve tunnels) is loaded. -You can now use this VM to do any testing or experimentation with KVM and Libvirt. +6. Log into the first VM (named "kvm-01" by default) using `vagrant ssh kvm-01`. In a separate terminal window, change to the directory where you placed the files for this environment and log into the second VM using `vagrant ssh kvm-02`. From here on, repeat all remaining steps on both VMs. + +7. Switch into the `ovs` directory. Compile OVS using the commands found in [the OVS INSTALL.md documentation](https://github.com/openvswitch/ovs/blob/master/INSTALL.md). For convenience, here are the steps required (without additional explanation; note that Ansible has already installed the necessary packages and prerequisites when Vagrant provisioned the VMs): + + $ ./boot.sh + $ ./configure --with-linux=/lib/modules/`uname -r`/build + $ make + $ sudo make install + $ sudo make modules_install + $ sudo modprobe openvswitch + $ sudo modprobe vport_geneve + $ sudo ovsdb-tool create /usr/local/etc/openvswitch/conf.db vswitchd/vswitch.ovsschema + $ sudo ovsdb-server --remote=punix:/usr/local/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --private-key=db:Open_vSwitch,SSL,private_key --certificate=db:Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --pidfile --detach + $ sudo ovs-vsctl --no-wait init + $ sudo ovs-vswitchd --pidfile --detach + + When this process is complete, you should be able to run `sudo ovs-vsctl show` without receiving any errors (no configuration will be shown because there is no configuration to show). + +8. Run `setup.sh` on each VM. This will create a network namespace, create a pair of veth interfaces, and configure the veth interfaces for testing. + +9. Configure OVS for Geneve tunneling using the following commands (note that you may need to preface each command with `sudo`): + + $ ovs-vsctl add-br br-int + $ ovs-vsctl add-port br-int geneve0 -- set interface geneve0 type=geneve options:remote_ip= + $ ovs-vsctl add-port veth0 + + For `` in the command above, substitute the IP address for the "eth1" interface of the _other_ VM. So, on `kvm-01`, you'd supply the IP address of "eth1" on `kvm-02`, and vice versa. + +10. From `kvm-01`, type `sudo ip netns exec testns ping -c 4 10.1.1.101` to verify connectivity between the test namespace on `kvm-01` with the test namespace on `kvm-02`. + +11. From `kvm-02`, type `sudo ip netns exec testns ping -c 4 10.1.1.100` to verify connectivity between the test namespace on `kvm-02` with the test namespace on `kvm-01`. + +Congratulations, you've just configured and tested Geneve tunneling with OVS! diff --git a/ovs-geneve/Vagrantfile b/ovs-geneve/Vagrantfile index 6f689c1..3bf5948 100644 --- a/ovs-geneve/Vagrantfile +++ b/ovs-geneve/Vagrantfile @@ -67,10 +67,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.cpus = machine['vcpu'] end # srv.vm.provider virtualbox end # config.vm.define - - # Provision the VM with Ansible - config.vm.provision 'ansible' do |ansible| - ansible.playbook = 'provision.yml' - end # config.vm.provision end # machines.each + + # Provision the VM with Ansible + config.vm.provision 'ansible' do |ansible| + ansible.playbook = 'provision.yml' + end # config.vm.provision end # Vagrant.configure diff --git a/ovs-geneve/geneve-net.xml b/ovs-geneve/geneve-net.xml deleted file mode 100644 index c06f652..0000000 --- a/ovs-geneve/geneve-net.xml +++ /dev/null @@ -1,6 +0,0 @@ - - geneve-net - - - - diff --git a/ovs-geneve/provision.yml b/ovs-geneve/provision.yml index b3e3a2d..10ac4b7 100644 --- a/ovs-geneve/provision.yml +++ b/ovs-geneve/provision.yml @@ -4,7 +4,7 @@ remote_user: "vagrant" tasks: - - name: "Install KVM and Libvirt" + - name: "Install necessary packages" apt: state: "present" update_cache: "yes" @@ -19,3 +19,19 @@ - linux-headers-3.19.0-49-generic - linux-image-3.19.0-49-generic - linux-image-extra-3.19.0-49-generic + + - name: "Clone OVS repository" + sudo: "no" + git: + repo: "https://github.com/openvswitch/ovs.git" + dest: "~/ovs" + update: "no" + + - name: "Copy in setup script" + sudo: "no" + copy: + src: "setup.sh" + dest: "/home/vagrant/setup.sh" + owner: "vagrant" + group: "vagrant" + mode: "0755" diff --git a/ovs-geneve/setup.sh b/ovs-geneve/setup.sh new file mode 100644 index 0000000..3bf3a33 --- /dev/null +++ b/ovs-geneve/setup.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Add new network namespace if it doesn't already exist +# Delete existing network namespace named "testns" +if [ -e "/var/run/netns/testns" ]; then + sudo ip netns del testns + echo "Existing network namespace removed" +fi + +# Create new "testns" namespace +sudo ip netns add testns +echo "New network namespace named \"testns\" created" + +# Create new veth pair, deleting existing veth pair if necessary +# If 'veth0' already exists, then delete it +if [ -e "/sysclass/net/veth0" ]; then + sudo ip link del veth0 type veth + echo "Existing veth pair removed" +fi + +# Create new veth pair +sudo ip link add veth0 type veth peer name veth1 +echo "New veth pair created" + +# Get last octet of IP address of VM for use later +IP=$(grep 'address' /etc/network/interfaces | tr "." " " | awk '{print $5'}) +#echo $IP +NS_IP="10.1.1.$IP/24" +#echo $NS_IP + +# Set up network namespace +# Move veth1 to testns if it still exists in primary namespace +if [ -e /sys/class/net/veth1 ]; then + sudo ip link set veth1 netns testns + echo "Moved veth1 to new network namespace" +fi + +# Add IP address to veth1 if IP address isn't already there +if [ `sudo ip netns exec testns ip addr list | grep 'inet ' | wc -l` -eq 0 ]; then + sudo ip netns exec testns ip addr add $NS_IP dev veth1 + echo "IP address $NS_IP assigned to veth1 in new network namespace" +fi + +# Set the veth pair to up +sudo ip netns exec testns ip link set veth1 up +sudo ip link set veth0 up