From 7199c87c32b94daec612bc022607bb86103bd8b2 Mon Sep 17 00:00:00 2001 From: Geoff Anderson Date: Mon, 14 Sep 2015 15:00:04 -0700 Subject: [PATCH] MINOR: Added scripts to automate Vagrant setup for system tests Updated testing README accordingly. Author: Geoff Anderson Reviewers: Ewen Cheslack-Postava, Gwen Shapira Closes #201 from granders/minor-vagrant-package-script --- tests/README.md | 44 ++++--------- tests/bootstrap-test-env.sh | 94 +++++++++++++++++++++++++++ vagrant/package-base-box.sh | 75 +++++++++++++++++++++ vagrant/system-test-Vagrantfile.local | 1 + 4 files changed, 181 insertions(+), 33 deletions(-) create mode 100755 tests/bootstrap-test-env.sh create mode 100755 vagrant/package-base-box.sh diff --git a/tests/README.md b/tests/README.md index bfeacbbe592..82d31daad5b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,51 +14,29 @@ https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk * Install Virtual Box from [https://www.virtualbox.org/](https://www.virtualbox.org/) (run `$ vboxmanage --version` to check if it's installed). * Install Vagrant >= 1.6.4 from [http://www.vagrantup.com/](http://www.vagrantup.com/) (run `vagrant --version` to check if it's installed). -* Install Vagrant Plugins: +* Install system test dependiences, including ducktape, a command-line tool and library for testing distributed systems. - # Required - # Note that vagrant-hostmanager v1.6.0 and up breaks our Vagrant scripts - $ vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0 - $ vagrant plugin install vagrant-cachier + $ python tests/setup.py develop -* Build a specific branch of Kafka - - $ cd kafka - $ git checkout $BRANCH - $ gradle - $ ./gradlew jar - -* Setup a testing cluster with Vagrant. Configure your Vagrant setup by creating the file - `Vagrantfile.local` in the directory of your Kafka checkout. For testing purposes, - `num_brokers` and `num_kafka` should be 0, and `num_workers` should be set high enough - to run all of you tests. An example resides in kafka/vagrant/system-test-Vagrantfile.local - - # Example Vagrantfile.local for use on local machine - # Vagrantfile.local should reside in the base Kafka directory - num_zookeepers = 0 - num_kafka = 0 - num_workers = 9 +* Run the bootstrap script to set up Vagrant for testing + + $ tests/bootstrap-test-env.sh -* Bring up the cluster (note that the initial provisioning process can be slow since it involves -installing dependencies and updates on every vm.): +* Bring up the test cluster $ vagrant up -* Install ducktape: +* Build the desired branch of Kafka - $ pip install ducktape - + $ git checkout $BRANCH + $ gradle # (only if necessary) + $ ./gradlew jar + * Run the system tests using ducktape: $ cd tests $ ducktape kafkatest/tests -* If you make changes to your Kafka checkout, you'll need to rebuild and resync to your Vagrant cluster: - - $ cd kafka - $ ./gradlew jar - $ vagrant rsync # Re-syncs build output to cluster - EC2 Quickstart -------------- This quickstart will help you run the Kafka system tests on EC2. In this setup, all logic is run diff --git a/tests/bootstrap-test-env.sh b/tests/bootstrap-test-env.sh new file mode 100755 index 00000000000..e12643a5060 --- /dev/null +++ b/tests/bootstrap-test-env.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script automates the process of setting up a local machine for running Kafka system tests + +# Helper function which prints version numbers so they can be compared lexically or numerically +function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d%03d\n", $1,$2,$3,$4); }'; } + +base_dir=`dirname $0`/.. +cd $base_dir + +echo "Checking Virtual Box installation..." +bad_vb=false +if [ -z `vboxmanage --version` ]; then + echo "It appears that Virtual Box is not installed. Please install and try again (see https://www.virtualbox.org/ for details)" + bad_vb=true +else + echo "Virtual Box looks good." +fi + +echo "Checking Vagrant installation..." +vagrant_version=`vagrant --version | egrep -o "\d+\.\d+\.\d+"` +bad_vagrant=false +if [ "$(version $vagrant_version)" -lt "$(version 1.6.4)" ]; then + echo "Found Vagrant version $vagrant_version. Please upgrade to 1.6.4 or higher (see http://www.vagrantup.com for details)" + bad_vagrant=true +else + echo "Vagrant installation looks good." +fi + +if [ "x$bad_vagrant" == "xtrue" -o "x$bad_vb" == "xtrue" ]; then + exit 1 +fi + +echo "Checking for necessary Vagrant plugins..." +install_hostmanager=false +hostmanager_version=`vagrant plugin list | grep vagrant-hostmanager | egrep -o "\d+\.\d+\.\d+"` +if [ -z "$hostmanager_version" ]; then + install_hostmanager=true +elif [ "$hostmanager_version" != "1.5.0" ]; then + echo "You have the wrong version of vagrant plugin vagrant-hostmanager. Uninstalling..." + vagrant plugin uninstall vagrant-hostmanager + install_hostmanager=true +fi +if [ "x$install_hostmanager" == "xtrue" ]; then + vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0 +fi + +echo "Creating and packaging a reusable base box for Vagrant..." +vagrant/package-base-box.sh + +# Set up Vagrantfile.local if necessary +if [ ! -e Vagrantfile.local ]; then + echo "Creating Vagrantfile.local..." + cp vagrant/system-test-Vagrantfile.local Vagrantfile.local +else + echo "Found an existing Vagrantfile.local. Keeping without overwriting..." +fi + +# Sanity check contents of Vagrantfile.local +echo "Checking Vagrantfile.local..." +vagrantfile_ok=true +num_brokers=`egrep -o "num_brokers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs` +num_zookeepers=`egrep -o "num_zookeepers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs` +num_workers=`egrep -o "num_workers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs` +if [ "x$num_brokers" == "x" -o "$num_brokers" != 0 ]; then + echo "Vagrantfile.local: bad num_brokers. Update to: num_brokers = 0" + vagrantfile_ok=false +fi +if [ "x$num_zookeepers" == "x" -o "$num_zookeepers" != 0 ]; then + echo "Vagrantfile.local: bad num_zookeepers. Update to: num_zookeepers = 0" + vagrantfile_ok=false +fi +if [ "x$num_workers" == "x" -o "$num_workers" == 0 ]; then + echo "Vagrantfile.local: bad num_workers (size of test cluster). Set num_workers high enough to run your tests." + vagrantfile_ok=false +fi + +if [ "$vagrantfile_ok" == "true" ]; then + echo "Vagrantfile.local looks good." +fi diff --git a/vagrant/package-base-box.sh b/vagrant/package-base-box.sh new file mode 100755 index 00000000000..5ac7f0e5356 --- /dev/null +++ b/vagrant/package-base-box.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script automates the process of creating and packaging +# a new vagrant base_box. For use locally (not aws). + +base_dir=`dirname $0`/.. +cd $base_dir + +backup_vagrantfile=backup_Vagrantfile.local +local_vagrantfile=Vagrantfile.local + +# Restore original Vagrantfile.local, if it exists +function revert_vagrantfile { + rm -f $local_vagrantfile + if [ -e $backup_vagrantfile ]; then + mv $backup_vagrantfile $local_vagrantfile + fi +} + +function clean_up { + echo "Cleaning up..." + vagrant destroy -f + rm -f package.box + revert_vagrantfile +} + +# Name of the new base box +base_box="kafkatest-worker" + +# vagrant VM name +worker_name="worker1" + +echo "Destroying vagrant machines..." +vagrant destroy -f + +echo "Removing $base_box from vagrant..." +vagrant box remove $base_box + +echo "Bringing up a single vagrant machine from scratch..." +if [ -e $local_vagrantfile ]; then + mv $local_vagrantfile $backup_vagrantfile +fi +echo "num_workers = 1" > $local_vagrantfile +echo "num_brokers = 0" >> $local_vagrantfile +echo "num_zookeepers = 0" >> $local_vagrantfile +vagrant up +up_status=$? +if [ $up_status != 0 ]; then + echo "Failed to bring up a template vm, please try running again." + clean_up + exit $up_status +fi + +echo "Packaging $worker_name..." +vagrant package $worker_name + +echo "Adding new base box $base_box to vagrant..." +vagrant box add $base_box package.box + +clean_up + diff --git a/vagrant/system-test-Vagrantfile.local b/vagrant/system-test-Vagrantfile.local index 7f280a4653c..0ec04af06fb 100644 --- a/vagrant/system-test-Vagrantfile.local +++ b/vagrant/system-test-Vagrantfile.local @@ -20,3 +20,4 @@ num_zookeepers = 0 num_brokers = 0 num_workers = 9 +base_box = "kafkatest-worker"