# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. #!/usr/bin/evn python # ================================================================= # system_test_runner.py # # - This script is the test driver for a distributed environment # system testing framework. It is located at the top level of the # framework hierachy (in this case - system_test/). # # - This test driver servers as an entry point to launch a series # of test suites (module) with multiple functionally similar test # cases which can be grouped together. # # - Please refer to system_test/README.txt for more details on # how to add test suite and test case. # # - In most cases, it is not necessary to make any changes to this # script. # ================================================================= from optparse import OptionParser from system_test_env import SystemTestEnv from utils import system_test_utils import logging.config import os import pprint import sys # load the config file for logging logging.config.fileConfig('logging.conf') # 'd' is an argument to be merged into the log message (see Python doc for logging). # In this case, corresponding class name can be appended to the end of the logging # message to facilitate debugging. d = {'name_of_class': '(system_test_runner)'} def main(): nLogger = logging.getLogger('namedLogger') aLogger = logging.getLogger('anonymousLogger') optionParser = OptionParser() optionParser.add_option("-p", "--print-test-descriptions-only", dest="printTestDescriptionsOnly", default=False, action="store_true", help="print test descriptions only - don't run the test") optionParser.add_option("-n", "--do-not-validate-remote-host", dest="doNotValidateRemoteHost", default=False, action="store_true", help="do not validate remote host (due to different kafka versions are installed)") (options, args) = optionParser.parse_args() print "\n" aLogger.info("=================================================") aLogger.info(" System Regression Test Framework") aLogger.info("=================================================") print "\n" testSuiteClassDictList = [] # SystemTestEnv is a class to provide all environement settings for this session # such as the SYSTEM_TEST_BASE_DIR, SYSTEM_TEST_UTIL_DIR, ... systemTestEnv = SystemTestEnv() if options.printTestDescriptionsOnly: systemTestEnv.printTestDescriptionsOnly = True if options.doNotValidateRemoteHost: systemTestEnv.doNotValidateRemoteHost = True if not systemTestEnv.printTestDescriptionsOnly: if not systemTestEnv.doNotValidateRemoteHost: if not system_test_utils.setup_remote_hosts(systemTestEnv): nLogger.error("Remote hosts sanity check failed. Aborting test ...", extra=d) print sys.exit(1) else: nLogger.info("SKIPPING : checking remote machines", extra=d) print # get all defined names within a module: definedItemList = dir(SystemTestEnv) aLogger.debug("=================================================") aLogger.debug("SystemTestEnv keys:") for item in definedItemList: aLogger.debug(" " + item) aLogger.debug("=================================================") aLogger.info("=================================================") aLogger.info("looking up test suites ...") aLogger.info("=================================================") # find all test suites in SYSTEM_TEST_BASE_DIR for dirName in os.listdir(systemTestEnv.SYSTEM_TEST_BASE_DIR): # make sure this is a valid testsuite directory if os.path.isdir(dirName) and dirName.endswith(systemTestEnv.SYSTEM_TEST_SUITE_SUFFIX): print nLogger.info("found a testsuite : " + dirName, extra=d) testModulePathName = os.path.abspath(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + dirName) if not systemTestEnv.printTestDescriptionsOnly: system_test_utils.setup_remote_hosts_with_testsuite_level_cluster_config(systemTestEnv, testModulePathName) # go through all test modules file in this testsuite for moduleFileName in os.listdir(testModulePathName): # make sure it is a valid test module if moduleFileName.endswith(systemTestEnv.SYSTEM_TEST_MODULE_EXT) \ and not moduleFileName.startswith("__"): # found a test module file nLogger.info("found a test module file : " + moduleFileName, extra=d) testModuleClassName = system_test_utils.sys_call("grep ^class " + testModulePathName + "/" + \ moduleFileName + " | sed 's/^class //g' | sed 's/(.*):.*//g'") testModuleClassName = testModuleClassName.rstrip('\n') # collect the test suite class data testSuiteClassDict = {} testSuiteClassDict["suite"] = dirName extLenToRemove = systemTestEnv.SYSTEM_TEST_MODULE_EXT.__len__() * -1 testSuiteClassDict["module"] = moduleFileName[:extLenToRemove] testSuiteClassDict["class"] = testModuleClassName testSuiteClassDictList.append(testSuiteClassDict) suiteName = testSuiteClassDict["suite"] moduleName = testSuiteClassDict["module"] className = testSuiteClassDict["class"] # add testsuite directory to sys.path such that the module can be loaded sys.path.append(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + suiteName) if not systemTestEnv.printTestDescriptionsOnly: aLogger.info("=================================================") aLogger.info("Running Test for : ") aLogger.info(" suite : " + suiteName) aLogger.info(" module : " + moduleName) aLogger.info(" class : " + className) aLogger.info("=================================================") # dynamically loading a module and starting the test class mod = __import__(moduleName) theClass = getattr(mod, className) instance = theClass(systemTestEnv) instance.runTest() print if not systemTestEnv.printTestDescriptionsOnly: totalFailureCount = 0 print print "========================================================" print " TEST REPORTS" print "========================================================" for systemTestResult in systemTestEnv.systemTestResultsList: for key in sorted(systemTestResult.iterkeys()): if key == "validation_status": print key, " : " testItemStatus = None for validatedItem in sorted(systemTestResult[key].iterkeys()): testItemStatus = systemTestResult[key][validatedItem] print " ", validatedItem, " : ", testItemStatus if "FAILED" == testItemStatus: totalFailureCount += 1 else: print key, " : ", systemTestResult[key] print print "========================================================" print print "========================================================" print "Total failures count : " + str(totalFailureCount) print "========================================================" print return totalFailureCount return -1 # ========================= # main entry point # ========================= sys.exit(main())