# Copyright 2012 Adobe Systems Inc.; # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Runs all the telemetry tests and pushes results to the server # # Make sure to run in from its folder! # # Env vars used: # - TestHelper.CHROMIUM_SRC: path to chromium src # - DEVICE_NAME: a label to identify the machine running the tests when submitting results # - CEF_HOME: path to CEF binaries (needed only if USE_CEF is set to True) # - USE_CEF: if run with CEF, set this to 'True' # import os import sys import shutil import subprocess import glob import platform import shutil class TestHelper(): GRUNT = "grunt" RESULTS_DIR = "/tmp/topcoat-telemetry" BROWSER = "system" BROWSER_EXEC = None SUBMIT_TYPE = "SHA" CHROMIUM_SRC = os.environ.get("CHROMIUM_SRC") DEVICE_NAME = os.environ.get("DEVICE_NAME") CEF_HOME = os.environ.get("CEF_HOME") USE_CEF = os.environ.get("USE_CEF") @staticmethod def init(targetPlatform, targetTheme): TestHelper._checkEnvVars() TestHelper._prepareProperties() TestHelper._prepareResultsDir() TestHelper._cleanTelemetryTests() TestHelper._prepareTelemetryTests(targetPlatform, targetTheme) @staticmethod def _getPlatform(): p = platform.platform().lower() if p.find('windows') != -1: return "Win" if p.find('darwin') != -1: return "Mac" if p.find('linux') != -1: return "Lin" @staticmethod def _checkEnvVars(): if not TestHelper.DEVICE_NAME: raise RuntimeError("Please set DEVICE_NAME env var (no spaces allowed yet)") if not TestHelper.CHROMIUM_SRC: raise RuntimeError("Please set CHROMIUM_SRC env var.") if not TestHelper.USE_CEF: raise RuntimeError("Please set USE_CEF env var.") if (TestHelper.USE_CEF == 'True') and (not TestHelper.CEF_HOME): raise RuntimeError("Please set CEF_HOME if you set USE_CEF to True") @staticmethod def _prepareProperties(): p = TestHelper._getPlatform() if p == "Win": TestHelper.GRUNT = "grunt.cmd" TestHelper.RESULTS_DIR = "C:\\tmp\\topcoat-telemetry" TestHelper.BROWSER = "exact" TestHelper.BROWSER_EXEC = "%s\\app\\cefclient.exe" % TestHelper.CEF_HOME if p == "Mac": if TestHelper.USE_CEF: TestHelper.BROWSER = "exact" TestHelper.BROWSER_EXEC = "%s/app/cefclient.app/Contents/MacOS/cefclient" % TestHelper.CEF_HOME if p == "Lin": TestHelper.BROWSER = "android-chrome-beta" @staticmethod def _prepareResultsDir(): print "runAll.py: Preparing results dir %s" % TestHelper.RESULTS_DIR if os.path.isdir(TestHelper.RESULTS_DIR): shutil.rmtree(TestHelper.RESULTS_DIR) os.makedirs(TestHelper.RESULTS_DIR) @staticmethod def _prepareTelemetryTests(targetPlatform, targetTheme): print "runAll.py: Preparing telemetry tests" subprocess.check_call([TestHelper.GRUNT, 'telemetry:'+targetPlatform+':'+targetTheme]) @staticmethod def _cleanTelemetryTests(): print "runAll.py: clean up CHROMIUM_SRC/tools/perf/page_sets/topcoat" path = TestHelper.CHROMIUM_SRC + "tools/perf/page_sets/topcoat" if os.path.exists(path): shutil.rmtree(path) @staticmethod def runTests(user_defined_test_list, how_many_rounds_to_run_the_test): print "runAll.py: Running telemetry tests, results in %s" % TestHelper.RESULTS_DIR telemetry_tests = ["loading_benchmark", "smoothness_benchmark"] if user_defined_test_list and len(user_defined_test_list) != 0: topcoat_test_files = user_defined_test_list else: topcoat_test_files = glob.glob(os.getcwd() + "/../perf/page_sets/*.json") def genCmd(): cmd = [ "python", TestHelper.CHROMIUM_SRC + "/tools/perf/run_multipage_benchmarks", "--browser=" + TestHelper.BROWSER, telemetry_test, TestHelper.CHROMIUM_SRC + "tools/perf/page_sets/%s" % topcoat_test_file, "-o", TestHelper.RESULTS_DIR + "/%s_%s-%s.txt" % (telemetry_test, topcoat_test_name, test_round) ] if TestHelper.BROWSER_EXEC: cmd.insert(3, "--browser-executable=" + TestHelper.BROWSER_EXEC) return cmd for tf in topcoat_test_files: topcoat_test_file = tf.split(os.sep)[-1] topcoat_test_name = topcoat_test_file.split(".")[0] print "runAll.py: Running tests for %s" % topcoat_test_name for telemetry_test in telemetry_tests: for test_round in range(how_many_rounds_to_run_the_test): cmd = genCmd() subprocess.check_call(cmd) @staticmethod def submitResults(git_cwd): print "runAll.py: Pushing telemetry data to the server" result_files = glob.glob(TestHelper.RESULTS_DIR + "/*.txt") for rf in result_files: subprocess.check_call([ TestHelper.GRUNT, "telemetry-submit:" + git_cwd, "--path=" + rf, "--device=" + TestHelper.DEVICE_NAME, "--test=" + rf.split(os.sep)[-1][:-6], "--type=" + TestHelper.SUBMIT_TYPE ]) if __name__ == "__main__": # Usage: # ./python runAll.py --platform=VALUE --theme=VALUE [--gitCWD=VALUE] [--test=VALUE] [--round=VALUE] # --platform= desktop or mobile # --theme= light or dark # [optional] --gitCWD=PATH_WHERE_YOU_WANT_TO_RUN_GIT_LOG, e.g. src/skins/button # [optional] --test=ONE_OR_MORE_TESTS_YOU_WANT_TO_RUN, e.g. topcoat_button.test.json # [optional] --test=HOW_MANY_ROUNDS_TO_RUN_THE_TEST, default is 1. platfrm = theme = git_cwd = test_list = None test_round = 1 args = sys.argv[1:] for arg in args: arg_key, arg_val = arg.split('=') if arg_key == '--platform': platfrm = arg_val elif arg_key == '--theme': theme = arg_val elif arg_key == '--gitCWD': git_cwd = arg_val elif arg_key == '--test': test_list = arg_val.split(',') elif arg_key == '--round': test_round = int(arg_val) else: print "%s is not recognized." if not platform or not theme: raise RuntimeError("ERROR: --platform and --theme must be set.") if not git_cwd: git_cwd = '' TestHelper.init(platfrm, theme) TestHelper.runTests(test_list, test_round) TestHelper.submitResults(git_cwd)