From c5c4de988890cb2ef16836dc386aa52c994faa90 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 9 Jun 2016 11:42:36 +0200 Subject: [PATCH 1/8] "versionsTest" renamed to "tests" --- .travis.yml | 2 +- {versionsTest => tests}/.gitignore | 0 {versionsTest => tests}/Makefile | 0 {versionsTest => tests}/test-zstd-versions.py | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename {versionsTest => tests}/.gitignore (100%) rename {versionsTest => tests}/Makefile (100%) rename {versionsTest => tests}/test-zstd-versions.py (100%) diff --git a/.travis.yml b/.travis.yml index 879aff8a9..e27c7981e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ matrix: env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" # - os: linux # sudo: required -# env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" +# env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest" - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 diff --git a/versionsTest/.gitignore b/tests/.gitignore similarity index 100% rename from versionsTest/.gitignore rename to tests/.gitignore diff --git a/versionsTest/Makefile b/tests/Makefile similarity index 100% rename from versionsTest/Makefile rename to tests/Makefile diff --git a/versionsTest/test-zstd-versions.py b/tests/test-zstd-versions.py similarity index 100% rename from versionsTest/test-zstd-versions.py rename to tests/test-zstd-versions.py From 9470b872c047191da95a337818c2d97618b8bcd0 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 9 Jun 2016 12:54:06 +0200 Subject: [PATCH 2/8] added test-zstd-speed.py --- tests/test-zstd-speed.py | 222 ++++++++++++++++++++++++++++++++++++ tests/test-zstd-versions.py | 12 +- 2 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 tests/test-zstd-speed.py diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py new file mode 100644 index 000000000..a69b1b751 --- /dev/null +++ b/tests/test-zstd-speed.py @@ -0,0 +1,222 @@ +#! /usr/bin/env python +# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py + +import argparse +import os +import string +import time +import traceback +from subprocess import Popen, PIPE + +repo_url = 'https://github.com/Cyan4973/zstd.git' +test_dir_name = 'speedTest' + + +def log(text): + print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text + +def execute(command, print_output=True): + log("> " + command) + popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd) + itout = iter(popen.stdout.readline, b"") + iterr = iter(popen.stderr.readline, b"") + stdout_lines = list(itout) + if print_output: + print ''.join(stdout_lines) + stderr_lines = list(iterr) + if stderr_lines: + print ''.join(stderr_lines) + popen.communicate() + if popen.returncode is not None and popen.returncode != 0: + raise RuntimeError(''.join(stderr_lines)) + return stdout_lines + stderr_lines +execute.cwd = None + + +def fetch(): + execute('git fetch -p') + output = execute('git branch -rl') + for line in output: + if "HEAD" in line: + output.remove(line) # remove "origin/HEAD -> origin/dev" + branches = map(lambda l: l.strip(), output) + return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches) + + +def notify(branch, commit, last_commit): + text_tmpl = string.Template('Changes since $last_commit:\r\n$commits') + branch = branch.split('/')[1] + fmt = '--format="%h: (%an) %s, %ar"' + if last_commit is None: + commits = execute('git log -n 10 %s %s' % (fmt, commit)) + else: + commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit)) + + text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)}) + print str("commits for %s: %s" % (commit, text)) + + +def compile(branch, commit, dry_run): + local_branch = string.split(branch, '/')[1] + version = local_branch.rpartition('-')[2] + version = version + '_' + commit + execute('git checkout -- . && git checkout ' + branch) + if not dry_run: + execute('VERSION=' + version + '; make clean zstdprogram') + + +def get_last_commit(resultsFileName): + if not os.path.isfile(resultsFileName): + return None, None, None + commit = None + cspeed = [] + dspeed = [] + with open(resultsFileName,'r') as f: + for line in f: + words = line.split() + if len(words) == 2: # branch + commit + commit = words[1]; + cspeed = [] + dspeed = [] + if (len(words) == 8): + cspeed.append(float(words[3])) + dspeed.append(float(words[5])) + #if commit != None: + # print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed) + return commit, cspeed, dspeed + + +def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg): + while os.getloadavg()[0] > maxLoadAvg: + print "bench loadavg=%.2f is higher than %s" % (os.getloadavg()[0], maxLoadAvg) + time.sleep(30) + start_load = str(os.getloadavg()) + result = execute('programs/zstd -qb1e' + str(lastCLevel) + ' ' + testFilePath) + end_load = str(os.getloadavg()) + linesExpected = lastCLevel + 2; + if len(result) != linesExpected: + print "len(result)=%d is different that expected %d" % (len(result), linesExpected) + return "" + with open(resultsFileName, "a") as myfile: + myfile.write(branch + " " + commit + "\n") + myfile.writelines(result) + myfile.close() + if (last_cspeed == None): + return "" + commit, cspeed, dspeed = get_last_commit(resultsFileName) + text = "" + for i in range(0, min(len(cspeed), len(last_cspeed))): + if (cspeed[i]/last_cspeed[i] < lower_limit): + text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i]) + if (dspeed[i]/last_dspeed[i] < lower_limit): + text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i]) + if text: + text += "maxLoadAvg=%s load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load) + return text + + +def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit): + with open(logFileName, "w") as myfile: + myfile.writelines(text) + myfile.close() + execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName) + + +def main(args, test_path, clone_path, testFilePaths): + print "test_path=%s" % test_path + print "clone_path=%s" % clone_path + print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths) + print "emails=%s" % args.emails + print "maxLoadAvg=%s" % args.maxLoadAvg + print "lowerLimit=%s" % args.lowerLimit + print "lastCLevel=%s" % args.lastCLevel + print "sleepTime=%s" % args.sleepTime + print "dry_run=%s" % args.dry_run + + for branch, commit in fetch(): + log("checking branch %s: head %s" % (branch, commit)) + try: + commitFileName = test_path + "/commit_" + branch.replace("/", "_") + if os.path.isfile(commitFileName): + last_commit = file(commitFileName, 'r').read() + else: + last_commit = None + file(commitFileName, 'w').write(commit) + + if commit == last_commit: + log("skipping branch %s: head %s already processed" % (branch, commit)) + else: + log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) + compile(branch, commit, args.dry_run) + + logFileName = test_path + "/log_" + branch.replace("/", "_") + text_to_send = [] + results_files = "" + for filePath in testFilePaths: + fileName = filePath.rpartition('/')[2] + resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName + last_commit, cspeed, dspeed = get_last_commit(resultsFileName) + + if not args.dry_run: + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + if text: + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " + if text_to_send: + send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit) + notify(branch, commit, last_commit) + except Exception as e: + stack = traceback.format_exc() + log("Error build %s, error %s" % (branch, str(e)) ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('testFileNames', help='file names list for speed test') + parser.add_argument('emails', help='e-mails to send warnings') + parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit e.g. 0.98', default=0.98) + parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) + parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) + parser.add_argument('--sleepTime', type=int, help='frequency of checking in seconds', default=60) + parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) + args = parser.parse_args() + + # check if test files are accessible + testFileNames = args.testFileNames.split() + testFilePaths = [] + for fileName in testFileNames: + if os.path.isfile(fileName): + testFilePaths.append(os.path.abspath(fileName)) + else: + raise RuntimeError("File not found: " + fileName) + + test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest + clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd + execute.cwd = clone_path + + # clone ZSTD repo if needed + if not os.path.isdir(test_path): + os.mkdir(test_path) + if not os.path.isdir(clone_path): + execute('git clone ' + repo_url) + if not os.path.isdir(clone_path): + raise RuntimeError("ZSTD clone not found: " + clone_path) + + while True: + pid = str(os.getpid()) + pidfile = "./speedTest.pid" + if os.path.isfile(pidfile): + print "%s already exists, exiting" % pidfile + else: + file(pidfile, 'w').write(pid) + try: + loadavg = os.getloadavg()[0] + if (loadavg <= args.maxLoadAvg): + main(args, test_path, clone_path, testFilePaths) + else: + print "loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg) + finally: + os.unlink(pidfile) + time.sleep(args.sleepTime) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 19e083a8e..437cd4c01 100644 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -11,7 +11,7 @@ import subprocess import sys repo_url = 'https://github.com/Cyan4973/zstd.git' -tmp_dir_name = 'versionsTest/zstdtest' +tmp_dir_name = 'tests/versionsTest' make_cmd = 'make' git_cmd = 'git' test_dat_src = 'README.md' @@ -110,8 +110,8 @@ def decompress_zst(tag): if __name__ == '__main__': error_code = 0 base_dir = os.getcwd() + '/..' # /path/to/zstd - tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/versionsTest/zstdtest - clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/versionsTest/zstdtest/zstd + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest + clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd programs_dir = base_dir + '/programs' # /path/to/zstd/programs os.makedirs(tmp_dir, exist_ok=True) @@ -130,14 +130,14 @@ if __name__ == '__main__': # Build all release zstd for tag in tags: os.chdir(base_dir) - dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd. + dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd. if not os.path.isfile(dst_zstd) or tag == head: if tag != head: - r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/ + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/ os.makedirs(r_dir, exist_ok=True) os.chdir(clone_dir) git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) - os.chdir(r_dir + '/programs') # /path/to/zstd/zstdtest//programs + os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest//programs make(['clean', 'zstd'], False) else: os.chdir(programs_dir) From bf0b4b034c4a72e8ea254567ba43fd5e2546fdb6 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 9 Jun 2016 12:57:18 +0200 Subject: [PATCH 3/8] update python scripts permissions --- tests/test-zstd-speed.py | 0 tests/test-zstd-versions.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/test-zstd-speed.py mode change 100644 => 100755 tests/test-zstd-versions.py diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py old mode 100644 new mode 100755 diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py old mode 100644 new mode 100755 From 348a53a842a2f831770fbb68861cd584d1b4b343 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 9 Jun 2016 13:14:21 +0200 Subject: [PATCH 4/8] test-zstd-speed.py: fixed git clone --- tests/test-zstd-speed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index a69b1b751..b1c345f69 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -194,15 +194,16 @@ if __name__ == '__main__': test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd - execute.cwd = clone_path # clone ZSTD repo if needed if not os.path.isdir(test_path): os.mkdir(test_path) if not os.path.isdir(clone_path): + execute.cwd = test_path execute('git clone ' + repo_url) if not os.path.isdir(clone_path): raise RuntimeError("ZSTD clone not found: " + clone_path) + execute.cwd = clone_path while True: pid = str(os.getpid()) From c1b154a8e8c9565a28f00c1fea084a9823bae0bc Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 10 Jun 2016 12:53:12 +0200 Subject: [PATCH 5/8] improved test-zstd-speed.py 1. Logs are much smaller now 2. Added "--message" 3. Added autodetection of mutt and mail --- tests/test-zstd-speed.py | 91 +++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index b1c345f69..bfe004a5f 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -15,7 +15,7 @@ test_dir_name = 'speedTest' def log(text): print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text -def execute(command, print_output=True): +def execute(command, print_output=False, print_error=True): log("> " + command) popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd) itout = iter(popen.stdout.readline, b"") @@ -24,15 +24,25 @@ def execute(command, print_output=True): if print_output: print ''.join(stdout_lines) stderr_lines = list(iterr) - if stderr_lines: + if print_output: print ''.join(stderr_lines) popen.communicate() if popen.returncode is not None and popen.returncode != 0: + if not print_output and print_error: + print ''.join(stderr_lines) raise RuntimeError(''.join(stderr_lines)) return stdout_lines + stderr_lines execute.cwd = None +def does_command_exist(command): + try: + execute(command, False, False); + except Exception as e: + return False + return True + + def fetch(): execute('git fetch -p') output = execute('git branch -rl') @@ -86,16 +96,17 @@ def get_last_commit(resultsFileName): return commit, cspeed, dspeed -def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg): +def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg, message): + sleepTime = 30 while os.getloadavg()[0] > maxLoadAvg: - print "bench loadavg=%.2f is higher than %s" % (os.getloadavg()[0], maxLoadAvg) - time.sleep(30) + log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime)) + time.sleep(sleepTime) start_load = str(os.getloadavg()) - result = execute('programs/zstd -qb1e' + str(lastCLevel) + ' ' + testFilePath) + result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath) end_load = str(os.getloadavg()) linesExpected = lastCLevel + 2; if len(result) != linesExpected: - print "len(result)=%d is different that expected %d" % (len(result), linesExpected) + log("ERROR: number of result lines=%d is different that expected %d" % (len(result), linesExpected)) return "" with open(resultsFileName, "a") as myfile: myfile.write(branch + " " + commit + "\n") @@ -111,30 +122,24 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP if (dspeed[i]/last_dspeed[i] < lower_limit): text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i]) if text: - text += "maxLoadAvg=%s load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load) + text = message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)) + text return text -def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit): +def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail): with open(logFileName, "w") as myfile: myfile.writelines(text) myfile.close() - execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName) + if have_mutt: + execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName) + elif have_mail: + execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName) + else: + log("e-mail cannot be sent (mail and mutt not found)") -def main(args, test_path, clone_path, testFilePaths): - print "test_path=%s" % test_path - print "clone_path=%s" % clone_path - print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths) - print "emails=%s" % args.emails - print "maxLoadAvg=%s" % args.maxLoadAvg - print "lowerLimit=%s" % args.lowerLimit - print "lastCLevel=%s" % args.lastCLevel - print "sleepTime=%s" % args.sleepTime - print "dry_run=%s" % args.dry_run - +def check_branches(args, test_path, testFilePaths, have_mutt, have_mail): for branch, commit in fetch(): - log("checking branch %s: head %s" % (branch, commit)) try: commitFileName = test_path + "/commit_" + branch.replace("/", "_") if os.path.isfile(commitFileName): @@ -158,28 +163,29 @@ def main(args, test_path, clone_path, testFilePaths): last_commit, cspeed, dspeed = get_last_commit(resultsFileName) if not args.dry_run: - text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) if text: - text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) if text: text_to_send.append(text) results_files += resultsFileName + " " if text_to_send: - send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit) + send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail) notify(branch, commit, last_commit) except Exception as e: stack = traceback.format_exc() - log("Error build %s, error %s" % (branch, str(e)) ) + log("ERROR: build %s, error %s" % (branch, str(e)) ) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('testFileNames', help='file names list for speed test') - parser.add_argument('emails', help='e-mails to send warnings') - parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit e.g. 0.98', default=0.98) + parser.add_argument('testFileNames', help='file names list for speed benchmark') + parser.add_argument('emails', help='list of e-mail addresses to send warnings') + parser.add_argument('--message', help='attach an additional message to e-mail') + parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98) parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) - parser.add_argument('--sleepTime', type=int, help='frequency of checking in seconds', default=60) + parser.add_argument('--sleepTime', type=int, help='frequency of repository checking in seconds', default=300) parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) args = parser.parse_args() @@ -195,6 +201,12 @@ if __name__ == '__main__': test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd + # check availability of e-mail senders + have_mutt = does_command_exist("mutt --help"); + have_mail = does_command_exist("mail -V"); + if not have_mutt and not have_mail + log("WARNING: e-mail senders mail and mutt not found") + # clone ZSTD repo if needed if not os.path.isdir(test_path): os.mkdir(test_path) @@ -205,19 +217,32 @@ if __name__ == '__main__': raise RuntimeError("ZSTD clone not found: " + clone_path) execute.cwd = clone_path + print "PARAMETERS:\ntest_path=%s" % test_path + print "clone_path=%s" % clone_path + print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths) + print "message=%s" % args.message + print "emails=%s" % args.emails + print "maxLoadAvg=%s" % args.maxLoadAvg + print "lowerLimit=%s" % args.lowerLimit + print "lastCLevel=%s" % args.lastCLevel + print "sleepTime=%s" % args.sleepTime + print "dry_run=%s" % args.dry_run + print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail) + while True: pid = str(os.getpid()) pidfile = "./speedTest.pid" if os.path.isfile(pidfile): - print "%s already exists, exiting" % pidfile + log("%s already exists, exiting" % pidfile) else: file(pidfile, 'w').write(pid) try: loadavg = os.getloadavg()[0] if (loadavg <= args.maxLoadAvg): - main(args, test_path, clone_path, testFilePaths) + check_branches(args, test_path, testFilePaths, have_mutt, have_mail) else: - print "loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg) + log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) finally: os.unlink(pidfile) + log("sleep for %s seconds" % args.sleepTime) time.sleep(args.sleepTime) From 2c07b977ea635a0168c547f78813556b0e65c098 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 10 Jun 2016 12:53:39 +0200 Subject: [PATCH 6/8] added tests\README.md --- tests/README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..7cea32a17 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,42 @@ +scripts for automated testing of zstd +================================ + +#### test-zstd-versions.py - script for testing zstd interoperability between versions + +This script creates `versionsTest` directory to which zstd repository is cloned. +Then all taged (released) versions of zstd are compiled. +In the following step interoperability between zstd versions is checked. + + +#### test-zstd-speed.py - script for testing zstd speed difference between commits + +This script creates `speedTest` directory to which zstd repository is cloned. +Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter). +After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. +If a new commit is found it's compiled and a speed benchmark for this commit is performed. +The results of the speed benchmark are compared to the previous results. +If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. +If second results are lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). +The speed benchmark is not performed if computers' load average is higher than `maxLoadAvg` (an optional parameter, default 0.75). + +The example usage: +```./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60``` + +The full list of parameters: +``` +positional arguments: + testFileNames file names list for speed benchmark + emails list of e-mail addresses to send warnings + +optional arguments: + -h, --help show this help message and exit + --message MESSAGE attach an additional message to e-mail + --lowerLimit LOWERLIMIT + send email if speed is lower than given limit + --maxLoadAvg MAXLOADAVG + maximum load average to start testing + --lastCLevel LASTCLEVEL + last compression level for testing + --sleepTime SLEEPTIME + frequency of repository checking in seconds +``` From b06c3e7d6c837eaded8d50c0c2d8e5befb15f150 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 10 Jun 2016 13:02:38 +0200 Subject: [PATCH 7/8] added how to run test-zstd-speed.py in background --- tests/README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/README.md b/tests/README.md index 7cea32a17..2e0aafaa2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,14 +13,21 @@ In the following step interoperability between zstd versions is checked. This script creates `speedTest` directory to which zstd repository is cloned. Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter). After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. -If a new commit is found it's compiled and a speed benchmark for this commit is performed. +If a new commit is found it is compiled and a speed benchmark for this commit is performed. The results of the speed benchmark are compared to the previous results. If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. -If second results are lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). -The speed benchmark is not performed if computers' load average is higher than `maxLoadAvg` (an optional parameter, default 0.75). +If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). +The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75). -The example usage: -```./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60``` +The example usage with two test files, one e-mail address, and with an additional message: +``` +./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 +``` + +To run the script in background please use: +``` +nohup ./test-zstd-speed.py testFileNames emails & +``` The full list of parameters: ``` From f1690296229415e3d2c8d62f471bd6480ad5579c Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 10 Jun 2016 13:59:08 +0200 Subject: [PATCH 8/8] Additional remarks for tests\README.md --- tests/README.md | 8 +++++++- tests/test-zstd-speed.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 2e0aafaa2..7b3795412 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,7 +17,13 @@ If a new commit is found it is compiled and a speed benchmark for this commit is The results of the speed benchmark are compared to the previous results. If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). -The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75). + +Additional remarks: +- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel +- Using the script with virtual machines can lead to large variations of speed results +- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75) +- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning + The example usage with two test files, one e-mail address, and with an additional message: ``` diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index bfe004a5f..3e5eaa738 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -204,7 +204,7 @@ if __name__ == '__main__': # check availability of e-mail senders have_mutt = does_command_exist("mutt --help"); have_mail = does_command_exist("mail -V"); - if not have_mutt and not have_mail + if not have_mutt and not have_mail: log("WARNING: e-mail senders mail and mutt not found") # clone ZSTD repo if needed