#!/bin/bash # Automatically execute the strongSwan test cases # # Copyright (C) 2004 Eric Marchionni, Patrik Rayo # Zuercher Hochschule Winterthur # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your # option) any later version. See . # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. DIR=$(dirname `readlink -f $0`) . $DIR/testing.conf . $DIR/scripts/function.sh SSHCONF="-F $DIR/ssh_config" [ -d $DIR/hosts ] || die "Directory 'hosts' not found" [ -d $DIR/tests ] || die "Directory 'tests' not found" [ -d $BUILDDIR ] || die "Directory '$BUILDDIR' does not exist, please run make-testing first" running_any $STRONGSWANHOSTS || die "Please start test environment before running $0" ln -sfT $DIR $TESTDIR/testing ############################################################################## # take care of new path and file variables # [ -d $TESTRESULTSDIR ] || mkdir $TESTRESULTSDIR TESTDATE=`date +%Y%m%d-%H%M-%S` TODAYDIR=$TESTRESULTSDIR/$TESTDATE mkdir $TODAYDIR TESTRESULTSHTML=$TODAYDIR/all.html INDEX=$TODAYDIR/index.html DEFAULTTESTSDIR=$TESTDIR/testing/tests GUEST_OUTPUT_DIR=/var/cache/testing-results SOURCEIP_ROUTING_TABLE=220 export LEAK_DETECTIVE_LOG=/var/log/leak-detective.log testnumber="0" failed_cnt="0" passed_cnt="0" subdir_cnt="0" ############################################################################## # parse arguments # function usage() { cat << EOF Usage: ${0##*/} [-h] [-v|-t] [-i|-e] [TESTDIRS] --help (-h) show usage information --verbose (-v) show complete logs on errors (implies -t) --timestamps (-t) show timestamps in console.log --pre (-i) run pretest script only (single test only) --post (-e) run posttest script only (single test only) TESTDIRS list of test directories (relative to testing/tests). wildcards (*) are supported. default is to run all tests. EOF } TESTDIRS=() while [ $# -gt 0 ]; do case $1 in -h|--help) usage exit ;; -v|--verbose) verbose=YES timestamps=YES ;; -t|--timestamps) timestamps=YES ;; -i|--pre) pretest_only=YES ;; -e|--post) posttest_only=YES ;; *) TESTDIRS+=("$1") ;; esac shift done function print_time() { [ "$timestamps" == "YES" ] && \ logtime=$(date +"%s.%N") && \ echo "$(date -d "0 +${logtime} sec -${teststart_ns} sec" +%S.%N) [$(date -d @${logtime} +%T.%N)] ~ " } ############################################################################## # copy default tests to $BUILDDIR # TESTSDIR=$BUILDDIR/tests [ -d $TESTSDIR ] || mkdir $TESTSDIR ############################################################################## # assign IP for each host to hostname # for host in $STRONGSWANHOSTS do eval ipv4_${host}="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`" eval ipv6_${host}="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`" case $host in moon) eval ipv4_moon1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" eval ipv6_moon1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" ;; sun) eval ipv4_sun1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" eval ipv6_sun1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" ;; alice) eval ipv4_alice1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" eval ipv6_alice1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" ;; venus) ;; bob) ;; carol) eval ipv4_carol1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" eval ipv6_carol1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" ;; dave) eval ipv4_dave1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" eval ipv6_dave1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`" ;; winnetou) ;; esac done ############################################################################## # open ssh sessions # for host in $STRONGSWANHOSTS do ssh $SSHCONF -N root@`eval echo \\\$ipv4_$host` >/dev/null 2>&1 & eval ssh_pid_$host="`echo $!`" do_on_exit kill `eval echo \\\$ssh_pid_$host` done ############################################################################## # determine actual software versions # [ -f $SHAREDDIR/.strongswan-version ] && SWANVERSION=`cat $SHAREDDIR/.strongswan-version` KERNELVERSION=`ssh $SSHCONF root@\$ipv4_winnetou uname -r 2>/dev/null` ############################################################################## # create header for the results html file # ENVIRONMENT_HEADER=$(cat <<@EOF @EOF ) cat > $INDEX <<@EOF strongSwan KVM Tests

strongSwan KVM Tests

$ENVIRONMENT_HEADER @EOF cat > $TESTRESULTSHTML <<@EOF strongSwan KVM Tests - All Tests
strongSwan KVM Tests / All Tests

All Tests

$ENVIRONMENT_HEADER @EOF echo "Guest kernel : $KERNELVERSION" echo "strongSwan : $SWANVERSION" echo "Date : $TESTDATE" echo ############################################################################## # trap CTRL-C to properly terminate a long run # function abort_tests() { echo -n "...aborting..." > /dev/tty aborted=YES } trap abort_tests INT ############################################################################## # enter specific test directory # if [ "${#TESTDIRS[@]}" -gt 0 ] then TESTS=$(printf "%s\n" "${TESTDIRS[@]}" | sort -u) else TESTS=$(ls $DEFAULTTESTSDIR) fi for SUBDIR in $TESTS do SUBTESTS="`basename $SUBDIR`" if [ $SUBTESTS = $SUBDIR ] then SUBTESTS="`ls $DEFAULTTESTSDIR/$SUBDIR`" else if [[ $SUBTESTS == *'*'* ]] then SUBTESTS="`basename -a $DEFAULTTESTSDIR/$SUBDIR`" fi SUBDIR="`dirname $SUBDIR`" fi if [ ! -d $TODAYDIR/$SUBDIR ] then mkdir $TODAYDIR/$SUBDIR if [ $testnumber == 0 ] then FIRST="Category" else FIRST=" " fi if [ $subdir_cnt != 0 ] then echo " " >> $INDEX echo " " >> $INDEX echo " " >> $INDEX subdir_cnt="0" fi echo " " >> $INDEX echo " ">> $INDEX echo " " >> $INDEX SUBTESTSINDEX=$TODAYDIR/$SUBDIR/index.html cat > $SUBTESTSINDEX <<@EOF strongSwan $SUBDIR Tests
strongSwan KVM Tests / $SUBDIR

strongSwan $SUBDIR Tests

Host `uname -a`
Guest kernel $KERNELVERSION
strongSwan $SWANVERSION
Date $TESTDATE
       
Number Test Time [s] Result
$subdir_cnt 
$FIRST$SUBDIR
@EOF fi for name in $SUBTESTS do let "testnumber += 1" let "subdir_cnt += 1" testname=$SUBDIR/$name log_action " $testnumber $testname:" teststart_ns=$(date +"%s.%N") teststart=$(date -d "@$teststart_ns" +"%s") if [ ! -d $DEFAULTTESTSDIR/${testname} ] then echo "is missing..skipped" continue fi [ ! -f $DEFAULTTESTSDIR/${testname}/description.txt ] && echo "!! File 'description.txt' is missing" && continue [ ! -f $DEFAULTTESTSDIR/${testname}/test.conf ] && echo "!! File 'test.conf' is missing" && continue [ ! -f $DEFAULTTESTSDIR/${testname}/pretest.dat ] && echo "!! File 'pretest.dat' is missing" && continue [ ! -f $DEFAULTTESTSDIR/${testname}/posttest.dat ] && echo "!! File 'posttest.dat' is missing" && continue [ ! -f $DEFAULTTESTSDIR/${testname}/evaltest.dat ] && echo "!! File 'evaltest.dat' is missing" && continue TESTRESULTDIR=$TODAYDIR/$testname mkdir -p $TESTRESULTDIR CONSOLE_LOG=$TESTRESULTDIR/console.log touch $CONSOLE_LOG TESTDIR=$TESTSDIR/${testname} unset RADIUSHOSTS unset DBHOSTS unset IPV6 unset SWANCTL source $DEFAULTTESTSDIR/${testname}/test.conf DBDIR=/etc/db.d STATUS="passed" function stop_tcpdump { # wait for packets to get processed, but don't wait longer than 1s eval ssh $SSHCONF root@\$ipv4_${1} "\"i=100; while [ \\\$i -gt 0 ]; do pkill -USR1 tcpdump; tail -1 /tmp/tcpdump.err.log | perl -n -e '/(\\d+).*?(\\d+)/; exit (\\\$1 == \\\$2)' || break; sleep 0.01; i=\\\$((\\\$i-1)); done;\"" echo "$(print_time)${1}# killall tcpdump" >> $CONSOLE_LOG eval ssh $SSHCONF root@\$ipv4_${1} "\"killall tcpdump; while true; do killall -q -0 tcpdump || break; sleep 0.01; done;\"" eval TDUP_${1}="false" echo "" >> $CONSOLE_LOG } ############################################################################ # skip this whole pretest block if we only execute the posttest script # if [ "$posttest_only" == "YES" ] then echo -n "(pre).." else ########################################################################## # copy test specific configurations to hosts and clear log files # $DIR/scripts/load-testconfig $testname ########################################################################## # clean up and initialize test hosts # declare -A INIT_OPTIONS=() for host in $TCPDUMPHOSTS do # all hosts currently capture on eth0 INIT_OPTIONS[${host}]="${INIT_OPTIONS[${host}]} -i eth0" eval TDUP_${host}="true" done for host in $DBHOSTS do INIT_OPTIONS[${host}]="${INIT_OPTIONS[${host}]} -d $DBDIR" done # initialize hosts in parallel WAIT_FOR=() for host in $STRONGSWANHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN /usr/local/bin/init-test ${INIT_OPTIONS[${host}]} & WAIT_FOR+=($!) done wait ${WAIT_FOR[@]} ########################################################################## # execute pre-test commands # echo -n "pre.." echo -e "PRE-TEST\n" >> $CONSOLE_LOG 2>&1 eval `awk -F "::" '{ if ($0 ~ /^#.*/) { printf("echo \"%s\"; ", $0); } else if ($2 != "") { printf("echo \"$(print_time)%s# %s\"; ", $1, $2) printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2) printf("echo;\n") } }' $TESTDIR/pretest.dat` >> $CONSOLE_LOG 2>&1 fi ############################################################################ ############################################################################ # skip this whole test block if we only execute the pre- or posttest script # if [ "$pretest_only" == "YES" -o "$posttest_only" == "YES" ] then echo -n "(test).." else ########################################################################## # get and evaluate test results # echo -n "test.." echo -e "\nTEST\n" >> $CONSOLE_LOG 2>&1 eval `awk -F "::" '{ host=$1 command=$2 pattern=$3 hit=$4 if ($0 ~ /^#.*/) { printf("echo \"%s\"; ", $0); next } else if (command == "") { next } printf("cmd_err=\044(mktemp --suff .err test.XXXXXX); ") printf("cmd_out=\044(mktemp --suff .out test.XXXXXX); ") printf("start_time=\044(print_time); ") if (command == "tcpdump") { printf("if [ \044TDUP_%s == \"true\" ]; then stop_tcpdump %s; fi; \n", host, host) printf("ssh \044SSHCONF root@\044ipv4_%s cat /tmp/tcpdump.log > \044cmd_out; ", host) } else { printf("ssh \044SSHCONF root@\044ipv4_%s %s >\044cmd_out 2>\044cmd_err; ", host, command) } printf("cmd_res=\044(cat \044cmd_out | grep \"%s\"); ", pattern) printf("cmd_exit=\044?; ") printf("cmd_fail=0; ") if (hit ~ /^[0-9]+$/) { printf("if [ \044(echo \"\044cmd_res\" | wc -l) -ne %d ] ", hit) } else { printf("if [ \044cmd_exit -eq 0 -a \"%s\" = \"NO\" ] ", hit) printf("|| [ \044cmd_exit -ne 0 -a \"%s\" = \"YES\" ] ", hit) } printf("; then STATUS=\"failed\"; cmd_fail=1; fi; \n") printf("if [ \044cmd_fail -ne 0 ]; then echo \"~~~~~~~ FAIL ~~~~~~~\"; fi; \n") if (command == "tcpdump") { printf("echo \"\044{start_time}%s# cat /tmp/tcpdump.log | grep \047%s\047 [%s]\"; ", host, pattern, hit) } else { printf("echo \"\044{start_time}%s# %s | grep \047%s\047 [%s]\"; ", host, command, pattern, hit) } printf("if [ -n \"\044cmd_res\" ]; then echo \"\044cmd_res\"; fi; \n") printf("cat \044cmd_err; \n") printf("if [ \044cmd_fail -ne 0 ]; then \n") printf("if [ -s \044cmd_out ]; then echo \"~~ output ~~~~~~~~~~\"; \n") printf("if [ \"\044verbose\" == \"YES\" ]; then cat \044cmd_out;\n") printf("else cat \044cmd_out | head; fi; fi; \n") printf("echo \"~~~~~~~~~~~~~~~~~~~~\"; fi; \n") printf("rm -f -- \044cmd_out \044cmd_err; \n") printf("echo; ") }' $TESTDIR/evaltest.dat` >> $CONSOLE_LOG 2>&1 ########################################################################## # collect test results from hosts # COLLECT_OPTIONS_BASE="-t $SOURCEIP_ROUTING_TABLE" if [ $name = "net2net-ip4-in-ip6-ikev2" -o $name = "net2net-ip6-in-ip4-ikev2" ] then COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -4 -6" IPROUTE_DSP="ip (-6) route list table $SOURCEIP_ROUTING_TABLE" IPTABLES_DSP="iptables -L ; ip6tables -L" IPTABLES_SAVE_DSP="iptables-save ; ip6tables-save" elif [ -n "$IPV6" ] then COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -6" IPROUTE_DSP="ip -6 route list table $SOURCEIP_ROUTING_TABLE" IPTABLES_DSP="ip6tables -L" IPTABLES_SAVE_DSP="ip6tables-save" else COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -4" IPROUTE_DSP="ip route list table $SOURCEIP_ROUTING_TABLE" IPTABLES_DSP="iptables -L" IPTABLES_SAVE_DSP="iptables-save" fi declare -A COLLECT_OPTIONS=() for host in $IPSECHOSTS do COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -i" if [ -z "$SWANCTL" ] then COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -l" fi done for host in $DBHOSTS do COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -d" done for host in $RADIUSHOSTS do COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -r" done # collect test results in parallel, copy them afterwards WAIT_FOR=() for host in ${!COLLECT_OPTIONS[@]} do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN /usr/local/bin/collect-results \ $COLLECT_OPTIONS_BASE ${COLLECT_OPTIONS[${host}]} ${GUEST_OUTPUT_DIR} & WAIT_FOR+=($!) done test ${#WAIT_FOR[@]} -gt 0 && wait ${WAIT_FOR[@]} WAIT_FOR=() for host in ${!COLLECT_OPTIONS[@]} do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN "tar -cf - -C ${GUEST_OUTPUT_DIR} ." | tar -xf - -C $TESTRESULTDIR/ & WAIT_FOR+=($!) done test ${#WAIT_FOR[@]} -gt 0 && wait ${WAIT_FOR[@]} chmod a+r $TESTRESULTDIR/* ########################################################################## # create index.html for the given test case # cat > $TESTRESULTDIR/index.html <<@EOF Test $testname
Guest kernel $KERNELVERSION
strongSwan $SWANVERSION
Date $TESTDATE
       
Number Test Result

Test $testname

Description

@EOF cat $TESTDIR/description.txt >> $TESTRESULTDIR/index.html cat >> $TESTRESULTDIR/index.html <<@EOF $VIRTHOSTS @EOF for host in $IPSECHOSTS do if [ -n "$SWANCTL" ] then echo >> $TESTRESULTDIR/${host}.swanctl.sas cat $TESTRESULTDIR/${host}.swanctl.pols >> \ $TESTRESULTDIR/${host}.swanctl.sas cat $TESTRESULTDIR/${host}.swanctl.algs >> \ $TESTRESULTDIR/${host}.swanctl.stats cat >> $TESTRESULTDIR/index.html <<@EOF

$host

 
@EOF else cat >> $TESTRESULTDIR/index.html <<@EOF

$host

@EOF fi done for host in $RADIUSHOSTS do cat >> $TESTRESULTDIR/index.html <<@EOF

$host

@EOF done cat >> $TESTRESULTDIR/index.html <<@EOF

tcpdump

    @EOF for host in $TCPDUMPHOSTS do cat >> $TESTRESULTDIR/index.html <<@EOF
  • $host tcpdump.log
    • @EOF done cat >> $TESTRESULTDIR/index.html <<@EOF
@EOF cat >> $TESTRESULTDIR/index.html <<@EOF
@EOF fi ############################################################################ ############################################################################ # skip this whole posttest block if we only execute the pretest script # if [ "$pretest_only" == "YES" ] then echo -n "(post)" else ########################################################################## # execute post-test commands # echo -n "post" echo -e "\nPOST-TEST\n" >> $CONSOLE_LOG 2>&1 eval `awk -F "::" '{ if ($0 ~ /^#.*/) { printf("echo \"%s\"; ", $0); } else if ($2 != "") { printf("echo \"$(print_time)%s# %s\"; ", $1, $2) printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2) printf("echo;\n") } }' $TESTDIR/posttest.dat` >> $CONSOLE_LOG 2>&1 ########################################################################## # check that IPsec state was cleaned up properly # for host in $IPSECHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} IPSECSTATE=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm state'` # ignore IPv4/v6 states created with IPComp SAs IPSECSTATEISSUE=`echo "$IPSECSTATE" | grep 'proto.*spi' | grep -v 'proto 4'` IPSECPOLICY=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm policy'` if [ -n "$IPSECSTATEISSUE" -o -n "$IPSECPOLICY" ] then echo -e "\n$host# ip xfrm state [NO]" >> $CONSOLE_LOG echo "$IPSECSTATE" >> $CONSOLE_LOG echo -e "\n$host# ip xfrm policy [NO]" >> $CONSOLE_LOG echo "$IPSECPOLICY" >> $CONSOLE_LOG STATUS="failed" fi done ########################################################################## # make sure there were no leaks # for host in $STRONGSWANHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} LEAKS=`ssh $SSHCONF $HOSTLOGIN 'cat $LEAK_DETECTIVE_LOG 2>/dev/null | grep -v "No leaks detected.*"'` if [ -n "$LEAKS" ] then echo -e "\n$host# cat $LEAK_DETECTIVE_LOG [NO]" >> $CONSOLE_LOG echo "$LEAKS" >> $CONSOLE_LOG echo "<<< $host $LEAK_DETECTIVE_LOG >>>" >> $CONSOLE_LOG STATUS="failed" fi done ########################################################################## # get a copy of /var/log/auth.log # for host in $IPSECHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN "grep -s -E 'charon|last message repeated|imcv|pt-tls-client' \ /var/log/auth.log" >> $TESTRESULTDIR/${host}.auth.log done ########################################################################## # get a copy of /var/log/daemon.log # for host in $IPSECHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN "grep -s -E 'systemd|swanctl|charon|last message repeated|imcv' \ /var/log/daemon.log" >> $TESTRESULTDIR/${host}.daemon.log done ########################################################################## # stop tcpdump if necessary # for host in $TCPDUMPHOSTS do if [ "`eval echo \\\$TDUP_${host}`" = "true" ] then stop_tcpdump $host fi eval HOSTLOGIN=root@\$ipv4_${host} scp $SSHCONF $HOSTLOGIN:/tmp/tcpdump.log \ $TESTRESULTDIR/${host}.tcpdump.log > /dev/null 2>&1 done ########################################################################## # remove database directory if needed # for host in $DBHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN "umount $DBDIR; rm -r $DBDIR" > /dev/null 2>&1 done ########################################################################## # copy default host config back if necessary # $DIR/scripts/restore-defaults $testname ########################################################################## # remove any charon.pid files that still may exist # for host in $IPSECHOSTS do eval HOSTLOGIN=root@\$ipv4_${host} ssh $SSHCONF $HOSTLOGIN 'if [ -f /var/run/charon.pid ]; then rm /var/run/charon.pid; echo " removed charon.pid on `hostname`"; fi' done fi ############################################################################ ########################################################################## # set counters # if [ $STATUS = "failed" ] then let "failed_cnt += 1" else let "passed_cnt += 1" fi ########################################################################## # write test status to html file # testend=$(date +%s) let "testend -= teststart" let "timetotal += testend" if [ $STATUS = "passed" ] then COLOR="green" log_status 0 else COLOR="red" log_status 1 fi cat >> $TESTRESULTSHTML << @EOF $testnumber $testname $testend $STATUS @EOF cat >> $SUBTESTSINDEX << @EOF $testnumber $name $STATUS   @EOF ########################################################################## # exit if aborted or running only pre-/posttest scripts # if [ -n "$aborted" -o "$pretest_only" == "YES" -o "$posttest_only" == "YES" ] then break 2 fi done done ############################################################################## # finish the results html file # printf -v timetotal_fmt '%02d:%02d:%02d' $((timetotal/3600)) $((timetotal%3600/60)) $((timetotal%60)) cat >> $TESTRESULTSHTML << @EOF      Passed$passed_cnt   Failed$failed_cnt   Time$timetotal_fmt (${timetotal}s)   @EOF if [ $subdir_cnt != 0 ] then cat >> $INDEX << @EOF $subdir_cnt   @EOF fi let "all_cnt = $passed_cnt + $failed_cnt" cat >> $INDEX << @EOF   all $all_cnt   Failed   $failed_cnt   @EOF echo echo_ok "Passed : $passed_cnt" echo_failed "Failed : $failed_cnt" echo echo "The results are available in $TODAYDIR" echo "or via the link http://$ipv4_winnetou/testresults/$TESTDATE" ENDDATE=`date +%Y%m%d-%H%M-%S` echo echo "Finished : $ENDDATE ($timetotal_fmt)"