Admin/isatest/isatest-check
author blanchet
Mon, 03 Feb 2014 23:38:33 +0100
changeset 55308 dc68f6fb88d2
parent 31582 4753c317d5c1
permissions -rwxr-xr-x
properly overwrite replay data from one compression iteration to another
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
22410
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     1
#!/usr/bin/env bash
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     2
#
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     3
# Author: Gerwin Klein, TU Muenchen
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     4
#
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     5
# DESCRIPTION: sends email for failed tests, checks for error.log,
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     6
#              generates development snapshot if test ok
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     7
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
     8
## global settings
22411
1956d895a4ed adjust paths
kleing
parents: 22410
diff changeset
     9
. ~/admin/isatest/isatest-settings
22410
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    10
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    11
# produce empty list for patterns like isatest-*.log if no 
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    12
# such file exists 
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    13
shopt -s nullglob
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    14
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    15
# mail program
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    16
MAIL=$HOME/bin/pmail
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    17
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    18
# tmp file for sending mail
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    19
TMP=/tmp/isatest-makedist.$$
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    20
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    21
export DISTPREFIX
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    22
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    23
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    24
## diagnostics
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    25
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    26
PRG="$(basename "$0")"
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    27
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    28
function usage()
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    29
{
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    30
  echo
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    31
  echo "Usage: $PRG"
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    32
  echo
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    33
  echo "   sends email for failed tests, checks for error.log,"
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    34
  echo "   generates development snapshot if test ok."
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    35
  echo "   To be called by cron."
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    36
  echo
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    37
  exit 1
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    38
}
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    39
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    40
function fail()
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    41
{
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    42
  echo "$1" >&2
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    43
  exit 2
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    44
}
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    45
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    46
## main
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    47
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    48
# check if tests are still running, wait for them a couple of hours
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    49
i=0
28526
a30b9cf3502e leave a log message when no snapshot is generated
kleing
parents: 28508
diff changeset
    50
while [ -n "$(ls $RUNNING)" -a $i -lt 40 ]; do 
a30b9cf3502e leave a log message when no snapshot is generated
kleing
parents: 28508
diff changeset
    51
    sleep 900
22410
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    52
    let "i = i+1"
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    53
done
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    54
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    55
FAIL=0
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    56
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    57
# still running -> give up
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    58
if [ -n "$(ls $RUNNING)" ]; then
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    59
    echo "Giving up waiting for test to finish at $(date)." > $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    60
    echo >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    61
    echo "Sessions still running:" >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    62
    echo "$(ls $RUNNING)" >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    63
    echo >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    64
    echo "Attaching all error logs collected so far." >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    65
    echo >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    66
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    67
    if [ -e $ERRORLOG ]; then
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    68
        cat $ERRORLOG >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    69
    fi
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    70
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    71
    echo "Have a nice day," >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    72
    echo "  isatest" >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    73
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    74
    for R in $MAILTO; do
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    75
        LOGS=$ERRORDIR/isatest*.log
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    76
        $MAIL "isabelle test taking too long" $R $TMP $LOGS
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    77
    done
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    78
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    79
    rm $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    80
    
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    81
    FAIL=1
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    82
elif [ -e $ERRORLOG ]; then
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    83
  # no tests running, check if there were errors
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    84
    cat $ERRORLOG > $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    85
    echo "Have a nice day," >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    86
    echo "  isatest" >> $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    87
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    88
    for R in $MAILTO; do
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    89
        LOGS=$ERRORDIR/isatest*.log
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    90
        $MAIL "isabelle test failed" $R $TMP $LOGS
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    91
    done
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    92
    
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    93
    rm $TMP
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    94
fi
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    95
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    96
# generate development snapshot page only for successful tests
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    97
# (failures in experimental sessions ok)
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
    98
if [ "$FAIL"=="0" -a "$(echo $ERRORDIR/isatest*[^e].log)" == "$(echo)" ]; then
28508
e8197ea2703b needs -b option for isabelle getenv
kleing
parents: 28504
diff changeset
    99
  (cd $HOME/devel-page; env DISTNAME=`$DISTPREFIX/Isabelle/bin/isabelle getenv -b ISABELLE_IDENTIFIER` make)
28539
bdb308737bfd do logging to MASTERLOG centrally (avoid multiple writers over NFS as
kleing
parents: 28526
diff changeset
   100
  log "generated development snapshot web page."
28526
a30b9cf3502e leave a log message when no snapshot is generated
kleing
parents: 28508
diff changeset
   101
else
28539
bdb308737bfd do logging to MASTERLOG centrally (avoid multiple writers over NFS as
kleing
parents: 28526
diff changeset
   102
  log "test failures, no web snapshot generated."
22410
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
   103
fi
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
   104
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
   105
exit 0
da313b67a04d moved all isatest/cron job related files to own directory
kleing
parents:
diff changeset
   106
## end