# HG changeset patch # User rosfran # Date 1177623277 -3600 # Node ID a3252ef8e61bb67556436bc3bd8f4535afd36251 # Parent 7da2a5e32fa6afb4cee34a6094e2b5866c2d78b0 [svn r607] Added some features to the regression tests. diff -r 7da2a5e32fa6 -r a3252ef8e61b gmyth/configure.ac --- a/gmyth/configure.ac Thu Apr 26 22:33:00 2007 +0100 +++ b/gmyth/configure.ac Thu Apr 26 22:34:37 2007 +0100 @@ -112,6 +112,25 @@ LDFLAGS="$LDFLAGS -lgcov" fi +AC_ARG_ENABLE(mudflap-debug, + AS_HELP_STRING([--enable-mudflap-debug], + [enable libmudflap debug (default = no)]), + [ + if test $enableval = no; then + USE_MUDFLAP_DEBUG=no; + else + USE_MUDFLAP_DEBUG=yes; + fi + ],[ + USE_MUDFLAP_DEBUG=no; + ] +) + +if [ test $USE_MUDFLAP_DEBUG = yes ]; then + CFLAGS="$CFLAGS -fmudflap -fmudflapth -O0" + LDFLAGS="$LDFLAGS -lmudflapth" +fi + # Check for pkgconfig AC_CHECK_PROG(HAVE_PKGCONFIG, pkg-config, yes, no) # Give error and exit if we don't have pkgconfig diff -r 7da2a5e32fa6 -r a3252ef8e61b gmyth/tests/Makefile.am --- a/gmyth/tests/Makefile.am Thu Apr 26 22:33:00 2007 +0100 +++ b/gmyth/tests/Makefile.am Thu Apr 26 22:34:37 2007 +0100 @@ -1,3 +1,4 @@ +INCLUDES = -I$(top_srcdir)/src -I$(top_builddir)/src LDADD = \ $(top_srcdir)/src/.libs/libgmyth.la diff -r 7da2a5e32fa6 -r a3252ef8e61b gmyth/tests/README --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/gmyth/tests/README Thu Apr 26 22:34:37 2007 +0100 @@ -0,0 +1,166 @@ +=== Regression Testing Scripts How-To === + +Authors: + Rafael Espindola + Rosfran Borges + +Requirements: + +The only additional dependency for running the tests is python 2.3 or +newer. Other then this, the tests can be run on any system that can +compile gmyth. + +The test system has some optional features that require some additional +tools: + +To use mudflap during the test, its headers must be installed. Package +libmudflap0-dev provides them on Debian. + +To test with valgrind, the valgrind package must be installed. It might +be necessary to tweak the suppression file. + +To compute line coverage, lcov must be installed. Unfortunately, the +current release of lcov has some problems finding the source code when +libtool is used. So it is necessary to download the cvs version +(http://ltp.cvs.sourceforge.net/ltp/utils/analysis/lcov/bin/lcov). + +Running the Tests: + +The tests can be run by the check target. I.E., by running "make check" +after compiling gmyth. The way in which the tests are run depends on +the options passed to configure: + +If the --enable-coverage option is used, the library and test programs +are compiled with support for generating line coverage +information. After running the tests, a summary will be available in +tests/regression/covere/index.html. + +If --enable-mudflap-debug is used, libmudflap is linked with +gmyth. Libmudflap will test for and report some common memory +management errors. + +If none of these options are used, valgrind will be used if it is +available. Otherwise, the tests will be runs without any special +tools. + +Basics of the test system: + +Each test consists of a small C program that links with gmyth and a +python script. The script runs the C program and interacts with it. By +doing so, it can control which code paths are executed. + +With this separation it is possible to reuse the C program in many +tests and use a higher level language for building the tests. + +If a script returns 0, the test is considered successful. If it +returns 77, the test is ignored. If it returns any other value, the +test is a failure. After running all tests, Automake reports how many +of them were successful. + +Declaring test programs and test scripts: + +The test programs are declared in the check_PROGRAMS variable in +Makefile.am and the test scripts in the TESTS variable. + +For example, gmyth_test_livetv is small program that contains just one control +point. test_livetv.py and test_epg.py are two scripts +that test different actions (tests) on the gmyth modules. To declare them, +the Makefile.am should contain: + +-------------------------------------- +check_PROGRAMS = gmyth_test_livetv +TESTS = test_livetv.py test_epg.py +-------------------------------------- + +Synchronizing the Tested Program and the Test Script: + +Automake will only run the test script. It is up to the script to run +the program to be tested. Since it takes some time for the test +program to start up, the test script must be informed when the test +program is ready. The test script must also be able to instruct the +program to exit + +In gmyth, the following technique is used. + the script starts the server "inside" a pipe + the test script tries to read a line from the pipe and blocks + after startup, the server prints "===SYNC===" to stdout + the read in the script succeeds + the server tries to read a line from stdin and blocks + the test script does the testing + the test script writes an empty line to the pipe + the read in the server succeeds + the test script waits for the server to exit + +In the test program this is implemented by the following template: + +-------------------------------------------------- +#include "common.h" +... +start the server +printf(SYNC_STRING); +fflush(NULL); +getchar(); +stop the server +-------------------------------------------------- + +In the test script the synchronization is done automatically inside +the start function: + +------------------------------------------------------- +p = start('gmyth_test_livetv') +------------------------------------------------------- + +The API + +Some utility function are provided that abstract some low level stuff +like were to find the libraries, the kind of test and error checking. + + +testing_tools.start(exe, ignore): + +Start the executable "exe" and ignore at most "ignore" libmudflap errors. + +The "ignore" argument exists because libmudflap doesn't has a builtin suppression +mechanism. + +When start return the synchronization described above is already +done. + +Start returns an object that represent the process. This object has the +following attributes: + + stdin: A pipe to the standard input of the process + stdout: A pipe to the standard output of the process + stderr: A pipe to the standard error of the process + + +testing_tools.finish(p): + +Stop the process "p" + +This function tries to stop the process nicely by using the protocol +described is the synchronization section. If the process fails to quit, +it is killed. + +Error checking is dependent of how the tests are being run: + + coverage or without any tools: check if the program returned 0 + mudflap: similar coverage, but also check that libmudflap didn't print any errors + valgrind: similar to coverage, but also check that valgrind didn't + find any errors or memory leaks. + +if no errors are found, finish returns 0. If any error is found, it returns 1. + +Example: +The following script just starts and stops the 'gmyth_test_livetv' program. + +-------------------------------------- +#!/usr/bin/python + +from test import start +import sys + +p = start('gmyth_test_epg', 2) +sys.exit(finish(p)) +---------------------------------- + diff -r 7da2a5e32fa6 -r a3252ef8e61b gmyth/tests/coverage.py --- a/gmyth/tests/coverage.py Thu Apr 26 22:33:00 2007 +0100 +++ b/gmyth/tests/coverage.py Thu Apr 26 22:34:37 2007 +0100 @@ -21,7 +21,7 @@ os.system('genhtml gmyth.info') exe = 'test' -exe = '.libs/' + exe +exe = [ '.libs/' + exe ] if is_coverage(exe): bdir = build_dir(exe) gen_cov_info(bdir)