[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
updated test scripts
From: |
Paul Kienzle |
Subject: |
updated test scripts |
Date: |
Tue, 2 May 2000 07:09:22 +0100 (BST) |
test.m
## Copyright (C) 2000 Paul Kienzle
##
## This program is free software and may be used for any purpose. This
## copyright notice must be maintained. Paul Kienzle is not responsible
## for the consequences of using this software.
## test('name')
## Perform all tests in the first file matching 'name', 'name.m' or
## 'name.cc' in your loadpath. Tests are done interactively, stopping
## when they encounter the first error.
##
## test('name', ['quiet'|'normal'|'verbose'])
## 'quiet': Don't do tests which require user interaction
## (e.g., looking at graphs or examining results vectors).
## 'normal': Do tests which require user interaction, but don't
## display tests as they happen.
## 'verbose': Display tests as they happen.
## Tests are done interactively, stopping at the first error.
##
## test('name', ['quiet'|'normal'|'verbose'], fid)
## 'quiet': Don't log all the tests as they happen, just the errors.
## 'normal': Log all tests as they happen, but don't do tests
## which require user interaction.
## 'verbose': Do tests which require user interaction.
## Batch processing. Write errors to already open file fid (hopefully
## then when octave crashes this file will tell you what was happening
## when it did). You can use stdout if you want to see the results as
## they happen.
##
## success = test(...)
## return true if all the tests succeeded.
##
## Lines which start with %! in file xxxx are test lines written in
## in Octave which are evaluated when you call the function test('xxxx').
## You can even embed them in C or C++ files if you do something like:
## #if 0
## %! disp('this is a test')
## #endif
##
## Since eval() will stop at the first error it encounters, you must
## divide your tests up into blocks, with anything in a separate
## block (indicated by << >>) evaluated separately. Blocks may not
## nest.
##
## There are a number of special block types:
## <<shared x,y,z>>
## Variables which are shared between all remaining blocks.
## Variables are all initialized to [], even if they are alreay
## defined.
## <<assert code::answer>>
## Evaluates code and fail if !all(all(ans=answer)). If ::answer
## is not specified, then all(all(ans)) must evaluate to true.
## The code may use any shared variables.
## <<usage name(args)>>
## Verify that usage info is printed when name(args) is evaluated.
## If name(args) is not given, then the name passed to test is
## used without arguments.
## <<error code>>
## The code should generate an error message. Test fails
## otherwise. You may use any shared variables in the code.
## <<noisy code>>
## The user will be asked to pause after this block of code
## is executed. Put all graphs and print statements in this
## kind of block.
## << code>>
## Arbitrary code. Don't do any user interaction (i.e., graphs
## or print statements). Assign values needed to shared
## variables but don't perform any tests. Use <<assert condition>>
## to test results. (well, if you do do tests, you can signal
## failure by a call to error("");
## <<# whatever>>
## don't process the block. Useful if you want to temporarily
## disable some tests. WARNING: because of my rather simplistic
## parsing, #<< whatever>> does NOT comment out the block.
## TODO: Consider protecting __xxx variables by eliminating them
## TODO: from the shared list automatically.
## TODO: Assertion values should be logged in the test file in
## TODO: batch mode. How to output an arbitrary variable
## TODO: pretty-printed to a file? "disp(v,fid)" would be nice.
## TODO: Verify that the error returned is a usage error. How??
## TODO: Detect warnings?
## TODO: Determine if error is elegant or if it is something obscure
## TODO: like index out of range or matrix contains empty
## TODO: Assertions need a tolerance variable for equality.
function __success_r = test (__name, __flag, __fid)
if (nargin < 2 || isempty(__flag))
__flag = 'normal';
endif
if (nargin < 3)
__fid = [];
endif
if (nargin < 1 || nargin > 3 || !isstr(__name) || !isstr(__flag))
usage("success = test('name', ['quiet'|'normal'|'verbose'], fid)");
endif
__batch = (!isempty(__fid));
if (strcmp(__flag, "normal"))
__alltests = !__batch;
__verbose = __batch;
elseif (strcmp(__flag, "quiet"))
__alltests = 0;
__verbose = 0;
elseif (strcmp(__flag, "verbose"))
__alltests = 1;
__verbose = 1;
else
error(["test unknown flag '" __flag "'"]);
endif
## information from test will be introduced by "key"
__status = "***** ";
__fail = "!!!!! ";
## decide if error messages should be collected
if (__batch)
fputs (__fid, [__status "processing " __name "\n" ]);
else
__fid = stdout;
endif
## locate the file to test
__file = file_in_loadpath (__name);
if (isempty (__file))
__file = file_in_loadpath ([__name ".m"]);
endif
if (isempty (__file))
__file = file_in_loadpath ([__name ".cc"]);
endif
if (isempty (__file))
fputs(__fid, [__fail __name " does not exist in path\n" ]);
if (nargout > 0) __success_r = 0; endif
return;
endif
## grab the test code from the file
__body = system([ "sed -n 's/^%!//p' " __file]);
if (isempty (__body))
fputs(__fid, [ __fail __file " has no tests available\n" ]);
if (nargout > 0) __success_r = 0; endif
return;
endif
## chop it up into blocks for evaluation
__blockstart = findstr ("<<", __body);
__blockend = findstr (">>", __body);
if (any (size (__blockstart) != size (__blockend))
|| any (__blockstart > __blockend))
fputs(__fid, [ __fail __file " has invalid << block structure>>\n" ]);
if (nargout > 0) __success_r = 0; endif
return;
endif
## ready to start tests ... if in batch mode, tell us what is happening
if (__verbose)
disp ([ __status __file ]);
endif
## assume all tests will pass
__all_success = 1;
## if no blocks given, pretend we have one big block
if isempty(__blockstart)
__blockstart = 1;
__blockend = length (__body) + 3;
__body = ["<< " __body ">>"];
endif
## process each block separately, initially with no shared variables
__shared = " ";
__shared_r = " ";
for __i=1:length(__blockstart)
## extract the block
__block = __body(__blockstart(__i)+2:__blockend(__i)-1);
## let the user/logfile know what is happening
if (__verbose)
fputs (__fid, [__status "<<" __block ">>\n"]);
endif
## pad with blanks for indexing purposes
__block = [ __block " " ];
## assume the block will succeed;
__success = 1;
__msg = [];
__show_assert = 0;
### SHARED
if strcmp (__block (1:6), "shared")
__code = deblank (__block (7:length(__block)));
## variables need default values
try
eval([strrep(__code,",","=[];") "=[];"]);
__success = 1; # in case the user is fiddling them
__msg = [];
if (length (__code) == 0)
__shared = " ";
__shared_r = " ";
else
__shared = __code;
__shared_r = ["[ " __code "] = "];
endif
catch
__success = 0;
__msg = [ __fail "shared variable initialization\n"];
end_try_catch
### ASSERT
elseif strcmp (__block (1:6), "assert")
__code = deblank (__block (7:length (__block)));
## extract answer experssion
__answer_idx = index(__code, "::");
if (__answer_idx != 0)
__answer_code = __code(__answer_idx+2:length(__code));
__code = __code(1:__answer_idx-1);
endif
## add __value to the returned variables
if (!strcmp(__shared, " "))
__returns = [ "[ __value, " __shared "] = " ];
else
__returns = "__value =";
endif
## evaluate test expression
try
eval(["function " __returns "__test__(" __shared ")\n" ...
__code ";\n__value = ans;\nendfunction"]);
eval([ __returns "__test__(" __shared ");"]);
__test_value = __value;
catch
__success = 0;
__msg = [ __fail "assert expression caused an error!\n" \
__error_text__];
end_try_catch
clear __test__;
## evaluate answer
if (__success)
if(__answer_idx>0)
try
eval(["function " __returns "__test__(" __shared ")\n" \
__answer_code ";\n__value = ans;\nendfunction"]);
eval([ __returns "__test__(" __shared ");"]);
if (!all(size(__test_value)==size(__value)) ||
!all(all(__test_value == __value)))
__success = 0;
__show_assert = 2;
__msg = [ __fail "assert expression != answer\n"];
endif
catch
__success = 0;
__msg = [ __fail "assert answer caused an error!\n" \
__error_text__];
end_try_catch
clear __test__;
elseif (!all (all (__test_value)))
__success = 0;
__show_assert = 1;
__msg = [ __fail "assert expression not all true\n"];
endif
endif
### USAGE
elseif strcmp (__block(1:5), "usage")
__code = deblank(__block (6:length (__block)));
if (length (__code) == 0)
__code = __name;
endif
__success = 0;
__msg = [ __fail "usage test does not cause an error!\n"];
try
eval ([__code ";"]);
catch
## if ( length (__error_text__) > 7 ...
## && strcmp (__error_text__(1:7), "usage: ") );
__success = 1;
__msg = [ __status "usage produced:\n" __error_text__];
## else
## __success = 0;
## __msg = [ __fail "usage produced an error!\n" __error_text__];
## endif
end_try_catch
### ERROR
elseif strcmp (__block(1:5), "error")
__code = deblank(__block (6:length (__block)));
try
eval(["function " __shared_r "__test__(" __shared ")\n" ...
__code "\nendfunction"]);
catch
__success = 0;
__msg = [ __fail "error test syntax error!\n" __error_text__];
end_try_catch
if (__success)
__success = 0;
__msg = [ __fail "error test does not cause an error!\n" ];
try
eval([ __shared_r "__test__(" __shared ");"]);
catch
__success = 1;
__msg = [ __status "error test caused the following error:\n" ...
__error_text__ ];
end_try_catch
clear __test__;
endif
### NOISY
elseif strcmp (__block(1:5), "noisy")
if (__alltests)
__code = deblank(__block (6:length (__block)));
try
eval(["function " __shared_r "__test__(" __shared ")\n" ...
__code "\nendfunction"]);
eval([__shared_r "__test__(" __shared ");"]);
input("Press <enter> to continue: ");
catch
__success = 0;
__msg = [ __status "noisy test caused an error!\n" __error_text__];
end_try_catch
clear __test__;
else
__msg = [ __status "skipping\n" ];
endif
### comment block
elseif strcmp (__block(1:1), "#")
__msg = [ __status "skipping\n" ];
### default code block
elseif isspace(__block(1))
__code = deblank(__block (2:length (__block)));
try
eval(["function " __shared_r "__test__(" __shared ")\n" ...
__code "\nendfunction"]);
eval([__shared_r "__test__(" __shared ");"]);
catch
__success = 0;
__msg = [ __fail "code error\n" __error_text__];
end_try_catch
clear __test__;
### unkown block
else
__success = 0;
__msg = [ __fail "unknown test type!\n"];
endif
## All done. Remember if we were successful and print any messages
if (__success == 0)
__all_success = 0;
## make sure the user knows what caused the error
if (!__verbose)
fputs (__fid, [__status "<<" __block ">>\n"]);
endif
fputs (__fid, __msg);
if (!__batch)
## display assert values
if (__show_assert >= 1)
disp([__fail "assert expression evaluates to:"]);
disp(__test_value);
endif
if (__show_assert == 2)
disp([__fail "assert answer evaluates to:"]);
disp(__value);
endif
## stop after one error if not in batch mode
if (nargout > 0) __success_r = 0; endif
return;
endif
elseif (__verbose && !isempty(__msg))
fputs (__fid, __msg);
endif
endfor
if (nargout > 0) __success_r = 0; endif
endfunction
%! text outside the blocks is not processed.
%! <<usage test>>
%! <<usage test(1,2,3,4)>>
%! <<error test("test", 'bogus'); >>
%! <<shared a,b,c>>
%! <<assert [isempty(a) isempty(b) isempty(c)]>>
%! << a=1; b=2; c=3;>>
%! <<assert [a b c]::[1 2 3]>>
%! <<noisy disp("here are the values for a b and c:")
%! a, b, c>>
%! << c=6;>>
%! <<assert [a b c]::[1 2 6]>>
%! <<shared __msg,__success>>
%! << __msg = "invisible only if 'quiet' since otherwise it is verbose\n";>>
%! <<assert !exist("a")>>
%! <<#this is a comment block. it can contain anything.>>
%! <<#
%! it is the # after the block open < < which determines this.>>
%! ## failure tests.
%! << error("Failure tests. To see them all, use test('test','verbose',1)");
%! >>
%! <<assert
%! "assertion is false, so fail";
%! [a b c]::[1 3 6]>>
%! <<shared a,b,c>>
%! <<bogus>>
%! <<usage toeplitz([1,2,3])>>
%! << syntax errors in this code>>
%! <<shared garbage in and garbage out>>
%! <<assert 5::5::5>>
%! <<usage garbage>>
%! <<error when it is a syntax error, then failure fails>>
%! <<error "this is an error because it succeeds.";>>
%! <<noisy syntax error)>>
%! <<error test('/etc/passwd');
%! test("nonexistent file");
%! ## These don't signal an error, so the test for an error fails. Note
%! ## that the call doesn't reference the current fid (it is unavailable),
%! ## so of course the informational message is not printed in the log.>>
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- updated test scripts,
Paul Kienzle <=