[cairo-commit] 3 commits - test/cairo-test.c test/make-html.pl test/README
Brian Ewins
brianewins at kemper.freedesktop.org
Fri Nov 9 17:34:46 PST 2007
test/README | 8 ++++
test/cairo-test.c | 21 ++++++++++-
test/make-html.pl | 102 +++++++++++++++++++++++-------------------------------
3 files changed, 72 insertions(+), 59 deletions(-)
New commits:
commit 77e9b05b2b011ce964287ff493c2b421977107ee
Author: Brian Ewins <Brian.Ewins at gmail.com>
Date: Sat Nov 10 01:34:03 2007 +0000
[test] support using a previous build as a reference
Allow using a previous test output directory as a source of
reference images. To make use of this, set the environment
variable 'CAIRO_REF_DIR' to point at an old test directory,
relative to the current test directory.
This is useful for testing backends when reference images haven't
been created yet, or which the current reference image structure
can't accomodate, like multiple font backends.
diff --git a/test/README b/test/README
index 90f0287..840e1ad 100644
--- a/test/README
+++ b/test/README
@@ -69,6 +69,14 @@ The work involved is similar the work described above for new bugs.
The only distinction is that the test is expected to pass so it
should not be added to the XFAIL_TESTS list.
+While working on a test
+-----------------------
+Before a bugfix or feature is ready, it may be useful to compare
+output from different builds. For convenience, you can set
+CAIRO_REF_DIR to point at a previous test directory, relative
+to the current test directory, and any previous output will be
+used by preference as reference images.
+
When a bug is fixed
-------------------
diff --git a/test/cairo-test.c b/test/cairo-test.c
index d2e4157..f036163 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -96,6 +96,7 @@ static const char *vector_ignored_tests[] = {
* context object there, (though not a whole lot). */
FILE *cairo_test_log_file = NULL;
const char *srcdir;
+const char *refdir;
/* Used to catch crashes in a test, such that we report it as such and
* continue testing, although one crasher may already have corrupted memory in
@@ -158,7 +159,20 @@ cairo_ref_name_for_test_target_format (const char *test_name,
{
char *ref_name = NULL;
- /* First look for a target/format-specific reference image. */
+ /* First look for a previous build for comparison. */
+ if (refdir) {
+ xasprintf (&ref_name, "%s/%s-%s-%s%s", refdir,
+ test_name,
+ target_name,
+ format,
+ CAIRO_TEST_PNG_SUFFIX);
+ if (access (ref_name, F_OK) != 0)
+ free (ref_name);
+ else
+ goto done;
+ }
+
+ /* Next look for a target/format-specific reference image. */
xasprintf (&ref_name, "%s/%s-%s-%s%s", srcdir,
test_name,
target_name,
@@ -477,6 +491,7 @@ cairo_test_expecting (cairo_test_t *test,
srcdir = getenv ("srcdir");
if (!srcdir)
srcdir = ".";
+ refdir = getenv ("CAIRO_REF_DIR");
cairo_test_init (test->name);
printf ("%s\n", test->description);
commit f8ee0cdf186bfb30bdeb487b63d42f274c1073ce
Author: Brian Ewins <Brian.Ewins at gmail.com>
Date: Sat Nov 10 01:34:01 2007 +0000
[test] Link to the correct reference image.
The generated test html guesses the names of the reference
images used in the tests. This is quite often wrong, and
makes it harder to change policy in cairo-test. Teach
make-html.pl to parse the log for the path to the reference image.
diff --git a/test/make-html.pl b/test/make-html.pl
index 9720c06..8bc961e 100755
--- a/test/make-html.pl
+++ b/test/make-html.pl
@@ -54,20 +54,40 @@ if ($#ARGV >= 0) { @files = @ARGV; } else { @files = <*.log>; }
foreach (<@files>) {
my $testname;
+ my $out_path, $diff_path, $ref_path;
my $fn = $_;
(open LOG, $fn) || next;
while (<LOG>) {
+ if (/^OUTPUT: (.*)$/) {
+ $out_path = $1;
+ next;
+ }
+ if (/^DIFFERENCE: (.*)$/) {
+ $diff_path = $1;
+ next;
+ }
+ if (/^REFERENCE: (.*)$/) {
+ $ref_path = $1;
+ next;
+ }
next unless /^TEST: (.*) TARGET: (.*) FORMAT: (.*) OFFSET: (.*) SIMILAR: (.*) RESULT: ([A-Z]*).*$/;
$testname = $1 if !defined($testname);
$tests->{$1} = {} unless $tests->{$1};
$tests->{$1}->{$2} = {} unless $tests->{$1}->{$2};
$tests->{$1}->{$2}->{$3} = {} unless $tests->{$1}->{$2}->{$3};
$tests->{$1}->{$2}->{$3}->{$4} = {} unless $tests->{$1}->{$2}->{$3}->{$4};
- $tests->{$1}->{$2}->{$3}->{$4}->{$5} = $6;
+ $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'out'} = $out_path;
+ $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'diff'} = $diff_path;
+ $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'ref'} = $ref_path;
+ $tests->{$1}->{$2}->{$3}->{$4}->{$5}->{'result'} = $6;
$teststats->{$2} = {"PASS" => 0, "FAIL" => 0, "XFAIL" => 0, "UNTESTED" => 0, "CRASHED" =>0}
unless $teststats->{$2};
($teststats->{$2}->{$6})++;
+
+ undef $out_path;
+ undef $diff_path;
+ undef $ref_path;
}
close LOG;
@@ -158,7 +178,6 @@ printl '<body>';
printl '<table border="1">';
print '<tr><th>Test</th>';
-print '<th>Ref</th>' if $config_show_ref;
foreach my $target (@targets) {
print '<th>', $target, '</th>';
@@ -166,7 +185,6 @@ foreach my $target (@targets) {
printl '</tr>';
print '<tr><td></td>';
-print '<td></td>' if $config_show_ref;
foreach my $target (@targets) {
print '<td>';
@@ -178,37 +196,6 @@ foreach my $target (@targets) {
}
printl '</tr>';
-sub testref {
- my ($test, $format, $rest) = @_;
- my $fmtstr = "";
- if ($format eq "rgb24") {
- $fmtstr = "-rgb24";
- }
-
- return "$test$fmtstr-ref.png";
-}
-
-sub testfiles {
- my ($test, $target, $format, $offset, $similar, $rest) = @_;
- my $fmtstr = "";
- my $offstr = "";
- my $simstr = "";
- if ($format eq "rgb24") {
- $fmtstr = "-rgb24";
- } elsif ($format eq "argb32") {
- $fmtstr = "-argb32";
- }
- if ($similar ne "0") {
- $simstr = "-similar";
- }
- if ($offset ne "0") {
- $offstr = "-" . $offset;
- }
-
- return ("out" => "$test-$target$fmtstr$simstr$offstr-out.png",
- "diff" => "$test-$target$fmtstr$simstr$offstr-diff.png");
-}
-
sub img_for {
my ($fn, $withlink) = @_;
@@ -237,35 +224,42 @@ foreach my $test (sort(keys %$tests)) {
if ($tgtdata) {
my $testres = $tgtdata->{$format}->{$offset}->{$similar};
if ($testres) {
- my %testfiles = testfiles($test, $target, $format,
- $offset, $similar);
- $testline .= "<td class=\"$testres\">";
+ my %testfiles;
+ $testfiles{'out'} = $testres->{'out'};
+ $testfiles{'diff'} = $testres->{'diff'};
+ $testfiles{'ref'} = $testres->{'ref'};
+
+ $testline .= "<td class=\"$testres->{'result'}\">";
$teststats{$target}{$testres}++;
- if ($testres eq "PASS") {
+ if ($testres->{'result'} eq "PASS") {
if ($config_show_all) {
- $testline .= img_for($testfiles{'out'},1);
+ $testline .= img_for($testfiles{'out'},1);
}
- } elsif ($testres eq "FAIL") {
+ } elsif ($testres->{'result'} eq "FAIL") {
if ($config_show_fail || $config_show_all) {
- $testline .= img_for($testfiles{'out'},1);
+ $testline .= img_for($testfiles{'out'},1);
$testline .= " ";
- $testline .= img_for($testfiles{'diff'},1);
+ $testline .= img_for($testfiles{'diff'},1);
+ $testline .= " ";
+ $testline .= img_for($testfiles{'ref'},1);
}
- } elsif ($testres eq "CRASHED") {
- $testline .= "!!!CRASHED!!!";
- } elsif ($testres eq "XFAIL") {
+ } elsif ($testres->{'result'} eq "CRASHED") {
+ $testline .= "!!!CRASHED!!!";
+ } elsif ($testres->{'result'} eq "XFAIL") {
#nothing
if ($config_show_all) {
- $testline .= img_for($testfiles{'out'},1);
+ $testline .= img_for($testfiles{'out'},1);
#$testline .= "<hr size=\"1\">";
$testline .= " ";
- $testline .= img_for($testfiles{'diff'},1);
+ $testline .= img_for($testfiles{'diff'},1);
+ $testline .= " ";
+ $testline .= img_for($testfiles{'ref'},1);
}
- } elsif ($testres eq "UNTESTED") {
+ } elsif ($testres->{'result'} eq "UNTESTED") {
#nothing
} else {
- $testline .= "UNSUPPORTED STATUS (update make-html.pl)";
- }
+ $testline .= "UNSUPPORTED STATUS '$testres->{'result'}' (update make-html.pl)";
+ }
$testline .= "</td>";
} else {
@@ -275,23 +269,18 @@ foreach my $test (sort(keys %$tests)) {
$testline .= '<td></td>';
}
}
- my $testref = testref($test, $format);
print '<tr><td>';
if ($config_show_inline) {
print "$test ($format/$offset) ";
print "(<a href=\"" . string_to_data("text/plain",$logs->{$test}) . "\">log</a>)";
} else {
- print "<a href=\"$testref\">", $test, ' (', $format, '/', $offset, ($similar ? ' similar' : ''), ')</a> ';
+ print $test, ' (', $format, '/', $offset, ($similar ? ' similar' : ''), ') ';
print "(<a href=\"$test.log\">log</a>)";
}
print '</td>';
- if ($config_show_ref) {
- print "<td>" . img_for($testref,1) . "</td>";
- }
-
print $testline;
print "</tr>\n";
@@ -301,4 +290,3 @@ foreach my $test (sort(keys %$tests)) {
}
print "</table></body></html>\n";
-
commit 15fc1bbefb49fefedca53c47ccfc7edace0acbfe
Author: Brian Ewins <Brian.Ewins at gmail.com>
Date: Sat Nov 10 01:33:47 2007 +0000
[test] log tested paths so that they can be parsed.
The test logs currently do not record the paths of
image output, the reference images tested against, and
the diffs created. This means that make-html.pl has to
duplicate the policy in cairo-test.c. Fix this by teaching
cairo-test.c to log the paths.
diff --git a/test/cairo-test.c b/test/cairo-test.c
index de8ed1b..d2e4157 100644
--- a/test/cairo-test.c
+++ b/test/cairo-test.c
@@ -387,6 +387,7 @@ cairo_test_for_target (cairo_test_t *test,
xunlink (png_name);
(target->write_to_png) (surface, png_name);
+ cairo_test_log ("OUTPUT: %s\n", png_name);
if (!ref_name) {
cairo_test_log ("Error: Cannot find reference image for %s/%s-%s-%s%s\n",srcdir,
test->name,
@@ -397,7 +398,8 @@ cairo_test_for_target (cairo_test_t *test,
goto UNWIND_CAIRO;
}
- cairo_test_log ("Comparing result against reference image: %s\n", ref_name);
+ cairo_test_log ("REFERENCE: %s\n", ref_name);
+ cairo_test_log ("DIFFERENCE: %s\n", diff_name);
if (target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED) {
diff_status= image_diff_flattened (png_name, ref_name, diff_name,
More information about the cairo-commit
mailing list