[cairo] cairo-perf cleanups

Chris Wilson chris at chris-wilson.co.uk
Tue Mar 13 03:32:36 PDT 2007


I'm having a little difficulty getting stable results out of cairo-perf.
I took a look through the source and have a fix for one bug and a couple
of cosmetic suggestions. Afterwards the results appear move consistent,
but I still worry that the default number of iterations for
cairo-perf-diff is insufficient. What I had thought had been implemented
(or was it only discussed?) was to ensure that each test ran for a
minimum length of time.
--
Chris Wilson
-------------- next part --------------
>From c8797eeec0dbd5f6f0626fa6b22348b2588d082e Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 09:43:01 +0000
Subject: [PATCH] Compute std.dev over values[min_valid : min_valid + num_valid]
---
 perf/cairo-stats.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/perf/cairo-stats.c b/perf/cairo-stats.c
index 2c6c7dc..43711a9 100644
--- a/perf/cairo-stats.c
+++ b/perf/cairo-stats.c
@@ -92,7 +92,7 @@ _cairo_stats_compute (cairo_stats_t		*st
     stats->median_ticks = values[min_valid + num_valid / 2];
 
     sum = 0.0;
-    for (i = min_valid; i < num_valid; i++) {
+    for (i = min_valid; i < min_valid + num_valid; i++) {
 	delta = values[i] - mean;
 	sum += delta * delta;
     }
-- 
1.4.1

-------------- next part --------------
>From 9e594a10a04b124724ca370ff5f230e2d665a72f Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 10:09:52 +0000
Subject: [PATCH] Discard statistically insignificant differences.

Discard as an "uninteresting change" if the two timings lie within 2
sigma of each other.
---
 perf/cairo-perf-diff-files.c |   16 +++++++++++++---
 perf/cairo-stats.c           |    3 ++-
 perf/cairo-stats.h           |    1 +
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/perf/cairo-perf-diff-files.c b/perf/cairo-perf-diff-files.c
index 8b6292a..8495efb 100644
--- a/perf/cairo-perf-diff-files.c
+++ b/perf/cairo-perf-diff-files.c
@@ -560,6 +560,7 @@ cairo_perf_report_diff (cairo_perf_repor
     }
 
     for (i = 0; i < num_diffs; i++) {
+	double min_value, max_value;
 	diff = &diffs[i];
 
 	change = diff->speedup;
@@ -572,9 +573,18 @@ cairo_perf_report_diff (cairo_perf_repor
 	if (change - 1.0 < min_change)
 	    continue;
 
-	/* Also discard as uninteresting if the change is less than
-	 * the sum each of the standard deviations. */
-	if (change - 1.0 < diff->old->stats.std_dev + diff->new->stats.std_dev)
+	/* Also discard as uninteresting if the change is the new value is
+	 * within 95% [2 sigma] of old value. */
+	min_value = diff->old->stats.mean * (1 - 2*diff->old->stats.std_dev);
+	max_value = diff->old->stats.mean * (1 + 2*diff->old->stats.std_dev);
+	if (diff->new->stats.mean > min_value &&
+		diff->new->stats.mean < max_value)
+	    continue;
+	/* similary, check vice versa. */
+	min_value = diff->new->stats.mean * (1 - 2*diff->new->stats.std_dev);
+	max_value = diff->new->stats.mean * (1 + 2*diff->new->stats.std_dev);
+	if (diff->old->stats.mean > min_value &&
+		diff->old->stats.mean < max_value)
 	    continue;
 
 	if (diff->speedup > 1.0 && ! printed_speedup) {
diff --git a/perf/cairo-stats.c b/perf/cairo-stats.c
index 43711a9..baa4e1d 100644
--- a/perf/cairo-stats.c
+++ b/perf/cairo-stats.c
@@ -61,7 +61,7 @@ _cairo_stats_compute (cairo_stats_t		*st
     qsort (values, num_values,
 	   sizeof (cairo_perf_ticks_t), _cairo_perf_ticks_cmp);
 
-    q1	 	= values[(1*num_values)/4];
+    q1		= values[(1*num_values)/4];
     q3		= values[(3*num_values)/4];
 
     iqr = q3 - q1;
@@ -100,4 +100,5 @@ _cairo_stats_compute (cairo_stats_t		*st
     /* Let's use a std. deviation normalized to the mean for easier
      * comparison. */
     stats->std_dev = sqrt(sum / num_valid) / mean;
+    stats->mean = mean;
 }
diff --git a/perf/cairo-stats.h b/perf/cairo-stats.h
index 3f8a988..48db0be 100644
--- a/perf/cairo-stats.h
+++ b/perf/cairo-stats.h
@@ -33,6 +33,7 @@ typedef struct _cairo_stats {
     cairo_perf_ticks_t median_ticks;
     double ticks_per_ms;
     double std_dev;
+    double mean;
     int iterations;
 } cairo_stats_t;
 
-- 
1.4.1

-------------- next part --------------
>From 38db7e352a7e02c1337ae9c9a2de6fe1478cff31 Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 10:19:15 +0000
Subject: [PATCH] Switch between millseconds and microseconds for timings.

Sometimes the timing is too small to display accurately in milliseconds
using %4.2f, so use microsecond results as appropiate.
---
 perf/cairo-perf-diff-files.c |   34 +++++++++++++++++++++++++---------
 1 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/perf/cairo-perf-diff-files.c b/perf/cairo-perf-diff-files.c
index 8495efb..b2a26e8 100644
--- a/perf/cairo-perf-diff-files.c
+++ b/perf/cairo-perf-diff-files.c
@@ -561,6 +561,7 @@ cairo_perf_report_diff (cairo_perf_repor
 
     for (i = 0; i < num_diffs; i++) {
 	double min_value, max_value;
+	double old_value, new_value;
 	diff = &diffs[i];
 
 	change = diff->speedup;
@@ -573,8 +574,13 @@ cairo_perf_report_diff (cairo_perf_repor
 	if (change - 1.0 < min_change)
 	    continue;
 
-	/* Also discard as uninteresting if the change is the new value is
-	 * within 95% [2 sigma] of old value. */
+	/* Also discard as uninteresting if the change is less than
+	 * the sum each of the standard deviations. */
+	if (change - 1.0 < hypot(diff->old->stats.std_dev, diff->new->stats.std_dev))
+	    continue;
+
+	/* And discard as uninteresting if the new mean is within
+	 * 95% [2 sigma] of old value. */
 	min_value = diff->old->stats.mean * (1 - 2*diff->old->stats.std_dev);
 	max_value = diff->old->stats.mean * (1 + 2*diff->old->stats.std_dev);
 	if (diff->new->stats.mean > min_value &&
@@ -598,14 +604,24 @@ cairo_perf_report_diff (cairo_perf_repor
 	    printed_slowdown = 1;
 	}
 
-	printf ("%5s-%-4s %26s-%-3d  %6.2f %4.2f%% -> %6.2f %4.2f%%: %5.2fx ",
+	printf ("%5s-%-4s %26s-%-3d  ",
 		diff->old->backend, diff->old->content,
-		diff->old->name, diff->old->size,
-		diff->old->stats.min_ticks / diff->old->stats.ticks_per_ms,
-		diff->old->stats.std_dev * 100,
-		diff->new->stats.min_ticks / diff->new->stats.ticks_per_ms,
-		diff->new->stats.std_dev * 100,
-		change);
+		diff->old->name, diff->old->size);
+	old_value = diff->old->stats.min_ticks / diff->old->stats.ticks_per_ms;
+	new_value = diff->new->stats.min_ticks / diff->new->stats.ticks_per_ms;
+	if (old_value < 1 || new_value < 1) {
+	    old_value *= 1000;
+	    new_value *= 1000;
+	    printf ("%6.2f?s %4.2f%% -> %6.2f?s %4.2f%%: %5.2fx ",
+		    old_value, diff->old->stats.std_dev * 100,
+		    new_value, diff->new->stats.std_dev * 100,
+		    change);
+	} else {
+	    printf ("%6.2fms %4.2f%% -> %6.2fms %4.2f%%: %5.2fx ",
+		    old_value, diff->old->stats.std_dev * 100,
+		    new_value, diff->new->stats.std_dev * 100,
+		    change);
+	}
 
 	if (diff->speedup > 1.0)
 	    printf ("speedup\n");
-- 
1.4.1



More information about the cairo mailing list