[cairo] cairo-perf cleanups
Chris Wilson
chris at chris-wilson.co.uk
Tue Mar 13 03:32:36 PDT 2007
I'm having a little difficulty getting stable results out of cairo-perf.
I took a look through the source and have a fix for one bug and a couple
of cosmetic suggestions. Afterwards the results appear move consistent,
but I still worry that the default number of iterations for
cairo-perf-diff is insufficient. What I had thought had been implemented
(or was it only discussed?) was to ensure that each test ran for a
minimum length of time.
--
Chris Wilson
-------------- next part --------------
>From c8797eeec0dbd5f6f0626fa6b22348b2588d082e Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 09:43:01 +0000
Subject: [PATCH] Compute std.dev over values[min_valid : min_valid + num_valid]
---
perf/cairo-stats.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/perf/cairo-stats.c b/perf/cairo-stats.c
index 2c6c7dc..43711a9 100644
--- a/perf/cairo-stats.c
+++ b/perf/cairo-stats.c
@@ -92,7 +92,7 @@ _cairo_stats_compute (cairo_stats_t *st
stats->median_ticks = values[min_valid + num_valid / 2];
sum = 0.0;
- for (i = min_valid; i < num_valid; i++) {
+ for (i = min_valid; i < min_valid + num_valid; i++) {
delta = values[i] - mean;
sum += delta * delta;
}
--
1.4.1
-------------- next part --------------
>From 9e594a10a04b124724ca370ff5f230e2d665a72f Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 10:09:52 +0000
Subject: [PATCH] Discard statistically insignificant differences.
Discard as an "uninteresting change" if the two timings lie within 2
sigma of each other.
---
perf/cairo-perf-diff-files.c | 16 +++++++++++++---
perf/cairo-stats.c | 3 ++-
perf/cairo-stats.h | 1 +
3 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/perf/cairo-perf-diff-files.c b/perf/cairo-perf-diff-files.c
index 8b6292a..8495efb 100644
--- a/perf/cairo-perf-diff-files.c
+++ b/perf/cairo-perf-diff-files.c
@@ -560,6 +560,7 @@ cairo_perf_report_diff (cairo_perf_repor
}
for (i = 0; i < num_diffs; i++) {
+ double min_value, max_value;
diff = &diffs[i];
change = diff->speedup;
@@ -572,9 +573,18 @@ cairo_perf_report_diff (cairo_perf_repor
if (change - 1.0 < min_change)
continue;
- /* Also discard as uninteresting if the change is less than
- * the sum each of the standard deviations. */
- if (change - 1.0 < diff->old->stats.std_dev + diff->new->stats.std_dev)
+ /* Also discard as uninteresting if the change is the new value is
+ * within 95% [2 sigma] of old value. */
+ min_value = diff->old->stats.mean * (1 - 2*diff->old->stats.std_dev);
+ max_value = diff->old->stats.mean * (1 + 2*diff->old->stats.std_dev);
+ if (diff->new->stats.mean > min_value &&
+ diff->new->stats.mean < max_value)
+ continue;
+ /* similary, check vice versa. */
+ min_value = diff->new->stats.mean * (1 - 2*diff->new->stats.std_dev);
+ max_value = diff->new->stats.mean * (1 + 2*diff->new->stats.std_dev);
+ if (diff->old->stats.mean > min_value &&
+ diff->old->stats.mean < max_value)
continue;
if (diff->speedup > 1.0 && ! printed_speedup) {
diff --git a/perf/cairo-stats.c b/perf/cairo-stats.c
index 43711a9..baa4e1d 100644
--- a/perf/cairo-stats.c
+++ b/perf/cairo-stats.c
@@ -61,7 +61,7 @@ _cairo_stats_compute (cairo_stats_t *st
qsort (values, num_values,
sizeof (cairo_perf_ticks_t), _cairo_perf_ticks_cmp);
- q1 = values[(1*num_values)/4];
+ q1 = values[(1*num_values)/4];
q3 = values[(3*num_values)/4];
iqr = q3 - q1;
@@ -100,4 +100,5 @@ _cairo_stats_compute (cairo_stats_t *st
/* Let's use a std. deviation normalized to the mean for easier
* comparison. */
stats->std_dev = sqrt(sum / num_valid) / mean;
+ stats->mean = mean;
}
diff --git a/perf/cairo-stats.h b/perf/cairo-stats.h
index 3f8a988..48db0be 100644
--- a/perf/cairo-stats.h
+++ b/perf/cairo-stats.h
@@ -33,6 +33,7 @@ typedef struct _cairo_stats {
cairo_perf_ticks_t median_ticks;
double ticks_per_ms;
double std_dev;
+ double mean;
int iterations;
} cairo_stats_t;
--
1.4.1
-------------- next part --------------
>From 38db7e352a7e02c1337ae9c9a2de6fe1478cff31 Mon Sep 17 00:00:00 2001
From: Chris Wilson <chris at chris-wilson.co.uk>
Date: Tue, 13 Mar 2007 10:19:15 +0000
Subject: [PATCH] Switch between millseconds and microseconds for timings.
Sometimes the timing is too small to display accurately in milliseconds
using %4.2f, so use microsecond results as appropiate.
---
perf/cairo-perf-diff-files.c | 34 +++++++++++++++++++++++++---------
1 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/perf/cairo-perf-diff-files.c b/perf/cairo-perf-diff-files.c
index 8495efb..b2a26e8 100644
--- a/perf/cairo-perf-diff-files.c
+++ b/perf/cairo-perf-diff-files.c
@@ -561,6 +561,7 @@ cairo_perf_report_diff (cairo_perf_repor
for (i = 0; i < num_diffs; i++) {
double min_value, max_value;
+ double old_value, new_value;
diff = &diffs[i];
change = diff->speedup;
@@ -573,8 +574,13 @@ cairo_perf_report_diff (cairo_perf_repor
if (change - 1.0 < min_change)
continue;
- /* Also discard as uninteresting if the change is the new value is
- * within 95% [2 sigma] of old value. */
+ /* Also discard as uninteresting if the change is less than
+ * the sum each of the standard deviations. */
+ if (change - 1.0 < hypot(diff->old->stats.std_dev, diff->new->stats.std_dev))
+ continue;
+
+ /* And discard as uninteresting if the new mean is within
+ * 95% [2 sigma] of old value. */
min_value = diff->old->stats.mean * (1 - 2*diff->old->stats.std_dev);
max_value = diff->old->stats.mean * (1 + 2*diff->old->stats.std_dev);
if (diff->new->stats.mean > min_value &&
@@ -598,14 +604,24 @@ cairo_perf_report_diff (cairo_perf_repor
printed_slowdown = 1;
}
- printf ("%5s-%-4s %26s-%-3d %6.2f %4.2f%% -> %6.2f %4.2f%%: %5.2fx ",
+ printf ("%5s-%-4s %26s-%-3d ",
diff->old->backend, diff->old->content,
- diff->old->name, diff->old->size,
- diff->old->stats.min_ticks / diff->old->stats.ticks_per_ms,
- diff->old->stats.std_dev * 100,
- diff->new->stats.min_ticks / diff->new->stats.ticks_per_ms,
- diff->new->stats.std_dev * 100,
- change);
+ diff->old->name, diff->old->size);
+ old_value = diff->old->stats.min_ticks / diff->old->stats.ticks_per_ms;
+ new_value = diff->new->stats.min_ticks / diff->new->stats.ticks_per_ms;
+ if (old_value < 1 || new_value < 1) {
+ old_value *= 1000;
+ new_value *= 1000;
+ printf ("%6.2f?s %4.2f%% -> %6.2f?s %4.2f%%: %5.2fx ",
+ old_value, diff->old->stats.std_dev * 100,
+ new_value, diff->new->stats.std_dev * 100,
+ change);
+ } else {
+ printf ("%6.2fms %4.2f%% -> %6.2fms %4.2f%%: %5.2fx ",
+ old_value, diff->old->stats.std_dev * 100,
+ new_value, diff->new->stats.std_dev * 100,
+ change);
+ }
if (diff->speedup > 1.0)
printf ("speedup\n");
--
1.4.1
More information about the cairo
mailing list