BazelBenchmarkTestRunner.cpp File Reference

#include "bench/Benchmark.h"
#include "include/core/SkBitmap.h"
#include "include/core/SkCanvas.h"
#include "include/core/SkColorType.h"
#include "include/core/SkGraphics.h"
#include "include/core/SkStream.h"
#include "include/core/SkString.h"
#include "include/core/SkSurface.h"
#include "include/encode/SkPngEncoder.h"
#include "include/private/base/SkAssert.h"
#include "include/private/base/SkTArray.h"
#include "src/base/SkTime.h"
#include "src/utils/SkJSONWriter.h"
#include "src/utils/SkOSPath.h"
#include "tools/AutoreleasePool.h"
#include "tools/ProcStats.h"
#include "tools/Stats.h"
#include "tools/flags/CommandLineFlags.h"
#include "tools/testrunners/benchmark/target/BenchmarkTarget.h"
#include "tools/testrunners/common/TestRunner.h"
#include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h"
#include "tools/testrunners/common/surface_manager/SurfaceManager.h"
#include "tools/timer/Timer.h"
#include <cinttypes>
#include <sstream>

Go to the source code of this file.

Classes
class	ResultsJSONWriter
struct	ResultsJSONWriter::SingleMeasurement
struct	ResultsJSONWriter::Result

Macros
#define	HUMANIZE(ms)   humanize(ms).c_str()

Functions
static	DEFINE_string (skip, "", "Space-separated list of test cases (regexps) to skip.")
static	DEFINE_string (match, "", "Space-separated list of test cases (regexps) to run. Will run all tests if omitted.")
static	DEFINE_string (gitHash, "", "Git hash to include in the results.json output file, which can be ingested by Perf.")
static	DEFINE_string (issue, "", "Changelist ID (e.g. a Gerrit changelist number) to include in the " "results.json output file, which can be ingested by Perf.")
static	DEFINE_string (patchset, "", "Patchset ID (e.g. a Gerrit patchset number) to include in the results.json " "output file, which can be ingested by Perf.")
static	DEFINE_string (key, "", "Space-separated key/value pairs common to all benchmarks. These will be " "included in the results.json output file, which can be ingested by Perf.")
static	DEFINE_string (links, "", "Space-separated name/URL pairs with additional information about the benchmark execution, " "for example links to the Swarming bot and task pages, named \"swarming_bot\" and " "\"swarming_task\", respectively. These links are included in the " "results.json output file, which can be ingested by Perf.")
static	DEFINE_string (outputDir, "", "Directory where to write any output JSON and PNG files. " "Optional when running under Bazel " "(e.g. \"bazel test //path/to:test\") as it defaults to " "$TEST_UNDECLARED_OUTPUTS_DIR.")
static	DEFINE_string (surfaceConfig, "", "Name of the Surface configuration to use (e.g. \"8888\"). This determines " "how we construct the SkSurface from which we get the SkCanvas that " "benchmarks will draw on. See file " "//tools/testrunners/common/surface_manager/SurfaceManager.h for details.")
static	DEFINE_string (cpuName, "", "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").")
static	DEFINE_string (gpuName, "", "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").")
static	DEFINE_bool (writePNGs, false, "Whether or not to write to the output directory any bitmaps produced by benchmarks.")
static	DEFINE_int (loops, 0, "The number of benchmark runs that constitutes a single sample.")
static	DEFINE_bool (autoTuneLoops, false, "Auto-tune (automatically determine) the number of benchmark runs that " "constitutes a single sample. Timings are only reported when auto-tuning.")
static	DEFINE_int (autoTuneLoopsMax, 1000000, "Maximum number of benchmark runs per single sample when auto-tuning. Ignored unless flag " "--autoTuneLoops is set.")
static	DEFINE_int (samples, 10, "Number of samples to collect for each benchmark.")
static	DEFINE_int (ms, 0, "For each benchmark, collect samples for this many milliseconds.")
static	DEFINE_int (flushEvery, 10, "Flush the results.json output file every n-th run. This file " "can be ingested by Perf.")
static	DEFINE_bool (csv, false, "Print status in CSV format.")
static	DEFINE_bool2 (quiet, q, false, "if true, do not print status updates.")
static	DEFINE_bool2 (verbose, v, false, "Enable verbose output from the test runner.")
static void	validate_flags (bool isBazelTest)
static double	now_ms ()
static void	warm_up_test_runner_once (BenchmarkTarget *target, int loops)
static int	sample_benchmark (BenchmarkTarget *target, int *loops, skia_private::TArray< double > *samples, skia_private::TArray< SkString > *statKeys, skia_private::TArray< double > *statValues)
static void	maybe_write_png (BenchmarkTarget *target, std::string outputDir)
SkString	humanize (double ms)
static SkString	to_string (int n)
static void	print_benchmark_stats (Stats *stats, skia_private::TArray< double > *samples, BenchmarkTarget *target, std::string surfaceConfig, int loops)
int	main (int argc, char **argv)

Variables
static bool	unused
AutoreleasePool	pool

Macro Definition Documentation

HUMANIZE

#define HUMANIZE

(

ms

)

humanize(ms).c_str()

Definition at line 451 of file BazelBenchmarkTestRunner.cpp.

Function Documentation

DEFINE_bool() [1/3]

static DEFINE_bool ( autoTuneLoops  , false  , "Auto-tune (automatically determine) the number of benchmark runs that " "constitutes a single sample. Timings are only reported when auto-tuning."    )

static

DEFINE_bool() [2/3]

static DEFINE_bool ( csv  , false  , "Print status in CSV format."    )

static

DEFINE_bool() [3/3]

static DEFINE_bool ( writePNGs  , false  , "Whether or not to write to the output directory any bitmaps produced by benchmarks."    )

static

DEFINE_bool2() [1/2]

static DEFINE_bool2 ( quiet  , q  , false  , "if  true, do not print status updates."    )

static

DEFINE_bool2() [2/2]

static DEFINE_bool2 ( verbose  , v  , false  , "Enable verbose output from the test runner."    )

static

DEFINE_int() [1/5]

static DEFINE_int ( autoTuneLoopsMax  , 1000000  , "Maximum number of benchmark runs per single sample when auto-tuning. Ignored unless flag " "--autoTuneLoops is set."    )

static

DEFINE_int() [2/5]

static DEFINE_int ( flushEvery  , 10  , "Flush the results.json output file every n-th run. This file " "can be ingested by Perf."    )

static

DEFINE_int() [3/5]

static DEFINE_int ( loops  , 0  , "The number of benchmark runs that constitutes a single sample."    )

static

DEFINE_int() [4/5]

static DEFINE_int ( ms  , 0  , "For each  benchmark, collect samples for this many milliseconds."    )

static

DEFINE_int() [5/5]

static DEFINE_int ( samples  , 10  , "Number of samples to collect for each benchmark."    )

static

DEFINE_string() [1/11]

static DEFINE_string ( cpuName  , ""  , "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\")."    )

static

DEFINE_string() [2/11]

static DEFINE_string ( gitHash  , ""  , "Git hash to include in the results.json output  file, which can be ingested by Perf."    )

static

DEFINE_string() [3/11]

static DEFINE_string ( gpuName  , ""  , "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\")."    )

static

DEFINE_string() [4/11]

static DEFINE_string ( issue  , ""  , "Changelist ID (e.g. a Gerrit changelist number) to include in the " "results.json output  file, which can be ingested by Perf."    )

static

DEFINE_string() [5/11]

static DEFINE_string ( key  , ""  , "Space-separated key/value pairs common to all benchmarks. These will be " "included in the results.json output  file, which can be ingested by Perf."    )

static

DEFINE_string() [6/11]

static DEFINE_string ( links  , ""  , "Space-separated name/URL pairs with additional information about the benchmark  execution, " "for example links to the Swarming bot and task  pages, named \"swarming_bot\" and " "\"swarming_task\"  , respectively. These links are included in the " "results.json output  file, which can be ingested by Perf."    )

static

DEFINE_string() [7/11]

static DEFINE_string ( match  , ""  , "Space-separated list of test cases (regexps) to run. Will run all tests if omitted."    )

static

DEFINE_string() [8/11]

static DEFINE_string ( outputDir  , ""  , "Directory where to write any output JSON and PNG files. " "Optional when running under Bazel " "(e.g. \"bazel test //path/to:test\") as it defaults to " "$TEST_UNDECLARED_OUTPUTS_DIR."    )

static

DEFINE_string() [9/11]

static DEFINE_string ( patchset  , ""  , "Patchset ID (e.g. a Gerrit patchset number) to include in the results.json " "output  file, which can be ingested by Perf."    )

static

DEFINE_string() [10/11]

static DEFINE_string ( skip  , ""  , "Space-separated list of test cases (regexps) to skip."    )

static

DEFINE_string() [11/11]

static DEFINE_string ( surfaceConfig  , ""  , "Name of the Surface configuration to use (e.g. \"8888\"). This determines " "how we construct the SkSurface from which we get the SkCanvas that " "benchmarks will draw on. See file " "//tools/testrunners/common/surface_manager/SurfaceManager.h for details."    )

static

humanize()

SkString humanize

(

double

ms

)

Definition at line 446 of file BazelBenchmarkTestRunner.cpp.

                             {
    if (FLAGS_verbose) return SkStringPrintf("%" PRIu64, (uint64_t)(ms * 1e6));
    return HumanizeMs(ms);
}

main()

int main	(	int	argc,
		char **	argv
	)

Definition at line 523 of file BazelBenchmarkTestRunner.cpp.

                                {
    TestRunner::InitAndLogCmdlineArgs(argc, argv);
 
    // When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in
    // $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag.
    //
    // See https://bazel.build/reference/test-encyclopedia#initial-conditions.
    std::string testUndeclaredOutputsDir;
    if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) {
        testUndeclaredOutputsDir = envVar;
    }
    bool isBazelTest = !testUndeclaredOutputsDir.empty();
 
    // Parse and validate flags.
    CommandLineFlags::Parse(argc, argv);
    validate_flags(isBazelTest);
 
    // TODO(lovisolo): Define an enum for surface configs and turn this flag into an enum value.
    std::string surfaceConfig = FLAGS_surfaceConfig[0];
 
    // Directory where we will write the output JSON file and any PNGs produced by the benchmarks.
    std::string outputDir =
            FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
 
    std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0];
    std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0];
 
    // Output JSON file.
    //
    // TODO(lovisolo): Define a constant with the file name, use it here and in flag descriptions.
    SkString jsonPath = SkOSPath::Join(outputDir.c_str(), "results.json");
    ResultsJSONWriter jsonWriter(jsonPath.c_str());
 
    if (FLAGS_gitHash.size() == 1) {
        jsonWriter.addGitHash(FLAGS_gitHash[0]);
    } else {
        TestRunner::Log(
                "Warning: No --gitHash flag was specified. Perf ingestion ignores JSON files that "
                "do not specify a Git hash. This is fine for local debugging, but CI tasks should "
                "always set the --gitHash flag.");
    }
    if (FLAGS_issue.size() == 1 && FLAGS_patchset.size() == 1) {
        jsonWriter.addChangelistInfo(FLAGS_issue[0], FLAGS_patchset[0]);
    }
 
    // Key.
    std::map<std::string, std::string> keyValuePairs = {
            // Add a key/value pair that nanobench will never use in order to avoid accidentally
            // polluting an existing trace.
            {"build_system", "bazel"},
    };
    for (int i = 1; i < FLAGS_key.size(); i += 2) {
        keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
    }
    keyValuePairs.merge(GetCompilationModeGoldAndPerfKeyValuePairs());
    jsonWriter.addKey(keyValuePairs);
 
    // Links.
    if (FLAGS_links.size()) {
        std::map<std::string, std::string> links;
        for (int i = 1; i < FLAGS_links.size(); i += 2) {
            links[FLAGS_links[i - 1]] = FLAGS_links[i];
        }
        jsonWriter.addLinks(links);
    }
 
    int runs = 0;
    bool missingCpuOrGpuWarningLogged = false;
    for (auto benchmarkFactory : BenchRegistry::Range()) {
        std::unique_ptr<Benchmark> benchmark(benchmarkFactory(nullptr));
 
        if (!TestRunner::ShouldRunTestCase(benchmark->getUniqueName(), FLAGS_match, FLAGS_skip)) {
            TestRunner::Log("Skipping %s", benchmark->getName());
            continue;
        }
 
        benchmark->delayedSetup();
 
        std::unique_ptr<BenchmarkTarget> target =
                BenchmarkTarget::FromConfig(surfaceConfig, benchmark.get());
        SkASSERT_RELEASE(target);
 
        if (benchmark->isSuitableFor(target->getBackend())) {
            // Print warning about missing cpu_or_gpu key if necessary.
            if (target->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" &&
                !missingCpuOrGpuWarningLogged) {
                TestRunner::Log(
                        "Warning: The surface is CPU-bound, but flag --cpuName was not provided. "
                        "Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
                missingCpuOrGpuWarningLogged = true;
            }
            if (target->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" &&
                !missingCpuOrGpuWarningLogged) {
                TestRunner::Log(
                        "Warning: The surface is GPU-bound, but flag --gpuName was not provided. "
                        "Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
                missingCpuOrGpuWarningLogged = true;
            }
 
            // Run benchmark and collect samples.
            int loops;
            skia_private::TArray<double> samples;
            skia_private::TArray<SkString> statKeys;
            skia_private::TArray<double> statValues;
            if (!sample_benchmark(target.get(), &loops, &samples, &statKeys, &statValues)) {
                // Sampling failed. A warning has already been printed.
                pool.drain();
                continue;
            }
 
            if (FLAGS_writePNGs) {
                // Save the bitmap produced by the benchmark to disk, if applicable. Not all
                // benchmarks produce bitmaps, e.g. those that use the "nonrendering" config.
                maybe_write_png(target.get(), outputDir);
            }
 
            // Building stats.plot often shows up in profiles, so skip building it when we're
            // not going to print it anyway.
            const bool want_plot = !FLAGS_quiet && !FLAGS_ms;
            Stats stats(samples, want_plot);
 
            print_benchmark_stats(&stats, &samples, target.get(), surfaceConfig, loops);
 
            ResultsJSONWriter::Result result;
            result.key = {
                    // Based on
                    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1566.
                    {"name", std::string(benchmark->getName())},
 
                    // Replaces the "config" and "extra_config" keys set by nanobench.
                    {"surface_config", surfaceConfig},
 
                    // Based on
                    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1578.
                    //
                    // TODO(lovisolo): Determine these dynamically when we add support for GMBench,
                    //                 SKPBench, etc.
                    {"source_type", "bench"},
                    {"bench_type", "micro"},
 
                    // Nanobench adds a "test" key consisting of "<unique name>_<width>_<height>",
                    // presumably with the goal of making the trace ID unique, see
                    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1456.
                    //
                    // However, we can accomplish unique trace IDs by expressing "<width>" and
                    // "<height>" as their own keys.
                    //
                    // Regarding the "<unique name>" part of the "test" key:
                    //
                    //  - Nanobench sets "<unique name>" to the result of
                    //    Benchmark::getUniqueName():
                    //    https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/Benchmark.h#41.
                    //
                    //  - Benchmark::getUniqueName() returns Benchmark::getName() except for the
                    //    following two cases.
                    //
                    //  - SKPBench::getUniqueName() returns "<name>_<scale>":
                    //    https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/SKPBench.cpp#33.
                    //
                    //  - SKPAnimationBench returns "<name>_<tag>":
                    //    https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/SKPAnimationBench.cpp#18.
                    //
                    // Therefore it is important that we add "<scale>" and "<tag>" as their own
                    // keys when we eventually add support for these kinds of benchmarks.
                    //
                    // Based on
                    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1456.
                    {"width", SkStringPrintf("%d", benchmark->getSize().width()).c_str()},
                    {"height", SkStringPrintf("%d", benchmark->getSize().height()).c_str()},
            };
            result.key.merge(target->getKeyValuePairs(cpuName, gpuName));
            result.measurements["ms"] = {
                    // Based on
                    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1571.
                    {.value = "min", .measurement = stats.min},
                    {.value = "ratio",
                     .measurement = sk_ieee_double_divide(stats.median, stats.min)},
            };
            if (!statKeys.empty()) {
                // Based on
                // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1580.
                //
                // Only SKPBench currently returns valid key/value pairs.
                SkASSERT(statKeys.size() == statValues.size());
                result.measurements["stats"] = {};
                for (int i = 0; i < statKeys.size(); i++) {
                    result.measurements["stats"].push_back(
                            {.value = statKeys[i].c_str(), .measurement = statValues[i]});
                }
            }
            jsonWriter.addResult(result);
 
            runs++;
            if (runs % FLAGS_flushEvery == 0) {
                jsonWriter.flush();
            }
 
            pool.drain();
        } else {
            if (FLAGS_verbose) {
                TestRunner::Log("Skipping \"%s\" because backend \"%s\" was unsuitable.\n",
                                target->getBenchmark()->getUniqueName(),
                                surfaceConfig.c_str());
            }
        }
    }
 
    BenchmarkTarget::printGlobalStats();
 
    SkGraphics::PurgeAllCaches();
 
    // Based on
    // https://skia.googlesource.com/skia/+/a063eaeaf1e09e4d6f42e0f44a5723622a46d21c/bench/nanobench.cpp#1668.
    jsonWriter.addResult({
            .key =
                    {
                            {"name", "memory_usage"},
                    },
            .measurements =
                    {
                            {"resident_set_size_mb",
                             {{.value = "max",
                               .measurement = double(sk_tools::getMaxResidentSetSizeMB())}}},
                    },
    });
 
    TestRunner::Log("JSON file written to: %s", jsonPath.c_str());
    TestRunner::Log("PNGs (if any) written to: %s", outputDir.c_str());
 
    return 0;
}

maybe_write_png()

static void maybe_write_png ( BenchmarkTarget *  target, std::string  outputDir  )

static

Definition at line 406 of file BazelBenchmarkTestRunner.cpp.

                                                                          {
    if (target->getBackend() == Benchmark::Backend::kNonRendering) {
        return;
    }
 
    SkString filename = SkStringPrintf("%s.png", target->getBenchmark()->getUniqueName());
    filename = SkOSPath::Join(outputDir.c_str(), filename.c_str());
 
    if (!target->getCanvas() ||
        target->getCanvas()->imageInfo().colorType() == kUnknown_SkColorType) {
        return;
    }
 
    SkBitmap bmp;
    bmp.allocPixels(target->getCanvas()->imageInfo());
    if (!target->getCanvas()->readPixels(bmp, 0, 0)) {
        TestRunner::Log("Warning: Could not read canvas pixels for benchmark \"%s\".",
                        target->getBenchmark()->getUniqueName());
        return;
    }
 
    SkFILEWStream stream(filename.c_str());
    if (!stream.isValid()) {
        TestRunner::Log("Warning: Could not write file \"%s\".", filename.c_str());
        return;
    }
    if (!SkPngEncoder::Encode(&stream, bmp.pixmap(), {})) {
        TestRunner::Log("Warning: Could not encode pixels from benchmark \"%s\" as PNG.",
                        target->getBenchmark()->getUniqueName());
        return;
    }
 
    if (FLAGS_verbose) {
        TestRunner::Log("PNG for benchmark \"%s\" written to: %s",
                        target->getBenchmark()->getUniqueName(),
                        filename.c_str());
    }
}

now_ms()

static double now_ms ( )

static

Definition at line 315 of file BazelBenchmarkTestRunner.cpp.

{ return SkTime::GetNSecs() * 1e-6; }

print_benchmark_stats()

static void print_benchmark_stats ( Stats *  stats, skia_private::TArray< double > *  samples, BenchmarkTarget *  target, std::string  surfaceConfig, int  loops  )

static

Definition at line 461 of file BazelBenchmarkTestRunner.cpp.

                                             {
    if (!FLAGS_autoTuneLoops) {
        TestRunner::Log("%4d/%-4dMB\t%s\t%s",
                        sk_tools::getCurrResidentSetSizeMB(),
                        sk_tools::getMaxResidentSetSizeMB(),
                        target->getBenchmark()->getUniqueName(),
                        surfaceConfig.c_str());
    } else if (FLAGS_quiet) {
        const char* mark = " ";
        const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
        if (stddev_percent > 5) mark = "?";
        if (stddev_percent > 10) mark = "!";
        TestRunner::Log("%10.2f %s\t%s\t%s",
                        stats->median * 1e3,
                        mark,
                        target->getBenchmark()->getUniqueName(),
                        surfaceConfig.c_str());
    } else if (FLAGS_csv) {
        const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
        SkDebugf("%g,%g,%g,%g,%g,%s,%s\n",
                 stats->min,
                 stats->median,
                 stats->mean,
                 stats->max,
                 stddev_percent,
                 surfaceConfig.c_str(),
                 target->getBenchmark()->getUniqueName());
    } else {
        const double stddev_percent = sk_ieee_double_divide(100 * sqrt(stats->var), stats->mean);
        TestRunner::Log("%4d/%-4dMB\t%d\t%s\t%s\t%s\t%s\t%.0f%%\t%s\t%s\t%s",
                        sk_tools::getCurrResidentSetSizeMB(),
                        sk_tools::getMaxResidentSetSizeMB(),
                        loops,
                        HUMANIZE(stats->min),
                        HUMANIZE(stats->median),
                        HUMANIZE(stats->mean),
                        HUMANIZE(stats->max),
                        stddev_percent,
                        FLAGS_ms ? to_string(samples->size()).c_str() : stats->plot.c_str(),
                        surfaceConfig.c_str(),
                        target->getBenchmark()->getUniqueName());
    }
 
    target->printStats();
 
    if (FLAGS_verbose) {
        std::ostringstream oss;
        oss << "Samples: ";
        for (int j = 0; j < samples->size(); j++) {
            oss << HUMANIZE((*samples)[j]) << "  ";
        }
        oss << target->getBenchmark()->getUniqueName();
        TestRunner::Log("%s", oss.str().c_str());
    }
}

sample_benchmark()

static int sample_benchmark ( BenchmarkTarget *  target, int *  loops, skia_private::TArray< double > *  samples, skia_private::TArray< SkString > *  statKeys, skia_private::TArray< double > *  statValues  )

static

Definition at line 342 of file BazelBenchmarkTestRunner.cpp.

                                                                    {
    target->setup();
 
    if (FLAGS_autoTuneLoops) {
        auto [autoTunedLoops, ok] = target->autoTuneLoops();
        if (!ok) {
            // Can't be timed. A warning has already been printed.
            target->tearDown();
            return false;
        }
        *loops = autoTunedLoops;
        if (*loops > FLAGS_autoTuneLoopsMax) {
            TestRunner::Log(
                    "Warning: Clamping loops from %d to %d (per the --autoTuneLoopsMax flag) for "
                    "benchmark \"%s\".",
                    *loops,
                    FLAGS_autoTuneLoopsMax,
                    target->getBenchmark()->getUniqueName());
            *loops = FLAGS_autoTuneLoopsMax;
        }
    } else {
        *loops = FLAGS_loops;
    }
 
    // Warm up the test runner to increase the chances of getting consistent measurements. Only
    // done once for the entire lifecycle of the test runner.
    warm_up_test_runner_once(target, *loops);
 
    // Each individual BenchmarkTarget must also be warmed up.
    target->warmUp(*loops);
 
    if (FLAGS_ms) {
        // Collect as many samples as possible for a specific duration of time.
        auto stop = now_ms() + FLAGS_ms;
        do {
            samples->push_back(target->time(*loops) / *loops);
            pool.drain();
        } while (now_ms() < stop);
    } else {
        // Collect an exact number of samples.
        samples->reset(FLAGS_samples);
        for (int s = 0; s < FLAGS_samples; s++) {
            (*samples)[s] = target->time(*loops) / *loops;
            pool.drain();
        }
    }
 
    // Scale each sample to the benchmark's own units, time/unit.
    for (double& sample : *samples) {
        sample *= (1.0 / target->getBenchmark()->getUnits());
    }
 
    target->dumpStats(statKeys, statValues);
    target->tearDown();
 
    return true;
}

to_string()

static SkString to_string ( int  n )

static

Definition at line 453 of file BazelBenchmarkTestRunner.cpp.

                                 {
    SkString str;
    str.appendS32(n);
    return str;
}

validate_flags()

static void validate_flags ( bool  isBazelTest )

static

Definition at line 150 of file BazelBenchmarkTestRunner.cpp.

                                             {
    TestRunner::FlagValidators::AllOrNone(
            {{"--issue", FLAGS_issue.size() > 0}, {"--patchset", FLAGS_patchset.size() > 0}});
    TestRunner::FlagValidators::StringAtMostOne("--issue", FLAGS_issue);
    TestRunner::FlagValidators::StringAtMostOne("--patchset", FLAGS_patchset);
    TestRunner::FlagValidators::StringEven("--key", FLAGS_key);
    TestRunner::FlagValidators::StringEven("--links", FLAGS_links);
 
    if (!isBazelTest) {
        TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir);
    }
    TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir);
 
    TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig);
    TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig);
 
    TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName);
    TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName);
 
    TestRunner::FlagValidators::ExactlyOne(
            {{"--loops", FLAGS_loops != 0}, {"--autoTuneLoops", FLAGS_autoTuneLoops}});
    if (!FLAGS_autoTuneLoops) {
        TestRunner::FlagValidators::IntGreaterOrEqual("--loops", FLAGS_loops, 1);
    }
 
    TestRunner::FlagValidators::IntGreaterOrEqual("--autoTuneLoopsMax", FLAGS_autoTuneLoopsMax, 1);
 
    TestRunner::FlagValidators::ExactlyOne(
            {{"--samples", FLAGS_samples != 0}, {"--ms", FLAGS_ms != 0}});
    if (FLAGS_ms == 0) {
        TestRunner::FlagValidators::IntGreaterOrEqual("--samples", FLAGS_samples, 1);
    }
    if (FLAGS_samples == 0) {
        TestRunner::FlagValidators::IntGreaterOrEqual("--ms", FLAGS_ms, 1);
    }
}

warm_up_test_runner_once()

static void warm_up_test_runner_once ( BenchmarkTarget *  target, int  loops  )

static

Definition at line 319 of file BazelBenchmarkTestRunner.cpp.

                                                                         {
    static bool warm = false;
    if (warm) {
        return;
    }
    if (FLAGS_ms < 1000) {
        // Run the first bench for 1000ms to warm up the test runner if FLAGS_ms < 1000.
        // Otherwise, the first few benches' measurements will be inaccurate.
        auto stop = now_ms() + 1000;
        do {
            target->time(loops);
            pool.drain();
        } while (now_ms() < stop);
    }
    warm = true;
}

Variable Documentation

pool

AutoreleasePool pool

Definition at line 313 of file BazelBenchmarkTestRunner.cpp.

unused

bool unused

static

Initial value:

=
        SkFlagInfo::CreateStringFlag("device-specific-bazel-config",
                                     nullptr,
                                     new CommandLineFlags::StringArray(),
                                     nullptr,
                                     "Ignored by this test runner.",
                                     nullptr)

Definition at line 142 of file BazelBenchmarkTestRunner.cpp.