39 "Space-separated list of test cases (regexps) to run. Will run all tests if omitted.");
45 "Git hash to include in the results.json output file, which can be ingested by Perf.");
49 "Changelist ID (e.g. a Gerrit changelist number) to include in the "
50 "results.json output file, which can be ingested by Perf.");
54 "Patchset ID (e.g. a Gerrit patchset number) to include in the results.json "
55 "output file, which can be ingested by Perf.");
59 "Space-separated key/value pairs common to all benchmarks. These will be "
60 "included in the results.json output file, which can be ingested by Perf.");
65 "Space-separated name/URL pairs with additional information about the benchmark execution, "
66 "for example links to the Swarming bot and task pages, named \"swarming_bot\" and "
67 "\"swarming_task\", respectively. These links are included in the "
68 "results.json output file, which can be ingested by Perf.");
81 "Directory where to write any output JSON and PNG files. "
82 "Optional when running under Bazel "
83 "(e.g. \"bazel test //path/to:test\") as it defaults to "
84 "$TEST_UNDECLARED_OUTPUTS_DIR.");
88 "Name of the Surface configuration to use (e.g. \"8888\"). This determines "
89 "how we construct the SkSurface from which we get the SkCanvas that "
90 "benchmarks will draw on. See file "
91 "//tools/testrunners/common/surface_manager/SurfaceManager.h for details.");
96 "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").");
101 "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").");
106 "Whether or not to write to the output directory any bitmaps produced by benchmarks.");
109static DEFINE_int(loops, 0,
"The number of benchmark runs that constitutes a single sample.");
114 "Auto-tune (automatically determine) the number of benchmark runs that "
115 "constitutes a single sample. Timings are only reported when auto-tuning.");
120 "Maximum number of benchmark runs per single sample when auto-tuning. Ignored unless flag "
121 "--autoTuneLoops is set.");
124static DEFINE_int(samples, 10,
"Number of samples to collect for each benchmark.");
127static DEFINE_int(ms, 0,
"For each benchmark, collect samples for this many milliseconds.");
131 "Flush the results.json output file every n-th run. This file "
132 "can be ingested by Perf.");
136static DEFINE_bool2(quiet, q,
false,
"if true, do not print status updates.");
138static DEFINE_bool2(verbose, v,
false,
"Enable verbose output from the test runner.");
147 "Ignored by this test runner.",
152 {{
"--issue", FLAGS_issue.size() > 0}, {
"--patchset", FLAGS_patchset.size() > 0}});
170 {{
"--loops", FLAGS_loops != 0}, {
"--autoTuneLoops", FLAGS_autoTuneLoops}});
171 if (!FLAGS_autoTuneLoops) {
178 {{
"--samples", FLAGS_samples != 0}, {
"--ms", FLAGS_ms != 0}});
182 if (FLAGS_samples == 0) {
212 std::map<std::string, std::string>
key;
219 , fAddingResults(false) {
225 assertNotAddingResults();
230 assertNotAddingResults();
236 assertNotAddingResults();
244 void addLinks(std::map<std::string, std::string> links) {
245 assertNotAddingResults();
247 for (
auto const& [
key,
value] : links) {
254 if (!fAddingResults) {
255 fAddingResults =
true;
270 for (
auto const& [
name, singleMeasurements] :
result.measurements) {
277 if (
SkIsFinite(singleMeasurement.measurement)) {
279 fJson.
appendCString(
"value", singleMeasurement.value.c_str());
294 if (fAddingResults) {
301 void assertNotAddingResults() {
302 if (fAddingResults) {
303 SK_ABORT(
"Cannot perform this operation after addResults() is called.");
320 static bool warm =
false;
324 if (FLAGS_ms < 1000) {
327 auto stop =
now_ms() + 1000;
331 }
while (
now_ms() < stop);
349 if (FLAGS_autoTuneLoops) {
350 auto [autoTunedLoops,
ok] =
target->autoTuneLoops();
356 *loops = autoTunedLoops;
357 if (*loops > FLAGS_autoTuneLoopsMax) {
359 "Warning: Clamping loops from %d to %d (per the --autoTuneLoopsMax flag) for "
362 FLAGS_autoTuneLoopsMax,
363 target->getBenchmark()->getUniqueName());
364 *loops = FLAGS_autoTuneLoopsMax;
367 *loops = FLAGS_loops;
379 auto stop =
now_ms() + FLAGS_ms;
383 }
while (
now_ms() < stop);
386 samples->
reset(FLAGS_samples);
387 for (
int s = 0;
s < FLAGS_samples;
s++) {
388 (*samples)[
s] =
target->time(*loops) / *loops;
394 for (
double& sample : *samples) {
395 sample *= (1.0 /
target->getBenchmark()->getUnits());
398 target->dumpStats(statKeys, statValues);
414 if (!
target->getCanvas() ||
421 if (!
target->getCanvas()->readPixels(bmp, 0, 0)) {
422 TestRunner::Log(
"Warning: Could not read canvas pixels for benchmark \"%s\".",
423 target->getBenchmark()->getUniqueName());
428 if (!stream.isValid()) {
433 TestRunner::Log(
"Warning: Could not encode pixels from benchmark \"%s\" as PNG.",
434 target->getBenchmark()->getUniqueName());
440 target->getBenchmark()->getUniqueName(),
447 if (FLAGS_verbose)
return SkStringPrintf(
"%" PRIu64, (uint64_t)(ms * 1e6));
451#define HUMANIZE(ms) humanize(ms).c_str()
464 std::string surfaceConfig,
466 if (!FLAGS_autoTuneLoops) {
470 target->getBenchmark()->getUniqueName(),
471 surfaceConfig.c_str());
472 }
else if (FLAGS_quiet) {
473 const char*
mark =
" ";
475 if (stddev_percent > 5)
mark =
"?";
476 if (stddev_percent > 10)
mark =
"!";
480 target->getBenchmark()->getUniqueName(),
481 surfaceConfig.c_str());
482 }
else if (FLAGS_csv) {
490 surfaceConfig.c_str(),
491 target->getBenchmark()->getUniqueName());
504 surfaceConfig.c_str(),
505 target->getBenchmark()->getUniqueName());
511 std::ostringstream oss;
513 for (
int j = 0; j < samples->
size(); j++) {
514 oss <<
HUMANIZE((*samples)[j]) <<
" ";
516 oss <<
target->getBenchmark()->getUniqueName();
523int main(
int argc,
char** argv) {
530 std::string testUndeclaredOutputsDir;
531 if (
char* envVar = std::getenv(
"TEST_UNDECLARED_OUTPUTS_DIR")) {
532 testUndeclaredOutputsDir = envVar;
534 bool isBazelTest = !testUndeclaredOutputsDir.empty();
541 std::string surfaceConfig = FLAGS_surfaceConfig[0];
544 std::string outputDir =
545 FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
547 std::string cpuName = FLAGS_cpuName.isEmpty() ?
"" : FLAGS_cpuName[0];
548 std::string gpuName = FLAGS_gpuName.isEmpty() ?
"" : FLAGS_gpuName[0];
556 if (FLAGS_gitHash.size() == 1) {
560 "Warning: No --gitHash flag was specified. Perf ingestion ignores JSON files that "
561 "do not specify a Git hash. This is fine for local debugging, but CI tasks should "
562 "always set the --gitHash flag.");
564 if (FLAGS_issue.size() == 1 && FLAGS_patchset.size() == 1) {
569 std::map<std::string, std::string> keyValuePairs = {
572 {
"build_system",
"bazel"},
574 for (
int i = 1; i < FLAGS_key.size(); i += 2) {
575 keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
578 jsonWriter.
addKey(keyValuePairs);
581 if (FLAGS_links.size()) {
582 std::map<std::string, std::string> links;
583 for (
int i = 1; i < FLAGS_links.size(); i += 2) {
584 links[FLAGS_links[i - 1]] = FLAGS_links[i];
590 bool missingCpuOrGpuWarningLogged =
false;
592 std::unique_ptr<Benchmark>
benchmark(benchmarkFactory(
nullptr));
601 std::unique_ptr<BenchmarkTarget>
target =
608 !missingCpuOrGpuWarningLogged) {
610 "Warning: The surface is CPU-bound, but flag --cpuName was not provided. "
611 "Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
612 missingCpuOrGpuWarningLogged =
true;
615 !missingCpuOrGpuWarningLogged) {
617 "Warning: The surface is GPU-bound, but flag --gpuName was not provided. "
618 "Perf traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
619 missingCpuOrGpuWarningLogged =
true;
633 if (FLAGS_writePNGs) {
641 const bool want_plot = !FLAGS_quiet && !FLAGS_ms;
642 Stats stats(samples, want_plot);
650 {
"name", std::string(
benchmark->getName())},
653 {
"surface_config", surfaceConfig},
660 {
"source_type",
"bench"},
661 {
"bench_type",
"micro"},
693 result.key.merge(
target->getKeyValuePairs(cpuName, gpuName));
694 result.measurements[
"ms"] = {
697 {.value =
"min", .measurement = stats.min},
701 if (!statKeys.
empty()) {
707 result.measurements[
"stats"] = {};
708 for (
int i = 0; i < statKeys.
size(); i++) {
709 result.measurements[
"stats"].push_back(
710 {.value = statKeys[i].c_str(), .measurement = statValues[i]});
716 if (runs % FLAGS_flushEvery == 0) {
723 TestRunner::Log(
"Skipping \"%s\" because backend \"%s\" was unsuitable.\n",
724 target->getBenchmark()->getUniqueName(),
725 surfaceConfig.c_str());
739 {
"name",
"memory_usage"},
743 {
"resident_set_size_mb",
static SkString to_string(int n)
static void maybe_write_png(BenchmarkTarget *target, std::string outputDir)
static void warm_up_test_runner_once(BenchmarkTarget *target, int loops)
SkString humanize(double ms)
static void validate_flags(bool isBazelTest)
static void print_benchmark_stats(Stats *stats, skia_private::TArray< double > *samples, BenchmarkTarget *target, std::string surfaceConfig, int loops)
static int sample_benchmark(BenchmarkTarget *target, int *loops, skia_private::TArray< double > *samples, skia_private::TArray< SkString > *statKeys, skia_private::TArray< double > *statValues)
#define DEFINE_bool(name, defaultValue, helpString)
#define DEFINE_int(name, defaultValue, helpString)
#define DEFINE_bool2(name, shortName, defaultValue, helpString)
#define DEFINE_string(name, defaultValue, helpString)
std::map< std::string, std::string > GetCompilationModeGoldAndPerfKeyValuePairs()
static bool match(const char *needle, const char *haystack)
#define SK_ABORT(message,...)
#define SkASSERT_RELEASE(cond)
@ kUnknown_SkColorType
uninitialized
void SK_SPI SkDebugf(const char format[],...) SK_PRINTF_LIKE(1
static bool SkIsFinite(T x, Pack... values)
static constexpr double sk_ieee_double_divide(double numer, double denom)
static bool skip(SkStream *stream, size_t amount)
static bool ok(int result)
SK_API SkString static SkString SkStringPrintf()
SkString HumanizeMs(double ms)
static std::unique_ptr< BenchmarkTarget > FromConfig(std::string surfaceConfig, Benchmark *benchmark)
static void printGlobalStats()
static void Parse(int argc, const char *const *argv)
void addLinks(std::map< std::string, std::string > links)
void addGitHash(std::string gitHash)
void addChangelistInfo(std::string issue, std::string patchset)
ResultsJSONWriter(const char *path)
void addResult(Result result)
void addKey(std::map< std::string, std::string > key)
void allocPixels(const SkImageInfo &info, size_t rowBytes)
const SkPixmap & pixmap() const
static bool CreateStringFlag(const char *name, const char *shortName, CommandLineFlags::StringArray *pStrings, const char *defaultValue, const char *helpString, const char *extendedHelpString)
static void PurgeAllCaches()
void appendS32(int32_t value)
void beginArray(const char *name=nullptr, bool multiline=true)
void beginObject(const char *name=nullptr, bool multiline=true)
void appendCString(const char *value)
void appendDoubleDigits(double value, int digits)
static SkString Join(const char *rootPath, const char *relativePath)
void appendS32(int32_t value)
const char * c_str() const
static void mark(SkCanvas *canvas, SkScalar x, SkScalar y, Fn &&fn)
SK_API bool Encode(SkWStream *dst, const SkPixmap &src, const Options &options)
void StringEven(std::string name, CommandLineFlags::StringArray flag)
void IntGreaterOrEqual(std::string name, int flag, int min)
void ExactlyOne(std::map< std::string, bool > flags)
void AllOrNone(std::map< std::string, bool > flags)
void StringNonEmpty(std::string name, CommandLineFlags::StringArray flag)
void StringAtMostOne(std::string name, CommandLineFlags::StringArray flag)
bool ShouldRunTestCase(const char *name, CommandLineFlags::StringArray &matchFlag, CommandLineFlags::StringArray &skipFlag)
void Log(const char *format,...) SK_PRINTF_LIKE(1
void InitAndLogCmdlineArgs(int argc, char **argv)
std::map< std::string, std::string > key
std::map< std::string, std::vector< SingleMeasurement > > measurements