BazelGMTestRunner.cpp File Reference

#include "gm/gm.h"
#include "include/core/SkBitmap.h"
#include "include/core/SkCanvas.h"
#include "include/core/SkColorSpace.h"
#include "include/core/SkColorType.h"
#include "include/core/SkImageInfo.h"
#include "include/core/SkStream.h"
#include "include/core/SkSurface.h"
#include "include/encode/SkPngEncoder.h"
#include "include/private/base/SkAssert.h"
#include "include/private/base/SkDebug.h"
#include "src/core/SkMD5.h"
#include "src/utils/SkJSONWriter.h"
#include "src/utils/SkOSPath.h"
#include "tools/HashAndEncode.h"
#include "tools/testrunners/common/TestRunner.h"
#include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h"
#include "tools/testrunners/common/surface_manager/SurfaceManager.h"
#include "tools/testrunners/gm/vias/Draw.h"
#include <algorithm>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <regex>
#include <set>
#include <sstream>
#include <string>

Go to the source code of this file.

Classes
struct	WritePNGAndJSONFilesResult

Functions
static	DEFINE_string (skip, "", "Space-separated list of test cases (regexps) to skip.")
static	DEFINE_string (match, "", "Space-separated list of test cases (regexps) to run. Will run all tests if omitted.")
static	DEFINE_string (outputDir, "", "Directory where to write any output .png and .json files. " "Optional when running under Bazel " "(e.g. \"bazel test //path/to:test\") as it defaults to " "$TEST_UNDECLARED_OUTPUTS_DIR.")
static	DEFINE_string (knownDigestsFile, "", "Plaintext file with one MD5 hash per line. This test runner will omit from " "the output directory any images with an MD5 hash in this file.")
static	DEFINE_string (key, "", "Space-separated key/value pairs common to all traces.")
static	DEFINE_string (surfaceConfig, "", "Name of the Surface configuration to use (e.g. \"8888\"). This determines " "how we construct the SkSurface from which we get the SkCanvas that GMs will " "draw on. See file //tools/testrunners/common/surface_manager/SurfaceManager.h for " "details.")
static	DEFINE_string (cpuName, "", "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").")
static	DEFINE_string (gpuName, "", "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").")
static	DEFINE_string (via, "direct", "Name of the \"via\" to use (e.g. \"picture_serialization\"). Optional.")
static WritePNGAndJSONFilesResult	write_png_and_json_files (std::string name, std::map< std::string, std::string > commonKeys, std::map< std::string, std::string > gmGoldKeys, std::map< std::string, std::string > surfaceGoldKeys, const SkBitmap &bitmap, const char *pngPath, const char *jsonPath, std::set< std::string > knownDigests)
static std::string	draw_result_to_string (skiagm::DrawResult result)
void	run_gm (std::unique_ptr< skiagm::GM > gm, std::string config, std::map< std::string, std::string > keyValuePairs, std::string cpuName, std::string gpuName, std::string outputDir, std::set< std::string > knownDigests)
std::set< std::string >	read_known_digests_file (std::string path)
int	main (int argc, char **argv)

Variables
static bool	unused
static int	gNumSuccessfulGMs = 0
static int	gNumFailedGMs = 0
static int	gNumSkippedGMs = 0
static bool	gMissingCpuOrGpuWarningLogged = false

Function Documentation

DEFINE_string() [1/9]

static DEFINE_string ( cpuName  , ""  , "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\")."    )

static

DEFINE_string() [2/9]

static DEFINE_string ( gpuName  , ""  , "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\")."    )

static

DEFINE_string() [3/9]

static DEFINE_string ( key  , ""  , "Space-separated key/value pairs common to all traces."    )

static

DEFINE_string() [4/9]

static DEFINE_string ( knownDigestsFile  , ""  , "Plaintext file with one MD5 hash per line. This test runner will omit from " "the output directory any images with an MD5 hash in this file."    )

static

DEFINE_string() [5/9]

static DEFINE_string ( match  , ""  , "Space-separated list of test cases (regexps) to run. Will run all tests if omitted."    )

static

DEFINE_string() [6/9]

static DEFINE_string ( outputDir  , ""  , "Directory where to write any output .png and .json files. " "Optional when running under Bazel " "(e.g. \"bazel test //path/to:test\") as it defaults to " "$TEST_UNDECLARED_OUTPUTS_DIR."    )

static

DEFINE_string() [7/9]

static DEFINE_string ( skip  , ""  , "Space-separated list of test cases (regexps) to skip."    )

static

DEFINE_string() [8/9]

static DEFINE_string ( surfaceConfig  , ""  , "Name of the Surface configuration to use (e.g. \"8888\"). This determines " "how we construct the SkSurface from which we get the SkCanvas that GMs will " "draw on. See file //tools/testrunners/common/surface_manager/SurfaceManager.h for " "details."    )

static

DEFINE_string() [9/9]

static DEFINE_string ( via  , "direct"  , "Name of the \"via\" to use (e.g. \"picture_serialization\"). Optional."    )

static

draw_result_to_string()

static std::string draw_result_to_string ( skiagm::DrawResult  result )

static

Definition at line 189 of file BazelGMTestRunner.cpp.

                                                                {
    switch (result) {
        case skiagm::DrawResult::kOk:
            return "Ok";
        case skiagm::DrawResult::kFail:
            return "Fail";
        case skiagm::DrawResult::kSkip:
            return "Skip";
        default:
            SkUNREACHABLE;
    }
}

main()

int main	(	int	argc,
		char **	argv
	)

Definition at line 340 of file BazelGMTestRunner.cpp.

                                {
    TestRunner::InitAndLogCmdlineArgs(argc, argv);
 
    // When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in
    // $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag.
    //
    // See https://bazel.build/reference/test-encyclopedia#initial-conditions.
    std::string testUndeclaredOutputsDir;
    if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) {
        testUndeclaredOutputsDir = envVar;
    }
    bool isBazelTest = !testUndeclaredOutputsDir.empty();
 
    // Parse and validate flags.
    CommandLineFlags::Parse(argc, argv);
    if (!isBazelTest) {
        TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir);
    }
    TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir);
    TestRunner::FlagValidators::StringAtMostOne("--knownDigestsFile", FLAGS_knownDigestsFile);
    TestRunner::FlagValidators::StringEven("--key", FLAGS_key);
    TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig);
    TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig);
    TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName);
    TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName);
    TestRunner::FlagValidators::StringAtMostOne("--via", FLAGS_via);
 
    std::string outputDir =
            FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
 
    auto knownDigests = std::set<std::string>();
    if (!FLAGS_knownDigestsFile.isEmpty()) {
        knownDigests = read_known_digests_file(FLAGS_knownDigestsFile[0]);
        TestRunner::Log(
                "Read %zu known digests from: %s", knownDigests.size(), FLAGS_knownDigestsFile[0]);
    }
 
    std::map<std::string, std::string> keyValuePairs;
    for (int i = 1; i < FLAGS_key.size(); i += 2) {
        keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
    }
    std::string config = FLAGS_surfaceConfig[0];
    std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0];
    std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0];
 
    // Execute all GM registerer functions, then run all registered GMs.
    for (const skiagm::GMRegistererFn& f : skiagm::GMRegistererFnRegistry::Range()) {
        std::string errorMsg = f();
        if (errorMsg != "") {
            SK_ABORT("Error while gathering GMs: %s", errorMsg.c_str());
        }
    }
    for (const skiagm::GMFactory& f : skiagm::GMRegistry::Range()) {
        std::unique_ptr<skiagm::GM> gm = f();
 
        if (!TestRunner::ShouldRunTestCase(gm->getName().c_str(), FLAGS_match, FLAGS_skip)) {
            TestRunner::Log("Skipping %s", gm->getName().c_str());
            gNumSkippedGMs++;
            continue;
        }
 
        run_gm(std::move(gm), config, keyValuePairs, cpuName, gpuName, outputDir, knownDigests);
    }
 
    // TODO(lovisolo): If running under Bazel, print command to display output files.
 
    TestRunner::Log(gNumFailedGMs > 0 ? "FAIL" : "PASS");
    TestRunner::Log(
            "%d successful GMs (images written to %s).", gNumSuccessfulGMs, outputDir.c_str());
    TestRunner::Log("%d failed GMs.", gNumFailedGMs);
    TestRunner::Log("%d skipped GMs.", gNumSkippedGMs);
    return gNumFailedGMs > 0 ? 1 : 0;
}

read_known_digests_file()

std::set< std::string > read_known_digests_file

(

std::string

path

)

Definition at line 313 of file BazelGMTestRunner.cpp.

                                                          {
    std::set<std::string> hashes;
    std::regex md5HashRegex("^[a-fA-F0-9]{32}$");
    std::ifstream f(path);
    std::string line;
    for (int lineNum = 1; std::getline(f, line); lineNum++) {
        // Trim left and right (https://stackoverflow.com/a/217605).
        auto isSpace = [](unsigned char c) { return !std::isspace(c); };
        std::string md5 = line;
        md5.erase(md5.begin(), std::find_if(md5.begin(), md5.end(), isSpace));
        md5.erase(std::find_if(md5.rbegin(), md5.rend(), isSpace).base(), md5.end());
 
        if (md5 == "") continue;
 
        if (!std::regex_match(md5, md5HashRegex)) {
            SK_ABORT(
                    "File '%s' passed via --knownDigestsFile contains an invalid entry on line "
                    "%d: '%s'",
                    path.c_str(),
                    lineNum,
                    line.c_str());
        }
        hashes.insert(md5);
    }
    return hashes;
}

run_gm()

void run_gm	(	std::unique_ptr< skiagm::GM >	gm,
		std::string	config,
		std::map< std::string, std::string >	keyValuePairs,
		std::string	cpuName,
		std::string	gpuName,
		std::string	outputDir,
		std::set< std::string >	knownDigests
	)

Definition at line 210 of file BazelGMTestRunner.cpp.

                                            {
    TestRunner::Log("GM: %s", gm->getName().c_str());
 
    // Create surface and canvas.
    std::unique_ptr<SurfaceManager> surfaceManager = SurfaceManager::FromConfig(
            config, SurfaceOptions{gm->getISize().width(), gm->getISize().height()});
    if (surfaceManager == nullptr) {
        SK_ABORT("Unknown --surfaceConfig flag value: %s.", config.c_str());
    }
 
    // Print warning about missing cpu_or_gpu key if necessary.
    if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" &&
         !gMissingCpuOrGpuWarningLogged)) {
        TestRunner::Log(
                "\tWarning: The surface is CPU-bound, but flag --cpuName was not provided. "
                "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
        gMissingCpuOrGpuWarningLogged = true;
    }
    if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" &&
         !gMissingCpuOrGpuWarningLogged)) {
        TestRunner::Log(
                "\tWarning: The surface is GPU-bound, but flag --gpuName was not provided. "
                "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
        gMissingCpuOrGpuWarningLogged = true;
    }
 
    // Set up GPU.
    TestRunner::Log("\tSetting up GPU...");
    SkString msg;
    skiagm::DrawResult result = gm->gpuSetup(surfaceManager->getSurface()->getCanvas(), &msg);
 
    // Draw GM into canvas if GPU setup was successful.
    SkBitmap bitmap;
    if (result == skiagm::DrawResult::kOk) {
        GMOutput output;
        std::string viaName = FLAGS_via.size() == 0 ? "" : (FLAGS_via[0]);
        TestRunner::Log("\tDrawing GM via \"%s\"...", viaName.c_str());
        output = draw(gm.get(), surfaceManager->getSurface().get(), viaName);
        result = output.result;
        msg = SkString(output.msg.c_str());
        bitmap = output.bitmap;
    }
 
    // Keep track of results. We will exit with a non-zero exit code in the case of failures.
    switch (result) {
        case skiagm::DrawResult::kOk:
            // We don't increment numSuccessfulGMs just yet. We still need to successfully save
            // its output bitmap to disk.
            TestRunner::Log("\tFlushing surface...");
            surfaceManager->flush();
            break;
        case skiagm::DrawResult::kFail:
            gNumFailedGMs++;
            break;
        case skiagm::DrawResult::kSkip:
            gNumSkippedGMs++;
            break;
        default:
            SkUNREACHABLE;
    }
 
    // Report GM result and optional message.
    TestRunner::Log("\tResult: %s", draw_result_to_string(result).c_str());
    if (!msg.isEmpty()) {
        TestRunner::Log("\tMessage: \"%s\"", msg.c_str());
    }
 
    // Save PNG and JSON file with MD5 hash to disk if the GM was successful.
    if (result == skiagm::DrawResult::kOk) {
        std::string name = std::string(gm->getName().c_str());
        SkString pngPath = SkOSPath::Join(outputDir.c_str(), (name + ".png").c_str());
        SkString jsonPath = SkOSPath::Join(outputDir.c_str(), (name + ".json").c_str());
 
        WritePNGAndJSONFilesResult pngAndJSONResult =
                write_png_and_json_files(gm->getName().c_str(),
                                         keyValuePairs,
                                         gm->getGoldKeys(),
                                         surfaceManager->getGoldKeyValuePairs(cpuName, gpuName),
                                         bitmap,
                                         pngPath.c_str(),
                                         jsonPath.c_str(),
                                         knownDigests);
        if (pngAndJSONResult.status == WritePNGAndJSONFilesResult::kError) {
            TestRunner::Log("\tERROR: %s", pngAndJSONResult.errorMsg.c_str());
            gNumFailedGMs++;
        } else if (pngAndJSONResult.status == WritePNGAndJSONFilesResult::kSkippedKnownDigest) {
            TestRunner::Log("\tSkipping known digest: %s", pngAndJSONResult.skippedDigest.c_str());
        } else {
            gNumSuccessfulGMs++;
            TestRunner::Log("\tPNG file written to: %s", pngPath.c_str());
            TestRunner::Log("\tJSON file written to: %s", jsonPath.c_str());
        }
    }
}

write_png_and_json_files()

static WritePNGAndJSONFilesResult write_png_and_json_files ( std::string  name, std::map< std::string, std::string >  commonKeys, std::map< std::string, std::string >  gmGoldKeys, std::map< std::string, std::string >  surfaceGoldKeys, const SkBitmap &  bitmap, const char *  pngPath, const char *  jsonPath, std::set< std::string >  knownDigests  )

static

Definition at line 114 of file BazelGMTestRunner.cpp.

                                        {
    HashAndEncode hashAndEncode(bitmap);
 
    // Compute MD5 hash.
    SkMD5 hash;
    hashAndEncode.feedHash(&hash);
    SkMD5::Digest digest = hash.finish();
    SkString md5 = digest.toLowercaseHexString();
 
    // Skip this digest if it's known.
    if (knownDigests.find(md5.c_str()) != knownDigests.end()) {
        return {
                .status = WritePNGAndJSONFilesResult::kSkippedKnownDigest,
                .skippedDigest = md5.c_str(),
        };
    }
 
    // Write PNG file.
    SkFILEWStream pngFile(pngPath);
    bool result = hashAndEncode.encodePNG(&pngFile,
                                          md5.c_str(),
                                          /* key= */ CommandLineFlags::StringArray(),
                                          /* properties= */ CommandLineFlags::StringArray());
    if (!result) {
        return {
                .status = WritePNGAndJSONFilesResult::kError,
                .errorMsg = "Error encoding or writing PNG to " + std::string(pngPath),
        };
    }
 
    // Validate GM-related Gold keys.
    if (gmGoldKeys.find("name") == gmGoldKeys.end()) {
        SK_ABORT("gmGoldKeys does not contain key \"name\"");
    }
    if (gmGoldKeys.find("source_type") == gmGoldKeys.end()) {
        SK_ABORT("gmGoldKeys does not contain key \"source_type\"");
    }
 
    // Validate surface-related Gold keys.
    if (surfaceGoldKeys.find("surface_config") == surfaceGoldKeys.end()) {
        SK_ABORT("surfaceGoldKeys does not contain key \"surface_config\"");
    }
 
    // Gather all Gold keys.
    std::map<std::string, std::string> keys = {
            {"build_system", "bazel"},
    };
    keys.merge(GetCompilationModeGoldAndPerfKeyValuePairs());
    keys.merge(commonKeys);
    keys.merge(surfaceGoldKeys);
    keys.merge(gmGoldKeys);
 
    // Write JSON file with MD5 hash and Gold key-value pairs.
    SkFILEWStream jsonFile(jsonPath);
    SkJSONWriter jsonWriter(&jsonFile, SkJSONWriter::Mode::kPretty);
    jsonWriter.beginObject();  // Root object.
    jsonWriter.appendString("md5", md5);
    jsonWriter.beginObject("keys");  // "keys" dictionary.
    for (auto const& [param, value] : keys) {
        jsonWriter.appendString(param.c_str(), SkString(value));
    }
    jsonWriter.endObject();  // "keys" dictionary.
    jsonWriter.endObject();  // Root object.
 
    return {.status = WritePNGAndJSONFilesResult::kSuccess};
}

Variable Documentation

gMissingCpuOrGpuWarningLogged

bool gMissingCpuOrGpuWarningLogged = false

static

Definition at line 206 of file BazelGMTestRunner.cpp.

gNumFailedGMs

int gNumFailedGMs = 0

static

Definition at line 203 of file BazelGMTestRunner.cpp.

gNumSkippedGMs

int gNumSkippedGMs = 0

static

Definition at line 204 of file BazelGMTestRunner.cpp.

gNumSuccessfulGMs

int gNumSuccessfulGMs = 0

static

Definition at line 202 of file BazelGMTestRunner.cpp.

unused

bool unused

static

Initial value:

=
        SkFlagInfo::CreateStringFlag("device-specific-bazel-config",
                                     nullptr,
                                     new CommandLineFlags::StringArray(),
                                     nullptr,
                                     "Ignored by this test runner.",
                                     nullptr)

Definition at line 98 of file BazelGMTestRunner.cpp.