collect_data Namespace Reference

Functions
def	get_shard_links (uri)
def	print_reencoded (text)
def	print_output_all (text)
def	print_output_div (shard, text, keywords)
def	get_output_sum (shard, text, should_print, s=[0, 0, 0, 0, 0, 0], divs=[])
def	get_stats (uri, output_type, keywords, output_csv)
def	main ()

Variables
	P_DIV = re.compile("(Isolate.+? !DIVERGENCE! (\n|.)+?)Isolate ", re.MULTILINE)
	P_SUM
	P_SHARD = re.compile(r".*make_a_fuzz_shard_(\d+)")

Function Documentation

get_output_sum()

def collect_data.get_output_sum	(		shard,
			text,
			should_print,
			s = [0, 0, 0, 0, 0, 0],
			divs = []
	)

Definition at line 79 of file collect_data.py.

def get_output_sum(shard, text, should_print, s=[0, 0, 0, 0, 0, 0], divs=[]):
    m = P_SUM.findall(text)
    if not m:
        sys.stderr.write("Failed to parse shard %s stdout for summary" % shard)
        return
    for test in m:
        if int(test[-1]) == 1:
            divs.append(shard)
        for i in range(len(s)):
            s[i] += int(test[i])
    if should_print:
        print(
            "Tests: %d Success: %d (Rerun: %d) Skipped: %d Timeout: %d "
            "Divergences: %d (failing shards: %s)    \r" %
            tuple(s + [", ".join(divs) if divs else "none"]),
            end="")
    return s
 
 

get_shard_links()

def collect_data.get_shard_links

(

uri

)

Definition at line 44 of file collect_data.py.

def get_shard_links(uri):
    links = []
    resp = requests.get(uri)
    soup = BeautifulSoup(resp.text, "html.parser")
    for a in soup.findAll("a"):
        if "stdout" in a.text:
            href = a["href"]
            if ("make_a_fuzz_shard" in href and "__trigger__" not in href):
                links.append(href + "?format=raw")
    return links
 
 

get_stats()

def collect_data.get_stats	(		uri,
			output_type,
			keywords,
			output_csv
	)

Definition at line 98 of file collect_data.py.

def get_stats(uri, output_type, keywords, output_csv):
    resp = requests.get(uri)
 
    if output_type == "all":
        print_output_all(resp.text)
    elif output_type == "div":
        shard = P_SHARD.findall(uri)[0]
        print_output_div(shard, resp.text, keywords)
    elif output_type == "sum":
        shard = P_SHARD.findall(uri)[0]
        should_print = not output_csv
        return get_output_sum(shard, resp.text, should_print)
    return None
 
 

main()

def collect_data.main

(

)

Definition at line 113 of file collect_data.py.

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--type",
        choices=("div", "sum", "all"),
        required=True,
        help=
        "Select output type (div: divergence report, sum: summary, all: complete stdout)"
    )
    parser.add_argument(
        "--filter",
        nargs="+",
        default=[],
        help="Do not include divergences containing these keywords.")
    parser.add_argument(
        "--output-csv",
        dest="output_csv",
        action="store_true",
        default=False,
        help=
        "Print output in CSV format to stdout. Only supported for --type=sum")
    parser.add_argument(
        "uri",
        type=str,
        help=
        "Uri of one make_a_fuzz run from https://ci.chromium.org/p/dart/builders/ci.sandbox/fuzz-linux."
    )
    args = parser.parse_args()
    if args.type != 'sum' and args.output_csv:
        print('Error: --output-csv can only be provided for --type=sum')
        return
 
    shard_links = get_shard_links(args.uri)
 
    if len(shard_links) == 0:
        print("Invalid run")
        sys.exit(-1)
        return
 
    for link in shard_links:
        stats = get_stats(link, args.type, args.filter, args.output_csv)
    if args.output_csv:
        print("%d,%d,%d,%d,%d,%d" % tuple(stats))
    else:
        print("")
    sys.exit(0)
 
 

print_output_all()

def collect_data.print_output_all

(

text

)

Definition at line 61 of file collect_data.py.

def print_output_all(text):
    print_reencoded(text)
 
 

print_output_div()

def collect_data.print_output_div	(		shard,
			text,
			keywords
	)

Definition at line 65 of file collect_data.py.

def print_output_div(shard, text, keywords):
    sys.stderr.write("Shard: " + shard + "  \r")
    m = P_DIV.findall(text)
    if m:
        for x in m:
            keep = True
            for word in keywords:
                if word in x[0]:
                    keep = False
                    break
            if keep:
                print_reencoded(x[0])
 
 

print_reencoded()

def collect_data.print_reencoded

(

text

)

Definition at line 56 of file collect_data.py.

def print_reencoded(text):
    # Re-encoding avoids breaking some terminals.
    print(text.encode("ascii", errors="ignore").decode("unicode-escape"))
 
 

Variable Documentation

P_DIV

collect_data.P_DIV = re.compile("(Isolate.+? !DIVERGENCE! (\n|.)+?)Isolate ", re.MULTILINE)

Definition at line 32 of file collect_data.py.

P_SHARD

collect_data.P_SHARD = re.compile(r".*make_a_fuzz_shard_(\d+)")

Definition at line 41 of file collect_data.py.

P_SUM

collect_data.P_SUM

Initial value:

=  re.compile(
    r"^Tests: (\d+) Success: (\d+) "
    r"\‍(Rerun: (\d+)\‍) Skipped: (\d+) "
    r"Timeout: (\d+) Divergences: (\d+)", re.MULTILINE)

Definition at line 35 of file collect_data.py.