rewrapper_dart Namespace Reference

Classes
class	Rewrapper

Functions
def	run_command (command, strategy)
def	load_package_config (exec_root)
def	resolve_uri (uri, exec_root, package_config, whole_dir=False)
def	list_imports (uri, exec_root, package_config)
def	find_inputs (uris, exec_root, package_config)
def	rewrite_absolute (arg, exec_root, working_directory)
def	main (argv)

Function Documentation

find_inputs()

def rewrapper_dart.find_inputs	(		uris,
			exec_root,
			package_config
	)

Definition at line 112 of file rewrapper_dart.py.

def find_inputs(uris, exec_root, package_config):
    inputs = set(uris)
    unexplored = set(uris)
    while unexplored:
        uri = unexplored.pop()
        imports = list_imports(uri, exec_root, package_config)
        for uri in imports:
            if not uri in inputs:
                inputs.add(uri)
                unexplored.add(uri)
    return inputs
 
 
# Rewrite absolute paths in an argument to be relative.

list_imports()

def rewrapper_dart.list_imports	(		uri,
			exec_root,
			package_config
	)

Definition at line 72 of file rewrapper_dart.py.

def list_imports(uri, exec_root, package_config):
    if uri.startswith('dart:'):
        return set()
    path = os.path.join(exec_root, resolve_uri(uri, exec_root, package_config))
    file = open(path, 'r')
    imports = set()
    for line in file.readlines():
        tokens = [token for token in re.split(r'\s+', line) if token != '']
        if not tokens or tokens[0] in [
                '//', '///', '/*', '*/', '#!', 'library', 'show'
        ]:
            continue
        # Imports must happen before definitions.
        if tokens[0] in ['const', 'class', 'enum']:
            break
        if 2 <= len(tokens
                   ) and tokens[0] == 'if' and tokens[1] == '(dart.library.io)':
            tokens = ['import'] + tokens[2:]
        if tokens[0] not in ['import', 'export', 'part']:
            continue
        if len(tokens) < 2:
            raise Exception(f'Bad import statement: {path}: {line}')
        if tokens[0] == 'part' and tokens[1] == 'of':
            continue
        token = tokens[1].replace('"', '').replace("'", '').replace(';', '')
        if token.startswith('dart:'):
            continue
        if not ':' in token:
            dirname = os.path.dirname(uri)
            while token.startswith('..'):
                token = token[3:]
                dirname = os.path.dirname(dirname)
            token = dirname + '/' + token
        imports.add(token)
    file.close()
    return imports
 
 
# Transitively compute the set of dart files needed to execute the specified
# entry point using the package config.

load_package_config()

def rewrapper_dart.load_package_config

(

exec_root

)

Definition at line 45 of file rewrapper_dart.py.

def load_package_config(exec_root):
    path = os.path.join(exec_root, '.dart_tool', 'package_config.json')
    with open(path, 'r') as file:
        return json.load(file)
 
 
# Resolves a Dart import URI using the package config.

main()

def rewrapper_dart.main

(

argv

)

Definition at line 683 of file rewrapper_dart.py.

def main(argv):
    # Like gn_run_binary, run programs relative to the build directory. The
    # command is assumed to invoke rewrapper and end its rewrapper arguments
    # with an -- argument.
    rewrapper_end = 0
    for i in range(len(argv)):
        if argv[i] == '--' and rewrapper_end == 0:
            rewrapper_end = i + 1
            if not '/' in argv[i + 1]:
                argv[i + 1] = './' + argv[i + 1]
            break
 
    rewrapper = Rewrapper(argv)
 
    if rewrapper.exec_root == None:
        raise Exception('No rewrapper --exec_root was specified')
 
    if not rewrapper.outputs:
        raise Exception('No output files were recognized')
 
    # Run the command directly if it's not supported for remote builds.
    if rewrapper.no_remote:
        run_command(argv[rewrapper_end:], 'local')
        return 0
 
    # Determine the set of input and output files.
    package_config = load_package_config(rewrapper.exec_root)
    if not rewrapper.depfiles:
        rewrapper.depfiles = [output + '.d' for output in rewrapper.outputs]
    output_files = rewrapper.outputs + rewrapper.depfiles
    inputs = find_inputs(rewrapper.entry_points, rewrapper.exec_root,
                         package_config)
    paths = set(
        resolve_uri(uri, rewrapper.exec_root, package_config, whole_dir=True)
        for uri in inputs)
    paths.add(os.path.join('.dart_tool', 'package_config.json'))
    for path in rewrapper.extra_paths:
        paths.add(path)
    # Ensure the working directory is included if no inputs are inside it.
    working_directory = rewrapper.rebase('.')
    if not any([path.startswith(working_directory) for path in paths]):
        paths.add(rewrapper.rebase('build.ninja.stamp'))
    paths = list(paths)
    paths.sort()
 
    # Construct the final rewrapped command line.
    command = [argv[1]]
    command.append('--labels=type=tool')
    command.append('--inputs=' + ','.join(paths))
    command.append('--output_files=' + ','.join(output_files))
    # Absolute paths must not be used with RBE, but since the build currently
    # heavily relies on them, work around this issue by rewriting the command
    # to instead use relative paths. The Dart SDK build rules needs to be fixed
    # rather than doing this, but this is an initial step towards that goal
    # which will land in subsequent follow up changes.
    command += argv[2:rewrapper_end] + [
        rewrite_absolute(arg, rewrapper.exec_root, working_directory)
        for arg in argv[rewrapper_end:]
    ]
 
    # Finally execute the command remotely.
    run_command(command, rewrapper.exec_strategy)
 
    # Until the depfiles are fixed so they don't contain absoiute paths, we need
    # to rewrite the absoute paths appropriately.
    for depfile in rewrapper.depfiles:
        lines = []
        try:
            with open(os.path.join(rewrapper.exec_root, depfile), 'r') as file:
                lines = file.readlines()
            lines = [
                line.replace('/b/f/w', rewrapper.exec_root) for line in lines
            ]
            with open(os.path.join(rewrapper.exec_root, depfile), 'w') as file:
                file.writelines(lines)
        except FileNotFoundError:
            pass
 
    return 0
 
 

resolve_uri()

def rewrapper_dart.resolve_uri	(		uri,
			exec_root,
			package_config,
			whole_dir = False
	)

Definition at line 52 of file rewrapper_dart.py.

def resolve_uri(uri, exec_root, package_config, whole_dir=False):
    if uri.startswith('package:'):
        match = re.search(r'package:([^/]*)/(.*)', uri)
        package_name = match.groups()[0]
        relative = match.groups()[1]
        package_data = next(pkg for pkg in package_config['packages']
                            if pkg['name'] == package_name)
        package_root = package_data['rootUri']
        package_root = package_root[3:]  # Remove leading ../
        package_uri = package_data['packageUri']
        if whole_dir:
            uri = package_root + '/' + package_uri
        else:
            uri = package_root + '/' + package_uri + relative
    return uri
 
 
# Lists the imports of a Dart file uri using the package config and a rough
# parser that recognizes fairly traditional imports. This is designed to be much
# faster than actually invoking the front end.

rewrite_absolute()

def rewrapper_dart.rewrite_absolute	(		arg,
			exec_root,
			working_directory
	)

Definition at line 126 of file rewrapper_dart.py.

def rewrite_absolute(arg, exec_root, working_directory):
    # The file:// schema does not work with relative paths as they are parsed as
    # the authority by the dart Uri class.
    arg = arg.replace('file:///' + exec_root, '../../')
    arg = arg.replace('file://' + exec_root, '../../')
    # Replace the absolute exec root by a relative path to the exec root.
    arg = arg.replace(exec_root, '../../')
    # Simplify paths going to the exec root and back into the out directory.
    # Carefully ensure the whole path isn't optimized away.
    if arg.endswith(f'../../{working_directory}/'):
        arg = arg.replace(f'../../{working_directory}/', '.')
    else:
        arg = arg.replace(f'../../{working_directory}/', '')
    return arg
 
 
# Parse the command line execution to recognize well known programs during the
# Dart SDK build, so the inputs and output files can be determined, and the
# command can be offloaded to RBE.
#
# RBE needs a command to run, a list of input files, and a list of output files,
# and it then executes the command remotely and caches the result. Absolute
# paths must not occur in the command as the remote execution will happen in
# another directory. However, since we currently rely on absolute paths, we work
# around the issue and rewrite the absolute paths accordingly until the problem
# is fixed on our end.
#
# This is a parser that handles nested commands executing each other, taking
# care to know whose options it is currently parsing, and extracting the
# appropriate information from each argument. Every invoked program and option
# during the build needs to be supported here, otherwise the remote command may
# be inaccurate and not have right inputs and outputs. Although maintaining this
# parser takes some effort, it is being paid back in the builders being sped up
# massively on cache hits, as well as speeding up any local developers that
# build code already built by the bots.
#
# To add a new program, recognize the entry point and define its parser method.
# To add a new option, parse the option in the appropriate method and either
# ignore it or recognize any input and output files. All invoked options needs
# be allowlisted here know we didn't accidentally misunderstand the invoked
# command when running it remotely.

run_command()

def rewrapper_dart.run_command	(		command,
			strategy
	)

Definition at line 21 of file rewrapper_dart.py.

def run_command(command, strategy):
    try:
        subprocess.check_output(command, stderr=subprocess.STDOUT)
        return 0
    except subprocess.CalledProcessError as e:
        print(e.output.decode("utf-8"))
        if strategy == 'remote':
            joined = ' '.join(command)
            print(f'''Failed to run command remotely: {joined}
 
If you're seeing this error on a bot and it doesn't happen locally, then you may
need to teach the script build/rbe/rewrapper_dart.py what the appropriate input
and outputs files of the command are. You can see the list of used inputs and
outputs in the rewrapper invocation above. To reproduce this error locally, try
forcing a remote build by setting the RBE_exec_strategy=remote environment
variable.
''')
        sys.exit(1)
    except OSError as e:
        print(e.strerror)
        sys.exit(1)
 
 
# Loads the package config file.