build Namespace Reference

Namespaces
namespace	android
namespace	api
namespace	canvaskit
namespace	chromebook
namespace	cmake
namespace	default
namespace	docker
namespace	pathkit
namespace	util

Classes
class	Artifact
class	ArtifactEventHandler
class	Page
class	Style

Functions
	find_images_directories ()
	find_artifacts ()
	build_for_deploy ()
	serve_for_development ()
	main ()
	BuildOptions ()
	NotifyBuildDone (build_config, success, start)
	UseRBE (out_dir)
	StartRBE (out_dir, env)
	StopRBE (env)
	BuildOneConfig (options, targets, target_os, mode, arch, sanitizer, env)
	RunOneBuildCommand (build_config, args, env)
	CheckCleanBuild (build_config, args, env)
	SanitizerEnvironmentVariables ()
	Build (configs, env, options)
	Main ()
	GetAllBuilders ()

Variables
	level
	fmt
	datefmt
	TOOL_DIR = os.path.dirname(os.path.realpath(__file__))
	SDK_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', '..', '..', '..'))
	WIKI_SOURCE_DIR = os.path.join(SDK_DIR, 'runtime', 'docs')
	STYLES_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', 'styles'))
	STYLES_INCLUDES_DIR = os.path.join(STYLES_DIR, 'includes')
	TEMPLATES_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', 'templates'))
	TEMPLATES_INCLUDES_DIR = os.path.join(TEMPLATES_DIR, 'includes')
str	PAGE_TEMPLATE = 'page.html'
str	OUTPUT_DIR = '/tmp/dart-vm-wiki'
	OUTPUT_CSS_DIR = os.path.join(OUTPUT_DIR, 'css')
	ignore_errors
	exist_ok
	parser = argparse.ArgumentParser()
	dest
	action
	default
	deploy
	args = parser.parse_args()
	is_dev_mode = not args.deploy
	deployment_root = args.deployment_root
	jinja2_env
	xref_extension = XrefExtension()
	HOST_OS = utils.GuessOS()
	HOST_CPUS = utils.GuessCpus()
	SCRIPT_DIR = os.path.dirname(sys.argv[0])
	DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
	AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()
str	usage
bool	rbe_started = False
	bootstrap_path = None
str	PYTHON_VERSION_COMPATIBILITY = "PY3"
list	DEPS
dict	PROPERTIES

Detailed Description

Tool used for rendering Dart Native Runtime wiki as HTML.

Usage: runtime/tools/wiki/build/build.py [--deploy]

If invoked without --deploy the tool would serve development version of the
wiki which supports fast edit-(auto)refresh cycle.

If invoked with --deploy it would build deployment version in the
/tmp/dart-vm-wiki directory.

Function Documentation

Build()

build.Build	(		configs,
			env,
			options
	)

Definition at line 232 of file build.py.

def Build(configs, env, options):
    # Build regular configs.
    rbe_builds = []
    for (build_config, args, rbe) in configs:
        if args is None:
            return 1
        if rbe:
            rbe_builds.append([env, args])
        elif RunOneBuildCommand(build_config, args, env=env) != 0:
            return 1
 
    # Run RBE builds in parallel.
    active_rbe_builds = []
    for (env, args) in rbe_builds:
        print(' '.join(args))
        process = subprocess.Popen(args, env=env)
        active_rbe_builds.append([args, process])
    while active_rbe_builds:
        time.sleep(0.1)
        for rbe_build in active_rbe_builds:
            (args, process) = rbe_build
            if process.poll() is not None:
                print(' '.join(args) + " done.")
                active_rbe_builds.remove(rbe_build)
                if process.returncode != 0:
                    for (_, to_kill) in active_rbe_builds:
                        to_kill.terminate()
                    return 1
 
    if options.check_clean:
        for (build_config, args, rbe) in configs:
            if CheckCleanBuild(build_config, args, env=env) != 0:
                return 1
 
    return 0
 
 

build_for_deploy()

build.build_for_deploy

(

)

Create a directory which can be deployed to static hosting.

Definition at line 244 of file build.py.

def build_for_deploy():
    """Create a directory which can be deployed to static hosting."""
    logging.info('Building wiki for deployment into %s', OUTPUT_DIR)
    Artifact.build_all()
    for images_dir in find_images_directories():
        src = os.path.join(WIKI_SOURCE_DIR, images_dir)
        dst = os.path.join(OUTPUT_DIR, images_dir)
        logging.info('Copying %s <- %s', dst, src)
        shutil.rmtree(dst, ignore_errors=True)
        shutil.copytree(src, dst)
 
    # Some images directories contain OmniGraffle source files which need
    # to be removed before
    logging.info('Removing image source files (*.graffle)')
    for graffle in Path(OUTPUT_DIR).rglob('*.graffle'):
        logging.info('... removing %s', graffle.as_posix())
 
 

BuildOneConfig()

build.BuildOneConfig	(		options,
			targets,
			target_os,
			mode,
			arch,
			sanitizer,
			env
	)

Definition at line 164 of file build.py.

def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer, env):
    build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer)
    out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
    using_rbe = False
    command = ['buildtools/ninja/ninja', '-C', out_dir]
    if options.verbose:
        command += ['-v']
    if UseRBE(out_dir):
        if options.no_start_rbe or StartRBE(out_dir, env):
            using_rbe = True
            command += [('-j%s' % str(options.j))]
            command += [('-l%s' % str(options.l))]
        else:
            exit(1)
    command += targets
    return (build_config, command, using_rbe)
 
 

BuildOptions()

build.BuildOptions

(

)

Definition at line 30 of file build.py.

def BuildOptions():
    parser = argparse.ArgumentParser(
        description='Runs GN (if necessary) followed by ninja',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
    config_group = parser.add_argument_group('Configuration Related Arguments')
    gn_py.AddCommonConfigurationArgs(config_group)
 
    gn_group = parser.add_argument_group('GN Related Arguments')
    gn_py.AddCommonGnOptionArgs(gn_group)
 
    other_group = parser.add_argument_group('Other Arguments')
    gn_py.AddOtherArgs(other_group)
 
    other_group.add_argument("-j",
                             type=int,
                             help='Ninja -j option for RBE builds.',
                             default=200 if sys.platform == 'win32' else 1000)
    other_group.add_argument("-l",
                             type=int,
                             help='Ninja -l option for RBE builds.',
                             default=64)
    other_group.add_argument("--no-start-rbe",
                             help="Don't try to start rbe",
                             default=False,
                             action='store_true')
    other_group.add_argument(
        "--check-clean",
        help="Check that a second invocation of Ninja has nothing to do",
        default=False,
        action='store_true')
 
    parser.add_argument('build_targets', nargs='*')
 
    return parser
 
 

CheckCleanBuild()

build.CheckCleanBuild	(		build_config,
			args,
			env
	)

Definition at line 196 of file build.py.

def CheckCleanBuild(build_config, args, env):
    args = args + ['-n', '-d', 'explain']
    print(' '.join(args))
    process = subprocess.Popen(args,
                               env=env,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               stdin=None)
    out, err = process.communicate()
    process.wait()
    if process.returncode != 0:
        return 1
    if 'ninja: no work to do' not in out.decode('utf-8'):
        print(err.decode('utf-8'))
        return 1
 
    return 0
 
 

find_artifacts()

build.find_artifacts

(

)

Find all wiki pages and styles and create corresponding Artifacts.

Definition at line 232 of file build.py.

def find_artifacts():
    """Find all wiki pages and styles and create corresponding Artifacts."""
    Artifact.all = {}
    for file in Path(WIKI_SOURCE_DIR).rglob('*.md'):
        name = file.relative_to(Path(WIKI_SOURCE_DIR)).as_posix().rsplit(
            '.', 1)[0]
        Page(name)
 
    for file in Path(STYLES_DIR).glob('*.scss'):
        Style(file.stem)
 
 

find_images_directories()

build.find_images_directories

(

)

Find all subdirectories called images within wiki.

Definition at line 224 of file build.py.

def find_images_directories():
    """Find all subdirectories called images within wiki."""
    return [
        f.relative_to(Path(WIKI_SOURCE_DIR)).as_posix()
        for f in Path(WIKI_SOURCE_DIR).rglob('images')
    ]
 
 

GetAllBuilders()

build.GetAllBuilders

(

)

Definition at line 21 of file build.py.

def GetAllBuilders():
  curl_command = [
      'curl',
      'https://ci.chromium.org/p/flutter/g/engine/builders',
  ]
  curl_result = subprocess.run(
      curl_command,
      universal_newlines=True,
      capture_output=True,
  )
  if curl_result.returncode != 0:
    print('Failed to fetch builder list: stderr:\n%s' % curl_result.stderr)
    return []
  sed_command = [
      'sed',
      '-En',
      's:.*aria-label="builder buildbucket/luci\\.flutter\\.prod/([^/]+)".*:\\1:p',
  ]
  sed_result = subprocess.run(
      sed_command,
      input=curl_result.stdout,
      capture_output=True,
      universal_newlines=True,
  )
  if sed_result.returncode != 0:
    print('Failed to fetch builder list: stderr:\n%s' % sed_result.stderr)
  return list(set(sed_result.stdout.splitlines()))
 
 

main()

build.main

(

)

Main entry point.

Definition at line 348 of file build.py.

def main():
    """Main entry point."""
    find_artifacts()
    if is_dev_mode:
        serve_for_development()
    else:
        build_for_deploy()
 
 

Main()

build.Main

(

)

Definition at line 269 of file build.py.

def Main():
    starttime = time.time()
    # Parse the options.
    parser = BuildOptions()
    options = parser.parse_args()
 
    targets = options.build_targets
 
    if not gn_py.ProcessOptions(options):
        parser.print_help()
        return 1
 
    # If binaries are built with sanitizers we should use those flags.
    # If the binaries are not built with sanitizers the flag should have no
    # effect.
    env = dict(os.environ)
    env.update(SanitizerEnvironmentVariables())
 
    # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
    #
    # See:
    #
    #   * https://openradar.appspot.com/radar?id=5608755232243712
    #   * https://github.com/dart-lang/sdk/issues/52411
    #
    # Remove these environment variables to avoid affecting clang's behaviors.
    if sys.platform == 'darwin':
        env.pop('CPATH', None)
        env.pop('LIBRARY_PATH', None)
        env.pop('SDKROOT', None)
 
    # Always run GN before building.
    gn_py.RunGnOnConfiguredConfigurations(options, env)
 
    # Build all targets for each requested configuration.
    configs = []
    for target_os in options.os:
        for mode in options.mode:
            for arch in options.arch:
                for sanitizer in options.sanitizer:
                    configs.append(
                        BuildOneConfig(options, targets, target_os, mode, arch,
                                       sanitizer, env))
 
    exit_code = Build(configs, env, options)
 
    endtime = time.time()
 
    StopRBE(env)
 
    if exit_code == 0:
        print("The build took %.3f seconds" % (endtime - starttime))
    return exit_code
 
 

NotifyBuildDone()

build.NotifyBuildDone	(		build_config,
			success,
			start
	)

Definition at line 67 of file build.py.

def NotifyBuildDone(build_config, success, start):
    if not success:
        print("BUILD FAILED")
 
    sys.stdout.flush()
 
    # Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY.
    notification_delay = float(
        os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max))
    if (time.time() - start) < notification_delay:
        return
 
    if success:
        message = 'Build succeeded.'
    else:
        message = 'Build failed.'
    title = build_config
 
    command = None
    if HOST_OS == 'macos':
        # Use AppleScript to display a UI non-modal notification.
        script = 'display notification  "%s" with title "%s" sound name "Glass"' % (
            message, title)
        command = "osascript -e '%s' &" % script
    elif HOST_OS == 'linux':
        if success:
            icon = 'dialog-information'
        else:
            icon = 'dialog-error'
        command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title)
    elif HOST_OS == 'win32':
        if success:
            icon = 'info'
        else:
            icon = 'error'
        command = (
            "powershell -command \""
            "[reflection.assembly]::loadwithpartialname('System.Windows.Forms')"
            "| Out-Null;"
            "[reflection.assembly]::loadwithpartialname('System.Drawing')"
            "| Out-Null;"
            "$n = new-object system.windows.forms.notifyicon;"
            "$n.icon = [system.drawing.systemicons]::information;"
            "$n.visible = $true;"
            "$n.showballoontip(%d, '%s', '%s', "
            "[system.windows.forms.tooltipicon]::%s);\"") % (
                5000,  # Notification stays on for this many milliseconds
                message,
                title,
                icon)
 
    if command:
        # Ignore return code, if this command fails, it doesn't matter.
        os.system(command)
 
 

RunOneBuildCommand()

build.RunOneBuildCommand	(		build_config,
			args,
			env
	)

Definition at line 182 of file build.py.

def RunOneBuildCommand(build_config, args, env):
    start_time = time.time()
    print(' '.join(args))
    process = subprocess.Popen(args, env=env, stdin=None)
    process.wait()
    if process.returncode != 0:
        NotifyBuildDone(build_config, success=False, start=start_time)
        return 1
    else:
        NotifyBuildDone(build_config, success=True, start=start_time)
 
    return 0
 
 

SanitizerEnvironmentVariables()

build.SanitizerEnvironmentVariables

(

)

Definition at line 215 of file build.py.

def SanitizerEnvironmentVariables():
    with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
        config = json.loads(fd.read())
        env = dict()
        for k, v in config['sanitizer_options'].items():
            env[str(k)] = str(v)
        symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None)
        if symbolizer_path:
            symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path))
            env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['LSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['UBSAN_SYMBOLIZER_PATH'] = symbolizer_path
        return env
 
 

serve_for_development()

build.serve_for_development

(

)

Serve wiki for development (with hot refresh).

Definition at line 270 of file build.py.

def serve_for_development():
    """Serve wiki for development (with hot refresh)."""
    logging.info('Serving wiki for development')
    Artifact.build_all()
 
    # Watch for file modifications and rebuild dependant artifacts when their
    # dependencies change.
    event_handler = ArtifactEventHandler()
    observer = Observer()
    observer.schedule(event_handler, TEMPLATES_DIR, recursive=False)
    observer.schedule(event_handler, WIKI_SOURCE_DIR, recursive=True)
    observer.schedule(event_handler, STYLES_DIR, recursive=True)
    observer.start()
 
    async def on_shutdown(app):
        for ws in app['websockets']:
            await ws.close(code=WSCloseCode.GOING_AWAY,
                           message='Server shutdown')
        observer.stop()
        observer.join()
 
    async def handle_artifact(name):
        source_path = os.path.join(OUTPUT_DIR, name)
        logging.info('Handling source path %s for %s', source_path, name)
        if source_path in Artifact.all:
            return web.FileResponse(source_path)
        else:
            return web.HTTPNotFound()
 
    async def handle_page(request):
        name = request.match_info.get('name', 'index.html')
        if name == '' or name.endswith('/'):
            name = name + 'index.html'
        return await handle_artifact(name)
 
    async def handle_css(request):
        name = request.match_info.get('name')
        return await handle_artifact('css/' + name)
 
    async def websocket_handler(request):
        logging.info('websocket connection open')
        ws = web.WebSocketResponse()
        await ws.prepare(request)
 
        loop = asyncio.get_event_loop()
 
        def notify():
            logging.info('requesting reload')
            asyncio.run_coroutine_threadsafe(ws.send_str('reload'), loop)
 
        Artifact.listeners.append(notify)
        request.app['websockets'].append(ws)
        try:
            async for msg in ws:
                if msg.type == WSMsgType.ERROR:
                    logging.error(
                        'websocket connection closed with exception %s',
                        ws.exception())
        finally:
            logging.info('websocket connection closing')
            Artifact.listeners.remove(notify)
            request.app['websockets'].remove(ws)
 
        logging.info('websocket connection closed')
        return ws
 
    app = web.Application()
    app['websockets'] = []
    for images_dir in find_images_directories():
        app.router.add_static('/' + images_dir,
                              os.path.join(WIKI_SOURCE_DIR, images_dir))
    app.router.add_get('/ws', websocket_handler)
    app.router.add_get('/css/{name}', handle_css)
    app.router.add_get('/{name:[^{}]*}', handle_page)
    app.on_shutdown.append(on_shutdown)
    web.run_app(app, access_log_format='"%r" %s')
 
 

StartRBE()

build.StartRBE	(		out_dir,
			env
	)

Definition at line 134 of file build.py.

def StartRBE(out_dir, env):
    global rbe_started, bootstrap_path
    if not rbe_started:
        rbe_dir = 'buildtools/reclient'
        with open(os.path.join(out_dir, 'args.gn'), 'r') as fp:
            for line in fp:
                if 'rbe_dir' in line:
                    words = line.split()
                    rbe_dir = words[2][1:-1]  # rbe_dir = "/path/to/rbe"
        bootstrap_path = os.path.join(rbe_dir, 'bootstrap')
        bootstrap_command = [bootstrap_path]
        process = subprocess.Popen(bootstrap_command, env=env)
        process.wait()
        if process.returncode != 0:
            print('Failed to start RBE')
            return False
        rbe_started = True
    return True
 
 

StopRBE()

build.StopRBE

(

env

)

Definition at line 154 of file build.py.

def StopRBE(env):
    global rbe_started, bootstrap_path
    if rbe_started:
        bootstrap_command = [bootstrap_path, '--shutdown']
        process = subprocess.Popen(bootstrap_command, env=env)
        process.wait()
        rbe_started = False
 
 
# Returns a tuple (build_config, command to run, whether rbe is used)

UseRBE()

build.UseRBE

(

out_dir

)

Definition at line 123 of file build.py.

def UseRBE(out_dir):
    args_gn = os.path.join(out_dir, 'args.gn')
    return 'use_rbe = true' in open(args_gn, 'r').read()
 
 
# Try to start RBE, but don't bail out if we can't. Instead print an error
# message, and let the build fail with its own error messages as well.

Variable Documentation

action

build.action

Definition at line 72 of file build.py.

args

build.args = parser.parse_args()

Definition at line 75 of file build.py.

AVAILABLE_ARCHS

build.AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()

Definition at line 22 of file build.py.

bootstrap_path

build.bootstrap_path = None

Definition at line 131 of file build.py.

DART_ROOT

build.DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))

Definition at line 21 of file build.py.

datefmt

build.datefmt

Definition at line 45 of file build.py.

default

build.default

Definition at line 73 of file build.py.

deploy

build.deploy

Definition at line 74 of file build.py.

deployment_root

build.deployment_root = args.deployment_root

Definition at line 78 of file build.py.

DEPS

list build.DEPS

Initial value:

=  [
  'depot_tools/gclient',
  'docker',
  'env',
  'infra',
  'recipe_engine/context',
  'recipe_engine/file',
  'recipe_engine/path',
  'recipe_engine/step',
  'run',
  'vars',
]

Definition at line 7 of file __init__.py.

dest

build.dest

Definition at line 72 of file build.py.

exist_ok

build.exist_ok

Definition at line 67 of file build.py.

fmt

build.fmt

Definition at line 44 of file build.py.

HOST_CPUS

build.HOST_CPUS = utils.GuessCpus()

Definition at line 19 of file build.py.

HOST_OS

build.HOST_OS = utils.GuessOS()

Definition at line 18 of file build.py.

ignore_errors

build.ignore_errors

Definition at line 66 of file build.py.

is_dev_mode

build.is_dev_mode = not args.deploy

Definition at line 77 of file build.py.

jinja2_env

build.jinja2_env

Initial value:

=  jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR),
                                lstrip_blocks=True,
                                trim_blocks=True)

Definition at line 81 of file build.py.

level

build.level

Definition at line 43 of file build.py.

OUTPUT_CSS_DIR

build.OUTPUT_CSS_DIR = os.path.join(OUTPUT_DIR, 'css')

Definition at line 63 of file build.py.

OUTPUT_DIR

build.OUTPUT_DIR = '/tmp/dart-vm-wiki'

Definition at line 62 of file build.py.

PAGE_TEMPLATE

str build.PAGE_TEMPLATE = 'page.html'

Definition at line 60 of file build.py.

parser

build.parser = argparse.ArgumentParser()

Definition at line 71 of file build.py.

PROPERTIES

dict build.PROPERTIES

Initial value:

=  {
  'buildername': Property(default=None),
}

Definition at line 22 of file __init__.py.

PYTHON_VERSION_COMPATIBILITY

str build.PYTHON_VERSION_COMPATIBILITY = "PY3"

Definition at line 5 of file __init__.py.

rbe_started

bool build.rbe_started = False

Definition at line 130 of file build.py.

SCRIPT_DIR

build.SCRIPT_DIR = os.path.dirname(sys.argv[0])

Definition at line 20 of file build.py.

SDK_DIR

build.SDK_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', '..', '..', '..'))

Definition at line 50 of file build.py.

STYLES_DIR

build.STYLES_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', 'styles'))

Definition at line 54 of file build.py.

STYLES_INCLUDES_DIR

build.STYLES_INCLUDES_DIR = os.path.join(STYLES_DIR, 'includes')

Definition at line 55 of file build.py.

TEMPLATES_DIR

build.TEMPLATES_DIR = os.path.relpath(os.path.join(TOOL_DIR, '..', 'templates'))

Definition at line 57 of file build.py.

TEMPLATES_INCLUDES_DIR

build.TEMPLATES_INCLUDES_DIR = os.path.join(TEMPLATES_DIR, 'includes')

Definition at line 58 of file build.py.

TOOL_DIR

build.TOOL_DIR = os.path.dirname(os.path.realpath(__file__))

Definition at line 49 of file build.py.

usage

str build.usage

Initial value:

=  """\
usage: %%prog [options] [targets]
 
This script invokes ninja to build Dart.
"""

Definition at line 24 of file build.py.

WIKI_SOURCE_DIR

build.WIKI_SOURCE_DIR = os.path.join(SDK_DIR, 'runtime', 'docs')

Definition at line 52 of file build.py.

xref_extension

build.xref_extension = XrefExtension()

Definition at line 127 of file build.py.