merge_and_upload_debug_symbols Namespace Reference

Functions
def	IsLinux ()
def	GetPackagingDir (out_dir)
def	CreateCIPDDefinition (target_arch, out_dir, symbol_dirs)
def	WriteCIPDDefinition (target_arch, out_dir, symbol_dirs)
def	CheckCIPDPackageExists (package_name, tag)
def	ProcessCIPDPackage (upload, cipd_yaml, engine_version, out_dir, target_arch)
def	HardlinkContents (dirA, dirB)
def	CalculateAbsoluteDirs (dirs)
def	main ()

Variables
	BUILD_ROOT_DIR = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..', '..'))

Function Documentation

CalculateAbsoluteDirs()

def merge_and_upload_debug_symbols.CalculateAbsoluteDirs

(

dirs

)

Definition at line 151 of file merge_and_upload_debug_symbols.py.

def CalculateAbsoluteDirs(dirs):
  results = []
  for directory in dirs:
    if os.path.isabs(directory):
      results.append(directory)
    else:
      results.append(os.path.join(BUILD_ROOT_DIR, directory))
  return results
 
 

CheckCIPDPackageExists()

def merge_and_upload_debug_symbols.CheckCIPDPackageExists	(		package_name,
			tag
	)

Check to see if the current package/tag combo has been published

Definition at line 62 of file merge_and_upload_debug_symbols.py.

def CheckCIPDPackageExists(package_name, tag):
  '''Check to see if the current package/tag combo has been published'''
  command = [
      'cipd',
      'search',
      package_name,
      '-tag',
      tag,
  ]
  stdout = subprocess.check_output(command)
  stdout = stdout if isinstance(stdout, str) else stdout.decode('UTF-8')
  match = re.search(r'No matching instances\.', stdout)
  if match:
    return False
  else:
    return True
 
 

CreateCIPDDefinition()

def merge_and_upload_debug_symbols.CreateCIPDDefinition	(		target_arch,
			out_dir,
			symbol_dirs
	)

Definition at line 37 of file merge_and_upload_debug_symbols.py.

def CreateCIPDDefinition(target_arch, out_dir, symbol_dirs):
  dir_name = os.path.basename(os.path.normpath(out_dir))
  pkg_def = """
package: flutter/fuchsia-debug-symbols-%s
description: Flutter and Dart runner debug symbols for Fuchsia. Target architecture %s.
install_mode: copy
data:
""" % (target_arch, target_arch)
  for symbol_dir in symbol_dirs:
    symbol_dir_name = os.path.basename(os.path.normpath(symbol_dir))
    data = '\n  - dir: %s' % (symbol_dir_name)
    pkg_def = pkg_def + data
  return pkg_def
 
 
# CIPD CLI needs the definition and data directory to be relative to each other.

GetPackagingDir()

def merge_and_upload_debug_symbols.GetPackagingDir

(

out_dir

)

Definition at line 33 of file merge_and_upload_debug_symbols.py.

def GetPackagingDir(out_dir):
  return os.path.abspath(out_dir)
 
 

HardlinkContents()

def merge_and_upload_debug_symbols.HardlinkContents	(		dirA,
			dirB
	)

Definition at line 123 of file merge_and_upload_debug_symbols.py.

def HardlinkContents(dirA, dirB):
  internal_symbol_dirs = []
  for src_dir, _, filenames in os.walk(dirA):
    for filename in filenames:
      # if a file contains 'dbg_success' in its name, it is a stamp file.
      # An example of this would be
      # '._dart_jit_runner_dbg_symbols_unstripped_dbg_success' these
      # are generated by GN and have to be ignored.
      if 'dbg_success' in filename:
        continue
      src = os.path.join(src_dir, filename)
      dest_dir = os.path.join(dirB, os.path.relpath(src_dir, dirA))
      try:
        os.makedirs(dest_dir)
        internal_symbol_dirs.append(dest_dir)
      except:
        pass
      dest = os.path.join(dest_dir, filename)
      if os.path.exists(dest):
        # The last two path components provide a content address for a .build-id entry.
        tokens = os.path.split(dest)
        name = os.path.join(tokens[-2], tokens[-1])
        print('%s already exists in destination; skipping linking' % name)
        continue
      os.link(src, dest)
  return internal_symbol_dirs
 
 

IsLinux()

def merge_and_upload_debug_symbols.IsLinux

(

)

Definition at line 27 of file merge_and_upload_debug_symbols.py.

def IsLinux():
  return platform.system() == 'Linux'
 
 
# out_dir here is of the format "/b/s/w/ir/k/recipe_cleanup/tmpIbWDdp"
# we need to place the cipd definition in this directory.

main()

def merge_and_upload_debug_symbols.main

(

)

Definition at line 161 of file merge_and_upload_debug_symbols.py.

def main():
  parser = argparse.ArgumentParser()
 
  parser.add_argument(
      '--symbol-dirs',
      required=True,
      nargs='+',
      help='Space separated list of directories that contain the debug symbols.'
  )
  parser.add_argument(
      '--out-dir',
      action='store',
      dest='out_dir',
      default=tempfile.mkdtemp(),
      help=(
          'Output directory where the executables will be placed defaults to an '
          'empty temp directory'
      )
  )
  parser.add_argument('--target-arch', type=str, choices=['x64', 'arm64'], required=True)
  parser.add_argument('--engine-version', required=True, help='Specifies the flutter engine SHA.')
 
  parser.add_argument('--upload', default=False, action='store_true')
 
  args = parser.parse_args()
 
  symbol_dirs = CalculateAbsoluteDirs(args.symbol_dirs)
  for symbol_dir in symbol_dirs:
    assert os.path.exists(symbol_dir) and os.path.isdir(symbol_dir)
 
  out_dir = args.out_dir
 
  if os.path.exists(out_dir):
    print('Directory: %s is not empty, deleting it.' % out_dir)
    shutil.rmtree(out_dir)
  os.makedirs(out_dir)
 
  internal_symbol_dirs = []
  for symbol_dir in symbol_dirs:
    internal_symbol_dirs += HardlinkContents(symbol_dir, out_dir)
 
  # make these unique
  internal_symbol_dirs = list(set(internal_symbol_dirs))
 
  arch = args.target_arch
  cipd_def = WriteCIPDDefinition(arch, out_dir, internal_symbol_dirs)
 
  # Set revision to HEAD if empty and remove upload. This is to support
  # presubmit workflows. An empty engine_version means this script is running
  # on presubmit.
  should_upload = args.upload
  engine_version = args.engine_version
  if not engine_version:
    engine_version = 'HEAD'
    should_upload = False
 
  ProcessCIPDPackage(should_upload, cipd_def, engine_version, out_dir, arch)
  return 0
 
 

ProcessCIPDPackage()

def merge_and_upload_debug_symbols.ProcessCIPDPackage	(		upload,
			cipd_yaml,
			engine_version,
			out_dir,
			target_arch
	)

Definition at line 80 of file merge_and_upload_debug_symbols.py.

def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
  _packaging_dir = GetPackagingDir(out_dir)
  tag = 'git_revision:%s' % engine_version
  package_name = 'flutter/fuchsia-debug-symbols-%s' % target_arch
  already_exists = CheckCIPDPackageExists(package_name, tag)
  if already_exists:
    print('CIPD package %s tag %s already exists!' % (package_name, tag))
 
  if upload and IsLinux() and not already_exists:
    command = [
        'cipd',
        'create',
        '-pkg-def',
        cipd_yaml,
        '-ref',
        'latest',
        '-tag',
        tag,
        '-verification-timeout',
        '10m0s',
    ]
  else:
    command = [
        'cipd', 'pkg-build', '-pkg-def', cipd_yaml, '-out',
        os.path.join(_packaging_dir, 'fuchsia-debug-symbols-%s.cipd' % target_arch)
    ]
 
  # Retry up to three times.  We've seen CIPD fail on verification in some
  # instances. Normally verification takes slightly more than 1 minute when
  # it succeeds.
  num_tries = 3
  for tries in range(num_tries):
    try:
      subprocess.check_call(command, cwd=_packaging_dir)
      break
    except subprocess.CalledProcessError as error:
      print('Failed %s times.\nError was: %s' % (tries + 1, error))
      if tries == num_tries - 1:
        raise
 
 
# Recursively hardlinks contents from one directory to another,
# skipping over collisions.

WriteCIPDDefinition()

def merge_and_upload_debug_symbols.WriteCIPDDefinition	(		target_arch,
			out_dir,
			symbol_dirs
	)

Definition at line 53 of file merge_and_upload_debug_symbols.py.

def WriteCIPDDefinition(target_arch, out_dir, symbol_dirs):
  _packaging_dir = GetPackagingDir(out_dir)
  yaml_file = os.path.join(_packaging_dir, 'debug_symbols.cipd.yaml')
  with open(yaml_file, 'w') as f:
    cipd_def = CreateCIPDDefinition(target_arch, out_dir, symbol_dirs)
    f.write(cipd_def)
  return yaml_file
 
 

Variable Documentation

BUILD_ROOT_DIR

merge_and_upload_debug_symbols.BUILD_ROOT_DIR = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..', '..'))

Definition at line 24 of file merge_and_upload_debug_symbols.py.