merge_and_upload_debug_symbols.py

Go to the documentation of this file.

#!/usr/bin/env python3
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
 
""" Merges the debug symbols and uploads them to cipd.
"""
 
import argparse
import collections
import json
import os
import platform
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
 
# Path to the engine root checkout. This is used to calculate absolute
# paths if relative ones are passed to the script.
BUILD_ROOT_DIR = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..', '..'))
 
 
def IsLinux():
  return platform.system() == 'Linux'
 
 
# out_dir here is of the format "/b/s/w/ir/k/recipe_cleanup/tmpIbWDdp"
# we need to place the cipd definition in this directory.
def GetPackagingDir(out_dir):
  return os.path.abspath(out_dir)
 
 
def CreateCIPDDefinition(target_arch, out_dir, symbol_dirs):
  dir_name = os.path.basename(os.path.normpath(out_dir))
  pkg_def = """
package: flutter/fuchsia-debug-symbols-%s
description: Flutter and Dart runner debug symbols for Fuchsia. Target architecture %s.
install_mode: copy
data:
""" % (target_arch, target_arch)
  for symbol_dir in symbol_dirs:
    symbol_dir_name = os.path.basename(os.path.normpath(symbol_dir))
    data = '\n  - dir: %s' % (symbol_dir_name)
    pkg_def = pkg_def + data
  return pkg_def
 
 
# CIPD CLI needs the definition and data directory to be relative to each other.
def WriteCIPDDefinition(target_arch, out_dir, symbol_dirs):
  _packaging_dir = GetPackagingDir(out_dir)
  yaml_file = os.path.join(_packaging_dir, 'debug_symbols.cipd.yaml')
  with open(yaml_file, 'w') as f:
    cipd_def = CreateCIPDDefinition(target_arch, out_dir, symbol_dirs)
    f.write(cipd_def)
  return yaml_file
 
 
def CheckCIPDPackageExists(package_name, tag):
  '''Check to see if the current package/tag combo has been published'''
  command = [
      'cipd',
      'search',
      package_name,
      '-tag',
      tag,
  ]
  stdout = subprocess.check_output(command)
  stdout = stdout if isinstance(stdout, str) else stdout.decode('UTF-8')
  match = re.search(r'No matching instances\.', stdout)
  if match:
    return False
  else:
    return True
 
 
def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
  _packaging_dir = GetPackagingDir(out_dir)
  tag = 'git_revision:%s' % engine_version
  package_name = 'flutter/fuchsia-debug-symbols-%s' % target_arch
  already_exists = CheckCIPDPackageExists(package_name, tag)
  if already_exists:
    print('CIPD package %s tag %s already exists!' % (package_name, tag))
 
  if upload and IsLinux() and not already_exists:
    command = [
        'cipd',
        'create',
        '-pkg-def',
        cipd_yaml,
        '-ref',
        'latest',
        '-tag',
        tag,
        '-verification-timeout',
        '10m0s',
    ]
  else:
    command = [
        'cipd', 'pkg-build', '-pkg-def', cipd_yaml, '-out',
        os.path.join(_packaging_dir, 'fuchsia-debug-symbols-%s.cipd' % target_arch)
    ]
 
  # Retry up to three times.  We've seen CIPD fail on verification in some
  # instances. Normally verification takes slightly more than 1 minute when
  # it succeeds.
  num_tries = 3
  for tries in range(num_tries):
    try:
      subprocess.check_call(command, cwd=_packaging_dir)
      break
    except subprocess.CalledProcessError as error:
      print('Failed %s times.\nError was: %s' % (tries + 1, error))
      if tries == num_tries - 1:
        raise
 
 
# Recursively hardlinks contents from one directory to another,
# skipping over collisions.
def HardlinkContents(dirA, dirB):
  internal_symbol_dirs = []
  for src_dir, _, filenames in os.walk(dirA):
    for filename in filenames:
      # if a file contains 'dbg_success' in its name, it is a stamp file.
      # An example of this would be
      # '._dart_jit_runner_dbg_symbols_unstripped_dbg_success' these
      # are generated by GN and have to be ignored.
      if 'dbg_success' in filename:
        continue
      src = os.path.join(src_dir, filename)
      dest_dir = os.path.join(dirB, os.path.relpath(src_dir, dirA))
      try:
        os.makedirs(dest_dir)
        internal_symbol_dirs.append(dest_dir)
      except:
        pass
      dest = os.path.join(dest_dir, filename)
      if os.path.exists(dest):
        # The last two path components provide a content address for a .build-id entry.
        tokens = os.path.split(dest)
        name = os.path.join(tokens[-2], tokens[-1])
        print('%s already exists in destination; skipping linking' % name)
        continue
      os.link(src, dest)
  return internal_symbol_dirs
 
 
def CalculateAbsoluteDirs(dirs):
  results = []
  for directory in dirs:
    if os.path.isabs(directory):
      results.append(directory)
    else:
      results.append(os.path.join(BUILD_ROOT_DIR, directory))
  return results
 
 
def main():
  parser = argparse.ArgumentParser()
 
  parser.add_argument(
      '--symbol-dirs',
      required=True,
      nargs='+',
      help='Space separated list of directories that contain the debug symbols.'
  )
  parser.add_argument(
      '--out-dir',
      action='store',
      dest='out_dir',
      default=tempfile.mkdtemp(),
      help=(
          'Output directory where the executables will be placed defaults to an '
          'empty temp directory'
      )
  )
  parser.add_argument('--target-arch', type=str, choices=['x64', 'arm64'], required=True)
  parser.add_argument('--engine-version', required=True, help='Specifies the flutter engine SHA.')
 
  parser.add_argument('--upload', default=False, action='store_true')
 
  args = parser.parse_args()
 
  symbol_dirs = CalculateAbsoluteDirs(args.symbol_dirs)
  for symbol_dir in symbol_dirs:
    assert os.path.exists(symbol_dir) and os.path.isdir(symbol_dir)
 
  out_dir = args.out_dir
 
  if os.path.exists(out_dir):
    print('Directory: %s is not empty, deleting it.' % out_dir)
    shutil.rmtree(out_dir)
  os.makedirs(out_dir)
 
  internal_symbol_dirs = []
  for symbol_dir in symbol_dirs:
    internal_symbol_dirs += HardlinkContents(symbol_dir, out_dir)
 
  # make these unique
  internal_symbol_dirs = list(set(internal_symbol_dirs))
 
  arch = args.target_arch
  cipd_def = WriteCIPDDefinition(arch, out_dir, internal_symbol_dirs)
 
  # Set revision to HEAD if empty and remove upload. This is to support
  # presubmit workflows. An empty engine_version means this script is running
  # on presubmit.
  should_upload = args.upload
  engine_version = args.engine_version
  if not engine_version:
    engine_version = 'HEAD'
    should_upload = False
 
  ProcessCIPDPackage(should_upload, cipd_def, engine_version, out_dir, arch)
  return 0
 
 
if __name__ == '__main__':
  sys.exit(main())