skp.create Namespace Reference

Functions
def	getenv (key)
def	get_flutter_skps (target_dir)
def	create_asset (chrome_src_path, browser_executable, target_dir, upload_to_partner_bucket, dm_path)
def	main ()

Variables
	FILE_DIR = os.path.dirname(os.path.abspath(__file__))
	INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
string	BROWSER_EXECUTABLE_ENV_VAR = 'SKP_BROWSER_EXECUTABLE'
string	CHROME_SRC_PATH_ENV_VAR = 'SKP_CHROME_SRC_PATH'
string	UPLOAD_TO_PARTNER_BUCKET_ENV_VAR = 'SKP_UPLOAD_TO_PARTNER_BUCKET'
string	DM_PATH_ENV_VAR = 'DM_PATH'
	SKIA_TOOLS = os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools')
string	PRIVATE_SKPS_GS = 'gs://skia-skps/private/skps'

Function Documentation

create_asset()

def skp.create.create_asset	(		chrome_src_path,
			browser_executable,
			target_dir,
			upload_to_partner_bucket,
			dm_path
	)

Create the SKP asset.

Creates the asset from 3 sources:
1. From Flutter's skp_generator tool.
2. The web pages defined in the tools/skp/page_sets/ directory.
3. Any private SKPs stored in $PRIVATE_SKPS_GS after running dm on
   them (see below).

The script runs the following cmd on the non-generated SKPs stored in
$PRIVATE_SKPS_GS -
`dm --config skp -w newskps/ --skps oldskps/ --src skp`
The cmd updates the version stored in the SKPs so that the versions in
them do not eventually become unsupported.

Definition at line 67 of file create.py.

                 upload_to_partner_bucket, dm_path):
  """Create the SKP asset.
 
  Creates the asset from 3 sources:
  1. From Flutter's skp_generator tool.
  2. The web pages defined in the tools/skp/page_sets/ directory.
  3. Any private SKPs stored in $PRIVATE_SKPS_GS after running dm on
     them (see below).
 
  The script runs the following cmd on the non-generated SKPs stored in
  $PRIVATE_SKPS_GS -
  `dm --config skp -w newskps/ --skps oldskps/ --src skp`
  The cmd updates the version stored in the SKPs so that the versions in
  them do not eventually become unsupported.
  """
  browser_executable = os.path.realpath(browser_executable)
  chrome_src_path = os.path.realpath(chrome_src_path)
  dm_path = os.path.realpath(dm_path)
  target_dir = os.path.realpath(target_dir)
 
  if not os.path.exists(target_dir):
    os.makedirs(target_dir)
 
  # 1. Flutter SKPs
  get_flutter_skps(target_dir)
 
  # 2. Skia's SKPs from tools/skp/page_sets/
  with utils.tmp_dir():
    if os.environ.get('CHROME_HEADLESS'):
      print('Starting xvfb')
      # Start Xvfb if running on a bot.
      try:
        xvfb_proc = subprocess.Popen([
            'sudo', 'Xvfb', ':0', '-screen', '0', '1280x1024x24'])
      except Exception:
        # It is ok if the above command fails, it just means that DISPLAY=:0
        # is already up.
        xvfb_proc = None
 
    print('Running webpages_playback to generate SKPs...')
    webpages_playback_cmd = [
      'python', '-u', os.path.join(SKIA_TOOLS, 'skp', 'webpages_playback.py'),
      '--page_sets', 'all',
      '--browser_executable', browser_executable,
      '--non-interactive',
      '--output_dir', os.getcwd(),
      '--chrome_src_path', chrome_src_path,
    ]
    if upload_to_partner_bucket:
      webpages_playback_cmd.append('--upload_to_partner_bucket')
    print('Running webpages_playback command:\n$ %s' %
        ' '.join(webpages_playback_cmd))
    try:
      subprocess.check_call(webpages_playback_cmd)
    finally:
      if xvfb_proc:
        try:
          xvfb_proc.kill()
        except OSError as e:
          print('Failed to kill xvfb process via Popen.kill();'
                ' attempting `sudo kill`...')
          try:
            subprocess.check_call(['sudo', 'kill', '-9', str(xvfb_proc.pid)])
          except subprocess.CalledProcessError as e:
            print('Failed to kill xvfb process via `sudo kill`;'
                  'this may cause a hang.')
      # Clean up any leftover browser instances. This can happen if there are
      # telemetry crashes, processes are not always cleaned up appropriately by
      # the webpagereplay and telemetry frameworks.
      procs = subprocess.check_output(['ps', 'ax']).decode()
      for line in procs.splitlines():
        if browser_executable in line:
          print(line)
          pid = line.strip().split(' ')[0]
          if pid != str(os.getpid()) and not 'python' in line:
            print('Kill browser process %s' % str(pid))
            try:
              subprocess.check_call(['kill', '-9', str(pid)])
            except subprocess.CalledProcessError as e:
              print(e)
          else:
            pass
        if 'Xvfb' in line:
          print(line)
          pid = line.strip().split(' ')[0]
          print('Kill Xvfb process %s' % str(pid))
          try:
            subprocess.check_call(['sudo', 'kill', '-9', str(pid)])
          except subprocess.CalledProcessError as e:
            print(e)
 
    src = os.path.join(os.getcwd(), 'playback', 'skps')
    for f in os.listdir(src):
      if f.endswith('.skp'):
        shutil.copyfile(os.path.join(src, f), os.path.join(target_dir, f))
    print('Done running webpages_playback.')
 
  # 3. Copy over private SKPs from Google storage into the target_dir.
  old_skps_dir = tempfile.mkdtemp()
  new_skps_dir = tempfile.mkdtemp()
  print('Copying non-generated SKPs from private GCS bucket...')
  subprocess.check_call([
    'gsutil', 'cp', os.path.join(PRIVATE_SKPS_GS, '*'), old_skps_dir])
  print('Updating non-generated SKP versions')
  subprocess.check_call([
      dm_path,
      '--config', 'skp',
      '-w', new_skps_dir,
      '--skps', old_skps_dir,
      '--src', 'skp'])
 
  # DM creates artifacts in 2 directory level- one for the config and one for
  # the output type. In this case it ends up creating ${temp_dir}/skp/skp/
  for f in os.listdir(os.path.join(new_skps_dir, 'skp', 'skp')):
    if f.endswith('.skp'):
      # Files generated by DM add a suffix based on the output type.
      # For "xyz.skp" DM will create "xyz.skp.skp". We strip out
      # the final ".skp" here.
      shutil.copyfile(
          os.path.join(new_skps_dir, 'skp', 'skp', f),
          os.path.join(target_dir, f.replace('.skp.skp', '.skp')))
  shutil.rmtree(old_skps_dir)
  shutil.rmtree(new_skps_dir)
 
 

get_flutter_skps()

def skp.create.get_flutter_skps

(

target_dir

)

Creates SKPs using Flutter's skp_generator tool.

Documentation is at https://github.com/flutter/tests/tree/master/skp_generator

Definition at line 45 of file create.py.

def get_flutter_skps(target_dir):
  """Creates SKPs using Flutter's skp_generator tool.
 
  Documentation is at https://github.com/flutter/tests/tree/master/skp_generator
  """
  with utils.tmp_dir():
    print('Retrieving Flutter SKPs...')
    utils.git_clone('https://github.com/flutter/tests.git', '.')
    os.chdir('skp_generator')
    subprocess.check_call(['bash', 'build.sh'])
    # Fix invalid SKP file names.
    for f in os.listdir('skps'):
      original_file_name = os.path.splitext(f)[0]
      new_file_name = ''.join([x if x.isalnum() else "_"
                               for x in original_file_name])
      if new_file_name != original_file_name:
        os.rename(os.path.join('skps', f),
                  os.path.join('skps', new_file_name + '.skp'))
    copy_tree('skps', target_dir)
    print('Done retrieving Flutter SKPs.')
 
 

getenv()

def skp.create.getenv

(

key

)

Definition at line 36 of file create.py.

def getenv(key):
  val = os.environ.get(key)
  if not val:
    print(('Environment variable %s not set; you should run this via '
           'create_and_upload.py.' % key), file=sys.stderr)
    sys.exit(1)
  return val
 
 

main()

def skp.create.main

(

)

Definition at line 193 of file create.py.

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('--target_dir', '-t', required=True)
  args = parser.parse_args()
 
  # Obtain flags from create_and_upload via environment variables, since
  # this script is called via `sk` and not directly.
  chrome_src_path = getenv(CHROME_SRC_PATH_ENV_VAR)
  browser_executable = getenv(BROWSER_EXECUTABLE_ENV_VAR)
  upload_to_partner_bucket = getenv(UPLOAD_TO_PARTNER_BUCKET_ENV_VAR) == '1'
  dm_path = getenv(DM_PATH_ENV_VAR)
 
  create_asset(chrome_src_path, browser_executable, args.target_dir,
               upload_to_partner_bucket, dm_path)
 
 

Variable Documentation

BROWSER_EXECUTABLE_ENV_VAR

string skp.create.BROWSER_EXECUTABLE_ENV_VAR = 'SKP_BROWSER_EXECUTABLE'

Definition at line 27 of file create.py.

CHROME_SRC_PATH_ENV_VAR

string skp.create.CHROME_SRC_PATH_ENV_VAR = 'SKP_CHROME_SRC_PATH'

Definition at line 28 of file create.py.

DM_PATH_ENV_VAR

string skp.create.DM_PATH_ENV_VAR = 'DM_PATH'

Definition at line 30 of file create.py.

FILE_DIR

skp.create.FILE_DIR = os.path.dirname(os.path.abspath(__file__))

Definition at line 21 of file create.py.

INFRA_BOTS_DIR

skp.create.INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))

Definition at line 22 of file create.py.

PRIVATE_SKPS_GS

string skp.create.PRIVATE_SKPS_GS = 'gs://skia-skps/private/skps'

Definition at line 33 of file create.py.

SKIA_TOOLS

skp.create.SKIA_TOOLS = os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools')

Definition at line 32 of file create.py.

UPLOAD_TO_PARTNER_BUCKET_ENV_VAR

string skp.create.UPLOAD_TO_PARTNER_BUCKET_ENV_VAR = 'SKP_UPLOAD_TO_PARTNER_BUCKET'

Definition at line 29 of file create.py.