create.py

Go to the documentation of this file.

#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
 
 
"""Create the SKP asset."""
 
 
from __future__ import print_function
import argparse
from distutils.dir_util import copy_tree
import os
import shutil
import subprocess
import sys
import tempfile
 
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
sys.path.insert(0, INFRA_BOTS_DIR)
import utils
 
 
BROWSER_EXECUTABLE_ENV_VAR = 'SKP_BROWSER_EXECUTABLE'
CHROME_SRC_PATH_ENV_VAR = 'SKP_CHROME_SRC_PATH'
UPLOAD_TO_PARTNER_BUCKET_ENV_VAR = 'SKP_UPLOAD_TO_PARTNER_BUCKET'
DM_PATH_ENV_VAR = 'DM_PATH'
 
SKIA_TOOLS = os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools')
PRIVATE_SKPS_GS = 'gs://skia-skps/private/skps'
 
 
def getenv(key):
  val = os.environ.get(key)
  if not val:
    print(('Environment variable %s not set; you should run this via '
           'create_and_upload.py.' % key), file=sys.stderr)
    sys.exit(1)
  return val
 
 
def get_flutter_skps(target_dir):
  """Creates SKPs using Flutter's skp_generator tool.
 
  Documentation is at https://github.com/flutter/tests/tree/master/skp_generator
  """
  with utils.tmp_dir():
    print('Retrieving Flutter SKPs...')
    utils.git_clone('https://github.com/flutter/tests.git', '.')
    os.chdir('skp_generator')
    subprocess.check_call(['bash', 'build.sh'])
    # Fix invalid SKP file names.
    for f in os.listdir('skps'):
      original_file_name = os.path.splitext(f)[0]
      new_file_name = ''.join([x if x.isalnum() else "_"
                               for x in original_file_name])
      if new_file_name != original_file_name:
        os.rename(os.path.join('skps', f),
                  os.path.join('skps', new_file_name + '.skp'))
    copy_tree('skps', target_dir)
    print('Done retrieving Flutter SKPs.')
 
 
def create_asset(chrome_src_path, browser_executable, target_dir,
                 upload_to_partner_bucket, dm_path):
  """Create the SKP asset.
 
  Creates the asset from 3 sources:
  1. From Flutter's skp_generator tool.
  2. The web pages defined in the tools/skp/page_sets/ directory.
  3. Any private SKPs stored in $PRIVATE_SKPS_GS after running dm on
     them (see below).
 
  The script runs the following cmd on the non-generated SKPs stored in
  $PRIVATE_SKPS_GS -
  `dm --config skp -w newskps/ --skps oldskps/ --src skp`
  The cmd updates the version stored in the SKPs so that the versions in
  them do not eventually become unsupported.
  """
  browser_executable = os.path.realpath(browser_executable)
  chrome_src_path = os.path.realpath(chrome_src_path)
  dm_path = os.path.realpath(dm_path)
  target_dir = os.path.realpath(target_dir)
 
  if not os.path.exists(target_dir):
    os.makedirs(target_dir)
 
  # 1. Flutter SKPs
  get_flutter_skps(target_dir)
 
  # 2. Skia's SKPs from tools/skp/page_sets/
  with utils.tmp_dir():
    if os.environ.get('CHROME_HEADLESS'):
      print('Starting xvfb')
      # Start Xvfb if running on a bot.
      try:
        xvfb_proc = subprocess.Popen([
            'sudo', 'Xvfb', ':0', '-screen', '0', '1280x1024x24'])
      except Exception:
        # It is ok if the above command fails, it just means that DISPLAY=:0
        # is already up.
        xvfb_proc = None
 
    print('Running webpages_playback to generate SKPs...')
    webpages_playback_cmd = [
      'python', '-u', os.path.join(SKIA_TOOLS, 'skp', 'webpages_playback.py'),
      '--page_sets', 'all',
      '--browser_executable', browser_executable,
      '--non-interactive',
      '--output_dir', os.getcwd(),
      '--chrome_src_path', chrome_src_path,
    ]
    if upload_to_partner_bucket:
      webpages_playback_cmd.append('--upload_to_partner_bucket')
    print('Running webpages_playback command:\n$ %s' %
        ' '.join(webpages_playback_cmd))
    try:
      subprocess.check_call(webpages_playback_cmd)
    finally:
      if xvfb_proc:
        try:
          xvfb_proc.kill()
        except OSError as e:
          print('Failed to kill xvfb process via Popen.kill();'
                ' attempting `sudo kill`...')
          try:
            subprocess.check_call(['sudo', 'kill', '-9', str(xvfb_proc.pid)])
          except subprocess.CalledProcessError as e:
            print('Failed to kill xvfb process via `sudo kill`;'
                  'this may cause a hang.')
      # Clean up any leftover browser instances. This can happen if there are
      # telemetry crashes, processes are not always cleaned up appropriately by
      # the webpagereplay and telemetry frameworks.
      procs = subprocess.check_output(['ps', 'ax']).decode()
      for line in procs.splitlines():
        if browser_executable in line:
          print(line)
          pid = line.strip().split(' ')[0]
          if pid != str(os.getpid()) and not 'python' in line:
            print('Kill browser process %s' % str(pid))
            try:
              subprocess.check_call(['kill', '-9', str(pid)])
            except subprocess.CalledProcessError as e:
              print(e)
          else:
            pass
        if 'Xvfb' in line:
          print(line)
          pid = line.strip().split(' ')[0]
          print('Kill Xvfb process %s' % str(pid))
          try:
            subprocess.check_call(['sudo', 'kill', '-9', str(pid)])
          except subprocess.CalledProcessError as e:
            print(e)
 
    src = os.path.join(os.getcwd(), 'playback', 'skps')
    for f in os.listdir(src):
      if f.endswith('.skp'):
        shutil.copyfile(os.path.join(src, f), os.path.join(target_dir, f))
    print('Done running webpages_playback.')
 
  # 3. Copy over private SKPs from Google storage into the target_dir.
  old_skps_dir = tempfile.mkdtemp()
  new_skps_dir = tempfile.mkdtemp()
  print('Copying non-generated SKPs from private GCS bucket...')
  subprocess.check_call([
    'gsutil', 'cp', os.path.join(PRIVATE_SKPS_GS, '*'), old_skps_dir])
  print('Updating non-generated SKP versions')
  subprocess.check_call([
      dm_path,
      '--config', 'skp',
      '-w', new_skps_dir,
      '--skps', old_skps_dir,
      '--src', 'skp'])
 
  # DM creates artifacts in 2 directory level- one for the config and one for
  # the output type. In this case it ends up creating ${temp_dir}/skp/skp/
  for f in os.listdir(os.path.join(new_skps_dir, 'skp', 'skp')):
    if f.endswith('.skp'):
      # Files generated by DM add a suffix based on the output type.
      # For "xyz.skp" DM will create "xyz.skp.skp". We strip out
      # the final ".skp" here.
      shutil.copyfile(
          os.path.join(new_skps_dir, 'skp', 'skp', f),
          os.path.join(target_dir, f.replace('.skp.skp', '.skp')))
  shutil.rmtree(old_skps_dir)
  shutil.rmtree(new_skps_dir)
 
 
def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('--target_dir', '-t', required=True)
  args = parser.parse_args()
 
  # Obtain flags from create_and_upload via environment variables, since
  # this script is called via `sk` and not directly.
  chrome_src_path = getenv(CHROME_SRC_PATH_ENV_VAR)
  browser_executable = getenv(BROWSER_EXECUTABLE_ENV_VAR)
  upload_to_partner_bucket = getenv(UPLOAD_TO_PARTNER_BUCKET_ENV_VAR) == '1'
  dm_path = getenv(DM_PATH_ENV_VAR)
 
  create_asset(chrome_src_path, browser_executable, args.target_dir,
               upload_to_partner_bucket, dm_path)
 
 
if __name__ == '__main__':
  main()