webpages_playback.SkPicturePlayback Class Reference

Inheritance diagram for webpages_playback.SkPicturePlayback:

Public Member Functions
def	__init__ (self, parse_options)
def	Run (self)

Public Attributes
	gs

Detailed Description

Class that archives or replays webpages and creates SKPs.

Definition at line 138 of file webpages_playback.py.

Constructor & Destructor Documentation

__init__()

def webpages_playback.SkPicturePlayback.__init__	(		self,
			parse_options
	)

Constructs a SkPicturePlayback BuildStep instance.

Definition at line 141 of file webpages_playback.py.

  def __init__(self, parse_options):
    """Constructs a SkPicturePlayback BuildStep instance."""
    assert parse_options.browser_executable, 'Must specify --browser_executable'
    self._browser_executable = parse_options.browser_executable
    self._browser_args = '--disable-setuid-sandbox'
    if parse_options.browser_extra_args:
      self._browser_args = '%s %s' % (
          self._browser_args, parse_options.browser_extra_args)
 
    self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path,
                                               CHROMIUM_PAGE_SETS_PATH)
    self._all_page_sets_specified = parse_options.page_sets == 'all'
    self._page_sets = self._ParsePageSets(parse_options.page_sets)
 
    self._record = parse_options.record
    self._skia_tools = parse_options.skia_tools
    self._non_interactive = parse_options.non_interactive
    self._upload = parse_options.upload
    self._skp_prefix = parse_options.skp_prefix
    data_store_location = parse_options.data_store
    if data_store_location.startswith(GS_PREFIX):
      self.gs = GoogleStorageDataStore(data_store_location)
    else:
      self.gs = LocalFileSystemDataStore(data_store_location)
    self._upload_to_partner_bucket = parse_options.upload_to_partner_bucket
    self._alternate_upload_dir = parse_options.alternate_upload_dir
    self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path,
                                                'tools', 'perf')
    self._catapult_dir = os.path.join(parse_options.chrome_src_path,
                                      'third_party', 'catapult')
 
    self._local_skp_dir = os.path.join(
        parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME)
    self._local_record_webpages_archive_dir = os.path.join(
        parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive')
 
    # List of SKP files generated by this script.
    self._skp_files = []
 

Member Function Documentation

Run()

def webpages_playback.SkPicturePlayback.Run

(

self

)

Run the SkPicturePlayback BuildStep.

Definition at line 207 of file webpages_playback.py.

  def Run(self):
    """Run the SkPicturePlayback BuildStep."""
 
    # Download the credentials file if it was not previously downloaded.
    if not os.path.isfile(CREDENTIALS_FILE_PATH):
      # Download the credentials.json file from Google Storage.
      self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH)
 
    if not os.path.isfile(CREDENTIALS_FILE_PATH):
      raise Exception("""Could not locate credentials file in the storage.
      Please create a credentials file in gs://%s that contains:
      {
        "google": {
          "username": "google_testing_account_username",
          "password": "google_testing_account_password"
        }
      }\n\n""" % CREDENTIALS_GS_PATH)
 
    # Delete any left over data files in the data directory.
    for archive_file in glob.glob(
        os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
      os.remove(archive_file)
 
    # Create the required local storage directories.
    self._CreateLocalStorageDirs()
 
    # Start the timer.
    start_time = time.time()
 
    # Loop through all page_sets.
    for page_set in self._page_sets:
      if os.path.basename(page_set) == '__init__.py':
        continue
      page_set_basename = os.path.basename(page_set).split('.')[0]
      page_set_json_name = page_set_basename + '.json'
      wpr_data_file_glob = (
          page_set.split(os.path.sep)[-1].split('.')[0] + '_*.wprgo')
      page_set_dir = os.path.dirname(page_set)
 
      if self._IsChromiumPageSet(page_set):
        print('Using Chromium\'s captured archives for Chromium\'s page sets.')
      elif self._record:
        # Create an archive of the specified webpages if '--record=True' is
        # specified.
        record_wpr_cmd = (
          'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
          'DISPLAY=%s' % X11_DISPLAY,
          os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
          '--extra-browser-args="%s"' % self._browser_args,
          '--browser=exact',
          '--browser-executable=%s' % self._browser_executable,
          '%s_page_set' % page_set_basename,
          '--page-set-base-dir=%s' % page_set_dir
        )
        for _ in range(RETRY_RECORD_WPR_COUNT):
          try:
            subprocess.check_call(' '.join(record_wpr_cmd), shell=True)
 
            # Copy over the created archive into the local webpages archive
            # directory.
            for wpr_data_file in glob.glob(os.path.join(
                LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file_glob)):
              shutil.copy(
                os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
                self._local_record_webpages_archive_dir)
            shutil.copy(
              os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
                           page_set_json_name),
              self._local_record_webpages_archive_dir)
 
            # Break out of the retry loop since there were no errors.
            break
          except Exception:
            # There was a failure continue with the loop.
            traceback.print_exc()
        else:
          # If we get here then record_wpr did not succeed and thus did not
          # break out of the loop.
          raise Exception('record_wpr failed for page_set: %s' % page_set)
 
      else:
        # Get the webpages archive so that it can be replayed.
        self._DownloadWebpagesArchive(wpr_data_file_glob, page_set_json_name)
 
      run_benchmark_cmd = [
          'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
          'DISPLAY=%s' % X11_DISPLAY,
          'timeout', '1800',
          os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
          '--extra-browser-args="%s"' % self._browser_args,
          '--browser=exact',
          '--browser-executable=%s' % self._browser_executable,
          SKP_BENCHMARK,
          '--page-set-name=%s' % page_set_basename,
          '--page-set-base-dir=%s' % page_set_dir,
          '--skp-outdir=%s' % TMP_SKP_DIR,
          '--also-run-disabled-tests',
      ]
 
      exclusions = PAGE_SETS_TO_EXCLUSIONS.get(os.path.basename(page_set))
      if exclusions:
        run_benchmark_cmd.append('--story-filter-exclude=' + exclusions)
 
      for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
        try:
          print('\n\n=======Capturing SKP of %s=======\n\n' % page_set)
          subprocess.check_call(' '.join(run_benchmark_cmd), shell=True)
        except subprocess.CalledProcessError:
          # There was a failure continue with the loop.
          traceback.print_exc()
          print('\n\n=======Retrying %s=======\n\n' % page_set)
          time.sleep(10)
          continue
 
        try:
          # Rename generated SKP files into more descriptive names.
          self._RenameSkpFiles(page_set)
        except InvalidSKPException:
          # There was a failure continue with the loop.
          traceback.print_exc()
          print('\n\n=======Retrying %s=======\n\n' % page_set)
          time.sleep(10)
          continue
 
        # Break out of the retry loop since there were no errors.
        break
      else:
        # If we get here then run_benchmark did not succeed and thus did not
        # break out of the loop.
        raise Exception('run_benchmark failed for page_set: %s' % page_set)
 
    print('\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
          time.time() - start_time))
 
    if self._skia_tools:
      render_pictures_cmd = [
          os.path.join(self._skia_tools, 'render_pictures'),
          '-r', self._local_skp_dir
      ]
      render_pdfs_cmd = [
          os.path.join(self._skia_tools, 'render_pdfs'),
          '-r', self._local_skp_dir
      ]
 
      for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
        print('\n\n=======Running %s=======' % ' '.join(tools_cmd))
        subprocess.check_call(tools_cmd)
 
      if not self._non_interactive:
        print('\n\n=======Running debugger=======')
        os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
                             self._local_skp_dir))
 
    print('\n\n')
 
    if self._upload:
      print('\n\n=======Uploading to %s=======\n\n' % self.gs.target_type())
      # Copy the directory structure in the root directory into Google Storage.
      dest_dir_name = ROOT_PLAYBACK_DIR_NAME
      if self._alternate_upload_dir:
        dest_dir_name = self._alternate_upload_dir
 
      self.gs.upload_dir_contents(
          self._local_skp_dir, dest_dir=dest_dir_name)
 
      print('\n\n=======New SKPs have been uploaded to %s =======\n\n' %
            posixpath.join(self.gs.target_name(), dest_dir_name,
                           SKPICTURES_DIR_NAME))
 
    else:
      print('\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type())
      print('Generated resources are available in %s\n\n' % self._local_skp_dir)
 
    if self._upload_to_partner_bucket:
      print('\n\n=======Uploading to Partner bucket %s =======\n\n' %
            PARTNERS_GS_BUCKET)
      partner_gs = GoogleStorageDataStore(PARTNERS_GS_BUCKET)
      timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d')
      upload_dir = posixpath.join(SKPICTURES_DIR_NAME, timestamp)
      try:
        partner_gs.delete_path(upload_dir)
      except subprocess.CalledProcessError:
        print('Cannot delete %s because it does not exist yet.' % upload_dir)
      print('Uploading %s to %s' % (self._local_skp_dir, upload_dir))
      partner_gs.upload_dir_contents(self._local_skp_dir, upload_dir)
      print('\n\n=======New SKPs have been uploaded to %s =======\n\n' %
            posixpath.join(partner_gs.target_name(), upload_dir))
 
    return 0
 

Member Data Documentation

gs

webpages_playback.SkPicturePlayback.gs

Definition at line 162 of file webpages_playback.py.

---

The documentation for this class was generated from the following file:

• third_party/skia/tools/skp/webpages_playback.py