Flutter Engine
The Flutter Engine
Loading...
Searching...
No Matches
Public Member Functions | Public Attributes | Protected Member Functions | Protected Attributes | List of all members
webpages_playback.SkPicturePlayback Class Reference
Inheritance diagram for webpages_playback.SkPicturePlayback:

Public Member Functions

 __init__ (self, parse_options)
 
 Run (self)
 

Public Attributes

 gs
 

Protected Member Functions

 _ParsePageSets (self, page_sets)
 
 _IsChromiumPageSet (self, page_set)
 
 _GetSkiaSkpFileName (self, page_set)
 
 _GetChromiumSkpFileName (self, page_set, site)
 
 _RenameSkpFiles (self, page_set)
 
 _CreateLocalStorageDirs (self)
 
 _DownloadWebpagesArchive (self, wpr_data_file, page_set_json_name)
 

Protected Attributes

 _browser_executable
 
 _browser_args
 
 _chrome_page_sets_path
 
 _all_page_sets_specified
 
 _page_sets
 
 _record
 
 _skia_tools
 
 _non_interactive
 
 _upload
 
 _skp_prefix
 
 _upload_to_partner_bucket
 
 _alternate_upload_dir
 
 _telemetry_binaries_dir
 
 _catapult_dir
 
 _local_skp_dir
 
 _local_record_webpages_archive_dir
 
 _skp_files
 

Detailed Description

Class that archives or replays webpages and creates SKPs.

Definition at line 138 of file webpages_playback.py.

Constructor & Destructor Documentation

◆ __init__()

webpages_playback.SkPicturePlayback.__init__ (   self,
  parse_options 
) 
Constructs a SkPicturePlayback BuildStep instance.

Definition at line 141 of file webpages_playback.py.

141 def __init__(self, parse_options):
142 """Constructs a SkPicturePlayback BuildStep instance."""
143 assert parse_options.browser_executable, 'Must specify --browser_executable'
144 self._browser_executable = parse_options.browser_executable
145 self._browser_args = '--disable-setuid-sandbox'
146 if parse_options.browser_extra_args:
147 self._browser_args = '%s %s' % (
148 self._browser_args, parse_options.browser_extra_args)
149
150 self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path,
151 CHROMIUM_PAGE_SETS_PATH)
152 self._all_page_sets_specified = parse_options.page_sets == 'all'
153 self._page_sets = self._ParsePageSets(parse_options.page_sets)
154
155 self._record = parse_options.record
156 self._skia_tools = parse_options.skia_tools
157 self._non_interactive = parse_options.non_interactive
158 self._upload = parse_options.upload
159 self._skp_prefix = parse_options.skp_prefix
160 data_store_location = parse_options.data_store
161 if data_store_location.startswith(GS_PREFIX):
162 self.gs = GoogleStorageDataStore(data_store_location)
163 else:
164 self.gs = LocalFileSystemDataStore(data_store_location)
165 self._upload_to_partner_bucket = parse_options.upload_to_partner_bucket
166 self._alternate_upload_dir = parse_options.alternate_upload_dir
167 self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path,
168 'tools', 'perf')
169 self._catapult_dir = os.path.join(parse_options.chrome_src_path,
170 'third_party', 'catapult')
171
172 self._local_skp_dir = os.path.join(
173 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME)
174 self._local_record_webpages_archive_dir = os.path.join(
175 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive')
176
177 # List of SKP files generated by this script.
178 self._skp_files = []
179

Member Function Documentation

◆ _CreateLocalStorageDirs()

webpages_playback.SkPicturePlayback._CreateLocalStorageDirs (   self)
protected 
Creates required local storage directories for this script.

Definition at line 457 of file webpages_playback.py.

457 def _CreateLocalStorageDirs(self):
458 """Creates required local storage directories for this script."""
459 for d in (self._local_record_webpages_archive_dir,
460 self._local_skp_dir):
461 if os.path.exists(d):
462 shutil.rmtree(d)
463 os.makedirs(d)
464

◆ _DownloadWebpagesArchive()

webpages_playback.SkPicturePlayback._DownloadWebpagesArchive (   self,
  wpr_data_file,
  page_set_json_name 
)
protected 
Downloads the webpages archive and its required page set from GS.

Definition at line 465 of file webpages_playback.py.

465 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name):
466 """Downloads the webpages archive and its required page set from GS."""
467 wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
468 wpr_data_file)
469 page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME,
470 'webpages_archive',
471 page_set_json_name)
472 gs = self.gs
473 if (gs.does_storage_object_exist(wpr_source) and
474 gs.does_storage_object_exist(page_set_source)):
475 gs.download_file(wpr_source, LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
476 gs.download_file(page_set_source,
477 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
478 page_set_json_name))
479 else:
480 raise Exception('%s and %s do not exist in %s!' % (gs.target_type(),
481 wpr_source, page_set_source))
482

◆ _GetChromiumSkpFileName()

webpages_playback.SkPicturePlayback._GetChromiumSkpFileName (   self,
  page_set,
  site 
)
protected 
Returns the SKP file name for Chromium page sets.

Definition at line 408 of file webpages_playback.py.

408 def _GetChromiumSkpFileName(self, page_set, site):
409 """Returns the SKP file name for Chromium page sets."""
410 # /path/to/http___mobile_news_sandbox_pt0 -> http___mobile_news_sandbox_pt0
411 _, webpage = os.path.split(site)
412 # http___mobile_news_sandbox_pt0 -> mobile_news_sandbox_pt0
413 for prefix in ('http___', 'https___', 'www_'):
414 if webpage.startswith(prefix):
415 webpage = webpage[len(prefix):]
416 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
417 ps_filename = os.path.basename(page_set)
418 # http___mobile_news_sandbox -> pagesetprefix_http___mobile_news_sandbox
419 basename = '%s_%s' % (CHROMIUM_PAGE_SETS_TO_PREFIX[ps_filename], webpage)
420 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
421

◆ _GetSkiaSkpFileName()

webpages_playback.SkPicturePlayback._GetSkiaSkpFileName (   self,
  page_set 
)
protected 
Returns the SKP file name for Skia page sets.

Definition at line 397 of file webpages_playback.py.

397 def _GetSkiaSkpFileName(self, page_set):
398 """Returns the SKP file name for Skia page sets."""
399 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
400 ps_filename = os.path.basename(page_set)
401 # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop
402 ps_basename, _ = os.path.splitext(ps_filename)
403 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
404 _, page_name, device = ps_basename.split('_')
405 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
406 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
407

◆ _IsChromiumPageSet()

webpages_playback.SkPicturePlayback._IsChromiumPageSet (   self,
  page_set 
)
protected 
Returns true if the specified page set is a Chromium page set.

Definition at line 203 of file webpages_playback.py.

203 def _IsChromiumPageSet(self, page_set):
204 """Returns true if the specified page set is a Chromium page set."""
205 return page_set.startswith(self._chrome_page_sets_path)
206

◆ _ParsePageSets()

webpages_playback.SkPicturePlayback._ParsePageSets (   self,
  page_sets 
)
protected

Definition at line 180 of file webpages_playback.py.

180 def _ParsePageSets(self, page_sets):
181 if not page_sets:
182 raise ValueError('Must specify at least one page_set!')
183 elif self._all_page_sets_specified:
184 # Get everything from the page_sets directory.
185 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
186 'page_sets')
187 ps = [os.path.join(page_sets_dir, page_set)
188 for page_set in os.listdir(page_sets_dir)
189 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and
190 page_set.endswith('.py')]
191 chromium_ps = [
192 os.path.join(self._chrome_page_sets_path, cr_page_set)
193 for cr_page_set in CHROMIUM_PAGE_SETS_TO_PREFIX]
194 ps.extend(chromium_ps)
195 elif '*' in page_sets:
196 # Explode and return the glob.
197 ps = glob.glob(page_sets)
198 else:
199 ps = page_sets.split(',')
200 ps.sort()
201 return ps
202

◆ _RenameSkpFiles()

webpages_playback.SkPicturePlayback._RenameSkpFiles (   self,
  page_set 
)
protected 
Rename generated SKP files into more descriptive names.

Look into the subdirectory of TMP_SKP_DIR and find the most interesting
.skp in there to be this page_set's representative .skp.

Throws InvalidSKPException if the chosen .skp is less than 1KB. This
typically happens when there is a 404 or a redirect loop. Anything greater
than 1KB seems to have captured at least some useful information.

Definition at line 422 of file webpages_playback.py.

422 def _RenameSkpFiles(self, page_set):
423 """Rename generated SKP files into more descriptive names.
424
425 Look into the subdirectory of TMP_SKP_DIR and find the most interesting
426 .skp in there to be this page_set's representative .skp.
427
428 Throws InvalidSKPException if the chosen .skp is less than 1KB. This
429 typically happens when there is a 404 or a redirect loop. Anything greater
430 than 1KB seems to have captured at least some useful information.
431 """
432 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
433 for site in subdirs:
434 if self._IsChromiumPageSet(page_set):
435 filename = self._GetChromiumSkpFileName(page_set, site)
436 else:
437 filename = self._GetSkiaSkpFileName(page_set)
438 filename = filename.lower()
439
440 if self._skp_prefix:
441 filename = '%s%s' % (self._skp_prefix, filename)
442
443 # We choose the largest .skp as the most likely to be interesting.
444 largest_skp = max(glob.glob(os.path.join(site, '*.skp')),
445 key=lambda path: os.stat(path).st_size)
446 dest = os.path.join(self._local_skp_dir, filename)
447 print('Moving', largest_skp, 'to', dest)
448 shutil.move(largest_skp, dest)
449 self._skp_files.append(filename)
450 shutil.rmtree(site)
451 skp_size = os.path.getsize(dest)
452 if skp_size < 1024:
453 raise InvalidSKPException(
454 'Size of %s is only %d. Something is wrong.' % (dest, skp_size))
455
456
print
void print(void *str)
Definition bridge.cpp:126
append
static void append(char **dst, size_t *count, const char *src, size_t n)
Definition editor.cpp:211
max
static float max(float r, float g, float b)
Definition hsl.cpp:49

◆ Run()

webpages_playback.SkPicturePlayback.Run (   self) 
Run the SkPicturePlayback BuildStep.

Definition at line 207 of file webpages_playback.py.

207 def Run(self):
208 """Run the SkPicturePlayback BuildStep."""
209
210 # Download the credentials file if it was not previously downloaded.
211 if not os.path.isfile(CREDENTIALS_FILE_PATH):
212 # Download the credentials.json file from Google Storage.
213 self.gs.download_file(CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH)
214
215 if not os.path.isfile(CREDENTIALS_FILE_PATH):
216 raise Exception("""Could not locate credentials file in the storage.
217 Please create a credentials file in gs://%s that contains:
218 {
219 "google": {
220 "username": "google_testing_account_username",
221 "password": "google_testing_account_password"
222 }
223 }\n\n""" % CREDENTIALS_GS_PATH)
224
225 # Delete any left over data files in the data directory.
226 for archive_file in glob.glob(
227 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
228 os.remove(archive_file)
229
230 # Create the required local storage directories.
231 self._CreateLocalStorageDirs()
232
233 # Start the timer.
234 start_time = time.time()
235
236 # Loop through all page_sets.
237 for page_set in self._page_sets:
238 if os.path.basename(page_set) == '__init__.py':
239 continue
240 page_set_basename = os.path.basename(page_set).split('.')[0]
241 page_set_json_name = page_set_basename + '.json'
242 wpr_data_file_glob = (
243 page_set.split(os.path.sep)[-1].split('.')[0] + '_*.wprgo')
244 page_set_dir = os.path.dirname(page_set)
245
246 if self._IsChromiumPageSet(page_set):
247 print('Using Chromium\'s captured archives for Chromium\'s page sets.')
248 elif self._record:
249 # Create an archive of the specified webpages if '--record=True' is
250 # specified.
251 record_wpr_cmd = (
252 'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
253 'DISPLAY=%s' % X11_DISPLAY,
254 os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
255 '--extra-browser-args="%s"' % self._browser_args,
256 '--browser=exact',
257 '--browser-executable=%s' % self._browser_executable,
258 '%s_page_set' % page_set_basename,
259 '--page-set-base-dir=%s' % page_set_dir
260 )
261 for _ in range(RETRY_RECORD_WPR_COUNT):
262 try:
263 subprocess.check_call(' '.join(record_wpr_cmd), shell=True)
264
265 # Copy over the created archive into the local webpages archive
266 # directory.
267 for wpr_data_file in glob.glob(os.path.join(
268 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file_glob)):
269 shutil.copy(
270 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
271 self._local_record_webpages_archive_dir)
272 shutil.copy(
273 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
274 page_set_json_name),
275 self._local_record_webpages_archive_dir)
276
277 # Break out of the retry loop since there were no errors.
278 break
279 except Exception:
280 # There was a failure continue with the loop.
281 traceback.print_exc()
282 else:
283 # If we get here then record_wpr did not succeed and thus did not
284 # break out of the loop.
285 raise Exception('record_wpr failed for page_set: %s' % page_set)
286
287 else:
288 # Get the webpages archive so that it can be replayed.
289 self._DownloadWebpagesArchive(wpr_data_file_glob, page_set_json_name)
290
291 run_benchmark_cmd = [
292 'PYTHONPATH=%s:%s:$PYTHONPATH' % (page_set_dir, self._catapult_dir),
293 'DISPLAY=%s' % X11_DISPLAY,
294 'timeout', '1800',
295 os.path.join(self._telemetry_binaries_dir, 'run_benchmark'),
296 '--extra-browser-args="%s"' % self._browser_args,
297 '--browser=exact',
298 '--browser-executable=%s' % self._browser_executable,
299 SKP_BENCHMARK,
300 '--page-set-name=%s' % page_set_basename,
301 '--page-set-base-dir=%s' % page_set_dir,
302 '--skp-outdir=%s' % TMP_SKP_DIR,
303 '--also-run-disabled-tests',
304 ]
305
306 exclusions = PAGE_SETS_TO_EXCLUSIONS.get(os.path.basename(page_set))
307 if exclusions:
308 run_benchmark_cmd.append('--story-filter-exclude=' + exclusions)
309
310 for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
311 try:
312 print('\n\n=======Capturing SKP of %s=======\n\n' % page_set)
313 subprocess.check_call(' '.join(run_benchmark_cmd), shell=True)
314 except subprocess.CalledProcessError:
315 # There was a failure continue with the loop.
316 traceback.print_exc()
317 print('\n\n=======Retrying %s=======\n\n' % page_set)
318 time.sleep(10)
319 continue
320
321 try:
322 # Rename generated SKP files into more descriptive names.
323 self._RenameSkpFiles(page_set)
324 except InvalidSKPException:
325 # There was a failure continue with the loop.
326 traceback.print_exc()
327 print('\n\n=======Retrying %s=======\n\n' % page_set)
328 time.sleep(10)
329 continue
330
331 # Break out of the retry loop since there were no errors.
332 break
333 else:
334 # If we get here then run_benchmark did not succeed and thus did not
335 # break out of the loop.
336 raise Exception('run_benchmark failed for page_set: %s' % page_set)
337
338 print('\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
339 time.time() - start_time))
340
341 if self._skia_tools:
342 render_pictures_cmd = [
343 os.path.join(self._skia_tools, 'render_pictures'),
344 '-r', self._local_skp_dir
345 ]
346 render_pdfs_cmd = [
347 os.path.join(self._skia_tools, 'render_pdfs'),
348 '-r', self._local_skp_dir
349 ]
350
351 for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
352 print('\n\n=======Running %s=======' % ' '.join(tools_cmd))
353 subprocess.check_call(tools_cmd)
354
355 if not self._non_interactive:
356 print('\n\n=======Running debugger=======')
357 os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
358 self._local_skp_dir))
359
360 print('\n\n')
361
362 if self._upload:
363 print('\n\n=======Uploading to %s=======\n\n' % self.gs.target_type())
364 # Copy the directory structure in the root directory into Google Storage.
365 dest_dir_name = ROOT_PLAYBACK_DIR_NAME
366 if self._alternate_upload_dir:
367 dest_dir_name = self._alternate_upload_dir
368
369 self.gs.upload_dir_contents(
370 self._local_skp_dir, dest_dir=dest_dir_name)
371
372 print('\n\n=======New SKPs have been uploaded to %s =======\n\n' %
373 posixpath.join(self.gs.target_name(), dest_dir_name,
374 SKPICTURES_DIR_NAME))
375
376 else:
377 print('\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type())
378 print('Generated resources are available in %s\n\n' % self._local_skp_dir)
379
380 if self._upload_to_partner_bucket:
381 print('\n\n=======Uploading to Partner bucket %s =======\n\n' %
382 PARTNERS_GS_BUCKET)
383 partner_gs = GoogleStorageDataStore(PARTNERS_GS_BUCKET)
384 timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d')
385 upload_dir = posixpath.join(SKPICTURES_DIR_NAME, timestamp)
386 try:
387 partner_gs.delete_path(upload_dir)
388 except subprocess.CalledProcessError:
389 print('Cannot delete %s because it does not exist yet.' % upload_dir)
390 print('Uploading %s to %s' % (self._local_skp_dir, upload_dir))
391 partner_gs.upload_dir_contents(self._local_skp_dir, upload_dir)
392 print('\n\n=======New SKPs have been uploaded to %s =======\n\n' %
393 posixpath.join(partner_gs.target_name(), upload_dir))
394
395 return 0
396

Member Data Documentation

◆ _all_page_sets_specified

webpages_playback.SkPicturePlayback._all_page_sets_specified
protected

Definition at line 152 of file webpages_playback.py.

◆ _alternate_upload_dir

webpages_playback.SkPicturePlayback._alternate_upload_dir
protected

Definition at line 166 of file webpages_playback.py.

◆ _browser_args

webpages_playback.SkPicturePlayback._browser_args
protected

Definition at line 145 of file webpages_playback.py.

◆ _browser_executable

webpages_playback.SkPicturePlayback._browser_executable
protected

Definition at line 144 of file webpages_playback.py.

◆ _catapult_dir

webpages_playback.SkPicturePlayback._catapult_dir
protected

Definition at line 169 of file webpages_playback.py.

◆ _chrome_page_sets_path

webpages_playback.SkPicturePlayback._chrome_page_sets_path
protected

Definition at line 150 of file webpages_playback.py.

◆ _local_record_webpages_archive_dir

webpages_playback.SkPicturePlayback._local_record_webpages_archive_dir
protected

Definition at line 174 of file webpages_playback.py.

◆ _local_skp_dir

webpages_playback.SkPicturePlayback._local_skp_dir
protected

Definition at line 172 of file webpages_playback.py.

◆ _non_interactive

webpages_playback.SkPicturePlayback._non_interactive
protected

Definition at line 157 of file webpages_playback.py.

◆ _page_sets

webpages_playback.SkPicturePlayback._page_sets
protected

Definition at line 153 of file webpages_playback.py.

◆ _record

webpages_playback.SkPicturePlayback._record
protected

Definition at line 155 of file webpages_playback.py.

◆ _skia_tools

webpages_playback.SkPicturePlayback._skia_tools
protected

Definition at line 156 of file webpages_playback.py.

◆ _skp_files

webpages_playback.SkPicturePlayback._skp_files
protected

Definition at line 178 of file webpages_playback.py.

◆ _skp_prefix

webpages_playback.SkPicturePlayback._skp_prefix
protected

Definition at line 159 of file webpages_playback.py.

◆ _telemetry_binaries_dir

webpages_playback.SkPicturePlayback._telemetry_binaries_dir
protected

Definition at line 167 of file webpages_playback.py.

◆ _upload

webpages_playback.SkPicturePlayback._upload
protected

Definition at line 158 of file webpages_playback.py.

◆ _upload_to_partner_bucket

webpages_playback.SkPicturePlayback._upload_to_partner_bucket
protected

Definition at line 165 of file webpages_playback.py.

◆ gs

webpages_playback.SkPicturePlayback.gs

Definition at line 162 of file webpages_playback.py.

The documentation for this class was generated from the following file:
Generated on Fri Apr 26 2024 06:22:37 for Flutter Engine by doxygen 1.9.8