upload_dm_results Namespace Reference

Functions
	RunSteps (api)
	GenTests (api)
	main (dm_dir, build_number, builder_name)

Variables
str	PYTHON_VERSION_COMPATIBILITY = "PY3"
list	DEPS
str	DM_JSON = 'dm.json'
str	VERBOSE_LOG = 'verbose.log'

Detailed Description

Upload DM output PNG files and JSON summary to Google Storage.

Function Documentation

GenTests()

upload_dm_results.GenTests

(

api

)

Definition at line 84 of file upload_dm_results.py.

def GenTests(api):
  builder = 'Upload-Test-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All'
  yield (
    api.test('normal_bot') +
    api.properties(buildername=builder,
                   gs_bucket='skia-infra-gm',
                   revision='abc123',
                   path_config='kitchen')
  )
 
  yield (
    api.test('alternate_bucket') +
    api.properties(buildername=builder,
                   gs_bucket='skia-infra-gm-alt',
                   revision='abc123',
                   path_config='kitchen')
  )
 
  yield (
    api.test('failed_once') +
    api.properties(buildername=builder,
                   gs_bucket='skia-infra-gm',
                   revision='abc123',
                   path_config='kitchen') +
    api.step_data('upload .png images', retcode=1)
  )
 
  yield (
    api.test('failed_all') +
    api.properties(buildername=builder,
                   gs_bucket='skia-infra-gm',
                   revision='abc123',
                   path_config='kitchen') +
    api.step_data('upload .png images', retcode=1) +
    api.step_data('upload .png images (attempt 2)', retcode=1) +
    api.step_data('upload .png images (attempt 3)', retcode=1) +
    api.step_data('upload .png images (attempt 4)', retcode=1) +
    api.step_data('upload .png images (attempt 5)', retcode=1)
  )
 
  yield (
      api.test('trybot') +
      api.properties.tryserver(
          gerrit_project='skia',
          gerrit_url='https://skia-review.googlesource.com/',
      ) +
      api.properties(
          buildername=builder,
          gs_bucket='skia-infra-gm',
          revision='abc123',
          path_config='kitchen')
  )

main()

upload_dm_results.main	(		dm_dir,
			build_number,
			builder_name
	)

Upload DM output PNG files and JSON summary to Google Storage.

  dm_dir:        path to PNG files and JSON summary    (str)
  build_number:  nth build on this builder             (str or int)
  builder_name:  name of this builder                  (str)

Definition at line 15 of file upload_dm_results.py.

def main(dm_dir, build_number, builder_name):
  """Upload DM output PNG files and JSON summary to Google Storage.
 
    dm_dir:        path to PNG files and JSON summary    (str)
    build_number:  nth build on this builder             (str or int)
    builder_name:  name of this builder                  (str)
  """
  # import gs_utils
  current_dir = os.path.dirname(os.path.abspath(__file__))
  sys.path.insert(0, os.path.join(current_dir, "../../../common/py/utils"))
  import gs_utils
 
  # Private, but Google-readable.
  ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE
  FINE_ACLS = [(
    gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN,
    'google.com',
    gs_utils.GSUtils.Permission.READ
  )]
 
  if not os.path.isfile(os.path.join(dm_dir, 'dm.json')):
    sys.exit("no dm.json file found in output directory.")
 
  # Move dm.json to its own directory to make uploading it easier.
  tmp = tempfile.mkdtemp()
  shutil.move(os.path.join(dm_dir, 'dm.json'),
              os.path.join(tmp,    'dm.json'))
 
  # Only images are left in dm_dir.  Upload any new ones.
  gs = gs_utils.GSUtils()
  gs.upload_dir_contents(dm_dir,
                         'skia-android-dm',
                         'dm-images-v1',
                         upload_if = gs.UploadIf.IF_NEW,
                         predefined_acl = ACL,
                         fine_grained_acl_list = FINE_ACLS)
 
 
  # /dm-json-v1/year/month/day/hour/build-number/builder/dm.json
  now = datetime.datetime.utcnow()
  summary_dest_dir = '/'.join(['dm-json-v1',
                               str(now.year ).zfill(4),
                               str(now.month).zfill(2),
                               str(now.day  ).zfill(2),
                               str(now.hour ).zfill(2),
                               str(build_number),
                               builder_name])
 
  # Upload the JSON summary.
  gs.upload_dir_contents(tmp,
                         'skia-android-dm',
                         summary_dest_dir,
                         predefined_acl = ACL,
                         fine_grained_acl_list = FINE_ACLS)
 
 
  # Just for hygiene, put dm.json back.
  shutil.move(os.path.join(tmp,    'dm.json'),
              os.path.join(dm_dir, 'dm.json'))
  os.rmdir(tmp)
 

RunSteps()

upload_dm_results.RunSteps

(

api

)

Definition at line 29 of file upload_dm_results.py.

def RunSteps(api):
  api.vars.setup()
  revision = api.properties['revision']
 
  results_dir = api.path['start_dir'].join('test')
 
  # Upload the images. It is *vital* that the images are uploaded first
  # so they exist whenever the json is processed.
  image_dest_path = 'gs://%s/dm-images-v1' % api.properties['gs_bucket']
  for ext in ['.png', '.pdf']:
    files_to_upload = api.file.glob_paths(
        'find %s images' % ext,
        results_dir,
        '*%s' % ext,
        test_data=['someimage.png'])
    # For some reason, glob returns results_dir when it should return nothing.
    files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)]
    if len(files_to_upload) > 0:
      api.gsutil.cp('%s images' % ext, results_dir.join('*%s' % ext),
                       image_dest_path, multithread=True)
 
  # Compute the directory to upload results to
  now = api.time.utcnow()
  summary_dest_path = '/'.join([
      'dm-json-v1',
      str(now.year ).zfill(4),
      str(now.month).zfill(2),
      str(now.day  ).zfill(2),
      str(now.hour ).zfill(2),
      revision,
      api.vars.builder_name,
      str(int(calendar.timegm(now.utctimetuple())))])
 
  # Trybot results are further siloed by issue/patchset.
  if api.vars.is_trybot:
    summary_dest_path = '/'.join(('trybot', summary_dest_path,
                                  str(api.vars.issue), str(api.vars.patchset)))
 
  summary_dest_path = 'gs://%s/%s' % (api.properties['gs_bucket'],
                                      summary_dest_path)
 
  # Directly upload dm.json and verbose.log if it exists
  json_file = results_dir.join(DM_JSON)
  log_file = results_dir.join(VERBOSE_LOG)
 
  api.gsutil.cp('dm.json', json_file,
                summary_dest_path + '/' + DM_JSON, extra_args=['-Z'])
 
  files = api.file.listdir('check for optional verbose.log file',
                           results_dir, test_data=['dm.json', 'verbose.log'])
  if log_file in files:
    api.gsutil.cp('verbose.log', log_file,
                  summary_dest_path + '/' + VERBOSE_LOG, extra_args=['-Z'])
 
 

Variable Documentation

DEPS

list upload_dm_results.DEPS

Initial value:

=  [
  'recipe_engine/file',
  'recipe_engine/json',
  'recipe_engine/path',
  'recipe_engine/properties',
  'recipe_engine/step',
  'recipe_engine/time',
  'gsutil',
  'vars',
]

Definition at line 13 of file upload_dm_results.py.

DM_JSON

str upload_dm_results.DM_JSON = 'dm.json'

Definition at line 25 of file upload_dm_results.py.

PYTHON_VERSION_COMPATIBILITY

str upload_dm_results.PYTHON_VERSION_COMPATIBILITY = "PY3"

Definition at line 11 of file upload_dm_results.py.

VERBOSE_LOG

str upload_dm_results.VERBOSE_LOG = 'verbose.log'

Definition at line 26 of file upload_dm_results.py.