safetynet_job.py

Go to the documentation of this file.

#!/usr/bin/env python3
# Copyright 2017 The PDFium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Looks for performance regressions on all pushes since the last run.
 
Run this nightly to have a periodical check for performance regressions.
 
Stores the results for each run and last checkpoint in a results directory.
"""
 
import argparse
import datetime
import json
import os
import sys
 
from common import PrintWithTime
from common import RunCommandPropagateErr
from githelper import GitHelper
from safetynet_conclusions import PrintConclusionsDictHumanReadable
 
 
class JobContext:
  """Context for a single run, including name and directory paths."""
 
  def __init__(self, args):
    self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    self.results_dir = args.results_dir
    self.last_revision_covered_file = os.path.join(self.results_dir,
                                                   'last_revision_covered')
    self.run_output_dir = os.path.join(self.results_dir,
                                       'profiles_%s' % self.datetime)
    self.run_output_log_file = os.path.join(self.results_dir,
                                            '%s.log' % self.datetime)
 
 
class JobRun:
  """A single run looking for regressions since the last one."""
 
  def __init__(self, args, context):
    """Constructor.
 
    Args:
      args: Namespace with arguments passed to the script.
      context: JobContext for this run.
    """
    self.git = GitHelper()
    self.args = args
    self.context = context
 
  def Run(self):
    """Searches for regressions.
 
    Will only write a checkpoint when first run, and on all subsequent runs
    a comparison is done against the last checkpoint.
 
    Returns:
      Exit code for the script: 0 if no significant changes are found; 1 if
      there was an error in the comparison; 3 if there was a regression; 4 if
      there was an improvement and no regression.
    """
    pdfium_src_dir = os.path.join(
        os.path.dirname(__file__), os.path.pardir, os.path.pardir)
    os.chdir(pdfium_src_dir)
 
    branch_to_restore = self.git.GetCurrentBranchName()
 
    if not self.args.no_checkout:
      self.git.FetchOriginMaster()
      self.git.Checkout('origin/main')
 
    # Make sure results dir exists
    if not os.path.exists(self.context.results_dir):
      os.makedirs(self.context.results_dir)
 
    if not os.path.exists(self.context.last_revision_covered_file):
      result = self._InitialRun()
    else:
      with open(self.context.last_revision_covered_file) as f:
        last_revision_covered = f.read().strip()
      result = self._IncrementalRun(last_revision_covered)
 
    self.git.Checkout(branch_to_restore)
    return result
 
  def _InitialRun(self):
    """Initial run, just write a checkpoint.
 
    Returns:
      Exit code for the script.
    """
    current = self.git.GetCurrentBranchHash()
 
    PrintWithTime('Initial run, current is %s' % current)
 
    self._WriteCheckpoint(current)
 
    PrintWithTime('All set up, next runs will be incremental and perform '
                  'comparisons')
    return 0
 
  def _IncrementalRun(self, last_revision_covered):
    """Incremental run, compare against last checkpoint and update it.
 
    Args:
      last_revision_covered: String with hash for last checkpoint.
 
    Returns:
      Exit code for the script.
    """
    current = self.git.GetCurrentBranchHash()
 
    PrintWithTime('Incremental run, current is %s, last is %s' %
                  (current, last_revision_covered))
 
    if not os.path.exists(self.context.run_output_dir):
      os.makedirs(self.context.run_output_dir)
 
    if current == last_revision_covered:
      PrintWithTime('No changes seen, finishing job')
      output_info = {
          'metadata':
              self._BuildRunMetadata(last_revision_covered, current, False)
      }
      self._WriteRawJson(output_info)
      return 0
 
    # Run compare
    cmd = [
        'testing/tools/safetynet_compare.py', '--this-repo',
        '--machine-readable',
        '--branch-before=%s' % last_revision_covered,
        '--output-dir=%s' % self.context.run_output_dir
    ]
    cmd.extend(self.args.input_paths)
 
    json_output = RunCommandPropagateErr(cmd)
 
    if json_output is None:
      return 1
 
    output_info = json.loads(json_output)
 
    run_metadata = self._BuildRunMetadata(last_revision_covered, current, True)
    output_info.setdefault('metadata', {}).update(run_metadata)
    self._WriteRawJson(output_info)
 
    PrintConclusionsDictHumanReadable(
        output_info,
        colored=(not self.args.output_to_log and not self.args.no_color),
        key='after')
 
    status = 0
 
    if output_info['summary']['improvement']:
      PrintWithTime('Improvement detected.')
      status = 4
 
    if output_info['summary']['regression']:
      PrintWithTime('Regression detected.')
      status = 3
 
    if status == 0:
      PrintWithTime('Nothing detected.')
 
    self._WriteCheckpoint(current)
 
    return status
 
  def _WriteRawJson(self, output_info):
    json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
    with open(json_output_file, 'w') as f:
      json.dump(output_info, f)
 
  def _BuildRunMetadata(self, revision_before, revision_after,
                        comparison_performed):
    return {
        'datetime': self.context.datetime,
        'revision_before': revision_before,
        'revision_after': revision_after,
        'comparison_performed': comparison_performed,
    }
 
  def _WriteCheckpoint(self, checkpoint):
    if not self.args.no_checkpoint:
      with open(self.context.last_revision_covered_file, 'w') as f:
        f.write(checkpoint + '\n')
 
 
def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('results_dir', help='where to write the job results')
  parser.add_argument(
      'input_paths',
      nargs='+',
      help='pdf files or directories to search for pdf files '
      'to run as test cases')
  parser.add_argument(
      '--no-checkout',
      action='store_true',
      help='whether to skip checking out origin/main. Use '
      'for script debugging.')
  parser.add_argument(
      '--no-checkpoint',
      action='store_true',
      help='whether to skip writing the new checkpoint. Use '
      'for script debugging.')
  parser.add_argument(
      '--no-color',
      action='store_true',
      help='whether to write output without color escape '
      'codes.')
  parser.add_argument(
      '--output-to-log',
      action='store_true',
      help='whether to write output to a log file')
  args = parser.parse_args()
 
  job_context = JobContext(args)
 
  if args.output_to_log:
    log_file = open(job_context.run_output_log_file, 'w')
    sys.stdout = log_file
    sys.stderr = log_file
 
  run = JobRun(args, job_context)
  result = run.Run()
 
  if args.output_to_log:
    log_file.close()
 
  return result
 
 
if __name__ == '__main__':
  sys.exit(main())