How to use _build_config method in Slash

Best Python code snippet using slash

cbuildbot_stages.py

Source:cbuildbot_stages.py

1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4"""Module containing the various stages that a builder runs."""5import contextlib6import datetime7import functools8import glob9import json10import logging11import math12import multiprocessing13import os14import Queue15import shutil16import sys17from chromite.buildbot import builderstage as bs18from chromite.buildbot import cbuildbot_commands as commands19from chromite.buildbot import cbuildbot_config20from chromite.buildbot import cbuildbot_results as results_lib21from chromite.buildbot import constants22from chromite.buildbot import lab_status23from chromite.buildbot import lkgm_manager24from chromite.buildbot import manifest_version25from chromite.buildbot import portage_utilities26from chromite.buildbot import repository27from chromite.buildbot import trybot_patch_pool28from chromite.buildbot import validation_pool29from chromite.lib import commandline30from chromite.lib import cros_build_lib31from chromite.lib import git32from chromite.lib import gs33from chromite.lib import toolchain34from chromite.lib import osutils35from chromite.lib import parallel36from chromite.lib import patch as cros_patch37_FULL_BINHOST = 'FULL_BINHOST'38_PORTAGE_BINHOST = 'PORTAGE_BINHOST'39_CROS_ARCHIVE_URL = 'CROS_ARCHIVE_URL'40_PRINT_INTERVAL = 141_VM_TEST_ERROR_MSG = """42!!!VMTests failed!!!43Logs are uploaded in the corresponding vm_test_results.tgz. This can be found44by clicking on the artifacts link in the "Report" Stage. Specifically look45for the test_harness/failed for the failing tests. For more46particulars, please refer to which test failed i.e. above see the47individual test that failed -- or if an update failed, check the48corresponding update directory.49"""50class NonHaltingBuilderStage(bs.BuilderStage):51 """Build stage that fails a build but finishes the other steps."""52 def Run(self):53 try:54 super(NonHaltingBuilderStage, self).Run()55 except results_lib.StepFailure:56 name = self.__class__.__name__57 cros_build_lib.Error('Ignoring StepFailure in %s', name)58class ForgivingBuilderStage(bs.BuilderStage):59 """Build stage that turns a build step red but not a build."""60 def _HandleStageException(self, exception):61 """Override and don't set status to FAIL but FORGIVEN instead."""62 return self._HandleExceptionAsWarning(exception)63class BoardSpecificBuilderStage(bs.BuilderStage):64 def __init__(self, options, build_config, board, suffix=None):65 super(BoardSpecificBuilderStage, self).__init__(options, build_config,66 suffix)67 self._current_board = board68 if not isinstance(board, basestring):69 raise TypeError('Expected string, got %r' % (board,))70 # Add a board name suffix to differentiate between various boards (in case71 # more than one board is built on a single builder.)72 if len(self._boards) > 1 or build_config['grouped']:73 self.name = '%s [%s]' % (self.name, board)74 def GetImageDirSymlink(self, pointer='latest-cbuildbot'):75 """Get the location of the current image."""76 buildroot, board = self._options.buildroot, self._current_board77 return os.path.join(buildroot, 'src', 'build', 'images', board, pointer)78class ArchivingStage(BoardSpecificBuilderStage):79 """Helper for stages that archive files.80 Attributes:81 archive_stage: The ArchiveStage instance for this board.82 bot_archive_root: The root path where output from this builder is stored.83 download_url: The URL where we can download artifacts.84 upload_url: The Google Storage location where we should upload artifacts.85 """86 PROCESSES = 1087 _BUILDBOT_ARCHIVE = 'buildbot_archive'88 _TRYBOT_ARCHIVE = 'trybot_archive'89 @classmethod90 def GetArchiveRoot(cls, buildroot, trybot=False):91 """Return the location where archive images are kept."""92 archive_base = cls._TRYBOT_ARCHIVE if trybot else cls._BUILDBOT_ARCHIVE93 return os.path.join(buildroot, archive_base)94 def __init__(self, options, build_config, board, archive_stage, suffix=None):95 super(ArchivingStage, self).__init__(options, build_config, board,96 suffix=suffix)97 self.archive_stage = archive_stage98 if options.remote_trybot:99 self.debug = options.debug_forced100 else:101 self.debug = options.debug102 self.version = archive_stage.GetVersion()103 gsutil_archive = self._GetGSUtilArchiveDir()104 self.upload_url = '%s/%s' % (gsutil_archive, self.version)105 trybot = not options.buildbot or options.debug106 archive_root = ArchivingStage.GetArchiveRoot(self._build_root, trybot)107 self.bot_archive_root = os.path.join(archive_root, self._bot_id)108 self.archive_path = os.path.join(self.bot_archive_root, self.version)109 if options.buildbot or options.remote_trybot:110 base_download_url = gs.PRIVATE_BASE_HTTPS_URL111 self.download_url = self.upload_url.replace('gs://', base_download_url)112 else:113 self.download_url = self.archive_path114 @contextlib.contextmanager115 def ArtifactUploader(self, queue=None, archive=True, strict=True):116 """Upload each queued input in the background.117 This context manager starts a set of workers in the background, who each118 wait for input on the specified queue. These workers run119 self.UploadArtifact(*args, archive=archive) for each input in the queue.120 Arguments:121 queue: Queue to use. Add artifacts to this queue, and they will be122 uploaded in the background. If None, one will be created on the fly.123 archive: Whether to automatically copy files to the archive dir.124 strict: Whether to treat upload errors as fatal.125 Returns:126 The queue to use. This is only useful if you did not supply a queue.127 """128 upload = lambda path: self.UploadArtifact(path, archive, strict)129 with parallel.BackgroundTaskRunner(upload, queue=queue,130 processes=self.PROCESSES) as bg_queue:131 yield bg_queue132 def PrintDownloadLink(self, filename, prefix=''):133 """Print a link to an artifact in Google Storage.134 Args:135 filename: The filename of the uploaded file.136 prefix: The prefix to put in front of the filename.137 """138 url = '%s/%s' % (self.download_url.rstrip('/'), filename)139 cros_build_lib.PrintBuildbotLink(prefix + filename, url)140 def UploadArtifact(self, path, archive=True, strict=True):141 """Upload generated artifact to Google Storage.142 Arguments:143 path: Path of local file to upload to Google Storage.144 archive: Whether to automatically copy files to the archive dir.145 strict: Whether to treat upload errors as fatal.146 """147 acl = None if self._build_config['internal'] else 'public-read'148 filename = path149 if archive:150 filename = commands.ArchiveFile(path, self.archive_path)151 try:152 commands.UploadArchivedFile(self.archive_path, self.upload_url, filename,153 self.debug, update_list=True, acl=acl)154 except cros_build_lib.RunCommandError as e:155 cros_build_lib.PrintBuildbotStepText('Upload failed')156 if strict:157 raise158 # Treat gsutil flake as a warning if it's the only problem.159 self._HandleExceptionAsWarning(e)160 def _GetGSUtilArchiveDir(self):161 if self._options.archive_base:162 gs_base = self._options.archive_base163 elif (self._options.remote_trybot or164 self._build_config['gs_path'] == cbuildbot_config.GS_PATH_DEFAULT):165 gs_base = constants.DEFAULT_ARCHIVE_BUCKET166 else:167 return self._build_config['gs_path']168 return '%s/%s' % (gs_base, self._bot_id)169class CleanUpStage(bs.BuilderStage):170 """Stages that cleans up build artifacts from previous runs.171 This stage cleans up previous KVM state, temporary git commits,172 clobbers, and wipes tmp inside the chroot.173 """174 option_name = 'clean'175 def _CleanChroot(self):176 commands.CleanupChromeKeywordsFile(self._boards,177 self._build_root)178 chroot_tmpdir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR,179 'tmp')180 if os.path.exists(chroot_tmpdir):181 cros_build_lib.SudoRunCommand(['rm', '-rf', chroot_tmpdir],182 print_cmd=False)183 cros_build_lib.SudoRunCommand(['mkdir', '--mode', '1777', chroot_tmpdir],184 print_cmd=False)185 def _DeleteChroot(self):186 chroot = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)187 if os.path.exists(chroot):188 cros_build_lib.RunCommand(['cros_sdk', '--delete', '--chroot', chroot],189 self._build_root,190 cwd=self._build_root)191 def _DeleteArchivedTrybotImages(self):192 """For trybots, clear all previus archive images to save space."""193 archive_root = ArchivingStage.GetArchiveRoot(self._build_root, trybot=True)194 shutil.rmtree(archive_root, ignore_errors=True)195 def _DeleteArchivedPerfResults(self):196 """Clear any previously stashed perf results from hw testing."""197 for result in glob.glob(os.path.join(198 self._options.log_dir, '*.%s' % HWTestStage.PERF_RESULTS_EXTENSION)):199 os.remove(result)200 def PerformStage(self):201 if (not (self._options.buildbot or self._options.remote_trybot)202 and self._options.clobber):203 if not commands.ValidateClobber(self._build_root):204 cros_build_lib.Die("--clobber in local mode must be approved.")205 # If we can't get a manifest out of it, then it's not usable and must be206 # clobbered.207 manifest = None208 if not self._options.clobber:209 try:210 manifest = git.ManifestCheckout.Cached(self._build_root, search=False)211 except (KeyboardInterrupt, MemoryError, SystemExit):212 raise213 except Exception, e:214 # Either there is no repo there, or the manifest isn't usable. If the215 # directory exists, log the exception for debugging reasons. Either216 # way, the checkout needs to be wiped since it's in an unknown217 # state.218 if os.path.exists(self._build_root):219 cros_build_lib.Warning("ManifestCheckout at %s is unusable: %s",220 self._build_root, e)221 if manifest is None:222 self._DeleteChroot()223 repository.ClearBuildRoot(self._build_root, self._options.preserve_paths)224 else:225 # Clean mount points first to be safe about deleting.226 commands.CleanUpMountPoints(self._build_root)227 commands.BuildRootGitCleanup(self._build_root, self._options.debug)228 tasks = [functools.partial(commands.BuildRootGitCleanup,229 self._build_root, self._options.debug),230 functools.partial(commands.WipeOldOutput, self._build_root),231 self._DeleteArchivedTrybotImages,232 self._DeleteArchivedPerfResults]233 if self._build_config['chroot_replace'] and self._options.build:234 tasks.append(self._DeleteChroot)235 else:236 tasks.append(self._CleanChroot)237 parallel.RunParallelSteps(tasks)238class PatchChangesStage(bs.BuilderStage):239 """Stage that patches a set of Gerrit changes to the buildroot source tree."""240 def __init__(self, options, build_config, patch_pool):241 """Construct a PatchChangesStage.242 Args:243 options, build_config: See arguments to bs.BuilderStage.__init__()244 patch_pool: A TrybotPatchPool object containing the different types of245 patches to apply.246 """247 bs.BuilderStage.__init__(self, options, build_config)248 self.patch_pool = patch_pool249 @staticmethod250 def _CheckForDuplicatePatches(_series, changes):251 conflicts = {}252 duplicates = []253 for change in changes:254 if change.id is None:255 cros_build_lib.Warning(256 "Change %s lacks a usable ChangeId; duplicate checking cannot "257 "be done for this change. If cherry-picking fails, this is a "258 "potential cause.", change)259 continue260 conflicts.setdefault(change.id, []).append(change)261 duplicates = [x for x in conflicts.itervalues() if len(x) > 1]262 if not duplicates:263 return changes264 for conflict in duplicates:265 cros_build_lib.Error(266 "Changes %s conflict with each other- they have same id %s.",267 ', '.join(map(str, conflict)), conflict[0].id)268 cros_build_lib.Die("Duplicate patches were encountered: %s", duplicates)269 @staticmethod270 def _FixIncompleteRemotePatches(series, changes):271 """Identify missing remote patches from older cbuildbot instances.272 Cbuildbot, prior to I8ab6790de801900c115a437b5f4ebb9a24db542f, uploaded273 a single patch per project- despite if there may have been a hundred274 patches actually pulled in by that patch. This method detects when275 we're dealing w/ the old incomplete version, and fills in those gaps."""276 broken = [x for x in changes277 if isinstance(x, cros_patch.UploadedLocalPatch)]278 if not broken:279 return changes280 changes = list(changes)281 known = cros_patch.PatchCache(changes)282 for change in broken:283 git_repo = series.GetGitRepoForChange(change)284 tracking = series.GetTrackingBranchForChange(change)285 branch = getattr(change, 'original_branch', tracking)286 for target in cros_patch.GeneratePatchesFromRepo(287 git_repo, change.project, tracking, branch, change.internal,288 allow_empty=True, starting_ref='%s^' % change.sha1):289 if target in known:290 continue291 known.Inject(target)292 changes.append(target)293 return changes294 def _PatchSeriesFilter(self, series, changes):295 if self._options.remote_version == 3:296 changes = self._FixIncompleteRemotePatches(series, changes)297 return self._CheckForDuplicatePatches(series, changes)298 def _ApplyPatchSeries(self, series, patch_pool, **kwargs):299 """Applies a patch pool using a patch series."""300 kwargs.setdefault('frozen', False)301 # Honor the given ordering, so that if a gerrit/remote patch302 # conflicts w/ a local patch, the gerrit/remote patch are303 # blamed rather than local (patch ordering is typically304 # local, gerrit, then remote).305 kwargs.setdefault('honor_ordering', True)306 kwargs['changes_filter'] = self._PatchSeriesFilter307 _applied, failed_tot, failed_inflight = series.Apply(308 list(patch_pool), **kwargs)309 failures = failed_tot + failed_inflight310 if failures:311 cros_build_lib.Die("Failed applying patches: %s",312 "\n".join(map(str, failures)))313 def PerformStage(self):314 class NoisyPatchSeries(validation_pool.PatchSeries):315 """Custom PatchSeries that adds links to buildbot logs for remote trys."""316 def ApplyChange(self, change, dryrun=False):317 if isinstance(change, cros_patch.GerritPatch):318 cros_build_lib.PrintBuildbotLink(str(change), change.url)319 elif isinstance(change, cros_patch.UploadedLocalPatch):320 cros_build_lib.PrintBuildbotStepText(str(change))321 return validation_pool.PatchSeries.ApplyChange(self, change,322 dryrun=dryrun)323 # If we're an external builder, ignore internal patches.324 helper_pool = validation_pool.HelperPool.SimpleCreate(325 cros_internal=self._build_config['internal'], cros=True)326 # Limit our resolution to non-manifest patches.327 patch_series = NoisyPatchSeries(328 self._build_root,329 force_content_merging=True,330 helper_pool=helper_pool,331 deps_filter_fn=lambda p: not trybot_patch_pool.ManifestFilter(p))332 self._ApplyPatchSeries(patch_series, self.patch_pool)333class BootstrapStage(PatchChangesStage):334 """Stage that patches a chromite repo and re-executes inside it.335 Attributes:336 returncode - the returncode of the cbuildbot re-execution. Valid after337 calling stage.Run().338 """339 option_name = 'bootstrap'340 def __init__(self, options, build_config, chromite_patch_pool,341 manifest_patch_pool=None):342 super(BootstrapStage, self).__init__(343 options, build_config, trybot_patch_pool.TrybotPatchPool())344 self.chromite_patch_pool = chromite_patch_pool345 self.manifest_patch_pool = manifest_patch_pool346 self.returncode = None347 def _ApplyManifestPatches(self, patch_pool):348 """Apply a pool of manifest patches to a temp manifest checkout.349 Arguments:350 filter_fn: Used to filter changes during dependency resolution.351 Returns:352 The path to the patched manifest checkout.353 Raises:354 Exception, if the new patched manifest cannot be parsed.355 """356 checkout_dir = os.path.join(self.tempdir, 'manfest-checkout')357 repository.CloneGitRepo(checkout_dir,358 self._build_config['manifest_repo_url'])359 patch_series = validation_pool.PatchSeries.WorkOnSingleRepo(360 checkout_dir, deps_filter_fn=trybot_patch_pool.ManifestFilter,361 tracking_branch=self._target_manifest_branch)362 self._ApplyPatchSeries(patch_series, patch_pool)363 # Create the branch that 'repo init -b <target_branch> -u <patched_repo>'364 # will look for.365 cmd = ['branch', '-f', self._target_manifest_branch, constants.PATCH_BRANCH]366 git.RunGit(checkout_dir, cmd)367 # Verify that the patched manifest loads properly. Propagate any errors as368 # exceptions.369 manifest = os.path.join(checkout_dir, self._build_config['manifest'])370 git.Manifest.Cached(manifest, manifest_include_dir=checkout_dir)371 return checkout_dir372 @staticmethod373 def _FilterArgsForApi(parsed_args, api_minor):374 """Remove arguments that are introduced after an api version."""375 def filter_fn(passed_arg):376 return passed_arg.opt_inst.api_version <= api_minor377 accepted, removed = commandline.FilteringParser.FilterArgs(378 parsed_args, filter_fn)379 if removed:380 cros_build_lib.Warning('The following arguments were removed due to api: '381 "'%s'" % ' '.join(removed))382 return accepted383 @classmethod384 def FilterArgsForTargetCbuildbot(cls, buildroot, cbuildbot_path, options):385 _, minor = cros_build_lib.GetTargetChromiteApiVersion(buildroot)386 args = [cbuildbot_path]387 args.extend(options.build_targets)388 args.extend(cls._FilterArgsForApi(options.parsed_args, minor))389 # Only pass down --cache-dir if it was specified. By default, we want390 # the cache dir to live in the root of each checkout, so this means that391 # each instance of cbuildbot needs to calculate the default separately.392 if minor >= 2 and options.cache_dir_specified:393 args += ['--cache-dir', options.cache_dir]394 return args395 #pylint: disable=E1101396 @osutils.TempDirDecorator397 def PerformStage(self):398 # The plan for the builders is to use master branch to bootstrap other399 # branches. Now, if we wanted to test patches for both the bootstrap code400 # (on master) and the branched chromite (say, R20), we need to filter the401 # patches by branch.402 filter_branch = self._target_manifest_branch403 if self._options.test_bootstrap:404 filter_branch = 'master'405 chromite_dir = os.path.join(self.tempdir, 'chromite')406 reference_repo = os.path.join(constants.SOURCE_ROOT, 'chromite', '.git')407 repository.CloneGitRepo(chromite_dir, constants.CHROMITE_URL,408 reference=reference_repo)409 git.RunGit(chromite_dir, ['checkout', filter_branch])410 def BranchAndChromiteFilter(patch):411 return (trybot_patch_pool.BranchFilter(filter_branch, patch) and412 trybot_patch_pool.ChromiteFilter(patch))413 patch_series = validation_pool.PatchSeries.WorkOnSingleRepo(414 chromite_dir, filter_branch,415 deps_filter_fn=BranchAndChromiteFilter)416 filtered_pool = self.chromite_patch_pool.FilterBranch(filter_branch)417 if filtered_pool:418 self._ApplyPatchSeries(patch_series, filtered_pool)419 cbuildbot_path = constants.PATH_TO_CBUILDBOT420 if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)):421 cbuildbot_path = 'chromite/buildbot/cbuildbot'422 cmd = self.FilterArgsForTargetCbuildbot(self.tempdir, cbuildbot_path,423 self._options)424 extra_params = ['--sourceroot=%s' % self._options.sourceroot]425 extra_params.extend(self._options.bootstrap_args)426 if self._options.test_bootstrap:427 # We don't want re-executed instance to see this.428 cmd = [a for a in cmd if a != '--test-bootstrap']429 else:430 # If we've already done the desired number of bootstraps, disable431 # bootstrapping for the next execution. Also pass in the patched manifest432 # repository.433 extra_params.append('--nobootstrap')434 if self.manifest_patch_pool:435 manifest_dir = self._ApplyManifestPatches(self.manifest_patch_pool)436 extra_params.extend(['--manifest-repo-url', manifest_dir])437 cmd += extra_params438 result_obj = cros_build_lib.RunCommand(439 cmd, cwd=self.tempdir, kill_timeout=30, error_code_ok=True)440 self.returncode = result_obj.returncode441class SyncStage(bs.BuilderStage):442 """Stage that performs syncing for the builder."""443 option_name = 'sync'444 output_manifest_sha1 = True445 def __init__(self, options, build_config):446 super(SyncStage, self).__init__(options, build_config)447 self.repo = None448 self.skip_sync = False449 self.internal = self._build_config['internal']450 def _GetManifestVersionsRepoUrl(self, read_only=False):451 return cbuildbot_config.GetManifestVersionsRepoUrl(452 self.internal,453 read_only=read_only)454 def Initialize(self):455 self._InitializeRepo()456 def _InitializeRepo(self, build_root=None, **kwds):457 if build_root is None:458 build_root = self._build_root459 manifest_url = self._options.manifest_repo_url460 if manifest_url is None:461 manifest_url = self._build_config['manifest_repo_url']462 kwds.setdefault('referenced_repo', self._options.reference_repo)463 kwds.setdefault('branch', self._target_manifest_branch)464 kwds.setdefault('manifest', self._build_config['manifest'])465 self.repo = repository.RepoRepository(manifest_url, build_root, **kwds)466 def GetNextManifest(self):467 """Returns the manifest to use."""468 return self._build_config['manifest']469 def ManifestCheckout(self, next_manifest):470 """Checks out the repository to the given manifest."""471 self._Print('\n'.join(['BUILDROOT: %s' % self.repo.directory,472 'TRACKING BRANCH: %s' % self.repo.branch,473 'NEXT MANIFEST: %s' % next_manifest]))474 if not self.skip_sync:475 self.repo.Sync(next_manifest)476 print >> sys.stderr, self.repo.ExportManifest(477 mark_revision=self.output_manifest_sha1)478 def PerformStage(self):479 self.Initialize()480 with osutils.TempDir() as tempdir:481 # Save off the last manifest.482 fresh_sync = True483 if os.path.exists(self.repo.directory) and not self._options.clobber:484 old_filename = os.path.join(tempdir, 'old.xml')485 try:486 old_contents = self.repo.ExportManifest()487 except cros_build_lib.RunCommandError as e:488 cros_build_lib.Warning(str(e))489 else:490 osutils.WriteFile(old_filename, old_contents)491 fresh_sync = False492 # Sync.493 self.ManifestCheckout(self.GetNextManifest())494 # Print the blamelist.495 if fresh_sync:496 cros_build_lib.PrintBuildbotStepText('(From scratch)')497 elif self._options.buildbot:498 lkgm_manager.GenerateBlameList(self.repo, old_filename)499class LKGMSyncStage(SyncStage):500 """Stage that syncs to the last known good manifest blessed by builders."""501 output_manifest_sha1 = False502 def GetNextManifest(self):503 """Override: Gets the LKGM."""504 # TODO(sosa): Should really use an initialized manager here.505 if self.internal:506 mv_dir = 'manifest-versions-internal'507 else:508 mv_dir = 'manifest-versions'509 manifest_path = os.path.join(self._build_root, mv_dir)510 manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True)511 manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo)512 return os.path.join(manifest_path, lkgm_manager.LKGMManager.LKGM_PATH)513class ChromeLKGMSyncStage(SyncStage):514 """Stage that syncs to the last known good manifest for Chrome."""515 output_manifest_sha1 = False516 def GetNextManifest(self):517 """Override: Gets the LKGM from the Chrome tree."""518 chrome_lkgm = commands.GetChromeLKGM(self._options.chrome_version)519 # We need a full buildspecs manager here as we need an initialized manifest520 # manager with paths to the spec.521 manifest_manager = manifest_version.BuildSpecsManager(522 source_repo=self.repo,523 manifest_repo=self._GetManifestVersionsRepoUrl(read_only=False),524 build_name=self._bot_id,525 incr_type='build',526 force=False,527 branch=self._target_manifest_branch)528 manifest_manager.BootstrapFromVersion(chrome_lkgm)529 return manifest_manager.GetLocalManifest(chrome_lkgm)530class ManifestVersionedSyncStage(SyncStage):531 """Stage that generates a unique manifest file, and sync's to it."""532 manifest_manager = None533 output_manifest_sha1 = False534 def __init__(self, options, build_config):535 # Perform the sync at the end of the stage to the given manifest.536 super(ManifestVersionedSyncStage, self).__init__(options, build_config)537 self.repo = None538 # If a builder pushes changes (even with dryrun mode), we need a writable539 # repository. Otherwise, the push will be rejected by the server.540 self.manifest_repo = self._GetManifestVersionsRepoUrl(read_only=False)541 # 1. If we're uprevving Chrome, Chrome might have changed even if the542 # manifest has not, so we should force a build to double check. This543 # means that we'll create a new manifest, even if there are no changes.544 # 2. If we're running with --debug, we should always run through to545 # completion, so as to ensure a complete test.546 self._force = self._chrome_rev or options.debug547 def HandleSkip(self):548 """Initializes a manifest manager to the specified version if skipped."""549 super(ManifestVersionedSyncStage, self).HandleSkip()550 if self._options.force_version:551 self.Initialize()552 self.ForceVersion(self._options.force_version)553 def ForceVersion(self, version):554 """Creates a manifest manager from given version and returns manifest."""555 return ManifestVersionedSyncStage.manifest_manager.BootstrapFromVersion(556 version)557 def Initialize(self):558 """Initializes a manager that manages manifests for associated stages."""559 increment = ('build' if self._target_manifest_branch == 'master'560 else 'branch')561 dry_run = self._options.debug562 self._InitializeRepo()563 # If chrome_rev is somehow set, fail.564 assert not self._chrome_rev, \565 'chrome_rev is unsupported on release builders.'566 ManifestVersionedSyncStage.manifest_manager = \567 manifest_version.BuildSpecsManager(568 source_repo=self.repo,569 manifest_repo=self.manifest_repo,570 manifest=self._build_config['manifest'],571 build_name=self._bot_id,572 incr_type=increment,573 force=self._force,574 branch=self._target_manifest_branch,575 dry_run=dry_run,576 master=self._build_config['master'])577 def GetNextManifest(self):578 """Uses the initialized manifest manager to get the next manifest."""579 assert self.manifest_manager, \580 'Must run GetStageManager before checkout out build.'581 to_return = self.manifest_manager.GetNextBuildSpec()582 previous_version = self.manifest_manager.GetLatestPassingSpec()583 target_version = self.manifest_manager.current_version584 # Print the Blamelist here.585 url_prefix = 'http://chromeos-images.corp.google.com/diff/report?'586 url = url_prefix + 'from=%s&to=%s' % (previous_version, target_version)587 cros_build_lib.PrintBuildbotLink('Blamelist', url)588 return to_return589 def PerformStage(self):590 self.Initialize()591 if self._options.force_version:592 next_manifest = self.ForceVersion(self._options.force_version)593 else:594 next_manifest = self.GetNextManifest()595 if not next_manifest:596 cros_build_lib.Info('Found no work to do.')597 if ManifestVersionedSyncStage.manifest_manager.DidLastBuildFail():598 raise results_lib.StepFailure('The previous build failed.')599 else:600 sys.exit(0)601 # Log this early on for the release team to grep out before we finish.602 if ManifestVersionedSyncStage.manifest_manager:603 self._Print('\nRELEASETAG: %s\n' % (604 ManifestVersionedSyncStage.manifest_manager.current_version))605 self.ManifestCheckout(next_manifest)606class LKGMCandidateSyncStage(ManifestVersionedSyncStage):607 """Stage that generates a unique manifest file candidate, and sync's to it."""608 sub_manager = None609 def __init__(self, options, build_config):610 super(LKGMCandidateSyncStage, self).__init__(options, build_config)611 # lkgm_manager deals with making sure we're synced to whatever manifest612 # we get back in GetNextManifest so syncing again is redundant.613 self.skip_sync = True614 def _GetInitializedManager(self, internal):615 """Returns an initialized lkgm manager."""616 increment = ('build' if self._target_manifest_branch == 'master'617 else 'branch')618 return lkgm_manager.LKGMManager(619 source_repo=self.repo,620 manifest_repo=cbuildbot_config.GetManifestVersionsRepoUrl(621 internal, read_only=False),622 manifest=self._build_config['manifest'],623 build_name=self._bot_id,624 build_type=self._build_config['build_type'],625 incr_type=increment,626 force=self._force,627 branch=self._target_manifest_branch,628 dry_run=self._options.debug,629 master=self._build_config['master'])630 def Initialize(self):631 """Override: Creates an LKGMManager rather than a ManifestManager."""632 self._InitializeRepo()633 ManifestVersionedSyncStage.manifest_manager = self._GetInitializedManager(634 self.internal)635 if (self._build_config['master'] and636 self._GetSlavesForMaster(self._build_config)):637 assert self.internal, 'Unified masters must use an internal checkout.'638 LKGMCandidateSyncStage.sub_manager = self._GetInitializedManager(False)639 def ForceVersion(self, version):640 manifest = super(LKGMCandidateSyncStage, self).ForceVersion(version)641 if LKGMCandidateSyncStage.sub_manager:642 LKGMCandidateSyncStage.sub_manager.BootstrapFromVersion(version)643 return manifest644 def GetNextManifest(self):645 """Gets the next manifest using LKGM logic."""646 assert self.manifest_manager, \647 'Must run Initialize before we can get a manifest.'648 assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \649 'Manifest manager instantiated with wrong class.'650 if self._build_config['master']:651 manifest = self.manifest_manager.CreateNewCandidate()652 if LKGMCandidateSyncStage.sub_manager:653 LKGMCandidateSyncStage.sub_manager.CreateFromManifest(manifest)654 return manifest655 else:656 return self.manifest_manager.GetLatestCandidate()657class CommitQueueSyncStage(LKGMCandidateSyncStage):658 """Commit Queue Sync stage that handles syncing and applying patches.659 This stage handles syncing to a manifest, passing around that manifest to660 other builders and finding the Gerrit Reviews ready to be committed and661 applying them into its out checkout.662 """663 def __init__(self, options, build_config):664 super(CommitQueueSyncStage, self).__init__(options, build_config)665 # Figure out the builder's name from the buildbot waterfall.666 builder_name = build_config['paladin_builder_name']667 self.builder_name = builder_name if builder_name else build_config['name']668 # The pool of patches to be picked up by the commit queue.669 # - For the master commit queue, it's initialized in GetNextManifest.670 # - For slave commit queues, it's initialized in SetPoolFromManifest.671 #672 # In all cases, the pool is saved to disk, and refreshed after bootstrapping673 # by HandleSkip.674 self.pool = None675 def HandleSkip(self):676 """Handles skip and initializes validation pool from manifest."""677 super(CommitQueueSyncStage, self).HandleSkip()678 filename = self._options.validation_pool679 if filename:680 self.pool = validation_pool.ValidationPool.Load(filename)681 else:682 self.SetPoolFromManifest(self.manifest_manager.GetLocalManifest())683 def ChangeFilter(self, pool, changes, non_manifest_changes):684 # First, look for changes that were tested by the Pre-CQ.685 changes_to_test = []686 for change in changes:687 status = pool.GetPreCQStatus(change)688 if status == manifest_version.BuilderStatus.STATUS_PASSED:689 changes_to_test.append(change)690 # If we only see changes that weren't verified by Pre-CQ, try all of the691 # changes. This ensures that the CQ continues to work even if the Pre-CQ is692 # down.693 if not changes_to_test:694 changes_to_test = changes695 return changes_to_test, non_manifest_changes696 def SetPoolFromManifest(self, manifest):697 """Sets validation pool based on manifest path passed in."""698 # Note that GetNextManifest() calls GetLatestCandidate() in this case,699 # so the repo will already be sync'd appropriately. This means that700 # AcquirePoolFromManifest doesn't need to sync.701 self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest(702 manifest, self._build_config['overlays'], self.repo,703 self._options.buildnumber, self.builder_name,704 self._build_config['master'], self._options.debug)705 def GetNextManifest(self):706 """Gets the next manifest using LKGM logic."""707 assert self.manifest_manager, \708 'Must run Initialize before we can get a manifest.'709 assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \710 'Manifest manager instantiated with wrong class.'711 if self._build_config['master']:712 try:713 # In order to acquire a pool, we need an initialized buildroot.714 if not git.FindRepoDir(self.repo.directory):715 self.repo.Initialize()716 pool = validation_pool.ValidationPool.AcquirePool(717 self._build_config['overlays'], self.repo,718 self._options.buildnumber, self.builder_name,719 self._options.debug, check_tree_open=not self._options.debug,720 changes_query=self._options.cq_gerrit_override,721 change_filter=self.ChangeFilter)722 # We only have work to do if there are changes to try.723 try:724 # Try our best to submit these but may have been overridden and won't725 # let that stop us from continuing the build.726 pool.SubmitNonManifestChanges()727 except validation_pool.FailedToSubmitAllChangesException as e:728 cros_build_lib.Warning(str(e))729 self.pool = pool730 except validation_pool.TreeIsClosedException as e:731 cros_build_lib.Warning(str(e))732 return None733 manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool)734 if LKGMCandidateSyncStage.sub_manager:735 LKGMCandidateSyncStage.sub_manager.CreateFromManifest(manifest)736 return manifest737 else:738 manifest = self.manifest_manager.GetLatestCandidate()739 if manifest:740 self.SetPoolFromManifest(manifest)741 self.pool.ApplyPoolIntoRepo()742 return manifest743 def PerformStage(self):744 """Performs normal stage and prints blamelist at end."""745 if self._options.force_version:746 self.HandleSkip()747 else:748 ManifestVersionedSyncStage.PerformStage(self)749class ManifestVersionedSyncCompletionStage(ForgivingBuilderStage):750 """Stage that records board specific results for a unique manifest file."""751 option_name = 'sync'752 def __init__(self, options, build_config, sync_stage, success):753 super(ManifestVersionedSyncCompletionStage, self).__init__(754 options, build_config)755 self.sync_stage = sync_stage756 self.success = success757 # Message that can be set that well be sent along with the status in758 # UpdateStatus.759 self.message = None760 def PerformStage(self):761 if ManifestVersionedSyncStage.manifest_manager:762 ManifestVersionedSyncStage.manifest_manager.UpdateStatus(763 success=self.success, message=self.message)764class ImportantBuilderFailedException(Exception):765 """Exception thrown when an important build fails to build."""766 pass767class LKGMCandidateSyncCompletionStage(ManifestVersionedSyncCompletionStage):768 """Stage that records whether we passed or failed to build/test manifest."""769 def _GetSlavesStatus(self):770 if self._options.debug:771 # In debug mode, nothing is uploaded to Google Storage, so we bypass772 # the extra hop and just look at what we have locally.773 status = manifest_version.BuilderStatus.GetCompletedStatus(self.success)774 status_obj = manifest_version.BuilderStatus(status, self.message)775 return {self._bot_id: status_obj}776 elif not self._build_config['master']:777 # Slaves only need to look at their own status.778 return ManifestVersionedSyncStage.manifest_manager.GetBuildersStatus(779 [self._bot_id])780 else:781 builders = self._GetSlavesForMaster(self._build_config)782 manager = ManifestVersionedSyncStage.manifest_manager783 sub_manager = LKGMCandidateSyncStage.sub_manager784 if sub_manager:785 public_builders = [b['name'] for b in builders if not b['internal']]786 statuses = sub_manager.GetBuildersStatus(public_builders)787 private_builders = [b['name'] for b in builders if b['internal']]788 statuses.update(manager.GetBuildersStatus(private_builders))789 else:790 statuses = manager.GetBuildersStatus([b['name'] for b in builders])791 return statuses792 def _AbortCQHWTests(self):793 """Abort any HWTests started by the CQ."""794 manifest_manager = ManifestVersionedSyncStage.manifest_manager795 if (cbuildbot_config.IsCQType(self._build_config['build_type']) and796 manifest_manager is not None and797 self._target_manifest_branch == 'master'):798 release_tag = manifest_manager.current_version799 if release_tag and not commands.HaveHWTestsBeenAborted(release_tag):800 commands.AbortHWTests(release_tag, self._options.debug)801 def HandleSuccess(self):802 # We only promote for the pfq, not chrome pfq.803 # TODO(build): Run this logic in debug mode too.804 if (not self._options.debug and805 cbuildbot_config.IsPFQType(self._build_config['build_type']) and806 self._build_config['master'] and807 self._target_manifest_branch == 'master' and808 ManifestVersionedSyncStage.manifest_manager is not None and809 self._build_config['build_type'] != constants.CHROME_PFQ_TYPE):810 ManifestVersionedSyncStage.manifest_manager.PromoteCandidate()811 if LKGMCandidateSyncStage.sub_manager:812 LKGMCandidateSyncStage.sub_manager.PromoteCandidate()813 def HandleValidationFailure(self, failing_statuses):814 cros_build_lib.PrintBuildbotStepWarnings()815 cros_build_lib.Warning('\n'.join([816 'The following builders failed with this manifest:',817 ', '.join(sorted(failing_statuses.keys())),818 'Please check the logs of the failing builders for details.']))819 def HandleValidationTimeout(self, inflight_statuses):820 cros_build_lib.PrintBuildbotStepWarnings()821 cros_build_lib.Warning('\n'.join([822 'The following builders took too long to finish:',823 ', '.join(sorted(inflight_statuses.keys())),824 'Please check the logs of these builders for details.']))825 def PerformStage(self):826 if ManifestVersionedSyncStage.manifest_manager:827 ManifestVersionedSyncStage.manifest_manager.UploadStatus(828 success=self.success, message=self.message)829 if not self.success and self._build_config['important']:830 self._AbortCQHWTests()831 statuses = self._GetSlavesStatus()832 failing_build_dict, inflight_build_dict = {}, {}833 for builder, status in statuses.iteritems():834 if status.Failed():835 failing_build_dict[builder] = status836 elif status.Inflight():837 inflight_build_dict[builder] = status838 if failing_build_dict or inflight_build_dict:839 if failing_build_dict:840 self.HandleValidationFailure(failing_build_dict)841 if inflight_build_dict:842 self.HandleValidationTimeout(inflight_build_dict)843 if failing_build_dict or inflight_build_dict:844 raise results_lib.StepFailure()845 else:846 self.HandleSuccess()847class CommitQueueCompletionStage(LKGMCandidateSyncCompletionStage):848 """Commits or reports errors to CL's that failed to be validated."""849 def HandleSuccess(self):850 if self._build_config['master']:851 self.sync_stage.pool.SubmitPool()852 # After submitting the pool, update the commit hashes for uprevved853 # ebuilds.854 manifest = git.ManifestCheckout.Cached(self._build_root)855 portage_utilities.EBuild.UpdateCommitHashesForChanges(856 self.sync_stage.pool.changes, self._build_root, manifest)857 if cbuildbot_config.IsPFQType(self._build_config['build_type']):858 super(CommitQueueCompletionStage, self).HandleSuccess()859 def HandleValidationFailure(self, failing_statuses):860 """Sends the failure message of all failing builds in one go."""861 super(CommitQueueCompletionStage, self).HandleValidationFailure(862 failing_statuses)863 if self._build_config['master']:864 failing_messages = [x.message for x in failing_statuses.itervalues()]865 self.sync_stage.pool.HandleValidationFailure(failing_messages)866 def HandleValidationTimeout(self, inflight_builders):867 super(CommitQueueCompletionStage, self).HandleValidationTimeout(868 inflight_builders)869 self.sync_stage.pool.HandleValidationTimeout()870 def PerformStage(self):871 if not self.success and self._build_config['important']:872 # This message is sent along with the failed status to the master to873 # indicate a failure.874 self.message = self.sync_stage.pool.GetValidationFailedMessage()875 super(CommitQueueCompletionStage, self).PerformStage()876 if ManifestVersionedSyncStage.manifest_manager:877 ManifestVersionedSyncStage.manifest_manager.UpdateStatus(878 success=self.success, message=self.message)879class PreCQSyncStage(SyncStage):880 """Sync and apply patches to test if they compile."""881 def __init__(self, options, build_config, patches):882 super(PreCQSyncStage, self).__init__(options, build_config)883 # The list of patches to test.884 self.patches = patches885 # The ValidationPool of patches to test. Initialized in PerformStage, and886 # refreshed after bootstrapping by HandleSkip.887 self.pool = None888 def HandleSkip(self):889 """Handles skip and loads validation pool from disk."""890 super(PreCQSyncStage, self).HandleSkip()891 filename = self._options.validation_pool892 if filename:893 self.pool = validation_pool.ValidationPool.Load(filename)894 def PerformStage(self):895 super(PreCQSyncStage, self).PerformStage()896 self.pool = validation_pool.ValidationPool.AcquirePreCQPool(897 self._build_config['overlays'], self._build_root,898 self._options.buildnumber, self._build_config['name'],899 dryrun=self._options.debug_forced, changes=self.patches)900 self.pool.ApplyPoolIntoRepo()901class PreCQCompletionStage(bs.BuilderStage):902 """Reports the status of a trybot run to Google Storage and Gerrit."""903 def __init__(self, options, build_config, sync_stage, success):904 super(PreCQCompletionStage, self).__init__(options, build_config)905 self.sync_stage = sync_stage906 self.success = success907 def PerformStage(self):908 # Update Gerrit and Google Storage with the Pre-CQ status.909 if self.success:910 self.sync_stage.pool.HandlePreCQSuccess()911 else:912 message = self.sync_stage.pool.GetValidationFailedMessage()913 self.sync_stage.pool.HandleValidationFailure([message])914class PreCQLauncherStage(SyncStage):915 """Scans for CLs and automatically launches Pre-CQ jobs to test them."""916 STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT917 STATUS_PASSED = validation_pool.ValidationPool.STATUS_PASSED918 STATUS_FAILED = validation_pool.ValidationPool.STATUS_FAILED919 STATUS_LAUNCHING = validation_pool.ValidationPool.STATUS_LAUNCHING920 STATUS_WAITING = validation_pool.ValidationPool.STATUS_WAITING921 # The number of minutes we allow before considering a launch attempt failed.922 # If this window isn't hit in a given launcher run, the window will start923 # again from scratch in the next run.924 LAUNCH_DELAY = 10925 def __init__(self, options, build_config):926 super(PreCQLauncherStage, self).__init__(options, build_config)927 self.skip_sync = True928 self.launching = {}929 self.retried = set()930 def _HasLaunchTimedOut(self, change):931 """Check whether a given |change| has timed out on its trybot launch.932 Assumes that the change is in the middle of being launched.933 Returns:934 True if the change has timed out. False otherwise.935 """936 diff = datetime.timedelta(minutes=self.LAUNCH_DELAY)937 return datetime.datetime.now() - self.launching[change] > diff938 def GetPreCQStatus(self, pool, changes):939 """Get the Pre-CQ status of a list of changes.940 Args:941 pool: The validation pool.942 changes: Changes to examine.943 Returns:944 busy: The set of CLs that are currently being tested.945 passed: The set of CLs that have been verified.946 """947 busy, passed = set(), set()948 for change in changes:949 status = pool.GetPreCQStatus(change)950 if status != self.STATUS_LAUNCHING:951 # The trybot has finished launching, so we should remove it from our952 # data structures.953 self.launching.pop(change, None)954 if status == self.STATUS_LAUNCHING:955 # The trybot is in the process of launching.956 busy.add(change)957 if change not in self.launching:958 self.launching[change] = datetime.datetime.now()959 elif self._HasLaunchTimedOut(change):960 if change in self.retried:961 msg = 'Failed twice to launch a Pre-CQ trybot for this change.'962 pool.SendNotification(change, '%(details)s', details=msg)963 pool.RemoveCommitReady(change)964 pool.UpdatePreCQStatus(change, self.STATUS_FAILED)965 self.retried.discard(change)966 else:967 # Try the change again.968 self.retried.add(change)969 pool.UpdatePreCQStatus(change, self.STATUS_WAITING)970 elif status == self.STATUS_INFLIGHT:971 # Once a Pre-CQ run actually starts, it'll set the status to972 # STATUS_INFLIGHT.973 busy.add(change)974 elif status == self.STATUS_FAILED:975 # The Pre-CQ run failed for this change. It's possible that we got976 # unlucky and this change was just marked as 'Not Ready' by a bot. To977 # test this, mark the CL as 'waiting' for now. If the CL is still marked978 # as 'Ready' next time we check, we'll know the CL is truly still ready.979 busy.add(change)980 pool.UpdatePreCQStatus(change, self.STATUS_WAITING)981 elif status == self.STATUS_PASSED:982 passed.add(change)983 return busy, passed984 def LaunchTrybot(self, pool, plan):985 """Launch a Pre-CQ run with the provided list of CLs.986 Args:987 plan: The list of patches to test in the Pre-CQ run.988 """989 cmd = ['cbuildbot', '--remote', '--nobootstrap',990 constants.PRE_CQ_BUILDER_NAME]991 if self._options.debug_forced:992 cmd.append('--debug')993 for patch in plan:994 number = cros_patch.FormatGerritNumber(995 patch.gerrit_number, force_internal=patch.internal)996 cmd += ['-g', number]997 cros_build_lib.RunCommand(cmd, cwd=self._build_root)998 for patch in plan:999 if pool.GetPreCQStatus(patch) != self.STATUS_PASSED:1000 pool.UpdatePreCQStatus(patch, self.STATUS_LAUNCHING)1001 def GetDisjointTransactionsToTest(self, pool, changes):1002 """Get the list of disjoint transactions to test.1003 Returns:1004 A list of disjoint transactions to test. Each transaction should be sent1005 to a different Pre-CQ trybot.1006 """1007 busy, passed = self.GetPreCQStatus(pool, changes)1008 # Create a list of disjoint transactions to test.1009 manifest = git.ManifestCheckout.Cached(self._build_root)1010 plans = pool.CreateDisjointTransactions(manifest)1011 for plan in plans:1012 # If any of the CLs in the plan are currently "busy" being tested,1013 # wait until they're done before launching our trybot run. This helps1014 # avoid race conditions.1015 #1016 # Similarly, if all of the CLs in the plan have already been validated,1017 # there's no need to launch a trybot run.1018 plan = set(plan)1019 if plan.issubset(passed):1020 logging.info('CLs already verified: %r', ' '.join(map(str, plan)))1021 elif plan.intersection(busy):1022 logging.info('CLs currently being verified: %r',1023 ' '.join(map(str, plan.intersection(busy))))1024 if plan.difference(busy):1025 logging.info('CLs waiting on verification of dependencies: %r',1026 ' '.join(map(str, plan.difference(busy))))1027 else:1028 yield plan1029 def ProcessChanges(self, pool, changes, _non_manifest_changes):1030 """Process a list of changes that were marked as Ready.1031 From our list of changes that were marked as Ready, we create a1032 list of disjoint transactions and send each one to a separate Pre-CQ1033 trybot.1034 Non-manifest changes are just submitted here because they don't need to be1035 verified by either the Pre-CQ or CQ.1036 """1037 # Submit non-manifest changes if we can.1038 if cros_build_lib.TreeOpen(1039 validation_pool.ValidationPool.STATUS_URL, 0, max_timeout=0):1040 try:1041 pool.SubmitNonManifestChanges(check_tree_open=False)1042 except validation_pool.FailedToSubmitAllChangesException as e:1043 cros_build_lib.Warning(str(e))1044 # Launch trybots for manifest changes.1045 for plan in self.GetDisjointTransactionsToTest(pool, changes):1046 self.LaunchTrybot(pool, plan)1047 # Tell ValidationPool to keep waiting for more changes until we hit1048 # its internal timeout.1049 return [], []1050 def PerformStage(self):1051 # Setup and initialize the repo.1052 super(PreCQLauncherStage, self).PerformStage()1053 # Loop through all of the changes until we hit a timeout.1054 validation_pool.ValidationPool.AcquirePool(1055 self._build_config['overlays'], self.repo,1056 self._options.buildnumber, constants.PRE_CQ_BUILDER_NAME,1057 dryrun=self._options.debug_forced,1058 changes_query=self._options.cq_gerrit_override,1059 check_tree_open=False, change_filter=self.ProcessChanges)1060class RefreshPackageStatusStage(bs.BuilderStage):1061 """Stage for refreshing Portage package status in online spreadsheet."""1062 def PerformStage(self):1063 commands.RefreshPackageStatus(buildroot=self._build_root,1064 boards=self._boards,1065 debug=self._options.debug)1066class InitSDKStage(bs.BuilderStage):1067 """Stage that is responsible for initializing the SDK."""1068 option_name = 'build'1069 def __init__(self, options, build_config):1070 super(InitSDKStage, self).__init__(options, build_config)1071 self._env = {}1072 if self._options.clobber:1073 self._env['IGNORE_PREFLIGHT_BINHOST'] = '1'1074 self._latest_toolchain = (self._build_config['latest_toolchain'] or1075 self._options.latest_toolchain)1076 if self._latest_toolchain and self._build_config['gcc_githash']:1077 self._env['USE'] = 'git_gcc'1078 self._env['GCC_GITHASH'] = self._build_config['gcc_githash']1079 def PerformStage(self):1080 chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)1081 replace = self._build_config['chroot_replace']1082 if os.path.isdir(self._build_root) and not replace:1083 try:1084 commands.RunChrootUpgradeHooks(self._build_root)1085 except results_lib.BuildScriptFailure:1086 cros_build_lib.PrintBuildbotStepText('Replacing broken chroot')1087 cros_build_lib.PrintBuildbotStepWarnings()1088 replace = True1089 if not os.path.isdir(chroot_path) or replace:1090 use_sdk = (self._build_config['use_sdk'] and not self._options.nosdk)1091 commands.MakeChroot(1092 buildroot=self._build_root,1093 replace=replace,1094 use_sdk=use_sdk,1095 chrome_root=self._options.chrome_root,1096 extra_env=self._env)1097class SetupBoardStage(InitSDKStage):1098 """Stage that is responsible for building host pkgs and setting up a board."""1099 option_name = 'build'1100 def __init__(self, options, build_config, boards=None):1101 super(SetupBoardStage, self).__init__(options, build_config)1102 if boards is not None:1103 self._boards = boards1104 def PerformStage(self):1105 # Calculate whether we should use binary packages.1106 usepkg = (self._build_config['usepkg_setup_board'] and1107 not self._latest_toolchain)1108 # We need to run chroot updates on most builders because they uprev after1109 # the InitSDK stage. For the SDK builder, we can skip updates because uprev1110 # is run prior to InitSDK. This is not just an optimization: It helps1111 # workaround http://crbug.com/2255091112 chroot_upgrade = (1113 self._build_config['build_type'] != constants.CHROOT_BUILDER_TYPE)1114 # Iterate through boards to setup.1115 chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)1116 for board_to_build in self._boards:1117 # Only update the board if we need to do so.1118 board_path = os.path.join(chroot_path, 'build', board_to_build)1119 if os.path.isdir(board_path) and not chroot_upgrade:1120 continue1121 commands.SetupBoard(self._build_root,1122 board=board_to_build,1123 usepkg=usepkg,1124 extra_env=self._env,1125 profile=self._options.profile or1126 self._build_config['profile'],1127 chroot_upgrade=chroot_upgrade)1128 chroot_upgrade = False1129 commands.SetSharedUserPassword(1130 self._build_root,1131 password=self._build_config['shared_user_password'])1132class UprevStage(bs.BuilderStage):1133 """Stage that uprevs Chromium OS packages that the builder intends to1134 validate.1135 """1136 option_name = 'uprev'1137 def __init__(self, options, build_config, boards=None, enter_chroot=True):1138 super(UprevStage, self).__init__(options, build_config)1139 self._enter_chroot = enter_chroot1140 if boards is not None:1141 self._boards = boards1142 def PerformStage(self):1143 # Perform other uprevs.1144 if self._build_config['uprev']:1145 overlays, _ = self._ExtractOverlays()1146 commands.UprevPackages(self._build_root,1147 self._boards,1148 overlays,1149 enter_chroot=self._enter_chroot)1150class SyncChromeStage(bs.BuilderStage):1151 """Stage that syncs Chrome sources if needed."""1152 option_name = 'managed_chrome'1153 def __init__(self, options, build_config):1154 super(SyncChromeStage, self).__init__(options, build_config)1155 # PerformStage() will fill this out for us.1156 self.chrome_version = None1157 def PerformStage(self):1158 # Perform chrome uprev.1159 chrome_atom_to_build = None1160 if self._chrome_rev:1161 chrome_atom_to_build = commands.MarkChromeAsStable(1162 self._build_root, self._target_manifest_branch,1163 self._chrome_rev, self._boards,1164 chrome_version=self._options.chrome_version)1165 kwargs = {}1166 if self._chrome_rev == constants.CHROME_REV_SPEC:1167 kwargs['revision'] = self._options.chrome_version1168 cpv = None1169 cros_build_lib.PrintBuildbotStepText('revision %s' % kwargs['revision'])1170 self.chrome_version = self._options.chrome_version1171 else:1172 cpv = portage_utilities.BestVisible(constants.CHROME_CP,1173 buildroot=self._build_root)1174 kwargs['tag'] = cpv.version_no_rev.partition('_')[0]1175 cros_build_lib.PrintBuildbotStepText('tag %s' % kwargs['tag'])1176 self.chrome_version = kwargs['tag']1177 useflags = self._build_config['useflags'] or []1178 commands.SyncChrome(self._build_root, self._options.chrome_root, useflags,1179 **kwargs)1180 if (self._chrome_rev and not chrome_atom_to_build and1181 self._options.buildbot and1182 self._build_config['build_type'] == constants.CHROME_PFQ_TYPE):1183 cros_build_lib.Info('Chrome already uprevved')1184 sys.exit(0)1185class PatchChromeStage(bs.BuilderStage):1186 """Stage that applies Chrome patches if needed."""1187 option_name = 'rietveld_patches'1188 def PerformStage(self):1189 for patch in ' '.join(self._options.rietveld_patches).split():1190 patch, colon, subdir = patch.partition(':')1191 if not colon:1192 subdir = 'src'1193 commands.PatchChrome(self._options.chrome_root, patch, subdir)1194class BuildPackagesStage(ArchivingStage):1195 """Build Chromium OS packages."""1196 option_name = 'build'1197 def __init__(self, options, build_config, board, archive_stage,1198 pgo_generate=False, pgo_use=False):1199 useflags = build_config['useflags'][:] if build_config['useflags'] else []1200 self._pgo_generate, self._pgo_use = pgo_generate, pgo_use1201 suffix = None1202 assert not pgo_generate or not pgo_use1203 if pgo_generate:1204 suffix = ' [%s]' % constants.USE_PGO_GENERATE1205 useflags.append(constants.USE_PGO_GENERATE)1206 elif pgo_use:1207 suffix = ' [%s]' % constants.USE_PGO_USE1208 useflags.append(constants.USE_PGO_USE)1209 super(BuildPackagesStage, self).__init__(options, build_config, board,1210 archive_stage, suffix=suffix)1211 self._env = {}1212 if useflags:1213 self._env['USE'] = ' '.join(useflags)1214 if self._options.chrome_root:1215 self._env['CHROME_ORIGIN'] = 'LOCAL_SOURCE'1216 if self._options.clobber:1217 self._env['IGNORE_PREFLIGHT_BINHOST'] = '1'1218 self._build_autotest = (self._build_config['build_tests'] and1219 self._options.tests)1220 def _GetArchitectures(self):1221 """Get the list of architectures built by this builder."""1222 return set(self._GetPortageEnvVar('ARCH', b) for b in self._boards)1223 def PerformStage(self):1224 # Wait for PGO data to be ready if needed.1225 if self._pgo_use:1226 cpv = portage_utilities.BestVisible(constants.CHROME_CP,1227 buildroot=self._build_root)1228 commands.WaitForPGOData(self._GetArchitectures(), cpv)1229 commands.Build(self._build_root,1230 self._current_board,1231 build_autotest=self._build_autotest,1232 usepkg=self._build_config['usepkg_build_packages'],1233 nowithdebug=self._build_config['nowithdebug'],1234 packages=self._build_config['packages'],1235 skip_chroot_upgrade=True,1236 chrome_root=self._options.chrome_root,1237 extra_env=self._env)1238class BuildImageStage(BuildPackagesStage):1239 """Build standard Chromium OS images."""1240 option_name = 'build'1241 def _BuildImages(self):1242 # We only build base, dev, and test images from this stage.1243 if self._pgo_generate:1244 images_can_build = set(['test'])1245 else:1246 images_can_build = set(['base', 'dev', 'test'])1247 images_to_build = set(self._build_config['images']).intersection(1248 images_can_build)1249 version = self.archive_stage.release_tag1250 disk_layout = self._build_config['disk_layout']1251 if self._pgo_generate:1252 disk_layout = constants.PGO_GENERATE_DISK_LAYOUT1253 if version:1254 version = '%s-pgo-generate' % version1255 rootfs_verification = self._build_config['rootfs_verification']1256 commands.BuildImage(self._build_root,1257 self._current_board,1258 sorted(images_to_build),1259 rootfs_verification=rootfs_verification,1260 version=version,1261 disk_layout=disk_layout,1262 extra_env=self._env)1263 # Update link to latest image.1264 latest_image = os.readlink(self.GetImageDirSymlink('latest'))1265 cbuildbot_image_link = self.GetImageDirSymlink()1266 if os.path.lexists(cbuildbot_image_link):1267 os.remove(cbuildbot_image_link)1268 os.symlink(latest_image, cbuildbot_image_link)1269 parallel.RunParallelSteps(1270 [self._BuildVMImage, self.ArchivePayloads,1271 lambda: self._GenerateAuZip(cbuildbot_image_link)])1272 def _BuildVMImage(self):1273 if self._build_config['vm_tests'] and not self._pgo_generate:1274 commands.BuildVMImageForTesting(1275 self._build_root,1276 self._current_board,1277 disk_layout=self._build_config['disk_vm_layout'],1278 extra_env=self._env)1279 def ArchivePayloads(self):1280 """Archives update payloads when they are ready."""1281 with osutils.TempDir(prefix='cbuildbot-payloads') as tempdir:1282 with self.ArtifactUploader() as queue:1283 if self._build_config['upload_hw_test_artifacts']:1284 image_path = os.path.join(self.GetImageDirSymlink(),1285 'chromiumos_test_image.bin')1286 # For non release builds, we are only interested in generating1287 # payloads for the purpose of imaging machines. This means we1288 # shouldn't generate delta payloads for n-1->n testing.1289 # TODO: Add a config flag for generating delta payloads instead.1290 if (self._build_config['build_type'] == constants.CANARY_TYPE and1291 not self._pgo_generate):1292 commands.GenerateNPlus1Payloads(1293 self._build_root, self.bot_archive_root, image_path, tempdir)1294 else:1295 commands.GenerateFullPayload(self._build_root, image_path, tempdir)1296 for payload in os.listdir(tempdir):1297 queue.put([os.path.join(tempdir, payload)])1298 def _GenerateAuZip(self, image_dir):1299 """Create au-generator.zip."""1300 if not self._pgo_generate:1301 commands.GenerateAuZip(self._build_root,1302 image_dir,1303 extra_env=self._env)1304 def _BuildAutotestTarballs(self):1305 with osutils.TempDir(prefix='cbuildbot-autotest') as tempdir:1306 with self.ArtifactUploader() as queue:1307 cwd = os.path.join(self._build_root, 'chroot', 'build',1308 self._current_board, 'usr', 'local')1309 # Find the control files in autotest/1310 control_files = commands.FindFilesWithPattern(1311 'control*', target='autotest', cwd=cwd)1312 # Tar the control files and the packages.1313 autotest_tarball = os.path.join(tempdir, 'autotest.tar')1314 input_list = control_files + ['autotest/packages']1315 commands.BuildTarball(self._build_root, input_list, autotest_tarball,1316 cwd=cwd, compressed=False)1317 queue.put([autotest_tarball])1318 # Tar up the test suites.1319 test_suites_tarball = os.path.join(tempdir, 'test_suites.tar.bz2')1320 commands.BuildTarball(self._build_root, ['autotest/test_suites'],1321 test_suites_tarball, cwd=cwd)1322 queue.put([test_suites_tarball])1323 def PerformStage(self):1324 # Build images and autotest tarball in parallel.1325 steps = []1326 if (self._build_config['upload_hw_test_artifacts'] or1327 self._build_config['archive_build_debug']) and self._build_autotest:1328 steps.append(self._BuildAutotestTarballs)1329 if self._build_config['images']:1330 steps.append(self._BuildImages)1331 parallel.RunParallelSteps(steps)1332class SignerTestStage(ArchivingStage):1333 """Run signer related tests."""1334 option_name = 'tests'1335 config_name = 'signer_tests'1336 # If the signer tests take longer than 30 minutes, abort. They usually take1337 # five minutes to run.1338 SIGNER_TEST_TIMEOUT = 18001339 def PerformStage(self):1340 if not self.archive_stage.WaitForRecoveryImage():1341 raise InvalidTestConditionException('Missing recovery image.')1342 with cros_build_lib.SubCommandTimeout(self.SIGNER_TEST_TIMEOUT):1343 commands.RunSignerTests(self._build_root, self._current_board)1344class UnitTestStage(BoardSpecificBuilderStage):1345 """Run unit tests."""1346 option_name = 'tests'1347 config_name = 'unittests'1348 # If the unit tests take longer than 70 minutes, abort. They usually take1349 # ten minutes to run.1350 #1351 # If the processes hang, parallel_emerge will print a status report after 601352 # minutes, so we picked 70 minutes because it gives us a little buffer time.1353 UNIT_TEST_TIMEOUT = 70 * 601354 def PerformStage(self):1355 with cros_build_lib.SubCommandTimeout(self.UNIT_TEST_TIMEOUT):1356 commands.RunUnitTests(self._build_root,1357 self._current_board,1358 full=(not self._build_config['quick_unit']),1359 nowithdebug=self._build_config['nowithdebug'],1360 blacklist=self._build_config['unittest_blacklist'])1361class VMTestStage(ArchivingStage):1362 """Run autotests in a virtual machine."""1363 option_name = 'tests'1364 config_name = 'vm_tests'1365 def _ArchiveTestResults(self, test_results_dir):1366 """Archives test results to Google Storage."""1367 test_tarball = commands.ArchiveTestResults(1368 self._build_root, test_results_dir, prefix='')1369 # Wait for breakpad symbols.1370 got_symbols = self.archive_stage.WaitForBreakpadSymbols()1371 filenames = commands.GenerateStackTraces(1372 self._build_root, self._current_board, test_tarball, self.archive_path,1373 got_symbols)1374 filenames.append(commands.ArchiveFile(test_tarball, self.archive_path))1375 cros_build_lib.Info('Uploading artifacts to Google Storage...')1376 with self.ArtifactUploader(archive=False, strict=False) as queue:1377 for filename in filenames:1378 queue.put([filename])1379 prefix = 'crash: ' if filename.endswith('.dmp.txt') else ''1380 self.PrintDownloadLink(filename, prefix)1381 def PerformStage(self):1382 # These directories are used later to archive test artifacts.1383 test_results_dir = commands.CreateTestRoot(self._build_root)1384 try:1385 test_type = self._build_config['vm_tests']1386 commands.RunTestSuite(self._build_root,1387 self._current_board,1388 self.GetImageDirSymlink(),1389 os.path.join(test_results_dir,1390 'test_harness'),1391 test_type=test_type,1392 whitelist_chrome_crashes=self._chrome_rev is None,1393 archive_dir=self.bot_archive_root)1394 if self._build_config['build_type'] == constants.CANARY_TYPE:1395 commands.RunDevModeTest(1396 self._build_root, self._current_board, self.GetImageDirSymlink())1397 except Exception:1398 cros_build_lib.Error(_VM_TEST_ERROR_MSG)1399 raise1400 finally:1401 self._ArchiveTestResults(test_results_dir)1402class TestTimeoutException(Exception):1403 """Raised when a critical test times out."""1404 pass1405class InvalidTestConditionException(Exception):1406 """Raised when pre-conditions for a test aren't met."""1407 pass1408class HWTestStage(ArchivingStage):1409 """Stage that runs tests in the Autotest lab."""1410 option_name = 'tests'1411 config_name = 'hw_tests'1412 PERF_RESULTS_EXTENSION = 'results'1413 def __init__(self, options, build_config, board, archive_stage, suite_config):1414 super(HWTestStage, self).__init__(options, build_config, board,1415 archive_stage,1416 suffix=' [%s]' % suite_config.suite)1417 self.suite_config = suite_config1418 self.wait_for_results = True1419 def _PrintFile(self, filename):1420 with open(filename) as f:1421 print f.read()1422 def _SendPerfResults(self):1423 """Sends the perf results from the test to the perf dashboard."""1424 result_file_name = '%s.%s' % (self.suite_config.suite,1425 HWTestStage.PERF_RESULTS_EXTENSION)1426 gs_results_file = '/'.join([self.upload_url, result_file_name])1427 gs_context = gs.GSContext()1428 gs_context.Copy(gs_results_file, self._options.log_dir)1429 # Prints out the actual result from gs_context.Copy.1430 logging.info('Copy of %s completed. Printing below:', result_file_name)1431 self._PrintFile(os.path.join(self._options.log_dir, result_file_name))1432 def _CheckAborted(self):1433 aborted = (self.archive_stage.release_tag and1434 commands.HaveHWTestsBeenAborted(self.archive_stage.release_tag))1435 if aborted:1436 cros_build_lib.PrintBuildbotStepText('aborted')1437 cros_build_lib.Warning('HWTests aborted')1438 return aborted1439 # Disable complaint about calling _HandleStageException.1440 # pylint: disable=W02121441 def _HandleStageException(self, exception):1442 """Override and don't set status to FAIL but FORGIVEN instead."""1443 # 2 for warnings returned by run_suite.py, or CLIENT_HTTP_CODE error1444 # returned by autotest_rpc_client.py. It is the former that we care about.1445 # 11, 12, 13 for cases when rpc is down, see autotest_rpc_errors.py.1446 codes_handled_as_warning = (2, 11, 12, 13)1447 if self.suite_config.critical:1448 return super(HWTestStage, self)._HandleStageException(exception)1449 is_lab_down = (isinstance(exception, lab_status.LabIsDownException) or1450 isinstance(exception, lab_status.BoardIsDisabledException))1451 is_warning_code = (isinstance(exception, cros_build_lib.RunCommandError) and1452 exception.result.returncode in codes_handled_as_warning)1453 if is_lab_down or is_warning_code or self._CheckAborted():1454 return self._HandleExceptionAsWarning(exception)1455 else:1456 return super(HWTestStage, self)._HandleStageException(exception)1457 def DealWithTimeout(self, exception):1458 if not self.suite_config.critical and not self.suite_config.fatal_timeouts:1459 return self._HandleExceptionAsWarning(exception)1460 return super(HWTestStage, self)._HandleStageException(exception)1461 def PerformStage(self):1462 if self._CheckAborted():1463 cros_build_lib.Info('Skipping HWTests as they have been aborted.')1464 return1465 build = '/'.join([self._bot_id, self.version])1466 if self._options.remote_trybot and self._options.hwtest:1467 debug = self._options.debug_forced1468 else:1469 debug = self._options.debug1470 try:1471 lab_status.CheckLabStatus(self._current_board)1472 with cros_build_lib.SubCommandTimeout(self.suite_config.timeout):1473 commands.RunHWTestSuite(build,1474 self.suite_config.suite,1475 self._current_board,1476 self.suite_config.pool,1477 self.suite_config.num,1478 self.suite_config.file_bugs,1479 self.wait_for_results,1480 debug)1481 if self.suite_config.copy_perf_results:1482 self._SendPerfResults()1483 except cros_build_lib.TimeoutError as exception:1484 return self.DealWithTimeout(exception)1485class AUTestStage(HWTestStage):1486 """Stage for au hw test suites that requires special pre-processing."""1487 def PerformStage(self):1488 """Wait for payloads to be staged and uploads its au control files."""1489 with osutils.TempDir() as tempdir:1490 tarball = commands.BuildAUTestTarball(1491 self._build_root, self._current_board, tempdir,1492 self.version, self.upload_url)1493 self.UploadArtifact(tarball)1494 super(AUTestStage, self).PerformStage()1495class ASyncHWTestStage(HWTestStage, ForgivingBuilderStage):1496 """Stage that fires and forgets hw test suites to the Autotest lab."""1497 def __init__(self, *args, **dargs):1498 super(ASyncHWTestStage, self).__init__(self, *args, **dargs)1499 self.wait_for_results = False1500class SDKPackageStage(bs.BuilderStage):1501 """Stage that performs preparing and packaging SDK files"""1502 # Version of the Manifest file being generated. Should be incremented for1503 # Major format changes.1504 MANIFEST_VERSION = '1'1505 _EXCLUDED_PATHS = ('usr/lib/debug', 'usr/local/autotest', 'packages', 'tmp')1506 def PerformStage(self):1507 tarball_name = 'built-sdk.tar.xz'1508 tarball_location = os.path.join(self._build_root, tarball_name)1509 chroot_location = os.path.join(self._build_root,1510 constants.DEFAULT_CHROOT_DIR)1511 board_location = os.path.join(chroot_location, 'build/amd64-host')1512 manifest_location = os.path.join(self._build_root,1513 '%s.Manifest' % tarball_name)1514 # Create a tarball of the latest SDK.1515 self.CreateSDKTarball(chroot_location, board_location, tarball_location)1516 # Create a package manifest for the tarball.1517 self.CreateManifestFromSDK(board_location, manifest_location)1518 # Create toolchain packages.1519 self.CreateRedistributableToolchains(chroot_location)1520 # Make sure the regular user has the permission to read.1521 cmd = ['chmod', 'a+r', tarball_location]1522 cros_build_lib.SudoRunCommand(cmd, cwd=board_location)1523 def CreateRedistributableToolchains(self, chroot_location):1524 osutils.RmDir(os.path.join(chroot_location,1525 constants.SDK_TOOLCHAINS_OUTPUT),1526 ignore_missing=True)1527 cros_build_lib.RunCommand(1528 ['cros_setup_toolchains', '--create-packages',1529 '--output-dir', os.path.join('/', constants.SDK_TOOLCHAINS_OUTPUT)],1530 enter_chroot=True)1531 def CreateSDKTarball(self, _chroot, sdk_path, dest_tarball):1532 """Creates an SDK tarball from a given source chroot.1533 Args:1534 chroot: A chroot used for finding compression tool.1535 sdk_path: Path to the root of newly generated SDK image.1536 dest_tarball: Path of the tarball that should be created.1537 """1538 # TODO(zbehan): We cannot use xz from the chroot unless it's1539 # statically linked.1540 extra_args = ['--exclude=%s/*' % path for path in self._EXCLUDED_PATHS]1541 # Options for maximum compression.1542 extra_env = { 'XZ_OPT' : '-e9' }1543 cros_build_lib.CreateTarball(1544 dest_tarball, sdk_path, sudo=True, extra_args=extra_args,1545 extra_env=extra_env)1546 def CreateManifestFromSDK(self, sdk_path, dest_manifest):1547 """Creates a manifest from a given source chroot.1548 Args:1549 sdk_path: Path to the root of the SDK to describe.1550 dest_manifest: Path to the manifest that should be generated.1551 """1552 package_data = {}1553 for key, version in portage_utilities.ListInstalledPackages(sdk_path):1554 package_data.setdefault(key, []).append((version, {}))1555 self._WriteManifest(package_data, dest_manifest)1556 def _WriteManifest(self, data, manifest):1557 """Encode manifest into a json file."""1558 json_input = dict(version=self.MANIFEST_VERSION, packages=data)1559 osutils.WriteFile(manifest, json.dumps(json_input))1560class SDKTestStage(bs.BuilderStage):1561 """Stage that performs testing an SDK created in a previous stage"""1562 option_name = 'tests'1563 def PerformStage(self):1564 tarball_location = os.path.join(self._build_root, 'built-sdk.tar.xz')1565 new_chroot_cmd = ['cros_sdk', '--chroot', 'new-sdk-chroot']1566 # Build a new SDK using the provided tarball.1567 cmd = new_chroot_cmd + ['--download', '--replace', '--nousepkg',1568 '--url', 'file://' + tarball_location]1569 cros_build_lib.RunCommand(cmd, cwd=self._build_root)1570 for board in self._boards:1571 cros_build_lib.PrintBuildbotStepText(board)1572 cmd = new_chroot_cmd + ['--', './setup_board',1573 '--board', board, '--skip_chroot_upgrade']1574 cros_build_lib.RunCommand(cmd, cwd=self._build_root)1575 cmd = new_chroot_cmd + ['--', './build_packages',1576 '--board', board, '--nousepkg', '--skip_chroot_upgrade']1577 cros_build_lib.RunCommand(cmd, cwd=self._build_root)1578class NothingToArchiveException(Exception):1579 """Thrown if ArchiveStage found nothing to archive."""1580 def __init__(self, message='No images found to archive.'):1581 super(NothingToArchiveException, self).__init__(message)1582class ArchiveStage(ArchivingStage):1583 """Archives build and test artifacts for developer consumption.1584 Attributes:1585 release_tag: The release tag. E.g. 2981.0.01586 version: The full version string, including the milestone.1587 E.g. R26-2981.0.0-b1231588 """1589 option_name = 'archive'1590 # This stage is intended to run in the background, in parallel with tests.1591 def __init__(self, options, build_config, board, release_tag,1592 chrome_version=None):1593 self.release_tag = release_tag1594 self._chrome_version = chrome_version1595 super(ArchiveStage, self).__init__(options, build_config, board, self)1596 self._breakpad_symbols_queue = multiprocessing.Queue()1597 self._recovery_image_status_queue = multiprocessing.Queue()1598 self._release_upload_queue = multiprocessing.Queue()1599 self._upload_queue = multiprocessing.Queue()1600 self._upload_symbols_queue = multiprocessing.Queue()1601 self._pkg_dir = os.path.join(1602 self._build_root, constants.DEFAULT_CHROOT_DIR,1603 'build', self._current_board, 'var', 'db', 'pkg')1604 # Setup the archive path. This is used by other stages.1605 self._SetupArchivePath()1606 @cros_build_lib.MemoizedSingleCall1607 def GetVersionInfo(self):1608 """Helper for picking apart various version bits"""1609 return manifest_version.VersionInfo.from_repo(self._build_root)1610 @cros_build_lib.MemoizedSingleCall1611 def GetVersion(self):1612 """Helper for calculating self.version."""1613 verinfo = self.GetVersionInfo()1614 calc_version = self.release_tag or verinfo.VersionString()1615 calc_version = 'R%s-%s' % (verinfo.chrome_branch, calc_version)1616 # Non-versioned builds need the build number to uniquify the image.1617 if not self.release_tag:1618 calc_version += '-b%s' % self._options.buildnumber1619 return calc_version1620 def WaitForRecoveryImage(self):1621 """Wait until artifacts needed by SignerTest stage are created.1622 Returns:1623 True if artifacts created successfully.1624 False otherwise.1625 """1626 cros_build_lib.Info('Waiting for recovery image...')1627 status = self._recovery_image_status_queue.get()1628 # Put the status back so other SignerTestStage instances don't starve.1629 self._recovery_image_status_queue.put(status)1630 return status1631 def _BreakpadSymbolsGenerated(self, success):1632 """Signal that breakpad symbols have been generated.1633 Arguments:1634 success: True to indicate the symbols were generated, else False.1635 """1636 self._breakpad_symbols_queue.put(success)1637 def WaitForBreakpadSymbols(self):1638 """Wait for the breakpad symbols to be generated.1639 Returns:1640 True if the breakpad symbols were generated.1641 False if the breakpad symbols were not generated within 20 mins.1642 """1643 success = False1644 cros_build_lib.Info('Waiting for breakpad symbols...')1645 try:1646 # TODO: Clean this up so that we no longer rely on a timeout1647 success = self._breakpad_symbols_queue.get(True, 1200)1648 except Queue.Empty:1649 cros_build_lib.Warning(1650 'Breakpad symbols were not generated within timeout period.')1651 return success1652 def _SetupArchivePath(self):1653 """Create a fresh directory for archiving a build."""1654 if self._options.buildbot:1655 # Buildbot: Clear out any leftover build artifacts, if present.1656 shutil.rmtree(self.archive_path, ignore_errors=True)1657 else:1658 # Clear the list of uploaded file if it exists1659 osutils.SafeUnlink(os.path.join(self.archive_path,1660 commands.UPLOADED_LIST_FILENAME))1661 os.makedirs(self.archive_path)1662 def RefreshMetadata(self, stage, final_status=None):1663 """Create a JSON of various metadata describing this build."""1664 config = self._build_config1665 acl = None if self._build_config['internal'] else 'public-read'1666 start_time = results_lib.Results.start_time1667 current_time = datetime.datetime.now()1668 start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)1669 current_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=current_time)1670 duration = '%s' % (current_time - start_time,)1671 sdk_verinfo = cros_build_lib.LoadKeyValueFile(1672 os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),1673 ignore_missing=True)1674 verinfo = self.GetVersionInfo()1675 metadata = {1676 # Version of the metadata format.1677 'metadata-version': '2',1678 # Data for this build.1679 'bot-config': config['name'],1680 'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True),1681 'boards': config['boards'],1682 'build-number': self._options.buildnumber,1683 'builder-name': os.environ.get('BUILDBOT_BUILDERNAME'),1684 'status': {1685 'current-time': current_time_stamp,1686 'status': final_status if final_status else 'running',1687 'summary': stage,1688 },1689 'time': {1690 'start': start_time_stamp,1691 'finish': current_time_stamp if final_status else '',1692 'duration': duration,1693 },1694 'version': {1695 'chrome': self._chrome_version,1696 'full': self.version,1697 'milestone': verinfo.chrome_branch,1698 'platform': self.release_tag or verinfo.VersionString(),1699 },1700 # Data for the toolchain used.1701 'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),1702 'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'),1703 }1704 if len(config['boards']) == 1:1705 toolchains = toolchain.GetToolchainsForBoard(config['boards'][0])1706 metadata['toolchain-tuple'] = (1707 toolchain.FilterToolchains(toolchains, 'default', True).keys() +1708 toolchain.FilterToolchains(toolchains, 'default', False).keys())1709 metadata['results'] = []1710 for name, result, description, run_time in results_lib.Results.Get():1711 timestr = datetime.timedelta(seconds=math.ceil(run_time))1712 if result in results_lib.Results.NON_FAILURE_TYPES:1713 status = 'passed'1714 else:1715 status = 'failed'1716 metadata['results'].append({1717 'name': name,1718 'status': status,1719 # The result might be a custom exception.1720 'summary': str(result),1721 'duration': '%s' % timestr,1722 'description': description,1723 'log': self.ConstructDashboardURL(stage=name),1724 })1725 metadata_json = os.path.join(self.archive_path, constants.METADATA_JSON)1726 # Stages may run in parallel, so we have to do atomic updates on this.1727 osutils.WriteFile(metadata_json, json.dumps(metadata), atomic=True)1728 commands.UploadArchivedFile(self.archive_path, self.upload_url,1729 os.path.basename(metadata_json),1730 debug=self.debug, acl=acl,1731 update_list=bool(final_status))1732 @staticmethod1733 def _SingleMatchGlob(path_pattern):1734 """Returns the last match (after sort) if multiple found."""1735 files = glob.glob(path_pattern)1736 files.sort()1737 if not files:1738 raise NothingToArchiveException('No %s found!' % path_pattern)1739 elif len(files) > 1:1740 cros_build_lib.PrintBuildbotStepWarnings()1741 cros_build_lib.Warning('Expecting one result for %s package, but '1742 'found multiple.', path_pattern)1743 return files[-1]1744 def ArchiveStrippedChrome(self):1745 """Generate and upload stripped Chrome package."""1746 cmd = ['strip_package', '--board', self._current_board,1747 constants.CHROME_PN]1748 cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True)1749 pkg_dir = os.path.join(1750 self._build_root, constants.DEFAULT_CHROOT_DIR, 'build',1751 self._current_board, 'stripped-packages')1752 chrome_tarball = self._SingleMatchGlob(1753 os.path.join(pkg_dir, constants.CHROME_CP) + '-*')1754 filename = os.path.basename(chrome_tarball)1755 os.link(chrome_tarball, os.path.join(self.archive_path, filename))1756 self._upload_queue.put([filename])1757 def BuildAndArchiveChromeSysroot(self):1758 """Generate and upload sysroot for building Chrome."""1759 assert self.archive_path.startswith(self._build_root)1760 in_chroot_path = git.ReinterpretPathForChroot(self.archive_path)1761 cmd = ['cros_generate_sysroot', '--out-dir', in_chroot_path, '--board',1762 self._current_board, '--package', constants.CHROME_CP]1763 cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True)1764 self._upload_queue.put([constants.CHROME_SYSROOT_TAR])1765 def ArchiveChromeEbuildEnv(self):1766 """Generate and upload Chrome ebuild environment."""1767 chrome_dir = self._SingleMatchGlob(1768 os.path.join(self._pkg_dir, constants.CHROME_CP) + '-*')1769 env_bzip = os.path.join(chrome_dir, 'environment.bz2')1770 with osutils.TempDir() as tempdir:1771 # Convert from bzip2 to tar format.1772 bzip2 = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)1773 cros_build_lib.RunCommand(1774 [bzip2, '-d', env_bzip, '-c'],1775 log_stdout_to_file=os.path.join(tempdir, constants.CHROME_ENV_FILE))1776 env_tar = os.path.join(self.archive_path, constants.CHROME_ENV_TAR)1777 cros_build_lib.CreateTarball(env_tar, tempdir)1778 self._upload_queue.put([os.path.basename(env_tar)])1779 def PerformStage(self):1780 buildroot = self._build_root1781 config = self._build_config1782 board = self._current_board1783 debug = self.debug1784 upload_url = self.upload_url1785 archive_path = self.archive_path1786 image_dir = self.GetImageDirSymlink()1787 extra_env = {}1788 if config['useflags']:1789 extra_env['USE'] = ' '.join(config['useflags'])1790 if not archive_path:1791 raise NothingToArchiveException()1792 # The following functions are run in parallel (except where indicated1793 # otherwise)1794 # \- BuildAndArchiveArtifacts1795 # \- ArchiveReleaseArtifacts1796 # \- ArchiveDebugSymbols1797 # \- ArchiveFirmwareImages1798 # \- BuildAndArchiveAllImages1799 # (builds recovery image first, then launches functions below)1800 # \- BuildAndArchiveFactoryImages1801 # \- ArchiveStandaloneTarballs1802 # \- ArchiveStandaloneTarball1803 # \- ArchiveZipFiles1804 # \- ArchiveHWQual1805 # \- PushImage (blocks on BuildAndArchiveAllImages)1806 # \- ArchiveStrippedChrome1807 # \- BuildAndArchiveChromeSysroot1808 # \- ArchiveChromeEbuildEnv1809 # \- ArchiveImageScripts1810 def ArchiveDebugSymbols():1811 """Generate debug symbols and upload debug.tgz."""1812 if config['archive_build_debug'] or config['vm_tests']:1813 success = False1814 try:1815 commands.GenerateBreakpadSymbols(buildroot, board)1816 success = True1817 finally:1818 self._BreakpadSymbolsGenerated(success)1819 # Kick off the symbol upload process in the background.1820 if config['upload_symbols']:1821 self._upload_symbols_queue.put([])1822 # Generate and upload tarball.1823 filename = commands.GenerateDebugTarball(1824 buildroot, board, archive_path, config['archive_build_debug'])1825 self._release_upload_queue.put([filename])1826 else:1827 self._BreakpadSymbolsGenerated(False)1828 def UploadSymbols():1829 """Upload generated debug symbols."""1830 if not debug:1831 commands.UploadSymbols(buildroot, board, config['chromeos_official'])1832 def BuildAndArchiveFactoryImages():1833 """Build and archive the factory zip file.1834 The factory zip file consists of the factory test image and the factory1835 install image. Both are built here.1836 """1837 # Build factory test image and create symlink to it.1838 factory_test_symlink = None1839 if 'factory_test' in config['images']:1840 alias = commands.BuildFactoryTestImage(buildroot, board, extra_env)1841 factory_test_symlink = self.GetImageDirSymlink(alias)1842 # Build factory install image and create a symlink to it.1843 factory_install_symlink = None1844 if 'factory_install' in config['images']:1845 alias = commands.BuildFactoryInstallImage(buildroot, board, extra_env)1846 factory_install_symlink = self.GetImageDirSymlink(alias)1847 if config['factory_install_netboot']:1848 commands.MakeNetboot(buildroot, board, factory_install_symlink)1849 # Build and upload factory zip.1850 if factory_install_symlink and factory_test_symlink:1851 image_root = os.path.dirname(factory_install_symlink)1852 filename = commands.BuildFactoryZip(1853 buildroot, board, archive_path, image_root)1854 self._release_upload_queue.put([filename])1855 def ArchiveStandaloneTarball(image_file):1856 """Build and upload a single tarball."""1857 self._release_upload_queue.put([commands.BuildStandaloneImageTarball(1858 archive_path, image_file)])1859 def ArchiveStandaloneTarballs():1860 """Build and upload standalone tarballs for each image."""1861 if config['upload_standalone_images']:1862 inputs = []1863 for image_file in glob.glob(os.path.join(image_dir, '*.bin')):1864 if os.path.basename(image_file) != 'chromiumos_qemu_image.bin':1865 inputs.append([image_file])1866 parallel.RunTasksInProcessPool(ArchiveStandaloneTarball, inputs)1867 def ArchiveZipFiles():1868 """Build and archive zip files.1869 This includes:1870 - image.zip (all images in one big zip file)1871 - the au-generator.zip used for update payload generation.1872 """1873 # Zip up everything in the image directory.1874 image_zip = commands.BuildImageZip(archive_path, image_dir)1875 self._release_upload_queue.put([image_zip])1876 # Archive au-generator.zip.1877 filename = 'au-generator.zip'1878 shutil.copy(os.path.join(image_dir, filename), archive_path)1879 self._release_upload_queue.put([filename])1880 def ArchiveHWQual():1881 """Build and archive the HWQual images."""1882 # TODO(petermayo): This logic needs to be exported from the BuildTargets1883 # stage rather than copied/re-evaluated here.1884 autotest_built = config['build_tests'] and self._options.tests and (1885 config['upload_hw_test_artifacts'] or config['archive_build_debug'])1886 if config['hwqual'] and autotest_built:1887 # Build the full autotest tarball for hwqual image. We don't upload it,1888 # as it's fairly large and only needed by the hwqual tarball.1889 cros_build_lib.Info('Archiving full autotest tarball locally ...')1890 tarball = commands.BuildFullAutotestTarball(self._build_root,1891 self._current_board,1892 image_dir)1893 commands.ArchiveFile(tarball, archive_path)1894 # Build hwqual image and upload to Google Storage.1895 hwqual_name = 'chromeos-hwqual-%s-%s' % (board, self.version)1896 filename = commands.ArchiveHWQual(buildroot, hwqual_name, archive_path,1897 image_dir)1898 self._release_upload_queue.put([filename])1899 def ArchiveFirmwareImages():1900 """Archive firmware images built from source if available."""1901 archive = commands.BuildFirmwareArchive(buildroot, board, archive_path)1902 if archive:1903 self._release_upload_queue.put([archive])1904 def BuildAndArchiveAllImages():1905 # Generate the recovery image. To conserve loop devices, we try to only1906 # run one instance of build_image at a time. TODO(davidjames): Move the1907 # image generation out of the archive stage.1908 # For recovery image to be generated correctly, BuildRecoveryImage must1909 # run before BuildAndArchiveFactoryImages.1910 if 'base' in config['images']:1911 commands.BuildRecoveryImage(buildroot, board, image_dir, extra_env)1912 self._recovery_image_status_queue.put(True)1913 if config['images']:1914 parallel.RunParallelSteps([BuildAndArchiveFactoryImages,1915 ArchiveHWQual,1916 ArchiveStandaloneTarballs,1917 ArchiveZipFiles])1918 def ArchiveImageScripts():1919 """Archive tarball of generated image manipulation scripts."""1920 target = os.path.join(archive_path, constants.IMAGE_SCRIPTS_TAR)1921 files = glob.glob(os.path.join(image_dir, '*.sh'))1922 files = [os.path.basename(f) for f in files]1923 cros_build_lib.CreateTarball(target, image_dir, inputs=files)1924 self._upload_queue.put([constants.IMAGE_SCRIPTS_TAR])1925 def PushImage():1926 # This helper script is only available on internal manifests currently.1927 if not config['internal']:1928 return1929 # Now that all data has been generated, we can upload the final result to1930 # the image server.1931 # TODO: When we support branches fully, the friendly name of the branch1932 # needs to be used with PushImages1933 sign_types = []1934 if config['name'].endswith('-%s' % cbuildbot_config.CONFIG_TYPE_FIRMWARE):1935 sign_types += ['firmware']1936 commands.PushImages(buildroot,1937 board=board,1938 branch_name='master',1939 archive_url=upload_url,1940 dryrun=debug or not config['push_image'],1941 profile=self._options.profile or config['profile'],1942 sign_types=sign_types)1943 def ArchiveReleaseArtifacts():1944 with self.ArtifactUploader(self._release_upload_queue, archive=False):1945 steps = [ArchiveDebugSymbols, BuildAndArchiveAllImages,1946 ArchiveFirmwareImages]1947 parallel.RunParallelSteps(steps)1948 PushImage()1949 def BuildAndArchiveArtifacts():1950 # Run archiving steps in parallel.1951 steps = [ArchiveReleaseArtifacts]1952 if config['images']:1953 steps.extend(1954 [self.ArchiveStrippedChrome, self.BuildAndArchiveChromeSysroot,1955 self.ArchiveChromeEbuildEnv, ArchiveImageScripts])1956 with parallel.BackgroundTaskRunner(1957 UploadSymbols, queue=self._upload_symbols_queue, processes=1):1958 with self.ArtifactUploader(self._upload_queue, archive=False):1959 parallel.RunParallelSteps(steps)1960 def MarkAsLatest():1961 # Update and upload LATEST file.1962 filename = 'LATEST-%s' % self._target_manifest_branch1963 latest_path = os.path.join(self.bot_archive_root, filename)1964 osutils.WriteFile(latest_path, self.version, mode='w')1965 commands.UploadArchivedFile(1966 self.bot_archive_root, self._GetGSUtilArchiveDir(), filename, debug)1967 try:1968 BuildAndArchiveArtifacts()1969 MarkAsLatest()1970 finally:1971 commands.RemoveOldArchives(self.bot_archive_root,1972 self._options.max_archive_builds)1973 def _HandleStageException(self, exception):1974 # Tell the HWTestStage not to wait for artifacts to be uploaded1975 # in case ArchiveStage throws an exception.1976 self._recovery_image_status_queue.put(False)1977 return super(ArchiveStage, self)._HandleStageException(exception)1978class UploadPrebuiltsStage(BoardSpecificBuilderStage):1979 """Uploads binaries generated by this build for developer use."""1980 option_name = 'prebuilts'1981 config_name = 'prebuilts'1982 def __init__(self, options, build_config, board, archive_stage, suffix=None):1983 super(UploadPrebuiltsStage, self).__init__(options, build_config, board,1984 suffix=suffix)1985 self._archive_stage = archive_stage1986 def GenerateCommonArgs(self):1987 """Generate common prebuilt arguments."""1988 generated_args = []1989 if self._options.debug:1990 generated_args.append('--debug')1991 profile = self._options.profile or self._build_config['profile']1992 if profile:1993 generated_args.extend(['--profile', profile])1994 # Generate the version if we are a manifest_version build.1995 if self._build_config['manifest_version']:1996 assert self._archive_stage, 'Archive stage missing for versioned build.'1997 version = self._archive_stage.GetVersion()1998 generated_args.extend(['--set-version', version])1999 if self._build_config['git_sync']:2000 # Git sync should never be set for pfq type builds.2001 assert not cbuildbot_config.IsPFQType(self._prebuilt_type)2002 generated_args.extend(['--git-sync'])2003 return generated_args2004 @classmethod2005 def _AddOptionsForSlave(cls, builder, board):2006 """Inner helper method to add upload_prebuilts args for a slave builder.2007 Returns:2008 An array of options to add to upload_prebuilts array that allow a master2009 to submit prebuilt conf modifications on behalf of a slave.2010 """2011 args = []2012 builder_config = cbuildbot_config.config[builder]2013 if builder_config['prebuilts']:2014 for slave_board in builder_config['boards']:2015 if builder_config['master'] and slave_board == board:2016 # Ignore self.2017 continue2018 args.extend(['--slave-board', slave_board])2019 slave_profile = builder_config['profile']2020 if slave_profile:2021 args.extend(['--slave-profile', slave_profile])2022 return args2023 def PerformStage(self):2024 """Uploads prebuilts for master and slave builders."""2025 prebuilt_type = self._prebuilt_type2026 board = self._current_board2027 binhosts = []2028 # Whether we publish public prebuilts.2029 public = (self._build_config['prebuilts'] == constants.PUBLIC)2030 # Common args we generate for all types of builds.2031 generated_args = self.GenerateCommonArgs()2032 # Args we specifically add for public/private build types.2033 public_args, private_args = [], []2034 # Public / private builders.2035 public_builders, private_builders = [], []2036 # Distributed builders that use manifest-versions to sync with one another2037 # share prebuilt logic by passing around versions.2038 if cbuildbot_config.IsPFQType(prebuilt_type):2039 # Public pfqs should upload host preflight prebuilts.2040 if prebuilt_type != constants.CHROME_PFQ_TYPE:2041 public_args.append('--sync-host')2042 # Deduplicate against previous binhosts.2043 binhosts.extend(self._GetPortageEnvVar(_PORTAGE_BINHOST, board).split())2044 binhosts.extend(self._GetPortageEnvVar(_PORTAGE_BINHOST, None).split())2045 for binhost in filter(None, binhosts):2046 generated_args.extend(['--previous-binhost-url', binhost])2047 if self._build_config['master'] and board == self._boards[-1]:2048 # The master builder updates all the binhost conf files, and needs to do2049 # so only once so as to ensure it doesn't try to update the same file2050 # more than once. As multiple boards can be built on the same builder,2051 # we arbitrarily decided to update the binhost conf files when we run2052 # upload_prebuilts for the last board. The other boards are treated as2053 # slave boards.2054 generated_args.append('--sync-binhost-conf')2055 for c in self._GetSlavesForMaster(self._build_config):2056 if c['prebuilts'] == constants.PUBLIC:2057 public_builders.append(c['name'])2058 public_args.extend(self._AddOptionsForSlave(c['name'], board))2059 elif c['prebuilts'] == constants.PRIVATE:2060 private_builders.append(c['name'])2061 private_args.extend(self._AddOptionsForSlave(c['name'], board))2062 # Upload the public prebuilts, if any.2063 if public_builders or public:2064 public_board = board if public else None2065 commands.UploadPrebuilts(2066 category=prebuilt_type, chrome_rev=self._chrome_rev,2067 private_bucket=False, buildroot=self._build_root,2068 board=public_board, extra_args=generated_args + public_args)2069 # Upload the private prebuilts, if any.2070 if private_builders or not public:2071 private_board = board if not public else None2072 commands.UploadPrebuilts(2073 category=prebuilt_type, chrome_rev=self._chrome_rev,2074 private_bucket=True, buildroot=self._build_root, board=private_board,2075 extra_args=generated_args + private_args)2076class DevInstallerPrebuiltsStage(UploadPrebuiltsStage):2077 config_name = 'dev_installer_prebuilts'2078 def PerformStage(self):2079 generated_args = generated_args = self.GenerateCommonArgs()2080 commands.UploadDevInstallerPrebuilts(2081 binhost_bucket=self._build_config['binhost_bucket'],2082 binhost_key=self._build_config['binhost_key'],2083 binhost_base_url=self._build_config['binhost_base_url'],2084 buildroot=self._build_root,2085 board=self._current_board,2086 extra_args=generated_args)2087class PublishUprevChangesStage(NonHaltingBuilderStage):2088 """Makes uprev changes from pfq live for developers."""2089 def PerformStage(self):2090 _, push_overlays = self._ExtractOverlays()2091 if push_overlays:2092 commands.UprevPush(self._build_root, push_overlays, self._options.debug)2093class ReportStage(bs.BuilderStage):2094 """Summarize all the builds."""2095 _HTML_HEAD = """<html>2096<head>2097 <title>Archive Index: %(board)s / %(version)s</title>2098</head>2099<body>2100<h2>Artifacts Index: %(board)s / %(version)s (%(config)s config)</h2>"""2101 def __init__(self, options, build_config, archive_stages, version):2102 bs.BuilderStage.__init__(self, options, build_config)2103 self._archive_stages = archive_stages2104 self._version = version if version else ''2105 def PerformStage(self):2106 acl = None if self._build_config['internal'] else 'public-read'2107 archive_urls = {}2108 for board_config, archive_stage in sorted(self._archive_stages.iteritems()):2109 board = board_config.board2110 head_data = {2111 'board': board,2112 'config': board_config.name,2113 'version': archive_stage.version,2114 }2115 head = self._HTML_HEAD % head_data2116 url = archive_stage.download_url2117 path = archive_stage.archive_path2118 upload_url = archive_stage.upload_url2119 # Generate the final metadata before we look at the uploaded list.2120 if results_lib.Results.BuildSucceededSoFar():2121 final_status = 'passed'2122 else:2123 final_status = 'failed'2124 archive_stage.RefreshMetadata('', final_status=final_status)2125 # Generate the index page needed for public reading.2126 uploaded = os.path.join(path, commands.UPLOADED_LIST_FILENAME)2127 if not os.path.exists(uploaded):2128 if (not self._build_config['compilecheck'] and2129 not self._options.compilecheck):2130 # UPLOADED doesn't exist. Normal if buildboard failed.2131 logging.warning('board %s did not make it to the archive stage; '2132 'skipping', board)2133 continue2134 files = osutils.ReadFile(uploaded).splitlines() + [2135 '.|Google Storage Index',2136 '..|',2137 ]2138 index = os.path.join(path, 'index.html')2139 commands.GenerateHtmlIndex(index, files, url_base=url, head=head)2140 commands.UploadArchivedFile(path, upload_url, os.path.basename(index),2141 debug=archive_stage.debug, acl=acl)2142 archive_urls[board] = archive_stage.download_url + '/index.html'2143 results_lib.Results.Report(sys.stdout, archive_urls=archive_urls,...

builder.py

Source:builder.py

...78 _cfg = {"sources": list(self.src_master.keys()),79 "gene_root": ['entrez_gene', 'ensembl_gene']}80 self._build_config = _cfg81 return _cfg82 def load_build_config(self, build):83 '''Load build config from src_build collection.'''84 src_build = get_src_build()85 self.src_build = src_build86 _cfg = src_build.find_one({'_id': build})87 if _cfg:88 self._build_config = _cfg89 else:90 raise ValueError('Cannot find build config named "%s"' % build)91 return _cfg92 def log_src_build(self, dict):93 '''put logging dictionary into the corresponding doc in src_build collection.94 if build_config is not loaded from src_build, nothing will be logged.95 '''96 src_build = getattr(self, 'src_build', None)97 if src_build:98 _cfg = src_build.find_one({'_id': self._build_config['_id']})99 _cfg['build'][-1].update(dict)100 src_build.update({'_id': self._build_config['_id']}, {"$set": {'build': _cfg['build']}})101 def log_building_start(self):102 if self.merge_logging:103 #setup logging104 logfile = 'databuild_{}_{}.log'.format('genedoc' + '_' + self._build_config['name'],105 time.strftime('%Y%m%d'))106 logfile = os.path.join(self.log_folder, logfile)107 setup_logfile(logfile)108 src_build = getattr(self, 'src_build', None)109 if src_build:110 #src_build.update({'_id': self._build_config['_id']}, {"$unset": {"build": ""}})111 d = {'status': 'building',112 'started_at': datetime.now(),113 'logfile': logfile,114 'target_backend': self.target.name}115 if self.target.name == 'mongodb':116 d['target'] = self.target.target_collection.name117 elif self.target.name == 'es':118 d['target'] = self.target.target_esidxer.ES_INDEX_NAME119 logging.info(pformat(d))120 src_build.update({'_id': self._build_config['_id']}, {"$push": {'build': d}})121 _cfg = src_build.find_one({'_id': self._build_config['_id']})122 if len(_cfg['build']) > self.max_build_status:123 #remove the first build status record124 src_build.update({'_id': self._build_config['_id']}, {"$pop": {'build': -1}})125 def _get_target_name(self):126 return 'genedoc_{}_{}_{}'.format(self._build_config['name'],127 get_timestamp(), get_random_string()).lower()128 def prepare_target(self, target_name=None):129 '''call self.update_backend() after validating self._build_config.'''130 if self.target.name == 'mongodb':131 _db = get_target_db()132 target_collection_name = target_name or self._get_target_name()133 self.target.target_collection = _db[target_collection_name]134 logging.info("Target: %s" % repr(target_collection_name))135 elif self.target.name == 'es':136 self.target.target_esidxer.ES_INDEX_NAME = target_name or self._get_target_name()137 self.target.target_esidxer._mapping = self.get_mapping()138 elif self.target.name == 'couchdb':139 self.target.db_name = target_name or ('genedoc' + '_' + self._build_config['name'])140 elif self.target.name == 'memory':141 self.target.target_name = target_name or ('genedoc' + '_' + self._build_config['name'])142 def get_src_master(self):143 src_master = get_src_master(self.src.client)144 self.src_master = dict([(src['_id'], src) for src in list(src_master.find())])145 def validate_src_collections(self,collection_list=None):146 if not collection_list:147 collection_list = set(self.src.collection_names())148 self.get_src_master()149 build_conf_src = self._build_config['sources']150 else:151 build_conf_src = collection_list152 logging.info("Sources: %s" % repr(build_conf_src))153 if self._build_config:154 for src in build_conf_src:155 assert src in self.src_master, '"%s" not found in "src_master"' % src156 assert src in collection_list, '"%s" not an existing collection in "%s"' % (src, self.src.name)157 else:158 raise ValueError('"build_config" cannot be empty.')159 def _load_entrez_geneid_d(self):160 self._entrez_geneid_d = loadobj(("entrez_gene__geneid_d.pyobj", self.src), mode='gridfs')161 def _load_ensembl2entrez_li(self):162 ensembl2entrez_li = loadobj(("ensembl_gene__2entrezgene_list.pyobj", self.src), mode='gridfs')163 #filter out those deprecated entrez gene ids164 logging.info(len(ensembl2entrez_li))165 ensembl2entrez_li = [(ensembl_id, self._entrez_geneid_d[int(entrez_id)]) for (ensembl_id, entrez_id) in ensembl2entrez_li166 if int(entrez_id) in self._entrez_geneid_d]167 logging.info(len(ensembl2entrez_li))168 ensembl2entrez = list2dict(ensembl2entrez_li, 0)169 self._idmapping_d_cache['ensembl_gene'] = ensembl2entrez170 def _save_idmapping_gridfs(self):171 '''saving _idmapping_d_cache into gridfs.'''172 idmapping_gridfs_d = {}173 if self._idmapping_d_cache:174 for id_type in self._idmapping_d_cache:175 filename = 'tmp_idmapping_d_cache_' + id_type176 dump2gridfs(self._idmapping_d_cache[id_type], filename, self.src)177 idmapping_gridfs_d[id_type] = filename178 return idmapping_gridfs_d179 def make_genedoc_root(self):180 if not self._entrez_geneid_d:181 self._load_entrez_geneid_d()182 if 'ensembl_gene' in self._build_config['gene_root']:183 self._load_ensembl2entrez_li()184 ensembl2entrez = self._idmapping_d_cache['ensembl_gene']185 if "species" in self._build_config:186 _query = {'taxid': {'$in': self._build_config['species']}}187 elif "species_to_exclude" in self._build_config:188 _query = {'taxid': {'$nin': self._build_config['species_to_exclude']}}189 else:190 _query = None191 geneid_set = []192 species_set = set()193 if "entrez_gene" in self._build_config['gene_root']:194 for doc_li in doc_feeder(self.src['entrez_gene'], inbatch=True, step=self.step, query=_query):195 #target_collection.insert(doc_li, manipulate=False, check_keys=False)196 self.target.insert(doc_li)197 geneid_set.extend([doc['_id'] for doc in doc_li])198 species_set |= set([doc['taxid'] for doc in doc_li])199 cnt_total_entrez_genes = len(geneid_set)200 cnt_total_species = len(species_set)201 logging.info('# of entrez Gene IDs in total: %d' % cnt_total_entrez_genes)202 logging.info('# of species in total: %d' % cnt_total_species)203 if "ensembl_gene" in self._build_config['gene_root']:204 cnt_ensembl_only_genes = 0205 cnt_total_ensembl_genes = 0206 for doc_li in doc_feeder(self.src['ensembl_gene'], inbatch=True, step=self.step, query=_query):207 _doc_li = []208 for _doc in doc_li:209 cnt_total_ensembl_genes += 1210 ensembl_id = _doc['_id']211 entrez_gene = ensembl2entrez.get(ensembl_id, None)212 if entrez_gene is None:213 #this is an Ensembl only gene214 _doc_li.append(_doc)215 cnt_ensembl_only_genes += 1216 geneid_set.append(_doc['_id'])217 if _doc_li:218 #target_collection.insert(_doc_li, manipulate=False, check_keys=False)219 self.target.insert(_doc_li)220 cnt_matching_ensembl_genes = cnt_total_ensembl_genes - cnt_ensembl_only_genes221 logging.info('# of ensembl Gene IDs in total: %d' % cnt_total_ensembl_genes)222 logging.info('# of ensembl Gene IDs match entrez Gene IDs: %d' % cnt_matching_ensembl_genes)223 logging.info('# of ensembl Gene IDs DO NOT match entrez Gene IDs: %d' % cnt_ensembl_only_genes)224 geneid_set = set(geneid_set)225 logging.info('# of total Root Gene IDs: %d' % len(geneid_set))226 _stats = {'total_entrez_genes': cnt_total_entrez_genes,227 'total_species': cnt_total_species,228 'total_ensembl_genes': cnt_total_ensembl_genes,229 'total_ensembl_genes_mapped_to_entrez': cnt_matching_ensembl_genes,230 'total_ensembl_only_genes': cnt_ensembl_only_genes,231 'total_genes': len(geneid_set)}232 self._stats = _stats233 self._src_version = self.get_src_version()234 self.log_src_build({'stats': _stats, 'src_version': self._src_version})235 return geneid_set236 def get_idmapping_d(self, src):237 if src in self._idmapping_d_cache:238 return self._idmapping_d_cache[src]239 else:240 self._load_ensembl2entrez_li()241 return self._idmapping_d_cache[src]242 #raise ValueError('cannot load "idmapping_d" for "%s"' % src)243 def merge(self, step=100000, restart_at=0,sources=None,target=None):244 t0 = time.time()245 self.validate_src_collections(sources)246 self.prepare_target(target_name=target)247 self.log_building_start()248 try:249 if self.using_ipython_cluster:250 if sources:251 raise NotImplemented("merge speficic sources not supported when using parallel")252 self._merge_ipython_cluster(step=step)253 else:254 self._merge_local(step=step, restart_at=restart_at,src_collection_list=sources)255 if self.target.name == 'es':256 logging.info("Updating metadata...")257 self.update_mapping_meta()258 t1 = round(time.time() - t0, 0)259 t = timesofar(t0)260 self.log_src_build({'status': 'success',261 'time': t,262 'time_in_s': t1,263 'timestamp': datetime.now()})264 finally:265 #do a simple validation here266 if getattr(self, '_stats', None):267 logging.info("Validating...")268 target_cnt = self.target.count()269 if target_cnt == self._stats['total_genes']:270 logging.info("OK [total count={}]".format(target_cnt))271 else:272 logging.info("Warning: total count of gene documents does not match [{}, should be {}]".format(target_cnt, self._stats['total_genes']))273 if self.merge_logging:274 sys.stdout.close()275 def merge_resume(self, build_config, at_collection, step=10000):276 '''resume a merging process after a failure.277 .merge_resume('mygene_allspecies', 'reporter')278 '''279 assert not self.using_ipython_cluster, "Abort. Can only resume merging in non-parallel mode."280 self.load_build_config(build_config)281 last_build = self._build_config['build'][-1]282 logging.info("Last build record:")283 logging.info(pformat(last_build))284 assert last_build['status'] == 'building', \285 "Abort. Last build does not need to be resumed."286 assert at_collection in self._build_config['sources'], \287 'Abort. Cannot resume merging from a unknown collection "{}"'.format(at_collection)288 assert last_build['target_backend'] == self.target.name, \289 'Abort. Re-initialized DataBuilder class using matching backend "{}"'.format(last_build['backend'])290 assert last_build.get('stats', None), \291 'Abort. Intital build stats are not available. You should restart the build from the scratch.'292 self._stats = last_build['stats']293 if ask('Continue to resume merging from "{}"?'.format(at_collection)) == 'Y':294 #TODO: resume logging295 target_name = last_build['target']296 self.validate_src_collections()297 self.prepare_target(target_name=target_name)298 src_cnt = 0299 for collection in self._build_config['sources']:300 if collection in ['entrez_gene', 'ensembl_gene']:301 continue302 src_cnt += 1303 if collection == at_collection:304 break305 self._merge_local(step=step, restart_at=src_cnt)306 if self.target.name == 'es':307 logging.info("Updating metadata...")308 self.update_mapping_meta()309 self.log_src_build({'status': 'success',310 'timestamp': datetime.now()})311 def _merge_ipython_cluster(self, step=100000):312 '''Do the merging on ipython cluster.'''313 from ipyparallel import Client, require314 from config import CLUSTER_CLIENT_JSON315 t0 = time.time()316 src_collection_list = [collection for collection in self._build_config['sources']317 if collection not in ['entrez_gene', 'ensembl_gene']]318 self.target.drop()319 self.target.prepare()320 geneid_set = self.make_genedoc_root()321 idmapping_gridfs_d = self._save_idmapping_gridfs()322 logging.info(timesofar(t0))323 rc = Client(CLUSTER_CLIENT_JSON)324 lview = rc.load_balanced_view()325 logging.info("\t# nodes in use: {}".format(len(lview.targets or rc.ids)))326 lview.block = False327 kwargs = {}328 target_collection = self.target.target_collection329 kwargs['server'], kwargs['port'] = target_collection.database.client.address330 kwargs['src_db'] = self.src.name331 kwargs['target_db'] = target_collection.database.name332 kwargs['target_collection_name'] = target_collection.name333 kwargs['limit'] = step334 @require('pymongo', 'time', 'types')335 def worker(kwargs):336 server = kwargs['server']337 port = kwargs['port']338 src_db = kwargs['src_db']339 target_db = kwargs['target_db']340 target_collection_name = kwargs['target_collection_name']341 src_collection = kwargs['src_collection']342 skip = kwargs['skip']343 limit = kwargs['limit']344 def load_from_gridfs(filename, db):345 import gzip346 import pickle347 import gridfs348 fs = gridfs.GridFS(db)349 fobj = fs.get(filename)350 gzfobj = gzip.GzipFile(fileobj=fobj)351 try:352 object = pickle.load(gzfobj)353 finally:354 gzfobj.close()355 fobj.close()356 return object357 def alwayslist(value):358 if value is None:359 return []360 if isinstance(value, (list, tuple)):361 return value362 else:363 return [value]364 conn = pymongo.MongoClient(server, port)365 src = conn[src_db]366 target_collection = conn[target_db][target_collection_name]367 idmapping_gridfs_name = kwargs.get('idmapping_gridfs_name', None)368 if idmapping_gridfs_name:369 idmapping_d = load_from_gridfs(idmapping_gridfs_name, src)370 else:371 idmapping_d = None372 cur = src[src_collection].find(skip=skip, limit=limit, timeout=False)373 cur.batch_size(1000)374 try:375 for doc in cur:376 _id = doc['_id']377 if idmapping_d:378 _id = idmapping_d.get(_id, None) or _id379 # there could be cases that idmapping returns multiple entrez_gene id.380 for __id in alwayslist(_id): 381 __id = str(__id)382 doc.pop('_id', None)383 doc.pop('taxid', None)384 target_collection.update({'_id': __id}, doc, manipulate=False, upsert=False)385 #target_collection.update({'_id': __id}, {'$set': doc},386 finally:387 cur.close()388 t0 = time.time()389 task_list = []390 for src_collection in src_collection_list:391 _kwargs = copy.copy(kwargs)392 _kwargs['src_collection'] = src_collection393 id_type = self.src_master[src_collection].get('id_type', None)394 if id_type:395 idmapping_gridfs_name = idmapping_gridfs_d[id_type]396 _kwargs['idmapping_gridfs_name'] = idmapping_gridfs_name397 cnt = self.src[src_collection].count()398 for s in range(0, cnt, step):399 __kwargs = copy.copy(_kwargs)400 __kwargs['skip'] = s401 task_list.append(__kwargs)402 logging.info("\t# of tasks: {}".format(len(task_list)))403 logging.info("\tsubmitting...")404 job = lview.map_async(worker, task_list)405 logging.info("done.")406 job.wait_interactive()407 logging.info("\t# of results returned: {}".format(len(job.result())))408 logging.info("\ttotal time: {}".format(timesofar(t0)))409 if self.shutdown_ipengines_after_done:410 logging.info("\tshuting down all ipengine nodes...")411 lview.shutdown()412 logging.info('Done.')413 def _merge_local(self, step=100000, restart_at=0, src_collection_list=None):414 if restart_at == 0 and src_collection_list is None:415 self.target.drop()416 self.target.prepare()417 geneid_set = self.make_genedoc_root()418 else:419 if not self._entrez_geneid_d:420 self._load_entrez_geneid_d()421 #geneid_set = set([x['_id'] for x in target_collection.find(projection=[], manipulate=False)])422 geneid_set = set(self.target.get_id_list())423 logging.info('\t%s' % len(geneid_set))424 if not src_collection_list:425 src_collection_list = self._build_config['sources']426 src_cnt = 0427 for collection in src_collection_list:428 if collection in ['entrez_gene', 'ensembl_gene']:429 continue430 src_cnt += 1431 id_type = self.src_master[collection].get('id_type', None)432 flag_need_id_conversion = id_type is not None433 if flag_need_id_conversion:434 idmapping_d = self.get_idmapping_d(id_type)435 else:436 idmapping_d = None437 if restart_at <= src_cnt:438 if self.use_parallel:439 self.doc_queue = []440 self._merge_parallel_ipython(collection, geneid_set,441 step=step, idmapping_d=idmapping_d)442 else:443 self._merge_sequential(collection, geneid_set,444 step=step, idmapping_d=idmapping_d)445 self.target.finalize()446 def _merge_sequential(self, collection, geneid_set, step=100000, idmapping_d=None):447 for doc in doc_feeder(self.src[collection], step=step):448 _id = doc['_id']449 if idmapping_d:450 _id = idmapping_d.get(_id, None) or _id451 for __id in alwayslist(_id): # there could be cases that idmapping returns multiple entrez_gene ids.452 __id = str(__id)453 if __id in geneid_set:454 doc.pop('_id', None)455 doc.pop('taxid', None)456 # target_collection.update({'_id': __id}, {'$set': doc},457 # manipulate=False,458 # upsert=False) #,safe=True)459 self.target.update(__id, doc)460 def _merge_parallel(self, collection, geneid_set, step=100000, idmapping_d=None):461 from multiprocessing import Process, Queue462 NUMBER_OF_PROCESSES = 8463 input_queue = Queue()464 input_queue.conn_pool = []465 def worker(q, target):466 while True:467 doc = q.get()468 if doc == 'STOP':469 break470 __id = doc.pop('_id')471 doc.pop('taxid', None)472 target.update(__id, doc)473 # target_collection.update({'_id': __id}, {'$set': doc},474 # manipulate=False,475 # upsert=False) #,safe=True)476 # Start worker processes477 for i in range(NUMBER_OF_PROCESSES):478 Process(target=worker, args=(input_queue, self.target)).start()479 for doc in doc_feeder(self.src[collection], step=step):480 _id = doc['_id']481 if idmapping_d:482 _id = idmapping_d.get(_id, None) or _id483 for __id in alwayslist(_id): # there could be cases that idmapping returns multiple entrez_gene ids.484 __id = str(__id)485 if __id in geneid_set:486 doc['_id'] = __id487 input_queue.put(doc)488 # Tell child processes to stop489 for i in range(NUMBER_OF_PROCESSES):490 input_queue.put('STOP')491 def _merge_parallel_ipython(self, collection, geneid_set, step=100000, idmapping_d=None):492 from IPython.parallel import Client, require493 rc = Client()494 dview = rc[:]495 #dview = rc.load_balanced_view()496 dview.block = False497 target_collection = self.target.target_collection498 dview['server'], dview['port'] = target_collection.database.client.address499 dview['database'] = target_collection.database.name500 dview['collection_name'] = target_collection.name501 def partition(lst, n):502 q, r = divmod(len(lst), n)503 indices = [q * i + min(i, r) for i in range(n + 1)]504 return [lst[indices[i]:indices[i + 1]] for i in range(n)]505 @require('pymongo', 'time')506 def worker(doc_li):507 conn = pymongo.MongoClient(server, port)508 target_collection = conn[database][collection_name]509 t0 = time.time()510 for doc in doc_li:511 __id = doc.pop('_id')512 doc.pop('taxid', None)513 target_collection.update({'_id': __id}, {'$set': doc},514 manipulate=False,515 upsert=False) # ,safe=True)516 logging.info('Done. [%.1fs]' % (time.time() - t0))517 for doc in doc_feeder(self.src[collection], step=step):518 _id = doc['_id']519 if idmapping_d:520 _id = idmapping_d.get(_id, None) or _id521 for __id in alwayslist(_id): # there could be cases that idmapping returns multiple entrez_gene ids.522 __id = str(__id)523 if __id in geneid_set:524 doc['_id'] = __id525 self.doc_queue.append(doc)526 if len(self.doc_queue) >= step:527 #dview.scatter('doc_li', self.doc_queue)528 #dview.apply_async(worker)529 dview.map_async(worker, partition(self.doc_queue, len(rc.ids)))530 self.doc_queue = []531 logging.info("!")532 def get_src_version(self):533 src_dump = get_src_dump(self.src.client)534 src_version = {}535 for src in src_dump.find():536 version = src.get('release', src.get('timestamp', None))537 if version:538 src_version[src['_id']] = version539 return src_version540 def get_last_src_build_stats(self):541 src_build = getattr(self, 'src_build', None)542 if src_build:543 _cfg = src_build.find_one({'_id': self._build_config['_id']})544 if _cfg['build'][-1].get('status', None) == 'success' and \545 _cfg['build'][-1].get('stats', None):546 stats = _cfg['build'][-1]['stats']547 return stats548 def get_target_collection(self):549 '''get the lastest target_collection from src_build record.'''550 src_build = getattr(self, 'src_build', None)551 if src_build:552 _cfg = src_build.find_one({'_id': self._build_config['_id']})553 if _cfg['build'][-1].get('status', None) == 'success' and \554 _cfg['build'][-1].get('target', None):555 target_collection = _cfg['build'][-1]['target']556 _db = get_target_db()557 target_collection = _db[target_collection]558 return target_collection559 def pick_target_collection(self, autoselect=True):560 '''print out a list of available target_collection, let user to pick one.'''561 target_db = get_target_db()562 target_collection_prefix = 'genedoc_' + self._build_config['name']563 target_collection_list = [target_db[name] for name in sorted(target_db.collection_names()) if name.startswith(target_collection_prefix)]564 if target_collection_list:565 logging.info("Found {} target collections:".format(len(target_collection_list)))566 logging.info('\n'.join(['\t{0:<5}{1.name:<45}\t{2}'.format(567 str(i + 1) + ':', target, target.count()) for (i, target) in enumerate(target_collection_list)]))568 logging.info()569 while 1:570 if autoselect:571 selected_idx = input("Pick one above [{}]:".format(len(target_collection_list)))572 else:573 selected_idx = input("Pick one above:")574 if autoselect:575 selected_idx = selected_idx or len(target_collection_list)576 try:577 selected_idx = int(selected_idx)578 break579 except ValueError:580 continue581 return target_collection_list[selected_idx - 1]582 else:583 logging.info("Found no target collections.")584 def get_mapping(self, enable_timestamp=True):585 '''collect mapping data from data sources.586 This is for GeneDocESBackend only.587 '''588 mapping = {}589 src_master = get_src_master(self.src.client)590 for collection in self._build_config['sources']:591 meta = src_master.find_one({"_id" : collection})592 if 'mapping' in meta:593 mapping.update(meta['mapping'])594 else:595 logging.info('Warning: "%s" collection has no mapping data.' % collection)596 mapping = {"properties": mapping,597 "dynamic": False}598 if enable_timestamp:599 mapping['_timestamp'] = {600 "enabled": True,601 }602 #allow source Compression603 #Note: no need of source compression due to "Store Level Compression"604 #mapping['_source'] = {'compress': True,}605 # 'compress_threshold': '1kb'}606 return mapping607 def update_mapping_meta(self):608 '''updating _meta field of ES mapping data, including index stats, versions.609 This is for GeneDocESBackend only.610 '''611 _meta = {}612 src_version = self.get_src_version()613 if src_version:614 _meta['src_version'] = src_version615 if getattr(self, '_stats', None):616 _meta['stats'] = self._stats617 if _meta:618 self.target.target_esidxer.update_mapping_meta({'_meta': _meta})619 def validate(self, build_config='mygene_allspecies', n=10):620 '''Validate merged genedoc, currently for ES backend only.'''621 import random622 import itertools623 import pyes624 self.load_build_config(build_config)625 last_build = self._build_config['build'][-1]626 logging.info("Last build record:")627 logging.info(pformat(last_build))628 #assert last_build['target_backend'] == 'es', '"validate" currently works for "es" backend only'629 target_name = last_build['target']630 self.validate_src_collections()631 self.prepare_target(target_name=target_name)632 logging.info("Validating...")633 target_cnt = self.target.count()634 stats_cnt = last_build['stats']['total_genes']635 if target_cnt == stats_cnt:636 logging.info("OK [total count={}]".format(target_cnt))637 else:638 logging.info("Warning: total count of gene documents does not match [{}, should be {}]".format(target_cnt, stats_cnt))639 if n > 0:640 for src in self._build_config['sources']:641 logging.info("\nSrc: %s" % src)642 # if 'id_type' in self.src_master[src] and self.src_master[src]['id_type'] != 'entrez_gene':643 # print "skipped."644 # continue645 cnt = self.src[src].count()646 fdr1 = doc_feeder(self.src[src], step=10000, s=cnt - n)647 rand_s = random.randint(0, cnt - n)648 fdr2 = doc_feeder(self.src[src], step=n, s=rand_s, e=rand_s + n)649 _first_exception = True650 for doc in itertools.chain(fdr1, fdr2):651 _id = doc['_id']652 try:653 es_doc = self.target.get_from_id(_id)654 except pyes.exceptions.NotFoundException:655 if _first_exception:656 logging.info()657 _first_exception = False658 logging.info("%s not found." % _id)659 continue660 for k in doc:661 if src == 'entrez_homologene' and k == 'taxid':662 # there is occasionally known error for taxid in homologene data.663 continue664 assert es_doc.get(k, None) == doc[k], (_id, k, es_doc.get(k, None), doc[k])665 def build_index(self, use_parallel=True):666 target_collection = self.get_target_collection()667 if target_collection:668 es_idxer = ESIndexer(mapping=self.get_mapping())669 es_idxer.ES_INDEX_NAME = 'genedoc_' + self._build_config['name']670 es_idxer.step = 10000671 es_idxer.use_parallel = use_parallel672 #es_idxer.s = 609000673 #es_idxer.conn.indices.delete_index(es_idxer.ES_INDEX_NAME)674 es_idxer.create_index()675 es_idxer.delete_index_type(es_idxer.ES_INDEX_TYPE, noconfirm=True)676 es_idxer.build_index(target_collection, verbose=False)677 es_idxer.optimize()678 else:679 logging.info("Error: target collection is not ready yet or failed to build.")680 def build_index2(self, build_config='mygene_allspecies', last_build_idx=-1, use_parallel=False, es_host=None, es_index_name=None, noconfirm=False):681 """Build ES index from last successfully-merged mongodb collection.682 optional "es_host" argument can be used to specified another ES host, otherwise default ES_HOST.683 optional "es_index_name" argument can be used to pass an alternative index name, otherwise same as mongodb collection name684 """685 self.load_build_config(build_config)686 assert "build" in self._build_config, "Abort. No such build records for config %s" % build_config687 last_build = self._build_config['build'][last_build_idx]688 logging.info("Last build record:")689 logging.info(pformat(last_build))690 assert last_build['status'] == 'success', \691 "Abort. Last build did not success."692 assert last_build['target_backend'] == "mongodb", \693 'Abort. Last build need to be built using "mongodb" backend.'694 assert last_build.get('stats', None), \695 'Abort. Last build stats are not available.'696 self._stats = last_build['stats']697 assert last_build.get('target', None), \698 'Abort. Last build target_collection is not available.'699 # Get the source collection to build the ES index700 # IMPORTANT: the collection in last_build['target'] does not contain _timestamp field,701 # only the "genedoc_*_current" collection does. When "timestamp" is enabled702 # in mappings, last_build['target'] collection won't be indexed by ES correctly,703 # therefore, we use "genedoc_*_current" collection as the source here:704 #target_collection = last_build['target']705 target_collection = "genedoc_{}_current".format(build_config)706 _db = get_target_db()707 target_collection = _db[target_collection]708 logging.info("")709 logging.info('Source: %s' % target_collection.name)710 _mapping = self.get_mapping()711 _meta = {}712 src_version = self.get_src_version()713 if src_version:714 _meta['src_version'] = src_version715 if getattr(self, '_stats', None):716 _meta['stats'] = self._stats717 if 'timestamp' in last_build:718 _meta['timestamp'] = last_build['timestamp']719 if _meta:720 _mapping['_meta'] = _meta721 es_index_name = es_index_name or target_collection.name722 es_idxer = ESIndexer(mapping=_mapping,723 es_index_name=es_index_name,724 es_host=es_host,725 step=5000)726 if build_config == 'mygene_allspecies':727 es_idxer.number_of_shards = 10 # default 5728 es_idxer.check()729 if noconfirm or ask("Continue to build ES index?") == 'Y':730 es_idxer.use_parallel = use_parallel731 #es_idxer.s = 609000732 if es_idxer.exists_index(es_idxer.ES_INDEX_NAME):733 if noconfirm or ask('Index "{}" exists. Delete?'.format(es_idxer.ES_INDEX_NAME)) == 'Y':734 es_idxer.conn.indices.delete(es_idxer.ES_INDEX_NAME)735 else:736 logging.info("Abort.")737 return738 es_idxer.create_index()739 #es_idxer.delete_index_type(es_idxer.ES_INDEX_TYPE, noconfirm=True)740 es_idxer.build_index(target_collection, verbose=False)741 # time.sleep(10) # pausing 10 second here742 # if es_idxer.wait_till_all_shards_ready():743 # print "Optimizing...", es_idxer.optimize()744 def sync_index(self, use_parallel=True):745 from utils import diff746 sync_src = self.get_target_collection()747 es_idxer = ESIndexer(self.get_mapping())748 es_idxer.ES_INDEX_NAME = sync_src.target_collection.name749 es_idxer.step = 10000750 es_idxer.use_parallel = use_parallel751 sync_target = databuild.backend.GeneDocESBackend(es_idxer)752 changes = diff.diff_collections(sync_src, sync_target)753 return changes754def main():755 if len(sys.argv) > 1:756 config = sys.argv[1]757 else:758 config = 'mygene_allspecies'759 use_parallel = '-p' in sys.argv760 sources = None # will build all sources761 target = None # will generate a new collection name762 # "target_col:src_col1,src_col2" will specifically merge src_col1763 # and src_col2 into existing target_col (instead of merging everything)764 if not use_parallel and len(sys.argv) > 2:765 target,tmp = sys.argv[2].split(":")766 sources = tmp.split(",")767 t0 = time.time()768 bdr = DataBuilder(backend='mongodb')769 bdr.load_build_config(config)770 bdr.using_ipython_cluster = use_parallel771 bdr.merge(sources=sources,target=target)772 logging.info("Finished. %s" % timesofar(t0))773if __name__ == '__main__':...

builderstage.py

Source:builderstage.py

1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4"""Module containing the base class for the stages that a builder runs."""5import copy6import os7import re8import sys9import time10import traceback11# We import mox so that we can identify mox exceptions and pass them through12# in our exception handling code.13try:14 import mox15except ImportError:16 mox = None17from chromite.buildbot import cbuildbot_config18from chromite.buildbot import cbuildbot_results as results_lib19from chromite.buildbot import portage_utilities20from chromite.buildbot import validation_pool21from chromite.lib import cros_build_lib22class BuilderStage(object):23 """Parent class for stages to be performed by a builder."""24 name_stage_re = re.compile(r'(\w+)Stage')25 # TODO(sosa): Remove these once we have a SEND/RECIEVE IPC mechanism26 # implemented.27 overlays = None28 push_overlays = None29 # Class variable that stores the branch to build and test30 _target_manifest_branch = None31 # Class should set this if they have a corresponding no<stage> option that32 # skips their stage.33 option_name = None34 # Class should set this if they have a corresponding setting in35 # self._build_config that skips their stage.36 config_name = None37 @staticmethod38 def SetManifestBranch(branch):39 BuilderStage._target_manifest_branch = branch40 @classmethod41 def StageNamePrefix(cls):42 return cls.name_stage_re.match(cls.__name__).group(1)43 def __init__(self, options, build_config, suffix=None):44 self._options = options45 self._bot_id = build_config['name']46 if not self._options.archive_base and self._options.remote_trybot:47 self._bot_id = 'trybot-' + self._bot_id48 self._build_config = copy.deepcopy(build_config)49 self.name = self.StageNamePrefix()50 if suffix:51 self.name += suffix52 self._boards = self._build_config['boards']53 self._build_root = os.path.abspath(self._options.buildroot)54 self._prebuilt_type = None55 if self._options.prebuilts and self._build_config['prebuilts']:56 self._prebuilt_type = self._build_config['build_type']57 # Determine correct chrome_rev.58 self._chrome_rev = self._build_config['chrome_rev']59 if self._options.chrome_rev:60 self._chrome_rev = self._options.chrome_rev61 def ConstructDashboardURL(self, stage=None):62 """Return the dashboard URL63 This is the direct link to buildbot logs as seen in build.chromium.org64 Args:65 stage: Link to a specific |stage|, otherwise the general buildbot log66 Returns:67 The fully formed URL68 """69 return validation_pool.ValidationPool.ConstructDashboardURL(70 self._build_config['overlays'], self._options.remote_trybot,71 self._build_config['name'], self._options.buildnumber, stage=stage)72 def _ExtractOverlays(self):73 """Extracts list of overlays into class."""74 overlays = portage_utilities.FindOverlays(75 self._build_config['overlays'], buildroot=self._build_root)76 push_overlays = portage_utilities.FindOverlays(77 self._build_config['push_overlays'], buildroot=self._build_root)78 # Sanity checks.79 # We cannot push to overlays that we don't rev.80 assert set(push_overlays).issubset(set(overlays))81 # Either has to be a master or not have any push overlays.82 assert self._build_config['master'] or not push_overlays83 return overlays, push_overlays84 def _Print(self, msg):85 """Prints a msg to stderr."""86 sys.stdout.flush()87 print >> sys.stderr, msg88 sys.stderr.flush()89 def _PrintLoudly(self, msg):90 """Prints a msg with loudly."""91 border_line = '*' * 6092 edge = '*' * 293 sys.stdout.flush()94 print >> sys.stderr, border_line95 msg_lines = msg.split('\n')96 # If the last line is whitespace only drop it.97 if not msg_lines[-1].rstrip():98 del msg_lines[-1]99 for msg_line in msg_lines:100 print >> sys.stderr, '%s %s' % (edge, msg_line)101 print >> sys.stderr, border_line102 sys.stderr.flush()103 def _GetPortageEnvVar(self, envvar, board):104 """Get a portage environment variable for the configuration's board.105 envvar: The environment variable to get. E.g. 'PORTAGE_BINHOST'.106 Returns:107 The value of the environment variable, as a string. If no such variable108 can be found, return the empty string.109 """110 cwd = os.path.join(self._build_root, 'src', 'scripts')111 if board:112 portageq = 'portageq-%s' % board113 else:114 portageq = 'portageq'115 binhost = cros_build_lib.RunCommand(116 [portageq, 'envvar', envvar], cwd=cwd, redirect_stdout=True,117 enter_chroot=True, error_code_ok=True)118 return binhost.output.rstrip('\n')119 @staticmethod120 def _GetSlavesForMaster(build_config, configs=None):121 """Gets the important builds corresponding to this master.122 Returns:123 A list of the slaves for this builder.124 """125 if configs is None:126 configs = cbuildbot_config.config127 builders = []128 assert build_config['manifest_version']129 assert build_config['master']130 for config in configs.itervalues():131 if (config['important'] and132 config['manifest_version'] and133 config['build_type'] == build_config['build_type'] and134 config['chrome_rev'] == build_config['chrome_rev'] and135 config['branch'] == build_config['branch']):136 builders.append(config)137 return builders138 def _Begin(self):139 """Can be overridden. Called before a stage is performed."""140 # Tell the buildbot we are starting a new step for the waterfall141 self._Print('\n@@@BUILD_STEP %s@@@\n' % self.name)142 self._PrintLoudly('Start Stage %s - %s\n\n%s' % (143 self.name, time.strftime('%H:%M:%S'), self.__doc__))144 def _Finish(self):145 """Can be overridden. Called after a stage has been performed."""146 self._PrintLoudly('Finished Stage %s - %s' %147 (self.name, time.strftime('%H:%M:%S')))148 def PerformStage(self):149 """Subclassed stages must override this function to perform what they want150 to be done.151 """152 def _HandleExceptionAsSuccess(self, _exception):153 """Use instead of HandleStageException to ignore an exception."""154 return results_lib.Results.SUCCESS, None155 @staticmethod156 def _StringifyException(exception):157 """Convert an exception into a string.158 This can only be called as part of an except block.159 """160 if isinstance(exception, results_lib.StepFailure):161 return str(exception)162 else:163 return traceback.format_exc()164 def _HandleExceptionAsWarning(self, exception):165 """Use instead of HandleStageException to treat an exception as a warning.166 This is used by the ForgivingBuilderStage's to treat any exceptions as167 warnings instead of stage failures.168 """169 cros_build_lib.PrintBuildbotStepWarnings()170 cros_build_lib.Warning(self._StringifyException(exception))171 return results_lib.Results.FORGIVEN, None172 def _HandleStageException(self, exception):173 """Called when PerformStage throws an exception. Can be overriden.174 Should return result, description. Description should be None if result175 is not an exception.176 """177 # Tell the user about the exception, and record it178 description = self._StringifyException(exception)179 cros_build_lib.PrintBuildbotStepFailure()180 cros_build_lib.Error(description)181 return exception, description182 def HandleSkip(self):183 """Run if the stage is skipped."""184 pass185 def Run(self):186 """Have the builder execute the stage."""187 if (self.option_name and not getattr(self._options, self.option_name) or188 self.config_name and not self._build_config[self.config_name]):189 self._PrintLoudly('Not running Stage %s' % self.name)190 self.HandleSkip()191 results_lib.Results.Record(self.name, results_lib.Results.SKIPPED)192 return193 record = results_lib.Results.PreviouslyCompletedRecord(self.name)194 if record:195 # Success is stored in the results log for a stage that completed196 # successfully in a previous run.197 self._PrintLoudly('Stage %s processed previously' % self.name)198 self.HandleSkip()199 results_lib.Results.Record(self.name, results_lib.Results.SUCCESS, None,200 float(record[2]))201 return202 start_time = time.time()203 # Set default values204 result = results_lib.Results.SUCCESS205 description = None206 sys.stdout.flush()207 sys.stderr.flush()208 self._Begin()209 try:210 self.PerformStage()211 except SystemExit as e:212 if e.code != 0:213 result, description = self._HandleStageException(e)214 raise215 except Exception as e:216 if mox is not None and isinstance(e, mox.Error):217 raise218 # Tell the build bot this step failed for the waterfall219 result, description = self._HandleStageException(e)220 if result not in (results_lib.Results.FORGIVEN,221 results_lib.Results.SUCCESS):222 raise results_lib.StepFailure()223 except BaseException as e:224 result, description = self._HandleStageException(e)225 raise226 finally:227 elapsed_time = time.time() - start_time228 results_lib.Results.Record(self.name, result, description,229 time=elapsed_time)230 self._Finish()231 sys.stdout.flush()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Slash automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_