Best Python code snippet using playwright-python
ebb.py
Source:ebb.py
...73 ''' Sets a property on the active node '''74 assert Scope._top is not None75 Scope._top.properties[name] = value76 @staticmethod77 def set_checked(name, value, expected_type):78 ''' Sets a property on the active node if it's not None, and checks it's79 type '''80 if value is None:81 return82 if expected_type is not None:83 assert isinstance(value, expected_type)84 Scope.set(name, value)85 @staticmethod86 def append(name, *values):87 ''' Appends values to a list property with key name on this node '''88 assert isinstance(name, basestring)89 assert Scope._top is not None90 properties = Scope._top.properties91 if name not in properties:92 properties[name] = []93 assert isinstance(properties[name], list)94 properties[name].extend(values)95 @staticmethod96 def update(name, key, value):97 ''' Adds a {key : value} entry on the dictionary named name on the top98 node '''99 assert isinstance(name, basestring)100 assert Scope._top is not None101 properties = Scope._top.properties102 if name not in properties:103 properties[name] = {}104 assert isinstance(properties[name], dict)105 properties[name][key] = value106 @staticmethod107 @contextlib.contextmanager108 def push(func, *args, **kwargs):109 ''' Pushes a scope and calls a function before yielding '''110 with Scope() as scope:111 func(*args, **kwargs)112 yield scope113 def get(self, name, default=None, public_only=False):114 ''' Search for a property up the tree and returns the first occurence115 '''116 assert isinstance(name, basestring)117 if name not in self.properties:118 value = None119 else:120 value = self.properties[name]121 if value is not None:122 if not isinstance(value, list) and not isinstance(value, dict):123 return value124 for scope in self._get_related_scopes(public_only):125 scope_value = scope.get(name, public_only=True)126 if isinstance(scope_value, list):127 if value is not None:128 assert isinstance(value, list)129 scope_value.extend(value)130 elif isinstance(scope_value, dict):131 if value is not None:132 assert isinstance(value, dict)133 scope_value.update(value)134 if scope_value is not None:135 value = scope_value136 if isinstance(value, list):137 return value[:]138 elif isinstance(value, dict):139 return value.copy()140 return value if value is not None else default141 def get_interpolated(self, name, default=None):142 ''' Search for a property up the tree, and returns it's value143 interpolated with this node and all of it's parents property values144 '''145 assert isinstance(name, basestring)146 result = self.get(name, default)147 return self.interpolate(result)148 def interpolate(self, value):149 ''' Interpolates value with all properties from current node and it's150 parents151 '''152 if hasattr(value, '__call__'):153 return value(self)154 elif isinstance(value, basestring):155 format_args = self.get_interpolation_values()156 try:157 return value.format(**format_args)158 except KeyError as error:159 raise KeyError('Key %s not found while interpolating %s on %s' % (error, value, self))160 elif isinstance(value, list):161 return [self.interpolate(it) for it in value]162 elif isinstance(value, dict):163 result = {}164 for key, it in value.iteritems():165 result[key] = self.interpolate(it)166 return result167 return value168 def get_rendered(self, name, default=None):169 ''' Search for a property up the tree, and returns it's value170 interpolated with this node and all of it's parents property values171 '''172 assert isinstance(name, basestring)173 result = self.get(name, default)174 return self.render(result)175 def render(self, value):176 ''' Interpolates value with all properties from current node and it's177 parents178 '''179 if isinstance(value, basestring):180 return _Renderer(value, self)181 elif isinstance(value, list):182 return [self.render(it) for it in value]183 elif isinstance(value, dict):184 result = {}185 for key, it in value.iteritems():186 result[key] = self.render(it)187 return result188 return value189 def get_interpolation_values(self, public_only=False):190 ''' Parses the tree bottom-up, getting string property values '''191 result = {}192 for scope in self._get_related_scopes(public_only):193 scope_values = scope.get_interpolation_values(True)194 result.update(scope_values)195 for key, value in self.properties.iteritems():196 if isinstance(value, basestring):197 result[key] = value198 return result199 def get_parent_of_type(self, parent_type):200 ''' Find closest parent of specified node type '''201 if self._parent is None:202 return None203 if isinstance(self._parent, parent_type):204 return self._parent205 return self._parent.get_parent_of_type(parent_type)206 def build(self, config):207 ''' Builds this node '''208 for child in self.children:209 child.build(config)210 self._build(config)211 def _get_related_scopes(self, public_only):212 if self._parent is not None:213 yield self._parent214 if not public_only:215 for it in self.children:216 if isinstance(it, Private):217 yield it218 def get_properties_names(self, prefix, public_only=False):219 ''' Returs all properties defined on this node and parent starting220 with given prefix '''221 result = set()222 for scope in self._get_related_scopes(public_only):223 result = result | scope.get_properties_names(prefix, True)224 for key, _ in self.properties.iteritems():225 if key.startswith(prefix + '_'):226 result.add(key)227 return result228 @abc.abstractmethod229 def _build(self, config):230 pass231 def _get_prefixed_properties(self, prefixes, rendered=None, raw=None):232 result = {}233 rendered = set([] if rendered is None else rendered)234 raw = set(() if raw is None else raw)235 if not isinstance(prefixes, tuple):236 assert isinstance(prefixes, str)237 prefixes = (prefixes,)238 props = {}239 for prefix in prefixes:240 interpolated_it = self.get_properties_names(prefix)241 raw_it = interpolated_it & raw242 rendered_it = interpolated_it & rendered243 interpolated_it = interpolated_it - raw_it - rendered_it244 props[prefix] = (interpolated_it, raw_it, rendered_it)245 for prefix, (interpolated_it, raw_it, rendered_it) in props.iteritems():246 for key in interpolated_it:247 key_without_prefix = key[len(prefix) + 1:]248 result[key_without_prefix] = self.get_interpolated(key)249 for key in raw_it:250 key_without_prefix = key[len(prefix) + 1:]251 result[key_without_prefix] = self.get(key)252 for key in rendered_it:253 key_without_prefix = key[len(prefix) + 1:]254 result[key_without_prefix] = self.get_rendered(key)255 # result[name] = self.properties[key]256 return result257 def _build_class(self,258 buildbot_class,259 prefixes,260 positional=None,261 raw=None,262 rendered=None,263 additional=None):264 kwargs = self._get_prefixed_properties(prefixes,265 rendered=rendered,266 raw=raw)267 if additional is not None:268 kwargs.update(additional)269 args = []270 if positional is not None:271 for name in positional:272 assert name in kwargs, '%s argument missing' % name273 args.append(kwargs[name])274 del kwargs[name]275 return buildbot_class(*args, **kwargs)276class Private(Scope):277 ''' Defines not inherited values on the parent scope '''278 def __init__(self):279 super(Private, self).__init__()280 def _get_related_scopes(self, public_only):281 return []282 def _build(self, config):283 pass284class Config(Scope):285 ''' Root config node '''286 def __init__(self):287 super(Config, self).__init__()288 self.buildbot_config = {}289 self._parsers = {}290 self._schedulers = {}291 self._slaves = []292 self._triggerables = {}293 self._locks = {}294 self._builders_scopes = {}295 self.buildbot_config['builders'] = []296 self.buildbot_config['schedulers'] = []297 self.buildbot_config['slaves'] = []298 self.buildbot_config['status'] = []299 self.buildbot_config['change_source'] = []300 self.buildbot_config['prioritizeBuilders'] = self._prioritize_builders301 self.slave_list_selector = None302 self.next_slave_selector = None303 @staticmethod304 def db(url, poll_interval=None):305 ''' Configures db parameters '''306 Scope.set_checked('db_db_url', url, basestring)307 Scope.set_checked('db_poll_interval', poll_interval, int)308 @staticmethod309 def site(title, title_url, buildbot_url):310 ''' Configures site parameters '''311 Scope.set_checked('base_title', title, basestring)312 Scope.set_checked('base_titleURL', title_url, basestring)313 Scope.set_checked('base_buildbotURL', buildbot_url, basestring)314 @staticmethod315 def logging(compression_limit=None,316 compression_method=None,317 max_size=None,318 max_tail_size=None):319 ''' Configures logging parameters '''320 Scope.set_checked('base_logCompressionLimit', compression_limit, int)321 Scope.set_checked('base_logCompressionMethod',322 compression_method,323 basestring)324 Scope.set_checked('base_logMaxSize', max_size, int)325 Scope.set_checked('base_logMaxTailSize', max_tail_size, int)326 @staticmethod327 def horizons(change_horizon=None,328 build_horizon=None,329 event_horizon=None,330 log_horizon=None):331 ''' Configures horizon parameters '''332 Scope.set_checked('base_changeHorizon', change_horizon, int)333 Scope.set_checked('base_buildHorizon', build_horizon, int)334 Scope.set_checked('base_eventHorizon', event_horizon, int)335 Scope.set_checked('base_logHorizon', log_horizon, int)336 @staticmethod337 def cache(changes=None,338 builds=None,339 chdicts=None,340 build_requests=None,341 source_stamps=None,342 ssdicts=None,343 objectids=None,344 usdicts=None):345 ''' Configures horizon parameters '''346 Scope.set_checked('cache_Changes', changes, int)347 Scope.set_checked('cache_Builds', builds, int)348 Scope.set_checked('cache_chdicts', chdicts, int)349 Scope.set_checked('cache_BuildRequests', build_requests, int)350 Scope.set_checked('cache_SourceStamps', source_stamps, int)351 Scope.set_checked('cache_ssdicts', ssdicts, int)352 Scope.set_checked('cache_objectids', objectids, int)353 Scope.set_checked('cache_usdicts', usdicts, int)354 @staticmethod355 def set_protocol(protocol, port):356 ''' Sets given protocol to given port '''357 Scope.update('protocols_%s' % protocol, 'port', port)358 @staticmethod359 def web_status(port, user, password):360 ''' Sets web status configuration '''361 Scope.set_checked('web_status_port', port, int)362 Scope.set_checked('web_status_user', user, str)363 Scope.set_checked('web_status_password', password, str)364 @staticmethod365 def add_renderer_handlers(*handlers):366 ''' Add rendering handlers that can udpate rendering arguments at build367 time '''368 Scope.append('config_renderer_handlers', *handlers)369 def get_builder(self, name):370 ''' Returns a declared Builder '''371 return self._builders_scopes[name]372 def get_slave(self, name):373 ''' Returns a declared slave '''374 for slave in self._slaves:375 if slave.get_interpolated('slave_name') == name:376 return slave377 return None378 def build_config(self):379 ''' Builds the buildbot config '''380 self.build(self)381 return self.buildbot_config382 def add_slave(self, slave):383 ''' Adds a slave for later tag filtering '''384 self._slaves.append(slave)385 def add_builder(self, builder, scope):386 ''' Adds a builder to this config '''387 self._builders_scopes[builder.name] = scope388 self.buildbot_config['builders'].append(builder)389 def get_slave_list(self, *tags):390 ''' Returns declared slaves matching *all* given tags391 Tags can be excluded if they start with â!â '''392 # TODO: allow the tag1|tag2 syntax for e.g. 'linux|win64'393 result = []394 wanted_tagset, unwanted_tagset = set(), set()395 for tag in tags:396 if tag[:1] != '!':397 wanted_tagset.add(tag)398 else:399 unwanted_tagset.add(tag[1:])400 for slave in self._slaves:401 slave_tagset = set()402 for tag in slave.get_interpolated('_slave_tags', []):403 slave_tagset.add(tag)404 if len(wanted_tagset - slave_tagset) == 0 and \405 len(unwanted_tagset.intersection(slave_tagset)) == 0:406 slave_name = slave.get_interpolated('slave_name')407 result.append(slave_name)408 if len(result) == 0:409 print 'Error : no slave found with tags %s and without tags %s' \410 % (wanted_tagset, unwanted_tagset)411 for slave in self._slaves:412 slave_tagset = set()413 for tag in slave.get_interpolated('_slave_tags', []):414 slave_tagset.add(tag)415 args = (slave.get_interpolated('slave_name'),416 wanted_tagset - slave_tagset)417 print 'Slave %s is missing tags %s' % args418 return result419 def _build(self, config):420 assert config == self421 conf_dict = config.buildbot_config422 # Db config423 conf_dict['db'] = self._get_prefixed_properties('db')424 conf_dict['caches'] = self._get_prefixed_properties('cache')425 conf_dict['protocols'] = self._get_prefixed_properties('protocols')426 conf_dict.update(self._get_prefixed_properties('base'))427 self._add_web_status()428 def _add_web_status(self):429 http_port = self.get('web_status_port')430 if http_port is None:431 return432 users = [(self.get_interpolated('web_status_user'),433 self.get_interpolated('web_status_password'))]434 auth = buildbot.status.web.auth.BasicAuth(users)435 authz = buildbot.status.web.authz.Authz(auth=auth,436 view=True,437 gracefulShutdown='auth',438 forceBuild='auth',439 forceAllBuilds='auth',440 pingBuilder='auth',441 stopBuild='auth',442 stopAllBuilds='auth',443 cancelPendingBuild='auth',444 showUsersPage='auth',445 cleanShutdown='auth')446 web_status = buildbot.status.html.WebStatus(http_port=http_port,447 authz=authz)448 self.buildbot_config['status'].append(web_status)449 def _prioritize_builders(self, _, builders):450 def _get_priority(builder):451 try:452 return self._builders_scopes[builder.name].get('_builder_priority', 0)453 except KeyError:454 return 99999455 builders.sort(key=_get_priority, reverse=True)456 return builders457class Slave(Scope):458 ''' Creates a new buildbot slave '''459 def __init__(self, name):460 super(Slave, self).__init__()461 self.properties['slave_name'] = name462 config = self.get_parent_of_type(Config)463 assert config is not None464 config.add_slave(self)465 @staticmethod466 def config(password=None,467 max_builds=None,468 keepalive_interval=300,469 missing_timeout=None):470 ''' Sets some buildbot slaves settings for current scope '''471 Scope.set_checked('slave_password', password, basestring)472 Scope.set_checked('slave_max_builds', max_builds, int)473 Scope.set_checked('slave_keepalive_interval', keepalive_interval, int)474 Scope.set_checked('slave_missing_timeout', missing_timeout, int)475 @staticmethod476 def add_property(key, value):477 ''' Adds a build property on this slave '''478 Scope.update('slave_properties', key, value)479 @staticmethod480 def add_notified_on_missing(*emails):481 ''' Adds emails to notify when slaves in scope are missing '''482 Scope.append('slave_notify_on_missing', *emails)483 @staticmethod484 def add_tags(*tags):485 ''' Adds specified tags to slaves in scope '''486 Scope.append('_slave_tags', *tags)487 def _build(self, config):488 slave = self._build_class(buildbot.buildslave.BuildSlave, 'slave',489 ['name', 'password'])490 config.buildbot_config['slaves'].append(slave)491class Builder(Scope):492 ''' Builder wrapper '''493 def __init__(self, name, category=None, description=None):494 super(Builder, self).__init__()495 self._accept_regex = None496 self._reject_regex = None497 self._factory = buildbot.process.factory.BuildFactory()498 self._nightly = None499 self.properties['builder_name'] = name500 if category is not None:501 self.properties['builder_category'] = category502 if description is not None:503 self.properties['builder_description'] = description504 def add_step(self, step):505 ''' Adds a step to this builder '''506 self._factory.addStep(step)507 def trigger_on_change(self, accept_regex='.*', reject_regex=None):508 ''' Triggers this build on change from source control '''509 self._accept_regex = accept_regex510 self._reject_regex = reject_regex511 def trigger_nightly(self,512 minute=None,513 hour=None,514 day_of_month=None,515 month=None,516 day_of_week=None):517 ''' Triggers this build nightly '''518 self._nightly = {'minute' : minute,519 'hour' : hour,520 'dayOfMonth': day_of_month,521 'month' : month,522 'dayOfWeek' : day_of_week}523 @staticmethod524 def config(name=None,525 category=None,526 build_dir=None,527 slave_build_dir=None,528 next_build=None,529 can_start_build=None,530 merge_requests=None,531 forcable=None,532 only_important=None,533 tree_stable_timer=None,534 file_is_important=None,535 project=None,536 priority=None):537 ''' Sets builder config values '''538 Scope.set_checked('builder_name', name, basestring)539 Scope.set_checked('builder_category', category, basestring)540 Scope.set_checked('builder_builddir', build_dir, None)541 Scope.set_checked('builder_slavebuilddir', slave_build_dir, basestring)542 Scope.set_checked('builder_nextBuild', next_build, None)543 Scope.set_checked('builder_canStartBuild', can_start_build, None)544 Scope.set_checked('builder_mergeRequests', merge_requests, None)545 Scope.set_checked('scheduler_onlyImportant', only_important, bool)546 Scope.set_checked('branch_scheduler_treeStableTimer', tree_stable_timer, int)547 Scope.set_checked('scheduler_fileIsImportant', file_is_important, None)548 Scope.set_checked('change_filter_project', project, basestring)549 Scope.set_checked('_builder_priority', priority, int)550 @staticmethod551 def mail_config(from_address=None,552 send_to_interested_users=None,553 subject=None,554 mode=None,555 add_logs=None,556 relay_host=None,557 smpt_port=None,558 use_tls=None,559 smtp_user=None,560 smtp_password=None,561 lookup=None,562 message_formatter=None,563 template_directory=None,564 template=None,565 body_type=None):566 ''' Sets mail related settings '''567 Scope.set_checked('mail_fromaddr', from_address, str)568 Scope.set_checked('mail_sendToInterestedUsers',569 send_to_interested_users, bool)570 Scope.set_checked('mail_subject', subject, str)571 Scope.set_checked('mail_mode', mode, None)572 Scope.set_checked('mail_addLogs', add_logs, None)573 Scope.set_checked('mail_relayhost', relay_host, str)574 Scope.set_checked('mail_smtpPort', smpt_port, int)575 Scope.set_checked('mail_useTls', use_tls, bool)576 Scope.set_checked('mail_smtpUser', smtp_user, str)577 Scope.set_checked('mail_smtpPassword', smtp_password, str)578 Scope.set_checked('mail_lookup', lookup, None)579 Scope.set_checked('_mail_message_formatter', message_formatter, None)580 Scope.set_checked('_mail_template_directory', template_directory, None)581 Scope.set_checked('_mail_template', template, None)582 Scope.set_checked('_mail_body_type', body_type, None)583 @staticmethod584 def add_extra_recipients(*emails):585 ''' Adds extra recipients to users '''586 Scope.append('mail_extraRecipients', *emails)587 @staticmethod588 def add_slave_tags(*tags):589 ''' Adds builder tags to current scope '''590 Scope.append('_builder_slave_tags', *tags)591 @staticmethod592 def add_env_variable(name, value):593 ''' Adds an envrionment variable to builders in scope '''594 Scope.update('builder_env', name, value)595 @staticmethod596 def add_tags(*tags):597 ''' Adds a tag to a builder '''598 Scope.append('builder_tags', *tags)599 @staticmethod600 def add_property(name, value):601 ''' Adds a property to builders in scope '''602 Scope.update('builder_properties', name, value)603 def _build(self, config):604 if config.slave_list_selector:605 slavenames = config.slave_list_selector(self)606 else:607 slave_tags = self.get_interpolated('_builder_slave_tags', [])608 slavenames = config.get_slave_list(*slave_tags)609 # TODO locks = get_locks('job', config, scope)610 args = {611 'slavenames': slavenames,612 'nextSlave': config.next_slave_selector,613 'factory': self._factory,614 }615 builder = self._build_class(buildbot.config.BuilderConfig, 'builder', additional=args)616 config.add_builder(builder, self)617 self._add_single_branch_scheduler(config)618 self._add_nightly_scheduler(config)619 self._add_mail_status(config)620 parent_trigger = self.get_parent_of_type(Trigger)621 if parent_trigger is not None:622 parent_trigger.add_builder(self.get_interpolated('builder_name'))623 def _add_single_branch_scheduler(self, config):624 if self._accept_regex is None:625 return626 builder_name = self.get_interpolated('builder_name')627 project_name = self.get_interpolated('project_name')628 args = {629 'filter_fn': _ChangeFilter(builder_name, project_name,630 self.interpolate(self._accept_regex),631 self.interpolate(self._reject_regex))632 }633 change_filter = self._build_class(buildbot.changes.filter.ChangeFilter,634 'change_filter',635 additional=args)636 args = {'name' : '%s single branch scheduler' % builder_name,637 'builderNames' : [builder_name],638 'change_filter' : change_filter,639 'reason' : 'A CL Triggered this build'}640 scheduler_class = buildbot.schedulers.basic.SingleBranchScheduler641 scheduler = self._build_class(scheduler_class,642 ('scheduler', 'branch_scheduler'),643 additional=args)644 config.buildbot_config['schedulers'].append(scheduler)645 def _add_nightly_scheduler(self, config):646 if self._nightly is None:647 return648 builder_name = self.get_interpolated('builder_name')649 args = {'name' : '%s nightly scheduler' % builder_name,650 'builderNames' : [builder_name],651 'branch' : None} #We don't use branches the way buildbot expects it652 for key, value in self._nightly.iteritems():653 if value is not None:654 args[key] = value655 scheduler_class = buildbot.schedulers.timed.Nightly656 scheduler = self._build_class(scheduler_class,657 'scheduler',658 additional=args)659 config.buildbot_config['schedulers'].append(scheduler)660 def _add_mail_status(self, config):661 extra_recipients = self.get_interpolated('mail_extraRecipients')662 send_mail = self.get_interpolated('mail_sendToInterestedUsers')663 if extra_recipients or send_mail:664 formatter = self.get('_mail_message_formatter')665 if formatter is None:666 formatter = _HtmlMailFormatter(self)667 args = {'messageFormatter': formatter,668 'builders': [self.get_interpolated('builder_name')]}669 mail_status = self._build_class(buildbot.status.mail.MailNotifier,670 'mail',671 additional=args)672 config.buildbot_config['status'].append(mail_status)673class Repository(Scope):674 ''' Change source base scope '''675 def __init__(self, name, is_polling_enabled):676 super(Repository, self).__init__()677 self.name = name678 self.is_polling_enabled = is_polling_enabled679 Scope.update('source_control_repositories', name, self)680 @staticmethod681 def config(poll_interval=None,682 poll_at_launch=None,683 hitsmax=None):684 ''' Common change source parameters '''685 Scope.set_checked('change_source_pollInterval', poll_interval, int)686 Scope.set_checked('change_source_pollAtLaunch', poll_at_launch, bool)687 Scope.set_checked('change_source_hitsmax', hitsmax, int)688 @abc.abstractmethod689 def get_sync_step(self, config, step_args):690 ''' Returns a step to sync this repository '''691 @abc.abstractmethod692 def _build_change_sources(self, config, args):693 ''' Creates a ChangeSource for this repository '''694 def _build(self, config):695 if not self.is_polling_enabled:696 return697 args = self._get_prefixed_properties('change_source')698 for change_source in self._build_change_sources(config, args):699 # XXX: this attribute works around a bug in buildbot that would700 # cause it to only trigger rebuilds for the first project using701 # that source702 change_source.compare_attrs.append('project')703 config.buildbot_config['change_source'].append(change_source)704class P4StreamSource(buildbot.changes.p4poller.P4Source):705 def __init__(self, **args):706 self._stream = None707 super(P4StreamSource, self).__init__(**args)708 @defer.inlineCallbacks709 #pylint: disable=invalid-name,missing-docstring710 def _get_process_output(self, args):711 base_get_process_output = super(P4StreamSource, self)._get_process_output712 # If action is not 'p4 changes', use the original function713 if 'changes' not in args:714 tmp = yield base_get_process_output(args)715 defer.returnValue(tmp)716 # Last argument is the location we're polling717 location, suffix = args[-1], ""718 if '...' in location:719 n = location.index('...')720 location, suffix = location[:n], location[n:]721 if self._stream:722 location = self._stream723 client = re.sub('[^a-zA-Z0-9]+', '-', 'poll-' + location).lower()724 # All arguments before changes are P4 options725 argc = args.index('changes')726 baseargs = args[:argc]727 # Check whether the location is a stream; otherwise, bail out728 tmp = yield base_get_process_output(baseargs + ['streams'])729 if 'Stream %s ' % location not in tmp:730 tmp = yield base_get_process_output(args)731 defer.returnValue(tmp)732 # Force p4base to be // in order to catch all changes to this client733 self._stream = location734 self.p4base = '//'735 # Check that our client references the stream736 tmp = yield base_get_process_output(baseargs + ['client', '-o', client])737 if 'Stream:\t%s' % location not in tmp:738 # Ensure the client exists739 p4clientcmd = '%s %s client' % (self.p4bin, ' '.join(baseargs))740 shargs = '%s -o %s | %s -i' % (p4clientcmd, client, p4clientcmd)741 tmp = yield utils.getProcessOutput('/bin/sh', ['-c', shargs])742 # Force switch the client stream743 tmp = yield base_get_process_output(baseargs + ['client', '-f', '-s', '-S', location, client])744 tmp = yield base_get_process_output(['-c', client] + args[:-1] + ['//%s%s' % (client, suffix)])745 defer.returnValue(tmp)746class P4Repository(Repository):747 ''' P4Repository handling '''748 def __init__(self, name, is_polling_enabled):749 super(P4Repository, self).__init__(name, is_polling_enabled)750 @staticmethod751 def config(port=None, user=None, password=None, client=None,752 binary=None, encoding=None, timezone=None, spec_options=None):753 ''' Common global p4 parameters '''754 # TODO : Add ticket management755 Scope.set_checked('p4_common_p4port', port, str)756 Scope.set_checked('p4_common_p4user', user, str)757 Scope.set_checked('p4_common_p4passwd', password, str)758 Scope.set_checked('p4_sync_p4client_spec_options', spec_options, str)759 Scope.set_checked('p4_sync_p4client', client, str)760 Scope.set_checked('p4_poll_p4bin', binary, str)761 Scope.set_checked('p4_poll_encoding', encoding, str)762 Scope.set_checked('p4_poll_server_tz', timezone, None)763 @staticmethod764 def add_views(*views):765 ''' Adds p4 mappings for current scope '''766 Scope.append('p4_sync_p4viewspec', *views)767 @staticmethod768 def set_stream(stream):769 ''' Adds p4 stream for current scope '''770 Scope.append('_p4stream', stream)771 # Hack the âViewâ entry. The view will be invalid, but it will be772 # overridden by the âStreamâ entry that we insert.773 # See master/buildbot/steps/source/p4.py in the buildbot sources774 # to understand why it works.775 Scope.append('p4_sync_p4viewspec',776 ('//ignored/', '...\n\nStream: ' + stream + '\n\n#'))777 def get_sync_step(self, _, step_args):778 ''' Returns sync step for this repository '''779 return self._build_class(buildbot.steps.source.p4.P4,780 ('p4_common', 'p4_sync'),781 rendered=['p4_sync_p4client'],782 additional=step_args)783 def _build_change_sources(self, config, args):784 paths_to_poll = []785 for (depot_path, _) in self.get('p4_sync_p4viewspec', []):786 if depot_path.startswith('//'):787 # Get depot from //depot/788 base = depot_path[2:-1]789 paths_to_poll.append(base)790 for base in paths_to_poll:791 split_file = lambda branchfile: (None, branchfile)792 args['split_file'] = split_file793 args['p4base'] = '//' + base794 project_name = self.get_interpolated('project_name')795 if project_name is not None:796 args['project'] = project_name797 p4 = self._build_class(P4StreamSource,798 ('p4_common', 'p4_poll'),799 additional=args)800 yield p4801class GitRepository(Repository):802 ''' Git repository '''803 def __init__(self, name, repo_url, is_polling_enabled):804 super(GitRepository, self).__init__(name, is_polling_enabled)805 Scope.set_checked('git_common_repourl', repo_url, str)806 @staticmethod807 def config(git_bin=None,808 use_time_stamps=None,809 encoding=None,810 branch=None,811 submodules=True,812 shallow=None,813 progress=None,814 retry_fetch=None,815 clobber_on_failure=None,816 method=None):817 ''' Common global git parameters '''818 Scope.set_checked('git_poll_gitbin', git_bin, str)819 Scope.set_checked('get_poll_usetimestamps', use_time_stamps, bool)820 Scope.set_checked('git_poll_encoding', encoding, str)821 Scope.set_checked('git_common_branch', branch, str)822 Scope.set_checked('git_sync_submodules', submodules, bool)823 Scope.set_checked('git_sync_shallow', shallow, bool)824 Scope.set_checked('git_sync_progress', progress, bool)825 Scope.set_checked('git_sync_retryFetch', retry_fetch, bool)826 Scope.set_checked('git_sync_clobberOnFailure', clobber_on_failure, bool)827 Scope.set_checked('git_sync_method', method, str)828 assert method in [None, 'clobber', 'fresh', 'clean', 'copy']829 def get_sync_step(self, _, step_args):830 ''' Returns sync step for this repository '''831 return self._build_class(buildbot.steps.source.git.Git,832 ('git_common', 'git_sync'),833 additional=step_args)834 def _build_change_sources(self, config, args):835 project_name = self.get_interpolated('project_name')836 if project_name is not None:837 args['project'] = project_name838 yield self._build_class(buildbot.changes.gitpoller.GitPoller,839 ('git_common', 'git_poll'),840 additional=args)841class Step(Scope):842 ''' Build step '''843 def __init__(self, name):844 super(Step, self).__init__()845 self.properties['step_name'] = name846 @staticmethod847 def config(halt_on_failure=None,848 flunk_on_warnings=None,849 flunk_on_failure=None,850 warn_on_warnings=None,851 warn_on_failure=None,852 always_run=None,853 description=None,854 description_done=None,855 do_step_if=None,856 hide_step_if=None,857 workdir=None,858 timeout=None):859 ''' Fail behavior settings '''860 Scope.set_checked('step_haltOnFailure', halt_on_failure, bool)861 Scope.set_checked('step_flunkOnWarnings', flunk_on_warnings, bool)862 Scope.set_checked('step_flunkOnFailure', flunk_on_failure, bool)863 Scope.set_checked('step_warnOnWarnings', warn_on_warnings, bool)864 Scope.set_checked('step_warnOnFailure', warn_on_failure, bool)865 Scope.set_checked('step_alwaysRun', always_run, bool)866 Scope.set_checked('step_description', description, str)867 Scope.set_checked('step_descriptionDone', description_done, str)868 Scope.set_checked('step_doStepIf', do_step_if, None)869 Scope.set_checked('step_hideStepIf', hide_step_if, None)870 Scope.set_checked('step_workdir', workdir, str)871 Scope.set_checked('step_timeout', timeout, int)872 @abc.abstractmethod873 def _get_step(self, config, step_args):874 ''' Builds the buildbot step '''875 pass876 def _build(self, config):877 builder = self.get_parent_of_type(Builder)878 if builder is None:879 step_name = self.get_interpolated('step_name')880 msg = 'Step %s not declared in Builder scope' % step_name881 raise Exception(msg)882 step_args = self._get_prefixed_properties('step')883 builder.add_step(self._get_step(config, step_args))884class Sync(Step):885 ''' Syncs a previoulsy declared repository '''886 def __init__(self, repo_name):887 super(Sync, self).__init__('sync %s' % repo_name)888 self._repo_name = repo_name889 @staticmethod890 def config(mode=None,891 always_use_latest=None,892 retry=None,893 log_environ=None):894 ''' Common sync steps config '''895 Scope.set_checked('sync_mode', mode, str)896 assert mode in ['incremental', 'full', None]897 Scope.set_checked('sync_alwaysUseLatest', always_use_latest, bool)898 Scope.set_checked('sync_retry', retry, tuple)899 assert retry is None or len(retry) == 2900 Scope.set_checked('sync_logEnviron', log_environ, bool)901 def _get_step(self, config, step_args):902 repos = self.get_interpolated('source_control_repositories')903 if repos is None or self._repo_name not in repos:904 msg = 'Unable to find repository %s in scope.' % self._repo_name905 raise Exception(msg)906 step_args.update(self._get_prefixed_properties('sync'))907 return repos[self._repo_name].get_sync_step(config, step_args)908class Command(Step):909 ''' Executes a shell command '''910 def __init__(self, name, command):911 super(Command, self).__init__(name)912 self._command = shlex.split(command.strip())913 @staticmethod914 def config(want_stdout=None, want_stderr=None, lazy_log_files=None,915 max_time=None, interrupt_signal=None, sigterm_time=None,916 initial_stdin=None):917 Scope.set_checked('shell_command_want_stdout', want_stdout, bool)918 Scope.set_checked('shell_command_want_stderr', want_stderr, bool)919 Scope.set_checked('shell_command_lazylogfiles', lazy_log_files, bool)920 Scope.set_checked('shell_command_maxTime', max_time, int)921 assert interrupt_signal in [None, 'KILL', 'TERM']922 Scope.set_checked('shell_command_interruptSignal', interrupt_signal, str)923 Scope.set_checked('shell_command_sigterm_time', sigterm_time, int)924 Scope.set_checked('shell_command_initialStdin', initial_stdin, int)925 @staticmethod926 def set_decode_rc(return_value, meaning):927 ''' Adds a return code meaning to this command '''928 assert isinstance(return_value, int)929 buildbot_enum = {'success' : buildbot.status.results.SUCCESS,930 'warnings' : buildbot.status.results.WARNINGS,931 'error' : buildbot.status.results.FAILURE}932 assert meaning in buildbot_enum933 Scope.update('shell_command_decodeRC',934 return_value,935 buildbot_enum[meaning])936 @staticmethod937 def set_log_file(name, path):938 ''' Adds a log file to this command '''939 Scope.update('shell_command_logfiles', name, path)940 def _get_step(self, config, step_args):941 step_args['command'] = self.render(self._command)942 return self._build_class(buildbot.steps.shell.ShellCommand,943 'shell_command',944 rendered=['shell_command_logfiles'],945 additional=step_args)946class Pylint(Step):947 ''' Runs pylint '''948 def __init__(self, command):949 super(Pylint, self).__init__('pylint')950 self._command = command951 def _get_step(self, config, step_args):952 step_args['command'] = self.render(shlex.split(self._command))953 return self._build_class(buildbot.steps.python.PyLint,954 (),955 additional=step_args)956class Sphinx(Step):957 ''' Builds sphinx documentation '''958 def __init__(self):959 super(Sphinx, self).__init__('build sphink documentation')960 @staticmethod961 def config(build_dir=None,962 source_dir=None,963 builder=None,964 sphinx=None,965 tags=None,966 defines=None,967 mode=None):968 Scope.set_checked('sphinx_sphinx_builddir', build_dir, str)969 Scope.set_checked('sphinx_sphinx_sourcedir', source_dir, str)970 Scope.set_checked('sphinx_sphinx_builder', builder, str)971 Scope.set_checked('sphinx_sphinx', sphinx, str)972 Scope.set_checked('sphinx_tags', tags, str)973 Scope.set_checked('sphinx_defines', defines, str)974 Scope.set_checked('sphinx_mode', mode, str)975 def _get_step(self, config, step_args):976 return self._build_class(buildbot.steps.python.Sphinx,977 ('sphinx'),978 additional=step_args)979class _ChangeFilter(object):980 ''' Callable filtering change matching a regular expression against modified981 files982 '''983 def __init__(self, builder, project, accept=None, reject=None):984 self._builder = builder985 self._project = project986 self._accept = accept987 self._reject = reject988 def __call__(self, change):989 msg_prefix = 'ChangeFilter: checking change %s with %s' % (change.revision, self._builder)990 if self._project != change.project:991 log.msg('%s: not our project (%s != %s)' % (msg_prefix, self._project, change.project))992 return False993 if change.who.lower() == 'buildbot':994 log.msg('%s: ignoring user buildbot' % msg_prefix)995 return False996 if '[skip]' in change.comments.lower():997 log.msg('%s: ignoring [skip] tag' % msg_prefix)998 return False999 if self._accept == '.*' and self._reject is None:1000 log.msg('%s: accepted (accept rule is .*)' % msg_prefix)1001 return True1002 for file_path in change.files:1003 is_accepted = self._accept is None or re.match(self._accept, file_path) is not None1004 is_refused = self._reject is not None and re.match(self._reject, file_path) is not None1005 if is_accepted and not is_refused:1006 log.msg('%s: accepted (%s)' % (msg_prefix, file_path))1007 return True1008 log.msg('%s: no file matching %s and not matching %s' % (msg_prefix, self._accept, self._reject))1009 return False1010class Trigger(Step):1011 ''' Triggers builders declared in child scope '''1012 def __init__(self, name, *builder_names):1013 super(Trigger, self).__init__(name)1014 self._builder_names = []1015 self._builder_names.extend(builder_names)1016 self._nightly = None1017 @staticmethod1018 @contextlib.contextmanager1019 def builder(name, category=None, description=None):1020 ''' Helper to trigger a single builder '''1021 with Trigger('%s-trigger' % name):1022 with Builder(name, category, description) as builder:1023 yield builder1024 @staticmethod1025 def config(wait_for_finish=None,1026 always_use_latest=None):1027 Scope.set_checked('trigger_waitForFinish', wait_for_finish, bool)1028 Scope.set_checked('trigger_alwaysUseLatest', always_use_latest, bool)1029 def nightly(self,1030 minute=None,1031 hour=None,1032 day_of_month=None,1033 month=None,1034 day_of_week=None):1035 ''' Uses a nightly triggerable instead of a triggerable '''1036 self._nightly = {'minute' : minute,1037 'hour' : hour,1038 'dayOfMonth': day_of_month,1039 'month' : month,1040 'dayOfWeek' : day_of_week}1041 def add_builder(self, builder_name):1042 ''' Adds a builder to this trigger '''...
configform.py
Source:configform.py
...597 Sets the state of all the gui components on this ConfigForm to match the 598 contents of the given Configuration object.599 '''600 601 def set_checked( checkbox, checked ):602 self.__update_checklist.SetItemChecked( \603 self.__update_checklist.Items.IndexOf(checkbox), checked )604 605 # 1. --- set get the parts in the checklist box (data tab)606 set_checked(ConfigForm.__SERIES_CB, config.update_series_b)607 set_checked(ConfigForm.__NUMBER_CB, config.update_number_b)608 set_checked(ConfigForm.__PUBLISHED_CB, config.update_published_b)609 set_checked(ConfigForm.__RELEASED_CB, config.update_released_b)610 set_checked(ConfigForm.__TITLE_CB, config.update_title_b)611 set_checked(ConfigForm.__CROSSOVERS_CB, config.update_crossovers_b)612 set_checked(ConfigForm.__WRITER_CB, config.update_writer_b)613 set_checked(ConfigForm.__PENCILLER_CB, config.update_penciller_b)614 set_checked(ConfigForm.__INKER_CB, config.update_inker_b)615 set_checked(ConfigForm.__COVER_ARTIST_CB,config.update_cover_artist_b)616 set_checked(ConfigForm.__COLORIST_CB, config.update_colorist_b)617 set_checked(ConfigForm.__LETTERER_CB, config.update_letterer_b)618 set_checked(ConfigForm.__EDITOR_CB, config.update_editor_b)619 set_checked(ConfigForm.__SUMMARY_CB, config.update_summary_b)620 set_checked(ConfigForm.__IMPRINT_CB, config.update_imprint_b)621 set_checked(ConfigForm.__PUBLISHER_CB, config.update_publisher_b)622 set_checked(ConfigForm.__VOLUME_CB, config.update_volume_b)623 set_checked(ConfigForm.__CHARACTERS_CB, config.update_characters_b)624 set_checked(ConfigForm.__TEAMS_CB, config.update_teams_b)625 set_checked(ConfigForm.__LOCATIONS_CB, config.update_locations_b)626 set_checked(ConfigForm.__WEBPAGE_CB, config.update_webpage_b)627 628 # 2. --- then get the parts in the other checkboxes (options tab)629 self.__ow_existing_cb.Checked = config.ow_existing_b630 self.__convert_imprints_cb.Checked = config.convert_imprints_b631 self.__autochoose_series_cb.Checked = config.autochoose_series_b632 self.__confirm_issue_cb.Checked = config.confirm_issue_b633 self.__ignore_blanks_cb.Checked = config.ignore_blanks_b634 self.__download_thumbs_cb.Checked = config.download_thumbs_b635 self.__preserve_thumbs_cb.Checked = config.preserve_thumbs_b636 self.__fast_rescrape_cb.Checked = config.fast_rescrape_b637 self.__rescrape_notes_cb.Checked = config.rescrape_notes_b638 self.__rescrape_tags_cb.Checked = config.rescrape_tags_b639 self.__summary_dialog_cb.Checked = config.summary_dialog_b640
...
cleaning_utils.py
Source:cleaning_utils.py
1import time2from core_data_modules.cleaners import Codes3from core_data_modules.data_models import Origin, Label4from core_data_modules.traced_data import Metadata5from core_data_modules.util import TimeUtils6class CleaningUtils(object):7 @staticmethod8 def make_label_from_cleaner_code(scheme, code, origin_id, origin_name="Pipeline Auto-Coder", date_time_utc=None,9 set_checked=False):10 """11 Constructs a new Label object from a code determined by a pipeline cleaner.12 :param scheme: Scheme which the `code` argument belongs to.13 :type scheme: core_data_modules.data_models.CodeScheme14 :param code: Code to construct the label from.15 :type code: Code16 :param origin_id: Identifier of the origin of this label.17 :type origin_id: str18 :param origin_name: Name of the origin of this label.19 :type origin_name: str20 :param date_time_utc: Date to set in the label as an ISO string in UTC, or None.21 If None, uses the current system time in UTC.22 :type date_time_utc: str | None23 :param set_checked: Whether to set the `checked` property of the returned Label.24 :type set_checked: bool25 :return: A new label.26 :rtype: Label27 """28 if date_time_utc is None:29 date_time_utc = TimeUtils.utc_now_as_iso_string()30 origin = Origin(origin_id, origin_name, "External")31 return Label(scheme.scheme_id, code.code_id, date_time_utc, origin, checked=set_checked)32 @classmethod33 def apply_cleaner_to_text(cls, cleaner, text, scheme, set_checked=False):34 """35 Applies a cleaning function to a text, and returns a label if the cleaned value wasn't NC.36 :param cleaner: Cleaning function to apply.37 :type cleaner: function of str -> str38 :param text: Text to apply the cleaner to.39 :type text: str40 :param scheme: Scheme containing codes which the string returned from the `cleaner` can be matched against.41 :type scheme: core_data_modules.data_models.CodeScheme42 :param set_checked: Whether to set the `checked` property of the applied Label.43 :type set_checked: bool44 """45 clean_value = cleaner(text)46 # Don't label data which the cleaners couldn't code47 if clean_value == Codes.NOT_CODED:48 return None49 # Construct a label for the clean_value returned by the cleaner50 code_id = scheme.get_code_with_match_value(clean_value)51 origin_id = Metadata.get_function_location(cleaner)52 return cls.make_label_from_cleaner_code(scheme, code_id, origin_id, set_checked=set_checked)53 @classmethod54 def apply_cleaner_to_traced_data_iterable(cls, user, data, raw_key, clean_key, cleaner, scheme, set_checked=False):55 """56 Applies a cleaning function to an iterable of TracedData objects, updating each with a new Label object.57 :param user: Identifier of the user running this program, for TracedData Metadata.58 :type user: str59 :param data: TracedData objects to apply the cleaner to.60 :type data: iterable of TracedData61 :param raw_key: Key in each TracedData object of the raw text to be cleaned.62 :type raw_key: str63 :param clean_key: Key in each TracedData object to write cleaned labels to.64 :type clean_key: str65 :param cleaner: Cleaning function to apply to each TracedData[`raw_key`].66 :type cleaner: function of str -> str67 :param scheme: Scheme containing codes which the strings returned from the `cleaner` can be matched against.68 :type scheme: core_data_modules.data_models.CodeScheme69 :param set_checked: Whether to set the `checked` property of the applied Labels.70 :type set_checked: bool71 """72 for td in data:73 # Skip data that isn't present74 if raw_key not in td:75 continue76 77 label = cls.apply_cleaner_to_text(cleaner, td[raw_key], scheme, set_checked)78 if label is None:79 continue...
system_manipulator.py
Source:system_manipulator.py
1import sys, os, math2import configparser3import datetime4import graphene5sys.path.append(os.environ['SPACE_SHIP_HOME'] + '/api/background/mappers')6from system_mapper import SystemMapper7sys.path.append(os.environ['SPACE_SHIP_HOME'] + '/recital/native')8import mongo_native9config = configparser.ConfigParser()10config.read(os.environ['SPACE_SHIP_HOME'] + '/databases.config')11TIMESTAMP_PATTERN = os.environ.get('TIMESTAMP_PATTERN') or config['FORMATS']['timestamp']12class CreateSystem(graphene.Mutation):13 class Arguments:14 name = graphene.String()15 serialnumber = graphene.Float()16 launched = graphene.String()17 checked = graphene.String()18 state = graphene.String()19 supervisor = graphene.String()20 type = graphene.String()21 ok = graphene.Boolean()22 system = graphene.Field(lambda: SystemMapper)23 def mutate(self, info, name, serialnumber, launched, checked, state, supervisor, type):24 system = SystemMapper.init_scalar(mongo_native.create_system(\25 name, serialnumber, datetime.datetime.strptime(launched, TIMESTAMP_PATTERN), datetime.datetime.strptime(checked, TIMESTAMP_PATTERN), state, supervisor, type\26 ))27 ok = True28 return CreateSystem(system = system, ok = ok)29class RemoveSystem(graphene.Mutation):30 class Arguments:31 id = graphene.String()32 ok = graphene.Boolean()33 system = graphene.Field(lambda: SystemMapper)34 def mutate(self, info, id):35 system = SystemMapper.init_scalar(mongo_native.remove_system(id))36 ok = True37 return RemoveSystem(system = system, ok = ok)38class UpdateSystems(graphene.Mutation):39 class Arguments:40 id = graphene.String(default_value = '')41 name = graphene.String(default_value = '')42 serialnumber = graphene.Float(default_value = float('nan'))43 launched = graphene.String(default_value = '')44 checked = graphene.String(default_value = '')45 state = graphene.String(default_value = '')46 supervisor = graphene.String(default_value = '')47 type = graphene.String(default_value = '')48 set_name = graphene.String(default_value = '')49 set_serialnumber = graphene.Float(default_value = float('nan'))50 set_launched = graphene.String(default_value = '')51 set_checked = graphene.String(default_value = '')52 set_state = graphene.String(default_value = '')53 set_supervisor = graphene.String(default_value = '')54 set_type = graphene.String(default_value = '')55 ok = graphene.Boolean()56 systems = graphene.List(lambda: SystemMapper)57 def mutate(self, info, id, name, serialnumber, launched, checked, state, supervisor, type, 58 set_name, set_serialnumber, set_launched, set_checked, set_state, set_supervisor, set_type):59 systems = [SystemMapper.init_scalar(item) for item in mongo_native.update_systems(\60 _id = ObjectId(id) if id else None, name = name, 61 serialnumber = serialnumber if not math.isnan(serialnumber) else None, 62 launched = datetime.datetime.strptime(launched, TIMESTAMP_PATTERN) if launched else None, 63 checked = datetime.datetime.strptime(checked, TIMESTAMP_PATTERN) if checked else None, 64 state = ObjectId(state) if state else None, supervisor = ObjectId(supervisor) if supervisor else None, 65 type = ObjectId(type) if type else None,66 set_name = set_name, 67 set_serialnumber = set_serialnumber if set_serialnumber != float('nan') else None, 68 set_launched = datetime.datetime.strptime(set_launched, TIMESTAMP_PATTERN) if set_launched else None, 69 set_checked = datetime.datetime.strptime(set_checked, TIMESTAMP_PATTERN) if set_checked else None, 70 set_state = ObjectId(set_state) if set_state else None, set_supervisor = ObjectId(set_supervisor) if set_supervisor else None, 71 set_type = ObjectId(set_type) if set_type else None72 )]73 ok = True...
LambdaTest’s Playwright tutorial will give you a broader idea about the Playwright automation framework, its unique features, and use cases with examples to exceed your understanding of Playwright testing. This tutorial will give A to Z guidance, from installing the Playwright framework to some best practices and advanced concepts.
Get 100 minutes of automation test minutes FREE!!