KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use join_hook method in autotest

Best Python code snippet using autotest_python

automode.py

Source:automode.py

1# -*- coding: utf-8 -*-2###3# Copyright (c) 2009-2010 by EliÃ¡n Hanisch <lambdae2@gmail.com>4#5# This program is free software; you can redistribute it and/or modify6# it under the terms of the GNU General Public License as published by7# the Free Software Foundation; either version 3 of the License, or8# (at your option) any later version.9#10# This program is distributed in the hope that it will be useful,11# but WITHOUT ANY WARRANTY; without even the implied warranty of12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the13# GNU General Public License for more details.14#15# You should have received a copy of the GNU General Public License16# along with this program.  If not, see <http://www.gnu.org/licenses/>.17###18###19#20#   Script for auto op/voice users.21#22#   It uses expressions for match user's usermasks when they join. Use at your own risk.23#24#   Commands:25#   * /automode: see /help automode26#27#   Settings:28#   * plugins.var.python.automode.enabled:29#     Self-explanatory, disables/enables automodes.30#     Valid values: 'on', 'off' Default: 'on'31#32#   2011-09-2033#   version 0.1.1: fix bug with channels with uppercase letters.34#35#   2014-04-1536#   version 0.1.2: fix bug where mode commands weren't sent properly37#38#   2016-06-2839#   version 0.1.3: support extended-join messages40#41#   2019-03-0942#   version 0.1.4: support python343###44from __future__ import print_function45SCRIPT_NAME    = "automode"46SCRIPT_AUTHOR  = "EliÃ¡n Hanisch <lambdae2@gmail.com>"47SCRIPT_VERSION = "0.1.4"48SCRIPT_LICENSE = "GPL3"49SCRIPT_DESC    = "Script for auto op/voice users when they join."50try:51    import weechat52    from weechat import prnt53    WEECHAT_RC_OK = weechat.WEECHAT_RC_OK54    import_ok = True55except ImportError:56    print ("This script must be run under WeeChat.")57    print ("Get WeeChat now at: http://weechat.flashtux.org/")58    import_ok = False59from fnmatch import fnmatch60def debug(s, *args):61    if not isinstance(s, basestring):62        s = str(s)63    if args:64        s = s %args65    prnt('', '%s\t%s' % (script_nick, s))66# settings67settings = { 'enabled': 'on' }68################69### Messages ###70script_nick = SCRIPT_NAME71def error(s, buffer=''):72    """Error msg"""73    weechat.prnt(buffer, '%s%s %s' %(weechat.prefix('error'), script_nick, s))74def say(s, buffer=''):75    """normal msg"""76    weechat.prnt(buffer, '%s\t%s' %(script_nick, s))77##############78### Config ###79boolDict = {'on':True, 'off':False}80def get_config_boolean(config):81    value = weechat.config_get_plugin(config)82    try:83        return boolDict[value]84    except KeyError:85        default = settings[config]86        error("Error while fetching config '%s'. Using default value '%s'." %(config, default))87        error("'%s' is invalid, allowed: 'on', 'off'" %value)88        return boolDict[default]89def get_config_list(config):90    value = weechat.config_get_plugin(config)91    if value:92        return value.split(',')93    else:94        return []95#################96### Functions ###97def find_matching_users(server, channel, pattern):98    # this is for check patterns when they are added99    infolist = weechat.infolist_get('irc_nick', '', '%s,%s' %(server, channel))100    L = []101    while weechat.infolist_next(infolist):102        nick = weechat.infolist_string(infolist, 'name')103        host = weechat.infolist_string(infolist, 'host')104        userhost = '%s!%s' %(nick, host)105        if fnmatch(userhost.lower(), pattern):106            L.append(nick)107    weechat.infolist_free(infolist)108    return L109def get_userhost(server, channel, nick):110    try:111        infolist = weechat.infolist_get('irc_nick', '', '%s,%s' %(server, channel))112        while weechat.infolist_next(infolist):113            _nick = weechat.infolist_string(infolist, 'name').lower()114            if _nick == nick:115                host = weechat.infolist_string(infolist, 'host')116                userhost = '%s!%s' %(nick, host)117                return userhost.lower()118    finally:119        weechat.infolist_free(infolist)120def get_patterns_in_config(filter):121    d = {}122    infolist = weechat.infolist_get('option', '', 'plugins.var.python.%s.%s' %(SCRIPT_NAME, filter))123    while weechat.infolist_next(infolist):124        name = weechat.infolist_string(infolist, 'option_name')125        name = name[len('python.%s.' %SCRIPT_NAME):]126        # channels might have dots in their names, so we'll strip type from right and server127        # from left. Lets hope that users doesn't use dots in server names.128        name, _, type = name.rpartition('.')129        if type not in ('op', 'halfop', 'voice'):130            # invalid option131            continue132        server, _, channel = name.partition('.')133        value = weechat.infolist_string(infolist, 'value')134        if not value:135            continue136        else:137            value = value.split(',')138        key = (server, channel)139        if key not in d:140            d[key] = {type:value}141        else:142            d[key][type] = value143    weechat.infolist_free(infolist)144    return d145########################146### Script callbacks ###147def join_cb(data, signal, signal_data):148    #debug('JOIN: %s %s', signal, signal_data)149    prefix, _, channel = signal_data.split()[:3]150    prefix = prefix[1:].lower()151    if channel[0] == ':':152        channel = channel[1:]153    server = signal[:signal.find(',')]154    for mode_type, shorthand in {'op':'o', 'halfop':'h', 'voice':'v'}.items():155        l = get_config_list('.'.join((server.lower(), channel.lower(), mode_type)))156        for pattern in l:157            #debug('checking: %r - %r', prefix, pattern)158            if fnmatch(prefix, pattern):159                buf = weechat.buffer_search('irc', '%s.%s' %(server, channel))160                if buf:161                    weechat.command(buf, '/wait 1 /mode {} +{} {}'.format(channel, shorthand, prefix[:prefix.find('!')]))162                return WEECHAT_RC_OK163    return WEECHAT_RC_OK164def command(data, buffer, args):165    global join_hook166    if not args:167        args = 'list'168    channel = weechat.buffer_get_string(buffer, 'localvar_channel')169    server = weechat.buffer_get_string(buffer, 'localvar_server')170    args = args.split()171    cmd = args[0]172    try:173        if cmd in ('add', 'del'):174            if not weechat.info_get('irc_is_channel', channel):175                error("Not an IRC channel buffer.")176                return WEECHAT_RC_OK177            type, match = args[1], args[2:]178            if type not in ('op', 'voice', 'halfop'):179                raise ValueError("valid values are 'op', 'halfop' and 'voice'.")180            if not match:181                raise ValueError("missing pattern or nick.")182            match = match[0].lower()183            config = '.'.join((server, channel.lower(), type))184            L = get_config_list(config)185            if cmd == 'add':186                # check if pattern is a nick187                if weechat.info_get('irc_is_nick', match):188                    userhost = get_userhost(server, channel, match)189                    if userhost:190                        match = userhost.lower()191                nicks = find_matching_users(server, channel, match)192                n = len(nicks)193                if n == 0:194                    say("'%s' added, matches 0 users." %match, buffer)195                elif n == 1:196                    say("'%s' added, matches 1 user: %s" %(match, nicks[0]),197                            buffer)198                elif n > 1:199                    say("'%s' added, matches %s%s%s users: %s" %(200                        match, weechat.color('lightred'), n, color_reset,201                        ' '.join(nicks)), buffer)202                if match not in L:203                    L.append(match)204            elif cmd == 'del':205                    if match not in L:206                        say("'%s' not found in %s.%s" %(match, server, channel), buffer)207                    else:208                        say("'%s' removed." %match, buffer)209                        del L[L.index(match)]210            if L:211                weechat.config_set_plugin(config, ','.join(L))212            else:213                weechat.config_unset_plugin(config)214        elif cmd == 'disable':215            if join_hook:216                weechat.unhook(join_hook)217            weechat.config_set_plugin('enabled', 'off')218            say("%s script disabled." %SCRIPT_NAME, buffer)219        elif cmd == 'enable':220            if join_hook:221                weechat.unhook(join_hook)222            join_hook = weechat.hook_signal('*,irc_in_join', 'join_cb', '')223            weechat.config_set_plugin('enabled', 'on')224            say("%s script enabled." %SCRIPT_NAME, buffer)225        elif cmd == 'list':226            if weechat.info_get('irc_is_channel', channel):227                filter = '%s.%s.*' %(server, channel)228            else:229                filter = '*'230                buffer = '' # print in core buffer231            if not get_config_boolean('enabled'):232                say('Automodes currently disabled.', buffer)233            patterns = get_patterns_in_config(filter)234            if not patterns:235                if buffer:236                    say('No automodes for %s.' %channel, buffer)237                else:238                    say('No automodes.', buffer)239                return WEECHAT_RC_OK240            for key, items in patterns.items():241                say('%s[%s%s.%s%s]' %(color_chat_delimiters,242                                      color_chat_buffer,243                                      key[0], key[1],244                                      color_chat_delimiters), buffer)245                for type, masks in items.items():246                    for mask in masks:247                        say('  %s%s%s: %s%s' %(color_chat_nick, type,248                                               color_chat_delimiters,249                                               color_reset,250                                               mask), buffer)251        else:252            raise ValueError("'%s' isn't a valid option. See /help %s" %(cmd, SCRIPT_NAME))253    except ValueError as e:254        error('Bad argument: %s' %e)255        return WEECHAT_RC_OK256    return WEECHAT_RC_OK257def completer(data, completion_item, buffer, completion):258    channel = weechat.buffer_get_string(buffer, 'localvar_channel')259    if not weechat.info_get('irc_is_channel', channel):260        return WEECHAT_RC_OK261    server = weechat.buffer_get_string(buffer, 'localvar_server')262    input = weechat.buffer_get_string(buffer, 'input')263    type = input.split()[2]264    patterns = get_patterns_in_config('%s.%s.%s' %(server, channel, type))265    if not patterns:266        return WEECHAT_RC_OK267    for mask in patterns[(server, channel)][type]:268        weechat.hook_completion_list_add(completion, mask, 0, weechat.WEECHAT_LIST_POS_END)269    return WEECHAT_RC_OK270if __name__ == '__main__' and import_ok and \271        weechat.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION, SCRIPT_LICENSE,272        SCRIPT_DESC, '', ''):273    # colors274    color_chat_delimiters = weechat.color('chat_delimiters')275    color_chat_nick       = weechat.color('chat_nick')276    color_reset           = weechat.color('reset')277    color_chat_buffer     = weechat.color('chat_buffer')278    # pretty [automode]279    script_nick = '%s[%s%s%s]%s' %(color_chat_delimiters,280                                   color_chat_nick,281                                   SCRIPT_NAME,282                                   color_chat_delimiters,283                                   color_reset)284    for opt, val in list(settings.items()):285        if not weechat.config_is_set_plugin(opt):286                weechat.config_set_plugin(opt, val)287    global join_hook288    if get_config_boolean('enabled'):289        join_hook = weechat.hook_signal('*,irc_in_join', 'join_cb', '')290    else:291        join_hook = ''292    weechat.hook_completion('automode_patterns', 'automode patterns', 'completer', '')293    weechat.hook_command(SCRIPT_NAME, SCRIPT_DESC ,294            "[ (add|del) <type> <nick|expression> | list | disable | enable ]",295            "       add: Adds a new automode for current channel. If a nick is given instead of an"296            " expression, it will use nick's exact usermask.\n"297            "       del: Removes an automode in current channel.\n"298            "      type: Specifies the user mode, it should be either 'op', 'halfop' or 'voice'.\n"299            "expression: Case insensible expression for match users when they join current channel."300            " It should be of the format 'nick!user@host', wildcards '?', '*', and character groups"301            " are allowed.\n"302            "      list: List automodes for current channel, or all automodes if current buffer"303            " isn't an IRC channel. This is the default action if no option is given.\n"304            "   disable: Disables the script.\n"305            "    enable: Enables the script.\n"306            "\n"307            "Be careful with the expressions you use, they must be specific and match only one"308            " user, if they are too vague, like 'nick!*' you might op users you don't want and lose"309            " control of your channel.",310            "add op|halfop|voice %(nicks)"\311            "||del op|halfop|voice %(automode_patterns)"\...

join.py

Source:join.py

1import warnings2from abc import ABC, abstractmethod3from typing import List4import torch5import torch.distributed as dist6class _JoinHook(ABC):7    r"""8    This defines a join hook, which provides two entry points in the join9    context manager: a main hook, which is called repeatedly while there exists10    a non-joined process, and a post-hook, which is called once all processes11    have joined.12    To implement a join hook for the generic join context manager, define a13    class that inherits from :class:`_JoinHook`, override ``main_hook()`` and14    ``post_hook()`` as appropriate, and override ``device()`` and15    ``process_group()`` to provide the device and process group information,16    respectively, which are needed for the join context manager implementation.17    """18    def main_hook(self):19        r"""20        This hook is called repeatedly while there exists a non-joined process21        to shadow collective communications in the forward pass, backward pass,22        and optimizer.23        """24        ...25    def post_hook(self, is_last_joiner: bool):26        r"""27        This hook is called after all processes have joined. It is passed an28        additional ``bool`` argument ``is_last_joiner``, which indicates if the29        rank is one of the last to join.30        Arguments:31            is_last_joiner (bool): ``True`` if the rank is one of the last to32                join; ``False`` otherwise.33        """34        ...35    @property36    @abstractmethod37    def device(self):38        r"""39        Returns the device from which to perform collective communications40        needed for the join context manager implementation itself.41        """42        ...43    @property44    @abstractmethod45    def process_group(self):46        r"""47        Returns the process group for join-related collective communications.48        """49        ...50class _Join():51    r"""52    This class defines the generic join context manager, which allows custom53    hooks to be called after a process joins. These hooks should shadow the54    collective communications of non-joined processes to prevent hanging and55    erroring and to ensure algorithmic correctness. Refer to :class:`_JoinHook`56    for details about the hook definition.57    .. warning::58        The context manager requires a ``dist.all_reduce(torch.ones(1))`` to be59        called on every non-joined process each time before it performs its60        collective communications in order to indicate that the process has not61        yet joined. For example, this can occur at the beginning of the forward62        pass.63    .. warning::64        If ``throw_on_early_termination`` is enabled, then the context manager65        additionally requires every non-joined process to participate in an66        all-reduce before it performs its collective communications in order to67        check if it should terminate due to detecting uneven inputs. This all-68        reduce should be of the form ``dist.all_reduce(torch.zeros(1))``; if69        the result is positive, then the process should terminate.70    .. warning::71        The context manager requires that all ``process_group`` attributes in72        the ``_JoinHook`` objects are the same. If there are multiple73        ``_JoinHook`` objects, then the ``device`` of the first is used. The74        process group and device information is used for checking for non-75        joined processes and for notifying processes to terminate if76        ``throw_on_early_termination`` is eanbled, both of which using an all-77        reduce.78    Arguments:79        join_hooks (List[_JoinHook]): a list of the :class:`_JoinHook` s to80            use; the hooks are iterated over in the given order.81        enable (bool): a flag enabling uneven input detection; setting to82            ``False`` disables the context manager's functionality and should83            only be set when the user knows the inputs will not be uneven84            (default: ``True``).85        throw_on_early_termination (bool): a flag controlling whether to raise86            an exception upon detecting uneven inputs (default: ``False``).87    """88    def __init__(89        self,90        join_hooks: List[_JoinHook],91        enable: bool = True,92        throw_on_early_termination: bool = False,93    ):94        if len(join_hooks) == 0:95            raise ValueError("The join context manager requires at least one join hook")96        self._join_hooks = join_hooks97        self._enable = enable98        self._throw_on_early_termination = throw_on_early_termination99        self._extract_dist_info()100    def _extract_dist_info(self):101        r"""102        Extracts the process group and device information from the join hooks.103        Preconditions:104            ``self._join_hooks`` is not ``None`` and is non-empty.105        Raises:106            ValueError107                If there are multiple conflicting ``process_group`` attributes108                among the ``_JoinHook`` objects.109        NOTE: The context manager uses the first specified device.110        """111        process_group = None112        device = None113        for join_hook in self._join_hooks:114            if process_group is None:115                process_group = join_hook.process_group116            elif process_group != join_hook.process_group:117                raise ValueError("Using join context manager with multiple process groups")118            if device is None:119                device = join_hook.device120        self._process_group = process_group121        self._rank = dist.get_rank(self._process_group)122        self._device = device123    def __enter__(self):124        ...125    def __exit__(self, type, value, traceback):126        r"""127        Repeatedly runs the main hooks until all processes join; then, runs128        the post-hooks.129        Raises:130            RuntimeError131                If ``throw_on_early_termination`` is enabled.132        """133        if not self._enable or type:134            return  # propagate the exception directly if one was raised135        all_procs_joined = False136        is_last_joiner = True137        i = 0138        WARN_THRESHOLD = 1000139        warnings.simplefilter("once")140        while not all_procs_joined:141            if i > WARN_THRESHOLD:142                warnings.warn(143                    "Detected uneven input skew of greater than "144                    f"{WARN_THRESHOLD}. This means that rank "145                    f"{self._rank} has at least {WARN_THRESHOLD} "146                    f"fewer inputs than other currently-active ranks. "147                    "This level of skew could lead to performance "148                    "degradataion during training."149                )150            # Shadow the all-reduce in non-joined processes151            num_nonjoined_procs = self._get_num_nonjoined_procs()152            if num_nonjoined_procs == 0:153                all_procs_joined = True154            else:155                if self._throw_on_early_termination:156                    self._notify_procs_to_terminate()157                # Run main hooks158                for join_hook in self._join_hooks:159                    join_hook.main_hook()160                is_last_joiner = False161                i += 1162        # Run post-hooks163        for join_hook in self._join_hooks:164            join_hook.post_hook(is_last_joiner)165    def _get_num_nonjoined_procs(self):166        r"""167        Returns the number of non-joined processes by shadowing an all-reduce168        in the non-joined processes.169        """170        num_nonjoined_procs = torch.zeros(1, device=self._device)171        dist.all_reduce(num_nonjoined_procs, group=self._process_group)172        return num_nonjoined_procs.item()173    def _notify_procs_to_terminate(self):174        r"""175        Schedules an all-reduce to notify non-joined processes to terminate176        and raises a ``RuntimeError`` indicating that the current process has177        exhausted its inputs.178        """179        ones = torch.ones(1, device=self._device)180        dist.all_reduce(ones, group=self._process_group)181        # NOTE: Raising `StopIteration` does not throw an error in Python 3.6182        # and throws a `RuntimeError` in Python 3.7+ (PEP 479), so we just183        # raise a `RuntimeError` here...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.