KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use member method in hypothesis

Best Python code snippet using hypothesis

enum.py

Source:enum.py

1import sys2from types import MappingProxyType, DynamicClassAttribute3# try _collections first to reduce startup cost4try:5    from _collections import OrderedDict6except ImportError:7    from collections import OrderedDict8__all__ = [9        'EnumMeta',10        'Enum', 'IntEnum', 'Flag', 'IntFlag',11        'auto', 'unique',12        ]13def _is_descriptor(obj):14    """Returns True if obj is a descriptor, False otherwise."""15    return (16            hasattr(obj, '__get__') or17            hasattr(obj, '__set__') or18            hasattr(obj, '__delete__'))19def _is_dunder(name):20    """Returns True if a __dunder__ name, False otherwise."""21    return (len(name) > 4 and22            name[:2] == name[-2:] == '__' and23            name[2] != '_' and24            name[-3] != '_')25def _is_sunder(name):26    """Returns True if a _sunder_ name, False otherwise."""27    return (len(name) > 2 and28            name[0] == name[-1] == '_' and29            name[1:2] != '_' and30            name[-2:-1] != '_')31def _make_class_unpicklable(cls):32    """Make the given class un-picklable."""33    def _break_on_call_reduce(self, proto):34        raise TypeError('%r cannot be pickled' % self)35    cls.__reduce_ex__ = _break_on_call_reduce36    cls.__module__ = '<unknown>'37_auto_null = object()38class auto:39    """40    Instances are replaced with an appropriate value in Enum class suites.41    """42    value = _auto_null43class _EnumDict(dict):44    """Track enum member order and ensure member names are not reused.45    EnumMeta will use the names found in self._member_names as the46    enumeration member names.47    """48    def __init__(self):49        super().__init__()50        self._member_names = []51        self._last_values = []52        self._ignore = []53    def __setitem__(self, key, value):54        """Changes anything not dundered or not a descriptor.55        If an enum member name is used twice, an error is raised; duplicate56        values are not checked for.57        Single underscore (sunder) names are reserved.58        """59        if _is_sunder(key):60            if key not in (61                    '_order_', '_create_pseudo_member_',62                    '_generate_next_value_', '_missing_', '_ignore_',63                    ):64                raise ValueError('_names_ are reserved for future Enum use')65            if key == '_generate_next_value_':66                setattr(self, '_generate_next_value', value)67            elif key == '_ignore_':68                if isinstance(value, str):69                    value = value.replace(',',' ').split()70                else:71                    value = list(value)72                self._ignore = value73                already = set(value) & set(self._member_names)74                if already:75                    raise ValueError('_ignore_ cannot specify already set names: %r' % (already, ))76        elif _is_dunder(key):77            if key == '__order__':78                key = '_order_'79        elif key in self._member_names:80            # descriptor overwriting an enum?81            raise TypeError('Attempted to reuse key: %r' % key)82        elif key in self._ignore:83            pass84        elif not _is_descriptor(value):85            if key in self:86                # enum overwriting a descriptor?87                raise TypeError('%r already defined as: %r' % (key, self[key]))88            if isinstance(value, auto):89                if value.value == _auto_null:90                    value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:])91                value = value.value92            self._member_names.append(key)93            self._last_values.append(value)94        super().__setitem__(key, value)95# Dummy value for Enum as EnumMeta explicitly checks for it, but of course96# until EnumMeta finishes running the first time the Enum class doesn't exist.97# This is also why there are checks in EnumMeta like `if Enum is not None`98Enum = None99class EnumMeta(type):100    """Metaclass for Enum"""101    @classmethod102    def __prepare__(metacls, cls, bases):103        # create the namespace dict104        enum_dict = _EnumDict()105        # inherit previous flags and _generate_next_value_ function106        member_type, first_enum = metacls._get_mixins_(bases)107        if first_enum is not None:108            enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None)109        return enum_dict110    def __new__(metacls, cls, bases, classdict):111        # an Enum class is final once enumeration items have been defined; it112        # cannot be mixed with other types (int, float, etc.) if it has an113        # inherited __new__ unless a new __new__ is defined (or the resulting114        # class will fail).115        #116        # remove any keys listed in _ignore_117        classdict.setdefault('_ignore_', []).append('_ignore_')118        ignore = classdict['_ignore_']119        for key in ignore:120            classdict.pop(key, None)121        member_type, first_enum = metacls._get_mixins_(bases)122        __new__, save_new, use_args = metacls._find_new_(classdict, member_type,123                                                        first_enum)124        # save enum items into separate mapping so they don't get baked into125        # the new class126        enum_members = {k: classdict[k] for k in classdict._member_names}127        for name in classdict._member_names:128            del classdict[name]129        # adjust the sunders130        _order_ = classdict.pop('_order_', None)131        # check for illegal enum names (any others?)132        invalid_names = set(enum_members) & {'mro', ''}133        if invalid_names:134            raise ValueError('Invalid enum member name: {0}'.format(135                ','.join(invalid_names)))136        # create a default docstring if one has not been provided137        if '__doc__' not in classdict:138            classdict['__doc__'] = 'An enumeration.'139        # create our new Enum type140        enum_class = super().__new__(metacls, cls, bases, classdict)141        enum_class._member_names_ = []               # names in definition order142        enum_class._member_map_ = OrderedDict()      # name->value map143        enum_class._member_type_ = member_type144        # save DynamicClassAttribute attributes from super classes so we know145        # if we can take the shortcut of storing members in the class dict146        dynamic_attributes = {k for c in enum_class.mro()147                              for k, v in c.__dict__.items()148                              if isinstance(v, DynamicClassAttribute)}149        # Reverse value->name map for hashable values.150        enum_class._value2member_map_ = {}151        # If a custom type is mixed into the Enum, and it does not know how152        # to pickle itself, pickle.dumps will succeed but pickle.loads will153        # fail.  Rather than have the error show up later and possibly far154        # from the source, sabotage the pickle protocol for this class so155        # that pickle.dumps also fails.156        #157        # However, if the new class implements its own __reduce_ex__, do not158        # sabotage -- it's on them to make sure it works correctly.  We use159        # __reduce_ex__ instead of any of the others as it is preferred by160        # pickle over __reduce__, and it handles all pickle protocols.161        if '__reduce_ex__' not in classdict:162            if member_type is not object:163                methods = ('__getnewargs_ex__', '__getnewargs__',164                        '__reduce_ex__', '__reduce__')165                if not any(m in member_type.__dict__ for m in methods):166                    _make_class_unpicklable(enum_class)167        # instantiate them, checking for duplicates as we go168        # we instantiate first instead of checking for duplicates first in case169        # a custom __new__ is doing something funky with the values -- such as170        # auto-numbering ;)171        for member_name in classdict._member_names:172            value = enum_members[member_name]173            if not isinstance(value, tuple):174                args = (value, )175            else:176                args = value177            if member_type is tuple:   # special case for tuple enums178                args = (args, )     # wrap it one more time179            if not use_args:180                enum_member = __new__(enum_class)181                if not hasattr(enum_member, '_value_'):182                    enum_member._value_ = value183            else:184                enum_member = __new__(enum_class, *args)185                if not hasattr(enum_member, '_value_'):186                    if member_type is object:187                        enum_member._value_ = value188                    else:189                        enum_member._value_ = member_type(*args)190            value = enum_member._value_191            enum_member._name_ = member_name192            enum_member.__objclass__ = enum_class193            enum_member.__init__(*args)194            # If another member with the same value was already defined, the195            # new member becomes an alias to the existing one.196            for name, canonical_member in enum_class._member_map_.items():197                if canonical_member._value_ == enum_member._value_:198                    enum_member = canonical_member199                    break200            else:201                # Aliases don't appear in member names (only in __members__).202                enum_class._member_names_.append(member_name)203            # performance boost for any member that would not shadow204            # a DynamicClassAttribute205            if member_name not in dynamic_attributes:206                setattr(enum_class, member_name, enum_member)207            # now add to _member_map_208            enum_class._member_map_[member_name] = enum_member209            try:210                # This may fail if value is not hashable. We can't add the value211                # to the map, and by-value lookups for this value will be212                # linear.213                enum_class._value2member_map_[value] = enum_member214            except TypeError:215                pass216        # double check that repr and friends are not the mixin's or various217        # things break (such as pickle)218        for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):219            class_method = getattr(enum_class, name)220            obj_method = getattr(member_type, name, None)221            enum_method = getattr(first_enum, name, None)222            if obj_method is not None and obj_method is class_method:223                setattr(enum_class, name, enum_method)224        # replace any other __new__ with our own (as long as Enum is not None,225        # anyway) -- again, this is to support pickle226        if Enum is not None:227            # if the user defined their own __new__, save it before it gets228            # clobbered in case they subclass later229            if save_new:230                enum_class.__new_member__ = __new__231            enum_class.__new__ = Enum.__new__232        # py3 support for definition order (helps keep py2/py3 code in sync)233        if _order_ is not None:234            if isinstance(_order_, str):235                _order_ = _order_.replace(',', ' ').split()236            if _order_ != enum_class._member_names_:237                raise TypeError('member order does not match _order_')238        return enum_class239    def __bool__(self):240        """241        classes/types should always be True.242        """243        return True244    def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):245        """Either returns an existing member, or creates a new enum class.246        This method is used both when an enum class is given a value to match247        to an enumeration member (i.e. Color(3)) and for the functional API248        (i.e. Color = Enum('Color', names='RED GREEN BLUE')).249        When used for the functional API:250        `value` will be the name of the new class.251        `names` should be either a string of white-space/comma delimited names252        (values will start at `start`), or an iterator/mapping of name, value pairs.253        `module` should be set to the module this class is being created in;254        if it is not set, an attempt to find that module will be made, but if255        it fails the class will not be picklable.256        `qualname` should be set to the actual location this class can be found257        at in its module; by default it is set to the global scope.  If this is258        not correct, unpickling will fail in some circumstances.259        `type`, if set, will be mixed in as the first base class.260        """261        if names is None:  # simple value lookup262            return cls.__new__(cls, value)263        # otherwise, functional API: we're creating a new Enum type264        return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start)265    def __contains__(cls, member):266        if not isinstance(member, Enum):267            import warnings268            warnings.warn(269                    "using non-Enums in containment checks will raise "270                    "TypeError in Python 3.8",271                    DeprecationWarning, 2)272        return isinstance(member, cls) and member._name_ in cls._member_map_273    def __delattr__(cls, attr):274        # nicer error message when someone tries to delete an attribute275        # (see issue19025).276        if attr in cls._member_map_:277            raise AttributeError(278                    "%s: cannot delete Enum member." % cls.__name__)279        super().__delattr__(attr)280    def __dir__(self):281        return (['__class__', '__doc__', '__members__', '__module__'] +282                self._member_names_)283    def __getattr__(cls, name):284        """Return the enum member matching `name`285        We use __getattr__ instead of descriptors or inserting into the enum286        class' __dict__ in order to support `name` and `value` being both287        properties for enum members (which live in the class' __dict__) and288        enum members themselves.289        """290        if _is_dunder(name):291            raise AttributeError(name)292        try:293            return cls._member_map_[name]294        except KeyError:295            raise AttributeError(name) from None296    def __getitem__(cls, name):297        return cls._member_map_[name]298    def __iter__(cls):299        return (cls._member_map_[name] for name in cls._member_names_)300    def __len__(cls):301        return len(cls._member_names_)302    @property303    def __members__(cls):304        """Returns a mapping of member name->value.305        This mapping lists all enum members, including aliases. Note that this306        is a read-only view of the internal mapping.307        """308        return MappingProxyType(cls._member_map_)309    def __repr__(cls):310        return "<enum %r>" % cls.__name__311    def __reversed__(cls):312        return (cls._member_map_[name] for name in reversed(cls._member_names_))313    def __setattr__(cls, name, value):314        """Block attempts to reassign Enum members.315        A simple assignment to the class namespace only changes one of the316        several possible ways to get an Enum member from the Enum class,317        resulting in an inconsistent Enumeration.318        """319        member_map = cls.__dict__.get('_member_map_', {})320        if name in member_map:321            raise AttributeError('Cannot reassign members.')322        super().__setattr__(name, value)323    def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1):324        """Convenience method to create a new Enum class.325        `names` can be:326        * A string containing member names, separated either with spaces or327          commas.  Values are incremented by 1 from `start`.328        * An iterable of member names.  Values are incremented by 1 from `start`.329        * An iterable of (member name, value) pairs.330        * A mapping of member name -> value pairs.331        """332        metacls = cls.__class__333        bases = (cls, ) if type is None else (type, cls)334        _, first_enum = cls._get_mixins_(bases)335        classdict = metacls.__prepare__(class_name, bases)336        # special processing needed for names?337        if isinstance(names, str):338            names = names.replace(',', ' ').split()339        if isinstance(names, (tuple, list)) and names and isinstance(names[0], str):340            original_names, names = names, []341            last_values = []342            for count, name in enumerate(original_names):343                value = first_enum._generate_next_value_(name, start, count, last_values[:])344                last_values.append(value)345                names.append((name, value))346        # Here, names is either an iterable of (name, value) or a mapping.347        for item in names:348            if isinstance(item, str):349                member_name, member_value = item, names[item]350            else:351                member_name, member_value = item352            classdict[member_name] = member_value353        enum_class = metacls.__new__(metacls, class_name, bases, classdict)354        # TODO: replace the frame hack if a blessed way to know the calling355        # module is ever developed356        if module is None:357            try:358                module = sys._getframe(2).f_globals['__name__']359            except (AttributeError, ValueError, KeyError) as exc:360                pass361        if module is None:362            _make_class_unpicklable(enum_class)363        else:364            enum_class.__module__ = module365        if qualname is not None:366            enum_class.__qualname__ = qualname367        return enum_class368    @staticmethod369    def _get_mixins_(bases):370        """Returns the type for creating enum members, and the first inherited371        enum class.372        bases: the tuple of bases that was given to __new__373        """374        if not bases:375            return object, Enum376        def _find_data_type(bases):377            for chain in bases:378                for base in chain.__mro__:379                    if base is object:380                        continue381                    elif '__new__' in base.__dict__:382                        if issubclass(base, Enum):383                            continue384                        return base385        # ensure final parent class is an Enum derivative, find any concrete386        # data type, and check that Enum has no members387        first_enum = bases[-1]388        if not issubclass(first_enum, Enum):389            raise TypeError("new enumerations should be created as "390                    "`EnumName([mixin_type, ...] [data_type,] enum_type)`")391        member_type = _find_data_type(bases) or object392        if first_enum._member_names_:393            raise TypeError("Cannot extend enumerations")394        return member_type, first_enum395    @staticmethod396    def _find_new_(classdict, member_type, first_enum):397        """Returns the __new__ to be used for creating the enum members.398        classdict: the class dictionary given to __new__399        member_type: the data type whose __new__ will be used by default400        first_enum: enumeration to check for an overriding __new__401        """402        # now find the correct __new__, checking to see of one was defined403        # by the user; also check earlier enum classes in case a __new__ was404        # saved as __new_member__405        __new__ = classdict.get('__new__', None)406        # should __new__ be saved as __new_member__ later?407        save_new = __new__ is not None408        if __new__ is None:409            # check all possibles for __new_member__ before falling back to410            # __new__411            for method in ('__new_member__', '__new__'):412                for possible in (member_type, first_enum):413                    target = getattr(possible, method, None)414                    if target not in {415                            None,416                            None.__new__,417                            object.__new__,418                            Enum.__new__,419                            }:420                        __new__ = target421                        break422                if __new__ is not None:423                    break424            else:425                __new__ = object.__new__426        # if a non-object.__new__ is used then whatever value/tuple was427        # assigned to the enum member name will be passed to __new__ and to the428        # new enum member's __init__429        if __new__ is object.__new__:430            use_args = False431        else:432            use_args = True433        return __new__, save_new, use_args434class Enum(metaclass=EnumMeta):435    """Generic enumeration.436    Derive from this class to define new enumerations.437    """438    def __new__(cls, value):439        # all enum instances are actually created during class construction440        # without calling this method; this method is called by the metaclass'441        # __call__ (i.e. Color(3) ), and by pickle442        if type(value) is cls:443            # For lookups like Color(Color.RED)444            return value445        # by-value search for a matching enum member446        # see if it's in the reverse mapping (for hashable values)447        try:448            return cls._value2member_map_[value]449        except KeyError:450            # Not found, no need to do long O(n) search451            pass452        except TypeError:453            # not there, now do long search -- O(n) behavior454            for member in cls._member_map_.values():455                if member._value_ == value:456                    return member457        # still not found -- try _missing_ hook458        try:459            exc = None460            result = cls._missing_(value)461        except Exception as e:462            exc = e463            result = None464        if isinstance(result, cls):465            return result466        else:467            ve_exc = ValueError("%r is not a valid %s" % (value, cls.__name__))468            if result is None and exc is None:469                raise ve_exc470            elif exc is None:471                exc = TypeError(472                        'error in %s._missing_: returned %r instead of None or a valid member'473                        % (cls.__name__, result)474                        )475            exc.__context__ = ve_exc476            raise exc477    def _generate_next_value_(name, start, count, last_values):478        for last_value in reversed(last_values):479            try:480                return last_value + 1481            except TypeError:482                pass483        else:484            return start485    @classmethod486    def _missing_(cls, value):487        raise ValueError("%r is not a valid %s" % (value, cls.__name__))488    def __repr__(self):489        return "<%s.%s: %r>" % (490                self.__class__.__name__, self._name_, self._value_)491    def __str__(self):492        return "%s.%s" % (self.__class__.__name__, self._name_)493    def __dir__(self):494        added_behavior = [495                m496                for cls in self.__class__.mro()497                for m in cls.__dict__498                if m[0] != '_' and m not in self._member_map_499                ]500        return (['__class__', '__doc__', '__module__'] + added_behavior)501    def __format__(self, format_spec):502        # mixed-in Enums should use the mixed-in type's __format__, otherwise503        # we can get strange results with the Enum name showing up instead of504        # the value505        # pure Enum branch506        if self._member_type_ is object:507            cls = str508            val = str(self)509        # mix-in branch510        else:511            cls = self._member_type_512            val = self._value_513        return cls.__format__(val, format_spec)514    def __hash__(self):515        return hash(self._name_)516    def __reduce_ex__(self, proto):517        return self.__class__, (self._value_, )518    # DynamicClassAttribute is used to provide access to the `name` and519    # `value` properties of enum members while keeping some measure of520    # protection from modification, while still allowing for an enumeration521    # to have members named `name` and `value`.  This works because enumeration522    # members are not set directly on the enum class -- __getattr__ is523    # used to look them up.524    @DynamicClassAttribute525    def name(self):526        """The name of the Enum member."""527        return self._name_528    @DynamicClassAttribute529    def value(self):530        """The value of the Enum member."""531        return self._value_532    @classmethod533    def _convert(cls, name, module, filter, source=None):534        """535        Create a new Enum subclass that replaces a collection of global constants536        """537        # convert all constants from source (or module) that pass filter() to538        # a new Enum called name, and export the enum and its members back to539        # module;540        # also, replace the __reduce_ex__ method so unpickling works in541        # previous Python versions542        module_globals = vars(sys.modules[module])543        if source:544            source = vars(source)545        else:546            source = module_globals547        # We use an OrderedDict of sorted source keys so that the548        # _value2member_map is populated in the same order every time549        # for a consistent reverse mapping of number to name when there550        # are multiple names for the same number rather than varying551        # between runs due to hash randomization of the module dictionary.552        members = [553                (name, source[name])554                for name in source.keys()555                if filter(name)]556        try:557            # sort by value558            members.sort(key=lambda t: (t[1], t[0]))559        except TypeError:560            # unless some values aren't comparable, in which case sort by name561            members.sort(key=lambda t: t[0])562        cls = cls(name, members, module=module)563        cls.__reduce_ex__ = _reduce_ex_by_name564        module_globals.update(cls.__members__)565        module_globals[name] = cls566        return cls567class IntEnum(int, Enum):568    """Enum where members are also (and must be) ints"""569def _reduce_ex_by_name(self, proto):570    return self.name571class Flag(Enum):572    """Support for flags"""573    def _generate_next_value_(name, start, count, last_values):574        """575        Generate the next value when not given.576        name: the name of the member577        start: the initital start value or None578        count: the number of existing members579        last_value: the last value assigned or None580        """581        if not count:582            return start if start is not None else 1583        for last_value in reversed(last_values):584            try:585                high_bit = _high_bit(last_value)586                break587            except Exception:588                raise TypeError('Invalid Flag value: %r' % last_value) from None589        return 2 ** (high_bit+1)590    @classmethod591    def _missing_(cls, value):592        original_value = value593        if value < 0:594            value = ~value595        possible_member = cls._create_pseudo_member_(value)596        if original_value < 0:597            possible_member = ~possible_member598        return possible_member599    @classmethod600    def _create_pseudo_member_(cls, value):601        """602        Create a composite member iff value contains only members.603        """604        pseudo_member = cls._value2member_map_.get(value, None)605        if pseudo_member is None:606            # verify all bits are accounted for607            _, extra_flags = _decompose(cls, value)608            if extra_flags:609                raise ValueError("%r is not a valid %s" % (value, cls.__name__))610            # construct a singleton enum pseudo-member611            pseudo_member = object.__new__(cls)612            pseudo_member._name_ = None613            pseudo_member._value_ = value614            # use setdefault in case another thread already created a composite615            # with this value616            pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)617        return pseudo_member618    def __contains__(self, other):619        if not isinstance(other, self.__class__):620            import warnings621            warnings.warn(622                    "using non-Flags in containment checks will raise "623                    "TypeError in Python 3.8",624                    DeprecationWarning, 2)625            return False626        return other._value_ & self._value_ == other._value_627    def __repr__(self):628        cls = self.__class__629        if self._name_ is not None:630            return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_)631        members, uncovered = _decompose(cls, self._value_)632        return '<%s.%s: %r>' % (633                cls.__name__,634                '|'.join([str(m._name_ or m._value_) for m in members]),635                self._value_,636                )637    def __str__(self):638        cls = self.__class__639        if self._name_ is not None:640            return '%s.%s' % (cls.__name__, self._name_)641        members, uncovered = _decompose(cls, self._value_)642        if len(members) == 1 and members[0]._name_ is None:643            return '%s.%r' % (cls.__name__, members[0]._value_)644        else:645            return '%s.%s' % (646                    cls.__name__,647                    '|'.join([str(m._name_ or m._value_) for m in members]),648                    )649    def __bool__(self):650        return bool(self._value_)651    def __or__(self, other):652        if not isinstance(other, self.__class__):653            return NotImplemented654        return self.__class__(self._value_ | other._value_)655    def __and__(self, other):656        if not isinstance(other, self.__class__):657            return NotImplemented658        return self.__class__(self._value_ & other._value_)659    def __xor__(self, other):660        if not isinstance(other, self.__class__):661            return NotImplemented662        return self.__class__(self._value_ ^ other._value_)663    def __invert__(self):664        members, uncovered = _decompose(self.__class__, self._value_)665        inverted = self.__class__(0)666        for m in self.__class__:667            if m not in members and not (m._value_ & self._value_):668                inverted = inverted | m669        return self.__class__(inverted)670class IntFlag(int, Flag):671    """Support for integer-based Flags"""672    @classmethod673    def _missing_(cls, value):674        if not isinstance(value, int):675            raise ValueError("%r is not a valid %s" % (value, cls.__name__))676        new_member = cls._create_pseudo_member_(value)677        return new_member678    @classmethod679    def _create_pseudo_member_(cls, value):680        pseudo_member = cls._value2member_map_.get(value, None)681        if pseudo_member is None:682            need_to_create = [value]683            # get unaccounted for bits684            _, extra_flags = _decompose(cls, value)685            # timer = 10686            while extra_flags:687                # timer -= 1688                bit = _high_bit(extra_flags)689                flag_value = 2 ** bit690                if (flag_value not in cls._value2member_map_ and691                        flag_value not in need_to_create692                        ):693                    need_to_create.append(flag_value)694                if extra_flags == -flag_value:695                    extra_flags = 0696                else:697                    extra_flags ^= flag_value698            for value in reversed(need_to_create):699                # construct singleton pseudo-members700                pseudo_member = int.__new__(cls, value)701                pseudo_member._name_ = None702                pseudo_member._value_ = value703                # use setdefault in case another thread already created a composite704                # with this value705                pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)706        return pseudo_member707    def __or__(self, other):708        if not isinstance(other, (self.__class__, int)):709            return NotImplemented710        result = self.__class__(self._value_ | self.__class__(other)._value_)711        return result712    def __and__(self, other):713        if not isinstance(other, (self.__class__, int)):714            return NotImplemented715        return self.__class__(self._value_ & self.__class__(other)._value_)716    def __xor__(self, other):717        if not isinstance(other, (self.__class__, int)):718            return NotImplemented719        return self.__class__(self._value_ ^ self.__class__(other)._value_)720    __ror__ = __or__721    __rand__ = __and__722    __rxor__ = __xor__723    def __invert__(self):724        result = self.__class__(~self._value_)725        return result726def _high_bit(value):727    """returns index of highest bit, or -1 if value is zero or negative"""728    return value.bit_length() - 1729def unique(enumeration):730    """Class decorator for enumerations ensuring unique member values."""731    duplicates = []732    for name, member in enumeration.__members__.items():733        if name != member.name:734            duplicates.append((name, member.name))735    if duplicates:736        alias_details = ', '.join(737                ["%s -> %s" % (alias, name) for (alias, name) in duplicates])738        raise ValueError('duplicate values found in %r: %s' %739                (enumeration, alias_details))740    return enumeration741def _decompose(flag, value):742    """Extract all members from the value."""743    # _decompose is only called if the value is not named744    not_covered = value745    negative = value < 0746    # issue29167: wrap accesses to _value2member_map_ in a list to avoid race747    #             conditions between iterating over it and having more pseudo-748    #             members added to it749    if negative:750        # only check for named flags751        flags_to_check = [752                (m, v)753                for v, m in list(flag._value2member_map_.items())754                if m.name is not None755                ]756    else:757        # check for named flags and powers-of-two flags758        flags_to_check = [759                (m, v)760                for v, m in list(flag._value2member_map_.items())761                if m.name is not None or _power_of_two(v)762                ]763    members = []764    for member, member_value in flags_to_check:765        if member_value and member_value & value == member_value:766            members.append(member)767            not_covered &= ~member_value768    if not members and value in flag._value2member_map_:769        members.append(flag._value2member_map_[value])770    members.sort(key=lambda m: m._value_, reverse=True)771    if len(members) > 1 and members[0].value == value:772        # we have the breakdown, don't need the value member itself773        members.pop(0)774    return members, not_covered775def _power_of_two(value):776    if value < 1:777        return False...

apache2_mod_proxy.py

Source:apache2_mod_proxy.py

1#!/usr/bin/python2# -*- coding: utf-8 -*-3# (c) 2016, Olivier Boukili <boukili.olivier@gmail.com>4#5# This file is part of Ansible.6#7# Ansible is free software: you can redistribute it and/or modify8# it under the terms of the GNU General Public License as published by9# the Free Software Foundation, either version 3 of the License, or10# (at your option) any later version.11#12# Ansible is distributed in the hope that it will be useful,13# but WITHOUT ANY WARRANTY; without even the implied warranty of14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the15# GNU General Public License for more details.16#17# You should have received a copy of the GNU General Public License18# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.19ANSIBLE_METADATA = {'metadata_version': '1.0',20                    'status': ['preview'],21                    'supported_by': 'community'}22DOCUMENTATION = '''23---24module: apache2_mod_proxy25author: Olivier Boukili (@oboukili)"26version_added: "2.2"27short_description: Set and/or get members' attributes of an Apache httpd 2.4 mod_proxy balancer pool28description:29  - Set and/or get members' attributes of an Apache httpd 2.4 mod_proxy balancer30    pool, using HTTP POST and GET requests. The httpd mod_proxy balancer-member31    status page has to be enabled and accessible, as this module relies on parsing32    this page. This module supports ansible check_mode, and requires BeautifulSoup33    python module.34options:35  balancer_url_suffix:36    default: /balancer-manager/37    description:38      - Suffix of the balancer pool url required to access the balancer pool39        status page (e.g. balancer_vhost[:port]/balancer_url_suffix).40    required: false41  balancer_vhost:42    default: None43    description:44      - (ipv4|ipv6|fqdn):port of the Apache httpd 2.4 mod_proxy balancer pool.45    required: true46  member_host:47    default: None48    description:49      - (ipv4|ipv6|fqdn) of the balancer member to get or to set attributes to.50        Port number is autodetected and should not be specified here.51        If undefined, apache2_mod_proxy module will return a members list of52        dictionaries of all the current balancer pool members' attributes.53    required: false54  state:55    default: None56    description:57      - Desired state of the member host.58        (absent|disabled),drained,hot_standby,ignore_errors can be59        simultaneously invoked by separating them with a comma (e.g. state=drained,ignore_errors).60    required: false61    choices: ["present", "absent", "enabled", "disabled", "drained", "hot_standby", "ignore_errors"]62  tls:63    default: false64    description:65      - Use https to access balancer management page.66    choices: ["true", "false"]67  validate_certs:68    default: true69    description:70      - Validate ssl/tls certificates.71    choices: ["true", "false"]72'''73EXAMPLES = '''74# Get all current balancer pool members' attributes:75- apache2_mod_proxy:76    balancer_vhost: 10.0.0.277# Get a specific member's attributes:78- apache2_mod_proxy:79    balancer_vhost: myws.mydomain.org80    balancer_suffix: /lb/81    member_host: node1.myws.mydomain.org82# Enable all balancer pool members:83- apache2_mod_proxy:84    balancer_vhost: '{{ myloadbalancer_host }}'85  register: result86- apache2_mod_proxy:87    balancer_vhost: '{{ myloadbalancer_host }}'88    member_host: '{{ item.host }}'89    state: present90  with_items: '{{ result.members }}'91# Gracefully disable a member from a loadbalancer node:92- apache2_mod_proxy:93    balancer_vhost: '{{ vhost_host }}'94    member_host: '{{ member.host }}'95    state: drained96  delegate_to: myloadbalancernode97- wait_for:98    host: '{{ member.host }}'99    port: '{{ member.port }}'100    state: drained101  delegate_to: myloadbalancernode102- apache2_mod_proxy:103    balancer_vhost: '{{ vhost_host }}'104    member_host: '{{ member.host }}'105    state: absent106  delegate_to: myloadbalancernode107'''108RETURN = '''109member:110    description: specific balancer member information dictionary, returned when apache2_mod_proxy module is invoked with member_host parameter.111    type: dict112    returned: success113    sample:114      {"attributes":115            {"Busy": "0",116            "Elected": "42",117            "Factor": "1",118            "From": "136K",119            "Load": "0",120            "Route": null,121            "RouteRedir": null,122            "Set": "0",123            "Status": "Init Ok ",124            "To": " 47K",125            "Worker URL": null126        },127        "balancer_url": "http://10.10.0.2/balancer-manager/",128        "host": "10.10.0.20",129        "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.20:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",130        "path": "/ws",131        "port": 8080,132        "protocol": "http",133        "status": {134            "disabled": false,135            "drained": false,136            "hot_standby": false,137            "ignore_errors": false138        }139      }140members:141    description: list of member (defined above) dictionaries, returned when apache2_mod_proxy is invoked with no member_host and state args.142    returned: success143    type: list144    sample:145      [{"attributes": {146            "Busy": "0",147            "Elected": "42",148            "Factor": "1",149            "From": "136K",150            "Load": "0",151            "Route": null,152            "RouteRedir": null,153            "Set": "0",154            "Status": "Init Ok ",155            "To": " 47K",156            "Worker URL": null157        },158        "balancer_url": "http://10.10.0.2/balancer-manager/",159        "host": "10.10.0.20",160        "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.20:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",161        "path": "/ws",162        "port": 8080,163        "protocol": "http",164        "status": {165            "disabled": false,166            "drained": false,167            "hot_standby": false,168            "ignore_errors": false169        }170        },171        {"attributes": {172            "Busy": "0",173            "Elected": "42",174            "Factor": "1",175            "From": "136K",176            "Load": "0",177            "Route": null,178            "RouteRedir": null,179            "Set": "0",180            "Status": "Init Ok ",181            "To": " 47K",182            "Worker URL": null183        },184        "balancer_url": "http://10.10.0.2/balancer-manager/",185        "host": "10.10.0.21",186        "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.21:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",187        "path": "/ws",188        "port": 8080,189        "protocol": "http",190        "status": {191            "disabled": false,192            "drained": false,193            "hot_standby": false,194            "ignore_errors": false}195        }196      ]197'''198import re199try:200    from BeautifulSoup import BeautifulSoup201except ImportError:202    HAS_BEAUTIFULSOUP = False203else:204    HAS_BEAUTIFULSOUP = True205# balancer member attributes extraction regexp:206EXPRESSION = r"(b=([\w\.\-]+)&w=(https?|ajp|wss?|ftp|[sf]cgi)://([\w\.\-]+):?(\d*)([/\w\.\-]*)&?[\w\-\=]*)"207# Apache2 server version extraction regexp:208APACHE_VERSION_EXPRESSION = r"Server Version: Apache/([\d.]+) \(([\w]+)\)"209def regexp_extraction(string, _regexp, groups=1):210    """ Returns the capture group (default=1) specified in the regexp, applied to the string """211    regexp_search = re.search(string=str(string), pattern=str(_regexp))212    if regexp_search:213        if regexp_search.group(groups) != '':214            return str(regexp_search.group(groups))215    return None216class BalancerMember(object):217    """ Apache 2.4 mod_proxy LB balancer member.218    attributes:219        read-only:220            host -> member host (string),221            management_url -> member management url (string),222            protocol -> member protocol (string)223            port -> member port (string),224            path -> member location (string),225            balancer_url -> url of this member's parent balancer (string),226            attributes -> whole member attributes (dictionary)227            module -> ansible module instance (AnsibleModule object).228        writable:229            status -> status of the member (dictionary)230    """231    def __init__(self, management_url, balancer_url, module):232        self.host = regexp_extraction(management_url, str(EXPRESSION), 4)233        self.management_url = str(management_url)234        self.protocol = regexp_extraction(management_url, EXPRESSION, 3)235        self.port = regexp_extraction(management_url, EXPRESSION, 5)236        self.path = regexp_extraction(management_url, EXPRESSION, 6)237        self.balancer_url = str(balancer_url)238        self.module = module239    def get_member_attributes(self):240        """ Returns a dictionary of a balancer member's attributes."""241        balancer_member_page = fetch_url(self.module, self.management_url)242        try:243            assert balancer_member_page[1]['status'] == 200244        except AssertionError:245            self.module.fail_json(msg="Could not get balancer_member_page, check for connectivity! " + balancer_member_page[1])246        else:247            try:248                soup = BeautifulSoup(balancer_member_page[0])249            except TypeError:250                self.module.fail_json(msg="Cannot parse balancer_member_page HTML! " + str(soup))251            else:252                subsoup = soup.findAll('table')[1].findAll('tr')253                keys = subsoup[0].findAll('th')254                for valuesset in subsoup[1::1]:255                    if re.search(pattern=self.host, string=str(valuesset)):256                        values = valuesset.findAll('td')257                        return dict((keys[x].string, values[x].string) for x in range(0, len(keys)))258    def get_member_status(self):259        """ Returns a dictionary of a balancer member's status attributes."""260        status_mapping = {'disabled':'Dis',261                          'drained':'Drn',262                          'hot_standby':'Stby',263                          'ignore_errors':'Ign'}264        status = {}265        actual_status = str(self.attributes['Status'])266        for mode in status_mapping.keys():267            if re.search(pattern=status_mapping[mode], string=actual_status):268                status[mode] = True269            else:270                status[mode] = False271        return status272    def set_member_status(self, values):273        """ Sets a balancer member's status attributes amongst pre-mapped values."""274        values_mapping = {'disabled':'&w_status_D',275                          'drained':'&w_status_N',276                          'hot_standby':'&w_status_H',277                          'ignore_errors':'&w_status_I'}278        request_body = regexp_extraction(self.management_url, EXPRESSION, 1)279        for k in values_mapping.keys():280            if values[str(k)]:281                request_body = request_body + str(values_mapping[k]) + '=1'282            else:283                request_body = request_body + str(values_mapping[k]) + '=0'284        response = fetch_url(self.module, self.management_url, data=str(request_body))285        try:286            assert response[1]['status'] == 200287        except AssertionError:288            self.module.fail_json(msg="Could not set the member status! " + self.host + " " + response[1]['status'])289    attributes = property(get_member_attributes)290    status = property(get_member_status, set_member_status)291class Balancer(object):292    """ Apache httpd 2.4 mod_proxy balancer object"""293    def __init__(self, host, suffix, module, members=None, tls=False):294        if tls:295            self.base_url = str(str('https://') + str(host))296            self.url = str(str('https://') + str(host) + str(suffix))297        else:298            self.base_url = str(str('http://') + str(host))299            self.url = str(str('http://') + str(host) + str(suffix))300        self.module = module301        self.page = self.fetch_balancer_page()302        if members is None:303            self._members = []304    def fetch_balancer_page(self):305        """ Returns the balancer management html page as a string for later parsing."""306        page = fetch_url(self.module, str(self.url))307        try:308            assert page[1]['status'] == 200309        except AssertionError:310            self.module.fail_json(msg="Could not get balancer page! HTTP status response: " + str(page[1]['status']))311        else:312            content = page[0].read()313            apache_version = regexp_extraction(content, APACHE_VERSION_EXPRESSION, 1)314            if not re.search(pattern=r"2\.4\.[\d]*", string=apache_version):315                self.module.fail_json(msg="This module only acts on an Apache2 2.4+ instance, current Apache2 version: " + str(apache_version))316            return content317    def get_balancer_members(self):318        """ Returns members of the balancer as a generator object for later iteration."""319        try:320            soup = BeautifulSoup(self.page)321        except TypeError:322            self.module.fail_json(msg="Cannot parse balancer page HTML! " + str(self.page))323        else:324            for element in soup.findAll('a')[1::1]:325                balancer_member_suffix = str(element.get('href'))326                try:327                    assert balancer_member_suffix is not ''328                except AssertionError:329                    self.module.fail_json(msg="Argument 'balancer_member_suffix' is empty!")330                else:331                    yield BalancerMember(str(self.base_url + balancer_member_suffix), str(self.url), self.module)332    members = property(get_balancer_members)333def main():334    """ Initiates module."""335    module = AnsibleModule(336        argument_spec=dict(337            balancer_vhost=dict(required=True, default=None, type='str'),338            balancer_url_suffix=dict(default="/balancer-manager/", type='str'),339            member_host=dict(type='str'),340            state=dict(type='str'),341            tls=dict(default=False, type='bool'),342            validate_certs=dict(default=True, type='bool')343        ),344        supports_check_mode=True345    )346    if HAS_BEAUTIFULSOUP is False:347        module.fail_json(msg="python module 'BeautifulSoup' is required!")348    if module.params['state'] is not None:349        states = module.params['state'].split(',')350        if (len(states) > 1) and (("present" in states) or ("enabled" in states)):351            module.fail_json(msg="state present/enabled is mutually exclusive with other states!")352        else:353            for _state in states:354                if _state not in ['present', 'absent', 'enabled', 'disabled', 'drained', 'hot_standby', 'ignore_errors']:355                    module.fail_json(msg="State can only take values amongst 'present', 'absent', 'enabled', 'disabled', 'drained', 'hot_standby', 'ignore_errors'.")356    else:357        states = ['None']358    mybalancer = Balancer(module.params['balancer_vhost'],359                          module.params['balancer_url_suffix'],360                          module=module,361                          tls=module.params['tls'])362    if module.params['member_host'] is None:363        json_output_list = []364        for member in mybalancer.members:365            json_output_list.append({366                "host": member.host,367                "status": member.status,368                "protocol": member.protocol,369                "port": member.port,370                "path": member.path,371                "attributes": member.attributes,372                "management_url": member.management_url,373                "balancer_url": member.balancer_url374            })375        module.exit_json(376            changed=False,377            members=json_output_list378        )379    else:380        changed = False381        member_exists = False382        member_status = {'disabled': False, 'drained': False, 'hot_standby': False, 'ignore_errors':False}383        for mode in member_status.keys():384            for state in states:385                if mode == state:386                    member_status[mode] = True387                elif mode == 'disabled' and state == 'absent':388                    member_status[mode] = True389        for member in mybalancer.members:390            if str(member.host) == str(module.params['member_host']):391                member_exists = True392                if module.params['state'] is not None:393                    member_status_before = member.status394                    if not module.check_mode:395                        member_status_after = member.status = member_status396                    else:397                        member_status_after = member_status398                    if member_status_before != member_status_after:399                        changed = True400                json_output = {401                    "host": member.host,402                    "status": member.status,403                    "protocol": member.protocol,404                    "port": member.port,405                    "path": member.path,406                    "attributes": member.attributes,407                    "management_url": member.management_url,408                    "balancer_url": member.balancer_url409                }410        if member_exists:411            module.exit_json(412                changed=changed,413                member=json_output414                )415        else:416            module.fail_json(msg=str(module.params['member_host']) + ' is not a member of the balancer ' + str(module.params['balancer_vhost']) + '!')417from ansible.module_utils.basic import AnsibleModule418from ansible.module_utils.urls import fetch_url419if __name__ == '__main__':...

test_emr_responses.py

Source:test_emr_responses.py

1# Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com>2#3# Permission is hereby granted, free of charge, to any person obtaining a4# copy of this software and associated documentation files (the5# "Software"), to deal in the Software without restriction, including6# without limitation the rights to use, copy, modify, merge, publish, dis-7# tribute, sublicense, and/or sell copies of the Software, and to permit8# persons to whom the Software is furnished to do so, subject to the fol-9# lowing conditions:10#11# The above copyright notice and this permission notice shall be included12# in all copies or substantial portions of the Software.13#14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-16# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT17# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 18# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,19# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS20# IN THE SOFTWARE.21# NOTE: These tests only cover the very simple cases I needed to test22# for the InstanceGroup fix.23import xml.sax24from boto import handler25from boto.emr import emrobject26from boto.resultset import ResultSet27from tests.compat import unittest28JOB_FLOW_EXAMPLE = b"""29<DescribeJobFlowsResponse30    xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15">31  <DescribeJobFlowsResult>32    <JobFlows>33      <member>34        <ExecutionStatusDetail>35          <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>36          <StartDateTime>2009-01-28T21:49:16Z</StartDateTime>37          <State>STARTING</State>38        </ExecutionStatusDetail>39        <BootstrapActions>40          <member>41            <BootstrapActionConfig>42              <ScriptBootstrapAction>43                <Args/>44                <Path>s3://elasticmapreduce/libs/hue/install-hue</Path>45              </ScriptBootstrapAction>46              <Name>Install Hue</Name>47            </BootstrapActionConfig>48          </member>49        </BootstrapActions>50        <VisibleToAllUsers>true</VisibleToAllUsers>51        <SupportedProducts>52          <member>Hue</member>53        </SupportedProducts>54        <Name>MyJobFlowName</Name>55        <LogUri>mybucket/subdir/</LogUri>56        <Steps>57          <member>58            <ExecutionStatusDetail>59              <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>60              <State>PENDING</State>61            </ExecutionStatusDetail>62            <StepConfig>63              <HadoopJarStep>64                <Jar>MyJarFile</Jar>65                <MainClass>MyMailClass</MainClass>66                <Args>67                  <member>arg1</member>68                  <member>arg2</member>69                </Args>70                <Properties/>71              </HadoopJarStep>72              <Name>MyStepName</Name>73              <ActionOnFailure>CONTINUE</ActionOnFailure>74            </StepConfig>75          </member>76        </Steps>77        <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId>78        <Instances>79          <Placement>80            <AvailabilityZone>us-east-1a</AvailabilityZone>81          </Placement>82          <SlaveInstanceType>m1.small</SlaveInstanceType>83          <MasterInstanceType>m1.small</MasterInstanceType>84          <Ec2KeyName>myec2keyname</Ec2KeyName>85          <InstanceCount>4</InstanceCount>86          <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps>87        </Instances>88      </member>89    </JobFlows>90  </DescribeJobFlowsResult>91  <ResponseMetadata>92    <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId>93  </ResponseMetadata>94</DescribeJobFlowsResponse>95"""96JOB_FLOW_COMPLETED = b"""97<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">98  <DescribeJobFlowsResult>99    <JobFlows>100      <member>101        <ExecutionStatusDetail>102          <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>103          <LastStateChangeReason>Steps completed</LastStateChangeReason>104          <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>105          <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>106          <State>COMPLETED</State>107          <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>108        </ExecutionStatusDetail>109        <BootstrapActions/>110        <Name>RealJobFlowName</Name>111        <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri>112        <Steps>113          <member>114            <StepConfig>115              <HadoopJarStep>116                <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar>117                <Args>118                  <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member>119                </Args>120                <Properties/>121              </HadoopJarStep>122              <Name>Setup Hadoop Debugging</Name>123              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>124            </StepConfig>125            <ExecutionStatusDetail>126              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>127              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>128              <State>COMPLETED</State>129              <EndDateTime>2010-10-21T01:04:22Z</EndDateTime>130            </ExecutionStatusDetail>131          </member>132          <member>133            <StepConfig>134              <HadoopJarStep>135                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>136                <Args>137                  <member>-mapper</member>138                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member>139                  <member>-reducer</member>140                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member>141                  <member>-input</member>142                  <member>s3://example.emrtest.data/raw/2010/10/20/*</member>143                  <member>-input</member>144                  <member>s3://example.emrtest.data/raw/2010/10/19/*</member>145                  <member>-input</member>146                  <member>s3://example.emrtest.data/raw/2010/10/18/*</member>147                  <member>-input</member>148                  <member>s3://example.emrtest.data/raw/2010/10/17/*</member>149                  <member>-input</member>150                  <member>s3://example.emrtest.data/raw/2010/10/16/*</member>151                  <member>-input</member>152                  <member>s3://example.emrtest.data/raw/2010/10/15/*</member>153                  <member>-input</member>154                  <member>s3://example.emrtest.data/raw/2010/10/14/*</member>155                  <member>-output</member>156                  <member>s3://example.emrtest.crunched/</member>157                </Args>158                <Properties/>159              </HadoopJarStep>160              <Name>testjob_Initial</Name>161              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>162            </StepConfig>163            <ExecutionStatusDetail>164              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>165              <StartDateTime>2010-10-21T01:04:22Z</StartDateTime>166              <State>COMPLETED</State>167              <EndDateTime>2010-10-21T01:36:18Z</EndDateTime>168            </ExecutionStatusDetail>169          </member>170          <member>171            <StepConfig>172              <HadoopJarStep>173                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>174                <Args>175                  <member>-mapper</member>176                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member>177                  <member>-reducer</member>178                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member>179                  <member>-input</member>180                  <member>s3://example.emrtest.crunched/*</member>181                  <member>-output</member>182                  <member>s3://example.emrtest.step1/</member>183                </Args>184                <Properties/>185              </HadoopJarStep>186              <Name>testjob_step1</Name>187              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>188            </StepConfig>189            <ExecutionStatusDetail>190              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>191              <StartDateTime>2010-10-21T01:36:18Z</StartDateTime>192              <State>COMPLETED</State>193              <EndDateTime>2010-10-21T01:37:51Z</EndDateTime>194            </ExecutionStatusDetail>195          </member>196          <member>197            <StepConfig>198              <HadoopJarStep>199                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>200                <Args>201                  <member>-mapper</member>202                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member>203                  <member>-reducer</member>204                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member>205                  <member>-input</member>206                  <member>s3://example.emrtest.crunched/*</member>207                  <member>-output</member>208                  <member>s3://example.emrtest.step2/</member>209                </Args>210                <Properties/>211              </HadoopJarStep>212              <Name>testjob_step2</Name>213              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>214            </StepConfig>215            <ExecutionStatusDetail>216              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>217              <StartDateTime>2010-10-21T01:37:51Z</StartDateTime>218              <State>COMPLETED</State>219              <EndDateTime>2010-10-21T01:39:32Z</EndDateTime>220            </ExecutionStatusDetail>221          </member>222          <member>223            <StepConfig>224              <HadoopJarStep>225                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>226                <Args>227                  <member>-mapper</member>228                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member>229                  <member>-reducer</member>230                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member>231                  <member>-input</member>232                  <member>s3://example.emrtest.step1/*</member>233                  <member>-output</member>234                  <member>s3://example.emrtest.step3/</member>235                </Args>236                <Properties/>237              </HadoopJarStep>238              <Name>testjob_step3</Name>239              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>240            </StepConfig>241            <ExecutionStatusDetail>242              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>243              <StartDateTime>2010-10-21T01:39:32Z</StartDateTime>244              <State>COMPLETED</State>245              <EndDateTime>2010-10-21T01:41:22Z</EndDateTime>246            </ExecutionStatusDetail>247          </member>248          <member>249            <StepConfig>250              <HadoopJarStep>251                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>252                <Args>253                  <member>-mapper</member>254                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member>255                  <member>-reducer</member>256                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member>257                  <member>-input</member>258                  <member>s3://example.emrtest.step1/*</member>259                  <member>-output</member>260                  <member>s3://example.emrtest.step4/</member>261                </Args>262                <Properties/>263              </HadoopJarStep>264              <Name>testjob_step4</Name>265              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>266            </StepConfig>267            <ExecutionStatusDetail>268              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>269              <StartDateTime>2010-10-21T01:41:22Z</StartDateTime>270              <State>COMPLETED</State>271              <EndDateTime>2010-10-21T01:43:03Z</EndDateTime>272            </ExecutionStatusDetail>273          </member>274        </Steps>275        <JobFlowId>j-3H3Q13JPFLU22</JobFlowId>276        <Instances>277          <SlaveInstanceType>m1.large</SlaveInstanceType>278          <MasterInstanceId>i-64c21609</MasterInstanceId>279          <Placement>280            <AvailabilityZone>us-east-1b</AvailabilityZone>281          </Placement>282          <InstanceGroups>283            <member>284              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>285              <InstanceRunningCount>0</InstanceRunningCount>286              <StartDateTime>2010-10-21T01:02:09Z</StartDateTime>287              <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime>288              <State>ENDED</State>289              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>290              <InstanceRequestCount>1</InstanceRequestCount>291              <InstanceType>m1.large</InstanceType>292              <Market>ON_DEMAND</Market>293              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>294              <InstanceRole>MASTER</InstanceRole>295              <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId>296              <Name>master</Name>297            </member>298            <member>299              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>300              <InstanceRunningCount>0</InstanceRunningCount>301              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>302              <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>303              <State>ENDED</State>304              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>305              <InstanceRequestCount>9</InstanceRequestCount>306              <InstanceType>m1.large</InstanceType>307              <Market>ON_DEMAND</Market>308              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>309              <InstanceRole>CORE</InstanceRole>310              <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId>311              <Name>slave</Name>312            </member>313          </InstanceGroups>314          <NormalizedInstanceHours>40</NormalizedInstanceHours>315          <HadoopVersion>0.20</HadoopVersion>316          <MasterInstanceType>m1.large</MasterInstanceType>317          <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName>318          <Ec2KeyName>myubersecurekey</Ec2KeyName>319          <InstanceCount>10</InstanceCount>320          <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps>321        </Instances>322      </member>323    </JobFlows>324  </DescribeJobFlowsResult>325  <ResponseMetadata>326    <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId>327  </ResponseMetadata>328</DescribeJobFlowsResponse>329"""330class TestEMRResponses(unittest.TestCase):331    def _parse_xml(self, body, markers):332        rs = ResultSet(markers)333        h = handler.XmlHandler(rs, None)334        xml.sax.parseString(body, h)335        return rs336    def _assert_fields(self, response, **fields):337        for field, expected in fields.items():338            actual = getattr(response, field)339            self.assertEquals(expected, actual,340                              "Field %s: %r != %r" % (field, expected, actual))341    def test_JobFlows_example(self):342        [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE,343                                    [('member', emrobject.JobFlow)])344        self._assert_fields(jobflow,345                            creationdatetime='2009-01-28T21:49:16Z',346                            startdatetime='2009-01-28T21:49:16Z',347                            state='STARTING',348                            instancecount='4',349                            jobflowid='j-3UN6WX5RRO2AG',350                            loguri='mybucket/subdir/',351                            name='MyJobFlowName',352                            availabilityzone='us-east-1a',353                            slaveinstancetype='m1.small',354                            masterinstancetype='m1.small',355                            ec2keyname='myec2keyname',356                            keepjobflowalivewhennosteps='true')357    def test_JobFlows_completed(self):358        [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED,359                                    [('member', emrobject.JobFlow)])360        self._assert_fields(jobflow,361                            creationdatetime='2010-10-21T01:00:25Z',362                            startdatetime='2010-10-21T01:03:59Z',363                            enddatetime='2010-10-21T01:44:18Z',364                            state='COMPLETED',365                            instancecount='10',366                            jobflowid='j-3H3Q13JPFLU22',367                            loguri='s3n://example.emrtest.scripts/jobflow_logs/',368                            name='RealJobFlowName',369                            availabilityzone='us-east-1b',370                            slaveinstancetype='m1.large',371                            masterinstancetype='m1.large',372                            ec2keyname='myubersecurekey',373                            keepjobflowalivewhennosteps='false')374        self.assertEquals(6, len(jobflow.steps))...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.