KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use update_mapping method in tempest

Best Python code snippet using tempest_python

build_index.py

Source:build_index.py

1#!/usr/bin/env python2# -*- coding: utf-8 -*-3'''4TODO:5       score-tuning:   1. Symbol6                    2. any ID7                    3. name8                    4. summary9                    5. other text (uniprot name, interpro desc, go term)10       case to consider:11                q=hypoxia inducible factor12                 if matching from the beginning, rank higher?13           appear first     # name: "hypoxia-inducible factor 1, alpha subunit (basic helix-loop-helix transcription factor)"14           appear latter    # name: "similar to egl nine homolog 1 (hypoxia-inducible factor prolyl hydroxylase 2) (hif-prolyl hydroxylase 2) (hif-ph2) (hph-2) (sm-20)"15    auto-correction:  1. multiple terms use default AND first, if no hit, do the OR automatically (with a note on UI)16'''17from functools import partial18from django.conf import settings19from django.db.models.signals import post_save, post_delete20from pyes.exceptions import (NotFoundException, IndexMissingException,21                             ElasticSearchException, TypeMissingException)22from biogps.utils import ask23from biogps.utils.models import queryset_iterator24from es_lib import get_es_conn25import logging26log = logging.getLogger('pyes')27if settings.DEBUG:28    log.setLevel(logging.DEBUG)29    if len(log.handlers) == 0:30        log_handler = logging.StreamHandler()31        log.addHandler(log_handler)32class BiogpsESIndexerBase(object):33    ES_HOST = settings.ES_HOST34    ES_INDEX_NAME = settings.ES_INDEXES['default']35    step = 10036    def __init__(self):37        self.conn = get_es_conn(self.ES_HOST, default_idx=[self.ES_INDEX_NAME])38    def check(self):39        '''print out ES server info for verification.'''40        print "Servers:", self.conn.servers41        print "Default indices:", self.conn.default_indices42        print "ES_INDEX_TYPE:", self.ES_INDEX_TYPE43    def create_index(self):44        try:45            print self.conn.open_index(self.ES_INDEX_NAME)46        #except NotFoundException:47        except IndexMissingException:48            print self.conn.create_index(self.ES_INDEX_NAME)49    def delete_index_type(self, index_type):50        '''Delete all indexes for a given index_type.'''51        index_name = self.ES_INDEX_NAME52        #Check if index_type exists53        try:54            mapping = self.conn.get_mapping(index_type, index_name)55        except TypeMissingException:56            print 'Error: index type "%s" does not exist in index "%s".' % (index_type, index_name)57            return58        path = '/%s/%s' % (index_name, index_type)59        if ask('Confirm to delete all data under "%s":' % path) == 'Y':60            return self.conn.delete_mapping(index_name, index_type)61    def verify_mapping(self, update_mapping=False):62        '''Verify if index and mapping exist, update mapping if mapping does not exist,63           or "update_mapping=True" explicitly64        '''65        conn = self.conn66        index_name = self.ES_INDEX_NAME67        index_type = self.ES_INDEX_TYPE68        #Test if index exists69        try:70            print "Opening index...", conn.open_index(index_name)71        except NotFoundException:72            print 'Error: index "%s" does not exist. Create it first.' % index_name73            return -174        try:75            cur_mapping = conn.get_mapping(index_type, index_name)76            empty_mapping = False77        except ElasticSearchException:78            #if no existing mapping available for index_type79            #force update_mapping to True80            empty_mapping = True81            update_mapping = True82#        empty_mapping = not cur_mapping[index_name].get(index_type, {})83#        if empty_mapping:84#            #if no existing mapping available for index_type85#            #force update_mapping to True86#            update_mapping = True87        if update_mapping:88            print "Updating mapping...",89            if not empty_mapping:90                print "\n\tRemoving existing mapping...",91                print conn.delete_mapping(index_name, index_type)92            _mapping = self.get_field_mapping()93            print conn.put_mapping(index_type,94                                   _mapping,95                                   [index_name])96    def index(self, doc, index_type, id=None):97        '''add a doc to the index. If id is not None, the existing doc will be98           updated.99        '''100        return self.conn.index(doc, self.ES_INDEX_NAME, index_type, id=id)101    def delete_index(self, index_type, id):102        '''delete a doc from the index based on passed id.'''103        return self.conn.delete(self.ES_INDEX_NAME, index_type, id)104    def optimize(self):105        return self.conn.optimize(self.ES_INDEX_NAME, wait_for_merge=True)106class BiogpsModelESIndexer(BiogpsESIndexerBase):107    '''The base class for indexing objects from BioGPSModel derived models,108       e.g., BioGPSPlugin, BiogpsGenereportLayout, etc.109    '''110    _model = None           # need to specify in each subclass111    ES_INDEX_TYPE = None    # need to specify in each subclass112    step = 100              # how many objects to retrieve in one queryset query113    def _get_field_mapping(self, extra_attrs={}):114        #field mapping templates115        id_type = {'store': "yes",116                   'index': 'not_analyzed',117                   'type': 'string',118                   'term_vector': 'with_positions_offsets'}119        text_type = {'store': "no",120              'index': 'analyzed',121              'type': 'string',122              'term_vector': 'with_positions_offsets'}123        date_type = {'store': "no",124              'index': 'not_analyzed',125              'type': 'date',126              'format': 'YYYY-MM-dd HH:mm:ss'}127        integer_type = {'type': 'integer'}128        float_type = {'type': 'float'}129        object_type = {'type': 'object'}130        boolean_type = {'type': 'boolean'}131        store_only = {'store': "yes",132                      'index': 'no',133                      'type': 'string'}134        disabled_object = {'type': 'object',135                           'enabled': False}136        disabled_string = {'type': 'string',137                           'index': 'no'}138        disabled_double = {'type': 'double',139                           'index': 'no'}140        disabled_integer = {'type': 'integer',141                             'index': 'no'}142        td = {'id_type': id_type,143              'text_type': text_type,144              'date_type': date_type,145              'integer_type': integer_type,146              'float_type': float_type,147              'boolean_type': boolean_type,148              'object_type': object_type,149              'store_only': store_only,150              'disabled_object':  disabled_object,151              'disabled_string':  disabled_string,152              'disabled_double':  disabled_double,153              'disabled_integer': disabled_integer}154        properties = {'in': id_type,155                      'id': id_type,156                      'created': date_type,157                      'lastmodified': date_type,158                      'role_permission': id_type,159                      'tags': id_type}160        for t_id in td.keys():161            for attr in extra_attrs.pop(t_id, []):162                properties[attr] = td[t_id]163        properties.update(extra_attrs)164        for f in properties:165            properties[f] = properties[f].copy()166        #some special settings167        #for tag field168        properties['tags']['index_name'] = 'tag'169        #for name field170        properties['name'] = {171             "type" : "multi_field",172             "store": "yes",173             "fields" : {174                "name" : {175                          'index': 'analyzed',176                          'type': 'string',177                          'boost': 2.0,178                          'term_vector': 'with_positions_offsets'179                         },180                "for_sort" : {181                              "type" : "string",182                              "index" : "not_analyzed"183                             }184              }185        }186        #for owner field187        properties['owner'] = {188            "store": "yes",189            "type": "object",190            "path": 'just_name',  #! This is important to make query "username:cwudemo" work, instead of using "owner.username:cwudemo"191            "properties" : {192                "username" : {193                              "type" : "string",194                              "index_name": "username",195                              "index": "not_analyzed",196                             },197                "name" : {198                           "type" : "string",199                          "index_name": "owner",200                          "index": "analyzed",201                         },202                "url" : {203                          "type" : "string",204                          "index": "no",205                         }206            }207        }208        #for rating_data field209        properties['rating_data'] = {210            "store": "yes",211            "type": "object",212            "properties": {213                "avg_stars": {"type": "short"},214                "total": {"type": "short"},215                "avg": {"type": "short"},216            }217        }218        mapping = {'properties': properties}219        # enable _source compression220        mapping["_source"] = {"enabled" : True,221                              "compress": True,222                              "compression_threshold": "1kb"}223#        #store "_all" for highlighting.224#        mapping["_all"] = {"store": "yes",225#                           "type": "string",226#                           "term_vector": "with_positions_offsets"}227        return mapping228    def get_field_mapping(self):229        raise NotImplementedError230    def build_index(self, update_mapping=False, bulk=True, verbose=False):231        conn = self.conn232        index_name = self.ES_INDEX_NAME233        index_type = self.ES_INDEX_TYPE234        self.verify_mapping(update_mapping=update_mapping)235        print "Building index..."236        cnt = 0237        for p in queryset_iterator(self._model, batch_size=self.step):238            doc = p.object_cvt(mode='es')239            conn.index(doc, index_name, index_type, doc['id'], bulk=bulk)240            cnt += 1241            if verbose:242                print cnt, ':', doc['id']243        print conn.flush()244        print conn.refresh()245        print 'Done! - {} docs indexed.'.format(cnt)246    def index_object(self, object_id):247        obj = self._model.objects.get(id=object_id)248        doc = obj.object_cvt(mode='es')249        print self.index(doc, self.ES_INDEX_TYPE, id=doc['id'])250class BiogpsPluginESIndexer(BiogpsModelESIndexer):251    '''A class for indexing all BiogpsPlugin objects.'''252    def __init__(self):253        from biogps.plugin.models import BiogpsPlugin254        self._model = BiogpsPlugin255        self.ES_INDEX_TYPE = self._model.short_name256        super(BiogpsModelESIndexer, self).__init__()257    def get_field_mapping(self):258        m_usage_data = {259            "store": "yes",260            "type": "object",261            "properties": {262                "users": {"type": "short"},263                "layouts": {"type": "short"},264            }265        }266        m = self._get_field_mapping(extra_attrs={'id_type': ['type', 'species'],267                                                 'text_type': ['name', 'description', 'short_description', 'url'],268                                                 "float_type": ['popularity'],269                                                 'disabled_object': ['options'],270                                                 'disabled_string': ['shortUrl', 'permission_style'],271                                                 'usage_data': m_usage_data272                                                 })273        return m274class BiogpsLayoutESIndexer(BiogpsModelESIndexer):275    '''A class for indexing all BiogpsGenereportLayout objects.'''276    def __init__(self):277        from biogps.layout.models import BiogpsGenereportLayout278        self._model = BiogpsGenereportLayout279        self.ES_INDEX_TYPE = self._model.short_name280        super(BiogpsModelESIndexer, self).__init__()281    def get_field_mapping(self):282        m = self._get_field_mapping(extra_attrs={'text_type': ['name', 'description'],283                                                 'disabled_string': ['permission_style'],284                                                 })285        #some special settings286#        m['name']['boost'] = 2.0287        return m288class BiogpsGenelistESIndexer(BiogpsModelESIndexer):289    '''A class for indexing all BiogpsGeneList objects.'''290    def __init__(self):291        from biogps.genelist.models import BiogpsGeneList292        self._model = BiogpsGeneList293        self.ES_INDEX_TYPE = self._model.short_name294        super(BiogpsModelESIndexer, self).__init__()295    def get_field_mapping(self):296        m = self._get_field_mapping(extra_attrs={'text_type': ['name', 'description'],297                                                 'disabled_string': ['permission_style'],298                                                 })299        #some special settings300#        m['name']['boost'] = 2.0301        return m302def _rebuild_x(delete_old=False, update_mapping=False, indexer=None):303    '''A convenient function for re-building indexes.304    '''305    es_indexer = indexer()306    if delete_old:307        es_indexer.delete_index_type(es_indexer.ES_INDEX_TYPE)308        update_mapping = True   # if delete_old is True, update_mapping should be True anyway309    es_indexer.build_index(update_mapping=update_mapping, bulk=True)310rebuild_plugin = partial(_rebuild_x, indexer=BiogpsPluginESIndexer)311rebuild_plugin.__doc__ = 'A convenient function for re-building all plugins.'312rebuild_layout = partial(_rebuild_x, indexer=BiogpsLayoutESIndexer)313rebuild_layout.__doc__ = 'A convenient function for re-building all layouts.'314rebuild_genelist = partial(_rebuild_x, indexer=BiogpsGenelistESIndexer)315rebuild_genelist.__doc__ = 'A convenient function for re-building all genelists.'316def rebuild_all(delete_old=False):317    '''A convenient function for re-building all biogpsmodel objects,318       not including genes and datasets.319    '''320    rebuild_plugin(delete_old)321    rebuild_layout(delete_old)322    rebuild_genelist(delete_old)323def on_the_fly_es_update_handler(sender, **kwargs):324    '''A post_save signal handler to update ES indexes whenever a BiogpsModel object325       is created/updated.326       To connect to a signal:327           post_save.connect(on_the_fly_es_update_handler, sender=BioGPSModel,328                             dispatch_uid="some_unique_id")329    '''330    if not getattr(settings, "SUSPEND_ES_UPDATE", None):331        object = kwargs['instance']332        es_indexer = BiogpsModelESIndexer()333        doc = object.object_cvt(mode='es')334        res = es_indexer.index(doc, object.short_name, object.id)335        return True336def on_the_fly_es_delete_handler(sender, **kwargs):337    '''A post_delete signal handler to delete ES indexes whenever a BiogpsModel object338       is deleted.339       To connect to a signal:340           post_delete.connect(on_the_fly_es_delete_handler, sender=BioGPSModel,341                               dispatch_uid="some_unique_id")342    '''343    if not getattr(settings, "SUSPEND_ES_UPDATE", None):344        object = kwargs['instance']345        es_indexer = BiogpsModelESIndexer()346        res = es_indexer.delete_index(object.short_name, object.id)347        return True348def set_on_the_fly_indexing(biogpsmodel):349    '''set input biogpsmodel for on the fly indexing:350          * add to index when a new object is created351          * update index when an object is updated352          * delete from index when an object is deleted353    '''354    post_save.connect(on_the_fly_es_update_handler, sender=biogpsmodel,355                      dispatch_uid=biogpsmodel.__name__ + 'update_indexer')356    post_delete.connect(on_the_fly_es_delete_handler, sender=biogpsmodel,...

feature_base.py

Source:feature_base.py

...28    A set of features that generates a pattern for the features it contains.29    This set will act like a mapping in that we map names to patterns.30    """31    mapping = {}32    def update_mapping(self):33        u"""34        Called every time we care about the mapping of names to features.35        """36        self.mapping = dict([(f.name, f) for f in iter(self)])37    @property38    def PATTERN(self):39        u"""40        Uses the mapping of names to features to return a PATTERN suitable41        for using the lib2to3 patcomp.42        """43        self.update_mapping()44        return u" |\n".join([pattern_unformatted % (f.name, f._pattern) for f in iter(self)])45    def __getitem__(self, key):46        u"""47        Implement a simple mapping to get patterns from names.48        """...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.