KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use section_logger method in molecule

Best Python code snippet using molecule_python

visualgenome.py

Source:visualgenome.py

...51            "|".join(r['synsets'])52        ]53    return extractor54def extract_relationship_dataframe(vg, limit=10, report=2e5):55    section = section_logger(1)56    data = []57    counter = 058    for img in vg['relationships']:59        if counter > limit:60            break61        else:62            counter += 163        if counter % report == 0:64            section("Loaded relationships in {} images".format(counter))65        rows = map(extract_relationship_data(img['image_id']), img['relationships'])66        data.extend(list(rows))67    rdf = pd.DataFrame(data, columns=['img', 'object', 'subject', 'predicate', 'synsets'])68    return rdf69def extract_object_data(img_id, img_w, img_h):70    def extractor(obj):71        return [72            obj['object_id'],73            "|".join(obj['synsets']),74            "|".join(obj['names']),75            img_id,76            obj['x']/img_w,77            obj['y']/img_h,78            obj['h']/img_h,79            obj['w']/img_w80        ]81    return extractor82def extract_object_dataframe(vg, limit=10, report=2e5):83    section = section_logger(1)84    data = []85    counter = 086    for img in vg['objects']:87        if counter > limit:88            break89        else:90            counter += 191        if counter % report == 0:92            section("Loaded objects in {} images".format(counter))93        current_image = vg['imageIdMap'][img['image_id']]94        assert current_image['image_id'] == img['image_id'], 'Position in the list {} does not correspond to ID {}'.format(img['image_id'], current_image['image_id'])95        rows = map(extract_object_data(img['image_id'], current_image['width'], current_image['height']), img['objects'])96        data.extend(rows)97    odf = pd.DataFrame(data, columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w'])98    return odf99def add_key_to_dict(id, collection):100    if id in collection:101        id_obj = collection[id]102    else:103        id_obj = len(collection)104        collection[id] = id_obj105    return id_obj106def extract_knowledge_graph(vg, limit=10, report=2e5):107    section = section_logger(1)108    counter = 0109    synset_ids = {}110    knowledge_graph = nx.DiGraph()111    for img in vg['relationships']:112        if counter > limit:113            break114        else:115            counter += 1116        if counter % report == 0:117            section("Loaded relationships in {} images".format(counter))118        for r in img['relationships']:119            object = r['object']['synsets']120            subject = r['subject']['synsets']121            predicate = r['synsets']122            for syn_obj in object:123                for syn_sub in subject:124                    id_obj = add_key_to_dict(syn_obj, synset_ids)125                    id_sub = add_key_to_dict(syn_sub, synset_ids)126                    if len(predicate) == 1:127                        predicate_str = predicate[0]128                    else:129                        predicate_str = 'relation.n.01'130                    knowledge_graph.add_edge(id_obj, id_sub, predicate=predicate_str)131    return knowledge_graph, synset_ids132def extract_data_analytics():133    section = section_logger()134    section('Extracting Visual Genome')135    vg = load_visual_genome()136    section('Generating DataFrames from the object files')137    odf = extract_object_dataframe(vg, int(1e6))138    section('Generating DataFrames from the relationship files')139    rdf = extract_relationship_dataframe(vg, int(1e6))140    section('Saving CSVs')141    odf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_objects.csv'))142    rdf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_relationships.csv'))143def extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, relation, predicate):144    hypernyms = wordnet_syn.__getattribute__(relation)()145    for hyp in hypernyms:146        hyp_name = hyp.name()147        if hyp_name in synset_ids:148            hyp_id = synset_ids[hyp_name]149        else:150            hyp_id = add_key_to_dict(hyp_name, synset_ids)151        graph.add_edge(syn_id, hyp_id, predicate=predicate)152    return graph, synset_ids153def add_wordnet_synsets(graph, synset_ids, report=2e5):154    old_synset_ids = dict(synset_ids)155    section = section_logger(1)156    counter = 0157    for syn_name, syn_id in old_synset_ids.items():158        counter += 1159        if counter % report == 0:160            section("Loaded relationships in {} images".format(counter))161        wordnet_syn = wordnet.synset(syn_name)162        extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hypernyms', 'generalize.v.01')163        extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hyponyms', 'specialize.v.01')164    return graph, synset_ids165def save_graph_info(graph, ids):166    nx.write_gml(graph, os.path.join(cf.VG_ANALYTICS, 'knowledge_graph.gml'))167    with open(os.path.join(cf.VG_ANALYTICS, 'synset_ids.json'), 'w') as outfile:168        json.dump(ids, outfile)169def compute_knowledge_graph():170    section = section_logger()171    section('Loading JSON files from dataset')172    vg = load_visual_genome()173    section('Generating Knowledge Graph')174    vi_graph, synset_ids = extract_knowledge_graph(vg, limit=1e10)175    section('Annotating with Wordnet synsets')176    wn_graph, synset_ids = add_wordnet_synsets(vi_graph, synset_ids)177    section('Saving graph')178    save_graph_info(wn_graph, synset_ids)179    return wn_graph180def load_graph_info():181    wn_graph = nx.read_gml(os.path.join(cf.VG_ANALYTICS, 'knowledge_graph.gml'))182    with open(os.path.join(cf.VG_ANALYTICS, 'synset_ids.json'), 'r') as infile:183        ids = json.load(infile)184    return wn_graph, ids...

settings.py

Source:settings.py

1"""2Django settings for redbutterfly project.3Generated by 'django-admin startproject' using Django 1.10.4For more information on this file, see5https://docs.djangoproject.com/en/1.10/topics/settings/6For the full list of settings and their values, see7https://docs.djangoproject.com/en/1.10/ref/settings/8"""9import os10import logging11from util.log_util import *12# Build paths inside the project like this: os.path.join(BASE_DIR, ...)13BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))14SECRET_KEY = 'mpjq@%1+hv@j90oxs0spt&9#4xeg0#g_n&9nnb)v_$-oz4!39$'15DEBUG = True16ALLOWED_HOSTS = ['*']17#default ini18DEFAULT_CONFIG_FILE = os.path.join(BASE_DIR, "conf/server.ini")19SECTION_REDIS = 'redis_server'20SECTION_LOGGER = 'server_log'21#logger info define22SERVICE_LOGGERFILE = os.path.join(BASE_DIR, "log/project_log/service.log")23CreatLogger('server', SERVICE_LOGGERFILE, SECTION_LOGGER)24server_logger = logging.getLogger('server')25# Application definition26INSTALLED_APPS = [27    'django.contrib.admin',28    'django.contrib.auth',29    'django.contrib.contenttypes',30    'django.contrib.sessions',31    'django.contrib.messages',32    'django.contrib.staticfiles',33    'rest_framework',34    'red',35]36MIDDLEWARE_CLASSES = [37    'django.middleware.security.SecurityMiddleware',38    'django.contrib.sessions.middleware.SessionMiddleware',39    'django.middleware.locale.LocaleMiddleware',40    'django.middleware.common.CommonMiddleware',41    'django.middleware.csrf.CsrfViewMiddleware',42    'django.contrib.auth.middleware.AuthenticationMiddleware',43    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',44    'django.contrib.messages.middleware.MessageMiddleware',45    'django.middleware.clickjacking.XFrameOptionsMiddleware',46]47ROOT_URLCONF = 'conf.urls'48TEMPLATES = [49    {50        'BACKEND': 'django.template.backends.django.DjangoTemplates',51        'DIRS': [os.path.join(BASE_DIR,'templates')],52        'APP_DIRS': True,53        'OPTIONS': {54            'context_processors': [55                'django.template.context_processors.debug',56                'django.template.context_processors.request',57                'django.contrib.auth.context_processors.auth',58                'django.contrib.messages.context_processors.messages',59            ],60        },61    },62]63TEMPLATE_CONTEXT_PROCESSORS = (64        "django.template.context_processors.request",65	"django.contrib.auth.context_processors.auth",66	"django.template.context_processors.debug",67	"django.template.context_processors.i18n",68	"django.template.context_processors.media",69	"django.template.context_processors.static",70	"django.template.context_processors.tz",71	"django.contrib.messages.context_processors.messages"72)73WSGI_APPLICATION = 'conf.wsgi.application'74DATABASES = {75    'default': {76        'ENGINE': 'django.db.backends.mysql',77        'NAME': 'redbutterfly',78	'USER':	'root',79	'PASSWORD': '426334',80	'HOST': '',81	'PORT': '',82    }83}84CACHES = {85    'default': {86        'BACKEND': 'django_redis.cache.RedisCache',87        'LOCATION': 'redis://127.0.0.1:6379/2',88	'OPTIONS': {89	    'CLIENT_CLASS': 'django_redis.client.DefaultClient',90	},91    },92}93REDIS_TIMEOUT=7*24*60*6094CUBES_REDIS_TIMEOUT=60*6095NEVER_REDIS_TIMEOUT=365*24*60*6096LOGIN_URL='/login/'97AUTH_USER_MODEL = 'red.User'98AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend',)99# Password validation100# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators101AUTH_PASSWORD_VALIDATORS = [102    {103        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',104    },105    {106        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',107    },108    {109        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',110    },111    {112        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',113    },114]115# Internationalization116# https://docs.djangoproject.com/en/1.10/topics/i18n/117LANGUAGE_CODE = 'en-us'118TIME_ZONE = 'UTC'119USE_I18N = True120USE_L10N = True121USE_TZ = True122# Static files (CSS, JavaScript, Images)123# https://docs.djangoproject.com/en/1.10/howto/static-files/124STATIC_URL = '/static/'125MEDIA_ROOT ='media/'...

restrictedopd.py

Source:restrictedopd.py

...11    obj_df.columns = ['id', 'name', 'number', 'occurrences']12    obj_df = obj_df.astype({'number': 'int32'})13    return obj_df14def extract_distributions(data):15    log = section_logger(1)16    log('Extracting distributions ')17    raw_data = data[['image_id', 'name']].reset_index()18    raw_data.columns = ['class', 'image_id', 'name']19    raw_data = raw_data.astype({'image_id': 'int32'})20    frequencies = raw_data.pivot_table(21        index='image_id',22        columns='name',23        values='class',24        aggfunc='count',25        fill_value=0.0)26    frequencies['sum'] = frequencies.sum(1)27    frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \28        frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0)29    img_objs = extract_global_objs(frequencies)30    obj_df = extract_objects(data)31    return img_objs, obj_df32def generate_vgopd_from_restrictedvg(output_path, input_path, top_objects, perc):33    section = section_logger()34    section('Loading RestrictedVG Dataset')35    restricted_definitions = load_dataframe(input_path)36    section('Creating distributions')37    image_df, obj_df = extract_distributions(restricted_definitions)38    section('Saving Raw DataFrame')39    save_raw_data(output_path, obj_df, image_df)40    section('Filtering objects')41    data_df = filter_top_objects(image_df, obj_df, top_objects)42    splits = split_distributions(data_df, perc)43    section('Saving final distribution')44    save_distributions(output_path, splits)45if __name__== "__main__":46    generate_vgopd_from_restrictedvg('/home/dani/Documentos/Proyectos/Doctorado/Datasets/RestrictedOPD/1000C',47                                     '/home/dani/Documentos/Proyectos/Doctorado/Datasets/restrictedGenome', 1000, 0.10)

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.