KaneAI - World’s First E2E Software Testing Agent.Book a Demo 
How to use tags_stats method in avocado
Best Python code snippet using avocado_python
main.py
Source:main.py 
1from typing import List, Set2import debug_load_envs # before import sly3import supervisely as sly4from project_commons import ProjectCommons5from ann_provider import AnnProvider, AnnMemCache, AnnDiskCacheRemovable, AnnDiskCachePersistent6from tags_stats import TagsStatsConstructor, TagsStats, TagMetaChecks7import globals as g8def beware_of_nonexistent_tags(selected_tags: List[str], project: ProjectCommons) -> None:9 project_tag_names = set(t.name for t in project.meta.tag_metas)10 nonexistent_tags = [t for t in selected_tags if t not in project_tag_names]11 if len(nonexistent_tags):12 raise ValueError(f'Tags not found in project: {nonexistent_tags}')13def debug_log_tag_stats(tag_meta: sly.TagMeta, tags_stats: TagsStats) -> None:14 name = tag_meta.name15 tag_checks = TagMetaChecks(tag_meta)16 sly.logger.debug(f'Tag info: {name=} '17 f'{tag_checks.has_appropriate_targets()=} '18 f'{tag_checks.has_appropriate_value_type()=} '19 f'{tags_stats.is_in_use(name)=} '20 f'{tags_stats.has_single_geom_type(name)=} '21 f'{tags_stats.objects_covered_cnt([name])=}')22def tag_is_appropriate(tag_name: str, project: ProjectCommons, tags_stats: TagsStats) -> bool:23 tag_meta = project.meta.tag_metas.get(tag_name)24 tag_checks = TagMetaChecks(tag_meta)25 if not tag_checks.has_appropriate_targets():26 sly.logger.warn(f'Inappropriate tag: wrong targets (like "images_only"). {tag_name=}')27 elif not tag_checks.has_appropriate_value_type():28 sly.logger.warn(f'Inappropriate tag: wrong value type (not None). {tag_name=}')29 elif not tags_stats.is_in_use(tag_name):30 sly.logger.warn(f'Inappropriate tag: not associated with any object. {tag_name=}')31 elif not tags_stats.has_single_geom_type(tag_name):32 raise ValueError(f'Inappropriate tag: associated with objects of different shapes. {tag_name=}')33 else:34 return True35 return False36def ensure_tag_set_is_appropriate(tag_names: List[str], tags_stats: TagsStats) -> None:37 if not tags_stats.have_not_intersected(tag_names):38 example = tags_stats.example_intersected(tag_names)39 raise ValueError(f'Inappropriate tag set: object associated with some tags is found. '40 f'Wrong tags example: {example}')41 class_names_rest = tags_stats.classes_not_covered_entirely(tag_names)42 class_tag_name_inters = set(tag_names).intersection(class_names_rest)43 if class_tag_name_inters:44 raise ValueError(f'Inappropriate tag set: some tag has same name with remaining class. '45 f'Wrong names: {class_tag_name_inters}')46 obj_covered_cnt = tags_stats.objects_covered_cnt(tag_names)47 total_obj_cnt = tags_stats.objects_count48 if class_names_rest:49 sly.logger.warn(f'There are objects in project which are not associated with any of selected tags. '50 f'Covered: {(obj_covered_cnt / total_obj_cnt):.1%}. '51 f'Total number of objects: {total_obj_cnt}. '52 f'Number of object covered by selected tags: {obj_covered_cnt}. '53 f'Remaining classes: {class_names_rest}')54 tags_with_images = tags_stats.tags_associated_with_images(tag_names)55 if tags_with_images:56 sly.logger.warn(f'Some of selected tags are associated with images, those associations will be remained. '57 f'Tag names: {tags_with_images}')58 sly.logger.info(f'Tags for conversion: {tag_names}')59class ProjectMetaConstructor:60 def __init__(self, src_project_meta: sly.ProjectMeta, tags_stats: TagsStats):61 self._src_project_meta = src_project_meta62 self._tags_stats = tags_stats63 def create_new_tags(self, appropriate_tags: List[str]) -> sly.TagMetaCollection:64 src_tag_names = set(t.name for t in self._src_project_meta.tag_metas)65 tag_names_remaining = self._tags_stats.tags_associated_with_images(appropriate_tags)66 tags_names_to_del = set(appropriate_tags) - tag_names_remaining67 res_tag_names = src_tag_names - tags_names_to_del68 res_tags_lst = [t.clone() for t in self._src_project_meta.tag_metas if t.name in res_tag_names]69 res_tags = sly.TagMetaCollection(res_tags_lst)70 return res_tags71 def create_new_classes(self, appropriate_tags: List[str]) -> sly.ObjClassCollection:72 class_names_remaining = self._tags_stats.classes_not_covered_entirely(appropriate_tags)73 res_classes_lst = [self._src_project_meta.obj_classes.get(c).clone() for c in class_names_remaining]74 for tag_name in appropriate_tags:75 src_tag_meta = self._src_project_meta.tag_metas.get(tag_name)76 geom_type = self._tags_stats.geometry_type(tag_name)77 new_cls = sly.ObjClass(name=tag_name,78 geometry_type=geom_type,79 color=src_tag_meta.color,80 hotkey=src_tag_meta.hotkey)81 res_classes_lst.append(new_cls)82 res_classes = sly.ObjClassCollection(res_classes_lst)83 return res_classes84 def create_new_project_meta(self, appropriate_tags: List[str]) -> sly.ProjectMeta:85 res_tags = self.create_new_tags(appropriate_tags)86 res_classes = self.create_new_classes(appropriate_tags)87 res_meta = self._src_project_meta.clone(obj_classes=res_classes, tag_metas=res_tags)88 return res_meta89class AnnConvertor:90 def __init__(self, appropriate_tags: List[str], src_meta: sly.ProjectMeta, res_meta: sly.ProjectMeta):91 self.tags_to_convert = set(appropriate_tags)92 self.src_meta = src_meta93 self.res_meta = res_meta94 def convert(self, ann: sly.Annotation):95 res_labels = self._convert_labels(ann.labels)96 res_img_tags = self._convert_tags(ann.img_tags, tags_to_rm=set())97 res_ann = ann.clone(labels=res_labels, img_tags=res_img_tags)98 return res_ann99 def _convert_tags(self, tags: sly.TagCollection, tags_to_rm: Set[str]):100 return sly.TagCollection([self._convert_tag(t) for t in tags if t.name not in tags_to_rm])101 def _convert_tag(self, tag: sly.Tag):102 new_tag_meta = self.res_meta.tag_metas.get(tag.name)103 return tag.clone(meta=new_tag_meta)104 def _convert_labels(self, labels: List[sly.Label]):105 return [self._convert_label(lbl) for lbl in labels]106 def _convert_label(self, label: sly.Label):107 important_tags = self.tags_to_convert.intersection(t.name for t in label.tags)108 assert len(important_tags) < 2109 if not important_tags:110 new_name = label.obj_class.name111 else:112 new_name = next(iter(important_tags))113 new_cls = self.res_meta.obj_classes.get(new_name)114 new_tags = self._convert_tags(label.tags, tags_to_rm=important_tags)115 return label.clone(obj_class=new_cls, tags=new_tags)116class DatasetShadowCreator:117 def __init__(self, api: sly.Api, res_project_id: int):118 self._api = api119 self._res_project_id = res_project_id120 self._map = {}121 def get_new(self, src_ds_info):122 res_info = self._map.get(src_ds_info.id)123 if res_info is None:124 res_info = self._api.dataset.create(self._res_project_id, src_ds_info.name,125 change_name_if_conflict=True)126 self._map[src_ds_info.id] = res_info127 return res_info128@sly.timeit129def tags_to_classes(api: sly.Api, selected_tags: List[str], result_project_name: str):130 project = ProjectCommons(api, g.project_id)131 if not result_project_name:132 result_project_name = f'{project.info.name} Untagged'133 # Step 0: collect tag associations134 beware_of_nonexistent_tags(selected_tags, project)135 # ann_cache = AnnDiskCachePersistent(g.temp_data_directory) # for debugging purposes136 if len(project) < g.anns_in_memory_limit:137 ann_cache = AnnMemCache()138 else:139 ann_cache = AnnDiskCacheRemovable(g.temp_data_directory)140 sly.logger.debug(f'Ann cache type: {type(ann_cache)}')141 ann_provider = AnnProvider(api, project, ann_cache=ann_cache)142 tags_stats_constructor = TagsStatsConstructor(project.meta)143 progress = sly.Progress('Collecting tags data', len(project), min_report_percent=5)144 for ann in ann_provider.get_anns():145 tags_stats_constructor.update_with_annotation(ann)146 progress.iter_done_report()147 tags_stats = tags_stats_constructor.get_stats()148 sly.logger.info('Tag statistics are collected', extra={149 'total_tags_cnt': len(project.meta.tag_metas),150 'total_objects_cnt': tags_stats.objects_count151 })152 for tag_meta in project.meta.tag_metas:153 debug_log_tag_stats(tag_meta, tags_stats)154 # Step 1: check selected tags155 appropriate_tag_names = [t for t in set(selected_tags) if tag_is_appropriate(t, project, tags_stats)]156 ensure_tag_set_is_appropriate(appropriate_tag_names, tags_stats)157 # Step 2: convert annotations & upload158 meta_constructor = ProjectMetaConstructor(project.meta, tags_stats)159 res_meta = meta_constructor.create_new_project_meta(appropriate_tag_names)160 sly.logger.info(f'Resulting tags: {sorted(t.name for t in res_meta.tag_metas)}')161 sly.logger.info(f'Resulting classes: {sorted(c.name for c in res_meta.obj_classes)}')162 res_project_info = api.project.create(g.workspace_id, result_project_name,163 type=sly.ProjectType.IMAGES, change_name_if_conflict=True)164 api.project.update_meta(res_project_info.id, res_meta.to_json())165 sly.logger.info(f'Resulting project name: {res_project_info.name!r}')166 ann_convertor = AnnConvertor(appropriate_tag_names, src_meta=project.meta, res_meta=res_meta)167 dataset_creator = DatasetShadowCreator(api, res_project_info.id)168 progress = sly.Progress('Converting classes', len(project))169 for ds_info, img_ids, img_hashes, img_names in project.iterate_batched():170 res_ds_info = dataset_creator.get_new(ds_info)171 anns = ann_provider.get_anns_by_img_ids(ds_info.id, img_ids)172 res_anns = [ann_convertor.convert(ann) for ann in anns]173 new_img_infos = api.image.upload_hashes(res_ds_info.id, names=img_names, hashes=img_hashes)174 api.annotation.upload_anns([i.id for i in new_img_infos], res_anns)175 progress.iters_done_report(len(img_ids))176 sly.logger.debug('Finished tags_to_classes')177if __name__ == '__main__':178 sly.logger.info(179 'Script arguments',180 extra={181 'context.teamId': g.team_id,182 'context.workspaceId': g.workspace_id,183 'modal.state.slyProjectId': g.project_id,184 'modal.state.selectedTags.tags': g.selected_tags,185 'modal.state.resultProjectName': g.res_project_name186 },187 )188 tags_to_classes(g.api, g.selected_tags, g.res_project_name)189 try:190 sly.app.fastapi.shutdown()191 except KeyboardInterrupt:...
concept_stats.py
Source:concept_stats.py
1#-----------------------------------------------------------------------2# concept_stats.py3#-----------------------------------------------------------------------4# Author: Brian Nemsick (bnensick3)5# Semester: Fall 20146# Team: Intelligent Review7#-----------------------------------------------------------------------8# Purpose: to calculate static concept statistics in ITS.9# Including:10# 1. Cross Corrolation P(A|B), given tag B is tagged, what is the 11# probability that A is also tagged.12# 2. Tag use in terms of total questions.13#-----------------------------------------------------------------------14import its_query as itsq15from its_definitions import get_category16class concept_stats(object):17 18 its_query_handle = None19 its_table_handle = None20 21 #-------------------------------------------------------------------22 # Index Defines (Constants) for general_stats23 #-------------------------------------------------------------------24 NAME = 0;25 TAGS_ID = 1;26 COUNT = 2;27 MAPPED_ID = 3;28 TOTAL_CORR = 4;29 general_stats = []30 31 #Used to map tags_id to indexes for cross correlation32 #cross_key[0] = tag033 cross_key = []34 35 #total_correlation[0] = total_correlation[cross_key[0]]36 total_correlation = []37 38 #total_questions39 total_questions = []40 41 #-------------------------------------------------------------------42 # Format of cross_correlation:43 #-------------------------------------------------------------------44 # Tag0 | Tag1 | ... | TagN|45 #-------------------------------------------------------------------46 # Tag0 1 | ... | ... |47 #-------------------------------------------------------------------48 # Tag1 ... 49 #-------------------------------------------------------------------50 # Tag2 ... 51 #-------------------------------------------------------------------52 cross_correlation = []53 54 55 def __init__(self):56 57 #Open connection to read from ITS database58 self.its_query_handle = itsq.its_query()59 self.its_query_handle.open_connection()60 61 #Calculate static concept statistics62 self.calc_concept_stats()63 64 #Close connection to read from ITS database65 self.its_query_handle.close_connection()66 67 def calc_concept_stats(self):68 69 #---------------------------------------------------------------70 # Tags Query -> general_stats71 #---------------------------------------------------------------72 # Find tags (actual name), tags_id, total occurrence for ITS73 # assignment 1-7.74 #---------------------------------------------------------------75 questions_id_query = "SELECT id FROM questions WHERE " + get_category(0)76 tags_stats_query = "SELECT tags.name, questions_tags.tags_id, COUNT(*) FROM questions_tags INNER JOIN tags ON tags.id = questions_tags.tags_id WHERE questions_tags.questions_id IN (" + questions_id_query + ") GROUP BY questions_tags.tags_id HAVING COUNT(*) >= 1" 77 tags_stats = self.its_query_handle.exec_its_query(tags_stats_query)78 79 num_tags = len(tags_stats) #Number of active tags80 81 #Insert into data arrays82 for i in range (0,num_tags):83 self.general_stats.append([str(tags_stats[i][self.NAME]),int(tags_stats[i][self.TAGS_ID]),int(tags_stats[i][self.COUNT]),i,0])84 self.cross_key.append(int(tags_stats[i][self.TAGS_ID]))85 self.total_questions.append(int(tags_stats[i][self.COUNT]))86 87 88 #---------------------------------------------------------------89 # Cross Corrolation Query -> cross_correlation90 #---------------------------------------------------------------91 # Find cross corrolation of all tags used by ITS92 # Defined as the P(A|B) - Conditional Probability93 #---------------------------------------------------------------94 for B in range (0, num_tags):95 96 #Calculate once to improve computation speed97 current_tags_id = str(self.general_stats[B][self.TAGS_ID])98 current_tags_total_questions = int(self.general_stats[B][self.COUNT])99 100 #Declare an empty 0's array101 current_cross_corr = [0] * num_tags102 cross_corr_concepts_query = "SELECT tags_id,COUNT(*) FROM questions_tags WHERE (questions_id IN (SELECT questions_id FROM questions_tags WHERE tags_id = " + current_tags_id + " )) AND questions_id IN (SELECT id FROM questions WHERE " + get_category(0) +") GROUP BY tags_id HAVING COUNT(*) >= 1"103 cross_corr_concepts = self.its_query_handle.exec_its_query(cross_corr_concepts_query)104 105 for A in range (0, len(cross_corr_concepts)):106 if (int(cross_corr_concepts[A][0]) != 0):107 ind = self.cross_key.index(int(cross_corr_concepts[A][0]))108 current_cross_corr[ind] = float(float(cross_corr_concepts[A][1])/current_tags_total_questions)109 110 #Add data to class variable111 self.cross_correlation.append(current_cross_corr)112 113 #Total correlation correction for tags with limited questions114 current_total_corr = sum(current_cross_corr)115 116 self.general_stats[B][self.TOTAL_CORR] = current_total_corr 117 self.total_correlation.append(current_total_corr)...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run avocado automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
