KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use get_subdir_list method in autotest

Best Python code snippet using autotest_python

analyze_utilities_v2.py

Source:analyze_utilities_v2.py

...22plt.rc('axes', titlesize=label_size)23plt.rc('axes', labelsize=label_size)24plt.rc('xtick', labelsize=tick_size)25plt.rc('ytick', labelsize=tick_size)26def get_subdir_list(path_to_root_dir: str):27    # search_string = pathlib.Path(path_to_root_dir) / '*'28    # subdir_list = glob.glob(str(search_string), recursive=False)29    p = pathlib.Path(path_to_root_dir)30    subdir_list = [d for d in p.iterdir() if d.is_dir()]31    return subdir_list32def merge_splitted_results_from_subdir(subdir_list: list):33    """34    Extract all adversarials.t from path sub directories, assert that the parameters are the same and unify them35    :param subdir_list: The path to the directory containing subdir that should be unified.36    :return: A unified adversarials object.37    """38    assert(len(subdir_list) > 1)39    params1 = load_dir_params(subdir_list[0])40    # Make sure all subdir contains the same configuration besides the indices:41    for i, subdir in enumerate(subdir_list[1:]):42        params2 = load_dir_params(subdir)43        assert(is_dicts_equal(params1["adv_attack_test"], params2["adv_attack_test"], ["test_start_idx", "test_end_idx"]))44        assert(is_dicts_equal(params1["model"], params2["model"]))45        assert(is_dicts_equal(params1["fit_to_sample"], params2["fit_to_sample"]))46    adv_l = []47    for i, subdir in enumerate(subdir_list):48        adv_l.append(torch.load(os.path.join(subdir, 'adversarials.t')))49    return adv_l[0].cat(adv_l), params150def is_dicts_equal(d1: dict, d2: dict, ignore_keys: Union[None, list] = None):51    if ignore_keys is None:52        ignore_keys = []53    return {k: v for k, v in d1.items() if k not in ignore_keys} == {k: v for k, v in d2.items() if k not in ignore_keys}54def load_dir_params(path: str):55    param_path = pathlib.Path(path) / 'params.json'56    if param_path.exists() is False:57        param_path = '..' / pathlib.Path(path) / 'params.json'58    param_path_l = glob.glob(str(param_path), recursive=False)59    assert len(param_path_l) == 1, "for param_path: {} found the following param files: {}".format(param_path, param_path_l)60    with open(param_path_l[0]) as f:61        params = json.load(f)62    if params["adv_attack_test"]["white_box"] is False:63        path_to_blackbox_dir = os.path.dirname(params["adv_attack_test"]["black_box_adv_path"])64        attack_params = load_dir_params(path_to_blackbox_dir)65        params["adv_attack_test"] = attack_params["adv_attack_test"]66        assert (params["adv_attack_test"]["white_box"])67    return params68def load_exp_result_from_dir(root_dir: str, indices: Union[list, None] = None):69    """70    Load adversarials results and param dict of an experiment. Results are contained in the root dir or inside multiple71     subdir.72    :param root_dir: The path to the root directory containing the results73    :param indices: if not None, include only results of indices74    :return: adversarial object and a param dict75    """76    subdir_list = get_subdir_list(root_dir)77    if(len(subdir_list) > 1):78        adv, params = merge_splitted_results_from_subdir(subdir_list)79    else:80        if len(subdir_list) == 1:81            root_dir = subdir_list[0]82        try:83            adv = torch.load(os.path.join(root_dir, 'adversarials.t'))84        except FileNotFoundError:85            adv = torch.load(os.path.join(root_dir, 'adversarials_compress.t'))86        params = load_dir_params(root_dir)87    if indices is not None:88        adv.correct = adv.correct[indices]89        adv.loss = adv.loss[indices]90    return adv, params91def compress_and_save_adv(subdir_list: list):92    """93    Save compressed adversarial object, without image data.94    :param subdir_list: a list of subdirectories for which the function will save a compressed version of adversarials.t95    """96    for i, subdir in enumerate(subdir_list):97        adv, params = load_exp_result_from_dir(subdir, None)98        if adv.original_sample is not None or adv.adversarial_sample is not None:99            print("Compress adversarial.t in directory: {}".format(subdir))100            adv.original_sample = None101            adv.adversarial_sample = None102            adv.dump(subdir, "adversarials_compress.t")103def results_dirs_to_df(subdir_list: list, indices: Union[list, None] = None) -> pd.DataFrame:104    """105    Load multiple experiments that are stored in subdir_list and save them into a DF.106    :param subdir_list: a list of folders containing the results107    :param indices: if not None, include only results of indices108    :return: A dataframe containing the results.109    """110    assert(len(subdir_list) >= 1)111    statistics_df = pd.DataFrame(columns=['acc', 'epsilon', 'mean loss', 'samples'])112    for i, subdir in enumerate(subdir_list):113        # params = load_dir_params(subdir)114        # if params["adv_attack_test"]["white_box"] is False:115        #     path_to_blackbox_dir = os.path.dirname(params["adv_attack_test"]["black_box_adv_path"])116        #     attack_params = load_dir_params(path_to_blackbox_dir)117        #     params["adv_attack_test"] = attack_params["adv_attack_test"]118        #     assert(params["adv_attack_test"]["white_box"])119        adv, params = load_exp_result_from_dir(subdir, indices)120        statistics_df.loc[i, "acc"] = adv.get_accuracy()121        statistics_df.loc[i, "mean loss"] = adv.get_mean_loss()122        statistics_df.loc[i, "mean size"] = adv.get_mean_loss()123        statistics_df.loc[i, "samples"] = len(adv.correct)124        if params["adv_attack_test"]['attack_type'] == 'natural':125            statistics_df.loc[i, "epsilon"] = 0126            statistics_df.loc[i, "iter"] = 0127            statistics_df.loc[i, "pgd_step"] = 0128            statistics_df.loc[i, "restarts"] = 0129        else:130            statistics_df.loc[i, "epsilon"] = params["adv_attack_test"]["epsilon"]131            statistics_df.loc[i, "iter"] = params["adv_attack_test"]["pgd_iter"]132            statistics_df.loc[i, "pgd_step"] = params["adv_attack_test"]["pgd_step"]133            statistics_df.loc[i, "restarts"] = params["adv_attack_test"]["pgd_test_restart_num"]134        statistics_df.loc[i, "lambda"] = params["fit_to_sample"]["epsilon"]135    return statistics_df136if __name__ == "__main__":137    print("start")138    # All tests were performed on the imagenet evaluation subset which includes all the samples for the first 100 classes (50 samples per each class for a total of 5000 samples).139    #140    # Experiments:141    # 1. pNML Accuracy Vs. Refinement strength (\lambda) - Shows the trade-off between natural and robust accuracy142    # 2. Accuracy Vs. PGD-attack strength (\epsilon); with\ without  pNML - Shows that pNML improve accuracy.143    # 3. Accuracy Vs. Adaptive-attack & PGD-attack strength (\epsilon) - Shows that adaptive attack is not working144    # 4. TODO: blackbox attack Accuracy Vs. Blackbox-attack for different epsilons strength (\epsilon) - Shows that pNML improve accuracy compared to blackbox without pNML145    # PGD attack properties: "pgd_iter": 50, "pgd_step": 0.00392156862, "pgd_test_restart_num": 10146    # For 1. "epsilon": 0.0156862745098 was selected.147    # For 2. lambda = 0.01176 (refinement strength)148    # For 3. "epsilon": [0.0156, 0.0313, 0.0627] was selected.; lambda = 0.01176(refinement strength). indices: 1 per label149    # 1.1. pNML Accuracy Vs. Refinement strength (\lambda)150    natural_acc_vs_lambda_res_path = "./output/imagenet_diff_fix_natural"151    pgd_acc_vs_lambda_res_path = "./output/imagenet_diff_fix_pgd"152    # 1.2.  Accuracy Vs. attack strength (\epsilon); with\ without  pNML153    pgd_nopnml_diff_eps_path = "./output/imagenet_pgd_diff_eps"154    pgd_pnml_diff_eps_path = "./output/imagenet_pgd_diff_eps_pnml_lambda_01176"155    # 1.3 Accuracy Vs. Adaptive-attack & PGD-attack strength (\epsilon)156    adaptive_diff_eps_path = "./output/imagenet_adaptive"157    # 1.1158    subdir_list = get_subdir_list(pgd_acc_vs_lambda_res_path)159    df_pgd = results_dirs_to_df(subdir_list)160    subdir_list = get_subdir_list(natural_acc_vs_lambda_res_path)161    df_natural = results_dirs_to_df(subdir_list)162    df_natural = df_natural.sort_values("lambda", ignore_index=True)163    df_pgd = df_pgd.sort_values("lambda", ignore_index=True)164    axes = plt.gca()165    # axes.set_xlim([0.0, 0.12])166    # axes.set_ylim([0.0, 1.0])167    plt.figure(0)168    plt.plot(df_natural['lambda'], df_natural['acc'], 'go-', label='Natural')169    plt.plot(df_pgd['lambda'], df_pgd['acc'], 'bo-', label='PGD')170    plt.legend(fontsize=12, ncol=1, loc=1)171    plt.xlabel('Refinement strength')172    plt.ylabel('Accuracy')173    plt.savefig(os.path.join(output_path, 'Natural_vs_refinement_strength.jpg'), dpi=200,174                bbox_inches=plt.tight_layout()) if is_save_fig else None175    plt.show()176    # 1.2177    subdir_list = get_subdir_list(pgd_nopnml_diff_eps_path)178    df_pgd_nopnml = results_dirs_to_df(subdir_list)179    subdir_list = get_subdir_list(pgd_pnml_diff_eps_path)180    df_pgd_pnml = results_dirs_to_df(subdir_list)181    df_pgd_nopnml = df_pgd_nopnml.sort_values("epsilon", ignore_index=True)182    df_pgd_pnml = df_pgd_pnml.sort_values("epsilon", ignore_index=True)183    axes = plt.gca()184    # axes.set_xlim([0.0, 0.12])185    # axes.set_ylim([0.0, 1.0])186    plt.figure(1)187    plt.plot(df_pgd_nopnml['epsilon'], df_pgd_nopnml['acc'], 'go--', label='no pNML')188    plt.plot(df_pgd_pnml['epsilon'], df_pgd_pnml['acc'], 'bo--', label='pNML')189    plt.legend(fontsize=12, ncol=1, loc=1)190    plt.xlabel('Attack strength')191    plt.ylabel('Accuracy')192    plt.savefig(os.path.join(output_path, 'pgd_pnml_and_nopnml.jpg'), dpi=200,193                bbox_inches=plt.tight_layout()) if is_save_fig else None194    plt.show()195    # 1.3196    indices = [i for i in range(0, 4999+1, 50)]197    subdir_list = get_subdir_list(adaptive_diff_eps_path)198    df_adaptive = results_dirs_to_df(subdir_list)199    df_adaptive = df_adaptive.sort_values("epsilon", ignore_index=True)200    subdir_list = get_subdir_list(pgd_pnml_diff_eps_path)201    df_pgd_pnml = results_dirs_to_df(subdir_list, indices)202    axes = plt.gca()203    plt.figure(2)204    plt.plot(df_adaptive['epsilon'], df_adaptive['acc'], 'go--', label='Adaptive')205    plt.plot(df_pgd_pnml['epsilon'], df_pgd_pnml['acc'], 'bo--', label='PGD')206    plt.legend(fontsize=12, ncol=1, loc=1)207    plt.xlabel('Attack strength')208    plt.ylabel('Accuracy')209    plt.savefig(os.path.join(output_path, 'adaptive_vs_pgd.jpg'), dpi=200,210                bbox_inches=plt.tight_layout()) if is_save_fig else None...

genqrc.py

Source:genqrc.py

...21    22    return '\n'.join(items)23    24def get_qrc_xml(topdir):25    subdirs = get_subdir_list(topdir)26    qresources = ["<RCC>"]27    for _dir in subdirs:28        qresources.append(get_qresource_xml_bydir(topdir, _dir))29    qresources.append("</RCC>")30    31    return '\n'.join(qresources)32def main(topdir, qrcfile):33    rcc_xml = get_qrc_xml(topdir)34    open("tmp.qrc", 'w').write(rcc_xml)35    return system("pyrcc4 -py3 tmp.qrc > %s" % qrcfile)36    37if __name__ == "__main__":38    argc = len(sys.argv)39    if 1 < argc < 3:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.