KaneAI - World’s First E2E Software Testing Agent.Book a Demo 
How to use config_files method in toolium
Best Python code snippet using toolium_python
eval_sweeps.py
Source:eval_sweeps.py 
1import os2import numpy as np3import sys4import argparse5sys.path.append(os.path.abspath('../../'))6from sweeper import Sweeper7def extract_line(lines, max_steps, interval=0):8 """9 Retrieves num_steps at which episodes were completed10 and returns an array where the value at index i represents11 the number of episodes completed until step i12 """13 step = 014 # rewards_over_time = np.zeros(max_steps//interval+1) # for GANModel training15 rewards_over_time = np.zeros(max_steps // interval + 1) # for DQN training16 try:17 for line in lines:18 if 'total steps' not in line:19 continue20 num_steps = float(line.split("|")[1].split(",")[0].split(" ")[-1])21 if num_steps > max_steps:22 break23 # reward = float(line.split("|")[1].split(",")[1].split(" ")[2]) # for GANModel training24 reward = float(line.split("|")[1].split(",")[2].split("/")[0].split(" ")[-1]) # for DQN training25 rewards_over_time[step] = reward26 step += 127 return rewards_over_time28 except:29 print(line)30 print('step:{}'.format(step))31 raise32def get_max_steps(lines):33 for line in lines[::-1]:34 if 'total steps' in line:35 max_steps = float(line.split("|")[1].split(",")[0].split(" ")[-1])36 return max_steps37 return -138def _eval_lines(config_file, last_points, start_idx, end_idx, max_steps, interval=10000):39 print('config_files: {}'.format(config_file))40 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))41 sweeper = Sweeper(os.path.join(project_root, config_file))42 eval = []43 eval_lines = []44 for k in range(sweeper.total_combinations):45 eval.append([])46 eval_lines.append([])47 for idx in range(start_idx, end_idx):48 cfg = sweeper.parse(idx)49 cfg.data_root = os.path.join(project_root, 'data', 'output')50 log_dir = cfg.get_log_dir()51 log_path = os.path.join(log_dir, 'log')52 try:53 with open(log_path, "r") as f:54 lines = f.readlines()55 except FileNotFoundError:56 continue57 if len(lines) == 0:58 continue59 # ugly parse based on the log_file format60 try:61 num_steps = get_max_steps(lines)62 if num_steps >= max_steps:63 assert idx % sweeper.total_combinations == cfg.param_setting64 avg_eval_steps = extract_line(lines, max_steps, interval=interval)65 # eval[id % sweeper.total_combinations].append(np.mean(avg_eval_steps[-int(len(avg_eval_steps)):]))66 eval[idx % sweeper.total_combinations].append(np.mean(avg_eval_steps[-last_points:]))67 # eval[id % sweeper.total_combinations].append(np.mean(avg_eval_steps))68 except IndexError:69 print(idx)70 raise71 summary = list(map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])), enumerate(eval)))72 summary = [x for x in summary if np.isnan(x[1]) == False]73 # new_summary = []74 # for s in summary:75 # if np.isnan(s[1]) == False:76 # new_summary.append(s)77 # print(summary[0])78 # print(new_summary[0])79 # quit()80 summary = sorted(summary, key=lambda s: s[1], reverse=True)81 for idx, mean, std, num_runs in summary:82 print("Param Setting # {:>3d} | Rewards: {:>10.10f} +/- {:>5.2f} ({:>2d} runs) {} | ".format(83 idx, mean, std, num_runs, sweeper.param_setting_from_id(idx)))84if __name__ == '__main__':85 parser = argparse.ArgumentParser(description="run_file")86 parser.add_argument('--cfg-file', default='default.json')87 parser.add_argument('--start', default=0, type=int)88 parser.add_argument('--end', default=72, type=int)89 parser.add_argument('--max-steps', default=50000, type=int)90 parser.add_argument('--interval', default=10, type=int)91 args = parser.parse_args()92 #############93 # Navigation94 # print("dqn_navigation")95 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_c.json', last_points=15, start_idx=0, end_idx=40, max_steps=300000, interval=10000)96 # # print("zach_navigation")97 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_navigation.json', last_points=15, start_idx=0, end_idx=200, max_steps=300000, interval=10000)98 # print("mve_avg_navigation")99 # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_navigation.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)100 # print("mve_avg_navigation_learnedv2")101 # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/navigation.json', last_points=15, start_idx=0, end_idx=60, max_steps=300000, interval=10000)102 # print("search_dt_navigation")103 # _eval_lines(config_files='experiment/archive/search_dt_perfect/navigation.json', last_points=15, start_idx=0, end_idx=80, max_steps=300000, interval=10000)104 #105 # print("search_dt_navigation_learnedv2")106 # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/navigation.json', last_points=15, start_idx=0, end_idx=80, max_steps=200000, interval=10000)107 # print("mve_sw_model")108 # _eval_lines(config_files='experiment/archive/mve_sw_model/navigation.json', last_points=15, start_idx=0, end_idx=240, max_steps=300000, interval=10000)109 # print("mve_learned/navigation")110 # _eval_lines(config_files='experiment/config_files_2/mve_learned/navigation.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)111 # #############112 #############113 # Cartpole114 #115 # print("dqn_cartpole")116 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)117 # print("mve_cartpole")118 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)119 # print("mve_avg_cartpole")120 # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)121 #122 # print("zach_cartpole")123 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole.json', last_points=12, start_idx=0, end_idx=200, max_steps=50000, interval=2000)124 # print("mve_cartpole_learned")125 # _eval_lines(config_files='experiment/archive/mve_avg_learned/cartpole.json', last_points=12, start_idx=0, end_idx=80, max_steps=50000, interval=2000)126 #127 # print("mve_sw_model")128 # _eval_lines(config_files='experiment/archive/mve_sw_model/cartpole.json', last_points=12, start_idx=0, end_idx=240, max_steps=50000, interval=2000)129 # print("dqn_cartpole")130 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole_b.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)131 #132 # print("zach_cartpole_b")133 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_b.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)134 #135 # print("zach_cartpole_c")136 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_c.json', last_points=12, start_idx=0, end_idx=200, max_steps=50000, interval=2000)137 # print("zach_cartpole_d")138 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_cartpole_d.json', last_points=12, start_idx=0,139 # end_idx=200, max_steps=50000, interval=2000)140 # #141 # print("search_dt_perfect")142 # _eval_lines(config_files='experiment/archive/search_dt_perfect/cartpole.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)143 # print("mve_avg_learnedv2")144 # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/cartpole.json', last_points=12, start_idx=0, end_idx=90, max_steps=50000, interval=2000)145 # print("search_dt_learnedv2")146 # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/cartpole.json', last_points=12, start_idx=0, end_idx=120, max_steps=50000, interval=2000)147 # print("mve_learned/cartpole")148 # _eval_lines(config_files='experiment/config_files_2/mve_learned/cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)149 # ############150 #############151 # Catcher Raw152 # print("dqn_catcher_raw")153 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_raw.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)154 #155 # print("dqn_catcher_raw_VF_sw")156 # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw.json', last_points=20, start_idx=0, end_idx=270, max_steps=200000, interval=5000)157 # print("dqn_catcher_raw_VF_sw_huber")158 # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw_huber.json', last_points=5, start_idx=0, end_idx=270, max_steps=200000, interval=5000)159 # print("dqn_catcher_raw_architecture_sweep")160 # _eval_lines(config_files='experiment/archive/model_free_v4/catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)161 #162 # print("dqn_catcher_raw_architecture_sweep_smaller_lr")163 # _eval_lines(config_files='experiment/archive/model_free_v4/catcher_raw_smaller_lr.json', last_points=20, start_idx=0, end_idx=150, max_steps=200000, interval=5000)164 # print("zach_catcher_raw")165 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)166 # print("zach_catcher_raw_b")167 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_raw_b.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)168 #169 # print("mve_avg_catcher_raw")170 # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)171 # print("mve_avg_catcher_learned")172 # _eval_lines(config_files='experiment/archive/mve_avg_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)173 #174 # print("mve_avg_catcher_learned_model_sw")175 # _eval_lines(config_files='experiment/archive/mve_sw_model/catcher_raw.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)176 # print("mve_avg_catcher_learned_error_aware")177 # _eval_lines(config_files='experiment/archive/mve_error_aware/catcher_raw.json', last_points=20, start_idx=0, end_idx=120, max_steps=200000, interval=5000)178 # print("mve_avg_catcher_learned_model_sweeps")179 # _eval_lines(config_files='experiment/archive/mve_sw_model/catcher_raw.json', last_points=20, start_idx=0, end_idx=240, max_steps=200000, interval=5000)180 # print("search_dt_perfect")181 # _eval_lines(config_files='experiment/archive/search_dt_perfect/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)182 # print("search_dt_learned")183 # _eval_lines(config_files='experiment/archive/search_dt_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)184 #185 # print("search_dt_learnedv2")186 # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)187 # print("mve_avg_catcher_learnedv2")188 # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/catcher_raw.json', last_points=20, start_idx=0, end_idx=80, max_steps=200000, interval=5000)189 #190 # print("mve_avg_catcher_learned_error_aware")191 # _eval_lines(config_files='experiment/archive/mve_error_awarev2/catcher_raw.json', last_points=20, start_idx=0, end_idx=120, max_steps=200000, interval=5000)192 # print("mve_learned")193 # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)194 # # print("mse_learned_mse")195 # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_mse.json', last_points=20, start_idx=0,196 # end_idx=200, max_steps=200000, interval=5000)197 # print("search_learned")198 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)199 # # print("search_learned_mse")200 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_mse.json', last_points=20, start_idx=0, end_idx=200, max_steps=200000, interval=5000)201 # print("mve_learned_v2_mse")202 # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v2.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)203 #204 # print("mve_learned_v3_huber")205 # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v3.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)206 #207 # print("mve_learned_v4_mse_highlr")208 # _eval_lines(config_files='experiment/config_files_2/mve_learned/catcher_raw_v4.json', last_points=20, start_idx=0, end_idx=90, max_steps=200000, interval=5000)209 #210 # print("search_learned_v2")211 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v2.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)212 #213 # print("search_learned_v3_huber")214 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v3.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)215 #216 # print("search_learned_v4_mse_highlr")217 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v4.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)218 # print("search_learned_v4_mse_huber")219 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_v5.json', last_points=20, start_idx=0, end_idx=180, max_steps=200000, interval=5000)220 # print("capacity_catcher_raw")221 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity_catcher_raw.json', last_points=20, start_idx=0, end_idx=1540, max_steps=200000, interval=5000)222 # print("dqn_catcher_raw_VF_sw_huber")223 # _eval_lines(config_files='experiment/config_files_2/model_free/catcher_raw_huber.json', last_points=20, start_idx=0, end_idx=270, max_steps=200000, interval=5000)224 # print("capacity_catcher_raw_8")225 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_8.json', last_points=20, start_idx=0, end_idx=20, max_steps=200000, interval=5000)226 # print("capacity_catcher_raw_16")227 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_16.json', last_points=20, start_idx=0, end_idx=30, max_steps=200000, interval=5000)228 # print("capacity_catcher_raw_32")229 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_32.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)230 # print("capacity_catcher_raw_64")231 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_64.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)232 # print("capacity_catcher_raw_128")233 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_128.json', last_points=20, start_idx=0, end_idx=60, max_steps=200000, interval=5000)234 # print("capacity_catcher_raw_256")235 # _eval_lines(config_files='experiment/config_files_2/search_learned/capacity/catcher_raw_256.json', last_points=20, start_idx=0, end_idx=70, max_steps=200000, interval=5000)236 # print("search_learned")237 # _eval_lines(config_files='experiment/config_files_2/search_learned/catcher_raw_huber.json', last_points=20, start_idx=0, end_idx=300, max_steps=200000, interval=5000)238 # ###########239 ############240 # Acrobot241 # print("dqn_acrobot")242 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_test.json', last_points=25, start_idx=0, end_idx=40, max_steps=100000, interval=2000)243 # print("mve_acrobot")244 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)245 #246 # print("mve_avg_acrobot")247 # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)248 # #249 # print("mve_avg_acrobot_learnedv2")250 # _eval_lines(config_files='experiment/archive/mve_avg_learnedv2/acrobot.json', last_points=25, start_idx=0, end_idx=60, max_steps=100000, interval=2000)251 # #252 # print("mve_sw_model")253 # _eval_lines(config_files='experiment/archive/mve_sw_model/acrobot.json', last_points=25, start_idx=0, end_idx=240, max_steps=100000, interval=2000)254 #255 # print("zach_acrobot")256 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_acrobot.json', last_points=25, start_idx=0, end_idx=200, max_steps=100000, interval=2000)257 #258 # print("search_dt_acrobot")259 # _eval_lines(config_files='experiment/archive/search_dt_perfect/acrobot.json', last_points=25, start_idx=0, end_idx=80, max_steps=100000, interval=2000)260 #261 # print("search_dt_acrobot_learnedv2")262 # _eval_lines(config_files='experiment/archive/search_dt_learnedv2/acrobot.json', last_points=25, start_idx=0, end_idx=80, max_steps=100000, interval=2000)263 # print("mve_learned/acrobot")264 # _eval_lines(config_files='experiment/config_files_2/mve_learned/acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)265 print("search_learned_fixed/acrobot")266 _eval_lines(config_file='experiment/config_files_2/search_learned_fixed/acrobot.json', last_points=25, start_idx=0, end_idx=500, max_steps=100000, interval=2000)267 print("search_learned_fixed/acrobot exponential")268 _eval_lines(config_file='experiment/config_files_2/search_learned_fixed/acrobot_exponential.json', last_points=25, start_idx=0, end_idx=500, max_steps=100000, interval=2000)269 # # ############270 ############271 # Catcher visual272 # print("dqn_catacher_visual")273 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', last_points=25, start_idx=0, end_idx=45, max_steps=500000, interval=10000)274 # print("mve_catcher_visual")275 # _eval_lines(config_files='experiment/archive/mve_avg_perfect/mve_catcher_visual.json', last_points=25, start_idx=0, end_idx=120, max_steps=500000, interval=10000)276 # print("zach_catcher_visual")277 # _eval_lines(config_files='experiment/archive/zach_perfect_2/zach_catcher_visual_b.json', last_points=25, start_idx=0, end_idx=150, max_steps=500000, interval=10000)278 # ############279 ## model_free280 # print("dqn_catacher_raw")281 # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_raw.json', start_idx=0, end_idx=120, max_steps=400000, interval=10000)282 # print("dqn_catacher_raw_2h")283 # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_raw_2h.json', start_idx=0, end_idx=60, max_steps=400000, interval=10000)284 # print("dqn_catacher_visual_gray")285 # _eval_lines(config_files='experiment/archive/model_free/dqn_catcher_visual_gray.json', start_idx=0, end_idx=24, max_steps=200000, interval=10000)286 ## model_free_v2287 # print("dqn_catacher_raw")288 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_raw.json', last_points=20, start_idx=0, end_idx=40, max_steps=200000, interval=5000)289 # print("dqn_acrobot")290 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_test.json', last_points=10, start_idx=0, end_idx=359, max_steps=200000, interval=10000)291 # print("dqn_catacher_lite")292 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_lite.json', start_idx=0, end_idx=60, max_steps=400000, interval=10000)293 # print("dqn_catacher_lite_b")294 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_lite_b.json', start_idx=0, end_idx=80, max_steps=600000, interval=10000)295 # print("dqn_catacher_visual")296 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', start_idx=0, end_idx=45, max_steps=500000, interval=10000)297 # print("dqn_acrobot")298 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_test.json', start_idx=0, end_idx=360, max_steps=200000, interval=10000)299 # print("mve_perfect_catcher_raw")300 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_catcher_raw.json', last_points=20, start_idx=0, end_idx=160, max_steps=200000, interval=5000)301 # print("dqn_acrobot")302 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_test.json', last_points=25, start_idx=0, end_idx=40, max_steps=100000, interval=2000)303 #304 # print("mve_acrobot")305 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_acrobot.json', last_points=25, start_idx=0, end_idx=160, max_steps=100000, interval=2000)306 # print("dqn_navigation")307 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_b.json', last_points=10, start_idx=0, end_idx=40, max_steps=200000, interval=10000)308 #309 # print("mve_navigation")310 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_navigation.json', last_points=10, start_idx=0, end_idx=160, max_steps=200000, interval=10000)311 # print("dqn_cartpole")312 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_cartpole.json', last_points=12, start_idx=0, end_idx=40, max_steps=50000, interval=2000)313 # #314 # print("mve_cartpole")315 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_cartpole.json', last_points=12, start_idx=0, end_idx=160, max_steps=50000, interval=2000)316 # print("dqn_navigation_b")317 # _eval_lines(config_files='experiment/archive/model_free_v3/dqn_navigation_c.json', last_points=15, start_idx=0, end_idx=40, max_steps=300000, interval=10000)318 #319 # print("mve_navigation_b")320 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_navigation_c.json', last_points=15, start_idx=0, end_idx=160, max_steps=300000, interval=10000)321 # print("zach_navigation")322 # _eval_lines(config_files='experiment/archive/zach_perfect/zach_navigation.json', last_points=10, start_idx=0, end_idx=160, max_steps=200000, interval=10000)323 # print("dqn_catacher_visual")324 # _eval_lines(config_files='experiment/archive/model_free_v2/dqn_catcher_visual.json', last_points=25, start_idx=0, end_idx=45, max_steps=500000, interval=10000)325 #326 # print("mve_catacher_visual")327 # _eval_lines(config_files='experiment/archive/mve_perfect/mve_catcher_visual.json', last_points=25,328 # start_idx=0, end_idx=60, max_steps=500000, interval=10000)329 # TRACKING330 # Catcher Raw331 # print("sprint_3/catcher_raw/horizon_avg")332 # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_avg.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)333 #334 #335 # print("sprint_3/catcher_raw/horizon_leaf")336 # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_leaf.json', last_points=20, start_idx=0, end_idx=50, max_steps=200000, interval=5000)337 #338 #339 # print("sprint_3/catcher_raw/horizon_avg_tune_vf_lr")340 # _eval_lines(config_files='experiment/config_files_3/search_learned/catcher_raw/horizon_avg_tune_vf_lr.json', last_points=20, start_idx=0, end_idx=360, max_steps=200000, interval=5000)341 #342 #343 #344 #345 #...
test_config.py
Source:test_config.py
1"""Do an integration test for config file usage."""23import os4import subprocess56HTML_TEST_FILES = os.path.abspath(os.path.dirname(__file__))789def test_config_valid():10 """Config test for valid HTML"""11 assert subprocess.call([12 'html5validator',13 f'--config={HTML_TEST_FILES}/config_files/valid.yaml']) == 0141516def test_config_invalid():17 """Config test for invalid HTML"""18 assert subprocess.call([19 'html5validator',20 f'--config={HTML_TEST_FILES}/config_files/invalid.yaml']) == 1212223def test_config_skip():24 """Config test for skipping files"""25 assert subprocess.call([26 'html5validator',27 f'--config={HTML_TEST_FILES}/config_files/skip.yaml']) == 2282930def test_config_invalid_with_css():31 """Config test for CSS and HTML"""32 assert subprocess.call([33 'html5validator',34 f'--config={HTML_TEST_FILES}/config_files/invalid_css.yaml'35 ]) == 3363738def test_config_invalid_css_only():39 """Config test for CSSS only"""40 assert subprocess.call([41 'html5validator',42 f"--config={HTML_TEST_FILES}/config_files/invalid_css_only.yaml"]) == 2434445def test_config_invalid_single_file():46 """Config test for invalid single file"""47 assert subprocess.call([48 'html5validator',49 f"--config={HTML_TEST_FILES}/config_files/invalid_single_file.yaml"50 ]) == 1515253def test_config_warning():54 """Config test for warnings"""55 assert subprocess.call([56 'html5validator',57 f"--config={HTML_TEST_FILES}/config_files/warning.yaml"]) == 1585960def test_config_warning_but_pass():61 """Config test for allowed warnings"""62 assert subprocess.call([63 'html5validator',64 f'--config={HTML_TEST_FILES}/config_files/warning_pass.yaml']) == 0656667def test_config_return_value():68 """Config test for error code return value"""69 assert subprocess.call([70 'html5validator',71 f'--config={HTML_TEST_FILES}/config_files/return_254.yaml']) == 25472 assert subprocess.call([73 'html5validator',74 f'--config={HTML_TEST_FILES}/config_files/return_255.yaml']) == 25575 assert subprocess.call([76 'html5validator',77 f'--config={HTML_TEST_FILES}/config_files/return_255_256.yaml']) == 255787980def test_config_angularjs():81 """Config test for angularjs"""82 assert subprocess.call([83 'html5validator',84 f'--config={HTML_TEST_FILES}/config_files/angularjs.yaml']) == 0858687def test_config_angularjs_no_output_with_ignore():88 """Make sure there is no spurious output when messages are ignored."""89 assert subprocess.check_output([90 'html5validator',91 f'--config={HTML_TEST_FILES}/config_files/angularjs.yaml']) == b''929394def test_config_angularjs_normal_quotes():95 """Config test for normal angularjs"""96 assert subprocess.call([97 'html5validator',98 f'--config={HTML_TEST_FILES}/config_files/angularjs_normal.yaml']) == 099100101def test_config_multiple_ignoreres():102 """Config test for multiple regex ignores"""103 assert subprocess.call([104 'html5validator',105 f'--config={HTML_TEST_FILES}/config_files/multiple_ignores.yaml']) == 0106107108def test_config_ignore_and_ignorere():109 """Config test for ignore and regex ignore"""110 assert subprocess.call([111 'html5validator',112 f'--config={HTML_TEST_FILES}/config_files/ignore_and_ignorere.yaml'113 ]) == 0114115116def test_config_stack_size():117 """Config test for stack size"""118 assert subprocess.call([119 'html5validator',120 f'--config={HTML_TEST_FILES}/config_files/stack_size.yaml']) == 0121122123def test_config_valid_format_flags():124 """Config test for output format for valid files"""125 assert subprocess.call([126 'html5validator',127 f'--config={HTML_TEST_FILES}/config_files/format_flags/text_valid.yaml'128 ]) == 0129 assert subprocess.call([130 'html5validator',131 f'--config={HTML_TEST_FILES}/config_files/format_flags/gnu_valid.yaml'132 ]) == 0133 assert subprocess.call([134 'html5validator',135 f'--config={HTML_TEST_FILES}/config_files/format_flags/json_valid.yaml'136 ]) == 0137 assert subprocess.call([138 'html5validator',139 f'--config={HTML_TEST_FILES}/config_files/format_flags/xml_valid.yaml'140 ]) == 0141142143def test_config_invalid_format_flags():144 """Config test for output format with invalid files"""145 assert subprocess.call([146 'html5validator',147 f'--config={HTML_TEST_FILES}/config_files/format_flags/'148 'text_invalid.yaml'149 ]) == 3150 assert subprocess.call([151 'html5validator',152 f'--config={HTML_TEST_FILES}/config_files/format_flags/'153 'gnu_invalid.yaml'154 ]) == 1155 assert subprocess.call([156 'html5validator',157 f'--config={HTML_TEST_FILES}/config_files/format_flags/'158 'json_invalid.yaml'159 ]) == 1160 assert subprocess.call([161 'html5validator',162 f'--config={HTML_TEST_FILES}/config_files/format_flags/'163 'xml_invalid.yaml'164 ]) == 8165166167def test_config_log_file():168 """Config test for log file"""169 assert subprocess.call([170 'html5validator',171 f'--config={HTML_TEST_FILES}/config_files/log_file.yaml']) == 0172173174def test_config_extra():175 """Config test for vnu extra arguments"""176 assert subprocess.call([177 'html5validator',178 f'--config={HTML_TEST_FILES}/config_files/extra.yaml']) == 0179180181def test_config_nofiles():182 """Command line test for missing files"""183 assert subprocess.call([184 "html5validator",185 f'--config={HTML_TEST_FILES}/config_files/no_files.yaml'186 ]) == 1187188189if __name__ == '__main__':190 test_config_valid()191 test_config_invalid()192 test_config_return_value()193 test_config_angularjs()194 test_config_multiple_ignoreres()195 test_config_ignore_and_ignorere()196 test_config_stack_size()197 test_config_valid_format_flags()198 test_config_invalid_format_flags()199 test_config_log_file()
...
configtest.py
Source:configtest.py
...39 'test-python-configs',40 'summary',41 ],42 require_config_file=require_config_file)43 def query_config_files(self):44 """This query method, much like others, caches its runtime45 settings in self.VAR so we don't have to figure out config_files46 multiple times.47 """48 if self.config_files:49 return self.config_files50 c = self.config51 if 'test_files' in c:52 self.config_files = c['test_files']53 return self.config_files54 self.debug("No --test-file(s) specified; defaulting to crawling the configs/ directory.")55 config_files = []56 for root, dirs, files in os.walk(os.path.join(sys.path[0], "..",57 "configs")):58 for name in files:59 # Hardcode =P60 if name.endswith(".json") or name.endswith(".py"):61 if not name.startswith("test_malformed"):62 config_files.append(os.path.join(root, name))63 self.config_files = config_files64 return self.config_files65 def list_config_files(self):66 """ Non-default action that is mainly here to demonstrate how67 non-default actions work in a mozharness script.68 """69 config_files = self.query_config_files()70 for config_file in config_files:71 self.info(config_file)72 def test_json_configs(self):73 """ Currently only "is this well-formed json?"74 """75 config_files = self.query_config_files()76 filecount = [0, 0]77 for config_file in config_files:78 if config_file.endswith(".json"):79 filecount[0] += 180 self.info("Testing %s." % config_file)81 contents = self.read_from_file(config_file, verbose=False)82 try:83 json.loads(contents)84 except ValueError:85 self.add_summary("%s is invalid json." % config_file,86 level="error")87 self.error(pprint.pformat(sys.exc_info()[1]))88 else:89 self.info("Good.")90 filecount[1] += 191 if filecount[0]:92 self.add_summary("%d of %d json config files were good." %93 (filecount[1], filecount[0]))94 else:95 self.add_summary("No json config files to test.")96 def test_python_configs(self):97 """Currently only "will this give me a config dictionary?"98 """99 config_files = self.query_config_files()100 filecount = [0, 0]101 for config_file in config_files:102 if config_file.endswith(".py"):103 filecount[0] += 1104 self.info("Testing %s." % config_file)105 global_dict = {}106 local_dict = {}107 try:108 execfile(config_file, global_dict, local_dict)109 except:110 self.add_summary("%s is invalid python." % config_file,111 level="error")112 self.error(pprint.pformat(sys.exc_info()[1]))113 else:...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run toolium automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
