Best Python code snippet using behave
rebuilder.py
Source:rebuilder.py
...25 self.op_graph = kwargs['graph']26 self.rebuild_ops = kwargs['rebuild_ops']27 28 @abstractmethod29 def rebuild(self, configs=None):30 pass31 32 '''33 @abstractmethod34 def __get_rebuild_config(self, index, op_dict):35 pass36 37 def get_down_ops(self, op):38 down_ops = []39 for aop in self.op_graph.get_ops():40 if (aop.get_input_num() > 0):# and (op.get_name() in [bop.get_name for bop in get_input_ops_list(aop)]):41 aop_input_names = [bop.get_name() for bop in get_input_ops_list(aop)]42 if op.get_name() in aop_input_names:43 down_ops.append(aop)44 return down_ops45 '''46class ReluRebuilder(BaseRebuilder):47 def __init__(self, *args, **kwargs):48 super().__init__(*args, **kwargs)49 50 def rebuild(self, graph, configs=None):51 if not isinstance(configs, dict):52 raise ValueError('Relu op rebuild configs should be dict')53 54 for relu_op in self.rebuild_ops:55 #input_ops = [node for node in self.op_graph.get_op(relu_op).get_input_ops()]56 input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(relu_op))]57 relu_rebuild_config = self.__get_rebuild_config(relu_op)58 59 #if relu_rebuild_config['relu_up_op'].get_type() != 'fix':60 input_ops[0].set_attr('fuse_relu', True)61 for down_op in relu_rebuild_config['relu_down_ops']:62 down_op.replace_input_ops(relu_rebuild_config['relu_op'], input_ops[0])63 graph.remove_op(relu_rebuild_config['relu_op'])64 65 def __get_rebuild_config(self, index):66 rebuild_config = {}67 rebuild_config['relu_op'] = self.op_graph.get_op(index)68 #rebuild_config['relu_up_op'] = self.op_graph.get_op(input_ops[0])69 70 relu_down_ops = get_down_ops(rebuild_config['relu_op'], self.op_graph)71 rebuild_config['relu_down_ops'] = relu_down_ops72 73 return rebuild_config74class FixRebuilder(BaseRebuilder):75 def __init__(self, *args, **kwargs):76 super().__init__(*args, **kwargs)77 78 def rebuild(self, graph, configs=None):79 if not isinstance(configs, dict):80 raise ValueError('Fix op rebuild configs should be dict')81 82 #print(self.rebuild_ops)83 for fix_op in self.rebuild_ops:84 #print(fix_op)85 #input_ops = [node for node in self.op_graph.get_op(fix_op).get_input_ops()]86 input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(fix_op))]87 configs[input_ops[0].get_name()] = self.op_graph.get_op(fix_op).get_attrs()88 fix_rebuild_config = self.__get_rebuild_config(fix_op)89 90 for down_op in fix_rebuild_config['fix_down_ops']:91 down_op.replace_input_ops(fix_rebuild_config['fix_op'], input_ops[0])92 graph.remove_op(fix_rebuild_config['fix_op'])93 94 def __get_rebuild_config(self, index):95 rebuild_config = {}96 rebuild_config['fix_op'] = self.op_graph.get_op(index)97 #rebuild_config['fix_up_op'] = self.op_graph.get_op(op_dict[index][0])98 99 fix_down_ops = get_down_ops(rebuild_config['fix_op'], self.op_graph)100 rebuild_config['fix_down_ops'] = fix_down_ops101 102 return rebuild_config103class MatmulRebuilder(BaseRebuilder):104 def __init__(self, *args, **kwargs):105 super().__init__(*args, **kwargs)106 107 def rebuild(self, graph, configs=None):108 if not isinstance(configs, dict):109 raise ValueError('Matmul op rebuild configs should be dict')110 for matmul_op in self.rebuild_ops:111 if not self.__should_rebuild(matmul_op):112 continue113 #input_ops = [node for node in self.op_graph.get_op(matmul_op).get_input_ops()]114 input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(matmul_op))]115 matmul_rebuild_config = self.__get_rebuild_config(matmul_op, input_ops)116 self.__rebuild_one_op(graph, matmul_rebuild_config, configs)117 118 def __should_rebuild(self, op):119 #if len(self.rebuild_ops[op]) == 3:120 if self.op_graph.get_op(op).get_input_num() == 3:121 return True122 else:123 return False124 125 def __get_rebuild_config(self, index, input_ops):126 rebuild_config = {}127 rebuild_config['matmul_op'] = self.op_graph.get_op(index)128 129 for node in input_ops:130 #node = self.op_graph.get_op(name)131 '''132 if node.get_type() == 'data':133 rebuild_config['input'] = node134 '''135 if (node.get_type() == 'const') and (node.get_output_tensor().ndim == 2):136 rebuild_config['weights'] = node137 elif (node.get_type() == 'const') and (node.get_output_tensor().ndim == 1):138 rebuild_config['bias'] = node139 else:140 rebuild_config['input'] = node141 142 matmul_down_ops = get_down_ops(rebuild_config['matmul_op'], self.op_graph)143 rebuild_config['matmul_down_ops'] = matmul_down_ops144 return rebuild_config145 146 def __rebuild_one_op(self, graph, rebuild_configs, fix_configs):147 matmul_op = rebuild_configs['matmul_op']148 149 wx_xt_ops: Dict[str, List[Op]] = {}150 wx_xt_ops['input'] = [rebuild_configs['input']]151 wx_xt_ops['input'].append(rebuild_configs['weights'])152 153 attrs: Dict[str, Any] = {}154 if matmul_op.get_type() == 'matmul':155 attrs['transpose_a'] = matmul_op.get_attr('transpose_a')156 attrs['transpose_b'] = matmul_op.get_attr('transpose_b')157 else:158 attrs['transpose_a'] = False159 attrs['transpose_b'] = True160 161 wx_xt_name = matmul_op.get_name() + '_matmul'162 wx_xt_mmul = graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)163 fix_configs[wx_xt_name] = fix_configs[matmul_op.get_name()]164 165 add_ops: Dict[str, List[Op]] = {}166 add_ops['input'] = [wx_xt_mmul]167 add_ops['input'].append(rebuild_configs['bias'])168 169 add_name = matmul_op.get_name() + '_eltwise'170 #add_name = matmul_op.get_name()171 #add_mmul = graph.create_op(add_name, 'eltwise', input_ops=add_ops)172 add_mmul = graph.create_op(add_name, 'add', input_ops=add_ops)173 fix_configs[add_name] = fix_configs[matmul_op.get_name()]174 175 if matmul_op.has_attr('fuse_relu'):176 add_mmul.set_attr('fuse_relu', True)177 178 for down_op in rebuild_configs['matmul_down_ops']:179 down_op.replace_input_ops(matmul_op, add_mmul)180 graph.remove_op(matmul_op)181class StridedSliceRebuilder(BaseRebuilder):182 def __init__(self, *args, **kwargs):183 super().__init__(*args, **kwargs)184 self.__fix_data = kwargs['fix_data']185 self.DATA_OPS = ['const']186 self.OPERATION_OPS = ['eltwise', 'matmul', 'linear', 'add']187 188 def rebuild(self, graph, configs=None):189 if not isinstance(configs, dict):190 raise ValueError('Strided_slice op rebuild configs should be dict')191 192 while len(self.rebuild_ops) > 0:193 #any_dict_index = list(self.rebuild_ops.keys())[0]194 any_index = self.rebuild_ops[0]195 stridedslice_rebuild_type, stridedslice_rebuild_config = self.__get_rebuild_config(any_index, self.rebuild_ops)196 self.__rebuild_one_op(graph, stridedslice_rebuild_type, stridedslice_rebuild_config, configs)197 198 def __get_rebuild_config(self, index, op_dict):199 rebuild_config = {}200 201 strided_op_list = []202 #input_op_name = op_dict[index][0]203 #input_ops = [node for node in self.op_graph.get_op(index).get_input_ops()]204 input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(index))]205 input_op_name = input_ops[0].get_name()206 207 for key in op_dict:208 #key_input_ops = [node for node in self.op_graph.get_op(key).get_input_ops()]209 key_input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(key))]210 #if op_dict[key][0] == input_op_name:211 if key_input_ops[0].get_name() == input_op_name:212 strided_op_list.append(self.op_graph.get_op(key))213 rebuild_config['stridedslice_ops'] = strided_op_list214 215 stridedslice_up_op = self.op_graph.get_op(input_op_name)216 rebuild_config['stridedslice_up_op'] = stridedslice_up_op217 rebuild_type, rebuild_config['params_ops'] = self.__get_legal_stridedslice_param(stridedslice_up_op)218 219 stridedslice_down_ops = {}220 for op in strided_op_list:221 down_ops = get_down_ops(op, self.op_graph)222 stridedslice_down_ops[op] = down_ops223 rebuild_config['stridedslice_down_ops'] = stridedslice_down_ops224 225 for op in strided_op_list:226 #del op_dict[op.get_name()]227 op_dict.remove(op.get_name())228 229 return rebuild_type, rebuild_config230 231 def __get_operation_stridedslice_param(self, up_op):232 param_ops_dict = {'input':None, 'weights':None, 'bias':None}233 234 input_num = up_op.get_input_num()235 #if (input_num == 2) and (up_op.get_type() == 'eltwise'):236 if (input_num == 2) and (up_op.get_type() == 'add'):237 input_ops = get_input_ops_list(up_op)238 input_ops_type = [input_op.get_type() for input_op in input_ops]239 if ('const' in input_ops_type) and (('matmul' in input_ops_type) or ('linear' in input_ops_type)):240 #if set(['const', 'matmul']) == set([input_op.get_type() for input_op in input_ops]):241 for op in input_ops:242 if op.get_type() == 'const':243 param_ops_dict['bias'] = op244 elif op.get_type() in GLOBAL_VARIABLE.MATMUL_OPS:245 if op.get_input_num() != 2:246 raise ValueError('Can not handle this kind of strided_slice op')247 for matmul_op in get_input_ops_list(op):248 if op.get_type() == 'data':249 param_ops_dict['input'] = matmul_op250 else:251 param_ops_dict['weights'] = matmul_op252 253 else:254 raise ValueError('Can not handle this kind of strided_slice op')255 256 elif (input_num > 1) and (up_op.get_type() in GLOBAL_VARIABLE.MATMUL_OPS):257 param_ops = get_input_ops_list(up_op)258 for op in param_ops:259 if op.get_type() == 'data':260 param_ops_dict['input'] = op261 elif op.get_output_tensor().ndim == 2:262 param_ops_dict['weights'] = op263 elif op.get_output_tensor().ndim == 1:264 param_ops_dict['bias'] = op265 else:266 raise ValueError('Can not handle this kind of strided_slice op')267 return param_ops_dict268 269 def __get_legal_stridedslice_param(self, up_op):270 if up_op.get_type() in self.DATA_OPS:271 stridedslice_type = 1272 param_ops_dict = {'const':up_op}273 elif up_op.get_type() in self.OPERATION_OPS:274 stridedslice_type = 2275 param_ops_dict = self.__get_operation_stridedslice_param(up_op)276 else:277 raise ValueError('Can not handle this kind of strided_slice op')278 return stridedslice_type, param_ops_dict279 280 def __rebuild_one_operation_op(self, graph, rebuild_configs, fix_configs):281 num = 0282 283 stridedslice_ops = rebuild_configs['stridedslice_ops'] 284 stridedslice_up_op = rebuild_configs['stridedslice_up_op']285 params_ops = rebuild_configs['params_ops']286 param_input = params_ops['input']287 param_weights = params_ops['weights']288 param_bias = params_ops['bias']289 stridedslice_down_ops = rebuild_configs['stridedslice_down_ops']290 291 for op in stridedslice_ops:292 op_attrs = op.get_attrs()293 begin = op_attrs['begin']294 end = op_attrs['end']295 end_mask = op_attrs['end_mask']296 if param_weights.get_name() in self.__fix_data:297 weights_data = self.__fix_data[param_weights.get_name()]298 else:299 #weights_data = param_weights.get_attrs()['data']300 #weights_data = param_weights.get_attr('data')301 weights_data = const_op_data(param_weights)302 if end_mask == 1:303 weights_tensor = weights_data[begin[1]:end[1], :]304 else:305 weights_tensor = weights_data[begin[1]:end[1], begin[0]:end[0]]306 307 weight_op_name = param_weights.get_name()+'_split_'+str(num)308 #weights_op = graph.create_op(name=weight_op_name, kind=param_weights.get_type(), tensor=weights_tensor)309 weights_op = graph.create_const_op(weight_op_name, weights_tensor)310 fix_configs[weight_op_name] = fix_configs[param_weights.get_name()] 311 312 wx_xt_ops: Dict[str, List[Op]] = {}313 wx_xt_ops['input'] = [param_input]314 #wx_xt_ops['input'].append(param_input)315 wx_xt_ops['input'].append(weights_op)316 317 attrs: Dict[str, Any] = {}318 if stridedslice_up_op.get_type() == 'matmul':319 attrs['transpose_a'] = stridedslice_up_op.get_attr('transpose_a')320 attrs['transpose_b'] = stridedslice_up_op.get_attr('transpose_b')321 else:322 attrs['transpose_a'] = False323 attrs['transpose_b'] = True324 325 wx_xt_name = op.get_name() + '_matmul'326 wx_xt_mmul = graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)327 fix_configs[wx_xt_name] = fix_configs[op.get_name()]328 329 if param_bias is not None:330 if param_bias.get_name() in self.__fix_data:331 bias_data = self.__fix_data[param_bias.get_name()]332 else:333 #bias_data = param_bias.get_attrs()['data']334 #bias_data = param_bias.get_attr('data')335 bias_data = const_op_data(param_bias)336 bias_tensor = bias_data[begin[1]:end[1]]337 bias_op_name = param_bias.get_name()+'_split_'+str(num)338 #bias_op = graph.create_op(name=bias_op_name, kind=param_bias.get_type(), tensor=bias_tensor)339 bias_op = graph.create_const_op(bias_op_name, bias_tensor)340 fix_configs[bias_op_name] = fix_configs[param_bias.get_name()]341 342 add_ops: Dict[str, List[Op]] = {}343 add_ops['input'] = [wx_xt_mmul]344 add_ops['input'].append(bias_op)345 add_name = op.get_name() + '_eltwise'346 #add_mmul = graph.create_op(add_name, 'eltwise', input_ops=add_ops)347 add_mmul = graph.create_op(add_name, 'add', input_ops=add_ops)348 fix_configs[add_name] = fix_configs[op.get_name()]349 350 for down_op in stridedslice_down_ops[op]:351 if param_bias is not None:352 down_op.replace_input_ops(op, add_mmul)353 else:354 down_op.replace_input_ops(op, wx_xt_mmul)355 graph.remove_op(op)356 num = num + 1357 graph.remove_op(stridedslice_up_op)358 359 def __rebuild_one_const_op(self, graph, rebuild_configs, fix_configs):360 num = 0361 362 stridedslice_ops = rebuild_configs['stridedslice_ops'] 363 stridedslice_up_op = rebuild_configs['stridedslice_up_op']364 const_op = rebuild_configs['const']365 stridedslice_down_ops = rebuild_configs['stridedslice_down_ops']366 367 for op in stridedslice_ops:368 op_attrs = op.get_attrs()369 begin = op_attrs['begin']370 end = op_attrs['end']371 end_mask = op_attrs['end_mask']372 if const_op.get_name() in self.__fix_data:373 const_data = self.__fix_data[const_op.get_name()]374 else:375 #const_data = const_op.get_attrs()['data']376 #const_data = const_op.get_attr('data')377 const_data = const_op_data(const_op)378 if end_mask == 1:379 const_tensor = const_data[begin[1]:end[1], :]380 else:381 const_tensor = const_data[begin[1]:end[1], begin[0]:end[0]]382 383 split_op_name = const_op.get_name()+'_split_'+str(num)384 #split_op = graph.create_op(name=split_op_name, kind=const_op.get_type(), tensor=const_tensor)385 split_op = graph.create_const_op(split_op_name, const_tensor)386 fix_configs[split_op_name] = fix_configs[const_op.get_name()] 387 388 for down_op in stridedslice_down_ops[op]:389 down_op.replace_input_ops(op, split_op)390 graph.remove_op(op)391 num = num + 1392 graph.remove_op(stridedslice_up_op)393 394 395 def __rebuild_one_op(self, graph, rebuild_type, rebuild_configs, fix_configs):396 if rebuild_type == 1:397 self.__rebuild_one_const_op(graph, rebuild_configs, fix_configs)398 elif rebuild_type == 2:399 self.__rebuild_one_operation_op(graph, rebuild_configs, fix_configs)400 401 402'''403class StridedSliceRebuilder(Rebuilder):404 def __init__(self, *args, **kwargs):405 super(StridedSliceRebuilder).__init__(*args, **kwargs)406 407 408 self.graph = configs['graph']409 self.fix_data = configs['fix_data']410 self.stridedslice_ops = configs['stridedslice_ops'] 411 self.stridedslice_up_op = configs['stridedslice_up_op']412 params_ops = configs['params_ops']413 self.param_input = params_ops['input']414 self.param_weights = params_ops['weights']415 self.param_bias = params_ops['bias']416 self.stridedslice_down_ops = configs['stridedslice_down_ops']417 418 def rebuild(self, fix_configs):419 num = 0420 for op in self.stridedslice_ops:421 op_attrs = op.get_attrs()422 begin = op_attrs['begin']423 end = op_attrs['end']424 end_mask = op_attrs['end_mask']425 if self.param_weights.get_name() in self.fix_data:426 weights_data = self.fix_data[self.param_weights.get_name()]427 else:428 #weights_data = self.param_weights.get_attrs()['data']429 weights_data = self.param_weights.get_attr('data')430 if end_mask == 1:431 weights_tensor = weights_data[begin[1]:end[1], :]432 else:433 weights_tensor = weights_data[begin[1]:end[1], begin[0]:end[0]]434 435 weight_op_name = self.param_weights.get_name()+'_split_'+str(num)436 weights_op = self.graph.create_op(name=weight_op_name, 437 kind=self.param_weights.get_type(), tensor=weights_tensor)438 fix_configs[weight_op_name] = fix_configs[self.param_weights.get_name()] 439 440 wx_xt_ops: Dict[str, List[Op]] = {}441 wx_xt_ops['input'] = [self.param_input]442 wx_xt_ops['input'].append(weights_op)443 444 attrs: Dict[str, Any] = {}445 attrs['transpose_a'] = False446 attrs['transpose_b'] = True447 448 wx_xt_name = op.get_name() + '_matmul'449 wx_xt_mmul = self.graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)450 fix_configs[wx_xt_name] = fix_configs[op.get_name()]451 452 if self.param_bias is not None:453 if self.param_bias.get_name() in self.fix_data:454 bias_data = self.fix_data[self.param_bias.get_name()]455 else:456 #bias_data = self.param_bias.get_attrs()['data']457 bias_data = self.param_bias.get_attr('data')458 bias_tensor = bias_data[begin[1]:end[1]]459 bias_op_name = self.param_bias.get_name()+'_split_'+str(num)460 bias_op = self.graph.create_op(name=bias_op_name, 461 kind=self.param_bias.get_type(), tensor=bias_tensor)462 fix_configs[bias_op_name] = fix_configs[self.param_bias.get_name()]463 464 add_ops: Dict[str, List[Op]] = {}465 add_ops['input'] = [wx_xt_mmul]466 add_ops['input'].append(bias_op)467 468 add_shape = wx_xt_mmul.get_output_tensor().dims469 add_tensor = np.zeros(add_shape, dtype=np.int64)470 471 add_name = op.get_name() + '_eltwise'472 #add_mmul = self.graph.create_op(add_name, 'eltwise', input_ops=add_ops, tensor=add_tensor)473 add_mmul = self.graph.create_op(add_name, 'add', input_ops=add_ops, tensor=add_tensor)474 fix_configs[add_name] = fix_configs[op.get_name()]475 476 for down_op in self.stridedslice_down_ops[op]:477 if self.param_bias is not None:478 down_op.replace_input_ops(op, add_mmul)479 else:480 down_op.replace_input_ops(op, wx_xt_mmul)481 self.graph.remove_op(op)482 num = num + 1483 self.graph.remove_op(self.stridedslice_up_op)484 485class FixRebuilder():486 def __init__(self, configs):487 self.graph = configs['graph']488 self.fix_op = configs['fix_op'] 489 self.fix_up_op = configs['fix_up_op']490 self.fix_down_ops = configs['fix_down_ops']491 492 def rebuild(self):493 for down_op in self.fix_down_ops:494 down_op.replace_input_ops(self.fix_op, self.fix_up_op)495 self.graph.remove_op(self.fix_op)...
test_rebuild.py
Source:test_rebuild.py
1from mock import patch, MagicMock, mock2from srv.modules.runners import rebuild3class TestRebuild():4 """5 """6 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)7 @patch('salt.runner.Runner', autospec=True)8 @patch('salt.client.LocalClient', autospec=True)9 def test_minions(self, localclient, runner, mm):10 rebuild.__opts__ = {}11 local = localclient.return_value12 local.cmd.return_value = {'data1.ceph': 'data1.ceph'}13 rr = rebuild.Rebuild(['data1.ceph'])14 assert rr.minions == ['data1.ceph']15 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)16 @patch('salt.runner.Runner', autospec=True)17 @patch('salt.client.LocalClient', autospec=True)18 def test_minions_targets(self, localclient, runner, mm):19 rebuild.__opts__ = {}20 local = localclient.return_value21 local.cmd.return_value = {'data1.ceph': 'data1.ceph',22 'data2.ceph':'data2.ceph'}23 rr = rebuild.Rebuild(['I@roles:storage'])24 assert rr.minions == ['data1.ceph', 'data2.ceph']25 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)26 @patch('salt.runner.Runner', autospec=True)27 @patch('salt.client.LocalClient', autospec=True)28 def test_minions_multiple(self, localclient, runner, mm):29 rebuild.__opts__ = {}30 local = localclient.return_value31 local.cmd.return_value = {'data1.ceph': 'data1.ceph',32 'data2.ceph':'data2.ceph'}33 rr = rebuild.Rebuild(['data*.ceph', 'data*.ceph'])34 assert rr.minions == ['data1.ceph', 'data2.ceph']35 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)36 @patch('salt.runner.Runner', autospec=True)37 @patch('salt.client.LocalClient', autospec=True)38 def test_osd_list(self, localclient, runner, mm):39 rebuild.__opts__ = {}40 local = localclient.return_value41 local.cmd.return_value = {'data1.ceph': [0, 1, 2]}42 rr = rebuild.Rebuild(['data*.ceph'])43 result = rr._osd_list('data1.ceph')44 assert result == [0, 1, 2]45 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)46 @patch('salt.runner.Runner', autospec=True)47 @patch('salt.client.LocalClient', autospec=True)48 def test_osd_list_no_match(self, localclient, runner, mm):49 rebuild.__opts__ = {}50 local = localclient.return_value51 local.cmd.return_value = {'data1.ceph': [0, 1, 2]}52 rr = rebuild.Rebuild(['data*.ceph'])53 result = rr._osd_list('data2.ceph')54 assert result == None55 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)56 @patch('salt.runner.Runner', autospec=True)57 @patch('salt.client.LocalClient', autospec=True)58 def test_validate_osd_df(self, localclient, runner, mm):59 rebuild.__opts__ = {}60 local = localclient.return_value61 local.cmd.return_value = {}62 mm.return_value = 'admin'63 rr = rebuild.Rebuild(['data*.ceph'])64 osd_df = {'admin': {'nodes': '', 'summary':{'total_kb_avail': '0'}}}65 result = rr._validate_osd_df(osd_df)66 assert result == True67 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)68 @patch('salt.runner.Runner', autospec=True)69 @patch('salt.client.LocalClient', autospec=True)70 def test_validate_osd_df_missing_mm(self, localclient, runner, mm):71 rebuild.__opts__ = {}72 local = localclient.return_value73 local.cmd.return_value = {}74 mm.return_value = 'admin'75 rr = rebuild.Rebuild(['data*.ceph'])76 osd_df = {'': {'nodes': '', 'summary':{'total_kb_avail': '0'}}}77 result = rr._validate_osd_df(osd_df)78 assert result == False79 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)80 @patch('salt.runner.Runner', autospec=True)81 @patch('salt.client.LocalClient', autospec=True)82 def test_validate_osd_df_missing_nodes(self, localclient, runner, mm):83 rebuild.__opts__ = {}84 local = localclient.return_value85 local.cmd.return_value = {}86 mm.return_value = 'admin'87 rr = rebuild.Rebuild(['data*.ceph'])88 osd_df = {'admin': {'': '', 'summary':{'total_kb_avail': '0'}}}89 result = rr._validate_osd_df(osd_df)90 assert result == False91 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)92 @patch('salt.runner.Runner', autospec=True)93 @patch('salt.client.LocalClient', autospec=True)94 def test_validate_osd_df_missing_summary(self, localclient, runner, mm):95 rebuild.__opts__ = {}96 local = localclient.return_value97 local.cmd.return_value = {}98 mm.return_value = 'admin'99 rr = rebuild.Rebuild(['data*.ceph'])100 osd_df = {'admin': {'nodes': '', '':{'total_kb_avail': '0'}}}101 result = rr._validate_osd_df(osd_df)102 assert result == False103 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)104 @patch('salt.runner.Runner', autospec=True)105 @patch('salt.client.LocalClient', autospec=True)106 def test_validate_osd_df_missing_total(self, localclient, runner, mm):107 rebuild.__opts__ = {}108 local = localclient.return_value109 local.cmd.return_value = {}110 mm.return_value = 'admin'111 rr = rebuild.Rebuild(['data*.ceph'])112 osd_df = {'admin': {'nodes': '', 'summary':{'': '0'}}}113 result = rr._validate_osd_df(osd_df)114 assert result == False115 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)116 @patch('salt.runner.Runner', autospec=True)117 @patch('salt.client.LocalClient', autospec=True)118 def test_safe(self, localclient, runner, mm):119 rebuild.__opts__ = {}120 local = localclient.return_value121 local.cmd.return_value = {'admin': {'nodes': [122 {'id': 0, 'kb_used': 10},123 {'id': 1, 'kb_used': 10}124 ],125 'summary':{'total_kb_avail': 30}}}126 mm.return_value = 'admin'127 rr = rebuild.Rebuild(['data*.ceph'])128 result = rr.safe(['0', '1'])129 assert result == True130 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)131 @patch('salt.runner.Runner', autospec=True)132 @patch('salt.client.LocalClient', autospec=True)133 def test_safe_fails(self, localclient, runner, mm):134 rebuild.__opts__ = {}135 local = localclient.return_value136 local.cmd.return_value = {'admin': {'nodes': [137 {'id': 0, 'kb_used': 10},138 {'id': 1, 'kb_used': 10}139 ],140 'summary':{'total_kb_avail': 10}}}141 mm.return_value = 'admin'142 rr = rebuild.Rebuild(['data*.ceph'])143 result = rr.safe(['0', '1'])144 assert result == False145 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)146 @patch('salt.runner.Runner', autospec=True)147 @patch('salt.client.LocalClient', autospec=True)148 def test_safe_fails_validation(self, localclient, runner, mm):149 rebuild.__opts__ = {}150 local = localclient.return_value151 local.cmd.return_value = {'': {'nodes': [152 {'id': 0, 'kb_used': 10},153 {'id': 1, 'kb_used': 10}154 ],155 'summary':{'total_kb_avail': 10}}}156 mm.return_value = 'admin'157 rr = rebuild.Rebuild(['data*.ceph'])158 result = rr.safe(['0', '1'])159 assert result == False160 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)161 @patch('salt.runner.Runner', autospec=True)162 @patch('salt.client.LocalClient', autospec=True)163 def test_check_failed(self, localclient, runner, mm):164 rebuild.__opts__ = {}165 rr = rebuild.Rebuild(['data*.ceph'])166 ret = {0: True, 1:True}167 result = rr._check_failed(ret, 'data1.ceph')168 assert result == False169 assert rr.skipped == []170 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)171 @patch('salt.runner.Runner', autospec=True)172 @patch('salt.client.LocalClient', autospec=True)173 def test_check_failed_finds_failure(self, localclient, runner, mm):174 rebuild.__opts__ = {}175 rr = rebuild.Rebuild(['data*.ceph'])176 ret = {0: False, 1:True}177 result = rr._check_failed(ret, 'data1.ceph')178 assert result == True179 assert rr.skipped == ['data1.ceph']180 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)181 @patch('salt.runner.Runner', autospec=True)182 @patch('salt.client.LocalClient', autospec=True)183 def test_check_failed_finds_error(self, localclient, runner, mm):184 rebuild.__opts__ = {}185 rr = rebuild.Rebuild(['data*.ceph'])186 ret = "An error message"187 result = rr._check_failed(ret, 'data1.ceph')188 assert result == True189 assert rr.skipped == ['data1.ceph']190 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)191 @patch('salt.runner.Runner', autospec=True)192 @patch('salt.client.LocalClient', autospec=True)193 def test_run(self, localclient, runner, mm):194 rebuild.__opts__ = {}195 rr = rebuild.Rebuild(['data*.ceph'])196 rr._disengaged = mock.Mock()197 rr._disengaged.return_value = True198 rr.minions = ['data1.ceph']199 rr._osd_list = mock.Mock()200 rr._osd_list.return_value = [0, 1]201 rr.safe = mock.Mock()202 rr.safe.return_value = True203 rr._busy_wait = mock.Mock()204 rr.runner.cmd = mock.Mock()205 rr.runner.cmd.return_value = {}206 rr._check_failed = mock.Mock()207 rr._check_failed.return_value = False208 rr._skipped_summary = mock.Mock()209 result = rr.run()210 assert result == ""211 assert rr._osd_list.called212 assert rr.safe.called213 assert rr.runner.cmd.called214 assert rr._check_failed.called215 assert rr._skipped_summary.called216 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)217 @patch('salt.runner.Runner', autospec=True)218 @patch('salt.client.LocalClient', autospec=True)219 def test_run_multiple(self, localclient, runner, mm):220 rebuild.__opts__ = {}221 rr = rebuild.Rebuild(['data*.ceph'])222 rr._disengaged = mock.Mock()223 rr._disengaged.return_value = True224 rr.minions = ['data1.ceph', 'data2.ceph']225 rr._osd_list = mock.Mock()226 rr._osd_list.return_value = [0, 1]227 rr.safe = mock.Mock()228 rr.safe.return_value = True229 rr._busy_wait = mock.Mock()230 rr.runner.cmd = mock.Mock()231 rr.runner.cmd.return_value = {}232 rr._check_failed = mock.Mock()233 rr._check_failed.return_value = False234 rr._skipped_summary = mock.Mock()235 result = rr.run()236 assert result == ""237 assert rr._osd_list.call_count == 2238 assert rr.safe.call_count == 2239 assert rr.runner.cmd.call_count == 4240 assert rr._check_failed.call_count == 2241 assert rr._skipped_summary.called242 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)243 @patch('salt.runner.Runner', autospec=True)244 @patch('salt.client.LocalClient', autospec=True)245 def test_run_check_only(self, localclient, runner, mm):246 rebuild.__opts__ = {}247 rr = rebuild.Rebuild(['data*.ceph'])248 rr._disengaged = mock.Mock()249 rr._disengaged.return_value = True250 rr.minions = ['data1.ceph']251 rr._osd_list = mock.Mock()252 rr._osd_list.return_value = [0, 1]253 rr.safe = mock.Mock()254 rr.safe.return_value = True255 rr.runner.cmd = mock.Mock()256 rr.runner.cmd.return_value = {}257 rr._check_failed = mock.Mock()258 rr._check_failed.return_value = False259 rr._skipped_summary = mock.Mock()260 result = rr.run(checkonly=True)261 assert result == ""262 assert rr._osd_list.called263 assert rr.safe.called264 assert rr.runner.cmd.called is False265 assert rr._check_failed.called is False266 assert rr._skipped_summary.called267 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)268 @patch('salt.runner.Runner', autospec=True)269 @patch('salt.client.LocalClient', autospec=True)270 def test_run_safety_engaged(self, localclient, runner, mm):271 rebuild.__opts__ = {}272 rr = rebuild.Rebuild(['data*.ceph'])273 rr._disengaged = mock.Mock()274 rr._disengaged.return_value = False275 rr.minions = ['data1.ceph']276 rr._osd_list = mock.Mock()277 rr._osd_list.return_value = [0, 1]278 rr.safe = mock.Mock()279 rr.safe.return_value = True280 rr.runner.cmd = mock.Mock()281 rr.runner.cmd.return_value = {}282 rr._check_failed = mock.Mock()283 rr._check_failed.return_value = False284 rr._skipped_summary = mock.Mock()285 result = rr.run()286 assert result == ""287 assert rr._osd_list.called is False288 assert rr.safe.called is False289 assert rr.runner.cmd.called is False290 assert rr._check_failed.called is False291 assert rr._skipped_summary.called is False292 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)293 @patch('salt.runner.Runner', autospec=True)294 @patch('salt.client.LocalClient', autospec=True)295 def test_run_no_remove(self, localclient, runner, mm):296 rebuild.__opts__ = {}297 rr = rebuild.Rebuild(['data*.ceph'])298 rr._disengaged = mock.Mock()299 rr._disengaged.return_value = True300 rr.minions = ['data1.ceph']301 rr._osd_list = mock.Mock()302 rr._osd_list.return_value = []303 rr.safe = mock.Mock()304 rr.safe.return_value = True305 rr.runner.cmd = mock.Mock()306 rr.runner.cmd.return_value = {}307 rr._check_failed = mock.Mock()308 rr._check_failed.return_value = False309 rr._skipped_summary = mock.Mock()310 result = rr.run()311 assert result == ""312 assert rr._osd_list.called313 assert rr.safe.called is False314 assert rr.runner.cmd.called315 assert rr._check_failed.called is False316 assert rr._skipped_summary.called317 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)318 @patch('salt.runner.Runner', autospec=True)319 @patch('salt.client.LocalClient', autospec=True)320 def test_run_not_safe(self, localclient, runner, mm):321 rebuild.__opts__ = {}322 rr = rebuild.Rebuild(['data*.ceph'])323 rr._disengaged = mock.Mock()324 rr._disengaged.return_value = True325 rr.minions = ['data1.ceph']326 rr._osd_list = mock.Mock()327 rr._osd_list.return_value = [0, 1]328 rr.safe = mock.Mock()329 rr.safe.return_value = False330 rr.runner.cmd = mock.Mock()331 rr.runner.cmd.return_value = {}332 rr._check_failed = mock.Mock()333 rr._check_failed.return_value = False334 rr._skipped_summary = mock.Mock()335 result = rr.run()336 assert result == ""337 assert rr._osd_list.called338 assert rr.safe.called339 assert rr.runner.cmd.called is False340 assert rr._check_failed.called is False341 assert rr._skipped_summary.called is False342 @patch('srv.modules.runners.rebuild.master_minion', autospec=True)343 @patch('salt.runner.Runner', autospec=True)344 @patch('salt.client.LocalClient', autospec=True)345 def test_run_remove_failed(self, localclient, runner, mm):346 rebuild.__opts__ = {}347 rr = rebuild.Rebuild(['data*.ceph'])348 rr._disengaged = mock.Mock()349 rr._disengaged.return_value = True350 rr.minions = ['data1.ceph']351 rr._osd_list = mock.Mock()352 rr._osd_list.return_value = [0, 1]353 rr.safe = mock.Mock()354 rr.safe.return_value = True355 rr._busy_wait = mock.Mock()356 rr.runner.cmd = mock.Mock()357 rr.runner.cmd.return_value = {}358 rr._check_failed = mock.Mock()359 rr._check_failed.return_value = True360 rr._skipped_summary = mock.Mock()361 result = rr.run()362 assert result == ""363 assert rr._osd_list.called364 assert rr.safe.called365 assert rr.runner.cmd.called366 assert rr._check_failed.called...
cell.py
Source:cell.py
...124 rebuild_ops = self.__get_rebuild_ops(op_type)125 if (op_type == 'relu') and (len(rebuild_ops) > 0): 126 rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}127 rebuilder = ReluRebuilder(**rebuild_configs)128 rebuilder.rebuild(self.__graph, self.__fix_configs)129 130 elif (op_type == 'fix') and (len(rebuild_ops) > 0):131 for fix_op in rebuild_ops:132 #input_ops = [op for op in self.__graph.get_op(fix_op).get_input_ops()]133 input_ops = [op for op in get_input_ops_list(self.__graph.get_op(fix_op))]134 #if self.__graph.get_op(op_input[fix_op][0]).get_type() == 'const':135 if input_ops[0].get_type() == 'const':136 self.__make_one_fix_data(input_ops[0].get_name(), fix_op)137 #self.__make_one_fix_data(op_input[fix_op][0], fix_op)138 139 rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}140 rebuilder = FixRebuilder(**rebuild_configs)141 rebuilder.rebuild(self.__graph, self.__fix_configs)142 #self._fuse_fix_config()143 144 elif (op_type == 'strided_slice') and (len(rebuild_ops) > 0):145 rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops, 'fix_data':self.__const_fix_data}146 rebuilder = StridedSliceRebuilder(**rebuild_configs)147 rebuilder.rebuild(self.__graph, self.__fix_configs)148 149 elif (op_type in GLOBAL_VARIABLE.MATMUL_OPS) and (len(rebuild_ops) > 0):150 rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}151 rebuilder = MatmulRebuilder(**rebuild_configs)152 rebuilder.rebuild(self.__graph, self.__fix_configs)153 154 self.__remove_invalid_const_data()155 '''156 for op in self.__graph.get_ops():157 print(op.get_name())158 print(op.get_type())159 '''160 '''161 def __rebuild_graph(self):162 for op_type in GLOBAL_VARIABLE.REBUILD_OPS_TYPE:163 if op_type == 'fix':164 fix_op_input = self.__get_op_input(op_type)165 166 for fix_op in fix_op_input:167 self.__fix_configs[fix_op_input[fix_op]] = self.__graph.get_op(fix_op).get_attrs()168 if self.__graph.get_op(fix_op_input[fix_op]).get_type() == 'const':169 self.__make_one_fix_data(fix_op_input[fix_op], fix_op)170 fix_rebuild_config = self.__get_fix_rebuild_config(fix_op, fix_op_input)171 fix_rebuild_config['graph'] = self.__graph172 fix_rebuilder = FixRebuilder(fix_rebuild_config)173 fix_rebuilder.rebuild()174 if op_type == 'strided_slice':175 #stridedslice_op_input = self.__get_stridedslice_op_input__()176 stridedslice_op_input = self.__get_op_input(op_type)177 print(stridedslice_op_input)178 179 while len(stridedslice_op_input) > 0:180 any_dict_index = list(stridedslice_op_input.keys())[0]181 stridedslice_rebuild_config = self.__get_stridedslice_rebuild_config(any_dict_index, stridedslice_op_input)182 stridedslice_rebuild_config['graph'] = self.__graph183 stridedslice_rebuild_config['fix_data'] = self.__const_fix_data184 stridedslice_rebuilder = StridedSliceRebuilder(stridedslice_rebuild_config)185 stridedslice_rebuilder.rebuild(self.__fix_configs)186 self.__remove_invalid_const_data()187 for op in self.__graph.get_ops():188 print(op.get_name())189 print(op.get_type())190 ''' 191 def reset(self, configs, xir_graph=None):192 pass193 194 def compile(self):195 parser_configs = {'ordered_nodes':self.__ordered_nodes,196 'graph':self.__graph,197 'fix_configs':self.__fix_configs,198 'fix_data':self.__const_fix_data}199 xir_parser = XirParser(**parser_configs)...
rebuild.py
Source:rebuild.py
1#!/usr/bin/python32'''3 (C) Copyright 2018-2021 Intel Corporation.4 SPDX-License-Identifier: BSD-2-Clause-Patent5'''6from daos_core_base import DaosCoreBase7class DaosCoreTestRebuild(DaosCoreBase):8 # pylint: disable=too-many-ancestors9 """Run just the daos_test rebuild tests.10 :avocado: recursive11 """12 def test_rebuild_0to10(self):13 """Jira ID: DAOS-277014 Test Description:15 Run daos_test -r -s3 -u subtests=0-1016 Use cases:17 Core tests for daos_test rebuild18 :avocado: tags=all,pr,daily_regression19 :avocado: tags=hw,ib2,medium20 :avocado: tags=unittest21 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_0to1022 """23 self.run_subtest()24 def test_rebuild_12to15(self):25 """Jira ID: DAOS-277026 Test Description:27 Run daos_test -r -s3 -u subtests=12-1528 Use cases:29 Core tests for daos_test rebuild30 :avocado: tags=all,pr,daily_regression31 :avocado: tags=hw,ib2,medium32 :avocado: tags=unittest33 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_12to1534 """35 self.run_subtest()36 def test_rebuild_16(self):37 """Jira ID: DAOS-277038 Test Description:39 Run daos_test -r -s3 -u subtests=1640 Use cases:41 Core tests for daos_test rebuild42 :avocado: tags=all,pr,daily_regression43 :avocado: tags=hw,ib2,medium44 :avocado: tags=unittest45 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1646 """47 self.run_subtest()48 def test_rebuild_17(self):49 """Jira ID: DAOS-277050 Test Description:51 Run daos_test -r -s3 -u subtests=1752 Use cases:53 Core tests for daos_test rebuild54 :avocado: tags=all,pr,daily_regression55 :avocado: tags=hw,ib2,medium56 :avocado: tags=unittest57 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1758 """59 self.run_subtest()60 def test_rebuild_18(self):61 """Jira ID: DAOS-277062 Test Description:63 Run daos_test -r -s3 -u subtests=1864 Use cases:65 Core tests for daos_test rebuild66 :avocado: tags=all,pr,daily_regression67 :avocado: tags=hw,ib2,medium68 :avocado: tags=unittest69 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1870 """71 self.run_subtest()72 def test_rebuild_19(self):73 """Jira ID: DAOS-277074 Test Description:75 Run daos_test -r -s3 -u subtests=1976 Use cases:77 Core tests for daos_test rebuild78 :avocado: tags=all,pr,daily_regression79 :avocado: tags=hw,ib2,medium80 :avocado: tags=unittest81 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1982 """83 self.run_subtest()84 def test_rebuild_20(self):85 """Jira ID: DAOS-277086 Test Description:87 Run daos_test -r -s3 -u subtests=2088 Use cases:89 Core tests for daos_test rebuild90 :avocado: tags=all,pr,daily_regression91 :avocado: tags=hw,ib2,medium92 :avocado: tags=unittest93 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_2094 """95 self.run_subtest()96 def test_rebuild_21(self):97 """Jira ID: DAOS-277098 Test Description:99 Run daos_test -r -s3 -u subtests=21100 Use cases:101 Core tests for daos_test rebuild102 :avocado: tags=all,pr,daily_regression103 :avocado: tags=hw,ib2,medium104 :avocado: tags=unittest105 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_21106 """107 self.run_subtest()108 def test_rebuild_22(self):109 """Jira ID: DAOS-2770110 Test Description:111 Run daos_test -r -s3 -u subtests=22112 Use cases:113 Core tests for daos_test rebuild114 :avocado: tags=all,pr,daily_regression115 :avocado: tags=hw,ib2,medium116 :avocado: tags=unittest117 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_22118 """119 self.run_subtest()120 def test_rebuild_23(self):121 """Jira ID: DAOS-2770122 Test Description:123 Run daos_test -r -s3 -u subtests=23124 Use cases:125 Core tests for daos_test rebuild126 :avocado: tags=all,pr,daily_regression127 :avocado: tags=hw,ib2,medium128 :avocado: tags=unittest129 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_23130 """131 self.run_subtest()132 def test_rebuild_24(self):133 """Jira ID: DAOS-2770134 Test Description:135 Run daos_test -r -s3 -u subtests=24136 Use cases:137 Core tests for daos_test rebuild138 :avocado: tags=all,pr,daily_regression139 :avocado: tags=hw,ib2,medium140 :avocado: tags=unittest141 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_24142 """143 self.run_subtest()144 def test_rebuild_25(self):145 """Jira ID: DAOS-2770146 Test Description:147 Run daos_test -r -s3 -u subtests=25148 Use cases:149 Core tests for daos_test rebuild150 :avocado: tags=all,pr,daily_regression151 :avocado: tags=hw,ib2,medium152 :avocado: tags=unittest153 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_25154 """155 self.run_subtest()156 def test_rebuild_26(self):157 """Jira ID: DAOS-2770158 Test Description:159 Run daos_test -r -s3 -u subtests=26160 Use cases:161 Core tests for daos_test rebuild162 :avocado: tags=all,pr,daily_regression163 :avocado: tags=hw,ib2,medium164 :avocado: tags=unittest165 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_26166 """167 self.run_subtest()168 def test_rebuild_27(self):169 """Jira ID: DAOS-2770170 Test Description:171 Run daos_test -r -s3 -u subtests=27172 Use cases:173 Core tests for daos_test rebuild174 :avocado: tags=all,pr,daily_regression175 :avocado: tags=hw,ib2,medium176 :avocado: tags=unittest177 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_27178 """179 self.run_subtest()180 def test_rebuild_28(self):181 """Jira ID: DAOS-2770182 Test Description:183 Run daos_test -r -s3 -u subtests=28184 Use cases:185 Core tests for daos_test rebuild186 :avocado: tags=all,pr,daily_regression187 :avocado: tags=hw,ib2,medium188 :avocado: tags=unittest189 :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_28190 """...
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Get 100 minutes of automation test minutes FREE!!