KaneAI - World’s First E2E Software Testing Agent.Book a Demo 
How to use skip_task method in Lemoncheesecake
Best Python code snippet using lemoncheesecake
upgrade.py
Source:upgrade.py 
...46 self.device["host"] = str(task.device.primary_ip.address.ip)47 else:48 msg = "No primary (mgmt) address"49 self.warning(msg)50 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)51 def debug(self, msg):52 log.debug(f"{self.log_id} - {msg}")53 self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - DEBUG - {msg}\n'54 self.task.save()55 def info(self, msg):56 log.info(f"{self.log_id} - {msg}")57 self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - INFO - {msg}\n'58 self.task.save()59 def warning(self, msg):60 log.warning(f"{self.log_id} - {msg}")61 self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - WARNING - {msg}\n'62 self.task.save()63 def error(self, msg):64 log.error(f"{self.log_id} - {msg}")65 self.task.log += f'{datetime.now(pytz.timezone(TIME_ZONE)).strftime("%Y-%m-%d %H:%M:%S")} - ERROR - {msg}\n'66 self.task.save()67 def action_task(self, action, msg, reason):68 self.task.status = action69 self.task.message = msg70 self.task.fail_reason = reason71 self.task.save()72 raise UpgradeException(73 reason=reason,74 message=msg,75 )76 def skip_task(self, msg="", reason=""):77 self.action_task(TaskStatusChoices.STATUS_SKIPPED, msg, reason)78 def drop_task(self, msg="", reason=""):79 self.action_task(TaskStatusChoices.STATUS_FAILED, msg, reason)80 def check(self):81 if not hasattr(self.task.device.device_type, "golden_image"):82 msg = f"No Golden Image for {self.task.device.device_type.model}"83 self.warning(msg)84 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)85 else:86 self.debug(87 f"Golden Image for {self.task.device.device_type.model} is {self.task.device.device_type.golden_image.sw}"88 )89 if self.task.start_time > self.task.scheduled_time + timedelta(hours=int(self.task.mw_duration)):90 msg = "Maintenance Window is over"91 self.warning(msg)92 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)93 else:94 self.debug("MW is still active")95 if self.task.task_type == TaskTypeChoices.TYPE_UPGRADE:96 q = get_queue(UPGRADE_QUEUE)97 active_jobs = q.started_job_registry.count98 non_ack = ScheduledTask.objects.filter(start_time__isnull=False, confirmed=False).count()99 if non_ack >= active_jobs + UPGRADE_THRESHOLD:100 msg = f"Reached failure threshold: Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack-active_jobs}, threshold: {UPGRADE_THRESHOLD}"101 self.warning(msg)102 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)103 else:104 self.debug(105 f"Unconfirmed: {non_ack}, active: {active_jobs}, failed: {non_ack - active_jobs}, threshold: {UPGRADE_THRESHOLD}"106 )107 else:108 self.debug(f"Task type is {self.task.task_type}, check against threshold was skipped")109 def connect_cli(self, **kwargs):110 def to_telnet(cli, **kwargs):111 try:112 cli.close()113 except Exception:114 pass115 cli = False116 if self.device["port"] != 23:117 self.debug("Swiching to telnet")118 self.device["port"] = 23119 self.device["transport"] = "telnet"120 cli = self.connect_cli(**kwargs)121 return cli122 cli = IOSXEDriver(**self.device, **kwargs)123 try:124 self.debug(f'Trying to connect via TCP/{self.device["port"]} ...')125 cli.open()126 except ScrapliAuthenticationFailed:127 self.debug(f'Incorrect username while connecting to the device via TCP/{self.device["port"]}')128 cli = to_telnet(cli, **kwargs)129 except ScrapliConnectionError:130 self.debug(f'Device closed connection on TCP/{self.device["port"]}')131 # raise132 cli = to_telnet(cli, **kwargs)133 except Exception:134 self.debug(f'Unknown error while connecting to the device via TCP/{self.device["port"]}')135 cli = to_telnet(cli, **kwargs)136 else:137 self.debug(f'Login successful while connecting to the device via TCP/{self.device["port"]}')138 return cli139 def is_alive(self):140 try:141 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:142 s.settimeout(self.device.get("timeout_socket", 5))143 s.connect((self.device["host"], 22))144 except Exception:145 self.debug("no response on TCP/22")146 try:147 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:148 s.settimeout(self.device.get("timeout_socket", 5))149 s.connect((self.device["host"], 23))150 except Exception:151 self.debug("no response on TCP/23")152 time.sleep(2)153 return False154 else:155 self.debug("got response on TCP/23")156 else:157 self.debug("got response on TCP/22")158 time.sleep(2)159 return True160 def check_device(self):161 pid = ""162 sn = ""163 cmd = [164 "show version",165 "dir /all",166 ]167 cli = self.connect_cli()168 if not cli:169 msg = "Can not connect to device CLI"170 self.error(msg)171 self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)172 output = cli.send_commands(cmd)173 cli.close()174 if output.failed:175 msg = "Can not collect outputs from device"176 self.error(msg)177 self.skip_task(msg, TaskFailReasonChoices.FAIL_CONFIG)178 self.debug("----------vv Outputs vv----------")179 self.debug(output.result)180 self.debug("----------^^ Outputs ^^----------")181 r = re.search(r"\n\w+\s+(\S+)\s+.*\(revision\s+", output[0].result)182 if r:183 pid = r.group(1)184 # pid = re.sub('\+','plus',r.group(1))185 self.info(f"PID: {r.group(1)}")186 else:187 msg = "Can not get device PID"188 self.error(msg)189 self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)190 r = re.search(r"\n.*\s+board\s+ID\s+(\S+)", output[0].result)191 if r:192 sn = r.group(1)193 self.info(f"SN: {sn}")194 else:195 msg = "Can not get device SN"196 self.error(msg)197 self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)198 if pid.upper() != self.task.device.device_type.model.upper() or sn.lower() != self.task.device.serial.lower():199 msg = "Device PID/SN does not match with NetBox data"200 self.error(msg)201 self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_CONFIG)202 self.info(f"Device {pid}/{sn} matches with NetBox data")203 self.files = output[1].textfsm_parse_output()204 self.file_system = self.files[0]["file_system"].strip("/")205 self.target_image = self.task.device.device_type.golden_image.sw.filename206 self.target_path = self.task.device.device_type.golden_image.sw.image.path207 self.image_on_device = list(filter(lambda x: x["name"] == self.target_image, self.files))208 self.debug(f"File system: {self.file_system}")209 self.debug(f"Target Image: {self.target_image}")210 self.debug(f"Target Path: {self.target_path}")211 self.debug(f"Target Image on box: {self.image_on_device}")212 return True213 def file_upload(self):214 if self.task.transfer_method == TaskTransferMethod.METHOD_FTP:215 cmd_copy = f"copy ftp://{FTP_USERNAME}:{FTP_PASSWORD}@{FTP_SERVER}/{self.target_image} {self.file_system}/{self.target_image}"216 elif self.task.transfer_method == TaskTransferMethod.METHOD_HTTP:217 cmd_copy = f"copy {HTTP_SERVER}{self.target_image} {self.file_system}/{self.target_image}"218 else:219 msg = "Unknown transfer method"220 self.error(msg)221 self.skip_task(msg, reason=TaskFailReasonChoices.FAIL_UPLOAD)222 config = [223 "file prompt quiet",224 "line vty 0 15",225 "exec-timeout 180 0",226 ]227 config_undo = [228 "no file prompt quiet",229 "line vty 0 15",230 "exec-timeout 30 0",231 ]232 cli = self.connect_cli(timeout_ops=7200, timeout_transport=7200)233 if not cli:234 msg = "Unable to connect to the device"235 self.error(msg)236 self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)237 if not len(self.image_on_device):238 self.info("No image on the device. Need to transfer")239 self.debug(240 f'Free on {self.file_system} {self.files[0]["total_free"]}, \241 Image size (+10%) {int(int(self.task.device.device_type.golden_image.sw.image.size)*1.1)}'242 )243 if int(self.files[0]["total_free"]) < int(244 int(self.task.device.device_type.golden_image.sw.image.size) * 1.1245 ):246 try:247 cli.close()248 except Exception:249 pass250 msg = f"No enough space on {self.file_system}"251 self.error(msg)252 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)253 self.info("Download image from FTP...")254 output = cli.send_configs(config)255 self.debug(f"Preparing for copy:\n{output.result}")256 if output.failed:257 try:258 cli.close()259 except Exception:260 pass261 msg = "Can not change configuration"262 self.error(msg)263 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)264 self.debug(f"Copy command: {cmd_copy}")265 output = cli.send_command(cmd_copy)266 self.debug(f"Copying process:\n{output.result}")267 if output.failed or not (re.search(r"OK", output.result) or re.search(r"bytes copied in", output.result)):268 try:269 cli.close()270 except Exception:271 pass272 msg = "Can not download image from server"273 self.error(msg)274 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)275 output = cli.send_configs(config_undo)276 self.debug(f"Rollback after copy:\n{output.result}")277 if output.failed:278 try:279 cli.close()280 except Exception:281 pass282 msg = "Can not do rollback configuration"283 self.error(msg)284 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPLOAD)285 else:286 self.info(f"Image {self.target_image} already exists")287 self.info("MD5 verification ...")288 md5 = cli.send_command(289 f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}"290 )291 self.debug(f"MD5 verication result:\n{md5.result[-200:]}")292 if md5.failed:293 try:294 cli.close()295 except Exception:296 pass297 msg = "Can not check MD5"298 self.error(msg)299 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)300 if re.search(r"Verified", md5.result):301 self.info("MD5 was verified")302 else:303 try:304 cli.close()305 except Exception:306 pass307 msg = "Wrong M5"308 self.error(msg)309 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)310 try:311 cli.close()312 except Exception:313 pass314 self.info("File was uploaded and verified")315 return True316 def device_reload(self):317 cmd = [318 "show run | i boot system",319 "show version",320 ]321 cli = self.connect_cli()322 if not cli:323 msg = "Unable to connect to the device"324 self.error(msg)325 self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)326 output = cli.send_commands(cmd)327 self.debug(f"Collected outputs:------vvvvv\n{output.result}\n-----^^^^^")328 if output.failed:329 try:330 cli.close()331 except Exception:332 pass333 msg = "Can not collect outputs for upgrade"334 self.error(msg)335 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)336 parsed = output[1].textfsm_parse_output()337 sw_current = parsed[0].get("version", "N/A")338 sw_target = self.task.device.device_type.golden_image.sw.version339 self.debug(f"Current version is {sw_current}")340 if sw_current.upper() == sw_target.upper():341 msg = f"Current version {sw_current} matches with target {sw_target}"342 self.warning(msg)343 self.info("Update custom field")344 self.task.device.custom_field_data[CF_NAME_SW_VERSION] = sw_current345 self.task.device.save()346 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)347 if not len(self.image_on_device):348 msg = "No target image on the box"349 self.error(msg)350 self.skip_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)351 self.info("Image exists on the box")352 cli.timeout_ops = 600353 self.info("MD5 verification ...")354 md5 = cli.send_command(355 f"verify /md5 {self.file_system}/{self.target_image} {self.task.device.device_type.golden_image.sw.md5sum}"356 )357 self.debug(f"MD5 verication result:\n{md5.result[-200:]}")358 if md5.failed:359 try:360 cli.close()361 except Exception:362 pass363 msg = "Can not check MD5"364 self.error(msg)365 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)366 if re.search(r"Verified", md5.result):367 self.info("MD5 was verified")368 else:369 try:370 cli.close()371 except Exception:372 pass373 msg = "Wrong M5"374 self.error(msg)375 self.skip_task(msg, TaskFailReasonChoices.FAIL_CHECK)376 cli.timeout_ops = 10377 self.info("Preparing boot system config")378 new_boot_lines = []379 old_boot_lines = output[0].result.splitlines()380 self.debug(f"Orginal boot lines:\n{old_boot_lines}")381 for line in old_boot_lines:382 new_boot_lines.append(f"no {line}")383 new_boot_lines.append(f"boot system {self.file_system}/{self.target_image}")384 if len(old_boot_lines):385 new_boot_lines.append(old_boot_lines[0])386 self.debug(f"New boot lines:\n{new_boot_lines}")387 output = cli.send_configs(new_boot_lines)388 self.debug(f"Changnig Boot vars:\n{output.result}")389 if output.failed:390 msg = "Unable to change bootvar"391 self.error(msg)392 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)393 else:394 self.info("Bootvar was changed")395 self.info("Write memory before reload")396 try:397 output = cli.send_command("write memory")398 except (ScrapliTimeout, ScrapliConnectionError):399 self.info("Interactive prompt was detected")400 time.sleep(2)401 cli.open()402 try:403 output_tmp = cli.send_interactive(404 [405 ("write", "[confirm]", False),406 ("\n", "#", False),407 ]408 )409 except (ScrapliTimeout, ScrapliConnectionError):410 msg = "Unable to write memory: ScrapliTimeout"411 self.error(msg)412 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)413 else:414 output = output_tmp415 if re.search(r"\[OK\]", output.result):416 self.info("Config was saved")417 else:418 msg = "Can not save config"419 self.error(msg)420 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)421 self.info("Reloading the box")422 try:423 output = cli.send_interactive(424 [425 ("reload in 1", "[confirm]", False),426 ("\n", "#", False),427 ]428 )429 except ScrapliTimeout:430 msg = "Unable to reload: ScrapliTimeout"431 self.error(msg)432 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)433 else:434 self.info("Reload was requested")435 try:436 cli.close()437 except Exception:438 pass439 return True440 def post_check(self):441 cmd = [442 "show version",443 ]444 cli = self.connect_cli()445 if not cli:446 msg = "Unable to connect to the device"447 self.error(msg)448 self.drop_task(msg, TaskFailReasonChoices.FAIL_CONNECT)449 output = cli.send_commands(cmd)450 self.debug(f"Commands output\n{output.result}")451 if output.failed:452 msg = "Can not collect outputs for post-chech"453 self.error(msg)454 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)455 parsed = output[0].textfsm_parse_output()456 self.info(f'New version is {parsed[0].get("version", "N/A")}')457 self.info("Write memory after reload")458 try:459 output = cli.send_command("write memory")460 except (ScrapliTimeout, ScrapliConnectionError):461 self.info("Interactive prompt was detected")462 time.sleep(2)463 cli.open()464 try:465 output_tmp = cli.send_interactive(466 [467 ("write", "[confirm]", False),468 ("\n", "#", False),469 ]470 )471 except (ScrapliTimeout, ScrapliConnectionError):472 msg = "Unable to write memory: ScrapliTimeout"473 self.error(msg)474 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)475 else:476 output = output_tmp477 if re.search(r"\[OK\]", output.result):478 self.info("Config was saved")479 else:480 msg = "Can not save config"481 self.error(msg)482 self.drop_task(msg, TaskFailReasonChoices.FAIL_UPGRADE)483 cli.close()484 self.info("Update custom field")485 self.task.device.custom_field_data[CF_NAME_SW_VERSION] = parsed[0].get("version", "N/A")486 self.task.device.save()487 self.info("Post-checks have been done")488 return True489 def execute_task(self):490 self.info(f"New Job {self.task.job_id} was started. Type {self.task.task_type}")491 self.info("Initial task checking...")492 self.check()493 self.info("Initial task check has been completed")494 self.info("Checking if device alive...")495 if not self.is_alive():496 msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is not reachable"497 self.warning(msg)498 self.skip_task(msg, TaskFailReasonChoices.FAIL_CONNECT)499 else:500 msg = f"Device {self.task.device.name}:{self.task.device.primary_ip.address.ip} is reachable"501 self.info(msg)502 self.info("Device valiation...")503 self.check_device()504 self.info("Device has been validated")505 if self.task.task_type == TaskTypeChoices.TYPE_UPLOAD:506 self.info("Uploadng image on the box...")507 self.file_upload()508 elif self.task.task_type == TaskTypeChoices.TYPE_UPGRADE:509 self.info("Reloading the box...")510 self.device_reload()511 hold_timer = 240512 self.info(f"Hold for {hold_timer} seconds")...
bot.py
Source:bot.py
...11from accounts import ACCOUNTS_LIST # list of api_ids and api_hashes12from messages import get_bot_dialogue, balance, withdraw_all13import keyboard14programm_is_running = True # changed by ctrl + shift + ] hotkey, exit while loop if false15async def skip_task(phone, err_msg, arg):16 '''17 Skips task(not always really skips)18 arg -- dictionary, {'tl_bot_chat': bot_chat_variable, 'msg': last_message(not string, but message object),19 'really_skip':True or False(read explanation), 'client': telethon_telgram_client}20 really_skip - some exceptions that occurs while getting on page do not mean that reward won't be given.21 I discovered that only captcha do not allow bot to get reward(see in captcha exception in main()). 22 So if variable is True(only captcha case) we really skip on new task if opposite, wait for default amount23 of seconds and go forward(if we had to wait more, main() will help by waiting new message)24 '''25 print(phone + ': ' + err_msg)26 if arg['really_skip']:27 await arg['client'](GetBotCallbackAnswerRequest(arg['tl_bot_chat'], arg['msg'].id, data=arg['msg'].reply_markup.rows[1].buttons[1].data))28 await asyncio.sleep(3)29 else:30 #define how long to wait31 msg = await arg['client'].get_messages(arg['tl_bot_chat'], limit=1)32 msg = msg[0]33 wait = str(msg.message).replace('You must stay on the site for', '').replace('seconds to get your reward.', '')34 wait = str(wait).replace('Please stay on the site for at least', '').replace('seconds...', '')35 try:36 wait = int(wait)37 except ValueError:38 print('Can not define how long to wait. Standart value is 15 seconds')39 wait = 1040 print('Wait for 15 seconds')41 await asyncio.sleep(wait + 5) # +5 for insurance42 43 print(phone + ': ' + 'Skipped!')44 45 46async def main(browser, accounts_list_slice):47 '''48 Main function where we choose account and talk to bot, then change when we need and repeat49 args:50 browser -- webdriver browser variable51 accounts_list_slice -- list of accounts that change each other if needed(when there is no task for very long)52 '''53 print('Main started...')54 start_session = time.time() # for calculating session duration55 global programm_is_running56 no_ads_iterator = 0 # count how many times there were no ads error57 account_iterator = 0 # read name58 while programm_is_running:59 #log into account60 log_data = accounts_list_slice[account_iterator]61 client = await tl.TelegramClient(log_data['phone'], log_data['api_id'], log_data['api_hash']).start()62 print('**Loged in as: ' + log_data['phone'] + '**')63 PREFIX_ID = log_data['phone'] + ': ' # should be in every print(), shows what number message relates to64 #get ltc bot dialogue65 tl_bot_chat = await get_bot_dialogue(client)66 print(PREFIX_ID + 'Found a LTC Click Bot chat')67 #get new link68 await client.send_message(tl_bot_chat, '/visit')69 print(PREFIX_ID + 'First /visit sent')70 71 #previous setup72 old_msg = None73 msg = await client.get_messages(tl_bot_chat, limit=1)74 msg = msg[0]75 for_skip_task = {'tl_bot_chat': tl_bot_chat, 'msg': None, 'really_skip':False, 'client': client}76 while programm_is_running:77 #wait if bot is lagging78 await asyncio.sleep(2)79 if re.search(r'there are no new ads available', msg.message) and programm_is_running:80 #if there is mo more ad81 no_ads_iterator += 1 #increment82 if no_ads_iterator >= 5:83 #if there is no ads for 5 times -> change account84 print(PREFIX_ID + 'No ads for this account now. Changing account...')85 account_iterator += 186 if account_iterator >= len(accounts_list_slice):87 account_iterator = 088 no_ads_iterator = 0 # new account starts at 0 no_ads_iterator89 await client.disconnect()90 break91 found_task = False92 #try for 5 more times93 print(PREFIX_ID + 'No ads observed. It may be a lie. Try /visit for 5 times in a row')94 for i in range(5):95 await client.send_message(tl_bot_chat, '/visit')96 await asyncio.sleep(2)97 msg = await client.get_messages(tl_bot_chat, limit=1)98 msg = msg[0]99 if not re.search(r'there are no new ads available', msg.message):100 #if found task break out of this function and go to the website101 print(PREFIX_ID + 'Found')102 found_task = True103 break104 print(PREFIX_ID + '#{} - No ads'.format(i))105 if not found_task:106 #if bot really do not have tasks for this account107 print(PREFIX_ID + 'Threre is no ad for {} times\nIf there will be no ad for {} times then change account'.format(no_ads_iterator, 5 - no_ads_iterator))108 print(PREFIX_ID + 'There is no more new ad. Sleep for 1 minute')109 print(PREFIX_ID + 'For exit press: ctrl + shift + ]')110 await asyncio.sleep(60) # sleep for a minute and check111 await client.send_message(tl_bot_chat, '/visit')112 print(PREFIX_ID + 'Get up and work!')113 await asyncio.sleep(2)114 115 #set time point before loop116 time_start = time.time()117 #reset msg118 msg = await client.get_messages(tl_bot_chat, limit=1) 119 msg = msg[0]120 #get message121 while msg == old_msg and programm_is_running:122 # exit only if new message123 msg = await client.get_messages(tl_bot_chat, limit=1)124 msg = msg[0]125 await asyncio.sleep(1)126 #check if we have waited for new message for too long127 if time.time() - time_start >= 95:128 try: 129 for_skip_task['msg'] = msg130 for_skip_task['really_skip'] = True131 await skip_task(log_data['phone'], 'There is no new message for too long', for_skip_task)132 for_skip_task['really_skip'] = False133 except AttributeError:134 print(PREFIX_ID + 'Last message was not a link')135 await client.send_message(tl_bot_chat, '/visit')136 await asyncio.sleep(5)137 msg = await client.get_messages(tl_bot_chat, limit=1)138 msg = msg[0]139 for_skip_task['msg'] = msg140 for_skip_task['really_skip'] = True141 try:142 await skip_task(log_data['phone'], 'Try skip for one more time', for_skip_task)143 except AttributeError:144 print(PREFIX_ID + 'Failed one more time')145 break146 finally:147 for_skip_task['really_skip'] = False148 #set new old_msg149 old_msg = msg150 # if got a url151 if re.search(r'Press', msg.message) and programm_is_running:152 no_ads_iterator = 0 # count how many times there were no ads error153 print(PREFIX_ID + 'Ad message sent: {}'.format(msg.date))154 visit_url = msg.reply_markup.rows[0].buttons[0].url155 print(PREFIX_ID + 'Ad URL: ' + visit_url)156 for_skip_task['msg'] = msg # for exceptions157 try: 158 #check for captcha159 url_site = urllib.request.urlopen(Request(visit_url, headers={'User-Agent' : 'Mozilla/5.0'}))160 captcha_str = url_site.read().decode('utf-8')161 url_site.close()162 if not re.search(r'reCAPTCHA', captcha_str):163 #go to URL164 browser.get(visit_url)165 print(PREFIX_ID + 'Page was opened succesfully\n\n')166 else:167 for_skip_task['really_skip'] = True168 await skip_task(log_data['phone'], 'Captcha was found on site. Skipping...', for_skip_task)169 for_skip_task['really_skip'] = False170 except selenium.common.exceptions.TimeoutException:171 await skip_task(log_data['phone'], 'Page loading timeout. Skipping...', for_skip_task)172 except TimeoutError:173 for_skip_task['really_skip'] = True174 await skip_task(log_data['phone'], 'Socket timeout. Skipping...', for_skip_task)175 for_skip_task['really_skip'] = False176 except ConnectionResetError:177 await skip_task(log_data['phone'], 'Connection reset. Skipping...', for_skip_task)178 except ConnectionRefusedError:179 await skip_task(log_data['phone'], 'Connection refused. Skipping...', for_skip_task)180 except urllib.error.HTTPError as e:181 await skip_task(log_data['phone'], 'Can not access the site. Skipping...' + str(e.reason), for_skip_task)182 except urllib.error.URLError:183 await skip_task(log_data['phone'], 'Bad certificate. Skipping...', for_skip_task)184 except UnicodeDecodeError:185 await skip_task(log_data['phone'], 'Can not decode text for captcha check')186 elif re.search(r'no longer valid', msg.message):187 # if skipped or some error appeared188 await client.send_message(tl_bot_chat, '/visit')189 190 print('Disconnecting with current client')191 await client.disconnect() 192def browser_setup():193 '''194 Setups one browser and returns browser variable195 '''196 #make browser headless197 print('Starting headless browser')198 options = webdriver.firefox.options.Options()199 options.headless = True # True or False...
worker.py
Source:worker.py
1# Copyright 2016 The Chromium Authors. All rights reserved.2# Use of this source code is governed by a BSD-style license that can be3# found in the LICENSE file.4import argparse5import json6import logging7import os8import random9import sys10import time11from googleapiclient import discovery12from oauth2client.client import GoogleCredentials13# NOTE: The parent directory needs to be first in sys.path to avoid conflicts14# with catapult modules that have colliding names, as catapult inserts itself15# into the path as the second element. This is an ugly and fragile hack.16_CLOUD_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),17 os.pardir)18sys.path.insert(0, os.path.join(_CLOUD_DIR, os.pardir))19# Add _CLOUD_DIR to the path to access common code through the same path as the20# frontend.21sys.path.append(_CLOUD_DIR)22from common.clovis_task import ClovisTask23import common.google_bigquery_helper24from common.google_instance_helper import GoogleInstanceHelper25from clovis_task_handler import ClovisTaskHandler26from failure_database import FailureDatabase27from google_storage_accessor import GoogleStorageAccessor28class Worker(object):29 def __init__(self, config, logger):30 """See README.md for the config format."""31 self._project_name = config['project_name']32 self._taskqueue_tag = config['taskqueue_tag']33 self._src_path = config['src_path']34 self._instance_name = config.get('instance_name')35 self._worker_log_path = config.get('worker_log_path')36 self._credentials = GoogleCredentials.get_application_default()37 self._logger = logger38 self._self_destruct = config.get('self_destruct')39 if self._self_destruct and not self._instance_name:40 self._logger.error('Self destruction requires an instance name.')41 # Separate the task storage path into the bucket and the base path under42 # the bucket.43 storage_path_components = config['task_storage_path'].split('/')44 self._bucket_name = storage_path_components[0]45 self._base_path_in_bucket = ''46 if len(storage_path_components) > 1:47 self._base_path_in_bucket = '/'.join(storage_path_components[1:])48 if not self._base_path_in_bucket.endswith('/'):49 self._base_path_in_bucket += '/'50 self._google_storage_accessor = GoogleStorageAccessor(51 credentials=self._credentials, project_name=self._project_name,52 bucket_name=self._bucket_name)53 if self._instance_name:54 failure_database_filename = \55 'failure_database_%s.json' % self._instance_name56 else:57 failure_database_filename = 'failure_dabatase.json'58 self._failure_database_path = os.path.join(self._base_path_in_bucket,59 failure_database_filename)60 # Recover any existing failures in case the worker died.61 self._failure_database = self._GetFailureDatabase()62 if self._failure_database.ToJsonDict():63 # Script is restarting after a crash, or there are already files from a64 # previous run in the directory.65 self._failure_database.AddFailure(FailureDatabase.DIRTY_STATE_ERROR,66 'failure_database')67 bigquery_service = common.google_bigquery_helper.GetBigQueryService(68 self._credentials)69 self._clovis_task_handler = ClovisTaskHandler(70 self._project_name, self._base_path_in_bucket, self._failure_database,71 self._google_storage_accessor, bigquery_service,72 config['binaries_path'], config['ad_rules_filename'],73 config['tracking_rules_filename'], self._logger, self._instance_name)74 self._UploadFailureDatabase()75 def Start(self):76 """Main worker loop.77 Repeatedly pulls tasks from the task queue and processes them. Returns when78 the queue is empty.79 """80 task_api = discovery.build('taskqueue', 'v1beta2',81 credentials=self._credentials)82 queue_name = 'clovis-queue'83 # Workaround for84 # https://code.google.com/p/googleappengine/issues/detail?id=1019985 project = 's~' + self._project_name86 while True:87 self._logger.debug('Fetching new task.')88 (clovis_task, task_id) = self._FetchClovisTask(project, task_api,89 queue_name)90 if not clovis_task:91 break92 self._logger.info('Processing task %s' % task_id)93 self._clovis_task_handler.Run(clovis_task)94 self._UploadFailureDatabase()95 self._logger.debug('Deleting task %s' % task_id)96 task_api.tasks().delete(project=project, taskqueue=queue_name,97 task=task_id).execute()98 self._logger.info('Finished task %s' % task_id)99 self._Finalize()100 def _GetFailureDatabase(self):101 """Downloads the failure database from CloudStorage."""102 self._logger.info('Downloading failure database')103 failure_database_string = self._google_storage_accessor.DownloadAsString(104 self._failure_database_path)105 return FailureDatabase(failure_database_string)106 def _UploadFailureDatabase(self):107 """Uploads the failure database to CloudStorage."""108 if not self._failure_database.is_dirty:109 return110 self._logger.info('Uploading failure database')111 self._google_storage_accessor.UploadString(112 self._failure_database.ToJsonString(),113 self._failure_database_path)114 self._failure_database.is_dirty = False115 def _FetchClovisTask(self, project_name, task_api, queue_name):116 """Fetches a ClovisTask from the task queue.117 Params:118 project_name(str): The name of the Google Cloud project.119 task_api: The TaskQueue service.120 queue_name(str): The name of the task queue.121 Returns:122 (ClovisTask, str): The fetched ClovisTask and its task ID, or (None, None)123 if no tasks are found.124 """125 response = task_api.tasks().lease(126 project=project_name, taskqueue=queue_name, numTasks=1, leaseSecs=600,127 groupByTag=True, tag=self._taskqueue_tag).execute()128 if (not response.get('items')) or (len(response['items']) < 1):129 return (None, None) # The task queue is empty.130 google_task = response['items'][0]131 task_id = google_task['id']132 # Delete the task without processing if it already failed multiple times.133 # TODO(droger): This is a workaround for internal bug b/28442122, revisit134 # once it is fixed.135 retry_count = google_task['retry_count']136 max_retry_count = 3137 skip_task = retry_count >= max_retry_count138 if skip_task:139 task_api.tasks().delete(project=project_name, taskqueue=queue_name,140 task=task_id).execute()141 clovis_task = ClovisTask.FromBase64(google_task['payloadBase64'])142 if retry_count > 0:143 self._failure_database.AddFailure('task_queue_retry',144 clovis_task.ToJsonString())145 self._UploadFailureDatabase()146 if skip_task:147 return self._FetchClovisTask(project_name, task_api, queue_name)148 return (clovis_task, task_id)149 def _Finalize(self):150 """Called before exiting."""151 self._logger.info('Done')152 self._clovis_task_handler.Finalize()153 # Upload the worker log.154 if self._worker_log_path:155 self._logger.info('Uploading worker log.')156 remote_log_path = os.path.join(self._base_path_in_bucket, 'worker_log')157 if self._instance_name:158 remote_log_path += '_' + self._instance_name159 self._google_storage_accessor.UploadFile(self._worker_log_path,160 remote_log_path)161 # Self destruct.162 if self._self_destruct:163 # Workaround for ComputeEngine internal bug b/28760288.164 random_delay = random.random() * 600.0 # Up to 10 minutes.165 self._logger.info(166 'Wait %.0fs to avoid load spikes on compute engine.' % random_delay)167 time.sleep(random_delay)168 self._logger.info('Starting instance destruction: ' + self._instance_name)169 google_instance_helper = GoogleInstanceHelper(170 self._credentials, self._project_name, self._logger)171 success = google_instance_helper.DeleteInstance(self._taskqueue_tag,172 self._instance_name)173 if not success:174 self._logger.error('Self destruction failed.')175 # Do not add anything after this line, as the instance might be killed at176 # any time.177if __name__ == '__main__':178 parser = argparse.ArgumentParser(179 description='ComputeEngine Worker for Clovis')180 parser.add_argument('--config', required=True,181 help='Path to the configuration file.')182 args = parser.parse_args()183 # Configure logging.184 logging.basicConfig(level=logging.WARNING,185 format='[%(asctime)s][%(levelname)s] %(message)s',186 datefmt='%y-%m-%d %H:%M:%S')187 logging.Formatter.converter = time.gmtime188 worker_logger = logging.getLogger('worker')189 worker_logger.setLevel(logging.INFO)190 worker_logger.info('Reading configuration')191 with open(args.config) as config_json:192 worker = Worker(json.load(config_json), worker_logger)...
task.py
Source:task.py
...70 if isinstance(dep_task.result, (TaskResultFailure, TaskResultSkipped)):71 reason = dep_task.result.reason72 else:73 reason = None74 skip_task(task, context, completed_task_queue, reason)75 return76 # skip task on external trigger if any77 skip_reason = context.is_task_to_be_skipped(task)78 if skip_reason:79 skip_task(task, context, completed_task_queue, reason=skip_reason)80 return81 # run task when all conditions are met82 run_task(task, context, completed_task_queue)83def schedule_tasks_to_be_run(tasks, context, pool, completed_tasks_queue):84 for task in tasks:85 pool.apply_async(handle_task, args=(task, context, completed_tasks_queue))86def skip_task(task, context, completed_task_queue, reason=""):87 _debug("skip task %s" % task)88 try:89 task.skip(context, reason)90 except Exception:91 task.result = TaskResultException(serialize_current_exception())92 else:93 task.result = TaskResultSkipped(reason)94 completed_task_queue.put(task)95def skip_all_tasks(tasks, remaining_tasks, completed_tasks, context, pool, completed_tasks_queue, reason):96 # schedule all tasks to be skipped...97 for task in remaining_tasks:98 pool.apply_async(skip_task, args=(task, context, completed_tasks_queue, reason))99 # ... and wait for their completion100 while len(completed_tasks) != len(tasks):...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run Lemoncheesecake automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
