Best Python code snippet using tempest_python
scrape.py
Source:scrape.py
1import concurrent.futures2import os3import html2text4import operator5import requests6from bs4 import BeautifulSoup7from jinja2 import Template8import yaml9from dateutil.parser import parse as date_parse10with open("templates/episode.md.j2") as f:11 TEMPLATE = Template(f.read())12def mkdir_safe(directory):13 try:14 os.mkdir(directory)15 except FileExistsError:16 pass17def get_list(soup, pre_title):18 """19 Blocks of links are preceded by a `p` saying what it is.20 """21 pre_element = soup.find("p", string=pre_title)22 if pre_element is None:23 return None24 return pre_element.find_next_sibling("ul")25def get_duration(seconds):26 minutes, seconds = divmod(seconds, 60)27 return f"{minutes} mins {seconds} secs"28def get_plain_title(title: str):29 """30 Get just the show title, without any numbering etc31 """32 # Remove number before colon33 title = title.split(":", 1)[-1]34 # Remove data after the pipe35 title = title.rsplit("|", 1)[0]36 # Strip any stray spaces37 return title.strip()38def create_episode(api_episode, show_config, output_dir):39 base_url = show_config['fireside_url']40 # RANT: What kind of API doesn't give the episode number?!41 episode_number = int(api_episode["url"].split("/")[-1])42 episode_number_padded = f"{episode_number:03}"43 publish_date = date_parse(api_episode['date_published'])44 output_file = f"{output_dir}/{publish_date.year}/episode-{episode_number_padded}.md"45 mkdir_safe(f"{output_dir}/{publish_date.year}")46 if os.path.isfile(output_file):47 print("Skipping", api_episode['url'], "as it already exists")48 return49 api_soup = BeautifulSoup(api_episode["content_html"], "html.parser")50 blurb = api_episode["summary"]51 sponsors = html2text.html2text(str(get_list(api_soup, "Sponsored By:")))52 links = html2text.html2text(str(get_list(api_soup, "Links:") or get_list(api_soup, "Episode Links:")))53 page_soup = BeautifulSoup(requests.get(api_episode["url"]).content, "html.parser")54 tags = []55 for link in page_soup.find_all("a", class_="tag"):56 tags.append(57 {"link": base_url + link.get("href"), "text": link.get_text().strip()}58 )59 # Sort tags by text60 tags = sorted(tags, key=operator.itemgetter("text"))61 hosts = []62 for host in page_soup.find_all("ul", class_="episode-hosts"):63 for link in host.find_all("a"):64 hosts.append(65 {"name": link.get("title"), "link": base_url + link.get("href")}66 )67 player_embed = page_soup.find("input", class_="copy-share-embed").get("value")68 show_attachment = api_episode["attachments"][0]69 output = TEMPLATE.render(70 {71 "title": api_episode["title"],72 "title_plain": get_plain_title(api_episode["title"]),73 "episode_number": episode_number,74 "episode_number_padded": episode_number_padded,75 "url": api_episode["url"],76 "audio": show_attachment["url"],77 "duration": get_duration(int(show_attachment['duration_in_seconds'])),78 "blurb": blurb,79 "sponsors": sponsors,80 "links": links,81 "hosts": hosts,82 "tags": tags,83 "player_embed": player_embed,84 "date_published": publish_date.date().isoformat(),85 "show_config": show_config86 }87 )88 with open(output_file, "w") as f:89 print("Saving", api_episode["url"])90 f.write(output)91def main():92 # Grab the config embedded in the mkdocs config93 with open("mkdocs.yml") as f:94 shows = yaml.load(f, Loader=yaml.SafeLoader)['extra']['shows']95 with concurrent.futures.ThreadPoolExecutor() as executor:96 futures = []97 for show_slug, show_config in shows.items():98 output_dir = f"docs/{show_slug}"99 mkdir_safe(output_dir)100 api_data = requests.get(show_config['fireside_url'] + "/json").json()101 for api_episode in api_data["items"]:102 futures.append(executor.submit(create_episode, api_episode, show_config, output_dir))103 # Drain to get exceptions. Still have to mash CTRL-C, though.104 for future in concurrent.futures.as_completed(futures):105 future.result()106if __name__ == "__main__":...
postrules.py
Source:postrules.py
1from __future__ import annotations2from typing import Optional, Union, List, Dict, Any3from dataclasses import dataclass4from .utils import flatten_list5PostRules = Optional[Dict[int, "PostRule"]]6@dataclass(frozen=True)7class PostRule:8 expand_quote_links: Optional[Union[bool, List[int]]] = None9 appended: Optional[List[int]] = None10 show_attachment: Optional[bool] = None11 @staticmethod12 def load_from_object(obj: Optional[Dict[str, Any]]) -> PostRule:13 obj = obj or dict()14 expand_quote_links = obj.get("expand-quote-links", None)15 if type(expand_quote_links) is int:16 expand_quote_links = [expand_quote_links]17 appended = obj.get("appended", None)18 if type(appended) is int:19 appended = [appended]20 elif isinstance(appended, list):21 appended = flatten_list(appended)22 return PostRule(23 expand_quote_links=expand_quote_links,24 appended=appended,25 show_attachment=obj.get("show-attachment", True),26 )27 @staticmethod28 def merge(old: PostRule, new: PostRule) -> PostRule:29 rule_dict = None30 if old != None:31 rule_dict = dict(old.__dict__)32 if new != None:33 if rule_dict != None:34 new = [(k, v)35 for k, v in new.__dict__.items() if v != None]36 rule_dict.update(new)37 else:38 rule_dict = dict(new.__dict__)39 if rule_dict != None:40 return PostRule(**rule_dict)...
39ded03760da_add_column_show_atta.py
Source:39ded03760da_add_column_show_atta.py
1"""add column show_attachment2Revision ID: 39ded03760da3Revises: None4Create Date: 2012-06-27 14:26:30.8260765"""6# revision identifiers, used by Alembic.7revision = '39ded03760da'8down_revision = None9from alembic import op10import sqlalchemy as sa11from sqlalchemy.sql import table, column12contact_form = table('contact_forms', column('show_attachment', sa.Boolean))13def upgrade():14 op.add_column('contact_forms', sa.Column('show_attachment', sa.Boolean))15 op.execute(contact_form.update().values({'show_attachment': True}))16 op.alter_column('contact_forms', 'show_attachment', nullable=False)17def downgrade():...
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Get 100 minutes of automation test minutes FREE!!