TestMu Conference is Back! Future of Testing + AIRegister for Free

How to use block_ads method in SeleniumBase

Best Python code snippet using SeleniumBase

AdBlock(Desktop_FireFox).py

Source:AdBlock(Desktop_FireFox).py

1# æ£å¼æ¸¬è©¦2# ç¼ºé»ï¼MITMèéURLsåæå»£ååè½çç¨å¼ç¢¼åå¨ä¸èµ·å¯«ï¼å°æªæ¨¡çµå3from adblockparser import AdblockRules4from browsermobproxy import Server5from selenium import webdriver6from selenium.common.exceptions import TimeoutException7from urllib.parse import urlparse8import time9import os10import pprint11class ProxyManger:12    __BMP = "/home/kimbelly/Tools/browsermob-proxy-2.1.4/bin/browsermob-proxy"13    def __init__(self):14        self.__server = Server(ProxyManger.__BMP)15        self.__client = None16    def start_server(self):17        self.__server.start()18        return self.__server19    def start_client(self):20        self.__client = self.__server.create_proxy(params={"trustAllServers": "true"})21        return self.__client22    @property23    def client(self):24        return self.__client25    @property26    def server(self):27        return self.__server28if "__main__" == __name__:29    proxy = ProxyManger()30    server = proxy.start_server()31    client = proxy.start_client()32    profile = webdriver.FirefoxProfile()33    profile.set_proxy(client.selenium_proxy())34    profile.set_preference("browser.cache.disk.enable", False)35    profile.set_preference("browser.cache.memory.enable", False)36    profile.set_preference("browser.cache.offline.enable", False)37    # Disable download popup38    profile.set_preference("browser.download.panel.shown", False)39    profile.set_preference("browser.download.animateNotifications", False)40    profile.set_preference("browser.download.show_plugins_in_list", False)41    profile.set_preference("dom.popup_maximum", 0)42    # profile.set_preference("network.http.use-cache", False)43    driver = webdriver.Firefox(executable_path='/home/kimbelly/Tools/geckodriver', firefox_profile=profile)44    driver.set_page_load_timeout(20)45    Path = "/home/kimbelly/Experiment/Block_Ads/FilterList/test/"46    file_list = os.listdir(Path)47    rules_dic = {}48    block_result_file = []49    i = 150    block_result_file.append(None)  # don't need block_result_file[0]51    for x in file_list:52        print(x + " = " + str(i))53        block_result_file.append(open("/home/kimbelly/Experiment/Block_Ads/Block_Result/Crawl_Result_5/Result" + str(i) + "_" + x + '.txt', 'w'))54        i += 155        with open(Path + x, 'rb') as blocklist_file:56            raw_rules = blocklist_file.read().decode('utf8').splitlines()57        rules_dic[x] = AdblockRules(raw_rules)58        blocklist_file.close()59    testURL_file = open('/home/kimbelly/Experiment/Crawl_TestURLs/Crawl_Result/Crawl_Result_5.txt', 'r')60    urls = testURL_file.read().splitlines()61    browseURL_count = 162    for url in urls:  # each test website (maybe contain ads)63        print(str(browseURL_count) + ". In the URL: " + url)64        domain = urlparse(url).netloc65        client.new_har(domain)66        try:67            driver.get(url)68            time.sleep(7)69        except TimeoutException as e:70            print(url + ": Page load timeout or Invalid URL... moving to next URL !!!")71        except:72            print("May be 'Reached error page' occurred")73        result = client.har74        url_arr = []75        url_set = []76        j = 177        for entry in result['log']['entries']:78            _url = entry['request']['url']79            url_arr.append(_url)80        url_set = list(set(url_arr))  # Delete duplicate urls81        block_flag = "False"82        ad_count = 083        for x in file_list:84            for elem in url_set:85                try:86                    block_flag = rules_dic[x].should_block(elem, {'script': True, 'image': True,87                                                                  'stylesheet': True, 'object': True,88                                                                  'xmlhttprequest': True, 'object-subrequest': True,89                                                                  'other': True, 'media': True, 'third-party': True})90                except:91                    print("Rule error......")92                    continue93                if block_flag:94                    # print("Block: " + elem)95                    block_result_file[j].write(elem + "\n")96                    ad_count += 197            print(x + ": Block #" + str(ad_count))98            ad_count = 099            j += 1100        browseURL_count += 1101    for k in range(1, len(block_result_file)):102        block_result_file[k].close()103    server.stop()...

AdBlock(All_Platform).py

Source:AdBlock(All_Platform).py

1# æ¨¡çµåï¼æèéHARåæå»£åä»¥èéAd URLsï¼éå©èåè½æéæå©åç¨å¼ç¢¼æªæ¡2from adblockparser import AdblockRules3from urllib.parse import urlparse4import os5# Prepare rules6Path = "/home/kimbelly/DataSet/Filter_Lists/test/"7file_list = os.listdir(Path)8rules_dic = {}9for x in file_list:10    print("Prepare: " + x)11    with open(Path + x, 'rb') as blocklist_file:12        raw_rules = blocklist_file.read().decode('utf8').splitlines()13    rules_dic[x] = AdblockRules(raw_rules)14    blocklist_file.close()15# Read HAR file16HAR_file = open('/home/kimbelly/DataSet/Get_HAR_Files/AndroidVersion/HAR_Uniq_1.txt', 'r')17urls = HAR_file.read().splitlines()18HAR_file.close()19# For each filter list, start to block20print("=====================================================")21for x in file_list:22    print("Use " + x + " to block ads:")23    block_result_file = open("/home/kimbelly/Analysis/Block_Ads/AndroidVersion/Block_Ads_Uniq_1/" + x + "_BlockResult",24                             'a+')25    ad_count = 026    for url in urls:27        block_flag = "False"28        try:29            block_flag = rules_dic[x].should_block(url, {'script': True, 'image': True, 'third-party': True,30                                                         'stylesheet': True, 'object': True, 'media': True,31                                                         'xmlhttprequest': True, 'object-subrequest': True,32                                                         'other': True,33                                                         'subdocument': True, 'background': True, 'xbl': True,34                                                         'ping': True, 'dtd': True, 'collapse': True,35                                                         'donottrack': True, 'websocket': True})36        except:37            print("Rule error......")38            continue39        if block_flag:40            block_result_file.write(url + "\n")41            ad_count += 142    print("Block: #" + str(ad_count))43    readMe_file = open("/home/kimbelly/Analysis/Block_Ads/AndroidVersion/Block_Ads_Uniq_1/ReadMe_1", 'a+')44    readMe_file.write(x + " block #" + str(ad_count) + '\n')45    block_result_file.close()46# Just test ~47'''with open("/home/kimbelly/DataSet/Filter_Lists/test/tmptest", 'rb') as blocklist_file:48    raw_rules = blocklist_file.read().decode('utf8').splitlines()49rules_dic = AdblockRules(raw_rules)50blocklist_file.close()51url = 'https://qq.com/ex?href=abc.com'52try:53    block_flag = rules_dic.should_block(url, {'script': True, 'image': True,'third-party': True,54                                                'stylesheet': True, 'object': True,'media': True,55                                                'xmlhttprequest': True, 'object-subrequest': True, 'other': True,56                                                'subdocument': True, 'background': True, 'xbl': True,57                                                'ping': True, 'dtd': True, 'collapse': True,58                                                'donottrack': True, 'websocket': True})59    if block_flag:60        print("yes")61    else:62        print("no")63except:...

httpbin.py

Source:httpbin.py

1# -*- coding: utf-8 -*-2from scrapy_zenscrape import ZenscrapeRequest3from scrapy import Spider4from httpbin.items import HttpbinItem5class HttpbinSpider(Spider):6    name = 'httpbin'7    start_urls = [8        'https://httpbin.org',9        'https://httpbin.org/headers?json',10    ]11    def start_requests(self):12        for url in self.start_urls:13            yield ZenscrapeRequest(url, params={14                # 'render': False,15                # 'block_ads': True,16                # 'block_resources': False,17                # 'premium': True,18                # 'location': 'fr',19                # 'wait_for': 5,20                # 'wait_for_css': '#swagger-ui',21            }, headers={22                # 'Accept-Language': 'En-US',23            }, cookies={24                # 'name_1': 'value_1',25            })26    def parse(self, response):27        body = response.body.decode(response.encoding)28        return HttpbinItem(29            url=response.url,30            status=response.status,31            body=body,32            headers=response.headers,...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.