How to use proxy_handler method in devtools-proxy

Best Python code snippet using devtools-proxy_python

xicidai_spider.py

Source:xicidai_spider.py Github

copy

Full Screen

1#coding:utf82import urllib2,urllib3import cookielib4import re5import random6# 设置是否开启代理池 0: 花钱代理 1 使用免费代理 2 不实用代理7proxy_switch = 18max_retry = 59# 创建cookiehandler10cookie = cookielib.CookieJar()11cookie_handler = urllib2.HTTPCookieProcessor(cookie)12# 创建代理handler13# proxy_handler = urllib2.ProxyHandler({'http' : '120.27.125.149:16816'})14def getProxyHandler(proxy_list):15 if proxy_switch == 0:16 proxy_handler = urllib2.ProxyHandler(17 {18 'http' : 'http://1752570559:wd0p04kd@120.27.125.149:16816',19 'https' : 'http://1752570559:wd0p04kd@120.27.125.149:16816'20 }21 ) # 使用私密代理22 elif proxy_switch == 1:23 # 免费代理，从爬取的代理池中读取24 print '用免费代理'25 # with open('proxy.txt','r') as f:26 #27 # proxy_list = f.readlines() #从文件中读取的代理列表28 # print proxy_list29 # proxy_poll = [] # 代理池30 # for proxy in proxy_list:31 # res = proxy.split(' ')32 # proxy_poll.append( #向代理池中加入一个代理33 # {34 # 'http' : res[2].lower() + '://' + res[0] + ':' + res[1],35 # 'https' : res[2].lower() + '://' + res[0] + ':' + res[1],36 # }37 # )38 print proxy_list39 proxy_poll = [] # 代理池40 for proxy in proxy_list:41 res = proxy.split(' ')42 proxy_poll.append( #向代理池中加入一个代理43 {44 'http' : res[2].lower() + '://' + res[0] + ':' + res[1],45 'https' : res[2].lower() + '://' + res[0] + ':' + res[1],46 }47 )48 if proxy_poll: #如果有代理则随机 ，否则不实用代理49 proxy = random.choice(proxy_poll)50 else:51 proxy = {}52 print proxy53 proxy_handler = urllib2.ProxyHandler(proxy)54 else:55 proxy_handler = urllib2.ProxyHandler() # 使用私密代理56 return proxy_handler57def getOpener(proxy_handler):58 opener = urllib2.build_opener(proxy_handler)59 # urllib2.install_opener(opener) # urllib2.urlopen时候都用这个opener去发请求60 urllib2.install_opener(opener)61 opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36')]62 return opener63def getPage():64 base_url = 'http://www.xicidaili.com/nn/'65 # base_url = 'http://www.baidu.com/s?wd=ip'66 # response = opener.open(base_url)67 # print response.read()68 # exit()69 start = raw_input('起始：')70 end = raw_input('结束：')71 with open('proxy.txt','r') as f:72 proxy_list = f.readlines() # 打开时候一定不能先清空73 with open('proxy.txt', 'w+') as f:74 for i in range(int(start), int(end)):75 fullurl = base_url + str(i)76 for i in range(max_retry): #循环去尝试open77 try:78 proxy_handler = getProxyHandler(proxy_list)79 opener = getOpener(proxy_handler)80 response = opener.open(fullurl,timeout=3)81 break82 except (urllib2.URLError,Exception) as e:83 response = None84 print str(e)85 if response is not None: # 多次尝试成功以后就进行html读取86 html = response.read()87 else:88 print '代理不可用，请检查！！'89 exit()90 # print html91 tr_pattern = re.compile(r'<tr.*?>.*?' + r'(?:<td.*?>(.*?)</td>.*?)' * 10 + r'.*?</tr>' ,re.S)92 tr_list = tr_pattern.findall(html)93 speed_pattern = re.compile(r'title="(.*?)"')94 for tr in tr_list:95 # 过滤响应小于2秒的96 res = speed_pattern.search(tr[6])97 speed = res.group(1)98 speed = speed.replace('秒','')99 # 存活期大于1天以上的100 if float(speed) < 2 and '天' in tr[8]:101 print 'crawling : %s' % tr[1]102 f.write("%s %s %s %s %s\n" % (tr[1],tr[2],tr[5],speed,tr[8],))103if __name__ == '__main__':104 getPage()...

Full Screen

Full Screen

compat.py

Source:compat.py Github

copy

Full Screen

1# -*- coding: utf-8 -*-2# flake8: noqa3"""Python 2/3 compatibility support."""4import sys5PY2 = sys.version_info[0] == 26PY3 = sys.version_info[0] == 37PY33 = sys.version_info[0:2] >= (3, 3)8if PY2:9 reload(sys)10 sys.setdefaultencoding('utf8')11 import urllib212 from urllib import urlencode13 from urllib2 import URLError14 from urllib2 import quote15 from urllib2 import unquote16 from urllib2 import urlopen17 from urlparse import parse_qsl18 from HTMLParser import HTMLParser19 def install_proxy(proxy_handler):20 """21 install global proxy.22 :param proxy_handler:23 :samp:`{"http":"http://my.proxy.com:1234", "https":"https://my.proxy.com:1234"}`24 :return:25 """26 proxy_support = urllib2.ProxyHandler(proxy_handler)27 opener = urllib2.build_opener(proxy_support)28 urllib2.install_opener(opener)29 def unescape(s):30 """Strip HTML entries from a string."""31 html_parser = HTMLParser()32 return html_parser.unescape(s)33 def unicode(s):34 """Encode a string to utf-8."""35 return s.encode('utf-8')36elif PY3:37 from urllib.error import URLError38 from urllib.parse import parse_qsl39 from urllib.parse import quote40 from urllib.parse import unquote41 from urllib.parse import urlencode42 from urllib.request import urlopen43 from urllib import request44 def install_proxy(proxy_handler):45 proxy_support = request.ProxyHandler(proxy_handler)46 opener = request.build_opener(proxy_support)47 request.install_opener(opener)48 def unicode(s):49 """No-op."""50 return s51 if PY33:52 from html.parser import HTMLParser53 def unescape(s):54 """Strip HTML entries from a string."""55 html_parser = HTMLParser()56 return html_parser.unescape(s)57 else:...

Full Screen

Full Screen

test_proxy.py

Source:test_proxy.py Github

copy

Full Screen

1#coding=utf-82import urllib23#不使用代理4import urllib25response = urllib2.urlopen('http://101.201.175.89/dist/#/login')6html = response.read()7print("#不使用代理")8print(html)9#使用代理访问10proxy_handler = urllib2.ProxyHandler({'http': '124.232.163.39:3128'})11opener = urllib2.build_opener(proxy_handler)12r = opener.open('http://101.201.175.89/dist/#/login')13print("#使用代理")14print(r.read())15# read16# urllib2是Python标准库，功能很强大，只是使用起来稍微麻烦一点。在Python 3中，urllib2不再保留，迁移到了urllib模块中。urllib2中通过ProxyHandler来设置使用代理服务器。17#18# proxy_handler = urllib2.ProxyHandler({'http': '121.193.143.249:80'})19# opener = urllib2.build_opener(proxy_handler)20# r = opener.open('http://httpbin.org/ip')21# print(r.read())22# 也可以用install_opener将配置好的opener安装到全局环境中，这样所有的urllib2.urlopen都会自动使用代理。23#24# urllib2.install_opener(opener)25# r = urllib2.urlopen('http://httpbin.org/ip')26# print(r.read())27# 在Python 3中，使用urllib。28#29# proxy_handler = urllib.request.ProxyHandler({'http': 'http://121.193.143.249:80/'})30# opener = urllib.request.build_opener(proxy_handler)31# r = opener.open('http://httpbin.org/ip')...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run devtools-proxy automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful