Using Proxies in Python 3 - python-3.x

Question

i am using proxies to make crawlers:
import requests
from bs4 import BeautifulSoup
import time
time.sleep(3)
import random
proxy_list = [
'66.82.144.29:8080',
'47.75.0.253:3129',
'217.119.82.14:8080' ]
proxies = random.choice(proxy_list)
for i in range(20):
url = "https://www.amazon.com/s/ref=sr_pg_{}".format(i) + "?fst=p90x%3A1%2Cas%3Aoff&rh=n%3A172282%2Cn%3A541966%2Cn%3A193870011%2Cn%3A172500%2Ck%3Acorsair+ddr4%2Cp_89%3ACorsair&page={}".format(i) + "&keywords=corsair+ddr4&ie=UTF8&qid=1522049082"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0'}
response = requests.get(url, verify=False, headers=headers, proxies=proxies)
soup = BeautifulSoup(response.text.encode('utf-8'), 'html.parser')
but there is error keep poping out
AttributeError Traceback (most recent call last)
<ipython-input-7-82c14c70f937> in <module>()
25 url = "https://www.amazon.com/s/ref=sr_pg_{}".format(i) + "?fst=p90x%3A1%2Cas%3Aoff&rh=n%3A172282%2Cn%3A541966%2Cn%3A193870011%2Cn%3A172500%2Ck%3Acorsair+ddr4%2Cp_89%3ACorsair&page={}".format(i) + "&keywords=corsair+ddr4&ie=UTF8&qid=1522049082"
26 headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0'}
---> 27 response = requests.get(url, verify=False, headers=headers, proxies=proxies)
28 soup = BeautifulSoup(response.text.encode('utf-8'), 'html.parser')
29 containers = soup.select('li.s-result-item.celwidget')
c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\api.py in get(url, params, **kwargs)
70
71 kwargs.setdefault('allow_redirects', True)
---> 72 return request('get', url, params=params, **kwargs)
73
74
c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\api.py in request(method, url, **kwargs)
56 # cases, and look like a memory leak in others.
57 with sessions.Session() as session:
---> 58 return session.request(method=method, url=url, **kwargs)
59
60
c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
497
498 settings = self.merge_environment_settings(
--> 499 prep.url, proxies, stream, verify, cert
500 )
501
c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\sessions.py in merge_environment_settings(self, url, proxies, stream, verify, cert)
669 if self.trust_env:
670 # Set environment's proxies.
--> 671 no_proxy = proxies.get('no_proxy') if proxies is not None else None
672 env_proxies = get_environ_proxies(url, no_proxy=no_proxy)
673 for (k, v) in env_proxies.items():
AttributeError: 'str' object has no attribute 'get'
what happens

Answer 1

When you pass proxies to the requests library the object passed should be a mapping from protocol to proxy.
import requests
proxies = {
'http': 'http://10.10.1.10:3128',
'https': 'http://10.10.1.10:1080',
}
requests.get('http://example.org', proxies=proxies)
See http://docs.python-requests.org/en/master/user/advanced/#proxies
So in your case try:
proxy_list = [
'http://66.82.144.29:8080',
'http://47.75.0.253:3129',
'http://217.119.82.14:8080' ]
selected = random.choice(proxy_list)
proxies = { protocol: selected for protocol in ('http', 'https') }
Note that you also have to include the protocol as the proxy server itself could be using http, https, or indeed socks5.