Opening a page with urllib2 handshake failure
<p>I am simply trying to open a webpage: <a href="https://close5.com/home/" rel="nofollow">https://close5.com/home/</a></p>

<p>And I keep getting differing errors concerning my ssl. Here are a couple of my attempts and their errors. I am open to a fix that works in either framework. My end goal is to use turn this page into a beautifulsoup4 soup.</p>

<p><strong>The error:</strong></p>

<pre><code>Traceback (most recent call last):
  File "test.py", line 54, in &lt;module&gt;
    print soup_maker_two(url)
  File "test.py", line 45, in soup_maker_two
    response = br.open(url)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_mechanize.py", line 203, in open
    return self._mech_open(url, data, timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_mechanize.py", line 230, in _mech_open
    response = UserAgentBase.open(self, request, data)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_opener.py", line 193, in open
    response = urlopen(self, req, data)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_urllib2_fork.py", line 344, in _open
    '_open', req)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_urllib2_fork.py", line 332, in _call_chain
    result = func(*args)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_urllib2_fork.py", line 1170, in https_open
    return self.do_open(conn_factory, req)
  File "/usr/local/lib/python2.7/dist-packages/mechanize/_urllib2_fork.py", line 1118, in do_open
    raise URLError(err)
urllib2.URLError: &lt;urlopen error [Errno 1] _ssl.c:510: error:14094410:SSL routines:SSL3_READ_BYTES:sslv3 alert handshake failure&gt;
</code></pre>

<p><strong>The code:</strong></p>

<pre><code>import mechanize
import ssl
from functools import wraps

def sslwrap(func):
    @wraps(func)
    def bar(*args, **kw):
        kw['ssl_version'] = ssl.PROTOCOL_TLSv1
        return func(*args, **kw)
    return bar

def soup_maker_two(url):
    br = mechanize.Browser()
    br.set_handle_robots(False)
    br.set_handle_equiv(False)
    br.set_handle_refresh(False)
    br.addheaders = [('User-agent', 'Firefox')]
    ssl.wrap_socket = sslwrap(ssl.wrap_socket)
    response = br.open(url)
    for f in br.forms():
        print f
    return 'hi'



if __name__ == "__main__":
    url = 'https://close5.com/'
    print soup_maker_two(url)
</code></pre>

<p>I have also tried got this error and code combo</p>

<p><strong>2nd Attempt</strong></p>

<p><strong>The error:</strong></p>

<pre><code>Traceback (most recent call last):
  File "test.py", line 29, in &lt;module&gt;
    print str(soup_maker(url))[0:1000]
  File "test.py", line 22, in soup_maker
    webpage = opener.open(req)
  File "/usr/lib/python2.7/urllib2.py", line 404, in open
    response = self._open(req, data)
  File "/usr/lib/python2.7/urllib2.py", line 422, in _open
    '_open', req)
  File "/usr/lib/python2.7/urllib2.py", line 382, in _call_chain
    result = func(*args)
  File "/usr/lib/python2.7/urllib2.py", line 1222, in https_open
    return self.do_open(httplib.HTTPSConnection, req)
  File "/usr/lib/python2.7/urllib2.py", line 1184, in do_open
    raise URLError(err)
urllib2.URLError: &lt;urlopen error [Errno 1] _ssl.c:510: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure&gt;
</code></pre>

<p><strong>The code:</strong></p>

<pre><code>from bs4 import BeautifulSoup
import urllib2

def soup_maker(url):
    class RedirectHandler(urllib2.HTTPRedirectHandler):
        def http_error_302(self, req, fp, code, msg, headers):
            result = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
            result.status = code
            return result



    hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
                                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                                'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
                                'Accept-Encoding': 'none',
                                'Accept-Language': 'en-US,en;q=0.8',
                                'Connection': 'keep-alive'}

    req = urllib2.Request(url,headers=hdr)
    opener = urllib2.build_opener(RedirectHandler())
    webpage = opener.open(req)
    soup = BeautifulSoup(webpage, "html5lib")
    return soup


if __name__ == "__main__":
    url = 'https://close5.com/home/'
    print str(soup_maker(url))[0:1000]
</code></pre>

<p><strong>EDIT 1</strong></p>

<p>from bs4 import BeautifulSoup
It was suggested that I use:</p>

<pre><code>def soup_maker(url):     
    soup = BeautifulSoup(requests.get(url).content, "html5lib")
    return soup

if __name__ == "__main__":
    import requests    
    url = 'https://close5.com/home/'
    print str(soup_maker(url))[:1000]
</code></pre>

<p>This code worked for Padraic, but does not work for me. I get the error:</p>

<pre><code>Traceback (most recent call last):
  File "test_3.py", line 10, in &lt;module&gt;
    print str(soup_maker(url))[:1000]
  File "test_3.py", line 4, in soup_maker
    soup = BeautifulSoup(requests.get(url).content, "html5lib")
  File "/usr/lib/python2.7/dist-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/sessions.py", line 455, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/sessions.py", line 558, in send
    r = adapter.send(request, **kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/adapters.py", line 385, in send
    raise SSLError(e)
requests.exceptions.SSLError: [Errno 1] _ssl.c:510: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure
</code></pre>

<p>Which is the same error as before. I am gonna guess it may have something to do with the fact that I am using Python 2.7.6, but I am uncertain. Also, I am unsure how to use that information to solve my issue.</p>

<p><strong>EDIT 2</strong></p>

<p>The issue may lie in the incorrect version of requests. I currently have <code>requests==2.2.1</code> in my <code>pip freeze</code></p>

<pre><code>sudo pip install -U requests
</code></pre>

<p>returns</p>

<pre><code>Downloading/unpacking requests from https://pypi.python.org/packages/2.7/r/requests/requests-2.9.1-py2.py3-none-any.whl#md5=58a444aaa02780ad01983f5f540e67b2
  Downloading requests-2.9.1-py2.py3-none-any.whl (501kB): 501kB downloaded
Installing collected packages: requests
  Found existing installation: requests 2.2.1
    Not uninstalling requests at /usr/lib/python2.7/dist-packages, owned by OS
Successfully installed requests
Cleaning up..
</code></pre>

<p><code>sudo pip2 install -U requests</code> returns the same thing</p>

<p><code>sudo pip uninstall requests</code> returns</p>

<pre><code>Not uninstalling requests at /usr/lib/python2.7/dist-packages, owned by OS
</code></pre>

<p>I am running ubuntu 14.04 and python 2.7.6 and requests 2.2.1</p>

<p><strong>Edit 3</strong></p>

<pre><code>sudo pip install --ignore-installed requests
</code></pre>

<p>gives</p>

<pre><code>Downloading/unpacking requests
  Downloading requests-2.9.1-py2.py3-none-any.whl (501kB): 501kB downloaded
Installing collected packages: requests
Successfully installed requests
Cleaning up...
</code></pre>

<p>but <code>sudo pip freeze</code> still gives <code>requests==2.2.1</code></p>

<p><strong>Edit 4</strong></p>

<p>After going through many suggestions I now have</p>

<pre><code>$python
Python 2.7.6 (default, Jun 22 2015, 18:00:18) 
[GCC 4.8.2] on linux2
Type "help", "copyright", "credits" or "license" for more information.
&gt;&gt;&gt; import requests;requests.__version__
'2.9.1'
&gt;&gt;&gt; url = 'https://close5.com/home/'
&gt;&gt;&gt; from bs4 import BeautifulSoup
&gt;&gt;&gt; soup = BeautifulSoup(requests.get(url).content, "html5lib")
/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/util/ssl_.py:315: SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name Indication) extension to TLS is not available on this platform. This may cause the server to present an incorrect TLS certificate, which can cause validation failures. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#snimissingwarning.
  SNIMissingWarning
/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/util/ssl_.py:120: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning
Traceback (most recent call last):
  File "&lt;stdin&gt;", line 1, in &lt;module&gt;
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 67, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 53, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 468, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 576, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 447, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: [Errno 1] _ssl.c:510: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure
&gt;&gt;&gt; 
</code></pre>