def post_ratelimited ( protocol , session , url , headers , data , allow_redirects = False , stream = False ) : thread_id = get_ident ( ) wait = 10 # seconds retry = 0 redirects = 0 # In Python 2, we want this to be a 'str' object so logging doesn't break (all formatting arguments are 'str'). # We activated 'unicode_literals' at the top of this file, so it would be a 'unicode' object unless we convert # to 'str' explicitly. This is a no-op for Python 3. log_msg = str ( '''\
Retry: %(retry)s
Waited: %(wait)s
Timeout: %(timeout)s
Session: %(session_id)s
Thread: %(thread_id)s
Auth type: %(auth)s
URL: %(url)s
HTTP adapter: %(adapter)s
Allow redirects: %(allow_redirects)s
Streaming: %(stream)s
Response time: %(response_time)s
Status code: %(status_code)s
Request headers: %(request_headers)s
Response headers: %(response_headers)s
Request data: %(xml_request)s
Response data: %(xml_response)s
''' ) log_vals = dict ( retry = retry , wait = wait , timeout = protocol . TIMEOUT , session_id = session . session_id , thread_id = thread_id , auth = session . auth , url = url , adapter = session . get_adapter ( url ) , allow_redirects = allow_redirects , stream = stream , response_time = None , status_code = None , request_headers = headers , response_headers = None , xml_request = data , xml_response = None , ) try : while True : _back_off_if_needed ( protocol . credentials . back_off_until ) log . debug ( 'Session %s thread %s: retry %s timeout %s POST\'ing to %s after %ss wait' , session . session_id , thread_id , retry , protocol . TIMEOUT , url , wait ) d_start = time_func ( ) # Always create a dummy response for logging purposes, in case we fail in the following r = DummyResponse ( url = url , headers = { } , request_headers = headers ) try : r = session . post ( url = url , headers = headers , data = data , allow_redirects = False , timeout = protocol . TIMEOUT , stream = stream ) except CONNECTION_ERRORS as e : log . debug ( 'Session %s thread %s: connection error POST\'ing to %s' , session . session_id , thread_id , url ) r = DummyResponse ( url = url , headers = { 'TimeoutException' : e } , request_headers = headers ) finally : log_vals . update ( retry = retry , wait = wait , session_id = session . session_id , url = str ( r . url ) , response_time = time_func ( ) - d_start , status_code = r . status_code , request_headers = r . request . headers , response_headers = r . headers , xml_response = '[STREAMING]' if stream else r . content , ) log . debug ( log_msg , log_vals ) if _may_retry_on_error ( r , protocol , wait ) : log . info ( "Session %s thread %s: Connection error on URL %s (code %s). Cool down %s secs" , session . session_id , thread_id , r . url , r . status_code , wait ) time . sleep ( wait ) # Increase delay for every retry retry += 1 wait *= 2 session = protocol . renew_session ( session ) continue if r . status_code in ( 301 , 302 ) : if stream : r . close ( ) url , redirects = _redirect_or_fail ( r , redirects , allow_redirects ) continue break except ( RateLimitError , RedirectError ) as e : log . warning ( e . value ) protocol . retire_session ( session ) raise except Exception as e : # Let higher layers handle this. Add full context for better debugging. log . error ( str ( '%s: %s\n%s' ) , e . __class__ . __name__ , str ( e ) , log_msg % log_vals ) protocol . retire_session ( session ) raise if r . status_code == 500 and r . content and is_xml ( r . content ) : # Some genius at Microsoft thinks it's OK to send a valid SOAP response as an HTTP 500 log . debug ( 'Got status code %s but trying to parse content anyway' , r . status_code ) elif r . status_code != 200 : protocol . retire_session ( session ) try : _raise_response_errors ( r , protocol , log_msg , log_vals ) # Always raises an exception finally : if stream : r . close ( ) log . debug ( 'Session %s thread %s: Useful response from %s' , session . session_id , thread_id , url ) return r , session
stack overflow python requests exceeded 30 redirects
There are two error - handling policies implemented here : a fail - fast policy intended for stand - alone scripts which fails on all responses except HTTP 200 . The other policy is intended for long - running tasks that need to respect rate - limiting errors from the server and paper over outages of up to 1 hour .
def post_ratelimited(protocol, session, url, headers, data, allow_redirects=False, stream=False):
    """
    There are two error-handling policies implemented here: a fail-fast policy intended for stand-alone scripts which
    fails on all responses except HTTP 200. The other policy is intended for long-running tasks that need to respect
    rate-limiting errors from the server and paper over outages of up to 1 hour.

    Wrap POST requests in a try-catch loop with a lot of error handling logic and some basic rate-limiting. If a request
    fails, and some conditions are met, the loop waits in increasing intervals, up to 1 hour, before trying again. The
    reason for this is that servers often malfunction for short periods of time, either because of ongoing data
    migrations or other maintenance tasks, misconfigurations or heavy load, or because the connecting user has hit a
    throttling policy limit.

    If the loop exited early, consumers of this package that don't implement their own rate-limiting code could quickly
    swamp such a server with new requests. That would only make things worse. Instead, it's better if the request loop
    waits patiently until the server is functioning again.

    If the connecting user has hit a throttling policy, then the server will start to malfunction in many interesting
    ways, but never actually tell the user what is happening. There is no way to distinguish this situation from other
    malfunctions. The only cure is to stop making requests.

    The contract on sessions here is to return the session that ends up being used, or retiring the session if we
    intend to raise an exception. We give up on max_wait timeout, not number of retries.

    An additional resource on handling throttling policies and client back off strategies:
        https://msdn.microsoft.com/en-us/library/office/jj945066(v=exchg.150).aspx#bk_ThrottlingBatch
    """
    thread_id = get_ident()
    wait = 10  # seconds
    retry = 0
    redirects = 0
    # In Python 2, we want this to be a 'str' object so logging doesn't break (all formatting arguments are 'str').
    # We activated 'unicode_literals' at the top of this file, so it would be a 'unicode' object unless we convert
    # to 'str' explicitly. This is a no-op for Python 3.
    log_msg = str('''\
Retry: %(retry)s
Waited: %(wait)s
Timeout: %(timeout)s
Session: %(session_id)s
Thread: %(thread_id)s
Auth type: %(auth)s
URL: %(url)s
HTTP adapter: %(adapter)s
Allow redirects: %(allow_redirects)s
Streaming: %(stream)s
Response time: %(response_time)s
Status code: %(status_code)s
Request headers: %(request_headers)s
Response headers: %(response_headers)s
Request data: %(xml_request)s
Response data: %(xml_response)s
''')
    log_vals = dict(
        retry=retry,
        wait=wait,
        timeout=protocol.TIMEOUT,
        session_id=session.session_id,
        thread_id=thread_id,
        auth=session.auth,
        url=url,
        adapter=session.get_adapter(url),
        allow_redirects=allow_redirects,
        stream=stream,
        response_time=None,
        status_code=None,
        request_headers=headers,
        response_headers=None,
        xml_request=data,
        xml_response=None,
    )
    try:
        while True:
            _back_off_if_needed(protocol.credentials.back_off_until)
            log.debug('Session %s thread %s: retry %s timeout %s POST\'ing to %s after %ss wait', session.session_id,
                      thread_id, retry, protocol.TIMEOUT, url, wait)
            d_start = time_func()
            # Always create a dummy response for logging purposes, in case we fail in the following
            r = DummyResponse(url=url, headers={}, request_headers=headers)
            try:
                r = session.post(url=url, headers=headers, data=data, allow_redirects=False, timeout=protocol.TIMEOUT,
                                 stream=stream)
            except CONNECTION_ERRORS as e:
                log.debug('Session %s thread %s: connection error POST\'ing to %s', session.session_id, thread_id, url)
                r = DummyResponse(url=url, headers={'TimeoutException': e}, request_headers=headers)
            finally:
                log_vals.update(
                    retry=retry,
                    wait=wait,
                    session_id=session.session_id,
                    url=str(r.url),
                    response_time=time_func() - d_start,
                    status_code=r.status_code,
                    request_headers=r.request.headers,
                    response_headers=r.headers,
                    xml_response='[STREAMING]' if stream else r.content,
                )
            log.debug(log_msg, log_vals)
            if _may_retry_on_error(r, protocol, wait):
                log.info("Session %s thread %s: Connection error on URL %s (code %s). Cool down %s secs",
                         session.session_id, thread_id, r.url, r.status_code, wait)
                time.sleep(wait)  # Increase delay for every retry
                retry += 1
                wait *= 2
                session = protocol.renew_session(session)
                continue
            if r.status_code in (301, 302):
                if stream:
                    r.close()
                url, redirects = _redirect_or_fail(r, redirects, allow_redirects)
                continue
            break
    except (RateLimitError, RedirectError) as e:
        log.warning(e.value)
        protocol.retire_session(session)
        raise
    except Exception as e:
        # Let higher layers handle this. Add full context for better debugging.
        log.error(str('%s: %s\n%s'), e.__class__.__name__, str(e), log_msg % log_vals)
        protocol.retire_session(session)
        raise
    if r.status_code == 500 and r.content and is_xml(r.content):
        # Some genius at Microsoft thinks it's OK to send a valid SOAP response as an HTTP 500
        log.debug('Got status code %s but trying to parse content anyway', r.status_code)
    elif r.status_code != 200:
        protocol.retire_session(session)
        try:
            _raise_response_errors(r, protocol, log_msg, log_vals)  # Always raises an exception
        finally:
            if stream:
                r.close()
    log.debug('Session %s thread %s: Useful response from %s', session.session_id, thread_id, url)
    return r, session