Commit 45a5c8c2 authored by Bruno Duyé's avatar Bruno Duyé

#254: better ser-side error handling

parent 9d74f07d
......@@ -66,9 +66,10 @@ def main():
global debug_mode
requests_session = Session()
# http://www.coglib.com/~icordasc/blog/2014/12/retries-in-requests.html
# backoff_factor=2 will make sleep for 2 * (2 ^ (retry_number - 1)), ie 0, 2, 4, 8, 16, 32 ...
requests_session.mount('http://', HTTPAdapter(max_retries=Retry(total=50,
backoff_factor=2, status_forcelist=[104, 500, 503, 504])))
# backoff_factor=2 will make sleep for 2 * (2 ^ (retry_number - 1)), ie 0, 2, 4, 8, 16, 32 ... up to 1 hour (for total=12)
requests_retry = Retry(total=12, backoff_factor=2, status_forcelist=[104, 500, 501, 502, 503, 504]) # retry when server return ont of this statuses
requests_session.mount('http://', HTTPAdapter(max_retries=requests_retry))
requests_session.mount('https://', HTTPAdapter(max_retries=requests_retry))
# Parse command line arguments
args = docopt(__doc__.format(self_filename=os.path.basename(__file__)))
target_dir = args['<target_dir>']
......@@ -206,11 +207,14 @@ def get_from_api(url, raise_on_errors=True):
sleeps_times = [1, 1, 2, 3, 5, 8]
sleep_time_index = 0 # sleep time index in sleeps_times array
while True:
response = requests_session.get(url)
content_str = response.content
if (content_str.startswith(b"<BEAAPI>") and b"Exceeded request quota per minute" in content_str) \
or (content_str.startswith(b"The service is unavailable")): # Tomcat error (https://git.nomics.world/dbnomics-fetchers/management/issues/251)
if content_str.startswith(b"The service is unavailable"):
try:
response_str = try_to_download(url)
except ConnectionError as e:
log.debug("Connection error => retry")
continue
if (response_str.startswith(b"<BEAAPI>") and b"Exceeded request quota per minute" in response_str) \
or (response_str.startswith(b"The service is unavailable")): # Tomcat error (https://git.nomics.world/dbnomics-fetchers/management/issues/251)
if response_str.startswith(b"The service is unavailable"):
log.debug("Tomcat error")
waiting_time = sleeps_times[sleep_time_index]
log.info("Exceeded request quota per minute or server internal error. Waiting {} seconds ...".format(waiting_time))
......@@ -222,17 +226,17 @@ def get_from_api(url, raise_on_errors=True):
break
# Convert json answer and return result
try:
content_dict = json.loads(content_str)
content_dict = json.loads(response_str)
except Exception as e:
response_head = content_str[:400]
response_head = response_str[:400]
log.exception(
"API call: {!r}\nException during conversion of API result to json. Full result saved to last_api_result.json file.\nHere's the head of the answer:".format(url, response_head))
with open('last_api_result.json', 'wb') as _f:
_f.write(content_str)
_f.write(response_str)
raise e
if debug_mode:
with open('last_api_result.json', 'wb') as _f:
_f.write(content_str)
_f.write(response_str)
error = raise_or_log_if_error(content_dict)
if error:
return None
......@@ -243,6 +247,23 @@ def get_from_api(url, raise_on_errors=True):
return results
def try_to_download(url):
""" Download given url and return content
- raise a ConnectionError when download failed
"""
global requests_session
log.debug("downloading {!r}".format(url))
try:
response = requests_session.get(url)
except Exception as e:
log.exception(e)
raise ConnectionError
if not response.ok:
msg = 'download url {!r} - status_code: {} - reason: {}'.format(url, response.status_code, response.reason)
raise ConnectionError(msg)
return response.content
def get_api_user_id_from_env():
"""Try getting API user ID from environnement variable and return it
"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment