Commit babe1984 authored by Bruno Duyé's avatar Bruno Duyé

Download: catch more API errors

parent 2edfb51e
......@@ -42,6 +42,7 @@ from requests import Session, exceptions
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util import Retry
from slugify import slugify
from toolz import dicttoolz
import ujson as json
......@@ -128,30 +129,43 @@ def get_from_api(url, raise_on_errors=True):
- exit_on_errors: if error is returned from API, raise an exception or only log error ?
def raise_or_log_if_error(json):
"""Raise an error if raise_on_errors is True, else return True if tehre's an error"""
"""Looks for API errors on different locations in json result.
- raise an error if raise_on_errors is True
- else return True if there's an error, and log it
assert json
if 'Error' in json:
error_str = json['Error']
error_dict = None
error_locations = [
# Errors are found at different places in json ...
['BEAAPI', 'Error'],
['BEAAPI', 'Results', 'Error'],
for location in error_locations:
if dicttoolz.get_in(location, json):
error_dict = dicttoolz.get_in(location, json)
error_description = dicttoolz.get_in(['ErrorDetail', 'Description'], error_dict) or error_dict.get('APIErrorDescription')
if error_dict:
if raise_on_errors:
log.error('API error while getting {!r}'.format(url))
raise APIException(error_str)
raise APIException(error_dict)
if json['Error'].get('ErrorDetail') and json['Error']['ErrorDetail'].get('Description') and \
json['Error']['ErrorDetail']['Description'] in '|'.join(API_ERRORS_WHITELIST):
if error_description in '|'.join(API_ERRORS_WHITELIST):
# This error is not to be logged as an error
log.debug('API error while getting {!r}'.format(url))
log.error('API error while getting {!r}'.format(url))
return True
return False
global debug_mode
assert url
log.debug("-> {}".format(url))
# Try to get answer, retrying if quota exceeded
sleeps_times = [1,1,2,3,5,8]
sleeps_times = [1, 1, 2, 3, 5, 8]
sleep_time_index = 0 # sleep time index in sleeps_times array
while True:
response = requests_session.get(url)
......@@ -175,13 +189,10 @@ def get_from_api(url, raise_on_errors=True):
if debug_mode:
with open('last_api_result.json', 'wb') as _f:
error = raise_or_log_if_error(content_dict['BEAAPI'])
error = raise_or_log_if_error(content_dict)
if error:
return None
results = content_dict['BEAAPI']['Results']
if error:
return None
return results
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment