Skip to content

GitLab

D
dbnomics-python-client
Commit 9d1e923c authored by Bruno Duyé's avatar Bruno Duyé
Browse files
Options

482 - fix wrong dimensions_values_labels + add tests + handle cases of no... 

482 - fix wrong dimensions_values_labels + add tests + handle cases of no dimensions_labels and/or no dimensions_values_labels
parent 9b9a1139
Pipeline #87019 passed with stage
    in 42 seconds
    Expand all Hide whitespace changes
    Inline Side-by-side
    Showing with 705 additions and 26 deletions
    .gitlab-ci.yml
    View file @ 9d1e923c
    ......@@ -9,7 +9,7 @@ Run tests:
    stage: test
    image: python:3.7-slim-stretch
    before_script:
    - pip install pytest
    - pip install pytest pytest-vcr
    - pip install --editable .
    script:
    - pytest
    ......
    dbnomics/__init__.py
    View file @ 9d1e923c
    ......@@ -21,6 +21,7 @@
    """Access DBnomics time series from Python."""
    from collections import defaultdict
    import itertools
    import json
    import logging
    ......@@ -28,6 +29,8 @@ import os
    import urllib.parse
    from urllib.parse import urljoin
    import vcr
    import pandas as pd
    import requests
    ......@@ -40,6 +43,13 @@ editor_apply_endpoint_nb_series_per_post = 100
    log = logging.getLogger(__name__)
    my_vcr = vcr.VCR(
    cassette_library_dir='fixtures/cassettes',
    record_mode='once',
    decode_compressed_response=True,
    )
    class TooManySeries(Exception):
    def __init__(self, num_found, max_nb_series):
    self.num_found = num_found
    ......@@ -185,18 +195,30 @@ def fetch_series_by_api_link(api_link, max_nb_series=None,
    """
    # Call API via `iter_series`, add dimensions labels and store result in `series_list`. Fill `datasets_dimensions`
    datasets_dimensions = None
    series_dims_by_dataset_code = {} # TODO doc
    series_dims_by_dataset_code = defaultdict(dict)
    # series_dims_by_dataset_code example:
    # {
    # 'WB/DB': {
    # 'EA19.1.0.0.0.ZUTN': { 'freq':'a', 'geo':'ea19', 'unit':'percentage-of-active-population'},
    # 'EA20.1.0.0.0.ZUTN': { 'freq':'a', 'geo':'ea20', 'unit':'percentage-of-active-population'},
    # ...
    # },
    # ...
    # }
    series_list = []
    for series_infos in iter_series(api_link, max_nb_series=max_nb_series):
    complete_dataset_code = series_infos['series']['provider_code'] + \
    '/' + series_infos['series']['dataset_code'] # ex 'AMECO/ZUTN'
    if datasets_dimensions is None:
    # Let see if there's only one dataset returned by API, or many datasets
    datasets_dimensions = series_infos['datasets_dimensions'] if 'datasets_dimensions' in series_infos else {
    # Only one dataset
    complete_dataset_code: series_infos['dataset_dimensions']
    }
    series_list.append(series_infos['series'])
    # Store series dimensions information for future use
    series_dims_by_dataset_code[complete_dataset_code] = series_infos['series']['dimensions']
    series_dims_by_dataset_code[complete_dataset_code][series_infos['series']
    ['series_code']] = series_infos['series']['dimensions']
    if len(series_list) == 0:
    return pd.DataFrame()
    ......@@ -211,10 +233,17 @@ def fetch_series_by_api_link(api_link, max_nb_series=None,
    # Add dimensions labels to flat_series
    complete_dataset_code = flat_series['provider_code'] + '/' + flat_series['dataset_code'] # ex: "AMECO/ZUTN"
    dataset_dimensions = datasets_dimensions[complete_dataset_code]
    for dimension_code, dimension_label in dataset_dimensions['dimensions_labels'].items():
    dimension_value_code = series_dims_by_dataset_code[complete_dataset_code][dimension_code]
    flat_series[dimension_label] = dict(dataset_dimensions['dimensions_values_labels']
    [dimension_code])[dimension_value_code]
    if 'dimensions_labels' in dataset_dimensions:
    dataset_dimensions_labels = dataset_dimensions['dimensions_labels']
    else:
    dataset_dimensions_labels = {dim_code: "{} (label)".format(dim_code)
    for dim_code in dataset_dimensions['dimensions_codes_order']}
    # Add dimensions values labels to current series
    if 'dimensions_values_labels' in dataset_dimensions:
    for dimension_code, dimension_label in dataset_dimensions_labels.items():
    dimension_value_code = series_dims_by_dataset_code[complete_dataset_code][series['series_code']][dimension_code]
    flat_series[dimension_label] = dict(dataset_dimensions['dimensions_values_labels']
    [dimension_code])[dimension_value_code]
    flat_series_list.append(flat_series)
    # Only applies if filters are used.
    ......@@ -231,32 +260,28 @@ def fetch_series_by_api_link(api_link, max_nb_series=None,
    for series in flat_series_list
    ] + filtered_series_list
    # `flat_series_list` is a list of dicts like [{"code": "A.B.C", "a_key": 9}, {"code": "X.Y.Z", "other_key": 42}]
    # Each series can have different keys so we want to do the union of all the keys of all the series. {"code", "a_key", "other_key"}
    # In the DataFrame the different columns will be sparse (there will be `NaN` values when a series does not have a specific key).
    # code | a_key | other_key
    # ----- | ----- | ---------
    # A.B.C | 9 | NaN
    # X.Y.Z | NaN | 42
    def union_sets(sets):
    return set.union(*sets)
    all_columns = union_sets([set(series.keys()) for series in flat_series_list])
    # Compute dimensions_labels_columns and dimensions_codes_columns
    dimensions_labels_columns = []
    dimensions_codes_columns = []
    # Compute dimensions_labels_columns_names and dimensions_codes_columns_names
    dimensions_labels_columns_names = []
    dimensions_codes_columns_names = []
    for complete_dataset_code in datasets_dimensions.keys():
    for dimension_code in datasets_dimensions[complete_dataset_code]['dimensions_codes_order']:
    dimensions_codes_columns.append(dimension_code)
    dimensions_labels_columns.append(
    datasets_dimensions[complete_dataset_code]['dimensions_labels'][dimension_code])
    dimensions_codes_columns_names.append(dimension_code)
    # We only add dimensions labels column if this information is present
    if 'dimensions_labels' in dataset_dimensions and 'dimensions_values_labels' in dataset_dimensions:
    dimensions_labels_columns_names.append(
    datasets_dimensions[complete_dataset_code]['dimensions_labels'][dimension_code])
    else:
    if 'dimensions_values_labels' in dataset_dimensions:
    # No dimensions labels but dimensions_values_labels -> we add " (label)" to the end of dimension code
    dimensions_labels_columns_names.append("{} (label)".format(dimension_code))
    # In the case there's no dimension_label nor dimensions_values_labels, we do not add any column
    # In the DataFrame we want to display the dimension columns at the right so we reorder them.
    ordered_columns = common_columns + dimensions_codes_columns + dimensions_labels_columns
    ordered_columns_names = common_columns + dimensions_codes_columns_names + dimensions_labels_columns_names
    # Build dataframe
    dataframes = (
    pd.DataFrame(data=series, columns=ordered_columns)
    pd.DataFrame(data=series, columns=ordered_columns_names)
    for series in flat_series_list
    )
    return pd.concat(objs=dataframes, sort=False)
    ......@@ -360,6 +385,7 @@ def iter_series(api_link, max_nb_series=None):
    total_nb_series = 0
    while True:
    # with vcr.use_cassette('tests/cassettes/test_fetch_series_without_dimensions_values_labels.yaml'):
    response_json = fetch_series_page(api_link, offset=total_nb_series)
    errors = response_json.get("errors")
    ......@@ -484,3 +510,4 @@ def grouper(n, iterable):
    def without_keys(d, keys):
    return {k: v for k, v in d.items() if k not in keys}
    setup.py
    View file @ 9d1e923c
    ......@@ -85,5 +85,6 @@ setup(
    ],
    tests_require=[
    'pytest',
    'pytest-vcr',
    ],
    )
    tests/cassettes/test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels.yaml 0 → 100644
    View file @ 9d1e923c
    interactions:
    - request:
    body: null
    headers:
    Accept:
    - '*/*'
    Accept-Encoding:
    - gzip, deflate
    Connection:
    - keep-alive
    User-Agent:
    - python-requests/2.21.0
    method: GET
    uri: https://api.db.nomics.world/v22/series?observations=1&series_ids=simu/lated/series2&offset=0
    response:
    body:
    string: |
    { "_meta": {
    "args": {
    "align_periods": false,
    "complete_missing_periods": false,
    "facets": false,
    "format": "json",
    "limit": 1000,
    "metadata": true,
    "observations": true,
    "offset": 0,
    "series_ids": [
    [
    "BLS",
    "is",
    "ISU00000000000000"
    ]
    ]
    },
    "version": "22.0.0"
    },
    "datasets": {
    "BLS/is": {
    "code": "is",
    "converted_at": "2019-04-12T16:59:00Z",
    "created_at": "2018-01-17T12:12:26Z",
    "description": "Occupational injury and illness counts and incidence rates are annual measures of the level and frequency of work-related injuries and illnesses. The rate equals the form of the number of injuries and illnesses per 100 full-time employees.",
    "dimensions_codes_order": [
    "area",
    "case_type",
    "data_type",
    "industry",
    "supersector"
    ],
    "indexed_at": "2019-04-13T07:51:04.095Z",
    "json_data_commit_ref": "0ace11084beae68061408e3a2cf9c02f6060c5cf",
    "name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
    "nb_series": 809690,
    "provider_code": "BLS",
    "provider_name": "U.S. Bureau of Labor Statistics"
    }
    },
    "errors": null,
    "providers": {
    "BLS": {
    "code": "BLS",
    "converted_at": "2019-09-07T09:00:40Z",
    "created_at": "2017-11-15T16:25:33Z",
    "indexed_at": "2019-09-07T09:02:20.155Z",
    "json_data_commit_ref": "857b0b8cc3dfdc476c79fbda3f22f7eaec216a81",
    "name": "U.S. Bureau of Labor Statistics",
    "region": "US",
    "slug": "bls",
    "website": "https://www.bls.gov/"
    }
    },
    "series": {
    "docs": [
    {
    "@frequency": "annual",
    "dataset_code": "is",
    "dataset_name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
    "dimensions": {
    "area": "000",
    "case_type": "0",
    "data_type": "0",
    "industry": "000000",
    "supersector": "000"
    },
    "indexed_at": "2019-04-13T07:51:04.095Z",
    "period": [
    "2014",
    "2015",
    "2016",
    "2017"
    ],
    "period_start_day": [
    "2014-01-01",
    "2015-01-01",
    "2016-01-01",
    "2017-01-01"
    ],
    "provider_code": "BLS",
    "series_code": "ISU00000000000000",
    "series_name": "All ownerships, All U.S. – RSE of Total recordable cases – Rate of illness cases per 10,000 full-time workers (Size class 0) – All workers – All",
    "value": [
    1.6,
    1.7,
    1.6,
    1.5
    ]
    }
    ],
    "limit": 1000,
    "num_found": 1,
    "offset": 0
    }
    }
    headers:
    Access-Control-Allow-Origin:
    - '*'
    Connection:
    - keep-alive
    Content-Type:
    - application/json
    Date:
    - Tue, 10 Sep 2019 10:45:46 GMT
    Server:
    - nginx/1.10.3
    Transfer-Encoding:
    - chunked
    content-length:
    - '76323'
    status:
    code: 200
    message: OK
    version: 1
    tests/cassettes/test_fetch_series_without_dimensions_values_labels.yaml 0 → 100644
    View file @ 9d1e923c
    interactions:
    - request:
    body: null
    headers:
    Accept:
    - '*/*'
    Accept-Encoding:
    - gzip, deflate
    Connection:
    - keep-alive
    User-Agent:
    - python-requests/2.21.0
    method: GET
    uri: https://api.db.nomics.world/v22/series?observations=1&series_ids=simu/lated/series1&offset=0
    response:
    body:
    string: |
    { "_meta": {
    "args": {
    "align_periods": false,
    "complete_missing_periods": false,
    "facets": false,
    "format": "json",
    "limit": 1000,
    "metadata": true,
    "observations": true,
    "offset": 0,
    "series_ids": [
    [
    "BLS",
    "is",
    "ISU00000000000000"
    ]
    ]
    },
    "version": "22.0.0"
    },
    "datasets": {
    "BLS/is": {
    "code": "is",
    "converted_at": "2019-04-12T16:59:00Z",
    "created_at": "2018-01-17T12:12:26Z",
    "description": "Occupational injury and illness counts and incidence rates are annual measures of the level and frequency of work-related injuries and illnesses. The rate equals the form of the number of injuries and illnesses per 100 full-time employees.",
    "dimensions_codes_order": [
    "area",
    "case_type",
    "data_type",
    "industry",
    "supersector"
    ],
    "dimensions_labels": {
    "area": "Area",
    "case_type": "Case Type",
    "data_type": "Data Type",
    "industry": "Industry",
    "supersector": "Supersector"
    },
    "indexed_at": "2019-04-13T07:51:04.095Z",
    "json_data_commit_ref": "0ace11084beae68061408e3a2cf9c02f6060c5cf",
    "name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
    "nb_series": 809690,
    "provider_code": "BLS",
    "provider_name": "U.S. Bureau of Labor Statistics"
    }
    },
    "errors": null,
    "providers": {
    "BLS": {
    "code": "BLS",
    "converted_at": "2019-09-07T09:00:40Z",
    "created_at": "2017-11-15T16:25:33Z",
    "indexed_at": "2019-09-07T09:02:20.155Z",
    "json_data_commit_ref": "857b0b8cc3dfdc476c79fbda3f22f7eaec216a81",
    "name": "U.S. Bureau of Labor Statistics",
    "region": "US",
    "slug": "bls",
    "website": "https://www.bls.gov/"
    }
    },
    "series": {
    "docs": [
    {
    "@frequency": "annual",
    "dataset_code": "is",
    "dataset_name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
    "dimensions": {
    "area": "000",
    "case_type": "0",
    "data_type": "0",
    "industry": "000000",
    "supersector": "000"
    },
    "indexed_at": "2019-04-13T07:51:04.095Z",
    "period": [
    "2014",
    "2015",
    "2016",
    "2017"
    ],
    "period_start_day": [
    "2014-01-01",
    "2015-01-01",
    "2016-01-01",
    "2017-01-01"
    ],
    "provider_code": "BLS",
    "series_code": "ISU00000000000000",
    "series_name": "All ownerships, All U.S. – RSE of Total recordable cases – Rate of illness cases per 10,000 full-time workers (Size class 0) – All workers – All",
    "value": [
    1.6,
    1.7,
    1.6,
    1.5
    ]
    }
    ],
    "limit": 1000,
    "num_found": 1,
    "offset": 0
    }
    }
    headers:
    Access-Control-Allow-Origin:
    - '*'
    Connection:
    - keep-alive
    Content-Type:
    - application/json
    Date:
    - Tue, 10 Sep 2019 10:45:46 GMT
    Server:
    - nginx/1.10.3
    Transfer-Encoding:
    - chunked
    content-length:
    - '76323'
    status:
    code: 200
    message: OK
    version: 1
    tests/test_client.py
    View file @ 9d1e923c
    ......@@ -19,6 +19,7 @@
    import logging
    import pytest
    import pandas as pd
    ......@@ -228,3 +229,41 @@ def test_fetch_series_with_filter_on_one_series_with_wrong_frequency(caplog):
    assert len(caplog.records) == 1
    assert caplog.records[0].levelname == 'ERROR'
    assert "Annual is already the lowest frequency" in caplog.records[0].message
    # --- Tests above uses VCR.py (https://vcrpy.readthedocs.io/en/latest/usage.html) to load fixtures ---
    @pytest.mark.vcr(decode_compressed_response=True)
    def test_fetch_series_without_dimensions_labels():
    df = fetch_series("WB", "DB", dimensions={
    "country": ["FR", "IT", "ES"],
    "indicator": ["IC.REG.COST.PC.FE.ZS.DRFN"],
    }) # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_labels.yaml file
    # Check that all expected columns are present
    expected_columns = {'indicator', 'country', 'indicator (label)', 'country (label)'}
    assert expected_columns & set(df.columns) == expected_columns, set(df.columns)
    # Check dimensions and dimensions_values_labels
    df_line = df.iloc[30]
    assert df_line['country'] == 'FR'
    assert df_line['country (label)'] == 'France'
    @pytest.mark.vcr(decode_compressed_response=True)
    def test_fetch_series_without_dimensions_values_labels():
    # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_values_labels.yaml file
    df = fetch_series('simu/lated/series1')
    # In the case of any dimensions_values_labels, we do not want dimensions_labels column to be added
    assert not 'Data Type' in list(df.columns)
    @pytest.mark.vcr(decode_compressed_response=True)
    def test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels():
    # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels.yaml file
    df = fetch_series('simu/lated/series2')
    # dimensions_labels column shouldn't exist
    assert not 'Data Type' in list(df.columns)
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment