Skip to content

D
dbnomics-python-client
Commit 2f701784 authored by Christophe Benz's avatar Christophe Benz
Browse files
Options

Merge branch 'issue_482-Python_client-Add_dimensions_labels_to_dataframes' into 'master' 

#482 python client - add dimensions labels to dataframes

Closes dbnomics-fetchers/management#482

See merge request !4
parents 8e244d0b 6a5bbfed
Pipeline #87236 failed with stages
      in 1 minute
      Expand all Hide whitespace changes
      Inline Side-by-side
      Showing with 1345 additions and 255 deletions
      .gitlab-ci.yml
      View file @ 2f701784
      ......@@ -9,7 +9,7 @@ Run tests:
      stage: test
      image: python:3.7-slim-stretch
      before_script:
      - pip install pytest
      - pip install pytest pytest-vcr
      - pip install --editable .
      script:
      - pytest
      ......
      README.md
      View file @ 2f701784
      ......@@ -8,7 +8,7 @@ This project relies on [Python Pandas](https://pandas.pydata.org/).
      A tutorial is available as a [Jupyter notebook](./index.ipynb).
      The "Binder" tool allows you to run it interactively in your browser. Click on [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/git/https%3A%2F%2Fgit.nomics.world%2Fdbnomics%2Fdbnomics-python-client.git/2817d45bee4d5cb0e88d4ffd19bcbd4f6248fb9b?filepath=index.ipynb) then wait a couple of seconds. After loading a list of files should be displayed. Click on `index.ipynb` to open the tutorial notebook, where you'll be able to play with the DBnomics Python client.
      The "Binder" tool allows you to run it interactively in your browser. Click on [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/git/https%3A%2F%2Fgit.nomics.world%2Fdbnomics%2Fdbnomics-python-client.git/master?filepath=index.ipynb) then wait a couple of seconds. After loading a list of files should be displayed. Click on `index.ipynb` to open the tutorial notebook, where you'll be able to play with the DBnomics Python client.
      ## Install
      ......
      dbnomics/__init__.py
      View file @ 2f701784
      This diff is collapsed.
      index.ipynb
      View file @ 2f701784
      This diff is collapsed.
      setup.py
      View file @ 2f701784
      ......@@ -85,5 +85,6 @@ setup(
      ],
      tests_require=[
      'pytest',
      'pytest-vcr',
      ],
      )
      tests/cassettes/test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels.yaml 0 → 100644
      View file @ 2f701784
      interactions:
      - request:
      body: null
      headers:
      Accept:
      - '*/*'
      Accept-Encoding:
      - gzip, deflate
      Connection:
      - keep-alive
      User-Agent:
      - python-requests/2.21.0
      method: GET
      uri: https://api.db.nomics.world/v22/series?observations=1&series_ids=simu/lated/series2&offset=0
      response:
      body:
      string: |
      { "_meta": {
      "args": {
      "align_periods": false,
      "complete_missing_periods": false,
      "facets": false,
      "format": "json",
      "limit": 1000,
      "metadata": true,
      "observations": true,
      "offset": 0,
      "series_ids": [
      [
      "BLS",
      "is",
      "ISU00000000000000"
      ]
      ]
      },
      "version": "22.0.0"
      },
      "datasets": {
      "BLS/is": {
      "code": "is",
      "converted_at": "2019-04-12T16:59:00Z",
      "created_at": "2018-01-17T12:12:26Z",
      "description": "Occupational injury and illness counts and incidence rates are annual measures of the level and frequency of work-related injuries and illnesses. The rate equals the form of the number of injuries and illnesses per 100 full-time employees.",
      "dimensions_codes_order": [
      "area",
      "case_type",
      "data_type",
      "industry",
      "supersector"
      ],
      "indexed_at": "2019-04-13T07:51:04.095Z",
      "json_data_commit_ref": "0ace11084beae68061408e3a2cf9c02f6060c5cf",
      "name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
      "nb_series": 809690,
      "provider_code": "BLS",
      "provider_name": "U.S. Bureau of Labor Statistics"
      }
      },
      "errors": null,
      "providers": {
      "BLS": {
      "code": "BLS",
      "converted_at": "2019-09-07T09:00:40Z",
      "created_at": "2017-11-15T16:25:33Z",
      "indexed_at": "2019-09-07T09:02:20.155Z",
      "json_data_commit_ref": "857b0b8cc3dfdc476c79fbda3f22f7eaec216a81",
      "name": "U.S. Bureau of Labor Statistics",
      "region": "US",
      "slug": "bls",
      "website": "https://www.bls.gov/"
      }
      },
      "series": {
      "docs": [
      {
      "@frequency": "annual",
      "dataset_code": "is",
      "dataset_name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
      "dimensions": {
      "area": "000",
      "case_type": "0",
      "data_type": "0",
      "industry": "000000",
      "supersector": "000"
      },
      "indexed_at": "2019-04-13T07:51:04.095Z",
      "period": [
      "2014",
      "2015",
      "2016",
      "2017"
      ],
      "period_start_day": [
      "2014-01-01",
      "2015-01-01",
      "2016-01-01",
      "2017-01-01"
      ],
      "provider_code": "BLS",
      "series_code": "ISU00000000000000",
      "series_name": "All ownerships, All U.S. – RSE of Total recordable cases – Rate of illness cases per 10,000 full-time workers (Size class 0) – All workers – All",
      "value": [
      1.6,
      1.7,
      1.6,
      1.5
      ]
      }
      ],
      "limit": 1000,
      "num_found": 1,
      "offset": 0
      }
      }
      headers:
      Access-Control-Allow-Origin:
      - '*'
      Connection:
      - keep-alive
      Content-Type:
      - application/json
      Date:
      - Tue, 10 Sep 2019 10:45:46 GMT
      Server:
      - nginx/1.10.3
      Transfer-Encoding:
      - chunked
      content-length:
      - '76323'
      status:
      code: 200
      message: OK
      version: 1
      tests/cassettes/test_fetch_series_without_dimensions_values_labels.yaml 0 → 100644
      View file @ 2f701784
      interactions:
      - request:
      body: null
      headers:
      Accept:
      - '*/*'
      Accept-Encoding:
      - gzip, deflate
      Connection:
      - keep-alive
      User-Agent:
      - python-requests/2.21.0
      method: GET
      uri: https://api.db.nomics.world/v22/series?observations=1&series_ids=simu/lated/series1&offset=0
      response:
      body:
      string: |
      { "_meta": {
      "args": {
      "align_periods": false,
      "complete_missing_periods": false,
      "facets": false,
      "format": "json",
      "limit": 1000,
      "metadata": true,
      "observations": true,
      "offset": 0,
      "series_ids": [
      [
      "BLS",
      "is",
      "ISU00000000000000"
      ]
      ]
      },
      "version": "22.0.0"
      },
      "datasets": {
      "BLS/is": {
      "code": "is",
      "converted_at": "2019-04-12T16:59:00Z",
      "created_at": "2018-01-17T12:12:26Z",
      "description": "Occupational injury and illness counts and incidence rates are annual measures of the level and frequency of work-related injuries and illnesses. The rate equals the form of the number of injuries and illnesses per 100 full-time employees.",
      "dimensions_codes_order": [
      "area",
      "case_type",
      "data_type",
      "industry",
      "supersector"
      ],
      "dimensions_labels": {
      "area": "Area",
      "case_type": "Case Type",
      "data_type": "Data Type",
      "industry": "Industry",
      "supersector": "Supersector"
      },
      "indexed_at": "2019-04-13T07:51:04.095Z",
      "json_data_commit_ref": "0ace11084beae68061408e3a2cf9c02f6060c5cf",
      "name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
      "nb_series": 809690,
      "provider_code": "BLS",
      "provider_name": "U.S. Bureau of Labor Statistics"
      }
      },
      "errors": null,
      "providers": {
      "BLS": {
      "code": "BLS",
      "converted_at": "2019-09-07T09:00:40Z",
      "created_at": "2017-11-15T16:25:33Z",
      "indexed_at": "2019-09-07T09:02:20.155Z",
      "json_data_commit_ref": "857b0b8cc3dfdc476c79fbda3f22f7eaec216a81",
      "name": "U.S. Bureau of Labor Statistics",
      "region": "US",
      "slug": "bls",
      "website": "https://www.bls.gov/"
      }
      },
      "series": {
      "docs": [
      {
      "@frequency": "annual",
      "dataset_code": "is",
      "dataset_name": "Occupational Injuries and Illnesses: Industry Data (2014 forward)",
      "dimensions": {
      "area": "000",
      "case_type": "0",
      "data_type": "0",
      "industry": "000000",
      "supersector": "000"
      },
      "indexed_at": "2019-04-13T07:51:04.095Z",
      "period": [
      "2014",
      "2015",
      "2016",
      "2017"
      ],
      "period_start_day": [
      "2014-01-01",
      "2015-01-01",
      "2016-01-01",
      "2017-01-01"
      ],
      "provider_code": "BLS",
      "series_code": "ISU00000000000000",
      "series_name": "All ownerships, All U.S. – RSE of Total recordable cases – Rate of illness cases per 10,000 full-time workers (Size class 0) – All workers – All",
      "value": [
      1.6,
      1.7,
      1.6,
      1.5
      ]
      }
      ],
      "limit": 1000,
      "num_found": 1,
      "offset": 0
      }
      }
      headers:
      Access-Control-Allow-Origin:
      - '*'
      Connection:
      - keep-alive
      Content-Type:
      - application/json
      Date:
      - Tue, 10 Sep 2019 10:45:46 GMT
      Server:
      - nginx/1.10.3
      Transfer-Encoding:
      - chunked
      content-length:
      - '76323'
      status:
      code: 200
      message: OK
      version: 1
      tests/test_client.py
      View file @ 2f701784
      ......@@ -19,6 +19,7 @@
      import logging
      import pytest
      import pandas as pd
      ......@@ -228,3 +229,41 @@ def test_fetch_series_with_filter_on_one_series_with_wrong_frequency(caplog):
      assert len(caplog.records) == 1
      assert caplog.records[0].levelname == 'ERROR'
      assert "Annual is already the lowest frequency" in caplog.records[0].message
      # --- Tests above uses VCR.py (https://vcrpy.readthedocs.io/en/latest/usage.html) to load fixtures ---
      @pytest.mark.vcr(decode_compressed_response=True)
      def test_fetch_series_without_dimensions_labels():
      df = fetch_series("WB", "DB", dimensions={
      "country": ["FR", "IT", "ES"],
      "indicator": ["IC.REG.COST.PC.FE.ZS.DRFN"],
      }) # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_labels.yaml file
      # Check that all expected columns are present
      expected_columns = {'indicator', 'country', 'indicator (label)', 'country (label)'}
      assert expected_columns & set(df.columns) == expected_columns, set(df.columns)
      # Check dimensions and dimensions_values_labels
      df_line = df.iloc[30]
      assert df_line['country'] == 'FR'
      assert df_line['country (label)'] == 'France'
      @pytest.mark.vcr(decode_compressed_response=True)
      def test_fetch_series_without_dimensions_values_labels():
      # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_values_labels.yaml file
      df = fetch_series('simu/lated/series1')
      # In the case of any dimensions_values_labels, we do not want dimensions_labels column to be added
      assert not 'Data Type' in list(df.columns)
      @pytest.mark.vcr(decode_compressed_response=True)
      def test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels():
      # Thanks to @pytest.mark.vcr decorator, this request result will be read from cassettes/test_fetch_series_without_dimensions_labels_nor_dimensions_values_labels.yaml file
      df = fetch_series('simu/lated/series2')
      # dimensions_labels column shouldn't exist
      assert not 'Data Type' in list(df.columns)
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment