Download datasets.json, flags.json and main zip files

download.py

@@ -24,21 +24,50 @@
 
 import argparse
 import logging
-import subprocess
-import sys
+import sys, os
+import requests
+import ujson
+from io import BytesIO
+from zipfile import ZipFile
 from pathlib import Path
 
+
+# Download main FAO file
+def download_file(path: Path, url):
+    """ Download, unzip and save FAO file to target_dir"""
+    print("Downloading database from ", url)
+    resp = requests.get(url)
+    with ZipFile(BytesIO(resp.content)) as zfile:
+        # Extract all files from zip and store it to path
+        zfile.extractall(path)
+
+
+def download_dataset_json(path: Path, url):
+    """ Download dataset.json file"""
+    elements = url.split("/")
+    r = requests.get(url)
+    print("Downloading " + elements[-1] + " data")
+    with open(str(path) + "/" + "datasets.json", 'wb') as f:
+        f.write(r.content)
+
+
+def download_flag_json(path: Path, url):
+    """ Download flag.json file"""
+    elements = url.split("/")
+    r = requests.get(url)
+    print("Downloading " + elements[-1] + " data")
+    with open(str(path) + "/" + "flags.json", 'wb') as f:
+        f.write(r.content)
+
+
 log = logging.getLogger(__name__)
 
 
 def main():
-    """ Download data: call download.sh """
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument('target_dir', type=Path,
-                        help='path of target directory')
-    parser.add_argument('--log', default='INFO',
-                        help='level of logging messages')
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('target_dir', type=Path, help='path of target directory')
+    parser.add_argument('--debug-http', action='store_true', help='display http.client debug messages')
+    parser.add_argument('--log', default='WARNING', help='level of logging messages')
     args = parser.parse_args()
 
     numeric_level = getattr(logging, args.log.upper(), None)
@@ -49,16 +78,29 @@ def main():
         level=numeric_level,
         stream=sys.stdout,  # Use stderr if script outputs data to stdout.
     )
+    logging.getLogger("urllib3").setLevel(logging.DEBUG if args.debug_http else logging.WARNING)
+    if args.debug_http:
+        http.client.HTTPConnection.debuglevel = 1
 
     target_dir = args.target_dir
     if not target_dir.exists():
         parser.error("Target dir {!r} not found".format(str(target_dir)))
 
-    log.info("Downloading data...")
-    command = f"./download.sh '{str(target_dir)}'"
-    subprocess.check_call(command, shell=True)
+    url = "http://fenixservices.fao.org/faostat/static/bulkdownloads/FAOSTAT.zip"
+
+    dataset_url = "http://fenixservices.fao.org/faostat/static/bulkdownloads/datasets_E.json"
+
+    flag_url = "http://fenixservices.fao.org/faostat/api/v1/en/definitions/types/flag"
+
+    # Download main FAO file
+    download_file(target_dir, url)
+
+    download_dataset_json(target_dir, dataset_url)
+
+    download_flag_json(target_dir, flag_url)
 
-    return 0
+    #dataset = ujson.loads(target_dir / "datasets.json")
+    #print(dataset)
 
 
 if __name__ == '__main__':