From 8cbdd2516c5e467a81f2ba1665cc24e5370df891 Mon Sep 17 00:00:00 2001
From: Constance de Quatrebarbes <constance.24barbes@cepremap.org>
Date: Tue, 3 Oct 2017 10:35:43 +0200
Subject: [PATCH] Add dares_to_source_data + settings: download xls* files and
html source pages
---
dares_to_source_data.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/dares_to_source_data.py b/dares_to_source_data.py
index 1807ee6..daf15f4 100644
--- a/dares_to_source_data.py
+++ b/dares_to_source_data.py
@@ -72,12 +72,11 @@ def main():
# #reset the repo by changing remote url of the git and push ?
# pass
for cat in TOP_CATEGORIES:
- # print(cat)
url = os.path.join(SOURCE_FILES_URL, cat["page_slug"])
resp = requests.get(url)
assert resp is not None, "requests.get() failed with url entrypoint %s" %url
assert resp.status_code in range(200, 399), "requests response.status_code == %s" %resp.status_code
- # download html source_file on it's page-slug
+ # download html source_file based one page-slug declared in SETTINGS
html_file_path = os.path.join(target_dir, cat["page_slug"]+".html")
with open(html_file_path, "w") as html_file:
html_file.write(resp.text)
--
GitLab