diff --git a/dares_to_source_data.py b/dares_to_source_data.py index 1807ee63c5ca751ac86de20704ab70861d785081..daf15f422cd7b71445015758b086918190d2544c 100644 --- a/dares_to_source_data.py +++ b/dares_to_source_data.py @@ -72,12 +72,11 @@ def main(): # #reset the repo by changing remote url of the git and push ? # pass for cat in TOP_CATEGORIES: - # print(cat) url = os.path.join(SOURCE_FILES_URL, cat["page_slug"]) resp = requests.get(url) assert resp is not None, "requests.get() failed with url entrypoint %s" %url assert resp.status_code in range(200, 399), "requests response.status_code == %s" %resp.status_code - # download html source_file on it's page-slug + # download html source_file based one page-slug declared in SETTINGS html_file_path = os.path.join(target_dir, cat["page_slug"]+".html") with open(html_file_path, "w") as html_file: html_file.write(resp.text)