Merge pull request #62 from oslokommune/DP-1619-public-data-no-auth

DP-1619 Allow unauthenticated access to public data

Author: simenheg

CHANGELOG.md

@@ -1,3 +1,8 @@
+## ?.?.?
+
+* Authentication is no longer necessary for downloading public ("green")
+  datasets.
+
 ## 0.6.1
 
 * `PostEvent.post_event` now also supports retries.

README.md

@@ -52,6 +52,7 @@ If environment variables are not available, the system will try to load from a d
 
 Table of contents:
 - [Upload data](#upload-data)
+- [Download data](#download-data)
 - [Sending events](#sending-events)
 - [Create and manage event streams](#create-and-manage-event-streams)
 - [Creating datasets with versions and editions](#creating-datasets-with-versions-and-editions)
@@ -60,7 +61,7 @@ Table of contents:
 ## Upload data
 
 When uploading data you need to refer to an existing dataset that you own, a version and an edition.
-If these are non existent then you can create them yourself. This can be achieved [using the sdk](#create-a-new-dataset-with-version-and-edition),
+If these are non existent then you can create them yourself. This can be achieved [using the sdk](#creating-datasets-with-versions-and-editions),
 or you can use our [command line interface](https://github.com/oslokommune/okdata-cli).
 
 
@@ -125,9 +126,15 @@ print(trace_events)
 
 ## Download data
 
-When downloading data you need to refer to an existing dataset that you own, a version and an edition.
-If these are non existent then you can create them yourself. This can be achieved [using the sdk](#create-a-new-dataset-with-version-and-edition),
-or you can use our [command line interface](https://github.com/oslokommune/okdata-cli).
+To download data you need to refer to a dataset that you have access to. This
+could be a public dataset, a restricted dataset you've been given access to, or
+a dataset that you own yourself. If the dataset is public, [authenticating
+yourself](#environment-variables) is not necessary.
+
+You will also need to refer to the specific version and edition of the dataset
+that you want to download. If this is your own dataset, make sure to create a
+[version and edition](#creating-datasets-with-versions-and-editions) before
+attempting to download it.
 
 ```python
 from okdata.sdk.data.download import Download

okdata/sdk/data/download.py

@@ -15,13 +15,14 @@ def __init__(self, config=None, auth=None, env=None):
         self.data_exporter_url = self.config.get("dataExporterUrl")
 
     def get_files(self, dataset_id, version, edition, retries=0):
-
-        get_download_urls_url = (
-            f"{self.data_exporter_url}/{dataset_id}/{version}/{edition}"
+        url = "{}/{}{}/{}/{}".format(
+            self.data_exporter_url,
+            "" if self.auth.token_provider else "public/",
+            dataset_id,
+            version,
+            edition,
         )
-
-        response = self.get(get_download_urls_url, retries=retries)
-        return response.json()
+        return self.get(url, retries=retries).json()
 
     def download(self, dataset_id, version, edition, output_path, retries=0):
         downloaded_files = []

tests/data/download_test.py

@@ -4,8 +4,6 @@
 
 from okdata.sdk.data.download import Download
 
-data_downloader = Download()
-
 file_name = "kake.csv"
 s3_key = f"procecced/raw/green/{file_name}"
 download_url = "https://www.dowload-stuff.com"
@@ -18,6 +16,20 @@
 
 
 def test_download(mock_home_dir, mock_http_calls):
+    data_downloader = Download()
+    output_path = f"{os.environ['HOME']}/my/path"
+    result = data_downloader.download(
+        dataset_id, version, edition, output_path=output_path
+    )
+    exp_output_file_path = f"{output_path}/{file_name}"
+    with open(exp_output_file_path, "r") as f:
+        assert str(f.read()) == test_file_content
+    assert result == {"files": [exp_output_file_path]}
+
+
+def test_download_public(mock_home_dir, mock_http_calls_public):
+    data_downloader = Download()
+    data_downloader.auth.token_provider = None
     output_path = f"{os.environ['HOME']}/my/path"
     result = data_downloader.download(
         dataset_id, version, edition, output_path=output_path
@@ -38,7 +50,21 @@ def mock_http_calls(requests_mock):
 
     requests_mock.register_uri(
         "GET",
-        f"{data_downloader.data_exporter_url}/{dataset_id}/{version}/{edition}",
+        f"{Download().data_exporter_url}/{dataset_id}/{version}/{edition}",
+        text=json.dumps([{"key": s3_key, "url": download_url}]),
+        status_code=200,
+    )
+
+    requests_mock.register_uri(
+        "GET", download_url, text=test_file_content, status_code=200
+    )
+
+
+@pytest.fixture(scope="function")
+def mock_http_calls_public(requests_mock):
+    requests_mock.register_uri(
+        "GET",
+        f"{Download().data_exporter_url}/public/{dataset_id}/{version}/{edition}",
         text=json.dumps([{"key": s3_key, "url": download_url}]),
         status_code=200,
     )