stream source upload

CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.3.4 (2020-02-06)
+- Stream add-source file upload with requests-toolbelt
+
 # 0.3.3 (2020-01-16)
 - Add confirmation prompt to delete-sources
 

requirements.txt

@@ -2,5 +2,6 @@ boto3==1.9.99
 Click==7.0
 cligj==0.5.0
 requests==2.21.0
+requests-toolbelt==0.9.1
 jsonschema==3.0.1
 jsonseq==1.0.0

tests/test_cli_sources.py

@@ -16,16 +16,42 @@
 @pytest.mark.usefixtures("token_environ")
 @mock.patch("requests.post")
 def test_cli_add_source(mock_request_post, MockResponse):
-    message = {"id": "mapbox://tileset-source/test-user/hello-world"}
-    mock_request_post.return_value = MockResponse(message, status_code=200)
+    okay_response = {"id": "mapbox://tileset-source/test-user/hello-world"}
+    mock_request_post.return_value = MockResponse(okay_response, status_code=200)
     runner = CliRunner()
-    result = runner.invoke(
+    validated_result = runner.invoke(
         add_source, ["test-user", "hello-world", "tests/fixtures/valid.ldgeojson"]
     )
-    assert result.exit_code == 0
+    assert validated_result.exit_code == 0
+
+    assert (
+        validated_result.output
+        == """{"id": "mapbox://tileset-source/test-user/hello-world"}\n"""
+    )
+
+
+@pytest.mark.usefixtures("token_environ")
+@mock.patch("requests.post")
+def test_cli_add_source_no_validation(mock_request_post, MockResponse):
+    error_response = {
+        "message": "Invalid file format. Only GeoJSON features are allowed."
+    }
+    mock_request_post.return_value = MockResponse(error_response, status_code=400)
+    runner = CliRunner()
+    no_validation_result = runner.invoke(
+        add_source,
+        [
+            "test-user",
+            "hello-again",
+            "tests/fixtures/invalid.ldgeojson",
+            "--no-validation",
+        ],
+    )
+    assert no_validation_result.exit_code == 1
 
     assert (
-        result.output == """{"id": "mapbox://tileset-source/test-user/hello-world"}\n"""
+        no_validation_result.exception.message
+        == '{"message": "Invalid file format. Only GeoJSON features are allowed."}'
     )
 
 

tilesets/scripts/cli.py

@@ -3,10 +3,9 @@
 import json
 import requests
 
-from io import BytesIO
-
 import click
 import cligj
+from requests_toolbelt import MultipartEncoder
 
 import tilesets
 from tilesets import utils, errors
@@ -303,35 +302,41 @@ def validate_source(features):
 @cli.command("add-source")
 @click.argument("username", required=True, type=str)
 @click.argument("id", required=True, type=str)
-@cligj.features_in_arg
+@click.argument("filename", required=True, type=str)
 @click.option("--no-validation", is_flag=True, help="Bypass source file validation")
 @click.option("--token", "-t", required=False, type=str, help="Mapbox access token")
 @click.option("--indent", type=int, default=None, help="Indent for JSON output")
 @click.pass_context
-def add_source(ctx, username, id, features, no_validation, token=None, indent=None):
+def add_source(ctx, username, id, filename, no_validation, token=None, indent=None):
     """Create/add a tileset source
 
     tilesets add-source <username> <id> <path/to/source/data>
     """
     mapbox_api = _get_api()
     mapbox_token = _get_token(token)
+    url = (
+        f"{mapbox_api}/tilesets/v1/sources/{username}/{id}?access_token={mapbox_token}"
+    )
 
-    with BytesIO() as io:
-        for feature in features:
-            url = f"{mapbox_api}/tilesets/v1/sources/{username}/{id}?access_token={mapbox_token}"
-            if not no_validation:
-                utils.validate_geojson(feature)
-
-            io.write((json.dumps(feature) + "\n").encode("utf-8"))
-
-        io.seek(0)
-
-        r = requests.post(url, files={"file": ("tileset-source", io)})
+    if not no_validation:
+        with open(filename) as ld_geojson_file:
+            for line in ld_geojson_file:
+                utils.validate_geojson(json.loads(line))
+
+    m = MultipartEncoder(fields={"file": ("file", open(filename, "rb"))})
+    resp = requests.post(
+        url,
+        data=m,
+        headers={
+            "Content-Disposition": "multipart/form-data",
+            "Content-type": m.content_type,
+        },
+    )
 
-    if r.status_code == 200:
-        click.echo(json.dumps(r.json(), indent=indent))
+    if resp.status_code == 200:
+        click.echo(json.dumps(resp.json(), indent=indent))
     else:
-        raise errors.TilesetsError(r.text)
+        raise errors.TilesetsError(resp.text)
 
 
 @cli.command("view-source")