Merge pull request #15 from socrata-platform/branweb/python3-upgrade

Branweb/python3 upgrade

.TestDockerfile

@@ -1,10 +1,9 @@
-FROM socrata/base-xenial
+FROM socrata/python3-bionic
 
 WORKDIR /app
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install python-mapnik python-pip
-RUN pip install --upgrade pip 
+    DEBIAN_FRONTEND=noninteractive apt-get -y install python3-mapnik
 
 RUN mkdir -p /app/carto_renderer
 

Dockerfile

@@ -1,16 +1,18 @@
-FROM socrata/base-xenial
+FROM socrata/python3-bionic
 
 WORKDIR /app
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install python-mapnik python-pip
-RUN pip install --upgrade pip 
+    DEBIAN_FRONTEND=noninteractive apt-get -y install python3-mapnik
 
 RUN mkdir -p /app/carto_renderer
 
 ENV LOG_LEVEL INFO
 
-ADD frozen.txt /app/
+COPY bin/freeze-reqs.sh /app/
+COPY dev-requirements.txt /app/
+RUN chmod +x /app/freeze-reqs.sh
+RUN /app/freeze-reqs.sh
 RUN pip install -r /app/frozen.txt
 
 COPY ship.d /etc/ship.d/

README.md

@@ -17,7 +17,7 @@ brew install mapnik
 
 Install Mapnik (Debian/Ubuntu):
 ```
-sudo apt-get install python-mapnik
+sudo apt-get install python3-mapnik
 ```
 
 Install Python Dependencies:
@@ -30,12 +30,6 @@ pip install -r dev-requirements.txt
 bin/start-renderer.sh --dev
 ```
 
-## Examples ##
-Render an image to `test.png`:
-```
-curl -o test.png localhost:4096/render -H 'Content-type: application/json' -d @carto_renderer/examples/main.json
-```
-
 ## Testing ##
 The tests are run using py.test and hypothesis
 

bin/dockerize.sh

@@ -3,6 +3,5 @@
 set -e
 
 cd "$(git rev-parse --show-toplevel 2>/dev/null)" 
-bin/freeze-reqs.sh
 docker build --rm -t carto-renderer .
 

bin/freeze-reqs.sh

@@ -3,17 +3,7 @@
 set -ev
 
 # Change to the project root.
-cd "$(git rev-parse --show-toplevel 2>/dev/null)"
-
-VENV_DIR="venv"
-if [ "${HUDSON_HOME}" ]; then
-    VENV_DIR="${HUDSON_HOME}/carto-renderer/${VENV_DIR}"
-fi
-
-if [ ! -d "${VENV_DIR}" ]; then
-    virtualenv "${VENV_DIR}"
-fi
-source "${VENV_DIR}"/bin/activate
+cd /app
 
 DEV_FILE='dev-requirements.txt'
 FROZEN_FILE='frozen.txt'

bin/start-renderer.sh

@@ -11,20 +11,19 @@ if [ '--dev' = "$1" ]; then
     fi
     source venv/bin/activate
 
-    if [ ! -d venv/lib/python2.7/site-packages/mapnik ]; then
-        ln -s "$MAPNIK_DIR" venv/lib/python2.7/site-packages/
+    if [ ! -d venv/lib/python3.6/site-packages/mapnik ]; then
+        ln -s "$MAPNIK_DIR" venv/lib/python3.6/site-packages/
     fi
 
     pip install --upgrade --requirement dev-requirements.txt
     PYTHONPATH=. carto_renderer/service.py
 else
     bin/dockerize.sh
-    rm frozen.txt
-
+
     if [ 'Darwin' = "$(uname)" ]; then
         OPTS=(-p 4096:4096 -e 'STYLE_HOST=docker.for.mac.localhost')
     else
-        OPTS=('--net=host')
+        OPTS=('--net=host' -e 'STYLE_HOST=localhost')
     fi
 
     docker run "${OPTS[@]}" -e STYLE_PORT=4097 -e LOG_LEVEL=INFO carto-renderer

carto_renderer/errors.py

@@ -10,9 +10,9 @@ class ServiceError(Exception):
     Base class for errors in this service.
     """
     def __init__(self, message, status_code, request_body=None):
-        super(ServiceError, self).__init__(message)
         self.status_code = status_code
         self.request_body = request_body
+        self.message = message
         logger = get_logger(self)
         if request_body:
             logger.error('Fatal Error (%d): "%s"; body: "%s"',
@@ -41,6 +41,6 @@ def __init__(self, keys, blob):
         message = ''
 
         beg = keys[:-1]
-        message = PayloadKeyError.msg.format("', '".join(beg), keys[-1])
+        message = PayloadKeyError.msg.format(b"', '".join(beg), keys[-1])
 
         super(PayloadKeyError, self).__init__(message, 400, request_body=blob)

carto_renderer/service.py

@@ -4,9 +4,9 @@
 """
 
 import json
-from urllib import quote_plus
+from urllib.parse import quote_plus
 
-from tornado import web
+from tornado import web, escape
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 from tornado.ioloop import IOLoop
 from tornado.options import define, parse_command_line, options
@@ -49,8 +49,7 @@ def render_png(tile, _zoom, xml, overscan):
     box_max = TILE_SIZE + overscan - 1
     map_tile.zoom_to_box(mapnik.Box2d(box_min, box_min, box_max, box_max))
 
-    for (name, features) in tile.items():
-        name = name.encode('ascii', 'ignore')
+    for (name, features) in list(tile.items()):
         source = mapnik.MemoryDatasource()
         map_layer = mapnik.Layer(name)
         map_layer.datasource = source
@@ -103,10 +102,10 @@ def extract_body(self):
         body = self.request.body
 
         try:
-            extracted = msgpack.loads(body)
+            extracted = msgpack.loads(body, raw=True)
         except Exception:
             logger.warn('Invalid message')
-            raise BadRequest('Could not parse message.', body)
+            raise BadRequest('Could not parse message.', escape.to_unicode(body))
         return extracted
 
     def _handle_request_exception(self, err):
@@ -119,13 +118,14 @@ def _handle_request_exception(self, err):
         logger.exception(err)
         if isinstance(err, ServiceError):
             status_code = err.status_code
+            payload['message'] = err.message
             if err.request_body:
                 payload['request_body'] = err.request_body
         else:
+            payload['message'] = str(err)
             status_code = 500
 
         payload['resultCode'] = status_code
-        payload['message'] = err.message
 
         self.clear()
         self.set_status(status_code)
@@ -168,16 +168,16 @@ class RenderHandler(BaseHandler):
 
     Expects a dictionary with 'style', 'zoom', and 'tile' values.
     """
-    keys = ['tile', 'zoom', 'style']
+    keys = [b'tile', b'zoom', b'style']
 
     def initialize(self, http_client, style_host, style_port):
         """Magic Tornado __init__ replacement."""
         self.http_client = http_client  # pragma: no cover
         self.style_host = style_host    # pragma: no cover
         self.style_port = style_port    # pragma: no cover
 
-    @web.asynchronous
-    def post(self):
+#    @web.asynchronous
+    async def post(self):
         """
         Actually render the png.
 
@@ -192,15 +192,15 @@ def post(self):
             raise PayloadKeyError(self.keys, geobody)
         else:
             try:
-                overscan = int(geobody['overscan'])
+                overscan = int(geobody[b'overscan'])
             except:
                 logger.warn('Invalid JSON; overscan must be an integer: %s',
                             geobody)
                 raise BadRequest('"overscan" must be an integer',
                                  request_body=geobody)
 
             try:
-                zoom = int(geobody['zoom'])
+                zoom = int(geobody[b'zoom'])
             except:
                 logger.warn('Invalid JSON; zoom must be an integer: %s',
                             geobody)
@@ -210,9 +210,9 @@ def post(self):
             path = 'http://{host}:{port}/style?style={css}'.format(
                 host=self.style_host,
                 port=self.style_port,
-                css=quote_plus(geobody['style']))
+                css=quote_plus(geobody[b'style']))
 
-            tile = geobody['tile']
+            tile = geobody[b'tile']
 
             def handle_response(response):
                 """
@@ -228,7 +228,7 @@ def handle_response(response):
 
                 logger.info('zoom: %d, num features: %d, len(xml): %d',
                             zoom,
-                            sum([len(layer) for layer in tile.values()]),
+                            sum([len(layer) for layer in list(tile.values())]),
                             len(xml))
                 logger.debug('xml: %s',
                              LogWrapper.Lazy(lambda: xml.replace('\n', ' ')))
@@ -241,7 +241,8 @@ def handle_response(response):
                 else {}
 
             req = HTTPRequest(path, headers=headers)
-            self.http_client.fetch(req, callback=handle_response)
+            resp = await self.http_client.fetch(req)
+            handle_response(resp)
 
 
 def main():  # pragma: no cover