From 3a4b3b440858519724503f2361d678bb25aecb4e Mon Sep 17 00:00:00 2001 From: Matthew Blissett Date: Fri, 1 Sep 2017 15:30:34 +0200 Subject: [PATCH 1/5] WIP: add buffers to layers (according to layer configuration). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multithread application — this is done badly, as I don't know the proper way to do it in Python. --- Dockerfile | 8 ++- postserve.html | 35 ++++++++++ postserve.js | 106 ++++++++++++++++++++++++++++ server.py | 182 +++++++++++++++++++++++++++++++++++++------------ 4 files changed, 287 insertions(+), 44 deletions(-) create mode 100644 postserve.html create mode 100644 postserve.js diff --git a/Dockerfile b/Dockerfile index 06bc6a4..4d06e0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.4 +FROM python:3.6 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app @@ -7,4 +7,8 @@ VOLUME /mapping COPY . /usr/src/app/ RUN pip install --no-cache-dir -r requirements.txt -CMD ["python", "-u","/usr/src/app/server.py"] +CMD ["python", "-u","/usr/src/app/server.py"] + +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + emacs-nox diff --git a/postserve.html b/postserve.html new file mode 100644 index 0000000..c43c7b5 --- /dev/null +++ b/postserve.html @@ -0,0 +1,35 @@ + + + + + EPSG:4326 map preview + + + + + + + + +
+
+
+ + + diff --git a/postserve.js b/postserve.js new file mode 100644 index 0000000..6814023 --- /dev/null +++ b/postserve.js @@ -0,0 +1,106 @@ +proj4.defs('EPSG:4326', "+proj=longlat +ellps=WGS84 +datum=WGS84 +units=degrees"); + +var resolutions = []; +var extent = 180.0; +var tile_size = 512; +var resolutions = Array(17).fill().map((_, i) => ( extent / tile_size / Math.pow(2, i) )); + +var layers = []; + +densityColours = ["#FFFF00", "#FFCC00", "#FF9900", "#FF6600", "#FF3300", "#FF0000"]; + +function createDensityStyle2() { + var point = new ol.style.Style({ + image: new ol.style.Circle({ + fill: new ol.style.Fill({color: '#FF0000'}), + radius: 1 + }), + fill: new ol.style.Fill({color: '#FF0000'}) + }); + + var styles = []; + return function(feature, resolution) { + var length = 0; + //console.log(feature); + var magnitude = Math.trunc(Math.min(5, Math.floor(Math.log(feature.get('total'))))) - 1; + //console.log("Colour ", magnitude, densityColours[magnitude]); + //styles[length++] = point; + styles[length++] = new ol.style.Style({ + image: new ol.style.Circle({ + fill: new ol.style.Fill({color: densityColours[magnitude]}), + radius: 1 + }), + fill: new ol.style.Fill({color: densityColours[magnitude]}) + }); + styles.length = length; + return styles; + }; +} + +function createStatsStyle() { + var fill = new ol.style.Fill({color: '#000000'}); + var stroke = new ol.style.Stroke({color: '#000000', width: 1}); + var text = new ol.style.Text({ + text: 'XYXYXY', + fill: fill, + stroke: stroke, + font: '16px "Open Sans", "Arial Unicode MS"' + }); + + + var styles = []; + return function(feature, resolution) { + var length = 0; + //console.log(feature); + text.setText('Occurrences: '+feature.get('total')); + console.log(feature.get('total')); + styles[length++] = new ol.style.Style({ + stroke: new ol.style.Stroke({color: '#000000'}), + text: text + }); + styles.length = length; + return styles; + }; +} + +var tileGrid = new ol.tilegrid.TileGrid({ + extent: ol.proj.get('EPSG:4326').getExtent(), + minZoom: 0, + maxZoom: 16, + resolutions: resolutions, + tileSize: 512, +}); + +layers['EPSG:4326'] = new ol.layer.VectorTile({ + source: new ol.source.VectorTile({ + projection: 'EPSG:4326', + format: new ol.format.MVT(), + tileGrid: tileGrid, + tilePixelRatio: 8, + url: 'http://mb.gbif.org:8080/tiles/{z}_{x}_{y}.pbf', + wrapX: false + }), + style: createStyle(), +}); + +layers['Grid'] = new ol.layer.Tile({ + extent: ol.proj.get('EPSG:4326').getExtent(), + source: new ol.source.TileDebug({ + projection: 'EPSG:4326', + tileGrid: tileGrid, + wrapX: false + }), +}); + +var map = new ol.Map({ + layers: [ + layers['EPSG:4326'], + layers['Grid'] + ], + target: 'map', + view: new ol.View({ + center: [10.7522, 59.9139], + projection: 'EPSG:4326', + zoom: 10 + }), +}); diff --git a/server.py b/server.py index 8122558..db47992 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,6 @@ import tornado.ioloop import tornado.web +import tornado.httpserver import io import os @@ -22,81 +23,178 @@ def GetTM2Source(file): tm2source = yaml.load(stream) return tm2source +buffer_sizes = [] + def GeneratePrepared(layers): queries = [] - prepared = "PREPARE gettile(geometry, numeric, numeric, numeric) AS " for layer in layers['Layer']: + buffer_size = str(layer['properties']['buffer-size']*2) + if (buffer_size not in buffer_sizes): + buffer_sizes.append(buffer_size) layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t" - layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox!,4096,0,true) AS mvtgeometry") - base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile" - queries.append(base_query.replace("!bbox!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4")) + layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox_nobuffer!,4096,"+buffer_size+",true) AS mvtgeometry") + layer_query = layer_query.replace("!bbox!", "!bbox_"+buffer_size+"_buffer!") + base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry,!bbox_nobuffer!,4096,"+buffer_size+",true) IS NOT NULL) AS tile" + queries.append(base_query.replace("!bbox_nobuffer!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4").replace("!bbox_"+buffer_size+"_buffer!", "$"+str(buffer_sizes.index(buffer_size)+5))) + + chunk = "" + for buffer_size in buffer_sizes: + chunk = chunk + ", geometry" + + prepared = "PREPARE gettile(geometry, numeric, numeric, numeric"+chunk+") AS " prepared = prepared + " UNION ALL ".join(queries) + ";" print(prepared) return(prepared) layers = GetTM2Source("/mapping/data.yml") prepared = GeneratePrepared(layers) -engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles')) -inspector = inspect(engine) -DBSession = sessionmaker(bind=engine) -session = DBSession() -session.execute(prepared) - -def bounds(zoom,x,y): - inProj = pyproj.Proj(init='epsg:4326') - outProj = pyproj.Proj(init='epsg:3857') - lnglatbbox = mercantile.bounds(x,y,zoom) - ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1])) - en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3])) + +def bounds(zoom,x,y,buff): + #inProj = pyproj.Proj(init='epsg:3575') + #outProj = pyproj.Proj(init='epsg:3575') + #lnglatbbox = mercantile.bounds(x,y,zoom) + #ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1])) + #en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3])) + + #map_width_in_metres = 2 * 2**0.5*6371007.2 + map_width_in_metres = 180.0 + #tile_width_in_pixels = 512.0 + #standardized_pixel_size = 0.00028 + #map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) + + tiles_down = 2**(zoom) + tiles_across = 2**(zoom) + + #arc x = x - 2**(zoom-1) + #arctic y = -(y - 2**(zoom-1)) - 1 + x = x - 2**(zoom) + y = -(y - 2**(zoom)) - 1 - 2**(zoom-1) + + print(x, y); + + tile_width_in_metres = (map_width_in_metres / tiles_across) + tile_height_in_metres = (map_width_in_metres / tiles_down) + ws = ((x - buff)*tile_width_in_metres, (y - buff)*tile_width_in_metres) + en = ((x+1+buff)*tile_height_in_metres, (y+1+buff)*tile_height_in_metres) + + print("Zoom, buffer", zoom, buff) + print("West: ", ws[0]) + print("South: ", ws[1]) + print("East: ", en[0]) + print("North: ", en[1]) + return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]} def zoom_to_scale_denom(zoom): # For !scale_denominator! # From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt + #map_width_in_metres = 2 * 2**0.5*6371007.2 # Arctic + #map_width_in_metres = 180.0 map_width_in_metres = 40075016.68557849 - tile_width_in_pixels = 256.0 + tile_width_in_pixels = 512.0 # This asks for a zoom level higher, since the tiles are doubled. standardized_pixel_size = 0.00028 map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size)) -def replace_tokens(query,s,w,n,e,scale_denom): - return query.replace("!bbox!","ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+"))").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","256").replace("!pixel_height!","256") +def replace_tokens(query,tilebounds,buffered_tilebounds,scale_denom,z): + s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) -def get_mvt(zoom,x,y): - try: # Sanitize the inputs - sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) - del zoom,x,y - except: - print('suspicious') - return 1 + start = query.replace("!bbox!","ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 4326)").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","512").replace("!pixel_height!","512") - scale_denom = zoom_to_scale_denom(sani_zoom) - tilebounds = bounds(sani_zoom,sani_x,sani_y) - s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) - final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!);" - sent_query = replace_tokens(final_query,s,w,n,e,scale_denom) - response = list(session.execute(sent_query)) - print(sent_query) - layers = filter(None,list(itertools.chain.from_iterable(response))) - final_tile = b'' - for layer in layers: - final_tile = final_tile + io.BytesIO(layer).getvalue() - return final_tile + for buffer_size in buffer_sizes: + token = "!bbox_"+buffer_size+"_buffer!" + idx = buffer_sizes.index(buffer_size) + + t = buffered_tilebounds[idx] + s,w,n,e = str(t['s']),str(t['w']),str(t['n']),str(t['e']) + start = start.replace(token,"ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 4326)") + + return start class GetTile(tornado.web.RequestHandler): + def get(self, zoom,x,y): + engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles')) + inspector = inspect(engine) + DBSession = sessionmaker(bind=engine) + self.session = DBSession() + print("Running prepare statement") + self.session.execute(prepared) + self.set_header("Content-Type", "application/x-protobuf") self.set_header("Content-Disposition", "attachment") self.set_header("Access-Control-Allow-Origin", "*") - response = get_mvt(zoom,x,y) - self.write(response) + response = self.get_mvt(zoom,x,y) + + self.session.close() + + #self.write(response) + + def get_mvt(self, zoom,x,y): + try: # Sanitize the inputs + sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) + del zoom,x,y + except: + print('suspicious') + return 1 + + scale_denom = zoom_to_scale_denom(sani_zoom) + tilebounds = bounds(sani_zoom,sani_x,sani_y,0) + #s_nobuffer,w_nobuffer,n_nobuffer,e_nobuffer = str(tilebounds_nobuffer['s']),str(tilebounds_nobuffer['w']),str(tilebounds_nobuffer['n']),str(tilebounds_nobuffer['e']) + + #tilebounds = bounds(sani_zoom,sani_x,sani_y,0.50) + #s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) + + buffered_tilebounds = [] + chunk = "" + for buffer_size in buffer_sizes: + t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0) + buffered_tilebounds.append(t) + + chunk = chunk + ", !bbox_"+buffer_size+"_buffer!" + + final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");" + sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom) + print('Final query', sent_query) + response = list(self.session.execute(sent_query)) + print (sani_zoom, sani_x, sani_y) + print(sent_query) + layers = filter(None,list(itertools.chain.from_iterable(response))) + final_tile = b'' + for layer in layers: + final_tile = final_tile + io.BytesIO(layer).getvalue() + self.write(final_tile) + #return final_tile + +# Buffers +# aeroway.yaml: buffer_size: 4 +# boundary.yaml: buffer_size: 4 +# building.yaml: buffer_size: 4 +# landcover.yaml: buffer_size: 4 +# landuse.yaml: buffer_size: 4 +# park.yaml: buffer_size: 4 +# transportation.yaml: buffer_size: 4 +# water.yaml: buffer_size: 4 +# waterway.yaml: buffer_size: 4 +# housenumber.yaml: buffer_size: 8 +# transportation_name.yaml: buffer_size: 8 +# graticules.yaml: buffer_size: 64 +# mountain_peak.yaml: buffer_size: 64 +# poi.yaml: buffer_size: 64 +# place.yaml: buffer_size: 256 +# water_name.yaml: buffer_size: 256 +# Need 4/256, 8/256, 64/256 and 256/256 def m(): if __name__ == "__main__": # Make this prepared statement from the tm2source - application = tornado.web.Application([(r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile)]) + application = tornado.web.Application([(r"/tiles/([0-9]+)_([0-9]+)_([0-9]+).pbf", GetTile)]) + + server = tornado.httpserver.HTTPServer(application) + server.bind(8080) + server.start(0) print("Postserve started..") - application.listen(8080) + #application.listen(8080) tornado.ioloop.IOLoop.instance().start() m() From 6aaef03c5e6a51340b84b2836963c3a9983cee63 Mon Sep 17 00:00:00 2001 From: Matthew Blissett Date: Mon, 4 Sep 2017 11:56:57 +0200 Subject: [PATCH 2/5] User interface. --- server.py | 5 ++++- postserve.html => static/index.html | 0 postserve.js => static/postserve.js | 28 +--------------------------- 3 files changed, 5 insertions(+), 28 deletions(-) rename postserve.html => static/index.html (100%) rename postserve.js => static/postserve.js (74%) diff --git a/server.py b/server.py index db47992..13a6e20 100644 --- a/server.py +++ b/server.py @@ -188,7 +188,10 @@ class GetTile(tornado.web.RequestHandler): def m(): if __name__ == "__main__": # Make this prepared statement from the tm2source - application = tornado.web.Application([(r"/tiles/([0-9]+)_([0-9]+)_([0-9]+).pbf", GetTile)]) + application = tornado.web.Application([ + (r"/tiles/([0-9]+)[/_]([0-9]+)[/_]([0-9]+).pbf", GetTile), + (r"/([^/]*)", tornado.web.StaticFileHandler, {"path": "./static", "default_filename": "index.html"}) + ]) server = tornado.httpserver.HTTPServer(application) server.bind(8080) diff --git a/postserve.html b/static/index.html similarity index 100% rename from postserve.html rename to static/index.html diff --git a/postserve.js b/static/postserve.js similarity index 74% rename from postserve.js rename to static/postserve.js index 6814023..a853e0f 100644 --- a/postserve.js +++ b/static/postserve.js @@ -37,32 +37,6 @@ function createDensityStyle2() { }; } -function createStatsStyle() { - var fill = new ol.style.Fill({color: '#000000'}); - var stroke = new ol.style.Stroke({color: '#000000', width: 1}); - var text = new ol.style.Text({ - text: 'XYXYXY', - fill: fill, - stroke: stroke, - font: '16px "Open Sans", "Arial Unicode MS"' - }); - - - var styles = []; - return function(feature, resolution) { - var length = 0; - //console.log(feature); - text.setText('Occurrences: '+feature.get('total')); - console.log(feature.get('total')); - styles[length++] = new ol.style.Style({ - stroke: new ol.style.Stroke({color: '#000000'}), - text: text - }); - styles.length = length; - return styles; - }; -} - var tileGrid = new ol.tilegrid.TileGrid({ extent: ol.proj.get('EPSG:4326').getExtent(), minZoom: 0, @@ -77,7 +51,7 @@ layers['EPSG:4326'] = new ol.layer.VectorTile({ format: new ol.format.MVT(), tileGrid: tileGrid, tilePixelRatio: 8, - url: 'http://mb.gbif.org:8080/tiles/{z}_{x}_{y}.pbf', + url: '/tiles/{z}_{x}_{y}.pbf', wrapX: false }), style: createStyle(), From dc12747c0f94cb421a2dea298b813d30306c82d7 Mon Sep 17 00:00:00 2001 From: Matthew Blissett Date: Mon, 4 Sep 2017 16:03:17 +0200 Subject: [PATCH 3/5] Rabbit WIP. --- queue.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + server.py | 110 ++++++++++++++++++--------------------------- 3 files changed, 157 insertions(+), 68 deletions(-) create mode 100644 queue.py diff --git a/queue.py b/queue.py new file mode 100644 index 0000000..0932778 --- /dev/null +++ b/queue.py @@ -0,0 +1,113 @@ +import server + +import pika +from msgpack import packb +import signal + +from json import loads as json_loads + +import sys +from sys import stderr + +mq_i = 0 +exiting = False + +to_queue_name = os.getenv('TO_QUEUE', 'made-tiles') +failed_queue_name = os.getenv('FAILED_QUEUE', 'failed-tiles') + +class Coordinate: + def __init__(self, row, column, zoom): + self.row = row + self.column = column + self.zoom = zoom + +def makeTile(ch, method, properties, data): + body = json_loads(data) + coord = Coordinate(body['coord']['row'], body['coord']['col'], body['coord']['zoom']) + path = '%d/%d/%d' % (coord.zoom, coord.column, coord.row) + + # Fetch a tile. + rendered = False + + global mq_i + global exiting + + while not rendered: + try: + content = server.get_mvt(coord.zoom, coord.column, coord.row) + + mq_i = mq_i + 1 + if (mq_i % 100 == 0): + print('ToMQ', mq_i, coord) + + msg = packb({ + "zoom": coord.zoom, + "column": coord.column, + "row": coord.row, + "tile": content + }) + + mq_done_channel.basic_publish(exchange='', routing_key=to_queue_name, body=msg) + + except: + # Something went wrong: try again? Log the error? + + print('Exception') + global exiting + if exiting: + print('Exiting.') + ch.close() + break + + print('Failed', coord, file=sys.stderr) + global dud_channel + dud_channel.basic_publish(exchange='', routing_key=failed_queue_name, body=data) + ch.basic_nack(delivery_tag = method.delivery_tag, requeue=False) + # MAX_ERRORS? + #if not error_list: + raise + #break + + else: + # Successfully got the tile. + rendered = True + ch.basic_ack(delivery_tag = method.delivery_tag) + +def exit_handler(signal, frame): + global exiting + exiting = True + global channel + global consumer_tag + print('You pressed Ctrl+C!') + channel.basic_cancel(consumer_tag=consumer_tag) + channel.close() + + sys.exit(0) + + +def m(): + if __name__ == "__main__": + mq_credentials = pika.PlainCredentials(os.getenv('MQ_USER','mblissett'), os.getenv('MQ_PASSWORD','mblissett')) + mq_connection = pika.BlockingConnection(pika.ConnectionParameters(os.getenv('MQ_HOST','mq.gbif.org'), os.getenv('MQ_PORT','5672'), os.getenv('MQ_VHOST','/users/mblissett'), mq_credentials)) + + global mq_done_channel + mq_done_channel = mq_connection.channel() + mq_done_channel.queue_declare(queue=os.getenv('TO_QUEUE','made-tiles')) + + global mq_dud_channel + mq_dud_channel = mq_connection.channel() + mq_dud_channel.queue_declare(queue=os.getenv('FAILED_QUEUE','failed-tiles')) + + global mq_channel + mq_channel = mq_connection.channel() + mq_channel.queue_declare(queue=os.getenv('FROM_QUEUE','do-tiles')) + mq_channel.basic_qos(prefetch_count=50) + global consumer_tag + consumer_tag = mq_channel.basic_consume(makeTile, queue=os.getenv('FROM_QUEUE','do-tiles')) + + signal.signal(signal.SIGINT, exit_handler) + + print('Waiting for messages. To exit press CTRL+C') + mq_channel.start_consuming() + +m() diff --git a/requirements.txt b/requirements.txt index bb95b17..fc0f2f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ mercantile==0.9.0 pyproj==1.9.5.1 pyyaml==3.12 psycopg2==2.6.2 +pika +msgpack-python diff --git a/server.py b/server.py index 13a6e20..b4e7334 100644 --- a/server.py +++ b/server.py @@ -49,6 +49,13 @@ def GeneratePrepared(layers): layers = GetTM2Source("/mapping/data.yml") prepared = GeneratePrepared(layers) +connection_string = 'postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles') +engine = create_engine(connection_string) +inspector = inspect(engine) +DBSession = sessionmaker(bind=engine) +session = DBSession() +print("Running prepare statement") +session.execute(prepared) def bounds(zoom,x,y,buff): #inProj = pyproj.Proj(init='epsg:3575') @@ -111,79 +118,45 @@ def replace_tokens(query,tilebounds,buffered_tilebounds,scale_denom,z): return start +def get_mvt(zoom,x,y): + try: # Sanitize the inputs + sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) + del zoom,x,y + except: + print('suspicious') + return 1 + + scale_denom = zoom_to_scale_denom(sani_zoom) + tilebounds = bounds(sani_zoom,sani_x,sani_y,0) + + buffered_tilebounds = [] + chunk = "" + for buffer_size in buffer_sizes: + t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0) + buffered_tilebounds.append(t) + + chunk = chunk + ", !bbox_"+buffer_size+"_buffer!" + + final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");" + sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom) + print('Final query', sent_query) + response = list(session.execute(sent_query)) + print (sani_zoom, sani_x, sani_y) + print(sent_query) + layers = filter(None,list(itertools.chain.from_iterable(response))) + final_tile = b'' + for layer in layers: + final_tile = final_tile + io.BytesIO(layer).getvalue() + return final_tile + class GetTile(tornado.web.RequestHandler): def get(self, zoom,x,y): - engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles')) - inspector = inspect(engine) - DBSession = sessionmaker(bind=engine) - self.session = DBSession() - print("Running prepare statement") - self.session.execute(prepared) - self.set_header("Content-Type", "application/x-protobuf") self.set_header("Content-Disposition", "attachment") self.set_header("Access-Control-Allow-Origin", "*") - response = self.get_mvt(zoom,x,y) - - self.session.close() - - #self.write(response) - - def get_mvt(self, zoom,x,y): - try: # Sanitize the inputs - sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) - del zoom,x,y - except: - print('suspicious') - return 1 - - scale_denom = zoom_to_scale_denom(sani_zoom) - tilebounds = bounds(sani_zoom,sani_x,sani_y,0) - #s_nobuffer,w_nobuffer,n_nobuffer,e_nobuffer = str(tilebounds_nobuffer['s']),str(tilebounds_nobuffer['w']),str(tilebounds_nobuffer['n']),str(tilebounds_nobuffer['e']) - - #tilebounds = bounds(sani_zoom,sani_x,sani_y,0.50) - #s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) - - buffered_tilebounds = [] - chunk = "" - for buffer_size in buffer_sizes: - t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0) - buffered_tilebounds.append(t) - - chunk = chunk + ", !bbox_"+buffer_size+"_buffer!" - - final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");" - sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom) - print('Final query', sent_query) - response = list(self.session.execute(sent_query)) - print (sani_zoom, sani_x, sani_y) - print(sent_query) - layers = filter(None,list(itertools.chain.from_iterable(response))) - final_tile = b'' - for layer in layers: - final_tile = final_tile + io.BytesIO(layer).getvalue() - self.write(final_tile) - #return final_tile - -# Buffers -# aeroway.yaml: buffer_size: 4 -# boundary.yaml: buffer_size: 4 -# building.yaml: buffer_size: 4 -# landcover.yaml: buffer_size: 4 -# landuse.yaml: buffer_size: 4 -# park.yaml: buffer_size: 4 -# transportation.yaml: buffer_size: 4 -# water.yaml: buffer_size: 4 -# waterway.yaml: buffer_size: 4 -# housenumber.yaml: buffer_size: 8 -# transportation_name.yaml: buffer_size: 8 -# graticules.yaml: buffer_size: 64 -# mountain_peak.yaml: buffer_size: 64 -# poi.yaml: buffer_size: 64 -# place.yaml: buffer_size: 256 -# water_name.yaml: buffer_size: 256 -# Need 4/256, 8/256, 64/256 and 256/256 + response = get_mvt(zoom,x,y) + self.write(response) def m(): if __name__ == "__main__": @@ -195,9 +168,10 @@ def m(): server = tornado.httpserver.HTTPServer(application) server.bind(8080) - server.start(0) + server.start(1) print("Postserve started..") #application.listen(8080) + tornado.ioloop.IOLoop.instance().start() m() From 30e37ac9216f2884200004b9aff26a0f8e516255 Mon Sep 17 00:00:00 2001 From: Matthew Blissett Date: Wed, 16 Jan 2019 17:44:49 +0100 Subject: [PATCH 4/5] Pull in requirements sooner. --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d06e0b..f0974c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,13 @@ WORKDIR /usr/src/app VOLUME /mapping -COPY . /usr/src/app/ +COPY requirements.txt /usr/src/app/ RUN pip install --no-cache-dir -r requirements.txt -CMD ["python", "-u","/usr/src/app/server.py"] - RUN DEBIAN_FRONTEND=noninteractive apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ emacs-nox + +CMD ["python", "-u","/usr/src/app/server.py"] + +COPY . /usr/src/app/ From aeed0f4737b300f73d3d86bc0880da197d8a654a Mon Sep 17 00:00:00 2001 From: Matthew Blissett Date: Thu, 11 Jul 2019 16:38:09 +0200 Subject: [PATCH 5/5] Move 4326 server and create 3575 server. --- Dockerfile | 3 +- make | 5 + queue_3575.py | 113 ++++++++++++++++++++++ queue.py => queue_4326.py | 10 +- run-consumers | 32 +++++++ server_3575.py | 185 ++++++++++++++++++++++++++++++++++++ server.py => server_4326.py | 17 ++-- start.sh | 3 + static/index_3575.html | 37 ++++++++ static/postserve_3575.js | 122 ++++++++++++++++++++++++ 10 files changed, 512 insertions(+), 15 deletions(-) create mode 100755 make create mode 100644 queue_3575.py rename queue.py => queue_4326.py (90%) create mode 100755 run-consumers create mode 100644 server_3575.py rename server.py => server_4326.py (96%) create mode 100644 start.sh create mode 100644 static/index_3575.html create mode 100644 static/postserve_3575.js diff --git a/Dockerfile b/Dockerfile index f0974c2..db478f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ emacs-nox -CMD ["python", "-u","/usr/src/app/server.py"] +#CMD ["python", "-u","/usr/src/app/server.py"] +CMD ["sh","/usr/src/app/start.sh"] COPY . /usr/src/app/ diff --git a/make b/make new file mode 100755 index 0000000..e7fea2a --- /dev/null +++ b/make @@ -0,0 +1,5 @@ +#!/bin/sh -e + +rm *~ -f +docker build -t mb.gbif.org:5000/postserve . +docker push mb.gbif.org:5000/postserve diff --git a/queue_3575.py b/queue_3575.py new file mode 100644 index 0000000..e307cf1 --- /dev/null +++ b/queue_3575.py @@ -0,0 +1,113 @@ +import server_3575 + +import os +import pika +from msgpack import packb +import signal + +from json import loads as json_loads + +import sys +from sys import stderr + +mq_i = 0 +exiting = False + +to_queue_name = os.getenv('MQ_DONE', 'made-tiles') +failed_queue_name = os.getenv('MQ_FAILED', 'failed-tiles') + +class Coordinate: + def __init__(self, row, column, zoom): + self.row = row + self.column = column + self.zoom = zoom + +def makeTile(ch, method, properties, data): + body = json_loads(data) + coord = Coordinate(body['coord']['row'], body['coord']['col'], body['coord']['zoom']) + path = '%d/%d/%d' % (coord.zoom, coord.column, coord.row) + + # Fetch a tile. + rendered = False + + global mq_i + global exiting + + while not rendered: + try: + content = server_3575.get_mvt(coord.zoom, coord.column, coord.row) + + mq_i = mq_i + 1 + if (mq_i % 100 == 0): + print('ToMQ', mq_i, coord) + + msg = packb({ + "zoom": coord.zoom, + "column": coord.column, + "row": coord.row, + "tile": content + }) + + mq_done_channel.basic_publish(exchange='', routing_key=to_queue_name, body=msg) + + except: + # Something went wrong: try again? Log the error? + + print('Exception') + global exiting + if exiting: + print('Exiting.') + ch.close() + break + + print('Failed', coord, file=sys.stderr) + mq_dud_channel.basic_publish(exchange='', routing_key=failed_queue_name, body=data) + ch.basic_nack(delivery_tag = method.delivery_tag, requeue=False) + # MAX_ERRORS? + #if not error_list: + raise + #break + + else: + # Successfully got the tile. + rendered = True + ch.basic_ack(delivery_tag = method.delivery_tag) + +def exit_handler(signal, frame): + global exiting + exiting = True + global channel + global consumer_tag + print('You pressed Ctrl+C!') + channel.basic_cancel(consumer_tag=consumer_tag) + channel.close() + + sys.exit(0) + + +def m(): + if __name__ == "__main__": + mq_credentials = pika.PlainCredentials(os.getenv('MQ_USERNAME','mblissett'), os.getenv('MQ_PASSWORD','xxxxxx')) + mq_connection = pika.BlockingConnection(pika.ConnectionParameters(os.getenv('MQ_HOST','mq.gbif.org'), int(os.getenv('MQ_PORT','5672')), os.getenv('MQ_VHOST','/users/mblissett'), mq_credentials)) + + global mq_done_channel + mq_done_channel = mq_connection.channel() + mq_done_channel.queue_declare(queue=os.getenv('MQ_DONE','made-tiles')) + + global mq_dud_channel + mq_dud_channel = mq_connection.channel() + mq_dud_channel.queue_declare(queue=os.getenv('MQ_FAILED','failed-tiles')) + + global mq_channel + mq_channel = mq_connection.channel() + mq_channel.queue_declare(queue=os.getenv('MQ_PENDING','do-tiles')) + mq_channel.basic_qos(prefetch_count=50) + global consumer_tag + consumer_tag = mq_channel.basic_consume(makeTile, queue=os.getenv('MQ_PENDING','do-tiles')) + + signal.signal(signal.SIGINT, exit_handler) + + print('Waiting for messages. To exit press CTRL+C') + mq_channel.start_consuming() + +m() diff --git a/queue.py b/queue_4326.py similarity index 90% rename from queue.py rename to queue_4326.py index 0932778..8c49fbb 100644 --- a/queue.py +++ b/queue_4326.py @@ -1,5 +1,6 @@ -import server +import server_4326 +import os import pika from msgpack import packb import signal @@ -60,8 +61,7 @@ def makeTile(ch, method, properties, data): break print('Failed', coord, file=sys.stderr) - global dud_channel - dud_channel.basic_publish(exchange='', routing_key=failed_queue_name, body=data) + mq_dud_channel.basic_publish(exchange='', routing_key=failed_queue_name, body=data) ch.basic_nack(delivery_tag = method.delivery_tag, requeue=False) # MAX_ERRORS? #if not error_list: @@ -87,8 +87,8 @@ def exit_handler(signal, frame): def m(): if __name__ == "__main__": - mq_credentials = pika.PlainCredentials(os.getenv('MQ_USER','mblissett'), os.getenv('MQ_PASSWORD','mblissett')) - mq_connection = pika.BlockingConnection(pika.ConnectionParameters(os.getenv('MQ_HOST','mq.gbif.org'), os.getenv('MQ_PORT','5672'), os.getenv('MQ_VHOST','/users/mblissett'), mq_credentials)) + mq_credentials = pika.PlainCredentials(os.getenv('MQ_USER','mblissett'), os.getenv('MQ_PASSWORD','xxxxxx')) + mq_connection = pika.BlockingConnection(pika.ConnectionParameters(os.getenv('MQ_HOST','mq.gbif.org'), int(os.getenv('MQ_PORT','5672')), os.getenv('MQ_VHOST','/users/mblissett'), mq_credentials)) global mq_done_channel mq_done_channel = mq_connection.channel() diff --git a/run-consumers b/run-consumers new file mode 100755 index 0000000..66f8c72 --- /dev/null +++ b/run-consumers @@ -0,0 +1,32 @@ +#!/bin/bash + +if [[ -z $PROCESS_COUNT ]]; then + PROCESS_COUNT=$(grep ^processor /proc/cpuinfo | wc -l) +fi + +if [[ "$VERBOSE" = "true" ]]; then + rm -Rfv logs + mkdir -p logs + out=logs/consumer_$$.log +else + out=/dev/null +fi + +while :; do + + if [[ $(pgrep -c -f queue_$PROJECTION.py) -lt $PROCESS_COUNT ]]; then + echo "Only" $(pgrep -c -f queue_$PROJECTION.py) "processes, starting another" + + nohup nice -n 5 python queue_$PROJECTION.py &> $out < /dev/null & + pid=$! + echo "Consumer started, pid $pid" + sleep 0.1 + [[ -f $out ]] && mv $out logs/consumer_$pid.log || echo "No $out logfile yet" + else + pgrep -a -f queue_$PROJECTION.py + sleep 15 + exit + fi + + sleep 5 +done diff --git a/server_3575.py b/server_3575.py new file mode 100644 index 0000000..9002327 --- /dev/null +++ b/server_3575.py @@ -0,0 +1,185 @@ +import tornado.ioloop +import tornado.web +import tornado.httpserver +import io +import os + +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy import create_engine +from sqlalchemy import inspect +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker + +import mercantile +import pyproj +import yaml +import sys +import itertools +import re + +def GetTM2Source(file): + with open(file,'r') as stream: + tm2source = yaml.load(stream) + return tm2source + +buffer_sizes = [] + +def GeneratePrepared(layers): + queries = [] + for layer in layers['Layer']: + buffer_size = str(layer['properties']['buffer-size']*2) + geometry_field = str(layer['Datasource']['geometry_field']) + if (buffer_size not in buffer_sizes): + buffer_sizes.append(buffer_size) + layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace + layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t" + layer_query = layer_query.replace(geometry_field, "ST_AsMVTGeom("+geometry_field+",!bbox_nobuffer!,4096,"+buffer_size+",true) AS mvtgeometry", 1) + layer_query = layer_query.replace("!bbox!", "!bbox_"+buffer_size+"_buffer!") + print('for re1',layer_query) + layer_query = re.sub(r"(layer_[a-z_]*)", r"\1_3575", layer_query) + layer_query = re.sub(r" z\(", r" z_3575(", layer_query) + print('for re2',layer_query) + and_or_where = 'AND' if 'WHERE' in layer_query else 'WHERE' + base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" "+and_or_where+" ST_AsMVTGeom("+geometry_field+",!bbox_nobuffer!,4096,"+buffer_size+",true) IS NOT NULL) AS tile" + queries.append(base_query.replace("!bbox_nobuffer!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4").replace("!bbox_"+buffer_size+"_buffer!", "$"+str(buffer_sizes.index(buffer_size)+5))) + print (base_query) + + chunk = "" + for buffer_size in buffer_sizes: + chunk = chunk + ", geometry" + + prepared = "PREPARE gettile(geometry, numeric, numeric, numeric"+chunk+") AS " + prepared = prepared + " UNION ALL ".join(queries) + ";" + print(prepared) + return(prepared) + +print("Starting up") +layers = GetTM2Source("/mapping/data.yml") +prepared = GeneratePrepared(layers) +connection_string = 'postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles') +engine = create_engine(connection_string) +inspector = inspect(engine) +DBSession = sessionmaker(bind=engine) +session = DBSession() +print("Running prepare statement") +session.execute(prepared) + +def bounds(zoom,x,y,buff): + print('Tile',zoom,x,y,'with buffer',buff) + #inProj = pyproj.Proj(init='epsg:4326') + #outProj = pyproj.Proj(init='epsg:3575') + #lnglatbbox = mercantile.bounds(x,y,zoom) + #ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1])) + #en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3])) + + map_width_in_metres = 2 * 2**0.5*6371007.2 + #map_width_in_metres = 180.0 + #tile_width_in_pixels = 512.0 + #standardized_pixel_size = 0.00028 + #map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) + + tiles_down = 2**(zoom) + tiles_across = 2**(zoom) + + x = x - 2**(zoom-1) + y = -(y - 2**(zoom-1)) - 1 + #x = x - 2**(zoom) + #y = -(y - 2**(zoom)) - 1 - 2**(zoom-1)# + + #print(x, y); + + tile_width_in_metres = (map_width_in_metres / tiles_across) + tile_height_in_metres = (map_width_in_metres / tiles_down) + ws = ((x - buff)*tile_width_in_metres, (y - buff)*tile_width_in_metres) + en = ((x+1+buff)*tile_height_in_metres, (y+1+buff)*tile_height_in_metres) + + print("Zoom, buffer", zoom, buff) + print("West: ", ws[0]) + print("South: ", ws[1]) + print("East: ", en[0]) + print("North: ", en[1]) + + return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]} + +def zoom_to_scale_denom(zoom): # For !scale_denominator! + # From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt + map_width_in_metres = 2 * 2**0.5*6371007.2 # Arctic + #map_width_in_metres = 180.0 + #map_width_in_metres = 40075016.68557849 + tile_width_in_pixels = 512.0 # This asks for a zoom level higher, since the tiles are doubled. + standardized_pixel_size = 0.00028 + map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) + return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size)) + +def replace_tokens(query,tilebounds,buffered_tilebounds,scale_denom,z): + s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) + + start = query.replace("!bbox!","ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 3575)").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","512").replace("!pixel_height!","512") + + for buffer_size in buffer_sizes: + token = "!bbox_"+buffer_size+"_buffer!" + idx = buffer_sizes.index(buffer_size) + + t = buffered_tilebounds[idx] + s,w,n,e = str(t['s']),str(t['w']),str(t['n']),str(t['e']) + start = start.replace(token,"ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 3575)") + + return start + +def get_mvt(zoom,x,y): + try: # Sanitize the inputs + sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) + del zoom,x,y + except: + print('suspicious') + return 1 + + scale_denom = zoom_to_scale_denom(sani_zoom) + tilebounds = bounds(sani_zoom,sani_x,sani_y,0) + + buffered_tilebounds = [] + chunk = "" + for buffer_size in buffer_sizes: + t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0) + buffered_tilebounds.append(t) + + chunk = chunk + ", !bbox_"+buffer_size+"_buffer!" + + final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");" + sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom) + print(sani_zoom, sani_x, sani_y, sent_query) + response = list(session.execute(sent_query)) + layers = filter(None,list(itertools.chain.from_iterable(response))) + final_tile = b'' + for layer in layers: + final_tile = final_tile + io.BytesIO(layer).getvalue() + return final_tile + +class GetTile(tornado.web.RequestHandler): + + def get(self, zoom,x,y): + self.set_header("Content-Type", "application/x-protobuf") + self.set_header("Content-Disposition", "attachment") + self.set_header("Access-Control-Allow-Origin", "*") + response = get_mvt(zoom,x,y) + self.write(response) + +def m(): + if __name__ == "__main__": + # Make this prepared statement from the tm2source + application = tornado.web.Application([ + (r"/[a-z/]+/([0-9]+)[/_]([0-9]+)[/_]([0-9]+).pbf", GetTile), + (r"/([^/]*)", tornado.web.StaticFileHandler, {"path": "./static", "default_filename": "index_3575.html"}) + ]) + + server = tornado.httpserver.HTTPServer(application) + server.bind(8080) + server.start(1) + print("Postserve started..") + #application.listen(8080) + + tornado.ioloop.IOLoop.instance().start() + +m() diff --git a/server.py b/server_4326.py similarity index 96% rename from server.py rename to server_4326.py index b4e7334..98a00d9 100644 --- a/server.py +++ b/server_4326.py @@ -47,6 +47,7 @@ def GeneratePrepared(layers): print(prepared) return(prepared) +print("Starting up") layers = GetTM2Source("/mapping/data.yml") prepared = GeneratePrepared(layers) connection_string = 'postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles') @@ -78,18 +79,18 @@ def bounds(zoom,x,y,buff): x = x - 2**(zoom) y = -(y - 2**(zoom)) - 1 - 2**(zoom-1) - print(x, y); + #print(x, y); tile_width_in_metres = (map_width_in_metres / tiles_across) tile_height_in_metres = (map_width_in_metres / tiles_down) ws = ((x - buff)*tile_width_in_metres, (y - buff)*tile_width_in_metres) en = ((x+1+buff)*tile_height_in_metres, (y+1+buff)*tile_height_in_metres) - print("Zoom, buffer", zoom, buff) - print("West: ", ws[0]) - print("South: ", ws[1]) - print("East: ", en[0]) - print("North: ", en[1]) + #print("Zoom, buffer", zoom, buff) + #print("West: ", ws[0]) + #print("South: ", ws[1]) + #print("East: ", en[0]) + #print("North: ", en[1]) return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]} @@ -139,10 +140,8 @@ def get_mvt(zoom,x,y): final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");" sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom) - print('Final query', sent_query) + print(sani_zoom, sani_x, sani_y, sent_query) response = list(session.execute(sent_query)) - print (sani_zoom, sani_x, sani_y) - print(sent_query) layers = filter(None,list(itertools.chain.from_iterable(response))) final_tile = b'' for layer in layers: diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..ba81869 --- /dev/null +++ b/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python -u ./server_$PROJECTION.py diff --git a/static/index_3575.html b/static/index_3575.html new file mode 100644 index 0000000..bbdb95a --- /dev/null +++ b/static/index_3575.html @@ -0,0 +1,37 @@ + + + + + EPSG:3575 map preview + + + + + + + + + + +
+
+
+ + + diff --git a/static/postserve_3575.js b/static/postserve_3575.js new file mode 100644 index 0000000..d57c4c6 --- /dev/null +++ b/static/postserve_3575.js @@ -0,0 +1,122 @@ +proj4.defs("EPSG:3575", "+proj=laea +lat_0=90 +lon_0=10 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs"); + +var halfWidth = Math.sqrt(2) * 6371007.2; +var extent = [-halfWidth, -halfWidth, halfWidth, halfWidth]; +ol.proj.get("EPSG:3575").setExtent(extent); + +var tile_size = 512; +var max_zoom = 16; +var resolutions = Array.from(new Array(max_zoom), (x,i) => (halfWidth/(tile_size*Math.pow(2,i-1)))); + +var layers = []; + +densityColours = ["#FFFF00", "#FFCC00", "#FF9900", "#FF6600", "#FF3300", "#FF0000"]; + +function createDensityStyle2() { + var point = new ol.style.Style({ + image: new ol.style.Circle({ + fill: new ol.style.Fill({color: '#FF0000'}), + radius: 1 + }), + fill: new ol.style.Fill({color: '#FF0000'}) + }); + + var styles = []; + return function(feature, resolution) { + var length = 0; + //console.log(feature); + var magnitude = Math.trunc(Math.min(5, Math.floor(Math.log(feature.get('total'))))) - 1; + //console.log("Colour ", magnitude, densityColours[magnitude]); + //styles[length++] = point; + styles[length++] = new ol.style.Style({ + image: new ol.style.Circle({ + fill: new ol.style.Fill({color: densityColours[magnitude]}), + radius: 1 + }), + fill: new ol.style.Fill({color: densityColours[magnitude]}) + }); + styles.length = length; + return styles; + }; +} + +var tile_grid = new ol.tilegrid.TileGrid({ + extent: ol.proj.get('EPSG:3575').getExtent(), + origin: [-halfWidth,halfWidth], + minZoom: 0, + maxZoom: max_zoom, + resolutions: resolutions, + tileSize: tile_size, +}); + +layers['EPSG:3575'] = new ol.layer.VectorTile({ + title: 'EPSG:3575 (Default size)', + type: 'base', + source: new ol.source.VectorTile({ + projection: 'EPSG:3575', + format: new ol.format.MVT(), + tileGrid: tile_grid, + tilePixelRatio: 8, + url: '/tiles/{z}_{x}_{y}.pbf', + wrapX: false + }), + style: createStyle(), +}); + +layers['Grid'] = new ol.layer.Tile({ + title: 'Tile grid', + type: 'Grid', + extent: ol.proj.get('EPSG:3575').getExtent(), + source: new ol.source.TileDebug({ + projection: 'EPSG:3575', + tileGrid: tile_grid, + wrapX: false + }), +}); + +// Show the tiles are 1:1 scale; cover 4096 screen pixels with the 4096px vector tiles. +var overzoom_tile_size = 4096; +var overzoom_resolutions = Array.from(new Array(max_zoom-3), (x,i) => (halfWidth/(overzoom_tile_size*Math.pow(2,i-1)))); +var overzoom_tile_grid = new ol.tilegrid.TileGrid({ + extent: ol.proj.get('EPSG:3575').getExtent(), + origin: [-halfWidth,halfWidth], + minZoom: 0, + maxZoom: max_zoom, + resolutions: overzoom_resolutions, + tileSize: overzoom_tile_size, +}); +layers['OZ_EPSG:3575'] = new ol.layer.VectorTile({ + title: 'Real size', + type: 'base', + source: new ol.source.VectorTile({ + projection: 'EPSG:3575', + format: new ol.format.MVT(), + tileGrid: overzoom_tile_grid, + tilePixelRatio: 1, + url: '/tiles/{z}_{x}_{y}.pbf', + wrapX: false + }), + style: createStyle(), +}); + +var map = new ol.Map({ + layers: [ + layers['Grid'], + new ol.layer.Group({ + title: 'Tiles', + layers: [ + layers['OZ_EPSG:3575'], + layers['EPSG:3575'], + ] + }) + ], + target: 'map', + view: new ol.View({ + center: [0, 0], + projection: 'EPSG:3575', + zoom: 2 + }), +}); + +var layerSwitcher = new ol.control.LayerSwitcher(); +map.addControl(layerSwitcher);