Move 4326 server and create 3575 server.
Matthew Blissett
5 years ago
parent
30e37ac921
commit
aeed0f4737
@ -0,0 +1,5 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
rm *~ -f
|
||||
docker build -t mb.gbif.org:5000/postserve .
|
||||
docker push mb.gbif.org:5000/postserve
|
||||
@ -0,0 +1,113 @@
|
||||
import server_3575
|
||||
|
||||
import os
|
||||
import pika
|
||||
from msgpack import packb
|
||||
import signal
|
||||
|
||||
from json import loads as json_loads
|
||||
|
||||
import sys
|
||||
from sys import stderr
|
||||
|
||||
mq_i = 0
|
||||
exiting = False
|
||||
|
||||
to_queue_name = os.getenv('MQ_DONE', 'made-tiles')
|
||||
failed_queue_name = os.getenv('MQ_FAILED', 'failed-tiles')
|
||||
|
||||
class Coordinate:
|
||||
def __init__(self, row, column, zoom):
|
||||
self.row = row
|
||||
self.column = column
|
||||
self.zoom = zoom
|
||||
|
||||
def makeTile(ch, method, properties, data):
|
||||
body = json_loads(data)
|
||||
coord = Coordinate(body['coord']['row'], body['coord']['col'], body['coord']['zoom'])
|
||||
path = '%d/%d/%d' % (coord.zoom, coord.column, coord.row)
|
||||
|
||||
# Fetch a tile.
|
||||
rendered = False
|
||||
|
||||
global mq_i
|
||||
global exiting
|
||||
|
||||
while not rendered:
|
||||
try:
|
||||
content = server_3575.get_mvt(coord.zoom, coord.column, coord.row)
|
||||
|
||||
mq_i = mq_i + 1
|
||||
if (mq_i % 100 == 0):
|
||||
print('ToMQ', mq_i, coord)
|
||||
|
||||
msg = packb({
|
||||
"zoom": coord.zoom,
|
||||
"column": coord.column,
|
||||
"row": coord.row,
|
||||
"tile": content
|
||||
})
|
||||
|
||||
mq_done_channel.basic_publish(exchange='', routing_key=to_queue_name, body=msg)
|
||||
|
||||
except:
|
||||
# Something went wrong: try again? Log the error?
|
||||
|
||||
print('Exception')
|
||||
global exiting
|
||||
if exiting:
|
||||
print('Exiting.')
|
||||
ch.close()
|
||||
break
|
||||
|
||||
print('Failed', coord, file=sys.stderr)
|
||||
mq_dud_channel.basic_publish(exchange='', routing_key=failed_queue_name, body=data)
|
||||
ch.basic_nack(delivery_tag = method.delivery_tag, requeue=False)
|
||||
# MAX_ERRORS?
|
||||
#if not error_list:
|
||||
raise
|
||||
#break
|
||||
|
||||
else:
|
||||
# Successfully got the tile.
|
||||
rendered = True
|
||||
ch.basic_ack(delivery_tag = method.delivery_tag)
|
||||
|
||||
def exit_handler(signal, frame):
|
||||
global exiting
|
||||
exiting = True
|
||||
global channel
|
||||
global consumer_tag
|
||||
print('You pressed Ctrl+C!')
|
||||
channel.basic_cancel(consumer_tag=consumer_tag)
|
||||
channel.close()
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def m():
|
||||
if __name__ == "__main__":
|
||||
mq_credentials = pika.PlainCredentials(os.getenv('MQ_USERNAME','mblissett'), os.getenv('MQ_PASSWORD','xxxxxx'))
|
||||
mq_connection = pika.BlockingConnection(pika.ConnectionParameters(os.getenv('MQ_HOST','mq.gbif.org'), int(os.getenv('MQ_PORT','5672')), os.getenv('MQ_VHOST','/users/mblissett'), mq_credentials))
|
||||
|
||||
global mq_done_channel
|
||||
mq_done_channel = mq_connection.channel()
|
||||
mq_done_channel.queue_declare(queue=os.getenv('MQ_DONE','made-tiles'))
|
||||
|
||||
global mq_dud_channel
|
||||
mq_dud_channel = mq_connection.channel()
|
||||
mq_dud_channel.queue_declare(queue=os.getenv('MQ_FAILED','failed-tiles'))
|
||||
|
||||
global mq_channel
|
||||
mq_channel = mq_connection.channel()
|
||||
mq_channel.queue_declare(queue=os.getenv('MQ_PENDING','do-tiles'))
|
||||
mq_channel.basic_qos(prefetch_count=50)
|
||||
global consumer_tag
|
||||
consumer_tag = mq_channel.basic_consume(makeTile, queue=os.getenv('MQ_PENDING','do-tiles'))
|
||||
|
||||
signal.signal(signal.SIGINT, exit_handler)
|
||||
|
||||
print('Waiting for messages. To exit press CTRL+C')
|
||||
mq_channel.start_consuming()
|
||||
|
||||
m()
|
||||
@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ -z $PROCESS_COUNT ]]; then
|
||||
PROCESS_COUNT=$(grep ^processor /proc/cpuinfo | wc -l)
|
||||
fi
|
||||
|
||||
if [[ "$VERBOSE" = "true" ]]; then
|
||||
rm -Rfv logs
|
||||
mkdir -p logs
|
||||
out=logs/consumer_$$.log
|
||||
else
|
||||
out=/dev/null
|
||||
fi
|
||||
|
||||
while :; do
|
||||
|
||||
if [[ $(pgrep -c -f queue_$PROJECTION.py) -lt $PROCESS_COUNT ]]; then
|
||||
echo "Only" $(pgrep -c -f queue_$PROJECTION.py) "processes, starting another"
|
||||
|
||||
nohup nice -n 5 python queue_$PROJECTION.py &> $out < /dev/null &
|
||||
pid=$!
|
||||
echo "Consumer started, pid $pid"
|
||||
sleep 0.1
|
||||
[[ -f $out ]] && mv $out logs/consumer_$pid.log || echo "No $out logfile yet"
|
||||
else
|
||||
pgrep -a -f queue_$PROJECTION.py
|
||||
sleep 15
|
||||
exit
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
@ -0,0 +1,185 @@
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
import tornado.httpserver
|
||||
import io
|
||||
import os
|
||||
|
||||
from sqlalchemy import Column, ForeignKey, Integer, String
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import mercantile
|
||||
import pyproj
|
||||
import yaml
|
||||
import sys
|
||||
import itertools
|
||||
import re
|
||||
|
||||
def GetTM2Source(file):
|
||||
with open(file,'r') as stream:
|
||||
tm2source = yaml.load(stream)
|
||||
return tm2source
|
||||
|
||||
buffer_sizes = []
|
||||
|
||||
def GeneratePrepared(layers):
|
||||
queries = []
|
||||
for layer in layers['Layer']:
|
||||
buffer_size = str(layer['properties']['buffer-size']*2)
|
||||
geometry_field = str(layer['Datasource']['geometry_field'])
|
||||
if (buffer_size not in buffer_sizes):
|
||||
buffer_sizes.append(buffer_size)
|
||||
layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace
|
||||
layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t"
|
||||
layer_query = layer_query.replace(geometry_field, "ST_AsMVTGeom("+geometry_field+",!bbox_nobuffer!,4096,"+buffer_size+",true) AS mvtgeometry", 1)
|
||||
layer_query = layer_query.replace("!bbox!", "!bbox_"+buffer_size+"_buffer!")
|
||||
print('for re1',layer_query)
|
||||
layer_query = re.sub(r"(layer_[a-z_]*)", r"\1_3575", layer_query)
|
||||
layer_query = re.sub(r" z\(", r" z_3575(", layer_query)
|
||||
print('for re2',layer_query)
|
||||
and_or_where = 'AND' if 'WHERE' in layer_query else 'WHERE'
|
||||
base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" "+and_or_where+" ST_AsMVTGeom("+geometry_field+",!bbox_nobuffer!,4096,"+buffer_size+",true) IS NOT NULL) AS tile"
|
||||
queries.append(base_query.replace("!bbox_nobuffer!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4").replace("!bbox_"+buffer_size+"_buffer!", "$"+str(buffer_sizes.index(buffer_size)+5)))
|
||||
print (base_query)
|
||||
|
||||
chunk = ""
|
||||
for buffer_size in buffer_sizes:
|
||||
chunk = chunk + ", geometry"
|
||||
|
||||
prepared = "PREPARE gettile(geometry, numeric, numeric, numeric"+chunk+") AS "
|
||||
prepared = prepared + " UNION ALL ".join(queries) + ";"
|
||||
print(prepared)
|
||||
return(prepared)
|
||||
|
||||
print("Starting up")
|
||||
layers = GetTM2Source("/mapping/data.yml")
|
||||
prepared = GeneratePrepared(layers)
|
||||
connection_string = 'postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles')
|
||||
engine = create_engine(connection_string)
|
||||
inspector = inspect(engine)
|
||||
DBSession = sessionmaker(bind=engine)
|
||||
session = DBSession()
|
||||
print("Running prepare statement")
|
||||
session.execute(prepared)
|
||||
|
||||
def bounds(zoom,x,y,buff):
|
||||
print('Tile',zoom,x,y,'with buffer',buff)
|
||||
#inProj = pyproj.Proj(init='epsg:4326')
|
||||
#outProj = pyproj.Proj(init='epsg:3575')
|
||||
#lnglatbbox = mercantile.bounds(x,y,zoom)
|
||||
#ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1]))
|
||||
#en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3]))
|
||||
|
||||
map_width_in_metres = 2 * 2**0.5*6371007.2
|
||||
#map_width_in_metres = 180.0
|
||||
#tile_width_in_pixels = 512.0
|
||||
#standardized_pixel_size = 0.00028
|
||||
#map_width_in_pixels = tile_width_in_pixels*(2.0**zoom)
|
||||
|
||||
tiles_down = 2**(zoom)
|
||||
tiles_across = 2**(zoom)
|
||||
|
||||
x = x - 2**(zoom-1)
|
||||
y = -(y - 2**(zoom-1)) - 1
|
||||
#x = x - 2**(zoom)
|
||||
#y = -(y - 2**(zoom)) - 1 - 2**(zoom-1)#
|
||||
|
||||
#print(x, y);
|
||||
|
||||
tile_width_in_metres = (map_width_in_metres / tiles_across)
|
||||
tile_height_in_metres = (map_width_in_metres / tiles_down)
|
||||
ws = ((x - buff)*tile_width_in_metres, (y - buff)*tile_width_in_metres)
|
||||
en = ((x+1+buff)*tile_height_in_metres, (y+1+buff)*tile_height_in_metres)
|
||||
|
||||
print("Zoom, buffer", zoom, buff)
|
||||
print("West: ", ws[0])
|
||||
print("South: ", ws[1])
|
||||
print("East: ", en[0])
|
||||
print("North: ", en[1])
|
||||
|
||||
return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]}
|
||||
|
||||
def zoom_to_scale_denom(zoom): # For !scale_denominator!
|
||||
# From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt
|
||||
map_width_in_metres = 2 * 2**0.5*6371007.2 # Arctic
|
||||
#map_width_in_metres = 180.0
|
||||
#map_width_in_metres = 40075016.68557849
|
||||
tile_width_in_pixels = 512.0 # This asks for a zoom level higher, since the tiles are doubled.
|
||||
standardized_pixel_size = 0.00028
|
||||
map_width_in_pixels = tile_width_in_pixels*(2.0**zoom)
|
||||
return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size))
|
||||
|
||||
def replace_tokens(query,tilebounds,buffered_tilebounds,scale_denom,z):
|
||||
s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e'])
|
||||
|
||||
start = query.replace("!bbox!","ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 3575)").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","512").replace("!pixel_height!","512")
|
||||
|
||||
for buffer_size in buffer_sizes:
|
||||
token = "!bbox_"+buffer_size+"_buffer!"
|
||||
idx = buffer_sizes.index(buffer_size)
|
||||
|
||||
t = buffered_tilebounds[idx]
|
||||
s,w,n,e = str(t['s']),str(t['w']),str(t['n']),str(t['e'])
|
||||
start = start.replace(token,"ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 3575)")
|
||||
|
||||
return start
|
||||
|
||||
def get_mvt(zoom,x,y):
|
||||
try: # Sanitize the inputs
|
||||
sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y)
|
||||
del zoom,x,y
|
||||
except:
|
||||
print('suspicious')
|
||||
return 1
|
||||
|
||||
scale_denom = zoom_to_scale_denom(sani_zoom)
|
||||
tilebounds = bounds(sani_zoom,sani_x,sani_y,0)
|
||||
|
||||
buffered_tilebounds = []
|
||||
chunk = ""
|
||||
for buffer_size in buffer_sizes:
|
||||
t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0)
|
||||
buffered_tilebounds.append(t)
|
||||
|
||||
chunk = chunk + ", !bbox_"+buffer_size+"_buffer!"
|
||||
|
||||
final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");"
|
||||
sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom)
|
||||
print(sani_zoom, sani_x, sani_y, sent_query)
|
||||
response = list(session.execute(sent_query))
|
||||
layers = filter(None,list(itertools.chain.from_iterable(response)))
|
||||
final_tile = b''
|
||||
for layer in layers:
|
||||
final_tile = final_tile + io.BytesIO(layer).getvalue()
|
||||
return final_tile
|
||||
|
||||
class GetTile(tornado.web.RequestHandler):
|
||||
|
||||
def get(self, zoom,x,y):
|
||||
self.set_header("Content-Type", "application/x-protobuf")
|
||||
self.set_header("Content-Disposition", "attachment")
|
||||
self.set_header("Access-Control-Allow-Origin", "*")
|
||||
response = get_mvt(zoom,x,y)
|
||||
self.write(response)
|
||||
|
||||
def m():
|
||||
if __name__ == "__main__":
|
||||
# Make this prepared statement from the tm2source
|
||||
application = tornado.web.Application([
|
||||
(r"/[a-z/]+/([0-9]+)[/_]([0-9]+)[/_]([0-9]+).pbf", GetTile),
|
||||
(r"/([^/]*)", tornado.web.StaticFileHandler, {"path": "./static", "default_filename": "index_3575.html"})
|
||||
])
|
||||
|
||||
server = tornado.httpserver.HTTPServer(application)
|
||||
server.bind(8080)
|
||||
server.start(1)
|
||||
print("Postserve started..")
|
||||
#application.listen(8080)
|
||||
|
||||
tornado.ioloop.IOLoop.instance().start()
|
||||
|
||||
m()
|
||||
@ -0,0 +1,37 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>EPSG:3575 map preview</title>
|
||||
<link rel="stylesheet" href="https://tile.gbif.org/ui/ol.css" type="text/css">
|
||||
<link rel="stylesheet" href="https://labs.gbif.org/~mblissett/2018/08/intersection-vector-tiles/ol3-layerswitcher-1.1.0.css" type="text/css"/>
|
||||
<script src="https://tile.gbif.org/ui/ol-debug.js"></script>
|
||||
<script src="https://tile.gbif.org/ui/proj4.js"></script>
|
||||
<script src="https://tile.gbif.org/ui/olms.js"></script>
|
||||
<script src="https://tile.gbif-dev.org/ui/common.js"></script>
|
||||
<script src="https://labs.gbif.org/~mblissett/2018/08/intersection-vector-tiles/ol3-layerswitcher-1.1.0.js"></script>
|
||||
<style>
|
||||
html {
|
||||
font-family: sans-serif;
|
||||
}
|
||||
.wrapper {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: lavender;
|
||||
}
|
||||
.map {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrapper">
|
||||
<div id="map" class="map"></div>
|
||||
</div>
|
||||
<script src="postserve_3575.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@ -0,0 +1,122 @@
|
||||
proj4.defs("EPSG:3575", "+proj=laea +lat_0=90 +lon_0=10 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs");
|
||||
|
||||
var halfWidth = Math.sqrt(2) * 6371007.2;
|
||||
var extent = [-halfWidth, -halfWidth, halfWidth, halfWidth];
|
||||
ol.proj.get("EPSG:3575").setExtent(extent);
|
||||
|
||||
var tile_size = 512;
|
||||
var max_zoom = 16;
|
||||
var resolutions = Array.from(new Array(max_zoom), (x,i) => (halfWidth/(tile_size*Math.pow(2,i-1))));
|
||||
|
||||
var layers = [];
|
||||
|
||||
densityColours = ["#FFFF00", "#FFCC00", "#FF9900", "#FF6600", "#FF3300", "#FF0000"];
|
||||
|
||||
function createDensityStyle2() {
|
||||
var point = new ol.style.Style({
|
||||
image: new ol.style.Circle({
|
||||
fill: new ol.style.Fill({color: '#FF0000'}),
|
||||
radius: 1
|
||||
}),
|
||||
fill: new ol.style.Fill({color: '#FF0000'})
|
||||
});
|
||||
|
||||
var styles = [];
|
||||
return function(feature, resolution) {
|
||||
var length = 0;
|
||||
//console.log(feature);
|
||||
var magnitude = Math.trunc(Math.min(5, Math.floor(Math.log(feature.get('total'))))) - 1;
|
||||
//console.log("Colour ", magnitude, densityColours[magnitude]);
|
||||
//styles[length++] = point;
|
||||
styles[length++] = new ol.style.Style({
|
||||
image: new ol.style.Circle({
|
||||
fill: new ol.style.Fill({color: densityColours[magnitude]}),
|
||||
radius: 1
|
||||
}),
|
||||
fill: new ol.style.Fill({color: densityColours[magnitude]})
|
||||
});
|
||||
styles.length = length;
|
||||
return styles;
|
||||
};
|
||||
}
|
||||
|
||||
var tile_grid = new ol.tilegrid.TileGrid({
|
||||
extent: ol.proj.get('EPSG:3575').getExtent(),
|
||||
origin: [-halfWidth,halfWidth],
|
||||
minZoom: 0,
|
||||
maxZoom: max_zoom,
|
||||
resolutions: resolutions,
|
||||
tileSize: tile_size,
|
||||
});
|
||||
|
||||
layers['EPSG:3575'] = new ol.layer.VectorTile({
|
||||
title: 'EPSG:3575 (Default size)',
|
||||
type: 'base',
|
||||
source: new ol.source.VectorTile({
|
||||
projection: 'EPSG:3575',
|
||||
format: new ol.format.MVT(),
|
||||
tileGrid: tile_grid,
|
||||
tilePixelRatio: 8,
|
||||
url: '/tiles/{z}_{x}_{y}.pbf',
|
||||
wrapX: false
|
||||
}),
|
||||
style: createStyle(),
|
||||
});
|
||||
|
||||
layers['Grid'] = new ol.layer.Tile({
|
||||
title: 'Tile grid',
|
||||
type: 'Grid',
|
||||
extent: ol.proj.get('EPSG:3575').getExtent(),
|
||||
source: new ol.source.TileDebug({
|
||||
projection: 'EPSG:3575',
|
||||
tileGrid: tile_grid,
|
||||
wrapX: false
|
||||
}),
|
||||
});
|
||||
|
||||
// Show the tiles are 1:1 scale; cover 4096 screen pixels with the 4096px vector tiles.
|
||||
var overzoom_tile_size = 4096;
|
||||
var overzoom_resolutions = Array.from(new Array(max_zoom-3), (x,i) => (halfWidth/(overzoom_tile_size*Math.pow(2,i-1))));
|
||||
var overzoom_tile_grid = new ol.tilegrid.TileGrid({
|
||||
extent: ol.proj.get('EPSG:3575').getExtent(),
|
||||
origin: [-halfWidth,halfWidth],
|
||||
minZoom: 0,
|
||||
maxZoom: max_zoom,
|
||||
resolutions: overzoom_resolutions,
|
||||
tileSize: overzoom_tile_size,
|
||||
});
|
||||
layers['OZ_EPSG:3575'] = new ol.layer.VectorTile({
|
||||
title: 'Real size',
|
||||
type: 'base',
|
||||
source: new ol.source.VectorTile({
|
||||
projection: 'EPSG:3575',
|
||||
format: new ol.format.MVT(),
|
||||
tileGrid: overzoom_tile_grid,
|
||||
tilePixelRatio: 1,
|
||||
url: '/tiles/{z}_{x}_{y}.pbf',
|
||||
wrapX: false
|
||||
}),
|
||||
style: createStyle(),
|
||||
});
|
||||
|
||||
var map = new ol.Map({
|
||||
layers: [
|
||||
layers['Grid'],
|
||||
new ol.layer.Group({
|
||||
title: 'Tiles',
|
||||
layers: [
|
||||
layers['OZ_EPSG:3575'],
|
||||
layers['EPSG:3575'],
|
||||
]
|
||||
})
|
||||
],
|
||||
target: 'map',
|
||||
view: new ol.View({
|
||||
center: [0, 0],
|
||||
projection: 'EPSG:3575',
|
||||
zoom: 2
|
||||
}),
|
||||
});
|
||||
|
||||
var layerSwitcher = new ol.control.LayerSwitcher();
|
||||
map.addControl(layerSwitcher);
|
||||
Loading…
Reference in New Issue