Browse Source

Add place import script

master
Skylar Ittner 3 years ago
parent
commit
93b99f8282
1 changed files with 151 additions and 0 deletions
  1. 151
    0
      import.php

+ 151
- 0
import.php View File

@@ -0,0 +1,151 @@
<?php

// Composer
require 'vendor/autoload.php';
// Settings file
require 'settings.php';

// Init database and stuff
$database;
try {
$database = new medoo([
'database_type' => DB_TYPE,
'database_name' => DB_NAME,
'server' => DB_SERVER,
'username' => DB_USER,
'password' => DB_PASS,
'charset' => DB_CHARSET
]);
} catch (Exception $ex) {
die("Database error.");
}

/**
* Checks if a string or whatever is empty.
* @param $str The thingy to check
* @return boolean True if it's empty or whatever.
*/
function is_empty($str) {
return (!isset($str) || $str == '' || $str == null);
}

error_reporting(E_ALL);
ini_set('display_errors', 'On');

//$file = $VARS['file'];
//$pass = $VARS['pass'];
if (is_empty($argv[1])) {
die("Please specify a json filename to process.");
}

$file_names = ["amenity_point", "amenity_polygon",
"geological_point", "geological_polygon",
"historic_point", "historic_polygon",
"leisure_point", "leisure_polygon",
"natural_point", "natural_polygon",
"shop_point", "shop_polygon",
"sport_point", "sport_polygon",
"tourism_point", "tourism_polygon"];

$files = [];
foreach ($file_names as $name) {
if (file_exists($argv[1] . $name . ".json")) {
$files[] = $argv[1] . $name . ".json";
}
}
$skipto = 0;

$time_start = microtime(true);

for ($j = 0; $j < count($files); $j++) {
$file = $files[$j];
echo "\n\n\n\n";
echo "Using $file as data source.\n";
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Starting import of $file\n", FILE_APPEND);
echo "[$file]: Starting to load source file...\n";
$raw = file_get_contents("$file");
echo "[$file]: Read complete. Starting to decode JSON...\n";
$json = json_decode($raw, true)['features'];
$raw = null; // Free up a bit of memory
echo "[$file]: JSON decode complete, inserting into DB...\n";
$total = count($json);
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] File $file loaded, starting DB import (approx. $total rows)\n", FILE_APPEND);
if ($skipto[$j] > 0) {
echo "[$file]: Skipping to record " . $skipto[$j] . "...\n";
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Skipping to record " . $skipto[$j] . "\n", FILE_APPEND);
}
set_time_limit(0);
$row_length = 0;
for ($i = $skipto[$j]; $i < $total; $i++) {
if ($i % 100 == 0) {
echo ".";
$row_length++;
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Processing record $i of $total\n", FILE_APPEND);
}
if ($row_length > 50) {
echo " [$i of $total]\n";
$row_length = 0;
}
$long = 0.00;
$lat = 0.00;

if ($json[$i]['geometry']['type'] == 'Polygon') {
// Find center of polygon
$coords = $json[$i]['geometry']['coordinates'][0];
$count = 0;
foreach ($coords as $co) {
if (!is_array($co[0]) && !is_array($co[1])) {
$long += $co[0];
$lat += $co[1];
$count++;
}
}
$long /= $count;
$lat /= $count;
} else if ($json[$i]['geometry']['type'] == 'Point') {
$long = $json[$i]['geometry']['coordinates'][0];
$lat = $json[$i]['geometry']['coordinates'][1];
} else {
// echo "\n[$file]: Warning: record $i not a known type, skipping.\n";
echo "x";
$row_length++;
continue;
}
if ($row_length > 50) {
echo " [$i of $total]\n";
$row_length = 0;
}

$id = $json[$i]['properties']['osm_id'];
$name = $json[$i]['properties']['name'];
if ($name == null) {
// $name = '';
// echo "[$file]: Warning: osmid $id has no name, skipping.\n";
continue;
}
if (!$database->has('places', ['osmid' => $id])) {
$database->insert('places', ['osmid' => $id, 'longitude' => $long, 'latitude' => $lat, 'name' => $name]);
} else {
echo "o";
$row_length++;
//echo "[$file]: Warning: osmid $id already in database, skipping.\n";
}
if ($row_length > 50) {
echo " [$i of $total]\n";
$row_length = 0;
}
$json[$i] = null; // Free up memory
}
echo "\n\n\n\n[$file]: Finished GeoJSON import.\n";
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Database import of $file complete.\n", FILE_APPEND);
// sleep(5);
}
$time_end = microtime(true);
$execution_time = ($time_end - $time_start) / 60;
echo "\n\n\n\n\n";
echo "-----------\n"
. "| DONE! |\n"
. "-----------\n";
echo "Total time: $execution_time minutes";
file_put_contents('import.log', "[" . date(DATE_ATOM) . "] All operations completed. Wall time: $execution_time minutes.\n", FILE_APPEND);


Loading…
Cancel
Save