diff --git a/import.php b/import.php new file mode 100644 index 0000000..3eb785e --- /dev/null +++ b/import.php @@ -0,0 +1,151 @@ + DB_TYPE, + 'database_name' => DB_NAME, + 'server' => DB_SERVER, + 'username' => DB_USER, + 'password' => DB_PASS, + 'charset' => DB_CHARSET + ]); +} catch (Exception $ex) { + die("Database error."); +} + +/** + * Checks if a string or whatever is empty. + * @param $str The thingy to check + * @return boolean True if it's empty or whatever. + */ +function is_empty($str) { + return (!isset($str) || $str == '' || $str == null); +} + +error_reporting(E_ALL); +ini_set('display_errors', 'On'); + +//$file = $VARS['file']; +//$pass = $VARS['pass']; +if (is_empty($argv[1])) { + die("Please specify a json filename to process."); +} + +$file_names = ["amenity_point", "amenity_polygon", + "geological_point", "geological_polygon", + "historic_point", "historic_polygon", + "leisure_point", "leisure_polygon", + "natural_point", "natural_polygon", + "shop_point", "shop_polygon", + "sport_point", "sport_polygon", + "tourism_point", "tourism_polygon"]; + +$files = []; +foreach ($file_names as $name) { + if (file_exists($argv[1] . $name . ".json")) { + $files[] = $argv[1] . $name . ".json"; + } +} +$skipto = 0; + +$time_start = microtime(true); + +for ($j = 0; $j < count($files); $j++) { + $file = $files[$j]; + echo "\n\n\n\n"; + echo "Using $file as data source.\n"; + file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Starting import of $file\n", FILE_APPEND); + echo "[$file]: Starting to load source file...\n"; + $raw = file_get_contents("$file"); + echo "[$file]: Read complete. Starting to decode JSON...\n"; + $json = json_decode($raw, true)['features']; + $raw = null; // Free up a bit of memory + echo "[$file]: JSON decode complete, inserting into DB...\n"; + $total = count($json); + file_put_contents('import.log', "[" . date(DATE_ATOM) . "] File $file loaded, starting DB import (approx. $total rows)\n", FILE_APPEND); + if ($skipto[$j] > 0) { + echo "[$file]: Skipping to record " . $skipto[$j] . "...\n"; + file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Skipping to record " . $skipto[$j] . "\n", FILE_APPEND); + } + set_time_limit(0); + $row_length = 0; + for ($i = $skipto[$j]; $i < $total; $i++) { + if ($i % 100 == 0) { + echo "."; + $row_length++; + file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Processing record $i of $total\n", FILE_APPEND); + } + if ($row_length > 50) { + echo " [$i of $total]\n"; + $row_length = 0; + } + $long = 0.00; + $lat = 0.00; + + if ($json[$i]['geometry']['type'] == 'Polygon') { + // Find center of polygon + $coords = $json[$i]['geometry']['coordinates'][0]; + $count = 0; + foreach ($coords as $co) { + if (!is_array($co[0]) && !is_array($co[1])) { + $long += $co[0]; + $lat += $co[1]; + $count++; + } + } + $long /= $count; + $lat /= $count; + } else if ($json[$i]['geometry']['type'] == 'Point') { + $long = $json[$i]['geometry']['coordinates'][0]; + $lat = $json[$i]['geometry']['coordinates'][1]; + } else { +// echo "\n[$file]: Warning: record $i not a known type, skipping.\n"; + echo "x"; + $row_length++; + continue; + } + if ($row_length > 50) { + echo " [$i of $total]\n"; + $row_length = 0; + } + + $id = $json[$i]['properties']['osm_id']; + $name = $json[$i]['properties']['name']; + if ($name == null) { +// $name = ''; +// echo "[$file]: Warning: osmid $id has no name, skipping.\n"; + continue; + } + if (!$database->has('places', ['osmid' => $id])) { + $database->insert('places', ['osmid' => $id, 'longitude' => $long, 'latitude' => $lat, 'name' => $name]); + } else { + echo "o"; + $row_length++; + //echo "[$file]: Warning: osmid $id already in database, skipping.\n"; + } + if ($row_length > 50) { + echo " [$i of $total]\n"; + $row_length = 0; + } + $json[$i] = null; // Free up memory + } + echo "\n\n\n\n[$file]: Finished GeoJSON import.\n"; + file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Database import of $file complete.\n", FILE_APPEND); +// sleep(5); +} +$time_end = microtime(true); +$execution_time = ($time_end - $time_start) / 60; +echo "\n\n\n\n\n"; +echo "-----------\n" + . "| DONE! |\n" + . "-----------\n"; +echo "Total time: $execution_time minutes"; +file_put_contents('import.log', "[" . date(DATE_ATOM) . "] All operations completed. Wall time: $execution_time minutes.\n", FILE_APPEND); +