You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

import.php 4.8KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. <?php
  2. // Composer
  3. require 'vendor/autoload.php';
  4. // Settings file
  5. require 'settings.php';
  6. // Init database and stuff
  7. $database;
  8. try {
  9. $database = new medoo([
  10. 'database_type' => DB_TYPE,
  11. 'database_name' => DB_NAME,
  12. 'server' => DB_SERVER,
  13. 'username' => DB_USER,
  14. 'password' => DB_PASS,
  15. 'charset' => DB_CHARSET
  16. ]);
  17. } catch (Exception $ex) {
  18. die("Database error.");
  19. }
  20. /**
  21. * Checks if a string or whatever is empty.
  22. * @param $str The thingy to check
  23. * @return boolean True if it's empty or whatever.
  24. */
  25. function is_empty($str) {
  26. return (!isset($str) || $str == '' || $str == null);
  27. }
  28. error_reporting(E_ALL);
  29. ini_set('display_errors', 'On');
  30. //$file = $VARS['file'];
  31. //$pass = $VARS['pass'];
  32. if (is_empty($argv[1])) {
  33. die("Please specify a json filename to process.");
  34. }
  35. $file_names = ["amenity_point", "amenity_polygon",
  36. "geological_point", "geological_polygon",
  37. "historic_point", "historic_polygon",
  38. "leisure_point", "leisure_polygon",
  39. "natural_point", "natural_polygon",
  40. "shop_point", "shop_polygon",
  41. "sport_point", "sport_polygon",
  42. "tourism_point", "tourism_polygon"];
  43. $files = [];
  44. foreach ($file_names as $name) {
  45. if (file_exists($argv[1] . $name . ".json")) {
  46. $files[] = $argv[1] . $name . ".json";
  47. }
  48. }
  49. $skipto = 0;
  50. $time_start = microtime(true);
  51. for ($j = 0; $j < count($files); $j++) {
  52. $file = $files[$j];
  53. echo "\n\n\n\n";
  54. echo "Using $file as data source.\n";
  55. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Starting import of $file\n", FILE_APPEND);
  56. echo "[$file]: Starting to load source file...\n";
  57. $raw = file_get_contents("$file");
  58. echo "[$file]: Read complete. Starting to decode JSON...\n";
  59. $json = json_decode($raw, true)['features'];
  60. $raw = null; // Free up a bit of memory
  61. echo "[$file]: JSON decode complete, inserting into DB...\n";
  62. $total = count($json);
  63. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] File $file loaded, starting DB import (approx. $total rows)\n", FILE_APPEND);
  64. if ($skipto[$j] > 0) {
  65. echo "[$file]: Skipping to record " . $skipto[$j] . "...\n";
  66. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Skipping to record " . $skipto[$j] . "\n", FILE_APPEND);
  67. }
  68. set_time_limit(0);
  69. $row_length = 0;
  70. for ($i = $skipto[$j]; $i < $total; $i++) {
  71. if ($i % 100 == 0) {
  72. echo ".";
  73. $row_length++;
  74. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Processing record $i of $total\n", FILE_APPEND);
  75. }
  76. if ($row_length > 50) {
  77. echo " [$i of $total]\n";
  78. $row_length = 0;
  79. }
  80. $long = 0.00;
  81. $lat = 0.00;
  82. if ($json[$i]['geometry']['type'] == 'Polygon') {
  83. // Find center of polygon
  84. $coords = $json[$i]['geometry']['coordinates'][0];
  85. $count = 0;
  86. foreach ($coords as $co) {
  87. if (!is_array($co[0]) && !is_array($co[1])) {
  88. $long += $co[0];
  89. $lat += $co[1];
  90. $count++;
  91. }
  92. }
  93. $long /= $count;
  94. $lat /= $count;
  95. } else if ($json[$i]['geometry']['type'] == 'Point') {
  96. $long = $json[$i]['geometry']['coordinates'][0];
  97. $lat = $json[$i]['geometry']['coordinates'][1];
  98. } else {
  99. // echo "\n[$file]: Warning: record $i not a known type, skipping.\n";
  100. echo "x";
  101. $row_length++;
  102. continue;
  103. }
  104. if ($row_length > 50) {
  105. echo " [$i of $total]\n";
  106. $row_length = 0;
  107. }
  108. $id = $json[$i]['properties']['osm_id'];
  109. $name = $json[$i]['properties']['name'];
  110. if ($name == null) {
  111. // $name = '';
  112. // echo "[$file]: Warning: osmid $id has no name, skipping.\n";
  113. continue;
  114. }
  115. if (!$database->has('places', ['osmid' => $id])) {
  116. $database->insert('places', ['osmid' => $id, 'longitude' => $long, 'latitude' => $lat, 'name' => $name]);
  117. } else {
  118. echo "o";
  119. $row_length++;
  120. //echo "[$file]: Warning: osmid $id already in database, skipping.\n";
  121. }
  122. if ($row_length > 50) {
  123. echo " [$i of $total]\n";
  124. $row_length = 0;
  125. }
  126. $json[$i] = null; // Free up memory
  127. }
  128. echo "\n\n\n\n[$file]: Finished GeoJSON import.\n";
  129. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] Database import of $file complete.\n", FILE_APPEND);
  130. // sleep(5);
  131. }
  132. $time_end = microtime(true);
  133. $execution_time = ($time_end - $time_start) / 60;
  134. echo "\n\n\n\n\n";
  135. echo "-----------\n"
  136. . "| DONE! |\n"
  137. . "-----------\n";
  138. echo "Total time: $execution_time minutes";
  139. file_put_contents('import.log', "[" . date(DATE_ATOM) . "] All operations completed. Wall time: $execution_time minutes.\n", FILE_APPEND);