lib/postgres/geo.sql

diff --git a/lib/postgres/continent.sql b/lib/postgres/continent.sql
index 2f458c3f63ed2bb2f1484a68fdd5e88cdb440893..e420499c636d3870cfd5e845381d41aeead8e6a3 100644 (file)
--- a/lib/postgres/continent.sql
+++ b/lib/postgres/continent.sql
@@ -6,5 +6,9 @@ CREATE TABLE IF NOT EXISTS continent (
 );
 BEGIN TRANSACTION;
 DELETE FROM continent;
-COPY continent FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/readme.txt | awk "/^Continent codes/ {ok=1} ok==1 && /^$/{exit} ok==1 {print \$0}" | tail -n +2 | sed -e "s/ : /\t/" -e "s/[\t ]\+geonameId=/\t/"' WITH (format 'text', NULL ''); 
+COPY continent FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/readme.txt \
+| awk "/^Continent codes/ {ok=1} ok==1 && /^$/{exit} ok==1 {print \$0}" \
+| tail -n +2 \
+| sed -e "s/ : /\t/" -e "s/[\t ]\+geonameId=/\t/" \
+' WITH (format 'text', NULL ''); 
 COMMIT;

diff --git a/lib/postgres/geo.sql b/lib/postgres/geo.sql
index c4f37a09c2d2ef79043df63ad3b9784af512e427..8c892c5f411068d1223090fe4ba6f7296f094763 100644 (file)
--- a/lib/postgres/geo.sql
+++ b/lib/postgres/geo.sql
@@ -25,8 +25,9 @@ CREATE TABLE geo (
   altnames varchar(10000),
   latitude numeric(7,4),
   longitude numeric(7,4),
-  feature_class char(1),
-  feature_code varchar(10),
+-- NB 06.01.18   feature_class char(1),
+-- NB 06.01.18   feature_code varchar(10),
+  feature varchar(10),
   country char(2),
   cc2 varchar(200),
 -- NB 06.01.18   admin1 varchar(20),
@@ -43,5 +44,10 @@ CREATE INDEX IF NOT EXISTS geo_country_idx ON geo (country);
 BEGIN TRANSACTION;
 DELETE FROM geo;
 -- NB 06.01.18 COPY geo FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/allCountries.zip | zcat' WITH (format 'text', NULL '');
-COPY geo FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/allCountries.zip | zcat | cut -f -10,15-' WITH (format 'text', NULL '');
+-- NB 06.01.18 COPY geo FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/allCountries.zip | zcat | cut -f -10,15-' WITH (format 'text', NULL '');
+COPY geo FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/allCountries.zip \
+| zcat \
+| awk "{\$7=\$7\".\"\$8;print}" FS="\t" OFS="\t" \
+| cut -f -7,9-10,15-
+' WITH (format 'text', NULL '');
 COMMIT;

diff --git a/lib/postgres/geofeature.sql b/lib/postgres/geofeature.sql
index dec6c0b3551f8900cf23b66ca95feef150a42a54..41aee3824d82b923d1c4297dfdc98196aeec6229 100644 (file)
--- a/lib/postgres/geofeature.sql
+++ b/lib/postgres/geofeature.sql
@@ -7,5 +7,12 @@ CREATE TABLE geofeature (
 -- CREATE INDEX IF NOT EXISTS geofeature_country_idx ON geofeature (country);
 BEGIN TRANSACTION;
 DELETE FROM geofeature;
-COPY geofeature FROM PROGRAM 'curl -s http://www.geonames.org/export/codes.html | tr -d "\n" | sed "s/<.tr>/\n/g" | grep "<a name=.*<td" | sed -E -e "s/(<.?t[^>]*> *)/\t/g" -e "s/^.*?=.([A-Z0-9\.]*)[^\t]+\t/\1/" -e "s/\t+/\t/g" | cut -f 1-3 | grep -v "not available"' WITH (format 'text', NULL '');
+COPY geofeature FROM PROGRAM 'curl -s http://www.geonames.org/export/codes.html \
+| tr -d "\n" \
+| sed "s/<.tr>/\n/g" \
+| grep "<a name=.*<td" \
+| sed -E -e "s/(<.?t[^>]*> *)/\t/g" -e "s/^.*?=.([A-Z0-9\.]*)[^\t]+\t/\1/" -e "s/\t+/\t/g" \
+| cut -f 1-3 \
+| grep -v "not available"
+' WITH (format 'text', NULL '');
 COMMIT;