update scripts

diff --git a/bin/html2csv b/bin/html2csv
index a33676620dab56028c206d86e0adcc8637b835f6..c20d8d22a0d97545dcac0ad55b0afc96e2a7503a 100755 (executable)
--- a/bin/html2csv
+++ b/bin/html2csv
@@ -38,7 +38,7 @@ my %Opt = (
   'html' => 0,
 );
 get_options(\%Opt);
-help() unless @ARGV;
+# NB 15.05.23 help() unless @ARGV;
 $main::_DATA_ = undef;
 
 #################################################################################

diff --git a/lib/postgres/continent.sql b/lib/postgres/continent.sql
index e420499c636d3870cfd5e845381d41aeead8e6a3..23099070568e310712a005b9ab8ff5a4b5ac0523 100644 (file)
--- a/lib/postgres/continent.sql
+++ b/lib/postgres/continent.sql
@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS continent (
 );
 BEGIN TRANSACTION;
 DELETE FROM continent;
+SELECT 'Update continent from download.geonames.org' as log;
 COPY continent FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/readme.txt \
 | awk "/^Continent codes/ {ok=1} ok==1 && /^$/{exit} ok==1 {print \$0}" \
 | tail -n +2 \

diff --git a/lib/postgres/country.sql b/lib/postgres/country.sql
index d2f6ab24eb356528555d49f30c7ec071522be418..3c585fba0412bc1be26ac8aa5040286c6d8bcbaf 100644 (file)
--- a/lib/postgres/country.sql
+++ b/lib/postgres/country.sql
@@ -44,6 +44,7 @@ CREATE INDEX IF NOT EXISTS country_geoid_idx ON country (geoid);
 CREATE INDEX IF NOT EXISTS country_capital_idx ON country (capital);
 BEGIN TRANSACTION;
 DELETE FROM country;
+SELECT 'Update country from download.geonames.org' as log;
 COPY country FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/countryInfo.txt \
 | grep -v "^#" \
 | sed "s/\\\\/\\\\\\\\/g" \

diff --git a/lib/postgres/css_entity.sh b/lib/postgres/css_entity.sh
index 3d7a46068bf19c264318b423521167e3488692df..e29af082b7dc351709a3e462bb44dd45b42a038c 100755 (executable)
--- a/lib/postgres/css_entity.sh
+++ b/lib/postgres/css_entity.sh
@@ -4,5 +4,6 @@ http_get https://www.w3schools.com/cssref/css_entities.php | \
 xmllint --html --xpath '/html/body//div[@id="main"]//table' - 2>/dev/null | \
 xml2csv - | \
 tail -n+2 | \
-sed -E "s,(\\\\),\1\1," \
-| awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \
+sed -E "s,(\\\\),\1\1," | \
+awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \
+

diff --git a/lib/postgres/css_function.sh b/lib/postgres/css_function.sh
index ded17c0855393b11a17a1fb9f3dc3d7c371ccefc..32387b4dc85fca5d1f7ded5947a5785ceed0aac3 100755 (executable)
--- a/lib/postgres/css_function.sh
+++ b/lib/postgres/css_function.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/css_functions.asp | \
+http_get https://www.w3schools.com/cssref/css_functions.php | \
 xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
 sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" | \
 xml2csv - | \

diff --git a/lib/postgres/css_reference.sh b/lib/postgres/css_reference.sh
index 94ca86426ffa0bf9b383c3416af28d8cc101f9de..0cd7a91aac1598f0235a349fb84722b00c9b856c 100755 (executable)
--- a/lib/postgres/css_reference.sh
+++ b/lib/postgres/css_reference.sh
@@ -1,8 +1,8 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/ \
-| html2txt \
-| perl -ne " \
-next unless /^align-content .*Specifies/ .. /^z-index/ and /^[a-z]/; \
-s/\s+/\t/; \
-chomp; /^(\S+)/; \
-print qq|\$_\thttps://www.w3schools.com/cssref/pr_\$1.asp\n|" \
+http_get https://www.w3schools.com/cssref/index.php | \
+xmllint --html --xpath '/html/body//div[@id="cssproperties"]//table' - 2>/dev/null | \
+perl -pe 's|<td><a href="([^"]+)">([^<]+)</a></td>|<td>https://www.w3schools.com/cssref/$1</td><td>$2</td>|' | \
+html2csv | \
+sed -E "s,(\\\\),\1\1," | \
+awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \
+

diff --git a/lib/postgres/css_selector.sh b/lib/postgres/css_selector.sh
index f73563fe0fb7d82201176248de2b19d28fb42724..028132aae6238bc460213ea3cecffdc63fc533da 100755 (executable)
--- a/lib/postgres/css_selector.sh
+++ b/lib/postgres/css_selector.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/css_selectors.asp \
+http_get https://www.w3schools.com/cssref/css_selectors.php \
 | xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null \
 | sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" \
 | xml2csv - \

diff --git a/lib/postgres/mime.sql b/lib/postgres/mime.sql
index 6921a5b2a4b3b07dcd444f48f119da26f91b6260..3929f66c001fbf3a9d55060968e38d3a749acf28 100644 (file)
--- a/lib/postgres/mime.sql
+++ b/lib/postgres/mime.sql
@@ -12,7 +12,7 @@ BEGIN TRANSACTION;
 DELETE FROM mime;
 CREATE TEMPORARY TABLE tmp_mime ON COMMIT DROP AS SELECT * FROM mime LIMIT 0;
 
-SELECT '>Stdin' as log;
+SELECT 'Update mime from stdin' as log;
 COPY mime (ext,type,name) FROM STDIN WITH (format 'text', NULL '');
 x3d    application/vnd.hzn-3d-crossword        3D Crossword Plugin
 3g2    video/3gpp2     3GP2
@@ -703,7 +703,7 @@ zmm application/vnd.handheld-entertainment+xml      ZVUE Media Manager
 zaz    application/vnd.zzazz.deck+xml  Zzazz Deck
 \.
 
-SELECT '>developer.mozilla.org' as log;
+SELECT 'Update mime from developer.mozilla.org' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'true && curl -s \
 https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types \
@@ -713,7 +713,7 @@ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Comp
 ' WITH (format 'text', NULL '');
 INSERT INTO mime SELECT * FROM tmp_mime ON CONFLICT DO NOTHING;
 
-SELECT '>gist.githubusercontent.com' as log;
+SELECT 'Update mime from gist.githubusercontent.com' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'true && curl -s \
 https://gist.githubusercontent.com/electerious/3d5a31a73cfd6423f835c074ab25fc06/raw/d48b8410e9aa6746cfd946bca21a1bb54c351c4e/Caddyfile \
@@ -721,7 +721,7 @@ https://gist.githubusercontent.com/electerious/3d5a31a73cfd6423f835c074ab25fc06/
 ' WITH (format 'text', NULL '');
 INSERT INTO mime SELECT * FROM tmp_mime ON CONFLICT DO NOTHING;
 
-SELECT '>svn.apache.org' as log;
+SELECT 'Update mime from svn.apache.org' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'curl -s \
 http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types \

diff --git a/lib/postgres/update.sh b/lib/postgres/update.sh
index f72e6d2082910c133e8621d25782bafbf812ace0..17e086ef28116bc3af2f4d91788b68f8c389a9e2 100755 (executable)
--- a/lib/postgres/update.sh
+++ b/lib/postgres/update.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -l
 #
 # lib/postgres/update.sh DB_NAME
-# Update table present in a db
+# Create/Update table present in a db (pub)
 #
 
 set -e
@@ -9,22 +9,20 @@ cd "$(dirname "$0")"
 
 DB=${1:?Db name missing}
 EREG="${2:-.}"
-#exec echo $EREG
 
-#export PATH="$PATH:$(pwd)"
-#export PATH="$PATH:."
 psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | grep -E "$EREG" | while read t; do
 
 #echo "$t" >&2; continue
        [ -r "$t.sql" ] || continue
-       echo "SELECT 'Update $t' as Comment;"
        grep -vE '^DROP' "$t.sql"
 
        copy=""
        if [ -e "$t.txt" ]; then
                copy="'$PWD/$t.txt'"
+               echo "SELECT 'Update $t from $t.txt' as log;"
        elif [ -x "$t.sh" ]; then
                copy="PROGRAM '$PWD/$t.sh'"
+               echo "SELECT 'Update $t from $t.sh' as log;"
        fi
 
        if [ -n "$copy" ]; then
@@ -35,5 +33,8 @@ COPY $t FROM $copy WITH (format 'text', NULL '');
 COMMIT;
 EOF
        fi
+       cat <<EOF
+SELECT 'Updated $t: ' || count(*) as log FROM $t;
+EOF
 
 done | PGOPTIONS='--client-min-messages=warning' psql "$DB" -qAt