From: Nicolas Boisselier <nicolas.boisselier@gmail.com>
Date: Sun, 14 May 2023 22:44:29 +0000 (+0200)
Subject: update scripts
X-Git-Url: https://git.nbdom.net/?a=commitdiff_plain;h=88c711b4d30d2a64dacda784976d67561ad88750;p=nb.git

update scripts
---

diff --git a/bin/html2csv b/bin/html2csv
index a3367662..c20d8d22 100755
--- a/bin/html2csv
+++ b/bin/html2csv
@@ -38,7 +38,7 @@ my %Opt = (
   'html' => 0,
 );
 get_options(\%Opt);
-help() unless @ARGV;
+# NB 15.05.23 help() unless @ARGV;
 $main::_DATA_ = undef;
 
 #################################################################################
diff --git a/lib/postgres/continent.sql b/lib/postgres/continent.sql
index e420499c..23099070 100644
--- a/lib/postgres/continent.sql
+++ b/lib/postgres/continent.sql
@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS continent (
 );
 BEGIN TRANSACTION;
 DELETE FROM continent;
+SELECT 'Update continent from download.geonames.org' as log;
 COPY continent FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/readme.txt \
 | awk "/^Continent codes/ {ok=1} ok==1 && /^$/{exit} ok==1 {print \$0}" \
 | tail -n +2 \
diff --git a/lib/postgres/country.sql b/lib/postgres/country.sql
index d2f6ab24..3c585fba 100644
--- a/lib/postgres/country.sql
+++ b/lib/postgres/country.sql
@@ -44,6 +44,7 @@ CREATE INDEX IF NOT EXISTS country_geoid_idx ON country (geoid);
 CREATE INDEX IF NOT EXISTS country_capital_idx ON country (capital);
 BEGIN TRANSACTION;
 DELETE FROM country;
+SELECT 'Update country from download.geonames.org' as log;
 COPY country FROM PROGRAM 'curl -s http://download.geonames.org/export/dump/countryInfo.txt \
 | grep -v "^#" \
 | sed "s/\\\\/\\\\\\\\/g" \
diff --git a/lib/postgres/css_entity.sh b/lib/postgres/css_entity.sh
index 3d7a4606..e29af082 100755
--- a/lib/postgres/css_entity.sh
+++ b/lib/postgres/css_entity.sh
@@ -4,5 +4,6 @@ http_get https://www.w3schools.com/cssref/css_entities.php | \
 xmllint --html --xpath '/html/body//div[@id="main"]//table' - 2>/dev/null | \
 xml2csv - | \
 tail -n+2 | \
-sed -E "s,(\\\\),\1\1," \
-| awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \
+sed -E "s,(\\\\),\1\1," | \
+awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \
+
diff --git a/lib/postgres/css_function.sh b/lib/postgres/css_function.sh
index ded17c08..32387b4d 100755
--- a/lib/postgres/css_function.sh
+++ b/lib/postgres/css_function.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/css_functions.asp | \
+http_get https://www.w3schools.com/cssref/css_functions.php | \
 xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
 sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" | \
 xml2csv - | \
diff --git a/lib/postgres/css_reference.sh b/lib/postgres/css_reference.sh
index 94ca8642..0cd7a91a 100755
--- a/lib/postgres/css_reference.sh
+++ b/lib/postgres/css_reference.sh
@@ -1,8 +1,8 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/ \
-| html2txt \
-| perl -ne " \
-next unless /^align-content .*Specifies/ .. /^z-index/ and /^[a-z]/; \
-s/\s+/\t/; \
-chomp; /^(\S+)/; \
-print qq|\$_\thttps://www.w3schools.com/cssref/pr_\$1.asp\n|" \
+http_get https://www.w3schools.com/cssref/index.php | \
+xmllint --html --xpath '/html/body//div[@id="cssproperties"]//table' - 2>/dev/null | \
+perl -pe 's|<td><a href="([^"]+)">([^<]+)</a></td>|<td>https://www.w3schools.com/cssref/$1</td><td>$2</td>|' | \
+html2csv | \
+sed -E "s,(\\\\),\1\1," | \
+awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \
+
diff --git a/lib/postgres/css_selector.sh b/lib/postgres/css_selector.sh
index f73563fe..028132aa 100755
--- a/lib/postgres/css_selector.sh
+++ b/lib/postgres/css_selector.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -l
-http_get https://www.w3schools.com/cssref/css_selectors.asp \
+http_get https://www.w3schools.com/cssref/css_selectors.php \
 | xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null \
 | sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" \
 | xml2csv - \
diff --git a/lib/postgres/mime.sql b/lib/postgres/mime.sql
index 6921a5b2..3929f66c 100644
--- a/lib/postgres/mime.sql
+++ b/lib/postgres/mime.sql
@@ -12,7 +12,7 @@ BEGIN TRANSACTION;
 DELETE FROM mime;
 CREATE TEMPORARY TABLE tmp_mime ON COMMIT DROP AS SELECT * FROM mime LIMIT 0;
 
-SELECT '>Stdin' as log;
+SELECT 'Update mime from stdin' as log;
 COPY mime (ext,type,name) FROM STDIN WITH (format 'text', NULL '');
 x3d	application/vnd.hzn-3d-crossword	3D Crossword Plugin
 3g2	video/3gpp2	3GP2
@@ -703,7 +703,7 @@ zmm	application/vnd.handheld-entertainment+xml	ZVUE Media Manager
 zaz	application/vnd.zzazz.deck+xml	Zzazz Deck
 \.
 
-SELECT '>developer.mozilla.org' as log;
+SELECT 'Update mime from developer.mozilla.org' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'true && curl -s \
 https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types \
@@ -713,7 +713,7 @@ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Comp
 ' WITH (format 'text', NULL '');
 INSERT INTO mime SELECT * FROM tmp_mime ON CONFLICT DO NOTHING;
 
-SELECT '>gist.githubusercontent.com' as log;
+SELECT 'Update mime from gist.githubusercontent.com' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'true && curl -s \
 https://gist.githubusercontent.com/electerious/3d5a31a73cfd6423f835c074ab25fc06/raw/d48b8410e9aa6746cfd946bca21a1bb54c351c4e/Caddyfile \
@@ -721,7 +721,7 @@ https://gist.githubusercontent.com/electerious/3d5a31a73cfd6423f835c074ab25fc06/
 ' WITH (format 'text', NULL '');
 INSERT INTO mime SELECT * FROM tmp_mime ON CONFLICT DO NOTHING;
 
-SELECT '>svn.apache.org' as log;
+SELECT 'Update mime from svn.apache.org' as log;
 DELETE FROM tmp_mime;
 COPY tmp_mime FROM PROGRAM 'curl -s \
 http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types \
diff --git a/lib/postgres/update.sh b/lib/postgres/update.sh
index f72e6d20..17e086ef 100755
--- a/lib/postgres/update.sh
+++ b/lib/postgres/update.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -l
 #
 # lib/postgres/update.sh DB_NAME
-# Update table present in a db
+# Create/Update table present in a db (pub)
 #
 
 set -e
@@ -9,22 +9,20 @@ cd "$(dirname "$0")"
 
 DB=${1:?Db name missing}
 EREG="${2:-.}"
-#exec echo $EREG
 
-#export PATH="$PATH:$(pwd)"
-#export PATH="$PATH:."
 psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | grep -E "$EREG" | while read t; do
 
 #echo "$t" >&2; continue
 	[ -r "$t.sql" ] || continue
-	echo "SELECT 'Update $t' as Comment;"
 	grep -vE '^DROP' "$t.sql"
 
 	copy=""
 	if [ -e "$t.txt" ]; then
 		copy="'$PWD/$t.txt'"
+		echo "SELECT 'Update $t from $t.txt' as log;"
 	elif [ -x "$t.sh" ]; then
 		copy="PROGRAM '$PWD/$t.sh'"
+		echo "SELECT 'Update $t from $t.sh' as log;"
 	fi
 
 	if [ -n "$copy" ]; then
@@ -35,5 +33,8 @@ COPY $t FROM $copy WITH (format 'text', NULL '');
 COMMIT;
 EOF
 	fi
+	cat <<EOF
+SELECT 'Updated $t: ' || count(*) as log FROM $t;
+EOF
 
 done | PGOPTIONS='--client-min-messages=warning' psql "$DB" -qAt