lib/postgres/html_tag.sh

diff --git a/lib/postgres/css_entity.sh b/lib/postgres/css_entity.sh
new file mode 100755 (executable)
index 0000000..83611c2
--- /dev/null
+++ b/lib/postgres/css_entity.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -l
+#(chr,entity,name)
+http_get https://www.w3schools.com/cssref/css_entities.asp | \
+xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
+xml2csv - | \
+tail -n+2 | \
+sed -E "s,(\\\\),\1\1," \
+| awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \

diff --git a/lib/postgres/css_entity.sql b/lib/postgres/css_entity.sql
index e1d27db584a4f2a6f228713bc8ff85a148d3cc7a..6e2188c6e41aedbd65d008a5c15987d7fc70c152 100644 (file)
--- a/lib/postgres/css_entity.sql
+++ b/lib/postgres/css_entity.sql
@@ -4,13 +4,3 @@ CREATE TABLE IF NOT EXISTS css_entity (
   chr varchar(2),
   name varchar(200)
 );
-BEGIN TRANSACTION;
-DELETE FROM css_entity;
-COPY css_entity (chr,entity,name) FROM PROGRAM '. /etc/profile && \
-http_get https://www.w3schools.com/cssref/css_entities.asp | \
-xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
-xml2csv - | \
-tail -n+2 | \
-sed -E "s,(\\\\),\\\\\1," \
-' WITH (format 'text', NULL ''); 
-COMMIT;

diff --git a/lib/postgres/css_function.sh b/lib/postgres/css_function.sh
new file mode 100755 (executable)
index 0000000..ded17c0
--- /dev/null
+++ b/lib/postgres/css_function.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -l
+http_get https://www.w3schools.com/cssref/css_functions.asp | \
+xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
+sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" | \
+xml2csv - | \
+grep "|" | \
+sed "s/|/\t/" \
+| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \

diff --git a/lib/postgres/css_function.sql b/lib/postgres/css_function.sql
index 8304e63767131a7bd3a6a765226f4e30566bdbc9..b094ab2a37b8d1cdc232b7d87207f1051866f5ef 100644 (file)
--- a/lib/postgres/css_function.sql
+++ b/lib/postgres/css_function.sql
@@ -4,14 +4,3 @@ CREATE TABLE IF NOT EXISTS css_function (
   description varchar(500),
   link varchar(300)
 );
-BEGIN TRANSACTION;
-DELETE FROM css_function;
-COPY css_function (link,name,description) FROM PROGRAM '. /etc/profile && \
-http_get https://www.w3schools.com/cssref/css_functions.asp | \
-xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
-sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" | \
-xml2csv - | \
-grep "|" | \
-sed "s/|/\t/" \
-' WITH (format 'text', NULL ''); 
-COMMIT;

diff --git a/lib/postgres/css_reference.sh b/lib/postgres/css_reference.sh
new file mode 100755 (executable)
index 0000000..94ca864
--- /dev/null
+++ b/lib/postgres/css_reference.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -l
+http_get https://www.w3schools.com/cssref/ \
+| html2txt \
+| perl -ne " \
+next unless /^align-content .*Specifies/ .. /^z-index/ and /^[a-z]/; \
+s/\s+/\t/; \
+chomp; /^(\S+)/; \
+print qq|\$_\thttps://www.w3schools.com/cssref/pr_\$1.asp\n|" \

diff --git a/lib/postgres/css_reference.sql b/lib/postgres/css_reference.sql
index 6c8e78ec55b9453d9a0e1b13e2aec6305fd6e86f..4b3d9a3b3f2c3bc7317c93ec32a2bb2b83bbe16c 100644 (file)
--- a/lib/postgres/css_reference.sql
+++ b/lib/postgres/css_reference.sql
@@ -4,21 +4,3 @@ CREATE TABLE IF NOT EXISTS css_reference (
   description varchar(500),
   link varchar(300)
 );
-BEGIN TRANSACTION;
-DELETE FROM css_reference;
-COPY css_reference (name,description,link) FROM PROGRAM '. /etc/profile && \
-http_get https://www.w3schools.com/cssref/ | \
-html2txt | \
-perl -ne " \
-next unless /^align-content .*Specifies/ .. /^z-index/ and /^[a-z]/; \
-s/\s+/\t/; \
-chomp; /^(\S+)/; \
-print qq|\$_\thttps://www.w3schools.com/cssref/pr_\$1.asp\n|" \
-' WITH (format 'text', NULL ''); 
--- NB 22.06.18 COPY css_reference (link,name,description) FROM PROGRAM '. /etc/profile && \
--- NB 22.06.18 http_get https://www.w3schools.com/cssref/ | \
--- NB 22.06.18 sed -E "s,<a href=\"([a-z][^\"]+)\">(.*?)</a>,https://www.w3schools.com/cssref/\1</td><td>\2,g" | \
--- NB 22.06.18 html2txt | \
--- NB 22.06.18 perl -ne "s/\s+/\t/ and s/\s+/\t/ and print if /^https.* [a-z]/"
--- NB 22.06.18 ' WITH (format 'text', NULL ''); 
-COMMIT;

diff --git a/lib/postgres/css_selector.sh b/lib/postgres/css_selector.sh
new file mode 100755 (executable)
index 0000000..f73563f
--- /dev/null
+++ b/lib/postgres/css_selector.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -l
+http_get https://www.w3schools.com/cssref/css_selectors.asp \
+| xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null \
+| sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" \
+| xml2csv - \
+| grep "|" \
+| sed "s/|/\t/" \
+| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$4"\t"$1}' \

diff --git a/lib/postgres/css_selector.sql b/lib/postgres/css_selector.sql
index 4c71eaca86288c3f05fcf3b6999074749f561e7c..10ffb40e0dac58f7288351ce4f075679e162b13e 100644 (file)
--- a/lib/postgres/css_selector.sql
+++ b/lib/postgres/css_selector.sql
@@ -5,14 +5,3 @@ CREATE TABLE IF NOT EXISTS css_selector (
   description varchar(500),
   link varchar(300)
 );
-BEGIN TRANSACTION;
-DELETE FROM css_selector;
-COPY css_selector (link,name,example,description) FROM PROGRAM '. /etc/profile && \
-http_get https://www.w3schools.com/cssref/css_selectors.asp | \
-xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \
-sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|</a><a,g" | \
-xml2csv - | \
-grep "|" | \
-sed "s/|/\t/" \
-' WITH (format 'text', NULL ''); 
-COMMIT;

diff --git a/lib/postgres/html_attribute.sh b/lib/postgres/html_attribute.sh
new file mode 100755 (executable)
index 0000000..564442b
--- /dev/null
+++ b/lib/postgres/html_attribute.sh
@@ -0,0 +1,9 @@
+#!/bin/bash -l
+http_get https://html.com/attributes/ \
+| sed -E "s,<(thrive_headline|header)[^.>]+>([^<]+)</\1>,<div>\2</div>," \
+| xmllint --html --xpath //table - 2>/dev/null \
+| xml2csv - \
+| sed -E \
+  -e "s/”/\"/g" -e "s,^[^<]+(<[^>]+>)[^\t]+,\L\1," \
+  -e "s,^(<([^ ]+) ([^=>]+)),https://html.com/attributes/\2-\3/\t\1," \
+| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \

diff --git a/lib/postgres/html_attribute.sql b/lib/postgres/html_attribute.sql
index dde3d17e3007ac727612eb34cb4f0ccdc7a4bb62..ea5919fc1ee32ead273dc03d9058c2b41544674f 100644 (file)
--- a/lib/postgres/html_attribute.sql
+++ b/lib/postgres/html_attribute.sql
@@ -4,15 +4,3 @@ CREATE TABLE IF NOT EXISTS html_attribute (
   description varchar(500),
   link varchar(200)
 );
-DELETE FROM html_attribute;
-BEGIN;
-COPY html_attribute (link,name,description) FROM PROGRAM '. /etc/profile && \
-http_get https://html.com/attributes/ | \
-sed -E "s,<(thrive_headline|header)[^.>]+>([^<]+)</\1>,<div>\2</div>," | \
-xmllint --html --xpath //table - 2>/dev/null | \
-xml2csv - | \
-sed -E \
-  -e "s/”/\"/g" -e "s,^[^<]+(<[^>]+>)[^\t]+,\L\1," \
-  -e "s,^(<([^ ]+) ([^=>]+)),https://html.com/attributes/\2-\3/\t\1," \
-' WITH (format 'text', NULL '');
-COMMIT;

diff --git a/lib/postgres/html_tag.sh b/lib/postgres/html_tag.sh
new file mode 100755 (executable)
index 0000000..7063c35
--- /dev/null
+++ b/lib/postgres/html_tag.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -l
+html2txt https://html.com/tags/ \
+| grep "^<" \
+| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://www.w3schools.com/TAGS/tag_\1.asp\t," \
+| awk 'BEGIN{FS="\t"}{print $1"\t"$3"\t"$2}' \

diff --git a/lib/postgres/html_tag.sql b/lib/postgres/html_tag.sql
index f05cd3019da4579aebc5ffd9825a153db11dc162..ea3372e301d210b4e9b86e4c74a914fbcb26b05f 100644 (file)
--- a/lib/postgres/html_tag.sql
+++ b/lib/postgres/html_tag.sql
@@ -4,11 +4,3 @@ CREATE TABLE IF NOT EXISTS html_tag (
   description varchar(500),
   link varchar(100)
 );
-BEGIN TRANSACTION;
-DELETE FROM html_tag;
-COPY html_tag (name,link,description) FROM PROGRAM '. /etc/profile && html2txt https://html.com/tags/ \
-| grep "^<" \
-| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://www.w3schools.com/TAGS/tag_\1.asp\t," \
-' WITH (format 'text', NULL ''); 
-COMMIT;
-

diff --git a/lib/postgres/nginx_var.sh b/lib/postgres/nginx_var.sh
new file mode 100755 (executable)
index 0000000..0acb665
--- /dev/null
+++ b/lib/postgres/nginx_var.sh
@@ -0,0 +1,6 @@
+#!/bin/bash -l
+http_get https://nginx.org/en/docs/varindex.html \
+| xmllint --html --xpath "/html/body//div[@id=\"content\"]//p" - 2>/dev/null \
+| sed -E -e "s,<a href=\"([^\"]+)\">([^<]+)</a> *(\(([^\)]+)\))?,https://nginx.org/en/docs/\1\t\2\t\4\n,g" -e "s,<[^>]+>,,g" \
+| grep -v "^ *$"  sed "s,(\([^(]\+\))$,\1,g" \
+| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \

diff --git a/lib/postgres/nginx_var.sql b/lib/postgres/nginx_var.sql
index fbc5aa8779486503df9f4f4b666e981e52bb41d6..145406ef1c317f3b3994ea72a8e873337ea60fba 100644 (file)
--- a/lib/postgres/nginx_var.sql
+++ b/lib/postgres/nginx_var.sql
@@ -1,4 +1,3 @@
-DROP TABLE IF EXISTS nginx_vars;
 DROP TABLE IF EXISTS nginx_var;
 CREATE TABLE IF NOT EXISTS nginx_var (
   name varchar(200),
@@ -7,11 +6,3 @@ CREATE TABLE IF NOT EXISTS nginx_var (
 );
 CREATE INDEX IF NOT EXISTS nginx_var_name ON nginx_var USING btree (name);
 CREATE INDEX IF NOT EXISTS nginx_var_module ON nginx_var USING btree (module);
-BEGIN TRANSACTION;
-DELETE FROM nginx_var;
-COPY nginx_var (link,name,module) FROM PROGRAM '. /etc/profile && http_get https://nginx.org/en/docs/varindex.html | \
-xmllint --html --xpath "/html/body//div[@id=\"content\"]//p" - 2>/dev/null | \
-sed -E -e "s,<a href=\"([^\"]+)\">([^<]+)</a> *(\(([^\)]+)\))?,https://nginx.org/en/docs/\1\t\2\t\4\n,g" -e "s,<[^>]+>,,g" | \
-grep -v "^ *$" |  sed "s,(\([^(]\+\))$,\1,g" \
-' WITH (format 'text', NULL ''); 
-COMMIT;

diff --git a/lib/postgres/update.sh b/lib/postgres/update.sh
index a7425de8dcf545a1b23c1e18e0016e4f89b02a43..f72e6d2082910c133e8621d25782bafbf812ace0 100755 (executable)
--- a/lib/postgres/update.sh
+++ b/lib/postgres/update.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -l
 #
 # lib/postgres/update.sh DB_NAME
 # Update table present in a db
@@ -8,11 +8,32 @@ set -e
 cd "$(dirname "$0")"
 
 DB=${1:?Db name missing}
+EREG="${2:-.}"
+#exec echo $EREG
 
-psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | while read t; do
+#export PATH="$PATH:$(pwd)"
+#export PATH="$PATH:."
+psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | grep -E "$EREG" | while read t; do
 
+#echo "$t" >&2; continue
        [ -r "$t.sql" ] || continue
        echo "SELECT 'Update $t' as Comment;"
        grep -vE '^DROP' "$t.sql"
 
+       copy=""
+       if [ -e "$t.txt" ]; then
+               copy="'$PWD/$t.txt'"
+       elif [ -x "$t.sh" ]; then
+               copy="PROGRAM '$PWD/$t.sh'"
+       fi
+
+       if [ -n "$copy" ]; then
+               cat <<EOF
+BEGIN TRANSACTION;
+DELETE FROM $t;
+COPY $t FROM $copy WITH (format 'text', NULL '');
+COMMIT;
+EOF
+       fi
+
 done | PGOPTIONS='--client-min-messages=warning' psql "$DB" -qAt