From: Nicolas Boisselier Date: Mon, 18 Mar 2019 03:31:51 +0000 (+0000) Subject: lib/postgres/html_tag.sh X-Git-Url: https://git.nbdom.net/?a=commitdiff_plain;h=c5df60a8ee288b58ec8b40caf38250aa4544c258;p=nb.git lib/postgres/html_tag.sh --- diff --git a/lib/postgres/css_entity.sh b/lib/postgres/css_entity.sh new file mode 100755 index 00000000..83611c28 --- /dev/null +++ b/lib/postgres/css_entity.sh @@ -0,0 +1,8 @@ +#!/bin/bash -l +#(chr,entity,name) +http_get https://www.w3schools.com/cssref/css_entities.asp | \ +xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \ +xml2csv - | \ +tail -n+2 | \ +sed -E "s,(\\\\),\1\1," \ +| awk 'BEGIN{FS="\t"}{print $2"\t"$1"\t"$3}' \ diff --git a/lib/postgres/css_entity.sql b/lib/postgres/css_entity.sql index e1d27db5..6e2188c6 100644 --- a/lib/postgres/css_entity.sql +++ b/lib/postgres/css_entity.sql @@ -4,13 +4,3 @@ CREATE TABLE IF NOT EXISTS css_entity ( chr varchar(2), name varchar(200) ); -BEGIN TRANSACTION; -DELETE FROM css_entity; -COPY css_entity (chr,entity,name) FROM PROGRAM '. /etc/profile && \ -http_get https://www.w3schools.com/cssref/css_entities.asp | \ -xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \ -xml2csv - | \ -tail -n+2 | \ -sed -E "s,(\\\\),\\\\\1," \ -' WITH (format 'text', NULL ''); -COMMIT; diff --git a/lib/postgres/css_function.sh b/lib/postgres/css_function.sh new file mode 100755 index 00000000..ded17c08 --- /dev/null +++ b/lib/postgres/css_function.sh @@ -0,0 +1,8 @@ +#!/bin/bash -l +http_get https://www.w3schools.com/cssref/css_functions.asp | \ +xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null | \ +sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|/dev/null | \ -sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|(.*?),https://www.w3schools.com/cssref/\1\2,g" | \ --- NB 22.06.18 html2txt | \ --- NB 22.06.18 perl -ne "s/\s+/\t/ and s/\s+/\t/ and print if /^https.* [a-z]/" --- NB 22.06.18 ' WITH (format 'text', NULL ''); -COMMIT; diff --git a/lib/postgres/css_selector.sh b/lib/postgres/css_selector.sh new file mode 100755 index 00000000..f73563fe --- /dev/null +++ b/lib/postgres/css_selector.sh @@ -0,0 +1,8 @@ +#!/bin/bash -l +http_get https://www.w3schools.com/cssref/css_selectors.asp \ +| xmllint --html --xpath "/html/body//div[@id=\"main\"]//table" - 2>/dev/null \ +| sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|/dev/null | \ -sed -E "s, href=\"([^\"]+)\",>https://www.w3schools.com/cssref/\1|]+>([^<]+),
\2
," \ +| xmllint --html --xpath //table - 2>/dev/null \ +| xml2csv - \ +| sed -E \ + -e "s/”/\"/g" -e "s,^[^<]+(<[^>]+>)[^\t]+,\L\1," \ + -e "s,^(<([^ ]+) ([^=>]+)),https://html.com/attributes/\2-\3/\t\1," \ +| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \ diff --git a/lib/postgres/html_attribute.sql b/lib/postgres/html_attribute.sql index dde3d17e..ea5919fc 100644 --- a/lib/postgres/html_attribute.sql +++ b/lib/postgres/html_attribute.sql @@ -4,15 +4,3 @@ CREATE TABLE IF NOT EXISTS html_attribute ( description varchar(500), link varchar(200) ); -DELETE FROM html_attribute; -BEGIN; -COPY html_attribute (link,name,description) FROM PROGRAM '. /etc/profile && \ -http_get https://html.com/attributes/ | \ -sed -E "s,<(thrive_headline|header)[^.>]+>([^<]+),
\2
," | \ -xmllint --html --xpath //table - 2>/dev/null | \ -xml2csv - | \ -sed -E \ - -e "s/”/\"/g" -e "s,^[^<]+(<[^>]+>)[^\t]+,\L\1," \ - -e "s,^(<([^ ]+) ([^=>]+)),https://html.com/attributes/\2-\3/\t\1," \ -' WITH (format 'text', NULL ''); -COMMIT; diff --git a/lib/postgres/html_tag.sh b/lib/postgres/html_tag.sh new file mode 100755 index 00000000..7063c358 --- /dev/null +++ b/lib/postgres/html_tag.sh @@ -0,0 +1,5 @@ +#!/bin/bash -l +html2txt https://html.com/tags/ \ +| grep "^<" \ +| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://www.w3schools.com/TAGS/tag_\1.asp\t," \ +| awk 'BEGIN{FS="\t"}{print $1"\t"$3"\t"$2}' \ diff --git a/lib/postgres/html_tag.sql b/lib/postgres/html_tag.sql index f05cd301..ea3372e3 100644 --- a/lib/postgres/html_tag.sql +++ b/lib/postgres/html_tag.sql @@ -4,11 +4,3 @@ CREATE TABLE IF NOT EXISTS html_tag ( description varchar(500), link varchar(100) ); -BEGIN TRANSACTION; -DELETE FROM html_tag; -COPY html_tag (name,link,description) FROM PROGRAM '. /etc/profile && html2txt https://html.com/tags/ \ -| grep "^<" \ -| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://www.w3schools.com/TAGS/tag_\1.asp\t," \ -' WITH (format 'text', NULL ''); -COMMIT; - diff --git a/lib/postgres/nginx_var.sh b/lib/postgres/nginx_var.sh new file mode 100755 index 00000000..0acb6653 --- /dev/null +++ b/lib/postgres/nginx_var.sh @@ -0,0 +1,6 @@ +#!/bin/bash -l +http_get https://nginx.org/en/docs/varindex.html \ +| xmllint --html --xpath "/html/body//div[@id=\"content\"]//p" - 2>/dev/null \ +| sed -E -e "s,([^<]+) *(\(([^\)]+)\))?,https://nginx.org/en/docs/\1\t\2\t\4\n,g" -e "s,<[^>]+>,,g" \ +| grep -v "^ *$" sed "s,(\([^(]\+\))$,\1,g" \ +| awk 'BEGIN{FS="\t"}{print $2"\t"$3"\t"$1}' \ diff --git a/lib/postgres/nginx_var.sql b/lib/postgres/nginx_var.sql index fbc5aa87..145406ef 100644 --- a/lib/postgres/nginx_var.sql +++ b/lib/postgres/nginx_var.sql @@ -1,4 +1,3 @@ -DROP TABLE IF EXISTS nginx_vars; DROP TABLE IF EXISTS nginx_var; CREATE TABLE IF NOT EXISTS nginx_var ( name varchar(200), @@ -7,11 +6,3 @@ CREATE TABLE IF NOT EXISTS nginx_var ( ); CREATE INDEX IF NOT EXISTS nginx_var_name ON nginx_var USING btree (name); CREATE INDEX IF NOT EXISTS nginx_var_module ON nginx_var USING btree (module); -BEGIN TRANSACTION; -DELETE FROM nginx_var; -COPY nginx_var (link,name,module) FROM PROGRAM '. /etc/profile && http_get https://nginx.org/en/docs/varindex.html | \ -xmllint --html --xpath "/html/body//div[@id=\"content\"]//p" - 2>/dev/null | \ -sed -E -e "s,([^<]+) *(\(([^\)]+)\))?,https://nginx.org/en/docs/\1\t\2\t\4\n,g" -e "s,<[^>]+>,,g" | \ -grep -v "^ *$" | sed "s,(\([^(]\+\))$,\1,g" \ -' WITH (format 'text', NULL ''); -COMMIT; diff --git a/lib/postgres/update.sh b/lib/postgres/update.sh index a7425de8..f72e6d20 100755 --- a/lib/postgres/update.sh +++ b/lib/postgres/update.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -l # # lib/postgres/update.sh DB_NAME # Update table present in a db @@ -8,11 +8,32 @@ set -e cd "$(dirname "$0")" DB=${1:?Db name missing} +EREG="${2:-.}" +#exec echo $EREG -psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | while read t; do +#export PATH="$PATH:$(pwd)" +#export PATH="$PATH:." +psql "$DB" -Atc '\dt' | grep '^public|' | cut -d'|' -f2 | grep -E "$EREG" | while read t; do +#echo "$t" >&2; continue [ -r "$t.sql" ] || continue echo "SELECT 'Update $t' as Comment;" grep -vE '^DROP' "$t.sql" + copy="" + if [ -e "$t.txt" ]; then + copy="'$PWD/$t.txt'" + elif [ -x "$t.sh" ]; then + copy="PROGRAM '$PWD/$t.sh'" + fi + + if [ -n "$copy" ]; then + cat <