From: Nicolas Boisselier <nicolas.boisselier@gmail.com>
Date: Wed, 25 Jan 2017 15:17:38 +0000 (+0000)
Subject: bin/html-table2csv
X-Git-Url: https://git.nbdom.net/?a=commitdiff_plain;h=20d3372626efc9be45a62a515c75782c1c54e1e0;p=nb.git

bin/html-table2csv
---

diff --git a/bin/html-table2csv b/bin/html-table2csv
new file mode 100755
index 00000000..470e9381
--- /dev/null
+++ b/bin/html-table2csv
@@ -0,0 +1,211 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+#use LWP::Simple qw/get/;
+use NB::Functions qw/html2txt str_trim/;
+#################################################################################
+#
+# VERSION
+#
+#################################################################################
+my $VERSION = '0.0.1';
+# NB 25.01.17
+# - create script: html-table2csv
+
+#################################################################################
+#
+# GLOBALS
+#
+#################################################################################
+my ($NAME) = $0 =~ m,([^/]+)$,;
+
+#################################################################################
+#
+# ARGS
+#
+#################################################################################
+my $VERBOSE = $main::VERBOSE = 1;
+my $DEBUG = $main::DEBUG = 0;
+
+my %Opt = (
+  'table' => undef,
+);
+get_options(\%Opt);
+help() unless @ARGV;
+$main::_DATA_ = undef;
+
+#################################################################################
+#
+# BEGIN
+#
+#################################################################################
+@ARGV = map {m,^\w+://, ? "curl -s '$_'" : $_} @ARGV if @ARGV;
+my $html = join('',<>);
+$html =~ s/\s+/ /g;
+
+my $table_num = 0;
+
+for my $table ($html =~ m,<table[^>]*>(.*?)<\s*/\s*table\s*>,gi) {
+  $table_num++;
+  next if defined $Opt{'table'} and $Opt{table} and $Opt{table} != $table_num;
+
+  for my $tr ($table =~ m,<tr[^>]*>(.*?)<\s*/\s*tr\s*>,gi) {
+    my @col;
+
+    for my $td ($tr =~ m,<t[dh][^>]*>(.*?)<\s*/\s*t[dh]\s*>,gi) {
+      $td = html2txt($td);
+      $td =~ s/\s+/ /g;
+      $td = str_trim($td);
+      push(@col,$td);
+    }
+
+    print join("\t",@col)."\n";
+
+  }
+
+}
+
+#################################################################################
+#
+# END
+#
+#################################################################################
+exit 0;
+
+#################################################################################
+#
+# Functions
+#
+#################################################################################
+sub help {
+#------------------------------------------------------------------------------
+# Print help and exit
+#------------------------------------------------------------------------------
+
+	require 'Pod/Usage.pm' unless $INC{'Pod/Usage.pm'};
+	require 'Pod/Perldoc.pm' unless $INC{'Pod/Perldoc.pm'};
+
+	# Substitutions
+	sub pod_env {
+		my $v = '';
+		eval '$v = ref(\\'.$_[0].') eq "ARRAY" ? join(" ",'.$_[0].') : '.$_[0].'; return defined $v ? $v : qq|UNDEF|;';
+		return $v;
+	}
+
+	$main::_DATA_ =~ s/([@\$][A-Z_a-z\{\}]+)/pod_env($1)/eg;
+
+	# Create tmp
+  my $in_file = (-e '/dev/shm' ? '/dev/shm' : '/tmp')."/$NAME.$$";
+  my $in;
+  open($in,">$in_file") or die "$NAME: Can't write into $in_file: $!";
+  print $in $main::_DATA_;
+	close $in;
+
+	# Output
+	open(STDOUT,"|perl -pe 's/\.$$//g'".(($ENV{PAGER}||'') eq 'less' ? "|less -FRi" : ""));
+	my $opts = {
+		-input => $in_file,
+		-ouput => \*STDOUT,
+		-exitval => 'noexit',
+		-sections => [qw(SYNOPSIS DESCRIPTION OPTIONS)],
+		-verbose => ($Opt{'help'} ? 99 : 3),
+	};
+
+	Pod::Usage::pod2usage($opts);
+  close STDOUT;
+  unlink $in_file if $in_file and -e $in_file;
+
+	exit 0;
+}
+
+#------------------------------------------------------------------------------
+# Print version and exit
+#------------------------------------------------------------------------------
+sub version { print "$NAME: version [$VERSION]\n"; exit 0; }
+
+#------------------------------------------------------------------------------
+# Get options from pod
+#------------------------------------------------------------------------------
+sub get_options {
+
+	use Getopt::Long qw(:config no_ignore_case no_auto_abbrev);
+
+	my @Opt;
+
+	sub pod_opt {
+		local $_;
+		my $o = shift;
+		$o =~ s/(=.|[\+\-\!]$)//;
+		$o = join(", ",map{"-$_"} split(/[\|,:;]/,$o));
+		return "$o";
+	}
+
+	while (<DATA>) {
+		s/option\[([^\]]+)\]/push(@Opt,$1) and pod_opt($1)/eg;
+		$main::_DATA_ .= $_;
+	}
+
+	GetOptions($_[0],@Opt) || exit -1;
+
+	help() if $_[0]{'help'} or $_[0]{'man'};
+	version() if $_[0]{'version'};
+
+	$main::VERBOSE = $VERBOSE = $_[0]{'verbose'} if defined $_[0]{'verbose'};
+	$main::DEBUG = $DEBUG = $_[0]{'debug'} if defined $_[0]{'debug'};
+
+}
+
+__DATA__
+
+=head1 NAME
+
+$NAME - Script to extract html table into csv
+
+=head1 SYNOPSIS
+
+Quick usage:
+
+=over
+
+=item $NAME --verbose
+
+=item $NAME --help
+
+=back
+
+=head1 OPTIONS
+
+ -option[table|t=i]   Only dump table number
+ -option[verbose|v+] Verbose mode: increase the verbosity level.
+ -option[debug+]     Debug mode: increase the verbosity level.
+ -option[version|V]  Print version (default: $VERSION)
+ -option[help|h|?]   Print a brief help message and exits.
+ -option[man]        Print the manual page and exits.
+
+=cut
+
+=head1 EXAMPLES
+
+...
+
+=head1 REQUIRES
+
+Getopt::Std, Pod::Usage
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright (C) 2017 Nicolas Boisselier
+
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+See <http://www.gnu.org/licenses/>.
+
+=head1 SEE ALSO
+
+perl(1), http://perldoc.perl.org/perlpodstyle.html
+
+=head1 AUTHOR
+
+Nicolas Boisselier <nicolas.boisselier@gmail.com>
+
+=cut
diff --git a/bin/vimplate b/bin/vimplate
new file mode 100755
index 00000000..563972d8
--- /dev/null
+++ b/bin/vimplate
@@ -0,0 +1,368 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+
+=head1 NAME
+
+vimplate - the vim template system.
+
+=cut
+
+use constant VERSION => '0.2.3';
+
+use POSIX qw(strftime cuserid setlocale LC_ALL);
+use English qw(-no_match_vars);
+use Getopt::Long qw(:config no_ignore_case );
+use Pod::Usage;
+
+my $vimplaterc='';
+
+=head1 DEPENDS on PACKAGES
+
+B<Template-Toolkit> http://search.cpan.org/~abw/Template-Toolkit-2.14
+
+please install Template-Toolkit on your system.
+
+=cut
+
+BEGIN {
+  eval { require Template; };
+  if ($EVAL_ERROR=~/Can't locate Template.pm/) {
+    print STDERR "$EVAL_ERROR";
+    print STDERR '-' x 60, "\n";
+    print STDERR "please install Template-Toolkit!\n";
+    print STDERR "example with $^X -MCPAN -e\"install Template\"\n";
+    print STDERR '-' x 60, "\n";
+    exit 1;
+  }
+}
+
+=head1 DEPENDS on SETTINGS
+
+B<variable HOME>
+
+on unix/bsd/linux the variable home is set.
+On Windows please set the variable home to the value
+where _vimplaterc should be locatet.
+
+=cut
+
+unless ( $ENV{'HOME'} ) {
+  print STDERR "Variable HOME isn't set!\n";
+  print STDERR "Please read the documentation.\n";
+  exit 1;
+}
+else {
+  if ( $^O =~ /Win/ ) {
+    $vimplaterc = $ENV{'HOME'} . '/_vimplaterc';
+    unless ( $ENV{'USER'} ) {
+      $ENV{'USER'}=$ENV{'USERNAME'};
+    }
+    else {
+      print STDERR "Variable USER isn't set!\n";
+      print STDERR "Please set this variable.\n";
+    }
+  }
+  else {
+    $vimplaterc = $ENV{'HOME'} . '/.vimplaterc';
+  }
+}
+
+=head1 SYNOPSIS
+
+=over 4
+
+=item vimplate <-template=<template>> [-out=<file>]
+               [-user=<user>] [-dir=<dir>] [-config=<file>]
+
+=item vimplate <-createconfig>
+
+=item vimplate <-listtemplates>
+
+=item vimplate <-listusers>
+
+=item vimplate <-version>
+
+=item vimplate <-help|-h|-?>
+
+=item vimplate <-man>
+
+=back
+
+=cut
+
+my %opt = ();
+GetOptions(
+            \%opt, 'template|t=s', '-out|o=s',
+                   'user=s', 'dir=s', 'config=s',
+                   'createconfig',
+                   'listtemplates!',
+                   'listusers!',
+                   'version!',
+                   'help|h|?!',
+                   'man!',
+  )
+  or pod2usage( -verbose => 0, -exitval => 1, -output => \*STDERR );
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<-help|-h|-?>
+
+Print a brief help message and exit.
+
+=cut
+
+if ( defined $opt{help} ) {
+  pod2usage( -verbose => 1, -exitval => 0 )
+}
+
+=item B<-man>
+
+Print the manual page and exit.
+
+=cut
+
+if ( defined $opt{man} ) {
+  pod2usage( -verbose => 2, -exitval => 0 )
+}
+
+=item B<-version>
+
+Print the version and exit.
+
+=cut
+
+if ( defined $opt{version} ) {
+  print 'vimplate version ' . VERSION . "\n";
+  exit 0;
+}
+
+=item B<-createconfig>
+
+Write a vimplate config to $HOME/.vimplaterc
+or %HOME%\_vimplaterc (on Windows) and exit.
+
+=cut
+
+if ( defined $opt{createconfig} ) {
+  if ( -f $vimplaterc ) {
+    print STDERR "I don't overwrite " . $vimplaterc . "\n";
+    print STDERR $vimplaterc . " already exist!\n";
+    exit 1;
+  }
+  eval {
+    open( F, '>', $vimplaterc )
+  };
+  if ($EVAL_ERROR) {
+    print STDERR "Can't write " . $vimplaterc . ": $!\n";
+    exit 1;
+  }
+  print F "# This is an example configuration.\n";
+  print F "# please see: http://napali.ch/vimplate\n";
+  print F "\n";
+  print F "# you can use \$Config::opt instead command options:\n";
+  print F "#   -user=<user> -dir=<dir>\n";
+  print F "\$Config::opt = {\n";
+  print F "                  dir  => '$ENV{HOME}/vimplate/Templates',\n";
+  print F "                  user => '$ENV{USER}',\n";
+  print F "};\n";
+  print F "\n";
+  print F "# we need \$Config::user with the option -user=<name>\n";
+  print F "\$Config::user = {\n";
+  print F "                   $ENV{USER}  => {\n";
+  print F "                                      firstname => 'yourFirstname',\n";
+  print F "                                      lastname  => '$ENV{USER}',\n";
+  print F "                                      mail      => '$ENV{USER}\@example.org',\n";
+  print F "                                      etc       => '...',\n";
+  print F "                   },\n";
+  print F "                   otherUser   => {\n";
+  print F "                                      firstname => 'otherFirstname',\n";
+  print F "                                      lastname  => 'otherLastname',\n";
+  print F "                                      mail      => 'otherMail\@example.org',\n";
+  print F "                   },\n";
+  print F "};\n";
+  print F "\n";
+  print F "# use \$Config::var for your own variables or subroutines\n";
+  print F "\$Config::var = {\n";
+  print F "                 yourArray => [ 'Perl', 'C', 'C++' ],\n";
+  print F "                 example   => sub{ time },\n";
+  print F "};\n";
+  close F
+    and print $vimplaterc . " written.\n";
+  exit 0;
+}
+
+{
+  package Config;
+  our $user;
+  our $var;
+  our $opt;
+  if ( -f $vimplaterc ) {
+    do $vimplaterc
+      or die "error in $vimplaterc: $!\n";
+  }
+}
+
+=item B<-listtemplate>
+
+Print the avaible templates and exit.
+
+=cut
+
+sub listTemplates {
+  opendir(DIR, $Config::opt->{dir});
+  my ($file, @files);
+  FILE: foreach $file (readdir(DIR)) {
+    next FILE if ($file!~/\.tt$/);
+    $file=~s/\.tt//;
+    push @files, $file;
+  }
+  close DIR;
+  return @files;
+}
+if ( defined $opt{listtemplates} ) {
+    print "$_\n" for listTemplates();
+    exit 0;
+}
+
+=item B<-listusers>
+
+Print the avaible users and exit.
+
+=cut
+
+sub listUsers {
+  return ( sort keys %$Config::user )
+}
+if ( defined $opt{listusers} ) {
+  print "$_\n" for listUsers();
+  exit 0;
+}
+
+=item B<-user|u=<username>>
+
+Use the information form user <username> while parsing templates.
+
+=cut
+
+if ( defined $opt{user} ) {
+  $Config::opt->{user}=$opt{user}
+}
+
+=item B<-dir|d=<templatedir>>
+
+Search templatefiles in <templatedir>.
+
+=cut
+
+if ( defined $opt{dir} ) {
+  $Config::opt->{dir}=$opt{dir}
+}
+
+=item B<-template=<templatefile>>
+
+Use the <templatefile>.
+
+=cut
+
+if ( defined $opt{template} ) {
+  my $tt = Template->new(
+                          {
+                            INCLUDE_PATH => $Config::opt->{dir},
+                            EVAL_PERL    => 1,
+                          }
+                        );
+
+  my $ttvar = {
+    user   => $Config::user->{ $Config::opt->{user} },
+    var    => $Config::var,
+    locale => sub {
+      if ( @_ > 0 ) {
+        setlocale( LC_ALL, shift );
+        return undef;
+      } else {
+        return setlocale(LC_ALL);
+      }
+    },
+    date => sub {
+      my $loc = setlocale(LC_ALL);
+      setlocale( LC_ALL, shift ) if ( @_ > 1 );
+      my $timestring = POSIX::strftime( shift, localtime );
+      setlocale( LC_ALL, $loc );
+      return $timestring;
+    },
+    uc => sub {
+      return uc $_[0];
+    },
+    ucfirst => sub {
+      return ucfirst $_[0];
+    },
+    lc => sub {
+      return lc $_[0];
+    },
+    choice => sub {
+      my $text=shift;
+      my $i=0;
+      print "$text\n";
+      foreach my $line (@_)
+      {
+        printf("%2d) %s\n", $i++, $line);
+      }
+      my $input=0;
+      do {
+        chomp($input=<STDIN>);
+        unless ($input=~/^\d+$/){$input=-1};
+      } while ( $input<0 or $input>=scalar(@_) );
+      return $_[$input];
+    },
+    input => sub {
+      print "$_[0]";
+      chomp(my $input=<STDIN>);
+      return $input;
+    },
+  };
+
+  $tt->process( $opt{template} . '.tt', $ttvar, $opt{out} );
+  if ( $EVAL_ERROR =~ /file error - .*: not found/ ) {
+    $tt->process( 
+      $ttvar->{'choice'}("choice: ", listTemplates()) . '.tt', $ttvar, $opt{out}
+    );
+  }
+  if ($EVAL_ERROR) {
+    print STDERR $EVAL_ERROR;
+    exit 1;
+  }
+  exit 0;
+}
+
+pod2usage( -verbose => 1, -exitval => 1, -output => \*STDERR );
+
+=back
+
+=cut
+
+__END__
+
+=head1 DESCRIPTION
+
+B<vimplate> Print a spezified template to standard output.
+
+=head1 AUTHOR
+
+Urs Stotz <stotz@gmx.ch>
+
+=head1 COPYRIGHT
+
+Copyright (c) 2004-2005, Urs Stotz <stotz@gmx.ch>
+
+All rights reserved. This program is free software; you can redistribute it
+and/or modify it under the same terms as Perl itself.
+
+=head1 SEE ALSO
+
+L<perl(1)|perl> L<Template(3)|Template>
+
+=cut
diff --git a/share/db/sys.sql b/share/db/sys.sql
index d86cbb50..f26ce745 100644
--- a/share/db/sys.sql
+++ b/share/db/sys.sql
@@ -28,3 +28,13 @@ CREATE TABLE shell_function (
   name VARCHAR(255) PRIMARY KEY,
   code BLOB
 );
+
+DROP TABLE IF EXISTS ovh_dedicated;
+CREATE TABLE ovh_dedicated (
+  offer VARCHAR(80),
+  proc VARCHAR(150),
+  cpu VARCHAR(150),
+  ram VARCHAR(150),
+  disk VARCHAR(150),
+  price float(5,2)
+);
diff --git a/share/db/update.sh b/share/db/update.sh
index 8dc00cd4..394d2e6f 100755
--- a/share/db/update.sh
+++ b/share/db/update.sh
@@ -47,6 +47,9 @@ data_src() {
     > "$tmp/port.csv"
   csv2table port "$tmp/port.csv"
 
+  curl -s https://www.ovh.co.uk/dedicated_servers/ | perl -MNB -e '$_=join("",<>);s/[\t\s\r\n]+/ /g;for $tr (/<tr[^>]+OffersList.*?>(.*?)<.tr>/g) { @r=(); for $td ($tr=~/<td.*?>(\w.*?)<.td>/g){ $td=~s/\s*<br>\s*/, /; push(@r,$td) if $td=str_trim(html2txt($td)) and $td!~/Configure/}; $r[5] =~ s/,//g; print $_ if @r==6 and $_=join("\t",map{s/[\t\s\r\n]+/ /g;s/ ex\. VAT.*$//; $_}@r)."\n" and $r[0]!~/Intel/ and $r[5]>0 and !$dbl{$_}++;}' > "$tmp/ovh_dedicated.csv"
+  csv2table ovh_dedicated "$tmp/ovh_dedicated.csv"
+
   return 0
   w3m -dump -cols 999999 https://en.wikipedia.org/wiki/List_of_HTTP_status_codes| \
     perl -ne 'next unless /^\d\d\d / .. /^ /;s/.\[\d+\]//g; s/^ *//; s/^(\d+) ([\w _-]+).*?\n$/$1\t$2\t/;print' \