import time
import datetime
from dateutil import parser
+import sys
# Function to fetch the rss feed and return the parsed RSS
def parseRSS( rss_url ):
# NB 22.06.18 print(key)
#print(feed['feed']['title'])
#print(feed['items'])
-# NB 23.06.18 for newsitem in feed['items']:
-# NB 23.06.18 print()
-# NB 23.06.18 for k,v in newsitem.items():
-# NB 23.06.18 print(k,' : ',v)
-# NB 23.06.18 return
+# NB 25.06.18 for newsitem in feed['items']:
+# NB 25.06.18 print('>',feed['feed']['title'])
+# NB 25.06.18 for k,v in newsitem.items():
+# NB 25.06.18 print(k)
+# NB 25.06.18 #print(k,' : ',v)
+# NB 25.06.18 break
+# NB 25.06.18 return
for newsitem in feed['items']:
line = [
]
#headlines.append(newsitem['content'][0]['value'].replace("\n",""))
#print(newsitem)
- # Channel Title
- line.append(feed['feed']['title'])
+ # Link
+ line.append(newsitem['link'])
+
+ # Channel Link
+ line.append(re.sub('/*$','',feed['feed']['link']))
+
+ # Title
+ if 'title' in newsitem:
+ line.append(newsitem['title'])
+ else:
+ continue
# Date
if 'published_parsed' in newsitem:
published = time.strftime('%F %T',newsitem["updated_parsed"])
else:
continue
-
line.append(published)
- # Title
- if 'title' in newsitem:
- line.append(newsitem['title'])
- else:
- continue
-
# Content
content = ''
if 'content' in newsitem:
html = newsitem['content'][0]['value']
elif 'description' in newsitem:
html = newsitem['description']
- elif 'link' in newsitem:
- content = newsitem['link']
- #print(content)
+ elif 'comments' in newsitem:
+ html = newsitem['comments']
+# NB 25.06.18 elif 'link' in newsitem:
+# NB 25.06.18 content = newsitem['link']
+# NB 25.06.18 #print(content)
else:
html = ''
if not content:
- soup = BeautifulSoup(html,"html.parser")
- content = re.sub(r'\s*\n',' ',soup.get_text())
+ content = html
+ #soup = BeautifulSoup(content,"html.parser")
+ #content = soup.get_text()
+ content = re.sub(r'\s*\n',' ',content)
#content = soup.get_text().replace("\n"," ")
line.append(content)
return headlines
-# A list to hold all headlines
-allheadlines = []
-
# List of RSS feeds that we will fetch and combine
-urls = {
+urls = [
#'https://feeds.feedburner.com/wrc/',
+ #'http://feeds.feedburner.com/CorbettReportRSS',
#'https://www.egaliteetreconciliation.fr/spip.php?page=backend',
- 'https://www.youtube.com/feeds/videos.xml?channel_id=UCUIjs9R044OjAxKzk0xhGoQ',
-}
+ #'https://pressfortruth.ca/index.php/tools/blocks/problog_list/rss?cID=147',
+ #'https://www.infowars.com/feed/custom_feed_rss',
+ #'https://www.youtube.com/feeds/videos.xml?channel_id=UCUIjs9R044OjAxKzk0xhGoQ',
+]
-for url in urls:
+for url in sys.argv[1:]:
getHeadlines( url )
-
+# NB 25.06.18 for url in urls:
+# NB 25.06.18 getHeadlines( url )
+# NB 25.06.18
.' frameborder="0"'
.' allowfullscreen'
.'></iframe>',
+ 'img' => function($str,$id) { return '<img alt="'.htmlentities($str).'" src="https://i1.ytimg.com/vi/'.$id.'/default.jpg" />'; },
],
'youtube' => [
#'exp' => 'www.youtube.com/watch\?v=([^/&\s]+)',
.' frameborder="0"'
.' allowfullscreen'
.'></iframe></a>',
+ 'img' => function($str,$id) { return '<img alt="'.htmlentities($str).'" src="https://i1.ytimg.com/vi/'.$id.'/default.jpg" />'; },
],
];
if (!preg_match(','.$site['exp'].',',$str,$m)) continue;
$id = $m[1];
$idsite = $k;
+
+ if (!empty($site['img'])) {
+ $fct = $site['img'];
+ $img = $fct($str,$id);
+ }
+
break;
}
if (empty($id)) {
+
$ext = preg_match(',^(\w+://|/).*\.(\w+)$,',$str,$m) ? $m[2] : '';
if ($ext) foreach (self::Types as $t => $e) {
if (strpos($t,'video/') !== 0 or $e != $ext) continue;
return '<video class="video" src="'.$str.'" controls />';
}
+
return;
}
return $init
. '<a class="video" href="'.$str.'"'
.' onclick="this.outerHTML=html_video[\''.$idsite.'\'].replace(\'<SRC>\',\''.$id.'\');return false"'
- .'>'.$str.'</a>';
-
- # NB 10.06.18: TODEL
- $id = '';
- if (!preg_match(',//www.youtube.com/watch\?v=([^&\s]+),',$str,$m)) return $str;
- $id = $m[1];
-
- $iframe = ''
- .'<iframe'
- .' class="video"'
- .' type="text/html"'
- .' src="//www.youtube.com/embed/<ID>?enablejsapi=1"'
- .' frameborder="0"'
- .'></iframe>'
- ;
-
- static $init = '';
- if (empty($init)) {
- $init = ''
- . '<script type="text/javascript"><!-- '."\n"
- . 'var html_video = document.createElement("div");'
- . 'html_video.innerHTML = "' . htmlentities($iframe) . '";'
- #. 'var html_video = "' . str_replace('"','\\"',htmlentities($iframe)) . '";'
- . 'html_video = html_video.childNodes.length === 0 ? "" : html_video.childNodes[0].nodeValue;'
- . "\n".' --></script>'
- ;
- }
-
- return $init
- . '<a class="video" href="#"'
- .' onclick="this.outerHTML=html_video.replace(\'<ID>\',\''.$id.'\')"'
- .'>'.$str.'</a>';
-
- $iframe = str_replace('<ID>',$id,$iframe);
+ .'>'
+ .(empty($img) ? $str : $img)
+ .'</a>';
}
if ($mime === null) $mime = self::fromContent($str);
if ($mime == 'text/plain') {
+
+ # Videos
if ($v = self::html_video($str)) return $v;
- elseif($v = preg_replace_callback(',(\w+://\S+),',function($v){
+
+ # A HREF
+ elseif($v = preg_replace_callback(',^(\w+://\S+),',function($v){
return '<a target="_blank" href="'.$v[1].'">'.htmlentities($v[1]).'</a>';
},$str) and $str != $v) return $v;
- #else preg_replace(',(\w+://\S+),e','<a href="$1">'.htmlentities('$1').'</a>',$str);
-
} elseif (preg_match('@^image/@',$mime)) {
return self::html_img_data($str);
DELETE FROM html_tag;
COPY html_tag (name,link,description) FROM PROGRAM '. /etc/profile && html2txt https://html.com/tags/ \
| grep "^<" \
-| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://html.com/tags/\1/\t," \
+| sed -E -e "s/ *HTML Tag *//" -e "s,^<([^>]+)>,\1\thttps://www.w3schools.com/TAGS/tag_\1.asp\t," \
' WITH (format 'text', NULL '');
COMMIT;
--- /dev/null
+DROP TABLE IF EXISTS website;
+CREATE TABLE IF NOT EXISTS website (
+ url varchar(200) PRIMARY KEY,
+ --id serial PRIMARY KEY,
+ lang char(3) REFERENCES lang (id),
+ name varchar(200),
+ description varchar(500),
+ --url varchar(200),
+ rss varchar(300)
+);
+BEGIN TRANSACTION;
+DELETE FROM website;
+COPY website FROM STDIN WITH (format 'text', NULL '');
+https://www.corbettreport.com eng The Corbett Report Open Source Intelligence News https://feeds.feedburner.com/CorbettReportRSS
+https://wearechange.or/ eng We Are Change Be the Change You Wish to See in the World https://feeds.feedburner.com/wrc
+https://www.egaliteetreconciliation.fr fra Egalite et Réconciliation Gauche du travail, Droite des valeurs : pour une réconciliation nationale ! https://www.egaliteetreconciliation.fr/spip.php?page=backend
+https://www.infowars.com eng INFOWARS Alex Jones' Infowars: There's a war on for your mind! https://www.infowars.com/feed/custom_feed_rss
+https://www.tvlibertes.com fra TVL TVLibertés - 1ère chaîne de réinformation. Nous, c'est vous https://www.tvlibertes.com/feed
+https://www.rt.com eng RT International RT is the first Russian 24/7 English-language news channel which brings the Russian view on global news. https://www.rt.com/rss
+https://pressfortruth.ca eng Press For Truth Press For Truth is a team of videographers, investigative journalists and political activists. We do our best to expose the global elite and their plans for a new world order. https://pressfortruth.ca/index.php/tools/blocks/problog_list/rss?cID=147
+\.
+-- 5|eng|Youtube INFO WARS NEWS||https://www.youtube.com/channel/UCUIjs9R044OjAxKzk0xhGoQ|https://www.youtube.com/feeds/videos.xml?channel_id=UCUIjs9R044OjAxKzk0xhGoQ
+COMMIT;
}
.rows .row img {
+ /*
height: 1.2em;
+ */
+ max-height: 4.8em;
}
.rows .row iframe.video {
height: 150px;
form.edit .fields textarea, form.edit .fields select {
width: 65%;
}
+form.edit .fields .field a {
+ display: block;
+}
form.edit .fields img {
max-height: 90px;
-html{font:90% 'Trebuchet MS',sans-serif;background-color:#eee}body{display:table;margin-left:auto;margin-right:auto;padding-left:1em;padding-right:1em;color:#444}@keyframes spin{0%{transform:rotate(0)}100%{transform:rotate(360deg)}}.loader{border:16px solid #fff;border-radius:50%;border-top:16px solid #000;padding:0;margin:0 auto 0 auto;width:8em;height:8em;animation:spin 2s linear infinite;display:block}body.loader{margin-top:8em}.loader *{display:none!important}h1{margin:0 0 .2em 0;font-size:180%;padding-bottom:.3em;margin-bottom:.5em;border-bottom:dashed 1px #444}h1 select{padding:0;border:none;background:inherit;font:inherit}a{color:inherit}div>a,li a,p a,td a{color:#3572b0}a,a:visited{text-decoration:none}.button:hover,a:hover{opacity:.8}ul{padding:0;margin:0}li{list-style:none;padding:0 0 0 .3em}table{border-collapse:collapse}td,th{padding:.2em .7em}th a.sort{float:left;margin-right:.1em}input:not([type]),input[type=''],input[type=date],input[type=email],input[type=password],input[type=text],input[type=url],select,textarea{border-radius:3px;border:solid 1px #ddd;padding:.3em .4em;box-shadow:0 1px 1px #eee}input[size]{max-width:70%}div.row div label:after,form label:after{content:':'}form input,form label,form select{vertical-align:middle;margin:.2em .4em .2em 0}.rows{padding:0}.rows form{display:inline-block}.block,.menu,div.row,form.edit,pre,table.rows,ul.row{border-radius:4px;border-spacing:0;padding:.5em .2em;margin-bottom:.5em;border:solid 1px #ddd;background-color:#fafafa;box-shadow:0 1px 2px #eee}table.rows{border-style:hidden;border-collapse:collapse;box-shadow:0 0 0 1px #ddd;table-layout:fixed}table.rows tr:nth-child(even) td{background-color:#fefefe}div.row div,table.rows td,table.rows th,ul.row li{border-bottom:solid 1px #ddd}div.row div,table.rows tr:last-child td,ul.row li{border-bottom:none}table.rows td,table.rows th{border-right:dashed 1px #ddd}.rows .count{text-align:right}table.rows th{text-align:left}table.rows td{padding:.3em .3em}table.rows.fixed{width:100%}div.row div label,table.rows.fixed td,table.rows.fixed th{white-space:nowrap;text-overflow:ellipsis;overflow:hidden}table.rows.fixed th{white-space:normal}table.rows td.delete,table.rows td.view{width:1%;white-space:nowrap}table.rows.fixed th.view{width:2em}table.rows.fixed th.delete{width:2.7em}div.rows{clear:both}div.row{float:left;margin-right:4px;min-width:24%}.nofloat{float:none}div.row div{margin:0;padding:.1em .2em;border-bottom:solid 1px #fff}.rows .row img{height:1.2em}.rows .row iframe.video{height:150px;width:250px}div.row div.view .button{margin-bottom:5px}div.row div.delete .button{margin-top:5px}div.row div:last-child{border-bottom:none;padding-bottom:0}div.row div.delete label,div.row div.view label{display:none}div.row div.delete,div.row div.view{text-align:center}div.row div label,ul.row li label{padding:.1em .5em .1em 0;min-width:20%;display:inline-block}.button{font:inherit;display:inline-block;text-decoration:none;line-height:1.6em;cursor:pointer;white-space:nowrap;box-sizing:border-box;margin:0 .1em 0 0;padding:0 .3em;border-radius:4px;border:none;color:#3572b0;background-color:#eee;background:linear-gradient(to bottom right,#eee,#ddd);box-shadow:0 1px 1px #bbb}.bottom,.center,.menu,.nav,.rows{clear:both;display:table}.bottom,.buttons,.center,.menu,.nav,.rows,td.button{margin-left:auto;margin-right:auto}div.buttons{text-align:center;margin-top:.5em}form.menu{padding:.5em;background-color:#fafafa;max-width:1024px}form.menu .button.add{float:right;margin-top:.2em}form.menu .criterias{border-top:dashed 1px #ddd;margin-top:.3em;padding-top:.3em;clear:both}form.menu .criterias span.label{border-right:solid 1px #fff;border-radius:3px;margin:0 .2em;white-space:nowrap;float:left}form.menu .criterias span.label:last-child{border-right:none}form.menu select:focus{max-width:auto}form.menu select{max-width:20%}form.menu .criterias input[type=text]{width:4em}div.row label,form.edit .fields label{width:25%;text-align:right;display:inline-block;margin-right:1em;vertical-align:top}div.row .label span,form.edit .fields .label span{display:inline-block;text-align:left;vertical-align:middle;margin-top:.2em}form.edit .fields .field{width:54%}form.edit .fields input[size],form.edit .fields select,form.edit .fields textarea{width:65%}form.edit .fields img{max-height:90px}iframe.video{float:left}
+html{font:90% 'Trebuchet MS',sans-serif;background-color:#eee}body{display:table;margin-left:auto;margin-right:auto;padding-left:1em;padding-right:1em;color:#444}@keyframes spin{0%{transform:rotate(0)}100%{transform:rotate(360deg)}}.loader{border:16px solid #fff;border-radius:50%;border-top:16px solid #000;padding:0;margin:0 auto 0 auto;width:8em;height:8em;animation:spin 2s linear infinite;display:block}body.loader{margin-top:8em}.loader *{display:none!important}h1{margin:0 0 .2em 0;font-size:180%;padding-bottom:.3em;margin-bottom:.5em;border-bottom:dashed 1px #444}h1 select{padding:0;border:none;background:inherit;font:inherit}a{color:inherit}div>a,li a,p a,td a{color:#3572b0}a,a:visited{text-decoration:none}.button:hover,a:hover{opacity:.8}ul{padding:0;margin:0}li{list-style:none;padding:0 0 0 .3em}table{border-collapse:collapse}td,th{padding:.2em .7em}th a.sort{float:left;margin-right:.1em}input:not([type]),input[type=''],input[type=date],input[type=email],input[type=password],input[type=text],input[type=url],select,textarea{border-radius:3px;border:solid 1px #ddd;padding:.3em .4em;box-shadow:0 1px 1px #eee}input[size]{max-width:70%}div.row div label:after,form label:after{content:':'}form input,form label,form select{vertical-align:middle;margin:.2em .4em .2em 0}.rows{padding:0}.rows form{display:inline-block}.block,.menu,div.row,form.edit,pre,table.rows,ul.row{border-radius:4px;border-spacing:0;padding:.5em .2em;margin-bottom:.5em;border:solid 1px #ddd;background-color:#fafafa;box-shadow:0 1px 2px #eee}table.rows{border-style:hidden;border-collapse:collapse;box-shadow:0 0 0 1px #ddd;table-layout:fixed}table.rows tr:nth-child(even) td{background-color:#fefefe}div.row div,table.rows td,table.rows th,ul.row li{border-bottom:solid 1px #ddd}div.row div,table.rows tr:last-child td,ul.row li{border-bottom:none}table.rows td,table.rows th{border-right:dashed 1px #ddd}.rows .count{text-align:right}table.rows th{text-align:left}table.rows td{padding:.3em .3em}table.rows.fixed{width:100%}div.row div label,table.rows.fixed td,table.rows.fixed th{white-space:nowrap;text-overflow:ellipsis;overflow:hidden}table.rows.fixed th{white-space:normal}table.rows td.delete,table.rows td.view{width:1%;white-space:nowrap}table.rows.fixed th.view{width:2em}table.rows.fixed th.delete{width:2.7em}div.rows{clear:both}div.row{float:left;margin-right:4px;min-width:24%}.nofloat{float:none}div.row div{margin:0;padding:.1em .2em;border-bottom:solid 1px #fff}.rows .row img{max-height:4.8em}.rows .row iframe.video{height:150px;width:250px}div.row div.view .button{margin-bottom:5px}div.row div.delete .button{margin-top:5px}div.row div:last-child{border-bottom:none;padding-bottom:0}div.row div.delete label,div.row div.view label{display:none}div.row div.delete,div.row div.view{text-align:center}div.row div label,ul.row li label{padding:.1em .5em .1em 0;min-width:20%;display:inline-block}.button{font:inherit;display:inline-block;text-decoration:none;line-height:1.6em;cursor:pointer;white-space:nowrap;box-sizing:border-box;margin:0 .1em 0 0;padding:0 .3em;border-radius:4px;border:none;color:#3572b0;background-color:#eee;background:linear-gradient(to bottom right,#eee,#ddd);box-shadow:0 1px 1px #bbb}.bottom,.center,.menu,.nav,.rows{clear:both;display:table}.bottom,.buttons,.center,.menu,.nav,.rows,td.button{margin-left:auto;margin-right:auto}div.buttons{text-align:center;margin-top:.5em}form.menu{padding:.5em;background-color:#fafafa;max-width:1024px}form.menu .button.add{float:right;margin-top:.2em}form.menu .criterias{border-top:dashed 1px #ddd;margin-top:.3em;padding-top:.3em;clear:both}form.menu .criterias span.label{border-right:solid 1px #fff;border-radius:3px;margin:0 .2em;white-space:nowrap;float:left}form.menu .criterias span.label:last-child{border-right:none}form.menu select:focus{max-width:auto}form.menu select{max-width:20%}form.menu .criterias input[type=text]{width:4em}div.row label,form.edit .fields label{width:25%;text-align:right;display:inline-block;margin-right:1em;vertical-align:top}div.row .label span,form.edit .fields .label span{display:inline-block;text-align:left;vertical-align:middle;margin-top:.2em}form.edit .fields .field{width:54%}form.edit .fields input[size],form.edit .fields select,form.edit .fields textarea{width:65%}form.edit .fields .field a{display:block}form.edit .fields img{max-height:90px}iframe.video{float:left}
projects / nb.git / commitdiff