use logging module for updater errors

17 files changed, 384 insertions, 123 deletions

diff --git a/bin/updater/Makefile b/bin/updater/Makefile
index 193ce39e7..0192126c6 100644
--- a/bin/updater/Makefile
+++ b/bin/updater/Makefile
@@ -2,12 +2,12 @@ all: install
 
 install: install-systemd
 
-preinit:
+preinitsetup:
 	mkdir -p /etc/owncloud/news
 	cp $(CURDIR)/example-config.ini /etc/owncloud/news/updater.ini
 	python3 setup.py install
 
-install-systemd: preinit
+install-systemd: preinitsetup
 	cp $(CURDIR)/systemd/owncloud-news-updater.service /etc/systemd/system/
 
 	@echo "Installed files. Please edit your config in /etc/owncloud/news/updater.ini and run:"
@@ -18,7 +18,7 @@ install-systemd: preinit
 	@echo "to reload the changes if you change the config in /etc/owncloud/news/updater.ini"
 
 
-install-sysvinit: preinit
+install-sysvinit: preinitsetup
 	cp $(CURDIR)/sysvinit/owncloud-news-updater /etc/rc.d/
 
 	@echo "Installed files. Please edit your config in /etc/owncloud/news/updater.ini and run:"
diff --git a/bin/updater/REAME.rst b/bin/updater/README.rst
index bc71b3a06..bc71b3a06 100644
--- a/bin/updater/REAME.rst
+++ b/bin/updater/README.rst
diff --git a/bin/updater/build/lib/owncloud_news/__init__.py b/bin/updater/build/lib/owncloud_news/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/bin/updater/build/lib/owncloud_news/__init__.py
diff --git a/bin/updater/build/lib/owncloud_news/__main__.py b/bin/updater/build/lib/owncloud_news/__main__.py
new file mode 100644
index 000000000..695c2d19b
--- /dev/null
+++ b/bin/updater/build/lib/owncloud_news/__main__.py
@@ -0,0 +1,3 @@
+from owncloud_news.application import main
+
+main()
\ No newline at end of file
diff --git a/bin/updater/build/lib/owncloud_news/application.py b/bin/updater/build/lib/owncloud_news/application.py
new file mode 100644
index 000000000..efd022dd3
--- /dev/null
+++ b/bin/updater/build/lib/owncloud_news/application.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Updater script for the news app which allows multiple feeds to be updated at
+once to speed up the update process. Built in cron has to be disabled in the
+news config, see the README.rst file in the top directory for more information.
+"""
+__author__ = 'Bernhard Posselt'
+__copyright__ = 'Copyright 2012-2014, Bernhard Posselt'
+__license__ = 'AGPL3+'
+__maintainer__ = 'Bernhard Posselt'
+__email__ = 'dev@bernhard-posselt.com'
+
+import sys
+import argparse
+import urllib
+import configparser
+
+from owncloud_news.updater import Updater
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--testrun',
+        help='Run update only once, DO NOT use this in a cron job, only \
+              recommended for testing', action='store_true')
+    parser.add_argument('--threads', '-t',
+        help='How many feeds should be fetched in parallel, defaults to 10',
+        default=10,
+        type=int)
+    parser.add_argument('--timeout', '-s',
+        help='Maximum number of seconds for updating a feed, \
+              defaults to 5 minutes',
+        default=5*60,
+        type=int)
+    parser.add_argument('--interval', '-i',
+        help='Update interval between fetching the next round of \
+            updates in minutes, defaults to 30 minutes. The update timespan \
+            will be subtracted from the interval.',
+        default=30,
+        type=int)
+    parser.add_argument('--config', '-c',
+        help='Path to config file where all parameters except can be defined \
+        as key values pair. An example is in bin/example_config.ini')
+    parser.add_argument('--user', '-u',
+        help='Admin username to log into ownCloud. Must be specified on the \
+        command line or in the config file.')
+    parser.add_argument('--password', '-p',
+        help='Admin password to log into ownCloud')
+    parser.add_argument('url',
+        help='The URL where owncloud is installed. Must be specified on the \
+        command line or in the config file.',
+        nargs='?')
+    args = parser.parse_args()
+
+    # read config file if given
+    if args.config:
+        config = configparser.ConfigParser()
+        files = config.read(args.config)
+
+        if len(files) <= 0:
+            print('Error: could not find config file %s' % args.config)
+            exit(1)
+
+        config_values = config['updater']
+        if 'user' in config_values:
+            args.user = config_values['user']
+        if 'password' in config_values:
+            args.password = config_values['password']
+        if 'testrun' in config_values:
+            args.testrun = config_values.getboolean('testrun')
+        if 'threads' in config_values:
+            args.threads = int(config_values['threads'])
+        if 'interval' in config_values:
+            args.interval = int(config_values['interval'])
+        if 'url' in config_values:
+            args.url = config_values['url']
+
+    # url and user must be specified either from the command line or in the
+    # config file
+    if not args.url or not args.user:
+        parser.print_help()
+        exit(1)
+
+    # create the updater and run the threads
+    updater = Updater(args.url, args.threads, args.interval, args.user,
+                      args.password, args.timeout, args.testrun)
+    updater.run()
+
+
+if __name__ == '__main__':
+    if sys.version_info < (3, 0):
+        print('Python 3.0 or higher is required to run this script')
+    else:
+        main()
+
+
diff --git a/bin/updater/build/lib/owncloud_news/updater.py b/bin/updater/build/lib/owncloud_news/updater.py
new file mode 100644
index 000000000..9107aa435
--- /dev/null
+++ b/bin/updater/build/lib/owncloud_news/updater.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import json
+import threading
+import requests
+import time
+import logging
+
+def check_status_code(response):
+    if response.status_code != 200:
+        raise Exception('Request failed with %i: %s' % (response.status_code,
+            response.text))
+
+
+class Updater:
+
+    def __init__(self, base_url, thread_num, interval, user, password, timeout,
+                 run_once):
+        self.thread_num = thread_num
+        self.interval = interval
+        self.base_url = base_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+        self.run_once = run_once
+
+        if self.base_url[-1] != '/':
+            self.base_url += '/'
+        self.base_url += 'index.php/apps/news/api/v1-2'
+
+        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
+        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
+        self.all_feeds_url = '%s/feeds/all' % self.base_url
+        self.update_url = '%s/feeds/update' % self.base_url
+
+        # logging
+        self.logger = logger.getLogger('ownCloud News Updater')
+        self.logger.setLevel(logging.INFO)
+
+        format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        self.logger.basicConfig(format=format)
+
+
+    def run(self):
+        while True:
+            self.start_time = time.time()  # reset clock
+
+            try:
+                # run the cleanup request and get all the feeds to update
+                auth = (self.user, self.password)
+
+                before = requests.get(self.before_cleanup_url, auth=auth)
+                check_status_code(before)
+
+                feeds_response = requests.get(self.all_feeds_url, auth=auth)
+                check_status_code(feeds_response)
+
+                feeds_json = feeds_response.text
+                feeds = json.loads(feeds_json)['feeds']
+
+                # start thread_num threads which update the feeds
+                threads = []
+                for num in range(0, self.thread_num):
+                    thread = UpdateThread(feeds, self.update_url, self.user,
+                        self.password, self.timeout, self.logger)
+                    thread.start()
+                    threads.append(thread)
+
+                for thread in threads:
+                    thread.join()
+
+                after = requests.get(self.after_cleanup_url, auth=auth)
+                check_status_code(after)
+
+                if self.run_once:
+                    return
+
+                # wait until the interval finished to run again and subtract
+                # the update run time from the interval
+                timeout = self.interval - int((time.time() - self.start_time))
+                if timeout > 0:
+                    time.sleep(timeout)
+
+            except (Exception) as e:
+                self.logger.error('%s: %s Trying again in 30 seconds' %
+                                   (self.base_url, e))
+                time.sleep(30)
+
+
+class UpdateThread(threading.Thread):
+
+    lock = threading.Lock()
+
+    def __init__(self, feeds, update_url, user, password, timeout, logger):
+        super().__init__()
+        self.feeds = feeds
+        self.update_url = update_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+        self.logger = logger
+
+    def run(self):
+        while True:
+            with UpdateThread.lock:
+                if len(self.feeds) > 0:
+                    feed = self.feeds.pop()
+                else:
+                    return
+
+            feed['feedId'] = feed['id']
+            del feed['id']
+
+            # call the update method of one feed
+            data = urllib.parse.urlencode(feed)
+            headers = {
+                'Content-type': 'application/json',
+                'Accept': 'text/plain'
+            }
+            url = '%s?%s' % (self.update_url, data)
+
+            try:
+                auth = (self.user, self.password)
+                request = requests.get(url, auth=auth, timeout=self.timeout)
+                check_status_code(request)
+            except (Exception) as e:
+                self.logger.error('%s: %s' % (url, e))
diff --git a/bin/updater/example-config.ini b/bin/updater/example-config.ini
index 67e9299b4..7d49aa7cf 100644
--- a/bin/updater/example-config.ini
+++ b/bin/updater/example-config.ini
@@ -5,6 +5,6 @@
 user = admin
 password = admin
 threads = 10
-interval = 1000
+interval = 300
 testrun = false
 url = http://localhost/owncloud
\ No newline at end of file
diff --git a/bin/updater/owncloud_news/application.py b/bin/updater/owncloud_news/application.py
index 1235f00f2..61eed3146 100755
--- a/bin/updater/owncloud_news/application.py
+++ b/bin/updater/owncloud_news/application.py
@@ -11,125 +11,10 @@ __maintainer__ = 'Bernhard Posselt'
 __email__ = 'dev@bernhard-posselt.com'
 
 import sys
-import time
-import json
 import argparse
-import threading
-import requests
-import urllib
 import configparser
 
-def check_status_code(response):
-    if response.status_code != 200:
-        raise Exception('Request failed with %i: %s' % (response.status_code,
-            response.text))
-
-class UpdateThread(threading.Thread):
-
-    lock = threading.Lock()
-
-    def __init__(self, feeds, update_url, user, password, timeout):
-        super().__init__()
-        self.feeds = feeds
-        self.update_url = update_url
-        self.user = user
-        self.password = password
-        self.timeout = timeout
-
-    def run(self):
-        while True:
-            with UpdateThread.lock:
-                if len(self.feeds) > 0:
-                    feed = self.feeds.pop()
-                else:
-                    return
-
-            feed['feedId'] = feed['id']
-            del feed['id']
-
-            # call the update method of one feed
-            data = urllib.parse.urlencode(feed)
-            headers = {
-                'Content-type': 'application/json',
-                'Accept': 'text/plain'
-            }
-            url = '%s?%s' % (self.update_url, data)
-
-            try:
-                auth = (self.user, self.password)
-                request = requests.get(url, auth=auth, timeout=self.timeout)
-                check_status_code(request)
-            except (Exception) as e:
-                print('%s: %s' % (url, e))
-
-
-
-class Updater:
-
-    def __init__(self, base_url, thread_num, interval, user, password, timeout,
-                 run_once):
-        self.thread_num = thread_num
-        self.interval = interval
-        self.base_url = base_url
-        self.user = user
-        self.password = password
-        self.timeout = timeout
-        self.run_once = run_once
-
-        if self.base_url[-1] != '/':
-            self.base_url += '/'
-        self.base_url += 'index.php/apps/news/api/v1-2'
-
-        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
-        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
-        self.all_feeds_url = '%s/feeds/all' % self.base_url
-        self.update_url = '%s/feeds/update' % self.base_url
-
-
-    def run(self):
-        while True:
-            self.start_time = time.time()  # reset clock
-
-            try:
-                # run the cleanup request and get all the feeds to update
-                auth = (self.user, self.password)
-
-                before = requests.get(self.before_cleanup_url, auth=auth)
-                check_status_code(before)
-
-                feeds_response = requests.get(self.all_feeds_url, auth=auth)
-                check_status_code(feeds_response)
-
-                feeds_json = feeds_response.text
-                feeds = json.loads(feeds_json)['feeds']
-
-                # start thread_num threads which update the feeds
-                threads = []
-                for num in range(0, self.thread_num):
-                    thread = UpdateThread(feeds, self.update_url, self.user,
-                        self.password, self.timeout)
-                    thread.start()
-                    threads.append(thread)
-
-                for thread in threads:
-                    thread.join()
-
-                after = requests.get(self.after_cleanup_url, auth=auth)
-                check_status_code(after)
-
-                if self.run_once:
-                    return
-
-                # wait until the interval finished to run again and subtract
-                # the update run time from the interval
-                timeout = self.interval - int((time.time() - self.start_time))
-                if timeout > 0:
-                    time.sleep(timeout)
-
-            except (Exception) as e:
-                print('%s: %s' % (self.base_url, e))
-                print('Trying again in 30 seconds')
-                time.sleep(30)
+from owncloud_news.updater import Updater
 
 
 def main():
@@ -197,7 +82,7 @@ def main():
 
     # create the updater and run the threads
     updater = Updater(args.url, args.threads, args.interval, args.user,
-        args.password, args.timeout, args.testrun)
+                      args.password, args.timeout, args.testrun)
     updater.run()
 
 
diff --git a/bin/updater/owncloud_news/updater.py b/bin/updater/owncloud_news/updater.py
new file mode 100644
index 000000000..f59c4f05d
--- /dev/null
+++ b/bin/updater/owncloud_news/updater.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+import json
+import threading
+import requests
+import time
+import logging
+import urllib
+
+def check_status_code(response):
+    if response.status_code != 200:
+        raise Exception('Request failed with %i: %s' % (response.status_code,
+            response.text))
+
+
+class Updater:
+
+    def __init__(self, base_url, thread_num, interval, user, password, timeout,
+                 run_once):
+        self.thread_num = thread_num
+        self.interval = interval
+        self.base_url = base_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+        self.run_once = run_once
+
+        if self.base_url[-1] != '/':
+            self.base_url += '/'
+        self.base_url += 'index.php/apps/news/api/v1-2'
+
+        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
+        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
+        self.all_feeds_url = '%s/feeds/all' % self.base_url
+        self.update_url = '%s/feeds/update' % self.base_url
+
+        # logging
+        format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        logging.basicConfig(format=format)
+        self.logger = logging.getLogger('ownCloud News Updater')
+        self.logger.setLevel(logging.INFO)
+
+
+    def run(self):
+        while True:
+            self.start_time = time.time()  # reset clock
+
+            try:
+                # run the cleanup request and get all the feeds to update
+                auth = (self.user, self.password)
+
+                before = requests.get(self.before_cleanup_url, auth=auth)
+                check_status_code(before)
+
+                feeds_response = requests.get(self.all_feeds_url, auth=auth)
+                check_status_code(feeds_response)
+
+                feeds_json = feeds_response.text
+                feeds = json.loads(feeds_json)['feeds']
+
+                # start thread_num threads which update the feeds
+                threads = []
+                for num in range(0, self.thread_num):
+                    thread = UpdateThread(feeds, self.update_url, self.user,
+                        self.password, self.timeout, self.logger)
+                    thread.start()
+                    threads.append(thread)
+
+                for thread in threads:
+                    thread.join()
+
+                after = requests.get(self.after_cleanup_url, auth=auth)
+                check_status_code(after)
+
+                if self.run_once:
+                    return
+
+                # wait until the interval finished to run again and subtract
+                # the update run time from the interval
+                timeout = self.interval - int((time.time() - self.start_time))
+                if timeout > 0:
+                    time.sleep(timeout)
+
+            except (Exception) as e:
+                self.logger.error('%s: %s Trying again in 30 seconds' %
+                                   (self.base_url, e))
+                time.sleep(30)
+
+
+class UpdateThread(threading.Thread):
+
+    lock = threading.Lock()
+
+    def __init__(self, feeds, update_url, user, password, timeout, logger):
+        super().__init__()
+        self.feeds = feeds
+        self.update_url = update_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+        self.logger = logger
+
+    def run(self):
+        while True:
+            with UpdateThread.lock:
+                if len(self.feeds) > 0:
+                    feed = self.feeds.pop()
+                else:
+                    return
+
+            feed['feedId'] = feed['id']
+            del feed['id']
+
+            # call the update method of one feed
+            data = urllib.parse.urlencode(feed)
+            headers = {
+                'Content-type': 'application/json',
+                'Accept': 'text/plain'
+            }
+            url = '%s?%s' % (self.update_url, data)
+
+            try:
+                auth = (self.user, self.password)
+                request = requests.get(url, auth=auth, timeout=self.timeout)
+                check_status_code(request)
+            except (Exception) as e:
+                self.logger.error('%s: %s' % (url, e))
diff --git a/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt b/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt
new file mode 100644
index 000000000..ec16370e6
--- /dev/null
+++ b/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt
@@ -0,0 +1,16 @@
+LICENSE.txt
+MANIFEST.in
+README.rst
+example-config.ini
+requirements.txt
+setup.py
+owncloud_news/__init__.py
+owncloud_news/__main__.py
+owncloud_news/application.py
+owncloud_news/updater.py
+owncloud_news_updater.egg-info/PKG-INFO
+owncloud_news_updater.egg-info/SOURCES.txt
+owncloud_news_updater.egg-info/dependency_links.txt
+owncloud_news_updater.egg-info/entry_points.txt
+owncloud_news_updater.egg-info/requires.txt
+owncloud_news_updater.egg-info/top_level.txt
\ No newline at end of file
diff --git a/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt b/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/bin/updater/owncloud_news_updater.egg-info/entry_points.txt b/bin/updater/owncloud_news_updater.egg-info/entry_points.txt
new file mode 100644
index 000000000..467bbbd89
--- /dev/null
+++ b/bin/updater/owncloud_news_updater.egg-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+owncloud-news-updater = owncloud_news.application:main
+
diff --git a/bin/updater/owncloud_news_updater.egg-info/requires.txt b/bin/updater/owncloud_news_updater.egg-info/requires.txt
new file mode 100644
index 000000000..b6bc7b49d
--- /dev/null
+++ b/bin/updater/owncloud_news_updater.egg-info/requires.txt
@@ -0,0 +1 @@
+requests==2.5.0
diff --git a/bin/updater/owncloud_news_updater.egg-info/top_level.txt b/bin/updater/owncloud_news_updater.egg-info/top_level.txt
new file mode 100644
index 000000000..03a09b50d
--- /dev/null
+++ b/bin/updater/owncloud_news_updater.egg-info/top_level.txt
@@ -0,0 +1 @@
+owncloud_news
diff --git a/bin/updater/requirements.txt b/bin/updater/requirements.txt
index f572f1c8d..8e9709c5b 100644
--- a/bin/updater/requirements.txt
+++ b/bin/updater/requirements.txt
@@ -1 +1 @@
-requests==2.2.1
\ No newline at end of file
+requests==2.5.0
\ No newline at end of file
diff --git a/bin/updater/setup.py b/bin/updater/setup.py
index 8c1173587..4fc64a63b 100644
--- a/bin/updater/setup.py
+++ b/bin/updater/setup.py
@@ -1,4 +1,4 @@
-from os import exit
+from sys import exit
 from xml.etree import ElementTree
 
 try:
diff --git a/bin/updater/systemd/owncloud-news.service b/bin/updater/systemd/owncloud-news.service
index 84c9cf132..b9873502f 100644
--- a/bin/updater/systemd/owncloud-news.service
+++ b/bin/updater/systemd/owncloud-news.service
@@ -3,6 +3,7 @@ After=default.target
 
 [Service]
 Type=simple
+User=nobody
 ExecStart=/usr/bin/python3 /usr/local/bin/owncloud-news-updater -c /etc/owncloud/news/config.ini
 
 [Install]