diff options
author | Bernhard Posselt <dev@bernhard-posselt.com> | 2014-12-04 12:04:48 +0100 |
---|---|---|
committer | Bernhard Posselt <dev@bernhard-posselt.com> | 2014-12-04 12:04:48 +0100 |
commit | 435314fcff2a954cd9544cacacc2d5c89cc9d2d4 (patch) | |
tree | d5114e6164475907814ae94109920bc286ec6aa1 /bin/updater | |
parent | 557d097bd4c080b60036241b770107d94c9ce116 (diff) |
use logging module for updater errors
Diffstat (limited to 'bin/updater')
-rw-r--r-- | bin/updater/Makefile | 6 | ||||
-rw-r--r-- | bin/updater/README.rst (renamed from bin/updater/REAME.rst) | 0 | ||||
-rw-r--r-- | bin/updater/build/lib/owncloud_news/__init__.py | 0 | ||||
-rw-r--r-- | bin/updater/build/lib/owncloud_news/__main__.py | 3 | ||||
-rw-r--r-- | bin/updater/build/lib/owncloud_news/application.py | 96 | ||||
-rw-r--r-- | bin/updater/build/lib/owncloud_news/updater.py | 127 | ||||
-rw-r--r-- | bin/updater/example-config.ini | 2 | ||||
-rwxr-xr-x | bin/updater/owncloud_news/application.py | 119 | ||||
-rw-r--r-- | bin/updater/owncloud_news/updater.py | 127 | ||||
-rw-r--r-- | bin/updater/owncloud_news_updater.egg-info/SOURCES.txt | 16 | ||||
-rw-r--r-- | bin/updater/owncloud_news_updater.egg-info/dependency_links.txt | 1 | ||||
-rw-r--r-- | bin/updater/owncloud_news_updater.egg-info/entry_points.txt | 3 | ||||
-rw-r--r-- | bin/updater/owncloud_news_updater.egg-info/requires.txt | 1 | ||||
-rw-r--r-- | bin/updater/owncloud_news_updater.egg-info/top_level.txt | 1 | ||||
-rw-r--r-- | bin/updater/requirements.txt | 2 | ||||
-rw-r--r-- | bin/updater/setup.py | 2 | ||||
-rw-r--r-- | bin/updater/systemd/owncloud-news.service | 1 |
17 files changed, 384 insertions, 123 deletions
diff --git a/bin/updater/Makefile b/bin/updater/Makefile index 193ce39e7..0192126c6 100644 --- a/bin/updater/Makefile +++ b/bin/updater/Makefile @@ -2,12 +2,12 @@ all: install install: install-systemd -preinit: +preinitsetup: mkdir -p /etc/owncloud/news cp $(CURDIR)/example-config.ini /etc/owncloud/news/updater.ini python3 setup.py install -install-systemd: preinit +install-systemd: preinitsetup cp $(CURDIR)/systemd/owncloud-news-updater.service /etc/systemd/system/ @echo "Installed files. Please edit your config in /etc/owncloud/news/updater.ini and run:" @@ -18,7 +18,7 @@ install-systemd: preinit @echo "to reload the changes if you change the config in /etc/owncloud/news/updater.ini" -install-sysvinit: preinit +install-sysvinit: preinitsetup cp $(CURDIR)/sysvinit/owncloud-news-updater /etc/rc.d/ @echo "Installed files. Please edit your config in /etc/owncloud/news/updater.ini and run:" diff --git a/bin/updater/REAME.rst b/bin/updater/README.rst index bc71b3a06..bc71b3a06 100644 --- a/bin/updater/REAME.rst +++ b/bin/updater/README.rst diff --git a/bin/updater/build/lib/owncloud_news/__init__.py b/bin/updater/build/lib/owncloud_news/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bin/updater/build/lib/owncloud_news/__init__.py diff --git a/bin/updater/build/lib/owncloud_news/__main__.py b/bin/updater/build/lib/owncloud_news/__main__.py new file mode 100644 index 000000000..695c2d19b --- /dev/null +++ b/bin/updater/build/lib/owncloud_news/__main__.py @@ -0,0 +1,3 @@ +from owncloud_news.application import main + +main()
\ No newline at end of file diff --git a/bin/updater/build/lib/owncloud_news/application.py b/bin/updater/build/lib/owncloud_news/application.py new file mode 100644 index 000000000..efd022dd3 --- /dev/null +++ b/bin/updater/build/lib/owncloud_news/application.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +Updater script for the news app which allows multiple feeds to be updated at +once to speed up the update process. Built in cron has to be disabled in the +news config, see the README.rst file in the top directory for more information. +""" +__author__ = 'Bernhard Posselt' +__copyright__ = 'Copyright 2012-2014, Bernhard Posselt' +__license__ = 'AGPL3+' +__maintainer__ = 'Bernhard Posselt' +__email__ = 'dev@bernhard-posselt.com' + +import sys +import argparse +import urllib +import configparser + +from owncloud_news.updater import Updater + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--testrun', + help='Run update only once, DO NOT use this in a cron job, only \ + recommended for testing', action='store_true') + parser.add_argument('--threads', '-t', + help='How many feeds should be fetched in parallel, defaults to 10', + default=10, + type=int) + parser.add_argument('--timeout', '-s', + help='Maximum number of seconds for updating a feed, \ + defaults to 5 minutes', + default=5*60, + type=int) + parser.add_argument('--interval', '-i', + help='Update interval between fetching the next round of \ + updates in minutes, defaults to 30 minutes. The update timespan \ + will be subtracted from the interval.', + default=30, + type=int) + parser.add_argument('--config', '-c', + help='Path to config file where all parameters except can be defined \ + as key values pair. An example is in bin/example_config.ini') + parser.add_argument('--user', '-u', + help='Admin username to log into ownCloud. Must be specified on the \ + command line or in the config file.') + parser.add_argument('--password', '-p', + help='Admin password to log into ownCloud') + parser.add_argument('url', + help='The URL where owncloud is installed. Must be specified on the \ + command line or in the config file.', + nargs='?') + args = parser.parse_args() + + # read config file if given + if args.config: + config = configparser.ConfigParser() + files = config.read(args.config) + + if len(files) <= 0: + print('Error: could not find config file %s' % args.config) + exit(1) + + config_values = config['updater'] + if 'user' in config_values: + args.user = config_values['user'] + if 'password' in config_values: + args.password = config_values['password'] + if 'testrun' in config_values: + args.testrun = config_values.getboolean('testrun') + if 'threads' in config_values: + args.threads = int(config_values['threads']) + if 'interval' in config_values: + args.interval = int(config_values['interval']) + if 'url' in config_values: + args.url = config_values['url'] + + # url and user must be specified either from the command line or in the + # config file + if not args.url or not args.user: + parser.print_help() + exit(1) + + # create the updater and run the threads + updater = Updater(args.url, args.threads, args.interval, args.user, + args.password, args.timeout, args.testrun) + updater.run() + + +if __name__ == '__main__': + if sys.version_info < (3, 0): + print('Python 3.0 or higher is required to run this script') + else: + main() + + diff --git a/bin/updater/build/lib/owncloud_news/updater.py b/bin/updater/build/lib/owncloud_news/updater.py new file mode 100644 index 000000000..9107aa435 --- /dev/null +++ b/bin/updater/build/lib/owncloud_news/updater.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import json +import threading +import requests +import time +import logging + +def check_status_code(response): + if response.status_code != 200: + raise Exception('Request failed with %i: %s' % (response.status_code, + response.text)) + + +class Updater: + + def __init__(self, base_url, thread_num, interval, user, password, timeout, + run_once): + self.thread_num = thread_num + self.interval = interval + self.base_url = base_url + self.user = user + self.password = password + self.timeout = timeout + self.run_once = run_once + + if self.base_url[-1] != '/': + self.base_url += '/' + self.base_url += 'index.php/apps/news/api/v1-2' + + self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url + self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url + self.all_feeds_url = '%s/feeds/all' % self.base_url + self.update_url = '%s/feeds/update' % self.base_url + + # logging + self.logger = logger.getLogger('ownCloud News Updater') + self.logger.setLevel(logging.INFO) + + format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + self.logger.basicConfig(format=format) + + + def run(self): + while True: + self.start_time = time.time() # reset clock + + try: + # run the cleanup request and get all the feeds to update + auth = (self.user, self.password) + + before = requests.get(self.before_cleanup_url, auth=auth) + check_status_code(before) + + feeds_response = requests.get(self.all_feeds_url, auth=auth) + check_status_code(feeds_response) + + feeds_json = feeds_response.text + feeds = json.loads(feeds_json)['feeds'] + + # start thread_num threads which update the feeds + threads = [] + for num in range(0, self.thread_num): + thread = UpdateThread(feeds, self.update_url, self.user, + self.password, self.timeout, self.logger) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + after = requests.get(self.after_cleanup_url, auth=auth) + check_status_code(after) + + if self.run_once: + return + + # wait until the interval finished to run again and subtract + # the update run time from the interval + timeout = self.interval - int((time.time() - self.start_time)) + if timeout > 0: + time.sleep(timeout) + + except (Exception) as e: + self.logger.error('%s: %s Trying again in 30 seconds' % + (self.base_url, e)) + time.sleep(30) + + +class UpdateThread(threading.Thread): + + lock = threading.Lock() + + def __init__(self, feeds, update_url, user, password, timeout, logger): + super().__init__() + self.feeds = feeds + self.update_url = update_url + self.user = user + self.password = password + self.timeout = timeout + self.logger = logger + + def run(self): + while True: + with UpdateThread.lock: + if len(self.feeds) > 0: + feed = self.feeds.pop() + else: + return + + feed['feedId'] = feed['id'] + del feed['id'] + + # call the update method of one feed + data = urllib.parse.urlencode(feed) + headers = { + 'Content-type': 'application/json', + 'Accept': 'text/plain' + } + url = '%s?%s' % (self.update_url, data) + + try: + auth = (self.user, self.password) + request = requests.get(url, auth=auth, timeout=self.timeout) + check_status_code(request) + except (Exception) as e: + self.logger.error('%s: %s' % (url, e)) diff --git a/bin/updater/example-config.ini b/bin/updater/example-config.ini index 67e9299b4..7d49aa7cf 100644 --- a/bin/updater/example-config.ini +++ b/bin/updater/example-config.ini @@ -5,6 +5,6 @@ user = admin password = admin threads = 10 -interval = 1000 +interval = 300 testrun = false url = http://localhost/owncloud
\ No newline at end of file diff --git a/bin/updater/owncloud_news/application.py b/bin/updater/owncloud_news/application.py index 1235f00f2..61eed3146 100755 --- a/bin/updater/owncloud_news/application.py +++ b/bin/updater/owncloud_news/application.py @@ -11,125 +11,10 @@ __maintainer__ = 'Bernhard Posselt' __email__ = 'dev@bernhard-posselt.com' import sys -import time -import json import argparse -import threading -import requests -import urllib import configparser -def check_status_code(response): - if response.status_code != 200: - raise Exception('Request failed with %i: %s' % (response.status_code, - response.text)) - -class UpdateThread(threading.Thread): - - lock = threading.Lock() - - def __init__(self, feeds, update_url, user, password, timeout): - super().__init__() - self.feeds = feeds - self.update_url = update_url - self.user = user - self.password = password - self.timeout = timeout - - def run(self): - while True: - with UpdateThread.lock: - if len(self.feeds) > 0: - feed = self.feeds.pop() - else: - return - - feed['feedId'] = feed['id'] - del feed['id'] - - # call the update method of one feed - data = urllib.parse.urlencode(feed) - headers = { - 'Content-type': 'application/json', - 'Accept': 'text/plain' - } - url = '%s?%s' % (self.update_url, data) - - try: - auth = (self.user, self.password) - request = requests.get(url, auth=auth, timeout=self.timeout) - check_status_code(request) - except (Exception) as e: - print('%s: %s' % (url, e)) - - - -class Updater: - - def __init__(self, base_url, thread_num, interval, user, password, timeout, - run_once): - self.thread_num = thread_num - self.interval = interval - self.base_url = base_url - self.user = user - self.password = password - self.timeout = timeout - self.run_once = run_once - - if self.base_url[-1] != '/': - self.base_url += '/' - self.base_url += 'index.php/apps/news/api/v1-2' - - self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url - self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url - self.all_feeds_url = '%s/feeds/all' % self.base_url - self.update_url = '%s/feeds/update' % self.base_url - - - def run(self): - while True: - self.start_time = time.time() # reset clock - - try: - # run the cleanup request and get all the feeds to update - auth = (self.user, self.password) - - before = requests.get(self.before_cleanup_url, auth=auth) - check_status_code(before) - - feeds_response = requests.get(self.all_feeds_url, auth=auth) - check_status_code(feeds_response) - - feeds_json = feeds_response.text - feeds = json.loads(feeds_json)['feeds'] - - # start thread_num threads which update the feeds - threads = [] - for num in range(0, self.thread_num): - thread = UpdateThread(feeds, self.update_url, self.user, - self.password, self.timeout) - thread.start() - threads.append(thread) - - for thread in threads: - thread.join() - - after = requests.get(self.after_cleanup_url, auth=auth) - check_status_code(after) - - if self.run_once: - return - - # wait until the interval finished to run again and subtract - # the update run time from the interval - timeout = self.interval - int((time.time() - self.start_time)) - if timeout > 0: - time.sleep(timeout) - - except (Exception) as e: - print('%s: %s' % (self.base_url, e)) - print('Trying again in 30 seconds') - time.sleep(30) +from owncloud_news.updater import Updater def main(): @@ -197,7 +82,7 @@ def main(): # create the updater and run the threads updater = Updater(args.url, args.threads, args.interval, args.user, - args.password, args.timeout, args.testrun) + args.password, args.timeout, args.testrun) updater.run() diff --git a/bin/updater/owncloud_news/updater.py b/bin/updater/owncloud_news/updater.py new file mode 100644 index 000000000..f59c4f05d --- /dev/null +++ b/bin/updater/owncloud_news/updater.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import json +import threading +import requests +import time +import logging +import urllib + +def check_status_code(response): + if response.status_code != 200: + raise Exception('Request failed with %i: %s' % (response.status_code, + response.text)) + + +class Updater: + + def __init__(self, base_url, thread_num, interval, user, password, timeout, + run_once): + self.thread_num = thread_num + self.interval = interval + self.base_url = base_url + self.user = user + self.password = password + self.timeout = timeout + self.run_once = run_once + + if self.base_url[-1] != '/': + self.base_url += '/' + self.base_url += 'index.php/apps/news/api/v1-2' + + self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url + self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url + self.all_feeds_url = '%s/feeds/all' % self.base_url + self.update_url = '%s/feeds/update' % self.base_url + + # logging + format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + logging.basicConfig(format=format) + self.logger = logging.getLogger('ownCloud News Updater') + self.logger.setLevel(logging.INFO) + + + def run(self): + while True: + self.start_time = time.time() # reset clock + + try: + # run the cleanup request and get all the feeds to update + auth = (self.user, self.password) + + before = requests.get(self.before_cleanup_url, auth=auth) + check_status_code(before) + + feeds_response = requests.get(self.all_feeds_url, auth=auth) + check_status_code(feeds_response) + + feeds_json = feeds_response.text + feeds = json.loads(feeds_json)['feeds'] + + # start thread_num threads which update the feeds + threads = [] + for num in range(0, self.thread_num): + thread = UpdateThread(feeds, self.update_url, self.user, + self.password, self.timeout, self.logger) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + after = requests.get(self.after_cleanup_url, auth=auth) + check_status_code(after) + + if self.run_once: + return + + # wait until the interval finished to run again and subtract + # the update run time from the interval + timeout = self.interval - int((time.time() - self.start_time)) + if timeout > 0: + time.sleep(timeout) + + except (Exception) as e: + self.logger.error('%s: %s Trying again in 30 seconds' % + (self.base_url, e)) + time.sleep(30) + + +class UpdateThread(threading.Thread): + + lock = threading.Lock() + + def __init__(self, feeds, update_url, user, password, timeout, logger): + super().__init__() + self.feeds = feeds + self.update_url = update_url + self.user = user + self.password = password + self.timeout = timeout + self.logger = logger + + def run(self): + while True: + with UpdateThread.lock: + if len(self.feeds) > 0: + feed = self.feeds.pop() + else: + return + + feed['feedId'] = feed['id'] + del feed['id'] + + # call the update method of one feed + data = urllib.parse.urlencode(feed) + headers = { + 'Content-type': 'application/json', + 'Accept': 'text/plain' + } + url = '%s?%s' % (self.update_url, data) + + try: + auth = (self.user, self.password) + request = requests.get(url, auth=auth, timeout=self.timeout) + check_status_code(request) + except (Exception) as e: + self.logger.error('%s: %s' % (url, e)) diff --git a/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt b/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt new file mode 100644 index 000000000..ec16370e6 --- /dev/null +++ b/bin/updater/owncloud_news_updater.egg-info/SOURCES.txt @@ -0,0 +1,16 @@ +LICENSE.txt +MANIFEST.in +README.rst +example-config.ini +requirements.txt +setup.py +owncloud_news/__init__.py +owncloud_news/__main__.py +owncloud_news/application.py +owncloud_news/updater.py +owncloud_news_updater.egg-info/PKG-INFO +owncloud_news_updater.egg-info/SOURCES.txt +owncloud_news_updater.egg-info/dependency_links.txt +owncloud_news_updater.egg-info/entry_points.txt +owncloud_news_updater.egg-info/requires.txt +owncloud_news_updater.egg-info/top_level.txt
\ No newline at end of file diff --git a/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt b/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/bin/updater/owncloud_news_updater.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/bin/updater/owncloud_news_updater.egg-info/entry_points.txt b/bin/updater/owncloud_news_updater.egg-info/entry_points.txt new file mode 100644 index 000000000..467bbbd89 --- /dev/null +++ b/bin/updater/owncloud_news_updater.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +owncloud-news-updater = owncloud_news.application:main + diff --git a/bin/updater/owncloud_news_updater.egg-info/requires.txt b/bin/updater/owncloud_news_updater.egg-info/requires.txt new file mode 100644 index 000000000..b6bc7b49d --- /dev/null +++ b/bin/updater/owncloud_news_updater.egg-info/requires.txt @@ -0,0 +1 @@ +requests==2.5.0 diff --git a/bin/updater/owncloud_news_updater.egg-info/top_level.txt b/bin/updater/owncloud_news_updater.egg-info/top_level.txt new file mode 100644 index 000000000..03a09b50d --- /dev/null +++ b/bin/updater/owncloud_news_updater.egg-info/top_level.txt @@ -0,0 +1 @@ +owncloud_news diff --git a/bin/updater/requirements.txt b/bin/updater/requirements.txt index f572f1c8d..8e9709c5b 100644 --- a/bin/updater/requirements.txt +++ b/bin/updater/requirements.txt @@ -1 +1 @@ -requests==2.2.1
\ No newline at end of file +requests==2.5.0
\ No newline at end of file diff --git a/bin/updater/setup.py b/bin/updater/setup.py index 8c1173587..4fc64a63b 100644 --- a/bin/updater/setup.py +++ b/bin/updater/setup.py @@ -1,4 +1,4 @@ -from os import exit +from sys import exit from xml.etree import ElementTree try: diff --git a/bin/updater/systemd/owncloud-news.service b/bin/updater/systemd/owncloud-news.service index 84c9cf132..b9873502f 100644 --- a/bin/updater/systemd/owncloud-news.service +++ b/bin/updater/systemd/owncloud-news.service @@ -3,6 +3,7 @@ After=default.target [Service] Type=simple +User=nobody ExecStart=/usr/bin/python3 /usr/local/bin/owncloud-news-updater -c /etc/owncloud/news/config.ini [Install] |