diff options
Diffstat (limited to 'bin/updater/owncloud_news_updater')
| -rw-r--r-- | bin/updater/owncloud_news_updater/__init__.py | 0 | ||||
| -rw-r--r-- | bin/updater/owncloud_news_updater/__main__.py | 3 | ||||
| -rwxr-xr-x | bin/updater/owncloud_news_updater/application.py | 128 | ||||
| -rw-r--r-- | bin/updater/owncloud_news_updater/updater.py | 217 |
4 files changed, 0 insertions, 348 deletions
diff --git a/bin/updater/owncloud_news_updater/__init__.py b/bin/updater/owncloud_news_updater/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/bin/updater/owncloud_news_updater/__init__.py +++ /dev/null diff --git a/bin/updater/owncloud_news_updater/__main__.py b/bin/updater/owncloud_news_updater/__main__.py deleted file mode 100644 index d144c2f44..000000000 --- a/bin/updater/owncloud_news_updater/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from owncloud_news_updater.application import main - -main()
\ No newline at end of file diff --git a/bin/updater/owncloud_news_updater/application.py b/bin/updater/owncloud_news_updater/application.py deleted file mode 100755 index e6bfb5e08..000000000 --- a/bin/updater/owncloud_news_updater/application.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -""" -Updater script for the news app which allows multiple feeds to be updated at -once to speed up the update process. Built in cron has to be disabled in the -news config, see the README.rst file in the top directory for more information. -""" -__author__ = 'Bernhard Posselt' -__copyright__ = 'Copyright 2012-2016, Bernhard Posselt' -__license__ = 'AGPL3+' -__maintainer__ = 'Bernhard Posselt' -__email__ = 'dev@bernhard-posselt.com' - -import os -import sys -import argparse -import configparser - -from owncloud_news_updater.updater import WebUpdater, ConsoleUpdater - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--testrun', - help='Run update only once, DO NOT use this in a cron job, only \ - recommended for testing', action='store_true') - parser.add_argument('--threads', '-t', - help='How many feeds should be fetched in parallel, defaults to 10', - default=10, - type=int) - parser.add_argument('--timeout', '-s', - help='Maximum number of seconds for updating a feed, \ - defaults to 5 minutes', - default=5*60, - type=int) - parser.add_argument('--interval', '-i', - help='Update interval between fetching the next round of \ - updates in seconds, defaults to 15 minutes. The update timespan \ - will be subtracted from the interval.', - default=15*60, - type=int) - parser.add_argument('--loglevel', '-l', - help='Log granularity, info will log all urls and received data, error \ - will only log errors', - default='error', - choices=['info', 'error']) - parser.add_argument('--config', '-c', - help='Path to config file where all parameters except can be defined \ - as key values pair. An example is in bin/example_config.ini') - parser.add_argument('--user', '-u', - help='Admin username to log into ownCloud. Must be specified on the \ - command line or in the config file if the updater should update over \ - HTTP') - parser.add_argument('--password', '-p', - help='Admin password to log into ownCloud if the updater should update \ - over HTTP') - parser.add_argument('url', - help='The URL or absolute path to the directory where owncloud is \ - installed. Must be specified on the command line or in the config \ - file. If the URL starts with http:// or https://, a user and password \ - are required. Otherwise updater tries to use the console based API \ - which was added in 8.1.0', - nargs='?') - args = parser.parse_args() - - # read config file if given - if args.config: - config = configparser.ConfigParser() - files = config.read(args.config) - - if len(files) <= 0: - print('Error: could not find config file %s' % args.config) - exit(1) - - config_values = config['updater'] - if 'user' in config_values: - args.user = config_values['user'] - if 'password' in config_values: - args.password = config_values['password'] - if 'testrun' in config_values: - args.testrun = config_values.getboolean('testrun') - if 'threads' in config_values: - args.threads = int(config_values['threads']) - if 'interval' in config_values: - args.interval = int(config_values['interval']) - if 'url' in config_values: - args.url = config_values['url'] - if 'loglevel' in config_values: - args.loglevel = config_values['loglevel'] - - if not args.url: - _exit(parser, 'No url or directory given') - - # if url starts with a /, the console based API will be used - isWeb = args.url.startswith('http://') or args.url.startswith('https://') - - # url and user must be specified either from the command line or in the - # config file - if isWeb and not args.user: - _exit(parser, 'Web API requires a user') - - if not isWeb and not os.path.isabs(args.url): - _exit(parser, ('Absolute path to ownCloud installation required, given ' - '%s') % args.url) - - if not isWeb and not os.path.isdir(args.url): - _exit(parser, '%s is not a directory' % args.url) - - # create the updater and run the threads - if isWeb: - updater = WebUpdater(args.url, args.threads, args.interval, - args.testrun, args.user, args.password, - args.timeout, args.loglevel) - else: - updater = ConsoleUpdater(args.url, args.threads, args.interval, - args.testrun, args.loglevel) - updater.run() - - -def _exit(parser, message): - print(message, file=sys.stderr) - parser.print_help() - exit(1) - -if __name__ == '__main__': - if sys.version_info < (3, 0): - print('Python 3.0 or higher is required to run this script') - else: - main() diff --git a/bin/updater/owncloud_news_updater/updater.py b/bin/updater/owncloud_news_updater/updater.py deleted file mode 100644 index 811ac76c2..000000000 --- a/bin/updater/owncloud_news_updater/updater.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -import sys -import traceback -import json -import threading -import requests -import time -import logging -import urllib -from subprocess import check_output - -def check_status_code(response): - if response.status_code != 200: - raise Exception('Request failed with %i: %s' % (response.status_code, - response.text)) - - -class Updater: - - def __init__(self, thread_num, interval, run_once, log_level): - self.thread_num = thread_num - self.run_once = run_once - self.interval = interval - # logging - format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - logging.basicConfig(format=format) - self.logger = logging.getLogger('ownCloud News Updater') - if log_level == 'info': - self.logger.setLevel(logging.INFO) - else: - self.logger.setLevel(logging.ERROR) - - def run(self): - if self.run_once: - self.logger.info('Running update once with %d threads' % - self.thread_num) - else: - self.logger.info(('Running update in an interval of %d seconds ' - 'using %d threads') % (self.interval, - self.thread_num)) - while True: - self.start_time = time.time() # reset clock - try: - self.before_update() - feeds = self.all_feeds() - - threads = [] - for num in range(0, self.thread_num): - thread = self.start_update_thread(feeds) - thread.start() - threads.append(thread) - for thread in threads: - thread.join() - - self.after_update() - - if self.run_once: - return - # wait until the interval finished to run again and subtract - # the update run time from the interval - update_duration_seconds = int((time.time() - self.start_time)) - timeout = self.interval - update_duration_seconds - if timeout > 0: - self.logger.info(('Finished updating in %d seconds, ' - 'next update in %d seconds') % - (update_duration_seconds, timeout)) - time.sleep(timeout) - except (Exception) as e: - self.logger.error('%s: Trying again in 30 seconds' % e) - traceback.print_exc(file=sys.stderr) - time.sleep(30) - - def before_update(self): - raise NotImplementedError - - def start_update_thread(self, feeds): - raise NotImplementedError - - def all_feeds(self): - raise NotImplementedError - - def after_update(self): - raise NotImplementedError - - -class UpdateThread(threading.Thread): - - lock = threading.Lock() - - def __init__(self, feeds, logger): - super().__init__() - self.feeds = feeds - self.logger = logger - - def run(self): - while True: - with WebUpdateThread.lock: - if len(self.feeds) > 0: - feed = self.feeds.pop() - else: - return - try: - self.logger.info('Updating feed with id %s and user %s' % - (feed['id'], feed['userId'])) - self.update_feed(feed) - except (Exception) as e: - self.logger.error(e) - traceback.print_exc(file=sys.stderr) - - def update_feed(self, feed): - raise NotImplementedError - - -class WebUpdater(Updater): - - def __init__(self, base_url, thread_num, interval, run_once, - user, password, timeout, log_level): - super().__init__(thread_num, interval, run_once, log_level) - self.base_url = base_url - self.auth = (user, password) - self.timeout = timeout - - if self.base_url[-1] != '/': - self.base_url += '/' - self.base_url += 'index.php/apps/news/api/v1-2' - - self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url - self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url - self.all_feeds_url = '%s/feeds/all' % self.base_url - self.update_url = '%s/feeds/update' % self.base_url - - def before_update(self): - self.logger.info('Calling before update url: %s' % self.before_cleanup_url) - before = requests.get(self.before_cleanup_url, auth=self.auth) - check_status_code(before) - - def start_update_thread(self, feeds): - return WebUpdateThread(feeds, self.logger, self.update_url, self.auth, - self.timeout) - - def all_feeds(self): - feeds_response = requests.get(self.all_feeds_url, auth=self.auth) - check_status_code(feeds_response) - feeds_json = feeds_response.text - self.logger.info('Received these feeds to update: %s' % feeds_json) - return json.loads(feeds_json)['feeds'] - - def after_update(self): - self.logger.info('Calling after update url: %s' % self.after_cleanup_url) - after = requests.get(self.after_cleanup_url, auth=self.auth) - check_status_code(after) - - -class WebUpdateThread(UpdateThread): - - def __init__(self, feeds, logger, update_url, auth, timeout): - super().__init__(feeds, logger) - self.update_url = update_url - self.auth = auth - self.timeout = timeout - - def update_feed(self, feed): - # rewrite parameters, a feeds id is mapped to feedId - feed['feedId'] = feed['id'] - del feed['id'] - - # turn the pyton dict into url parameters - data = urllib.parse.urlencode(feed) - headers = { - 'Accept': 'text/plain' - } - url = '%s?%s' % (self.update_url, data) - request = requests.get(url, auth=self.auth, timeout=self.timeout) - check_status_code(request) - - -class ConsoleUpdater(Updater): - - def __init__(self, directory, thread_num, interval, run_once, log_level): - super().__init__(thread_num, interval, run_once, log_level) - self.directory = directory.rstrip('/') - base_command = ['php', '-f', self.directory + '/occ'] - self.before_cleanup_command = base_command + ['news:updater:before-update'] - self.all_feeds_command = base_command + ['news:updater:all-feeds'] - self.update_feed_command = base_command + ['news:updater:update-feed'] - self.after_cleanup_command = base_command + ['news:updater:after-update'] - - def before_update(self): - self.logger.info('Running before update command %s' % - ' '.join(self.before_cleanup_command)) - check_output(self.before_cleanup_command) - - def start_update_thread(self, feeds): - return ConsoleUpdateThread(feeds, self.logger, self.update_feed_command) - - def all_feeds(self): - feeds_json = check_output(self.all_feeds_command).strip() - feeds_json = str(feeds_json, 'utf-8') - self.logger.info('Received these feeds to update: %s' % feeds_json) - return json.loads(feeds_json)['feeds'] - - def after_update(self): - self.logger.info('Running after update command %s' % - ' '.join(self.after_cleanup_command)) - check_output(self.before_cleanup_command) - - -class ConsoleUpdateThread(UpdateThread): - - def __init__(self, feeds, logger, update_base_command): - super().__init__(feeds, logger) - self.update_base_command = update_base_command - - def update_feed(self, feed): - command = self.update_base_command + [str(feed['id']), feed['userId']] - self.logger.info('Running update command %s' % ' '.join(command)) - check_output(command) |