From 2a56b3bdec2a20ab6886532663c16edd684f3fc0 Mon Sep 17 00:00:00 2001 From: Bernhard Posselt Date: Thu, 4 Dec 2014 12:20:00 +0100 Subject: gitignore --- .gitignore | 2 +- bin/updater/build/lib/owncloud_news/__init__.py | 0 bin/updater/build/lib/owncloud_news/__main__.py | 3 - bin/updater/build/lib/owncloud_news/application.py | 95 --------------- bin/updater/build/lib/owncloud_news/updater.py | 127 --------------------- 5 files changed, 1 insertion(+), 226 deletions(-) delete mode 100644 bin/updater/build/lib/owncloud_news/__init__.py delete mode 100644 bin/updater/build/lib/owncloud_news/__main__.py delete mode 100644 bin/updater/build/lib/owncloud_news/application.py delete mode 100644 bin/updater/build/lib/owncloud_news/updater.py diff --git a/.gitignore b/.gitignore index a485e6073..9a89d3bb1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ PKG-INFO __pycache__ dist/ owncloud_news_updater.egg-info -updater/build +bin/updater/build # just sane ignores .*.sw[po] diff --git a/bin/updater/build/lib/owncloud_news/__init__.py b/bin/updater/build/lib/owncloud_news/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bin/updater/build/lib/owncloud_news/__main__.py b/bin/updater/build/lib/owncloud_news/__main__.py deleted file mode 100644 index 695c2d19b..000000000 --- a/bin/updater/build/lib/owncloud_news/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from owncloud_news.application import main - -main() \ No newline at end of file diff --git a/bin/updater/build/lib/owncloud_news/application.py b/bin/updater/build/lib/owncloud_news/application.py deleted file mode 100644 index 61eed3146..000000000 --- a/bin/updater/build/lib/owncloud_news/application.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -""" -Updater script for the news app which allows multiple feeds to be updated at -once to speed up the update process. Built in cron has to be disabled in the -news config, see the README.rst file in the top directory for more information. -""" -__author__ = 'Bernhard Posselt' -__copyright__ = 'Copyright 2012-2014, Bernhard Posselt' -__license__ = 'AGPL3+' -__maintainer__ = 'Bernhard Posselt' -__email__ = 'dev@bernhard-posselt.com' - -import sys -import argparse -import configparser - -from owncloud_news.updater import Updater - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--testrun', - help='Run update only once, DO NOT use this in a cron job, only \ - recommended for testing', action='store_true') - parser.add_argument('--threads', '-t', - help='How many feeds should be fetched in parallel, defaults to 10', - default=10, - type=int) - parser.add_argument('--timeout', '-s', - help='Maximum number of seconds for updating a feed, \ - defaults to 5 minutes', - default=5*60, - type=int) - parser.add_argument('--interval', '-i', - help='Update interval between fetching the next round of \ - updates in minutes, defaults to 30 minutes. The update timespan \ - will be subtracted from the interval.', - default=30, - type=int) - parser.add_argument('--config', '-c', - help='Path to config file where all parameters except can be defined \ - as key values pair. An example is in bin/example_config.ini') - parser.add_argument('--user', '-u', - help='Admin username to log into ownCloud. Must be specified on the \ - command line or in the config file.') - parser.add_argument('--password', '-p', - help='Admin password to log into ownCloud') - parser.add_argument('url', - help='The URL where owncloud is installed. Must be specified on the \ - command line or in the config file.', - nargs='?') - args = parser.parse_args() - - # read config file if given - if args.config: - config = configparser.ConfigParser() - files = config.read(args.config) - - if len(files) <= 0: - print('Error: could not find config file %s' % args.config) - exit(1) - - config_values = config['updater'] - if 'user' in config_values: - args.user = config_values['user'] - if 'password' in config_values: - args.password = config_values['password'] - if 'testrun' in config_values: - args.testrun = config_values.getboolean('testrun') - if 'threads' in config_values: - args.threads = int(config_values['threads']) - if 'interval' in config_values: - args.interval = int(config_values['interval']) - if 'url' in config_values: - args.url = config_values['url'] - - # url and user must be specified either from the command line or in the - # config file - if not args.url or not args.user: - parser.print_help() - exit(1) - - # create the updater and run the threads - updater = Updater(args.url, args.threads, args.interval, args.user, - args.password, args.timeout, args.testrun) - updater.run() - - -if __name__ == '__main__': - if sys.version_info < (3, 0): - print('Python 3.0 or higher is required to run this script') - else: - main() - - diff --git a/bin/updater/build/lib/owncloud_news/updater.py b/bin/updater/build/lib/owncloud_news/updater.py deleted file mode 100644 index f59c4f05d..000000000 --- a/bin/updater/build/lib/owncloud_news/updater.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 - -import json -import threading -import requests -import time -import logging -import urllib - -def check_status_code(response): - if response.status_code != 200: - raise Exception('Request failed with %i: %s' % (response.status_code, - response.text)) - - -class Updater: - - def __init__(self, base_url, thread_num, interval, user, password, timeout, - run_once): - self.thread_num = thread_num - self.interval = interval - self.base_url = base_url - self.user = user - self.password = password - self.timeout = timeout - self.run_once = run_once - - if self.base_url[-1] != '/': - self.base_url += '/' - self.base_url += 'index.php/apps/news/api/v1-2' - - self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url - self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url - self.all_feeds_url = '%s/feeds/all' % self.base_url - self.update_url = '%s/feeds/update' % self.base_url - - # logging - format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - logging.basicConfig(format=format) - self.logger = logging.getLogger('ownCloud News Updater') - self.logger.setLevel(logging.INFO) - - - def run(self): - while True: - self.start_time = time.time() # reset clock - - try: - # run the cleanup request and get all the feeds to update - auth = (self.user, self.password) - - before = requests.get(self.before_cleanup_url, auth=auth) - check_status_code(before) - - feeds_response = requests.get(self.all_feeds_url, auth=auth) - check_status_code(feeds_response) - - feeds_json = feeds_response.text - feeds = json.loads(feeds_json)['feeds'] - - # start thread_num threads which update the feeds - threads = [] - for num in range(0, self.thread_num): - thread = UpdateThread(feeds, self.update_url, self.user, - self.password, self.timeout, self.logger) - thread.start() - threads.append(thread) - - for thread in threads: - thread.join() - - after = requests.get(self.after_cleanup_url, auth=auth) - check_status_code(after) - - if self.run_once: - return - - # wait until the interval finished to run again and subtract - # the update run time from the interval - timeout = self.interval - int((time.time() - self.start_time)) - if timeout > 0: - time.sleep(timeout) - - except (Exception) as e: - self.logger.error('%s: %s Trying again in 30 seconds' % - (self.base_url, e)) - time.sleep(30) - - -class UpdateThread(threading.Thread): - - lock = threading.Lock() - - def __init__(self, feeds, update_url, user, password, timeout, logger): - super().__init__() - self.feeds = feeds - self.update_url = update_url - self.user = user - self.password = password - self.timeout = timeout - self.logger = logger - - def run(self): - while True: - with UpdateThread.lock: - if len(self.feeds) > 0: - feed = self.feeds.pop() - else: - return - - feed['feedId'] = feed['id'] - del feed['id'] - - # call the update method of one feed - data = urllib.parse.urlencode(feed) - headers = { - 'Content-type': 'application/json', - 'Accept': 'text/plain' - } - url = '%s?%s' % (self.update_url, data) - - try: - auth = (self.user, self.password) - request = requests.get(url, auth=auth, timeout=self.timeout) - check_status_code(request) - except (Exception) as e: - self.logger.error('%s: %s' % (url, e)) -- cgit v1.2.3