summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBernhard Posselt <dev@bernhard-posselt.com>2014-12-04 12:20:00 +0100
committerBernhard Posselt <dev@bernhard-posselt.com>2014-12-04 12:20:00 +0100
commit2a56b3bdec2a20ab6886532663c16edd684f3fc0 (patch)
treee42c642ca95568d956b77d8c74b3081305d9b646
parent8136e7a33d6405e76c2ff483a5a4e14a5d4f7e75 (diff)
gitignore
-rw-r--r--.gitignore2
-rw-r--r--bin/updater/build/lib/owncloud_news/__init__.py0
-rw-r--r--bin/updater/build/lib/owncloud_news/__main__.py3
-rw-r--r--bin/updater/build/lib/owncloud_news/application.py95
-rw-r--r--bin/updater/build/lib/owncloud_news/updater.py127
5 files changed, 1 insertions, 226 deletions
diff --git a/.gitignore b/.gitignore
index a485e6073..9a89d3bb1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,7 +14,7 @@ PKG-INFO
__pycache__
dist/
owncloud_news_updater.egg-info
-updater/build
+bin/updater/build
# just sane ignores
.*.sw[po]
diff --git a/bin/updater/build/lib/owncloud_news/__init__.py b/bin/updater/build/lib/owncloud_news/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/bin/updater/build/lib/owncloud_news/__init__.py
+++ /dev/null
diff --git a/bin/updater/build/lib/owncloud_news/__main__.py b/bin/updater/build/lib/owncloud_news/__main__.py
deleted file mode 100644
index 695c2d19b..000000000
--- a/bin/updater/build/lib/owncloud_news/__main__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from owncloud_news.application import main
-
-main() \ No newline at end of file
diff --git a/bin/updater/build/lib/owncloud_news/application.py b/bin/updater/build/lib/owncloud_news/application.py
deleted file mode 100644
index 61eed3146..000000000
--- a/bin/updater/build/lib/owncloud_news/application.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env python3
-"""
-Updater script for the news app which allows multiple feeds to be updated at
-once to speed up the update process. Built in cron has to be disabled in the
-news config, see the README.rst file in the top directory for more information.
-"""
-__author__ = 'Bernhard Posselt'
-__copyright__ = 'Copyright 2012-2014, Bernhard Posselt'
-__license__ = 'AGPL3+'
-__maintainer__ = 'Bernhard Posselt'
-__email__ = 'dev@bernhard-posselt.com'
-
-import sys
-import argparse
-import configparser
-
-from owncloud_news.updater import Updater
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--testrun',
- help='Run update only once, DO NOT use this in a cron job, only \
- recommended for testing', action='store_true')
- parser.add_argument('--threads', '-t',
- help='How many feeds should be fetched in parallel, defaults to 10',
- default=10,
- type=int)
- parser.add_argument('--timeout', '-s',
- help='Maximum number of seconds for updating a feed, \
- defaults to 5 minutes',
- default=5*60,
- type=int)
- parser.add_argument('--interval', '-i',
- help='Update interval between fetching the next round of \
- updates in minutes, defaults to 30 minutes. The update timespan \
- will be subtracted from the interval.',
- default=30,
- type=int)
- parser.add_argument('--config', '-c',
- help='Path to config file where all parameters except can be defined \
- as key values pair. An example is in bin/example_config.ini')
- parser.add_argument('--user', '-u',
- help='Admin username to log into ownCloud. Must be specified on the \
- command line or in the config file.')
- parser.add_argument('--password', '-p',
- help='Admin password to log into ownCloud')
- parser.add_argument('url',
- help='The URL where owncloud is installed. Must be specified on the \
- command line or in the config file.',
- nargs='?')
- args = parser.parse_args()
-
- # read config file if given
- if args.config:
- config = configparser.ConfigParser()
- files = config.read(args.config)
-
- if len(files) <= 0:
- print('Error: could not find config file %s' % args.config)
- exit(1)
-
- config_values = config['updater']
- if 'user' in config_values:
- args.user = config_values['user']
- if 'password' in config_values:
- args.password = config_values['password']
- if 'testrun' in config_values:
- args.testrun = config_values.getboolean('testrun')
- if 'threads' in config_values:
- args.threads = int(config_values['threads'])
- if 'interval' in config_values:
- args.interval = int(config_values['interval'])
- if 'url' in config_values:
- args.url = config_values['url']
-
- # url and user must be specified either from the command line or in the
- # config file
- if not args.url or not args.user:
- parser.print_help()
- exit(1)
-
- # create the updater and run the threads
- updater = Updater(args.url, args.threads, args.interval, args.user,
- args.password, args.timeout, args.testrun)
- updater.run()
-
-
-if __name__ == '__main__':
- if sys.version_info < (3, 0):
- print('Python 3.0 or higher is required to run this script')
- else:
- main()
-
-
diff --git a/bin/updater/build/lib/owncloud_news/updater.py b/bin/updater/build/lib/owncloud_news/updater.py
deleted file mode 100644
index f59c4f05d..000000000
--- a/bin/updater/build/lib/owncloud_news/updater.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import threading
-import requests
-import time
-import logging
-import urllib
-
-def check_status_code(response):
- if response.status_code != 200:
- raise Exception('Request failed with %i: %s' % (response.status_code,
- response.text))
-
-
-class Updater:
-
- def __init__(self, base_url, thread_num, interval, user, password, timeout,
- run_once):
- self.thread_num = thread_num
- self.interval = interval
- self.base_url = base_url
- self.user = user
- self.password = password
- self.timeout = timeout
- self.run_once = run_once
-
- if self.base_url[-1] != '/':
- self.base_url += '/'
- self.base_url += 'index.php/apps/news/api/v1-2'
-
- self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
- self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
- self.all_feeds_url = '%s/feeds/all' % self.base_url
- self.update_url = '%s/feeds/update' % self.base_url
-
- # logging
- format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
- logging.basicConfig(format=format)
- self.logger = logging.getLogger('ownCloud News Updater')
- self.logger.setLevel(logging.INFO)
-
-
- def run(self):
- while True:
- self.start_time = time.time() # reset clock
-
- try:
- # run the cleanup request and get all the feeds to update
- auth = (self.user, self.password)
-
- before = requests.get(self.before_cleanup_url, auth=auth)
- check_status_code(before)
-
- feeds_response = requests.get(self.all_feeds_url, auth=auth)
- check_status_code(feeds_response)
-
- feeds_json = feeds_response.text
- feeds = json.loads(feeds_json)['feeds']
-
- # start thread_num threads which update the feeds
- threads = []
- for num in range(0, self.thread_num):
- thread = UpdateThread(feeds, self.update_url, self.user,
- self.password, self.timeout, self.logger)
- thread.start()
- threads.append(thread)
-
- for thread in threads:
- thread.join()
-
- after = requests.get(self.after_cleanup_url, auth=auth)
- check_status_code(after)
-
- if self.run_once:
- return
-
- # wait until the interval finished to run again and subtract
- # the update run time from the interval
- timeout = self.interval - int((time.time() - self.start_time))
- if timeout > 0:
- time.sleep(timeout)
-
- except (Exception) as e:
- self.logger.error('%s: %s Trying again in 30 seconds' %
- (self.base_url, e))
- time.sleep(30)
-
-
-class UpdateThread(threading.Thread):
-
- lock = threading.Lock()
-
- def __init__(self, feeds, update_url, user, password, timeout, logger):
- super().__init__()
- self.feeds = feeds
- self.update_url = update_url
- self.user = user
- self.password = password
- self.timeout = timeout
- self.logger = logger
-
- def run(self):
- while True:
- with UpdateThread.lock:
- if len(self.feeds) > 0:
- feed = self.feeds.pop()
- else:
- return
-
- feed['feedId'] = feed['id']
- del feed['id']
-
- # call the update method of one feed
- data = urllib.parse.urlencode(feed)
- headers = {
- 'Content-type': 'application/json',
- 'Accept': 'text/plain'
- }
- url = '%s?%s' % (self.update_url, data)
-
- try:
- auth = (self.user, self.password)
- request = requests.get(url, auth=auth, timeout=self.timeout)
- check_status_code(request)
- except (Exception) as e:
- self.logger.error('%s: %s' % (url, e))