add systemd scripts for updater

9 files changed, 370 insertions, 0 deletions

diff --git a/bin/updater/Makefile b/bin/updater/Makefile
new file mode 100644
index 000000000..5e16847e7
--- /dev/null
+++ b/bin/updater/Makefile
@@ -0,0 +1,27 @@
+all: install
+
+install: install-systemd
+
+install-systemd:
+	mkdir -p /etc/owncloud/news
+
+	# install systemd unit file
+	cp $(CURDIR)/example-config.ini /etc/owncloud/news/updater.ini
+	cp $(CURDIR)/systemd/owncloud-news.service /etc/systemd/system/
+	python3 setup.py install
+
+	@echo "Installed files. Please edit your config in /etc/owncloud/news/updater.ini and run:"
+	@echo "    systemctl enable owncloud-news"
+	@echo "    systemctl start owncloud-news"
+	@echo "to run the updater on startup and:"
+	@echo "    systemctl restart owncloud-news"
+	@echo "if you change the config in /etc/owncloud/news/updater.ini"
+
+make uninstall:
+	rm -rf /etc/owncloud/news
+	rm -rf /etc/systemd/systemd/owncloud-news.service
+	python3 setup.py uninstall
+
+	@echo "Uninstalled files. Please run: "
+	@echo "    systemctl disable owncloud-news"
+	@echo "to remove it from boot"
\ No newline at end of file
diff --git a/bin/updater/REAME.rst b/bin/updater/REAME.rst
new file mode 100644
index 000000000..454c7554f
--- /dev/null
+++ b/bin/updater/REAME.rst
@@ -0,0 +1,71 @@
+ownCloud News Updater
+=====================
+
+ownCloud does not require people to install threading or multiprocessing libraries. Because the feed update process is mainly limited by I/O, parallell fetching of RSS feed updates can speed up the updating process a lot. In addition the cronjob can get `into a deadlock <https://github.com/owncloud/core/issues/3221>`_ which will halt cause your feeds not to be updated. This can be soled by using a script that uses the `updater API <https://github.com/owncloud/news/wiki/Cron-1.2>`_
+
+Preinstallation
+---------------
+
+To run the updates via an external threaded script the cron updater has to be disabled. To do that go to the admin section an uncheck the "Use ownCloud cron" checkbox or open **owncloud/data/news/config/config.ini** set:
+
+::
+    useCronUpdates = true
+
+to
+
+::
+    useCronUpdates = false
+
+
+Installation: No init system
+----------------------------
+
+If you decide against using an init system to run the script simply run::
+
+    sudo setup.py install
+
+Then you can run the updater daemon using::
+
+    owncloud-news-updater --user USERNAME --password PASSWORD http://yourcloud.com
+
+or if you are using a config file::
+
+    owncloud-news-updater -c /path/to/config
+
+
+
+Installation: SystemD
+---------------------
+
+To install the script for systemd run::
+
+    sudo make install-systemd
+
+Then edit the config in **/etc/owncloud/news/updater.ini** with your details and run::
+
+    owncloud-news-updater -c /etc/owncloud/news/updater.ini
+
+to test your settings. If everything worked out fine, enable the systemd unit with::
+
+    sudo systemctl enable owncloud-news
+    sudo systemctl start owncloud-news
+
+Finally turn off the cron updates in the owncloud admin interface
+
+
+Self signed certificates
+------------------------
+
+Should you use a self signed certificate over SSL, first consider getting a free valid cert signed by `StartSSL <http://startssl.com>`_. If you don't want to get a valid certificate, you need to add it to the installed certs::
+
+    cat /path/to/your/cert/cacert.pem >> /usr/local/lib/python3.X/dist-packages/requests/cacert.pem
+
+The directories might vary depending on your distribution and Python version.
+
+
+Development
+-----------
+
+If you want to edit the python code and test it run::
+
+    python3 -m owncloud_news -c /path/to/config.ini
\ No newline at end of file
diff --git a/bin/updater/example-config.ini b/bin/updater/example-config.ini
new file mode 100644
index 000000000..67e9299b4
--- /dev/null
+++ b/bin/updater/example-config.ini
@@ -0,0 +1,10 @@
+# all values are optional and can be left out to require the argument being
+# passed in from the command line
+
+[updater]
+user = admin
+password = admin
+threads = 10
+interval = 1000
+testrun = false
+url = http://localhost/owncloud
\ No newline at end of file
diff --git a/bin/updater/owncloud_news/__init__.py b/bin/updater/owncloud_news/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/bin/updater/owncloud_news/__init__.py
diff --git a/bin/updater/owncloud_news/__main__.py b/bin/updater/owncloud_news/__main__.py
new file mode 100644
index 000000000..695c2d19b
--- /dev/null
+++ b/bin/updater/owncloud_news/__main__.py
@@ -0,0 +1,3 @@
+from owncloud_news.application import main
+
+main()
\ No newline at end of file
diff --git a/bin/updater/owncloud_news/application.py b/bin/updater/owncloud_news/application.py
new file mode 100755
index 000000000..1235f00f2
--- /dev/null
+++ b/bin/updater/owncloud_news/application.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""
+Updater script for the news app which allows multiple feeds to be updated at
+once to speed up the update process. Built in cron has to be disabled in the
+news config, see the README.rst file in the top directory for more information.
+"""
+__author__ = 'Bernhard Posselt'
+__copyright__ = 'Copyright 2012-2014, Bernhard Posselt'
+__license__ = 'AGPL3+'
+__maintainer__ = 'Bernhard Posselt'
+__email__ = 'dev@bernhard-posselt.com'
+
+import sys
+import time
+import json
+import argparse
+import threading
+import requests
+import urllib
+import configparser
+
+def check_status_code(response):
+    if response.status_code != 200:
+        raise Exception('Request failed with %i: %s' % (response.status_code,
+            response.text))
+
+class UpdateThread(threading.Thread):
+
+    lock = threading.Lock()
+
+    def __init__(self, feeds, update_url, user, password, timeout):
+        super().__init__()
+        self.feeds = feeds
+        self.update_url = update_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+
+    def run(self):
+        while True:
+            with UpdateThread.lock:
+                if len(self.feeds) > 0:
+                    feed = self.feeds.pop()
+                else:
+                    return
+
+            feed['feedId'] = feed['id']
+            del feed['id']
+
+            # call the update method of one feed
+            data = urllib.parse.urlencode(feed)
+            headers = {
+                'Content-type': 'application/json',
+                'Accept': 'text/plain'
+            }
+            url = '%s?%s' % (self.update_url, data)
+
+            try:
+                auth = (self.user, self.password)
+                request = requests.get(url, auth=auth, timeout=self.timeout)
+                check_status_code(request)
+            except (Exception) as e:
+                print('%s: %s' % (url, e))
+
+
+
+class Updater:
+
+    def __init__(self, base_url, thread_num, interval, user, password, timeout,
+                 run_once):
+        self.thread_num = thread_num
+        self.interval = interval
+        self.base_url = base_url
+        self.user = user
+        self.password = password
+        self.timeout = timeout
+        self.run_once = run_once
+
+        if self.base_url[-1] != '/':
+            self.base_url += '/'
+        self.base_url += 'index.php/apps/news/api/v1-2'
+
+        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
+        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
+        self.all_feeds_url = '%s/feeds/all' % self.base_url
+        self.update_url = '%s/feeds/update' % self.base_url
+
+
+    def run(self):
+        while True:
+            self.start_time = time.time()  # reset clock
+
+            try:
+                # run the cleanup request and get all the feeds to update
+                auth = (self.user, self.password)
+
+                before = requests.get(self.before_cleanup_url, auth=auth)
+                check_status_code(before)
+
+                feeds_response = requests.get(self.all_feeds_url, auth=auth)
+                check_status_code(feeds_response)
+
+                feeds_json = feeds_response.text
+                feeds = json.loads(feeds_json)['feeds']
+
+                # start thread_num threads which update the feeds
+                threads = []
+                for num in range(0, self.thread_num):
+                    thread = UpdateThread(feeds, self.update_url, self.user,
+                        self.password, self.timeout)
+                    thread.start()
+                    threads.append(thread)
+
+                for thread in threads:
+                    thread.join()
+
+                after = requests.get(self.after_cleanup_url, auth=auth)
+                check_status_code(after)
+
+                if self.run_once:
+                    return
+
+                # wait until the interval finished to run again and subtract
+                # the update run time from the interval
+                timeout = self.interval - int((time.time() - self.start_time))
+                if timeout > 0:
+                    time.sleep(timeout)
+
+            except (Exception) as e:
+                print('%s: %s' % (self.base_url, e))
+                print('Trying again in 30 seconds')
+                time.sleep(30)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--testrun',
+        help='Run update only once, DO NOT use this in a cron job, only \
+              recommended for testing', action='store_true')
+    parser.add_argument('--threads', '-t',
+        help='How many feeds should be fetched in parallel, defaults to 10',
+        default=10,
+        type=int)
+    parser.add_argument('--timeout', '-s',
+        help='Maximum number of seconds for updating a feed, \
+              defaults to 5 minutes',
+        default=5*60,
+        type=int)
+    parser.add_argument('--interval', '-i',
+        help='Update interval between fetching the next round of \
+            updates in minutes, defaults to 30 minutes. The update timespan \
+            will be subtracted from the interval.',
+        default=30,
+        type=int)
+    parser.add_argument('--config', '-c',
+        help='Path to config file where all parameters except can be defined \
+        as key values pair. An example is in bin/example_config.ini')
+    parser.add_argument('--user', '-u',
+        help='Admin username to log into ownCloud. Must be specified on the \
+        command line or in the config file.')
+    parser.add_argument('--password', '-p',
+        help='Admin password to log into ownCloud')
+    parser.add_argument('url',
+        help='The URL where owncloud is installed. Must be specified on the \
+        command line or in the config file.',
+        nargs='?')
+    args = parser.parse_args()
+
+    # read config file if given
+    if args.config:
+        config = configparser.ConfigParser()
+        files = config.read(args.config)
+
+        if len(files) <= 0:
+            print('Error: could not find config file %s' % args.config)
+            exit(1)
+
+        config_values = config['updater']
+        if 'user' in config_values:
+            args.user = config_values['user']
+        if 'password' in config_values:
+            args.password = config_values['password']
+        if 'testrun' in config_values:
+            args.testrun = config_values.getboolean('testrun')
+        if 'threads' in config_values:
+            args.threads = int(config_values['threads'])
+        if 'interval' in config_values:
+            args.interval = int(config_values['interval'])
+        if 'url' in config_values:
+            args.url = config_values['url']
+
+    # url and user must be specified either from the command line or in the
+    # config file
+    if not args.url or not args.user:
+        parser.print_help()
+        exit(1)
+
+    # create the updater and run the threads
+    updater = Updater(args.url, args.threads, args.interval, args.user,
+        args.password, args.timeout, args.testrun)
+    updater.run()
+
+
+if __name__ == '__main__':
+    if sys.version_info < (3, 0):
+        print('Python 3.0 or higher is required to run this script')
+    else:
+        main()
+
+
diff --git a/bin/updater/requirements.txt b/bin/updater/requirements.txt
new file mode 100644
index 000000000..f572f1c8d
--- /dev/null
+++ b/bin/updater/requirements.txt
@@ -0,0 +1 @@
+requests==2.2.1
\ No newline at end of file
diff --git a/bin/updater/setup.py b/bin/updater/setup.py
new file mode 100644
index 000000000..8c1173587
--- /dev/null
+++ b/bin/updater/setup.py
@@ -0,0 +1,39 @@
+from os import exit
+from xml.etree import ElementTree
+
+try:
+    from setuptools import setup, find_packages
+except ImportError as e:
+    print('Could not find setuptools. Did you install the package?')
+    exit(1)
+
+with open('requirements.txt', 'r') as infile:
+    install_requires = infile.read().split('\n')
+
+with open('README.rst', 'r') as infile:
+    long_description = infile.read()
+
+# parse version from info.xml
+tree = ElementTree.parse('../../appinfo/info.xml')
+for element in tree.findall('version'):
+    version = element.text
+
+setup (
+    name = 'owncloud-news-updater',
+    version = version,
+    description = 'ownCloud news updater',
+    long_description = long_description,
+    author = 'Bernhard Posselt',
+    author_email = 'dev@bernhard-posselt.com',
+    url = 'https://github.com/owncloud/news',
+    packages = find_packages(),
+    include_package_data = True,
+    license = 'AGPL',
+    install_requires = install_requires,
+    keywords = ['owncloud', 'news', 'updater'],
+    entry_points = {
+        'console_scripts': [
+            'owncloud-news-updater = owncloud_news.application:main'
+        ]
+    }
+)
diff --git a/bin/updater/systemd/owncloud-news.service b/bin/updater/systemd/owncloud-news.service
new file mode 100644
index 000000000..84c9cf132
--- /dev/null
+++ b/bin/updater/systemd/owncloud-news.service
@@ -0,0 +1,9 @@
+[Unit]
+After=default.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /usr/local/bin/owncloud-news-updater -c /etc/owncloud/news/config.ini
+
+[Install]
+WantedBy=default.target
\ No newline at end of file