From 84c2ba538c3cc199720d24c0bf680bf496b46826 Mon Sep 17 00:00:00 2001 From: Benjamin Brahmer Date: Mon, 24 Oct 2022 17:16:08 +0200 Subject: Create tests for the updating and purging functions via the API Signed-off-by: Benjamin Brahmer --- .github/workflows/updater-test.yml | 80 +++++++++++ .gitignore | 3 + .gitmodules | 3 + docs/admin.md | 7 +- tests/test_helper/bats-assert | 2 +- tests/test_helper/php-feed-generator | 1 + tests/updater/helpers/settings.bash | 8 ++ tests/updater/update.bats | 251 +++++++++++++++++++++++++++++++++++ 8 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/updater-test.yml create mode 160000 tests/test_helper/php-feed-generator create mode 100644 tests/updater/helpers/settings.bash create mode 100644 tests/updater/update.bats diff --git a/.github/workflows/updater-test.yml b/.github/workflows/updater-test.yml new file mode 100644 index 000000000..3f862d2aa --- /dev/null +++ b/.github/workflows/updater-test.yml @@ -0,0 +1,80 @@ +name: Updater Tests +on: + pull_request + +env: + POSTGRES_PASSWORD: nc_test_db + MYSQL_USER: nc_test + MYSQL_PASSWORD: nc_test_db + MYSQL_DATABASE: nc_test + MYSQL_PORT: 3800 + +jobs: + integration: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + name: "Update Test: Nextcloud ${{ matrix.nextcloud }} - PHP ${{ matrix.php-versions }}" + strategy: + matrix: + php-versions: ['8.1'] + nextcloud: ['stable26'] + database: ['sqlite'] + experimental: [false] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + extensions: pdo_sqlite,pdo_mysql,pdo_pgsql,gd,zip + coverage: none + + - name: Setup BATS & httpie + run: sudo apt-get install -y httpie && npm install -g bats@1.7.0 + + - name: Set up server + uses: SMillerDev/nextcloud-actions/setup-nextcloud@main + with: + version: ${{ matrix.nextcloud }} + cron: true + database-type: ${{ matrix.database }} + database-host: localhost + database-port: 5432 + database-name: postgres + database-user: postgres + database-password: ${{ env.POSTGRES_PASSWORD }} + + - name: Prime app build + run: make + + - name: Configure server with app + uses: SMillerDev/nextcloud-actions/setup-nextcloud-app@main + with: + app: 'news' + check-code: false + force: ${{ matrix.experimental }} + + - name: Install composer install php-feed-generator + working-directory: ../server + run: composer install -d apps/news/tests/test_helper/php-feed-generator + + - name: Run Updater tests + working-directory: ../server + run: | + php -S localhost:8080 &> /tmp/webserver.log & + cd apps/news/tests/test_helper/feeds && php -S localhost:8090 &> /tmp/feedserver.log & + + sleep 2 + + cd ${{ github.workspace }}/../server + + bats apps/news/tests/updater + + # Kill php server + kill %1 + kill %2 + diff --git a/.gitignore b/.gitignore index 7f9ed3550..c219d71ff 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ site/ #bats tests/api/helpers/settings-override.bash +tests/test_helper/feeds/test.xml +tests/test_helper/feeds/feed1.xml +tests/test_helper/feeds/feed2.xml # python PKG-INFO diff --git a/.gitmodules b/.gitmodules index d74af0778..b3b42f3c2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "tests/test_helper/bats-assert"] path = tests/test_helper/bats-assert url = https://github.com/bats-core/bats-assert.git +[submodule "tests/test_helper/php-feed-generator"] + path = tests/test_helper/php-feed-generator + url = https://github.com/Grotax/php-feed-generator.git diff --git a/docs/admin.md b/docs/admin.md index 0b2cef164..b645219e5 100644 --- a/docs/admin.md +++ b/docs/admin.md @@ -11,8 +11,11 @@ Alternatively you may use an [external updater](https://nextcloud.github.io/news Auto purging automatically removes the oldest read items of every feed after every update. The value you enter here is used as the limit of read items per feed, unless the feed comes with more items in it's feed. -For example you have the default value of 200 and the feed has 210 items in it's feed. -In this case the limit will be 210 instead of 200. +The individual limit per feed is only adjusted when it's bigger. Let's say last feed update came with 210 items, +then that will be the limit for that feed as long as no bigger update with more items is fetched. +In this case the limit will be 210 instead of 200, for that feed. + +This is needed to prevent items from reappearing in the feed. ## Purge unread items This changes the behavior of the auto purging to also purge unread items. This is useful if you have users with a lot of unread items. diff --git a/tests/test_helper/bats-assert b/tests/test_helper/bats-assert index ffe84ea5d..78fa631d1 160000 --- a/tests/test_helper/bats-assert +++ b/tests/test_helper/bats-assert @@ -1 +1 @@ -Subproject commit ffe84ea5dd43b568851549b3e241db150c12929c +Subproject commit 78fa631d1370562d2cd4a1390989e706158e7bf0 diff --git a/tests/test_helper/php-feed-generator b/tests/test_helper/php-feed-generator new file mode 160000 index 000000000..7cc160481 --- /dev/null +++ b/tests/test_helper/php-feed-generator @@ -0,0 +1 @@ +Subproject commit 7cc16048103bf31e9a4b54293e8bbc6049a874c1 diff --git a/tests/updater/helpers/settings.bash b/tests/updater/helpers/settings.bash new file mode 100644 index 000000000..460cdfa17 --- /dev/null +++ b/tests/updater/helpers/settings.bash @@ -0,0 +1,8 @@ +export user=admin +export NC_FEED="http://localhost:8090/Nextcloud.rss" +export HEISE_FEED="http://localhost:8090/heise.xml" +export BASE_URLv1="http://localhost:8080/index.php/apps/news/api/v1-2" +export NC_HOST="http://localhost:8080" +export TEST_FEED="http://localhost:8090/test.xml" +export FEED1="http://localhost:8090/feed1.xml" +export FEED2="http://localhost:8090/feed2.xml" \ No newline at end of file diff --git a/tests/updater/update.bats b/tests/updater/update.bats new file mode 100644 index 000000000..9db1ec4bf --- /dev/null +++ b/tests/updater/update.bats @@ -0,0 +1,251 @@ +#!/usr/bin/env bats + +setup_file(){ + load "helpers/settings" + + if test -f "tests/api/helpers/settings-override.bash"; then + load "helpers/settings-override" + fi + + export APP_PASSWORD=$(NC_PASS=${user} ./occ user:add-app-password ${user} --password-from-env | grep -Po '([A-Z|a-z|0-9]{72})') +} + +teardown_file(){ + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${NC_HOST}/ocs/v2.php/core/apppassword OCS-APIRequest:true +} + +setup() { + load "../test_helper/bats-support/load" + load "../test_helper/bats-assert/load" + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 10 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml +} + +TESTSUITE="Update" + +# +# This testsuite is not intended to test the api but rather the update and purge functions. +# + +teardown() { + # delete all feeds + FEED_IDS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + for i in $FEED_IDS; do + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${BASE_URLv1}/feeds/$i > /dev/null + done + + # delete all folders + FOLDER_IDS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/folders | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + for i in $FOLDER_IDS; do + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${BASE_URLv1}/folders/$i > /dev/null + done +} + +@test "[$TESTSUITE] Test simple update" { + # Create Feed + FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)') + # Get Items + ID_LIST1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + # Trigger Update + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + # Get Items again + ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + assert_equal "${ID_LIST1[*]}" "${ID_LIST2[*]}" +} + +@test "[$TESTSUITE] Test simple update with new content" { + # Create Feed + FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)') + # Get Items + ID_LIST1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 15 -s 9 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + + # Trigger Update + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + # Get Items again + ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + output="${ID_LIST2[*]}" + + # Check that they are not equal but that they match partially. + assert_not_equal "${ID_LIST1[*]}" "${ID_LIST2[*]}" + assert_output --partial "${ID_LIST1[*]}" +} + +@test "[$TESTSUITE] Test purge with small feed" { + # Generate Feed with 210 items. + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 0 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + # Create Feed + FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)') + + # Trigger Update + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 50 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + + # Trigger Update + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 100 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + + # Trigger Update + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 150 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + + # Trigger Update + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 200 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + + # Get Items + ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # get biggest item ID + max=${ID_LIST[0]} + for n in "${ID_LIST[@]}" ; do + ((n > max)) && max=$n + done + + # mark all items of feed as read, returns nothing + STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + + # cleanup, purge items + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update + + # Get unread Items, should be empty + output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')" + + # Get all items, also read items + ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + assert_equal $STATUS_CODE 200 + # check if amount is as expected + assert_equal "${#ID_LIST2[@]}" 200 + + # unread items should be empty + assert_output "" +} + +@test "[$TESTSUITE] Test purge with more items than default limit 200" { + # Generate Feed with 210 items. + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + # Create Feed + FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)') + # Get Items + ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # get biggest item ID + max=${ID_LIST[0]} + for n in "${ID_LIST[@]}" ; do + ((n > max)) && max=$n + done + + # mark all items of feed as read, returns nothing + STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + + # cleanup, purge items + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update + + # Get unread Items, should be empty + output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')" + + # Get all items, also read items + ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + assert_equal $STATUS_CODE 200 + # check if amount is as expected + assert_equal "${#ID_LIST2[@]}" 210 + assert_output "" +} + +@test "[$TESTSUITE] Test Update and pruge with feed item>200; items<200" { + # Generate Feed with 210 items. + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + # Create Feed + FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)') + # Get Items + ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # get biggest item ID + max=${ID_LIST[0]} + for n in "${ID_LIST[@]}" ; do + ((n > max)) && max=$n + done + + # mark all items of feed as read, returns nothing + STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + # cleanup, purge items + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update + + FIRST_UPDATE="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')" + + assert_equal "${FIRST_UPDATE}" "" + + # Generate feed "update" items id 190 + 40 items = id 230 + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 40 -s 190 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID + + # Get Items + ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # get biggest item ID + max=${ID_LIST[0]} + for n in "${ID_LIST[@]}" ; do + ((n > max)) && max=$n + done + + # mark all items of feed as read, returns nothing + STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + + SECOND_UPDATE="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')" + + assert_equal "${SECOND_UPDATE}" "" + + # cleanup, purge items + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update + + # Get all items, also read items + READ_ITEMS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # stays at the 210 limit https://github.com/nextcloud/news/blob/ec74c1b5f3712594c7ea2139c8dfdff15d1ef826/lib/Service/FeedServiceV2.php#L287 + assert_equal "${#READ_ITEMS[@]}" 210 +} + +@test "[$TESTSUITE] Test purge with two feeds with different item count limit" { + # Generate Feed with 260 items. + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 260 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/feed1.xml + # Generate Feed with 210 items. + php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/feed2.xml + + # Create Feeds + FEED1ID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$FEED1 | grep -Po '"id":\K([0-9]+)') + FEED2ID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$FEED2 | grep -Po '"id":\K([0-9]+)') + # Get Items + ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + # get biggest item ID, it is enough to do this one time + max=${ID_LIST[0]} + for n in "${ID_LIST[@]}" ; do + ((n > max)) && max=$n + done + + # mark all items of both feeds as read, returns nothing + STATUS_CODE1=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEED1ID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + STATUS_CODE2=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEED2ID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)') + + # cleanup, purge items + http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update + + # Get unread Items, should be empty + output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')" + + # Get all items of Feed Nr. 1 and 2, including read items + ID_LIST_FEED1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items id="$FEED1ID" type=0 | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + ID_LIST_FEED2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items id="$FEED2ID" type=0 | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')) + + + assert_equal $STATUS_CODE1 200 + assert_equal $STATUS_CODE2 200 + # check if amount is as expected + assert_equal "${#ID_LIST_FEED1[@]}" 260 + assert_equal "${#ID_LIST_FEED2[@]}" 210 + assert_output "" +} \ No newline at end of file -- cgit v1.2.3