From d18eea69350a28cbce748c05b5fe03a465dfbd94 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Sun, 29 Jan 2023 22:59:57 +0000 Subject: paperless-ngx: Build frontend from source --- pkgs/applications/office/paperless-ngx/default.nix | 77 +++++++++++++++++----- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix index c87e54e09154..7a49cf6e960a 100644 --- a/pkgs/applications/office/paperless-ngx/default.nix +++ b/pkgs/applications/office/paperless-ngx/default.nix @@ -1,6 +1,8 @@ { lib -, fetchurl +, fetchFromGitHub +, buildNpmPackage , nixosTests +, gettext , python3 , ghostscript , imagemagickBig @@ -12,10 +14,18 @@ , unpaper , poppler_utils , liberation_ttf -, fetchFromGitHub }: let + version = "1.11.3"; + + src = fetchFromGitHub { + owner = "paperless-ngx"; + repo = "paperless-ngx"; + rev = "refs/tags/v${version}"; + hash = "sha256-d1O8bAXB7NPymibagWUgMDZ9LAZ6QpawmT5D5yw4j+w="; + }; + # Use specific package versions required by paperless-ngx python = python3.override { packageOverrides = self: super: { @@ -78,20 +88,47 @@ let unpaper poppler_utils ]; + + frontend = buildNpmPackage { + pname = "paperless-ngx-frontend"; + inherit version src; + + npmDepsHash = "sha256-ZjIWU8i2Riz4KmC1Woi0uaJ+uNyuU27XwFqvo4bCC5k="; + + nativeBuildInputs = [ + python3 + ]; + + postPatch = '' + cd src-ui + ''; + + CYPRESS_INSTALL_BINARY = "0"; + NG_CLI_ANALYTICS = "false"; + + npmBuildFlags = [ + "--" "--configuration" "production" + ]; + + installPhase = '' + runHook preInstall + mkdir -p $out/lib/paperless-ui + mv ../src/documents/static/frontend $out/lib/paperless-ui/ + runHook postInstall + ''; + }; in -python.pkgs.pythonPackages.buildPythonApplication rec { +python.pkgs.buildPythonApplication rec { pname = "paperless-ngx"; - version = "1.11.3"; + format = "other"; - # Fetch the release tarball instead of a git ref because it contains the prebuilt frontend - src = fetchurl { - url = "https://github.com/paperless-ngx/paperless-ngx/releases/download/v${version}/${pname}-v${version}.tar.xz"; - hash = "sha256-wGNkdczgV+UDd9ZO+BXMSWotpetE/+c/jJAAH+6SXps="; - }; + inherit version src; - format = "other"; + nativeBuildInputs = [ + gettext + ]; - propagatedBuildInputs = with python.pkgs.pythonPackages; [ + propagatedBuildInputs = with python.pkgs; [ aioredis arrow asgiref @@ -192,15 +229,23 @@ python.pkgs.pythonPackages.buildPythonApplication rec { zope_interface ]; - # Compile manually because `pythonRecompileBytecodeHook` only works for - # files in `python.sitePackages` + postBuild = '' + # Compile manually because `pythonRecompileBytecodeHook` only works + # for files in `python.sitePackages` ${python.interpreter} -OO -m compileall src + + # Collect static files + ${python.interpreter} src/manage.py collectstatic --clear --no-input + + # Compile string translations using gettext + ${python.interpreter} src/manage.py compilemessages ''; installPhase = '' - mkdir -p $out/lib - cp -r . $out/lib/paperless-ngx + mkdir -p $out/lib/paperless-ngx + cp -r {src,static,LICENSE,gunicorn.conf.py} $out/lib/paperless-ngx + ln -s ${frontend}/lib/paperless-ui/frontend $out/lib/paperless-ngx/static/ chmod +x $out/lib/paperless-ngx/src/manage.py makeWrapper $out/lib/paperless-ngx/src/manage.py $out/bin/paperless-ngx \ --prefix PYTHONPATH : "$PYTHONPATH" \ @@ -250,7 +295,7 @@ python.pkgs.pythonPackages.buildPythonApplication rec { ]; passthru = { - inherit python path; + inherit python path frontend; tests = { inherit (nixosTests) paperless; }; }; -- cgit v1.2.3 From 7018ef588677057eae83b611fd63bc6ebcd4b3ae Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 30 Jan 2023 02:35:34 +0100 Subject: python3Packages.redis: fix hiredis extra name --- pkgs/development/python-modules/redis/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/redis/default.nix b/pkgs/development/python-modules/redis/default.nix index 649a1192afd0..d16cff4f9280 100644 --- a/pkgs/development/python-modules/redis/default.nix +++ b/pkgs/development/python-modules/redis/default.nix @@ -41,7 +41,7 @@ buildPythonPackage rec { ]; passthru.optional-dependencies = { - hidredis = [ + hiredis = [ hiredis ]; ocsp = [ -- cgit v1.2.3 From 1b3a601be2c06913a48d65992df61da8ef370ffc Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 30 Jan 2023 02:37:00 +0100 Subject: paperless-ngx: 1.11.3 -> 1.12.2 https://github.com/paperless-ngx/paperless-ngx/releases/tag/v1.12.0 https://github.com/paperless-ngx/paperless-ngx/releases/tag/v1.12.1 https://github.com/paperless-ngx/paperless-ngx/releases/tag/v1.12.2 --- pkgs/applications/office/paperless-ngx/default.nix | 50 ++++++++++++++-------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix index 7a49cf6e960a..28314d48fe51 100644 --- a/pkgs/applications/office/paperless-ngx/default.nix +++ b/pkgs/applications/office/paperless-ngx/default.nix @@ -17,13 +17,13 @@ }: let - version = "1.11.3"; + version = "1.12.2"; src = fetchFromGitHub { owner = "paperless-ngx"; repo = "paperless-ngx"; rev = "refs/tags/v${version}"; - hash = "sha256-d1O8bAXB7NPymibagWUgMDZ9LAZ6QpawmT5D5yw4j+w="; + hash = "sha256-1QufnRD2Nbc4twRZ4Yrf3ae1BRGves8tJ/M7coWnRPI="; }; # Use specific package versions required by paperless-ngx @@ -93,7 +93,7 @@ let pname = "paperless-ngx-frontend"; inherit version src; - npmDepsHash = "sha256-ZjIWU8i2Riz4KmC1Woi0uaJ+uNyuU27XwFqvo4bCC5k="; + npmDepsHash = "sha256-fp0Gy3018u2y6jaUM9bmXU0SVjyEJdsvkBqbmb8S10Y="; nativeBuildInputs = [ python3 @@ -130,21 +130,25 @@ python.pkgs.buildPythonApplication rec { propagatedBuildInputs = with python.pkgs; [ aioredis - arrow + amqp + anyio asgiref async-timeout attrs autobahn automat + billiard bleach - blessed celery certifi cffi channels-redis channels - chardet + charset-normalizer click + click-didyoumean + click-plugins + click-repl coloredlogs concurrent-log-handler constantly @@ -155,18 +159,16 @@ python.pkgs.buildPythonApplication rec { django-cors-headers django-extensions django-filter - django-picklefield django djangorestframework filelock - fuzzywuzzy gunicorn h11 hiredis httptools humanfriendly + humanize hyperlink - imagehash idna imap-tools img2pdf @@ -177,9 +179,11 @@ python.pkgs.buildPythonApplication rec { langdetect lxml msgpack + mysqlclient nltk numpy ocrmypdf + packaging pathvalidate pdf2image pdfminer-six @@ -187,6 +191,7 @@ python.pkgs.buildPythonApplication rec { pillow pluggy portalocker + prompt-toolkit psycopg2 pyasn1-modules pyasn1 @@ -195,7 +200,6 @@ python.pkgs.buildPythonApplication rec { python-dateutil python-dotenv python-gnupg - levenshtein python-magic pytz pyyaml @@ -208,27 +212,34 @@ python.pkgs.buildPythonApplication rec { scikit-learn scipy service-identity - six - sortedcontainers + setproctitle + sniffio sqlparse threadpoolctl tika + tornado tqdm - twisted.optional-dependencies.tls + twisted txaio + tzdata tzlocal urllib3 uvicorn uvloop + vine watchdog - watchgod + watchfiles wcwidth + webencodings websockets whitenoise whoosh + zipp zope_interface - ]; - + ] + ++ redis.optional-dependencies.hiredis + ++ twisted.optional-dependencies.tls + ++ uvicorn.optional-dependencies.standard; postBuild = '' # Compile manually because `pythonRecompileBytecodeHook` only works @@ -255,12 +266,12 @@ python.pkgs.buildPythonApplication rec { --prefix PATH : "${path}" ''; - nativeCheckInputs = with python.pkgs.pythonPackages; [ + nativeCheckInputs = with python.pkgs; [ + factory_boy + imagehash pytest-django pytest-env - pytest-sugar pytest-xdist - factory_boy pytestCheckHook ]; @@ -302,6 +313,7 @@ python.pkgs.buildPythonApplication rec { meta = with lib; { description = "Tool to scan, index, and archive all of your physical documents"; homepage = "https://paperless-ngx.readthedocs.io/"; + changelog = "https://github.com/paperless-ngx/paperless-ngx/releases/tag/v${version}"; license = licenses.gpl3Only; maintainers = with maintainers; [ lukegb gador erikarvstedt ]; }; -- cgit v1.2.3 From 4fc0f24120eec931c5c7ea95dcfad327563c048f Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Mon, 30 Jan 2023 18:52:21 +0100 Subject: paperless-ngx: Reduce output size By dropping the tests and its assets we further reduce the output size from 23M to just 8.4M. --- pkgs/applications/office/paperless-ngx/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix index 28314d48fe51..61c607c422e8 100644 --- a/pkgs/applications/office/paperless-ngx/default.nix +++ b/pkgs/applications/office/paperless-ngx/default.nix @@ -266,6 +266,11 @@ python.pkgs.buildPythonApplication rec { --prefix PATH : "${path}" ''; + postFixup = '' + # Remove tests with samples (~14M) + find $out/lib/paperless-ngx -type d -name tests -exec rm -rv {} + + ''; + nativeCheckInputs = with python.pkgs; [ factory_boy imagehash -- cgit v1.2.3 From a7d112b30a36fd4f0500fed41c62249461624c98 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 30 Jan 2023 20:20:16 +0100 Subject: nixos/paperless: Reindex documents after updating to 1.12.x This causes a reindex of all documents to allow for comments made before 1.12.x to be searchable. Also change the format of the version file to just include the version, not the whole store path. This simplifies version comparisons and causes migrations to run only when the version has changed. Co-Authored-By: Martin Weinelt --- nixos/modules/services/misc/paperless.nix | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/paperless.nix b/nixos/modules/services/misc/paperless.nix index 1dddd147ac09..667f16d98f82 100644 --- a/nixos/modules/services/misc/paperless.nix +++ b/nixos/modules/services/misc/paperless.nix @@ -226,9 +226,26 @@ in # Auto-migrate on first run or if the package has changed versionFile="${cfg.dataDir}/src-version" - if [[ $(cat "$versionFile" 2>/dev/null) != ${pkg} ]]; then + version=$(cat "$versionFile" 2>/dev/null || echo 0) + + if [[ $version != ${pkg.version} ]]; then ${pkg}/bin/paperless-ngx migrate - echo ${pkg} > "$versionFile" + + # Parse old version string format for backwards compatibility + version=$(echo "$version" | grep -ohP '[^-]+$') + + versionLessThan() { + target=$1 + [[ $({ echo "$version"; echo "$target"; } | sort -V | head -1) != "$target" ]] + } + + if versionLessThan 1.12.0; then + # Reindex documents as mentioned in https://github.com/paperless-ngx/paperless-ngx/releases/tag/v1.12.1 + echo "Reindexing documents, to allow searching old comments. Required after the 1.12.x upgrade." + ${pkg}/bin/paperless-ngx document_index reindex + fi + + echo ${pkg.version} > "$versionFile" fi '' + optionalString (cfg.passwordFile != null) '' -- cgit v1.2.3