Merge pull request #213489 from mweinelt/paperless-update

paperless-ngx: 1.11.3 -> 1.12.2

3 files changed, 113 insertions, 34 deletions

diff --git a/nixos/modules/services/misc/paperless.nix b/nixos/modules/services/misc/paperless.nix
index 1dddd147ac09..667f16d98f82 100644
--- a/nixos/modules/services/misc/paperless.nix
+++ b/nixos/modules/services/misc/paperless.nix
@@ -226,9 +226,26 @@ in
 
         # Auto-migrate on first run or if the package has changed
         versionFile="${cfg.dataDir}/src-version"
-        if [[ $(cat "$versionFile" 2>/dev/null) != ${pkg} ]]; then
+        version=$(cat "$versionFile" 2>/dev/null || echo 0)
+
+        if [[ $version != ${pkg.version} ]]; then
           ${pkg}/bin/paperless-ngx migrate
-          echo ${pkg} > "$versionFile"
+
+          # Parse old version string format for backwards compatibility
+          version=$(echo "$version" | grep -ohP '[^-]+$')
+
+          versionLessThan() {
+            target=$1
+            [[ $({ echo "$version"; echo "$target"; } | sort -V | head -1) != "$target" ]]
+          }
+
+          if versionLessThan 1.12.0; then
+            # Reindex documents as mentioned in https://github.com/paperless-ngx/paperless-ngx/releases/tag/v1.12.1
+            echo "Reindexing documents, to allow searching old comments. Required after the 1.12.x upgrade."
+            ${pkg}/bin/paperless-ngx document_index reindex
+          fi
+
+          echo ${pkg.version} > "$versionFile"
         fi
       ''
       + optionalString (cfg.passwordFile != null) ''
diff --git a/pkgs/applications/office/paperless-ngx/default.nix b/pkgs/applications/office/paperless-ngx/default.nix
index c87e54e09154..61c607c422e8 100644
--- a/pkgs/applications/office/paperless-ngx/default.nix
+++ b/pkgs/applications/office/paperless-ngx/default.nix
@@ -1,6 +1,8 @@
 { lib
-, fetchurl
+, fetchFromGitHub
+, buildNpmPackage
 , nixosTests
+, gettext
 , python3
 , ghostscript
 , imagemagickBig
@@ -12,10 +14,18 @@
 , unpaper
 , poppler_utils
 , liberation_ttf
-, fetchFromGitHub
 }:
 
 let
+  version = "1.12.2";
+
+  src = fetchFromGitHub {
+    owner = "paperless-ngx";
+    repo = "paperless-ngx";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-1QufnRD2Nbc4twRZ4Yrf3ae1BRGves8tJ/M7coWnRPI=";
+  };
+
   # Use specific package versions required by paperless-ngx
   python = python3.override {
     packageOverrides = self: super: {
@@ -78,36 +88,67 @@ let
     unpaper
     poppler_utils
   ];
+
+  frontend = buildNpmPackage {
+    pname = "paperless-ngx-frontend";
+    inherit version src;
+
+    npmDepsHash = "sha256-fp0Gy3018u2y6jaUM9bmXU0SVjyEJdsvkBqbmb8S10Y=";
+
+    nativeBuildInputs = [
+      python3
+    ];
+
+    postPatch = ''
+      cd src-ui
+    '';
+
+    CYPRESS_INSTALL_BINARY = "0";
+    NG_CLI_ANALYTICS = "false";
+
+    npmBuildFlags = [
+      "--" "--configuration" "production"
+    ];
+
+    installPhase = ''
+      runHook preInstall
+      mkdir -p $out/lib/paperless-ui
+      mv ../src/documents/static/frontend $out/lib/paperless-ui/
+      runHook postInstall
+    '';
+  };
 in
-python.pkgs.pythonPackages.buildPythonApplication rec {
+python.pkgs.buildPythonApplication rec {
   pname = "paperless-ngx";
-  version = "1.11.3";
+  format = "other";
 
-  # Fetch the release tarball instead of a git ref because it contains the prebuilt frontend
-  src = fetchurl {
-    url = "https://github.com/paperless-ngx/paperless-ngx/releases/download/v${version}/${pname}-v${version}.tar.xz";
-    hash = "sha256-wGNkdczgV+UDd9ZO+BXMSWotpetE/+c/jJAAH+6SXps=";
-  };
+  inherit version src;
 
-  format = "other";
+  nativeBuildInputs = [
+    gettext
+  ];
 
-  propagatedBuildInputs = with python.pkgs.pythonPackages; [
+  propagatedBuildInputs = with python.pkgs; [
     aioredis
-    arrow
+    amqp
+    anyio
     asgiref
     async-timeout
     attrs
     autobahn
     automat
+    billiard
     bleach
-    blessed
     celery
     certifi
     cffi
     channels-redis
     channels
-    chardet
+    charset-normalizer
     click
+    click-didyoumean
+    click-plugins
+    click-repl
     coloredlogs
     concurrent-log-handler
     constantly
@@ -118,18 +159,16 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
     django-cors-headers
     django-extensions
     django-filter
-    django-picklefield
     django
     djangorestframework
     filelock
-    fuzzywuzzy
     gunicorn
     h11
     hiredis
     httptools
     humanfriendly
+    humanize
     hyperlink
-    imagehash
     idna
     imap-tools
     img2pdf
@@ -140,9 +179,11 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
     langdetect
     lxml
     msgpack
+    mysqlclient
     nltk
     numpy
     ocrmypdf
+    packaging
     pathvalidate
     pdf2image
     pdfminer-six
@@ -150,6 +191,7 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
     pillow
     pluggy
     portalocker
+    prompt-toolkit
     psycopg2
     pyasn1-modules
     pyasn1
@@ -158,7 +200,6 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
     python-dateutil
     python-dotenv
     python-gnupg
-    levenshtein
     python-magic
     pytz
     pyyaml
@@ -171,36 +212,51 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
     scikit-learn
     scipy
     service-identity
-    six
-    sortedcontainers
+    setproctitle
+    sniffio
     sqlparse
     threadpoolctl
     tika
+    tornado
     tqdm
-    twisted.optional-dependencies.tls
+    twisted
     txaio
+    tzdata
     tzlocal
     urllib3
     uvicorn
     uvloop
+    vine
     watchdog
-    watchgod
+    watchfiles
     wcwidth
+    webencodings
     websockets
     whitenoise
     whoosh
+    zipp
     zope_interface
-  ];
+  ]
+  ++ redis.optional-dependencies.hiredis
+  ++ twisted.optional-dependencies.tls
+  ++ uvicorn.optional-dependencies.standard;
 
-  # Compile manually because `pythonRecompileBytecodeHook` only works for
-  # files in `python.sitePackages`
   postBuild = ''
+    # Compile manually because `pythonRecompileBytecodeHook` only works
+    # for files in `python.sitePackages`
     ${python.interpreter} -OO -m compileall src
+
+    # Collect static files
+    ${python.interpreter} src/manage.py collectstatic --clear --no-input
+
+    # Compile string translations using gettext
+    ${python.interpreter} src/manage.py compilemessages
   '';
 
   installPhase = ''
-    mkdir -p $out/lib
-    cp -r . $out/lib/paperless-ngx
+    mkdir -p $out/lib/paperless-ngx
+    cp -r {src,static,LICENSE,gunicorn.conf.py} $out/lib/paperless-ngx
+    ln -s ${frontend}/lib/paperless-ui/frontend $out/lib/paperless-ngx/static/
     chmod +x $out/lib/paperless-ngx/src/manage.py
     makeWrapper $out/lib/paperless-ngx/src/manage.py $out/bin/paperless-ngx \
       --prefix PYTHONPATH : "$PYTHONPATH" \
@@ -210,12 +266,17 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
       --prefix PATH : "${path}"
   '';
 
-  nativeCheckInputs = with python.pkgs.pythonPackages; [
+  postFixup = ''
+    # Remove tests with samples (~14M)
+    find $out/lib/paperless-ngx -type d -name tests -exec rm -rv {} +
+  '';
+
+  nativeCheckInputs = with python.pkgs; [
+    factory_boy
+    imagehash
     pytest-django
     pytest-env
-    pytest-sugar
     pytest-xdist
-    factory_boy
     pytestCheckHook
   ];
 
@@ -250,13 +311,14 @@ python.pkgs.pythonPackages.buildPythonApplication rec {
   ];
 
   passthru = {
-    inherit python path;
+    inherit python path frontend;
     tests = { inherit (nixosTests) paperless; };
   };
 
   meta = with lib; {
     description = "Tool to scan, index, and archive all of your physical documents";
     homepage = "https://paperless-ngx.readthedocs.io/";
+    changelog = "https://github.com/paperless-ngx/paperless-ngx/releases/tag/v${version}";
     license = licenses.gpl3Only;
     maintainers = with maintainers; [ lukegb gador erikarvstedt ];
   };
diff --git a/pkgs/development/python-modules/redis/default.nix b/pkgs/development/python-modules/redis/default.nix
index 649a1192afd0..d16cff4f9280 100644
--- a/pkgs/development/python-modules/redis/default.nix
+++ b/pkgs/development/python-modules/redis/default.nix
@@ -41,7 +41,7 @@ buildPythonPackage rec {
   ];
 
   passthru.optional-dependencies = {
-    hidredis = [
+    hiredis = [
       hiredis
     ];
     ocsp = [