linuxPackages.virtualboxGuestAdditions: apply mp-r0drv-linux.c patch

These don't use a the virtualbox sources, but an iso as src, and we need
to add the kernel 5.3 patch aswell.

As for some reason the source files are present on the .iso with Windows
Line endings (sic!), call dos2unix first.

Unfortunately, we can't use the same kernel-5.3-fix.patch as virtualbox
itself, as some files are missing and paths are different.

2 files changed, 59 insertions, 5 deletions

diff --git a/pkgs/applications/virtualization/virtualbox/guest-additions/default.nix b/pkgs/applications/virtualization/virtualbox/guest-additions/default.nix
index 08b6fa63e84c..e20fbf73190e 100644
--- a/pkgs/applications/virtualization/virtualbox/guest-additions/default.nix
+++ b/pkgs/applications/virtualization/virtualbox/guest-additions/default.nix
@@ -1,5 +1,5 @@
 { stdenv, fetchurl, lib, patchelf, cdrkit, kernel, which, makeWrapper
-, zlib, xorg, dbus, virtualbox }:
+, zlib, xorg, dbus, virtualbox, dos2unix }:
 
 let
   version = virtualbox.version;
@@ -32,9 +32,6 @@ in stdenv.mkDerivation {
   KERN_DIR = "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build";
   KERN_INCL = "${kernel.dev}/lib/modules/${kernel.modDirVersion}/source/include";
 
-  # If you add a patch you probably need this.
-  #patchFlags = [ "-p1" "-d" "install/src/vboxguest-${version}" ];
-
   hardeningDisable = [ "pic" ];
 
   NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration";
@@ -42,11 +39,18 @@ in stdenv.mkDerivation {
   nativeBuildInputs = [ patchelf makeWrapper ];
   buildInputs = [ cdrkit ] ++ kernel.moduleBuildDependencies;
 
-  postPatch = ''
+
+  prePatch = ''
     substituteInPlace src/vboxguest-${version}/vboxvideo/vbox_ttm.c \
       --replace "<ttm/" "<drm/ttm/"
+    ${dos2unix}/bin/dos2unix src/vboxguest-${version}/vboxguest/r0drv/linux/mp-r0drv-linux.c
   '';
 
+  patchFlags = [ "-p1" "-d" "src/vboxguest-${version}" ];
+  # Kernel 5.3 fix, should be fixed with VirtualBox 6.0.14
+  # https://www.virtualbox.org/ticket/18911
+  patches = [ ./kernel-5.3-fix.patch ];
+
   unpackPhase = ''
     ${if stdenv.hostPlatform.system == "i686-linux" || stdenv.hostPlatform.system == "x86_64-linux" then ''
         isoinfo -J -i $src -x /VBoxLinuxAdditions.run > ./VBoxLinuxAdditions.run
diff --git a/pkgs/applications/virtualization/virtualbox/guest-additions/kernel-5.3-fix.patch b/pkgs/applications/virtualization/virtualbox/guest-additions/kernel-5.3-fix.patch
new file mode 100644
index 000000000000..0bdd9ec10563
--- /dev/null
+++ b/pkgs/applications/virtualization/virtualbox/guest-additions/kernel-5.3-fix.patch
@@ -0,0 +1,50 @@
+--- a/vboxguest/r0drv/linux/mp-r0drv-linux.c
++++ a/vboxguest/r0drv/linux/mp-r0drv-linux.c
+@@ -283,12 +283,15 @@
+     if (RTCpuSetCount(&OnlineSet) > 1)
+     {
+         /* Fire the function on all other CPUs without waiting for completion. */
+-# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
++# if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
++        smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
++# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
+         int rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
++        Assert(!rc); NOREF(rc);
+ # else
+         int rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
+-# endif
+         Assert(!rc); NOREF(rc);
++# endif
+     }
+ #endif
+
+@@ -326,7 +329,6 @@
+ {
+ #ifdef CONFIG_SMP
+     IPRT_LINUX_SAVE_EFL_AC();
+-    int rc;
+     RTMPARGS Args;
+
+     RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
+@@ -337,14 +339,17 @@
+     Args.cHits = 0;
+
+     RTThreadPreemptDisable(&PreemptState);
+-# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
+-    rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
++# if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
++    smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
++# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
++    int rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
++    Assert(rc == 0); NOREF(rc);
+ # else /* older kernels */
+-    rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
++    int rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
++    Assert(rc == 0); NOREF(rc);
+ # endif /* older kernels */
+     RTThreadPreemptRestore(&PreemptState);
+
+-    Assert(rc == 0); NOREF(rc);
+     IPRT_LINUX_RESTORE_EFL_AC();
+ #else
+     RT_NOREF(pfnWorker, pvUser1, pvUser2);