patch 9.0.1682: sodium encryption is not portablev9.0.1682

Problem: crypt: sodium encryption is not portable
Solution: use little-endian byte order for sodium encrypted files

As mentioned in #12586, sodium encryption only works on little ending
architectures, because reading and writing the sodium encryption
parameters are stored in the encrypted files in an arch-dependent way.

This of course fails for big-endian architectures like s390.

So make sure to use little-endian byte order when reading and writing
sodium encrypted files.

fixes: #12586
closes: 12655

2 files changed, 90 insertions, 15 deletions

diff --git a/src/crypt.c b/src/crypt.c
index 9f0f68a2d4..d5d216b35f 100644
--- a/src/crypt.c
+++ b/src/crypt.c
@@ -77,6 +77,12 @@ typedef struct {
 static int crypt_sodium_init_(cryptstate_T *state, char_u *key, crypt_arg_T *arg);
 static long crypt_sodium_buffer_decode(cryptstate_T *state, char_u *from, size_t len, char_u **buf_out, int last);
 static long crypt_sodium_buffer_encode(cryptstate_T *state, char_u *from, size_t len, char_u **buf_out, int last);
+# if defined(FEAT_SODIUM) || defined(PROTO)
+static void crypt_long_long_to_char(long long n, char_u *s);
+static void crypt_int_to_char(int n, char_u *s);
+static long long crypt_char_to_long_long(char_u *s);
+static int crypt_char_to_int(char_u *s);
+#endif
 #if defined(FEAT_EVAL) && defined(FEAT_SODIUM)
 static void crypt_sodium_report_hash_params(unsigned long long opslimit, unsigned long long ops_def, size_t memlimit, size_t mem_def, int alg, int alg_def);
 #endif
@@ -966,35 +972,45 @@ crypt_sodium_init_(
 	// "cat_add" should not be NULL, check anyway for safety
 	if (state->method_nr == CRYPT_M_SOD2 && arg->cat_add != NULL)
 	{
-	    memcpy(arg->cat_add, &opslimit, sizeof(opslimit));
-	    arg->cat_add += sizeof(opslimit);
+	    char_u	buffer[20];
+	    char_u	*p = buffer;
+	    vim_memset(buffer, 0, 20);
+
+	    crypt_long_long_to_char(opslimit, p);
+	    p += sizeof(opslimit);
 
-	    memcpy(arg->cat_add, &memlimit, sizeof(memlimit));
-	    arg->cat_add += sizeof(memlimit);
+	    crypt_long_long_to_char(memlimit, p);
+	    p += sizeof(memlimit);
 
-	    memcpy(arg->cat_add, &alg, sizeof(alg));
-	    arg->cat_add += sizeof(alg);
+	    crypt_int_to_char(alg, p);
+	    memcpy(arg->cat_add, buffer, sizeof(opslimit) + sizeof(memlimit) + sizeof(alg));
 	}
     }
     else
     {
+	char_u	buffer[20];
+	char_u	*p = buffer;
+	vim_memset(buffer, 0, 20);
+	int	size = sizeof(opslimit) +
+	    sizeof(memlimit) + sizeof(alg);
+
 	// Reading parameters from file
-	if (arg->cat_add_len
-		    < (int)(sizeof(opslimit) + sizeof(memlimit) + sizeof(alg)))
+	if (arg->cat_add_len < size)
 	{
 	    sodium_free(sd_state);
 	    return FAIL;
 	}
 
 	// derive the key from the file header
-	memcpy(&opslimit, arg->cat_add, sizeof(opslimit));
-	arg->cat_add += sizeof(opslimit);
-
-	memcpy(&memlimit, arg->cat_add, sizeof(memlimit));
-	arg->cat_add += sizeof(memlimit);
+	memcpy(p, arg->cat_add, size);
+	arg->cat_add += size;
 
-	memcpy(&alg, arg->cat_add, sizeof(alg));
-	arg->cat_add += sizeof(alg);
+	opslimit = crypt_char_to_long_long(p);
+	p += sizeof(opslimit);
+	memlimit = crypt_char_to_long_long(p);
+	p += sizeof(memlimit);
+	alg = crypt_char_to_int(p);
+	p += sizeof(alg);
 
 #ifdef FEAT_EVAL
 	crypt_sodium_report_hash_params(opslimit,
@@ -1327,6 +1343,63 @@ crypt_sodium_report_hash_params(
     }
 }
 #endif
+
+    static void
+crypt_long_long_to_char(long long n, char_u *s)
+{
+    int i;
+    for (i = 0; i < 8; i++)
+    {
+	s[i] = (char_u)(n & 0xff);
+	n = (unsigned)n >> 8;
+    }
+}
+
+    static void
+crypt_int_to_char(int n, char_u *s)
+{
+    int i;
+    for (i = 0; i < 4; i++)
+    {
+	s[i] = (char_u)(n & 0xff);
+	n = (unsigned)n >> 8;
+    }
+}
+
+    static long long
+crypt_char_to_long_long(char_u *s)
+{
+    unsigned long long    retval = 0;
+    int i;
+    for (i = 7; i >= 0; i--)
+    {
+	if (i == 7)
+	    retval = s[i];
+	else
+	    retval |= s[i];
+	if (i > 0)
+	    retval <<= 8;
+    }
+    return retval;
+}
+
+    static int
+crypt_char_to_int(char_u *s)
+{
+    int retval = 0;
+    int i;
+
+    for (i = 3; i >= 0; i--)
+    {
+	if (i == 3)
+	    retval = s[i];
+	else
+	    retval |= s[i];
+	if (i > 0)
+	    retval <<= 8;
+    }
+    return retval;
+}
 # endif
 
 #endif // FEAT_CRYPT
diff --git a/src/version.c b/src/version.c
index 39f637c185..6903b261d7 100644
--- a/src/version.c
+++ b/src/version.c
@@ -696,6 +696,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1682,
+/**/
     1681,
 /**/
     1680,