randfile.c: .rnd can become orphaned on VMS.

Submitted by: David North

1 files changed, 43 insertions, 23 deletions

diff --git a/crypto/rand/randfile.c b/crypto/rand/randfile.c
index 0c7f9e6dfb..c4e6160779 100644
--- a/crypto/rand/randfile.c
+++ b/crypto/rand/randfile.c
@@ -83,6 +83,14 @@
 #define BUFSIZE	1024
 #define RAND_DATA 1024
 
+#ifdef OPENSSL_SYS_VMS
+/* This declaration is a nasty hack to get around vms' extension to fopen
+ * for passing in sharing options being disabled by our /STANDARD=ANSI89 */
+static FILE *(*const vms_fopen)(const char *, const char *, ...) =
+    (FILE *(*)(const char *, const char *, ...))fopen;
+#define VMS_OPEN_ATTRS "shr=get,put,upd,del","ctx=bin,stm","rfm=stm","rat=none","mrs=0"
+#endif
+
 /* #define RFILE ".rnd" - defined in ../../e_os.h */
 
 /* Note that these functions are intended for seed files only.
@@ -108,7 +116,11 @@ int RAND_load_file(const char *file, long bytes)
 #endif
 	if (bytes == 0) return(ret);
 
+#ifdef OPENSSL_SYS_VMS
+	in=vms_fopen(file,"rb",VMS_OPEN_ATTRS);
+#else
 	in=fopen(file,"rb");
+#endif
 	if (in == NULL) goto err;
 #if defined(S_IFBLK) && defined(S_IFCHR) && !defined(OPNESSL_NO_POSIX_IO)
 	if (sb.st_mode & (S_IFBLK | S_IFCHR)) {
@@ -171,19 +183,46 @@ int RAND_write_file(const char *file)
 	}
 #endif
 
-#if defined(O_CREAT) && !defined(OPENSSL_SYS_WIN32) && !defined(OPENSSL_NO_POSIX_IO)
+#if defined(O_CREAT) && !defined(OPENSSL_NO_POSIX_IO) && !defined(OPENSSL_SYS_VMS)
 	{
-	/* For some reason Win32 can't write to files created this way */
-	
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
 	/* chmod(..., 0600) is too late to protect the file,
 	 * permissions should be restrictive from the start */
-	int fd = open(file, O_CREAT, 0600);
+	int fd = open(file, O_WRONLY|O_CREAT|O_BINARY, 0600);
 	if (fd != -1)
 		out = fdopen(fd, "wb");
 	}
 #endif
+
+#ifdef OPENSSL_SYS_VMS
+	/* VMS NOTE: Prior versions of this routine created a _new_
+	 * version of the rand file for each call into this routine, then
+	 * deleted all existing versions named ;-1, and finally renamed
+	 * the current version as ';1'. Under concurrent usage, this
+	 * resulted in an RMS race condition in rename() which could
+	 * orphan files (see vms message help for RMS$_REENT). With the
+	 * fopen() calls below, openssl/VMS now shares the top-level
+	 * version of the rand file. Note that there may still be
+	 * conditions where the top-level rand file is locked. If so, this
+	 * code will then create a new version of the rand file. Without
+	 * the delete and rename code, this can result in ascending file
+	 * versions that stop at version 32767, and this routine will then
+	 * return an error. The remedy for this is to recode the calling
+	 * application to avoid concurrent use of the rand file, or
+	 * synchronize usage at the application level. Also consider
+	 * whether or not you NEED a persistent rand file in a concurrent
+	 * use situation. 
+	 */
+
+	out = vms_fopen(file,"rb+",VMS_OPEN_ATTRS);
+	if (out == NULL)
+		out = vms_fopen(file,"wb",VMS_OPEN_ATTRS);
+#else
 	if (out == NULL)
 		out = fopen(file,"wb");
+#endif
 	if (out == NULL) goto err;
 
 #ifndef NO_CHMOD
@@ -205,25 +244,6 @@ int RAND_write_file(const char *file)
 		ret+=i;
 		if (n <= 0) break;
                 }
-#ifdef OPENSSL_SYS_VMS
-	/* Try to delete older versions of the file, until there aren't
-	   any */
-	{
-	char *tmpf;
-
-	tmpf = OPENSSL_malloc(strlen(file) + 4);  /* to add ";-1" and a nul */
-	if (tmpf)
-		{
-		strcpy(tmpf, file);
-		strcat(tmpf, ";-1");
-		while(delete(tmpf) == 0)
-			;
-		rename(file,";1"); /* Make sure it's version 1, or we
-				      will reach the limit (32767) at
-				      some point... */
-		}
-	}
-#endif /* OPENSSL_SYS_VMS */
 
 	fclose(out);
 	OPENSSL_cleanse(buf,BUFSIZE);