Add VMS version of app_dirname()

Related to #3709

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4008)

1 files changed, 52 insertions, 3 deletions

diff --git a/apps/apps.c b/apps/apps.c
index 3b713f4acc..ef573552eb 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -25,6 +25,12 @@
 #endif
 #include <ctype.h>
 #include <errno.h>
+#ifdef __VMS
+# include <descrip.h>
+# include <iledef.h>
+# include <fscndef.h>
+# include <starlet.h>
+#endif
 #include <openssl/err.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
@@ -2397,14 +2403,57 @@ static char *posix_dirname(char *path)
 }
 
 /*
- * TODO: implement app_dirname for Windows
- * and VMS.
+ * TODO: implement app_dirname for Windows.
  */
-#if !defined(_WIN32) && !defined(__VMS)
+#if !defined(_WIN32)
 char *app_dirname(char *path)
 {
     return posix_dirname(path);
 }
+#elif defined(__VMS)
+/*
+ * sys$filescan fills the given item list with pointers into the original
+ * path string, so all we need to do is to find the file name and simply
+ * put a NUL byte wherever the FSCN$_NAME pointer points.  If there is no
+ * file name part and the path string isn't the empty string, we know for
+ * a fact that the whole string is a directory spec and return it as is.
+ * Otherwise or if that pointer is the starting address of the original
+ * path string, we know to return "sys$disk:[]", which corresponds to the
+ * Unixly ".".
+ *
+ * If sys$filescan returns an error status, we know that this is not
+ * parsable as a VMS file spec, and then use the fallback, in case we
+ * have a Unix type path.
+ */
+char *app_dirname(char *path)
+{
+    char *ret = "sys$disk:[]";
+    struct dsc$descriptor_s dsc_path = { 0 };
+    ile2 itemlist[] = {
+        {0, FSCN$_NAME, 0},
+        {0, 0, 0}
+    };
+    int fields;
+    int status;
+
+    dsc_path.dsc$a_pointer = path;
+    dsc_path.dsc$w_length = strlen(path);
+    status = sys$filescan(&dsc_path, itemlist, &fields, 0, 0);
+
+    if (!(status & 1))
+        return posix_dirname(path);
+
+    if ((fields & (1 << FSCN$_NAME)) == 0) {
+        if (dsc_path.dsc$w_length != 0)
+            ret = path;
+    } else if (itemlist[0].ile2$ps_bufaddr != path) {
+        if (itemlist[0].ile2$ps_bufaddr != path) {
+            *itemlist[0].ile2$ps_bufaddr = '\0';
+            ret = path;
+        }
+    }
+    return ret;
+}
 #endif
 
 /* raw_read|write section */