VMS: don't use /DSF, turn off CALL_DEBUG instead

It turns out that /DSF didn't do any good for our purposes.  Instead,
remove the CALL_DEBUG flag from any image we link.  This ensures that
we can have debugging information in the image files, but don't
automatically end up in a debugging session upon image activation.

Unfortunately, this means the CALL_DEBUG must be turned on when there
is a need to run with the debugger activated, and to turn it off when
done.  This has been documented in NOTES.VMS.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2957)

5 files changed, 33 insertions, 12 deletions

diff --git a/Configurations/10-main.conf b/Configurations/10-main.conf
index c3d2e94792..09c0f2d0eb 100644
--- a/Configurations/10-main.conf
+++ b/Configurations/10-main.conf
@@ -1786,7 +1786,7 @@ sub vms_info {
                                    release => "/OPTIMIZE/NODEBUG"),
         defines          => add("OPENSSL_USE_NODELETE"),
         lflags           => picker(default => "/MAP='F\$PARSE(\".MAP\",\"\$\@\")'",
-                                   debug   => "/DSF='F\$PARSE(\"\$(BLDDIR).DSF\",\"\$\@\")'/TRACEBACK",
+                                   debug   => "/DEBUG/TRACEBACK",
                                    release => "/NODEBUG/NOTRACEBACK"),
         lib_cflags       => add("/NAMES=(AS_IS,SHORTENED)/EXTERN_MODEL=STRICT_REFDEF"),
         # no_inst_bin_cflags is used instead of bin_cflags by descrip.mms.tmpl
diff --git a/Configurations/descrip.mms.tmpl b/Configurations/descrip.mms.tmpl
index a57188d86f..166c6c6a40 100644
--- a/Configurations/descrip.mms.tmpl
+++ b/Configurations/descrip.mms.tmpl
@@ -314,11 +314,11 @@ uninstall : uninstall_docs uninstall_sw
 # use $(LIBS), $(PROGRAMS), $(GENERATED) and $(ENGINES)directly.
 libclean :
         {- join("\n\t", map { "- DELETE $_.OLB;*" } @{$unified_info{libraries}}) || "@ !" -}
-        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.DSF;*,$_.MAP;*,$_.OPT;*" } @shlibs) || "@ !" -}
+        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.MAP;*,$_.OPT;*" } @shlibs) || "@ !" -}
 
 clean : libclean
-        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.DSF;*,$_.OPT;*" } @{$unified_info{programs}}) || "@ !" -}
-        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.DSF;*,$_.OPT;*" } @{$unified_info{engines}}) || "@ !" -}
+        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.OPT;*" } @{$unified_info{programs}}) || "@ !" -}
+        {- join("\n\t", map { "- DELETE $_.EXE;*,$_.OPT;*" } @{$unified_info{engines}}) || "@ !" -}
         {- join("\n\t", map { "- DELETE $_;*" } @{$unified_info{scripts}}) || "@ !" -}
         {- join("\n\t", map { "- DELETE $_;*" } @generated) || "@ !" -}
         - DELETE [...]*.MAP;*
@@ -673,6 +673,7 @@ $shlib.EXE : $lib.OLB $deps $ordinalsfile
         $write_opt
         CLOSE OPT_FILE
         LINK \$(LDFLAGS)/SHARE=\$\@ $shlib.OPT/OPT \$(EX_LIBS)
+        SET IMAGE/FLAGS=(NOCALL_DEBUG) \$\@
         DELETE $shlib.SYMVEC;*
         PURGE $shlib.EXE,$shlib.OPT,$shlib.MAP
 EOF
@@ -712,6 +713,7 @@ $lib.EXE : $deps
         $write_opt2
         CLOSE OPT_FILE
         LINK \$(LDFLAGS)/SHARE=\$\@ $lib.OPT/OPT \$(EX_LIBS)
+        SET IMAGE/FLAGS=(NOCALL_DEBUG) \$\@
         - PURGE $lib.EXE,$lib.OPT,$lib.MAP
 EOF
   }
@@ -793,6 +795,7 @@ $bin.EXE : $deps
         -@ IF .NOT. link_severity THEN TYPE $bin.LINKLOG
         -@ DELETE $bin.LINKLOG;*
         @ IF .NOT. link_severity THEN SPAWN/WAIT/NOLOG EXIT 'link_status'
+        SET IMAGE/FLAGS=(NOCALL_DEBUG) $bin.EXE
         - PURGE $bin.EXE,$bin.OPT
 EOF
   }
diff --git a/NOTES.VMS b/NOTES.VMS
index 3e9a57e805..48743720f6 100644
--- a/NOTES.VMS
+++ b/NOTES.VMS
@@ -56,6 +56,26 @@
  to use.
 
 
+ About debugging
+ ---------------
+
+ If you build for debugging, the default on VMS is that image
+ activation starts the debugger automatically, giving you a debug
+ prompt.  Unfortunately, this disrupts all other uses, such as running
+ test programs in the test framework.
+
+ As a compromise, we're turning off the flag that makes the debugger
+ start automatically.  If there is a program that you need to debug,
+ you need to turn that flag back on first, for example:
+
+    $ set image /flag=call_debug [.test]evp_test.exe
+
+ Then just run it and you will find yourself in a debuggin session.
+ When done, we recomment that you turn that flag back off:
+
+    $ set image /flag=nocall_debug [.test]evp_test.exe
+
+
  Checking the distribution
  -------------------------
 
diff --git a/util/local_shlib.com.in b/util/local_shlib.com.in
index daceffc506..e49aa15c77 100644
--- a/util/local_shlib.com.in
+++ b/util/local_shlib.com.in
@@ -2,10 +2,9 @@ ${-
   use File::Spec::Functions qw(rel2abs);
 
   my $bldtop = rel2abs($config{builddir});
-  our %names = ( ( map { $_ => $bldtop.$_.".EXE" }
-                   map { $unified_info{sharednames}->{$_} || () }
-                   @{$unified_info{libraries}} ),
-                 'DBG$IMAGE_DSF_PATH' => $bldtop );
+  our %names = ( map { $_ => $bldtop.$_.".EXE" }
+                 map { $unified_info{sharednames}->{$_} || () }
+                 @{$unified_info{libraries}} );
   "" -}
 $       ! Create a local environment with the shared library logical names
 $       ! properly set.  Undo this with unlocal_shlib.com
diff --git a/util/unlocal_shlib.com.in b/util/unlocal_shlib.com.in
index 67120c6e78..dd4fd2a9dd 100644
--- a/util/unlocal_shlib.com.in
+++ b/util/unlocal_shlib.com.in
@@ -2,10 +2,9 @@ ${-
   use File::Spec::Functions qw(rel2abs);
 
   my $bldtop = rel2abs($config{builddir});
-  our %names = ( ( map { $_ => $bldtop.$_.".EXE" }
-                   map { $unified_info{sharednames}->{$_} || () }
-                   @{$unified_info{libraries}} ),
-                 'DBG$IMAGE_DSF_PATH' => $bldtop );
+  our %names = ( map { $_ => $bldtop.$_.".EXE" }
+                 map { $unified_info{sharednames}->{$_} || () }
+                 @{$unified_info{libraries}} );
   "" -}
 $       ! Remove the local environment created by local_shlib.com
 $