MINI MINI MANI MO

Path : /opt/oracle/product/18c/dbhomeXE/rdbms/admin/
File Upload :
Current File : //opt/oracle/product/18c/dbhomeXE/rdbms/admin/rman.pm

# 
# $Header: rdbms/admin/rman.pm /main/3 2017/07/10 09:21:47 akruglik Exp $
#
# rman.pm
# 
# Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      rman.pm - rman perl Module 
#
#    DESCRIPTION
#      This module defines subroutines which can be used to execute one or 
#      more RMAN statements or a RMAN script in 
#      - a non-Consolidated Database, 
#      - all Containers of a Consolidated Database, or 
#      - a specified Container of a Consolidated Database
#
#    NOTES
#
#    MODIFIED   (MM/DD/YY)
#    akruglik    07/05/17 - LRG 20421900: a number of catcon subroutines allow
#                           caller to pass path to sqlplus binary or just a
#                           string sqlplus
#    molagapp    09/01/16 - reduce duplicate functions with catcon.pm
#    lexuxu      06/09/15 - Creation
# 


package rman;

use 5.006;
use strict;
use warnings;
use English;
use IO::Handle;       # to flush buffers
use Term::ReadKey;    # to not echo password
use IPC::Open2;       # to perform 2-way communication 
use File::Spec ();    
use catcon qw( catconImportRmanVariables TimeStamp get_connect_string 
               sureunlink exec_DB_script print_exec_DB_script_output 
               get_instance_status_and_name get_con_id find_in_path
               send_sig_to_procs get_num_procs get_CDB_indicator 
               build_connect_string valid_src_dir valid_log_dir 
               validate_script_path get_log_file_base_path get_fed_root_info
               get_fed_pdb_names);

require Exporter;

our @ISA = qw(Exporter);

# Items to export into callers namespace by default. Note: do not export
# names by default without a very good reason. Use EXPORT_OK instead.
# Do not simply export all your public functions/methods/constants.

# This allows declaration       use rman ':all';
# If you do not need this, moving things directly into @EXPORT or @EXPORT_OK
# will save memory.
our %EXPORT_TAGS = ( 'all' => [ qw(
        
) ] );

our @EXPORT_OK = qw ( rmanTest rmanInit rmanExec rmanWrapUp rmanRootonly 
                      rmanIgnore rmanFedRoot rmanEZConnect rmanVerbose 
                      rmanHostPort rmanAllInst); 

our @EXPORT = qw(
        
);
our $VERSION = '0.01';

##############################################################################
# Helper function Section
##############################################################################


# set_log_file_base_path
#
# Parameters:
#   - log directory, as specified by the user
#   - base for log file names
#   - an indicator of whether to produce debugging info
#
# Returns
#   - 0  ERROR 
#   - Log Base File Name
#
my $RMANOUT = *STDOUT;

sub set_log_file_base_path ($$$) 
{
   my ($LogDir, $LogBase, $DebugOn) = @_;

   my $RetLogFilePathBase = 0;

   # NOTE:
   # log directory and base for log file names is handled first since we 
   # need them to open the rman output file
   # if directory for log file(s) has been specified, verify that it exists 
   # and is writable
   #
   if (!valid_log_dir($LogDir)) 
   {
      # use STDERR because CATCONOUT is yet to be opened
      print STDERR "rman.pl: set_log_file_base_path: ".
                   "Unexpected error returned by valid_log_dir\n";
      return 0;
   }
      
   # get around catcon: get_log_file_base_path unexpected output.
   catconImportRmanVariables($RMANOUT, $DebugOn); 

   $RetLogFilePathBase =
      get_log_file_base_path($LogDir, $LogBase);

   # open RMANCONOUT. If opened successfully, all output generated 
   # should be written to RMANOUT to ensure that we can find it after 
   # running an lrg on the farm or through some other Perl 
   if (!open($RMANOUT, ">>", $RetLogFilePathBase."_rman_".$$.".lst")) 
   {
      print STDERR "rman.pl: set_log_file_base_path: unable to open ".
            $RetLogFilePathBase."_rman_".$$.".lst as RMANOUT\n";
      return 0;
   }

   print STDERR "rman.pl: ALL rman-related output will be written to [".
                $RetLogFilePathBase."_rman_".$$.".lst]\n";

   # make RMANOUT "hot" so diagnostic and error message output does not get 
   # buffered
   select((select($RMANOUT), $|=1)[0]);

   log_msg("rman.pl: See [${RetLogFilePathBase}*.log] files for ".
           "output generated by scripts\n");
   log_msg("rman.pl: See [${RetLogFilePathBase}*.lst] files for ".
           "spool files, if any\n");

   my $ts = TimeStamp();

   # do not dump the banner to STDERR
   print $RMANOUT <<msg;
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

$VERSION
\trmanInit: start logging rman output at $ts

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

msg

    return $RetLogFilePathBase;
}

# log_msg - add supplied string to both STDERR and RMANOUT 
sub log_msg($) 
{
   my ($LogMsg) = @_;

   print STDERR $LogMsg;
   print $RMANOUT $LogMsg if (defined $RMANOUT);
}

#
# printToRman- send RMAN command to pipe and execute it 
#
# parameters:
#   - function name calling printToRman (IN)
#   - RMAN file handle (IN)
#   - RMAN statement to be executed (IN)
#   - tail appending to the end of RMAN statement, normally \n (IN)
#   - an indicator of whether to produce debugging info (IN
#
# Returns: None 
#
sub printToRman ($$$$$) 
{
   my ($func, $fh, $stmt, $tail, $DebugOn) = @_;

   if ($DebugOn) 
   {
      my $printableStmt = $stmt;

      my $debugQry = 
      qq{select '$func(): $printableStmt' as rman_statement from dual\n/\n};
      print {$fh} $debugQry;
   }

   print {$fh} qq#$stmt$tail#;
}

#
# start_processes - start processes
#
# parameters:
#   - number of processes to start (IN)
#   - base for constructing log file names (IN)
#   - reference to an array of file handles; will be obtained as a 
#     side-effect of calling open() (OUT)
#   - reference to an array of process ids; will be obtained by calls 
#     to open (OUT)
#   - reference to an array of Container names; array will be empty if 
#     we are operating against a non-Consolidated DB (IN)
#   - Array of PDB status which is opend by RMAN (OUT) 
#   - Array of PDB Pointer mapping Procids to PDBStatus (OUT) 
#   - connect string array of PDBs used to connect to a DB (IN)
#   - connect string array of PDBs used to connect to a DB, hidden passwd (IN)
#   - an indicator of whether to produce debugging info (IN)
#   - an indicator of whether processes are being started for the first 
#     time (IN)
#   - RMAN done Command
#
# Returns: 1 if an error is encountered
#
sub start_processes ($$\@\@\@\@\@\@\@$$$) 
{
   my ($NumProcs, $LogFilePathBase, $FileHandles, $ProcIds, 
       $Containers, $PDBStatus, $PDBPointers, $ConnectString, 
       $ConnectStringDiag, $DebugOn, $FirstTime, $DoneCmd) =  @_;

   my $ps;

   # Array of length $NumProcs to be filled with positions of which
   # PDBs are available to open in @PDBStatus.
   my @PDB_Pointer = (0) x $NumProcs;

   if ($DebugOn) 
   {
      log_msg("start_processes: will start $NumProcs processes\n");
   }

   if (@$Containers and ($#$Containers >= 0))
   {
      # Initialize @rman_PDBStatus for start_process to pick PDBs from
      @$PDBStatus = (0) x (scalar @$Containers);

      if (scalar @$Containers > $NumProcs)
      {
         my $PDBidx;
         my $idx = 0;

         for ($PDBidx= 0; $PDBidx <= $#$Containers; $PDBidx++)
         {
             # Initialize $PDBStatus array, for example $PDBStatus(1,1,0,0)
             # means that PDB0, PDB1 are ready to be used by two available
             # procs. PDBStatus=0 remains waiting when a process has finished
             # its job.
             # PDB_Pointer is the array telling us which PDB being opened by
             # a certain process, for example, the value of $PDB_Pointer[0]
             # is the Container ID(offset to $PDBStatus) opened by ProcId=0 
             if ($PDBidx < $NumProcs) 
             {
                # PDBStatus contains indicator of whether corresponding PDBs
                # in @Containers should be open in start_process
                $PDBStatus->[$PDBidx] = 1; 

                #PDB_Pointer contains the position of @$PDBStatus which is 1
                $PDB_Pointer[$idx++] = $PDBidx;
             }
             else
             {
                last;
             }
         }
      }
      elsif (scalar(@$Containers) == $NumProcs )
      {
         # All PDBs will be opend in available processes - ready for use
         @$PDBStatus = (1) x (scalar @$Containers);

         @PDB_Pointer = (0..$#$Containers);
      }
      else
      {
         log_msg("Process Numbers greater than PDB numbers is". 
                 "not expected\n");
         return 1;
      }
   }
   else
   {
      if ($NumProcs != 1)
      {
         log_msg("start_processes: Non-CDB should not start more than one
                 processes\n");
         return 1;
      }
    
      @$PDBStatus = (1);
      @PDB_Pointer = (0);
   }

   @$PDBPointers = @PDB_Pointer;

   # 14787047: Save STDOUT so we can restore it after we start each process
   if (!open(SAVED_STDOUT, ">&STDOUT"))
   {
      log_msg("start_processes: failed to open SAVED_STDOUT\n");
      return 1;
   }

   # Keep Perl happy and avoid the warning "SAVED_STDOUT used only once"
   print SAVED_STDOUT "";

   for ($ps=0; $ps < $NumProcs; $ps++) 
   {
      #my $LogFile = $LogFilePathBase.$ps."_".$$.".log";
      my $LogFile = $LogFilePathBase."_".$ps.".log";

      # If starting for the first time, open for write; otherwise append
      if ($FirstTime)
      {
         if (!open (STDOUT,">", "$LogFile"))
         {
            log_msg("start_processes: failed to open STDOUT (1)\n");
            return 1;
         }
      } 
      else 
      {
        close (STDOUT);
        if (!open (STDOUT,"+>>", "$LogFile")) 
        {
           log_msg("start_processes: failed to open STDOUT (2)\n");
           return 1;
        }
      }

      my $id = open($FileHandles->[$ps], "|-", "rman");

      if (!$id) 
      {
         log_msg("start_processes: failed to open pipe to RMAN\n");
         return 1;
      }

      push(@$ProcIds, $id);

      if ($DebugOn) 
      {
         log_msg("start_processes: process $ps (id = $ProcIds->[$#$ProcIds]) ".
                 "will use log file $LogFile\n");
      }

      # file handle for the current process
      my $fh = $FileHandles->[$ps];
      my $PDBString = $ConnectString->[$PDB_Pointer[$ps]];
      my $PDBStringDiag = $ConnectStringDiag->[$PDB_Pointer[$ps]];

      print $fh "connect target $PDBString\n";

      if ($DebugOn) 
      {
         log_msg(qq#start_processes: connected using $PDBStringDiag\n#);
      }

      $fh->flush;
   }

   # 14787047: Restore saved stdout
   close (STDOUT);
   open (STDOUT, ">&SAVED_STDOUT");

   return create_done_files($NumProcs, $LogFilePathBase,
                            $FileHandles, $ProcIds,
                            $DebugOn, $DoneCmd);
}

#
# done_file_name_prefix - generate prefix of a name for a "done" file using 
# file name base and the id of a process to which the "done" file belongs
#
sub done_file_name_prefix($$) 
{
   my ($FilePathBase, $ProcId) =  @_;

   return $FilePathBase."_rman_".$ProcId;
}


#
# done_file_name - generate name for a "done" file using file name base 
#                  and the id of a process to which the "done" dile belongs
#
sub done_file_name($$) 
{
   my ($FilePathBase, $ProcId) =  @_;

   return done_file_name_prefix($FilePathBase, $ProcId).".done";
}

#
# create_done_files - Creates "done" files to be created for all processes.
#                     next_proc() will look for these files to determine
#                     whether a given process is available to take on the
#                     next script or SQL statement
#
# parameters:
#   - number of processes to start (IN)
#   - base for constructing log file names (IN)
#   - reference to an array of file handles (IN)
#   - reference to an array of process ids (IN)
#   - an indicator of whether to produce debugging info (IN)
#   - Done Command (IN)
#
sub create_done_files ($$$$$$) 
{
   my ($NumProcs, $LogFilePathBase, $FileHandles_REF,
       $ProcIds_REF, $DebugOn, $DoneCmd) =  @_;

   for (my $CurProc = 0; $CurProc < $NumProcs; $CurProc++)
   {
      my $DoneFile =
         done_file_name($LogFilePathBase, $ProcIds_REF->[$CurProc]);
   
      if (! -e $DoneFile) 
      {
         my $DoneCmdFile = sprintf($DoneCmd, $DoneFile);
         # "done" file does not exist - cause it to be created
         printToRman("create_done_files", $FileHandles_REF->[$CurProc],
                     $DoneCmdFile, "\n", $DebugOn);

         # flush the file so a subsequent test for file existence does 
         # not fail due to buffering
         $FileHandles_REF->[$CurProc]->flush;

         if ($DebugOn) 
         {
            my $msg = <<msg;
create_done_files: sent "$DoneCmd $DoneFile"
    to process $CurProc (id = $ProcIds_REF->[$CurProc]) to indicate its 
    availability
msg
            log_msg($msg);
         }
      } 
      elsif (! -f $DoneFile) 
      {
         log_msg(
         qq#create_done_files: "done" file name collision: $DoneFile\n#);

         return 1;
      } 
      else 
      {
         if ($DebugOn) 
         {
            log_msg(
            qq#create_done_files: "done" file $DoneFile already exists\n#);
         }
      }
   }

   return 0;
}

#
# end_processes - end all processes
#
# parameters:
#   - index of the first process to end (IN)
#   - index of the last process to end (IN)
#   - reference to an array of file handles (IN)
#   - reference to an array of process ids; will be cleared of its 
#     elements (OUT)
#   - an indicator of whether to produce debugging info (IN)
#   - log file path base
#
sub end_processes ($$\@\@$$) 
{
   my ($FirstProcIdx, $LastProcIdx, $FileHandles, $ProcIds, $DebugOn,
       $LogFilePathBase) =  @_;

   my $ps;

   if ($FirstProcIdx < 0) 
   {
      log_msg("end_processes: FirstProcIdx ($FirstProcIdx) was less than 0\n");
      return 1;
   }

   if ($LastProcIdx < $FirstProcIdx)
   {
      log_msg("end_processes: LastProcIdx ($LastProcIdx) was ".
              "less than  FirstProcIdx ($FirstProcIdx)\n");
      return 1;
   }

   if ($DebugOn)
   {
      log_msg("end_processes: will end processes $FirstProcIdx ".
              "to $LastProcIdx\n");
   }

   $SIG{CHLD} = 'IGNORE';

   for ($ps = $FirstProcIdx; $ps <= $LastProcIdx; $ps++)
   {
      if ($FileHandles->[$ps]) 
      {
         print {$FileHandles->[$ps]} "EXIT;\n";
         close ($FileHandles->[$ps]);
         $FileHandles->[$ps] = undef;
      } 
      elsif ($DebugOn) 
      {
        log_msg("end_processes: process $ps has already been stopped\n");
      }
   }

   clean_up_compl_files($LogFilePathBase, $ProcIds, $FirstProcIdx, $LastProcIdx,
                        $DebugOn);

   splice @$ProcIds, $FirstProcIdx, $LastProcIdx - $FirstProcIdx + 1;

   if ($DebugOn)
   {
      log_msg("end_processes: ended processes $FirstProcIdx to $LastProcIdx\n");
   }

   return 0;
}

# clean_up_compl_files - purge files indicating completion of processes.
#                        Purging will start from the first process given
#                        and end at the last process given.
#      
# parameters:
#    - base of process completion file name (IN)
#    - reference to an array of process ids (IN)
#    - First process whose completion files are to be purged (IN)
#    - Last processes whose completion files are to be purged (IN)
#    - an indicator of whether to print debugging info (IN)
# 
sub clean_up_compl_files ($$$$$) 
{
   my ($FileNameBase, $ProcIds, $FirstProc, $LastProc, $DebugOn) =  @_;

   my $ps;

   if ($DebugOn)
   {
      log_msg("clean_up_compl_files: FileNameBase = $FileNameBase, ".
              "FirstProc = $FirstProc, LastProc = $LastProc\n");
      log_msg(qq#ProcIds=@$ProcIds\n#);
   }

   for ($ps=$FirstProc; $ps <= $LastProc; $ps++) 
   {
      my $DoneFile = done_file_name($FileNameBase, $ProcIds->[$ps]);

      if (-e $DoneFile && -f $DoneFile)
      {
         if ($DebugOn)
         {
            log_msg("clean_up_compl_files: call sureunlink to ".
                    "remove $DoneFile\n");
         }

         sureunlink($DoneFile);

         if ($DebugOn) 
         {
            log_msg(qq#clean_up_compl_files: removed $DoneFile\n#);
         }
      }
   }
}

#
# next_proc - determine next process which has finished work assigned to it
#             (and is available to execute a script or an RMAN statement)
#
# Description:
#   This subroutine will look for a file ("done" file) indicating that the 
#   process has finished running a script or a statement which was sent to it.
#   Once such file is located, it will be deleted and a number of a process 
#   to which that file corresponded will be returned to the caller
#
# Parameters:
#   - number of processes from which we may choose the next available process
#   - total number of processes which were started (used when we try to
#     determine if some processes may have died)
#   - number of the process starting with which to start our search; if the 
#     supplied number is greater than $ProcsUsed, it will be reset to 0
#   - reference to an array of booleans indicating status of which processes 
#     should be ignored; may be undefined
#   - base for names of files whose presense will indicate that a process has 
#     completed
#   - reference to an array of process ids
#   - an indicator of whether we are running under Windows
#   - an indicator of whether to display diagnostic info
#
sub next_proc ($$$$$$$$$$) 
{
   my ($ProcsUsed, $NumProcs, $StartingProc, $ProcsToIgnore,
       $FilePathBase, $ProcIds, $PDBStatus, $PDB_Pointer, 
       $Windows, $DebugOn) = @_;

   if ($DebugOn)
   {
      my $msg = <<next_proc_DEBUG;
  running next_proc(ProcsUsed    = $ProcsUsed, 
                    NumProcs     = $NumProcs,
                    StartingProc = $StartingProc,
                    FilePathBase = $FilePathBase,
                    Windows      = $Windows,
                    DebugOn      = $DebugOn);

next_proc_DEBUG
      log_msg($msg);
   }

   my $Pick_Idle = 0;

   # This is a check against PDBStatus array, if there's one or more PDB 
   # in the status of "ready", next_proc should not return a process
   # referring to a PDB in other status. 
   # This is to prevent an undesigned situation that one PDB
   # is finishing its job too fast and that proc is trying to continuous
   # take on new tasks and calls switch_proc to switch to another PDB
   # while other process just remains idle. This causes problem because  
   # some idle PDB is waiting to be picked up but in such case switch_proc
   # will be called while there might be no available PDB to take 
   # (switch_proc takes free prcoess based on if its PDBStatus is 1 or a 
   # Donefile is present). 
   foreach my $CurConStatus (@$PDBStatus) 
   {
      if ($CurConStatus == 1)
      {
         # Set pick_idle flag to force next_proc to return proc of status 1
         $Pick_Idle = 1;
      }
   }

   # process number will be used to construct names of "done" files
   # before executing the while loop for the first time, it will be set to 
   # $StartingProc.  If that number is outside of [0, $ProcsUsed-1], 
   # it [$CurProc] will be reset to 0; for subsequent iterations through the 
   # while loop, $CurProc will start with 0
   my $CurProc = ($StartingProc >= 0 && $StartingProc <= $ProcsUsed-1)
                  ? $StartingProc : 0;

   # look for *.done files which will indicate which processes have 
   # completed their work
   #
   # we may end up waiting a while before finding an available process and if 
   # debugging is turned on, user's screen may be flooded with 
   # next_proc: Skip checking process ...
   # and
   # next_proc: Checking if process ... is available
   # messages.  
   #
   # To avoid this, we will print this message every 10 second or so.  Since 
   # we check for processes becoming available every 0.01 of a second (or so), 
   # we will report generate debugging messages every 1000-th time through the 
   # loop
   my $itersBetweenMsgs = 1000;

   for (my $numIters = 0; ; $numIters++)
   {
      # Make sure no rman processes died (Windows only).
      # For Unix this handle through signals.
      if ($Windows)
      {
         # sending signal 0 to processes whose ids are stored in an array will 
         # return a number of processes to which this signal could be 
         # delivered, ergo which are alive
         my $LiveProcs = send_sig_to_procs(@$ProcIds, 0);

         if ($NumProcs != $LiveProcs) 
         {
            log_msg(qq#next_proc: total processes ($NumProcs) != number of live processes ($LiveProcs); giving up\n#);

            rmanWrapUp();
            send_sig_to_procs(@$ProcIds, 9);
            clean_up_compl_files($FilePathBase, $ProcIds, 0, $ProcsUsed-1,
                             $DebugOn);
            rman_HandleSigchld();

            return -1;
         }
      }

      for (; $CurProc < $ProcsUsed; $CurProc++) 
      {
         if ( $ProcsToIgnore && @$ProcsToIgnore
              && $ProcsToIgnore->[$CurProc])
         {
            if ($DebugOn && $numIters % $itersBetweenMsgs == 0) 
            {
               log_msg("next_proc: Skip checking process $CurProc\n");
            }
            next;
         }

         if ($Pick_Idle && ($PDBStatus->[$PDB_Pointer->[$CurProc]] != 1))
         {
            if ($DebugOn && $numIters % $itersBetweenMsgs == 0) 
            {
               log_msg("next_proc: Skip checking process $CurProc".
                       " since there's another preferred prcoess".
                       " with status \"ready\"\n");
            }
            next;
         }

         # file which will indicate that process $CurProc finished its work
         my $DoneFile = done_file_name($FilePathBase, $ProcIds->[$CurProc]);

         if ($DebugOn && $numIters % $itersBetweenMsgs == 0) 
         {
            log_msg("next_proc: Checking if process $CurProc (id = ".
                    "$ProcIds->[$CurProc]) is available\n");
         }

         # Is file is present, remove the "done" file (thus making this process 
         # appear "busy") and return process number to the caller.
         if (-e $DoneFile)
         {
            if ($DebugOn)
            {
              log_msg("next_proc: call sureunlink to remove $DoneFile\n");
            }

            sureunlink($DoneFile);

            if ($DebugOn)
            {
               log_msg("next_proc: process $CurProc is available\n");
            }

            return $CurProc;
         }
      }

      select (undef, undef, undef, 0.01);

      $CurProc = 0;
   }

  return -1;  # this statement will never be reached
}


#
# wait_for_completion - wait for completion of processes
#
# Description:
#   This subroutine will wait for processes to indicate that they've 
#   completed their work by creating a "done" file.  Since it will use 
#   next_proc() (which deleted "done" files of processes whose ids it returns) 
#   to find processes which are done running, this subroutine will generate 
#   "done" files once all processes are done to indicate that they are 
#   available to take on more work.
#
# Parameters:
#   - number of processes which were used (and whose completion we need to 
#     confirm)
#   - total number of processes which were started
#   - base for generating names of files whose presense will indicate that a 
#     process has completed
#   - references to an array of process file handles
#   - reference to a array of process ids
#   - array of PDB Status
#   - array of process id mapping to ConId 
#   - command which will be sent to a process to cause it to generate a 
#     "done" file
#   - an indicator of whether we are running under Windows
#   - an indicator of whether to display diagnostic info
#
sub wait_for_completion ($$$\@\@\@\@$$$) 
{
   my ($ProcsUsed, $NumProcs, $FilePathBase, $ProcFileHandles,
       $ProcIds, $PDBStatus, $PDB_Pointer, $DoneCmd, $Windows, 
       $DebugOn) = @_;

   if ($DebugOn)
   {
      my $msg = <<wait_for_completion_DEBUG;
  running wait_for_completion(ProcsUsed             = $ProcsUsed, 
                              FilePathBase          = $FilePathBase,
                              DoneCmd               = $DoneCmd,
                              Windows               = $Windows,
                              DebugOn               = $DebugOn);

wait_for_completion_DEBUG
      log_msg($msg);
   }

   my $CurProc = 0;

   if ($DebugOn)
   {
      log_msg("wait_for_completion: waiting for $ProcsUsed ".
              "processes to complete\n");
   }

   # look for *.done files which will indicate which processes have 
   # completed their work
   my $NumProcsCompleted = 0;  # how many processes have completed

   # This array will be used to keep track of processes which have completed 
   # so as to avoid checking for existence of files which have already been 
   # seen and removed
   my @ProcsCompleted = (0) x $ProcsUsed;

   while ($NumProcsCompleted < $ProcsUsed)
   {
      $CurProc = next_proc($ProcsUsed, $NumProcs, $CurProc + 1, 
                           \@ProcsCompleted, $FilePathBase, 
                           $ProcIds, $PDBStatus, $PDB_Pointer, 
                           $Windows, $DebugOn);

      if ($CurProc < 0) 
      {
         log_msg(qq#wait_for_completion: unexpected error in next_proc()\n#);
         return 1;
      }

      if ($DebugOn) 
      {
         log_msg("wait_for_completion: process $CurProc is done\n");
      }

      $NumProcsCompleted++; # one more process has comleted

      # remember that this process has completed so next_proc does not try to 
      # check its status
      $ProcsCompleted[$CurProc] = 1;
   }

   if ($DebugOn) 
   {
      log_msg("wait_for_completion: All $NumProcsCompleted ".
              "processes have completed\n");
   }
   # issue statements to cause "done" files to be created to indicate
   # that all $ProcsUsed processes are ready to take on more work
   for ($CurProc = 0; $CurProc < $ProcsUsed; $CurProc++) 
   {
      # file which will indicate that process $CurProc finished its work
      my $DoneFile = done_file_name($FilePathBase, $ProcIds->[$CurProc]);
      my $DoneCmdFile = sprintf($DoneCmd, $DoneFile);

      printToRman("wait_for_completion", $ProcFileHandles->[$CurProc],
                  $DoneCmdFile, "\n", $DebugOn);

      # flush the file so a subsequent test for file existence does 
      # not fail due to buffering
      $ProcFileHandles->[$CurProc]->flush;

      if ($DebugOn) 
      {
         my $msg = <<msg;
wait_for_completion: sent "$DoneCmd $DoneFile" 
    to process $CurProc (id = $ProcIds->[$CurProc]) to indicate that it is 
    available to take on more work
msg
         log_msg($msg);
      }
  }

  return 0;
}

#
# pickNextProc - pick a process to run next statement or script
# 
# Note pickNextProc picks a free process and internally ends and restarts
# a new process to switch to another PDB since RMAN cannot switch containers
# if it's a new process, then just pick that without call switch_proc
#
# Parameters:
#   (IN) unless indicated otherwise
#   - number of processes from which we may choose the next available process 
#   - total number of processes which were started (used when we try to
#     determine if some processes may have died)
#   - number of the process starting with which to start our search; if the 
#     supplied number is greater than $ProcsUsed, it will be reset to 0
#   - base for names of files whose presense will indicate that a process has 
#     completed
#   - reference to an array of PDB Status 
#   - reference to an array of PDB Pointers 
#   - reference to an array of connection strings
#   - reference to an array of connection strings, hidden passwd for logs
#   - reference to an array of file handles 
#   - reference to an array of process ids
#   - Done Command to generate done file 
#   - an indicator of whether we are running under Windows
#   - an indicator of whether to display diagnostic info
#
sub pickNextProc ($$$$\@\@\@\@\@\@$$$) 
{
   my ($ProcsUsed, $NumProcs, $StartingProc, $FilePathBase,
       $PDBStatus, $PDB_Pointer, $PDBConnString, $PDBConnStringDiag,
       $ProcFileHandles, $ProcIds, $DoneCmd, $Windows, $DebugOn) = @_;

   if ($DebugOn)
   {
      log_msg("pickNextProc: calling pickNextProc\n");
   }

   # find next available process
   my $CurProc = next_proc($ProcsUsed, $NumProcs, $StartingProc, undef,
                           $FilePathBase, $ProcIds, $PDBStatus, 
                           $PDB_Pointer, $Windows, $DebugOn);
   if ($CurProc < 0)
   {
      # some unexpected error was encountered
      log_msg("pickNextProc: unexpected error in next_proc\n");

      return -1;
   }

   # If CurProc corresponds to a PDB of status 1 (ready), then update 
   # the stauts to 2(running scripts) ,reutrn this proc to run 
   # scripts/statements
   # If CurProc corresponds to a PDB of status 2 (running), it means 
   # the proc has finished one PDB's job, update to status 3(completed)
   # and switch to another PDB.
   # If CurProc corresponds to a PDB of other status, someting is wrong,
   # report error.
   if ($PDBStatus->[$PDB_Pointer->[$CurProc]] == 1)
   {
      # update PDBStatus to running scripts/statements status
      $PDBStatus->[$PDB_Pointer->[$CurProc]] = 2;

      return $CurProc;
   }
   elsif ($PDBStatus->[$PDB_Pointer->[$CurProc]] == 2)
   {
      # update PDBStatus to status "completed"
      $PDBStatus->[$PDB_Pointer->[$CurProc]] = 3;

      # Since RMAN cannot switch PDBs in its session, to pick next available
      # processes to open the next waiting PDB we will end CurProc and start
      # another new proc for the next PDB in line.
      if (switch_proc($FilePathBase, $ProcsUsed, $CurProc, $PDBStatus, 
                      $PDB_Pointer, $ProcFileHandles, $ProcIds, 
                      $PDBConnString, $PDBConnStringDiag, $DoneCmd, $DebugOn))
      {
         log_msg("pickNextProc: unexpected error in switch_proc\n");
         return -1;
      }

      # update PDBStatus to running scripts/statements status
      # This is a New PDB since $PDBPointer has been updated
      # in switch_proc
      $PDBStatus->[$PDB_Pointer->[$CurProc]] = 2;

      return $CurProc;
   }
   else
   {
      log_msg("pickNextProc: Process $CurProc returned corresponds ".
              "to a Container Databae status ".
              "$PDBStatus->[$PDB_Pointer->[$CurProc]] is not expected\n"); 
      return -1;
   }
}


#
# switch_proc - switch Curproc to a new process for another PDB  
#
# Parameters:
#   - base for constructing log/done file names (IN) 
#   - number of processes from which we may pick the next available process(IN) 
#   - Offset of the current process to switch (IN)
#   - Array of the PDB status opened in processes (OUT)
#   - Array of Pointers to PDB status array (OUT)
#   - File handles of each process (OUT)
#   - reference to an array of process ids (OUT)
#   - Connect strings for each PDB (IN)
#   - Connect strings for each PDB, hidden passwd (IN)
#   - RMAN done Command (IN)
#   - indicator of whether Debug info should be outputted (IN)
#
# Returns: 1 if an error is encountered
#
sub switch_proc ($$$$$$$$$$) 
{
   my ($FilePathBase, $ProcsUsed, $CurProc, $PDBStatus, $PDB_Pointer,
       $ProcFileHandles, $ProcIds, $PDBConnString, $PDBConnStringDiag,
       $DoneCmd, $DebugOn) = @_;

   if ($DebugOn)
   {
      log_msg("switch_proc: calling switch_proc\n");
   }

   # All PDBs are opend by RMAN, no reason to call switch_proc for
   # an unused PDB
   if (scalar @$PDBStatus == $ProcsUsed)
   {
      log_msg("switch_proc: number of PDBs equals number of processes, ".
              "switch_proc is not expected to be called\n");
      return 1;
   }

   if ($DebugOn)
   {
      log_msg("switch_proc: will end process $CurProc\n");
   }

   $SIG{CHLD} = 'IGNORE';

   # End CurProc first
   if ($ProcFileHandles->[$CurProc])
   {
      print {$ProcFileHandles->[$CurProc]} "EXIT;\n";
      close ($ProcFileHandles->[$CurProc]);
      $ProcFileHandles->[$CurProc] = undef;
   }
   elsif ($DebugOn)
   {
      log_msg("switch_proc: process $CurProc has already been stopped\n");
   }

   if ($DebugOn)
   {
      log_msg("switch_proc: successfully ended process $CurProc ".
              "with process id $ProcIds->[$CurProc]\n");
      log_msg("switch_proc: Now replace it with a new process\n");
   }

   $SIG{CHLD} = \&rman_HandleSigchld;

   # restart a new proc

   # 14787047: Save STDOUT so we can restore it after we start each process
   if (!open(SAVED_STDOUT_R, ">&STDOUT"))
   {
      log_msg("switch_proc: failed to open SAVED_STDOUT\n");
      return 1;
   }

   # Keep Perl happy and avoid the warning "SAVED_STDOUT_R used only once"
   print SAVED_STDOUT_R "";

   #my $LogFile = $FilePathBase.$CurProc."_".$$.".log";
   my $LogFile = $FilePathBase."_".$CurProc.".log";

   # append to the existing log file
   close (STDOUT);
   if (!open (STDOUT,"+>>", "$LogFile"))
   {
      log_msg("switch_proc: failed to open STDOUT\n");
      return 1;
   }

   my $id = open($ProcFileHandles->[$CurProc], "|-", "rman");

   if (!$id)
   {
      log_msg("switch_proc: failed to open pipe to RMAN\n");
      return 1;
   }

   $ProcIds->[$CurProc] = $id;

   if ($DebugOn)
   {
      log_msg("switch_proc: process $CurProc (id = $ProcIds->[$CurProc]) ".
              "will use log file $LogFile\n");
   }
   
   # file handle for the current process
   my $fh = $ProcFileHandles->[$CurProc];

   # construct PDB_Pointer to map @Containers and @ProcIds,
   # $PDB_Pointer->[$CurProc] represents the offset to @Containers
   # and @PDBStatus and note that values in @PDBStatus represent
   # the status of the PDB used by RMAN (0-unused, 1-being used, 2-completed)
   my $PDBidx;
   my $idx = 0;
   my $found = 0;

   if ($DebugOn)
   {
      log_msg("switch_proc: Status of PDBStatus is @$PDBStatus\n"); 
      log_msg("switch_proc: Status of PDB_Pointer is @$PDB_Pointer\n"); 
   }

   for ($PDBidx= 0; $PDBidx <= $#$PDBStatus; $PDBidx++)
   {
      if ($PDBStatus->[$PDBidx] == 0)
      {
         # found the PDB to be opened
         $found = 1;

         # This is the PDB we pick to open next by RMAN 
         my $CurCon = $PDB_Pointer->[$CurProc];
 
         if ($PDBStatus->[$CurCon] != 3)
         {
            #PDBinPRoc->[$CurCon] must be 2 since it was being used by RMAN
            log_msg("switch_proc: PDBStatus->[$CurCon] value is not 3".
                    " (completed), not expected\n");
            return 1;
         }

         # Update PDB_Pointer to PDBidx which is the available PDB index 
         $PDB_Pointer->[$CurProc] = $PDBidx;

         # Update PDBStatus for a newly opening PDB - Status "ready"
         $PDBStatus->[$PDBidx] = 1;

         last;
      }
   }

   # return error if cannot find a PDB to switch
   if (!$found)
   {
      log_msg("switch_proc: failed to find available PDB to open\n");
      return 1;
   }

   if ($DebugOn)
   {
      log_msg("switch_proc: updated status of PDBStatus".
              "is @$PDBStatus\n"); 
      log_msg("switch_proc: updated status of PDB_Pointer ".
              "is @$PDB_Pointer\n"); 
   }
                
   my $PDBString = $PDBConnString->[$PDB_Pointer->[$CurProc]];
   my $PDBStringDiag = $PDBConnStringDiag->[$PDB_Pointer->[$CurProc]];

   print $fh "connect target $PDBString\n";

   if ($DebugOn)
   {
      log_msg("switch_proc: connected using $PDBStringDiag\n");
   }

   $fh->flush;

   # 14787047: Restore saved stdout
   close (STDOUT);
   open (STDOUT, ">&SAVED_STDOUT_R");
   
   # create done file for this new process
   my $DoneFile =
         done_file_name($FilePathBase, $ProcIds->[$CurProc]);

   # No done file was created for this new process, so no such file
   # should exist.
   if ( -e $DoneFile)
   {
      if ($DebugOn)
      {
         log_msg("switch_proc: done file $DoneFile should not exist\n");
      }
      return 1;
   }

   if ($DebugOn)
   {
      log_msg("switch_proc: successfully restarted process $CurProc ".
              "with process id $ProcIds->[$CurProc]\n");
   }

   return 0;
}

#
# validate_con_names - determine whether Container name(s) supplied
#     by the caller are valid (i.e. that the DB is Consolidated and contains a 
#     Container with specified names) and modify a list of Containers against 
#     which scripts/statements will be run as directed by the caller:  
#     - if the caller indicated that a list of Container names to be validated
#       refers to Containers against which scripts/statements should not be 
#       run, remove them from $Containers
#     - otherwise, remove names of Containers which did not appear on the 
#       list of Container names to be validated from $Containers
#
# Parameters:
#   - reference to a string containing space-delimited name(s) of Container
#   - an indicator of whether the above list refers to Containers against 
#     which scripts/statements should not be run
#   - reference to an array of Container names, if any
#   - reference to an array of indicators of whether a corresponding 
#     Container is open (Y/N)
#   - an indicator of whether non-existent PDBs should be ignored
#   - an indicator of whether debugging information should be produced
#
# Returns:
#   An empty list if an error was encountered or if the list of names of 
#   Containers to be excluded consisted of names of all Containers of a CDB.
#   A list of names of Containers which were either explicitly included or 
#   were NOT explicitly excluded by the caller
#
sub validate_con_names (\$$\@\@$$) 
{
   my ($ConNameStr, $Exclude, $Containers, $IsOpen, $Force, $DebugOn) = @_;

   if ($DebugOn) 
   {
       my $msg = <<validate_con_names_DEBUG;
running validate_con_names(
  ConNameStr   = $$ConNameStr, 
  Exclude      = $Exclude, 
  Containers   = @$Containers,
  IsOpen       = @$IsOpen,
  Force        = $Force)
validate_con_names_DEBUG
      log_msg($msg);
   }

   if (!${$ConNameStr}) 
   {
      # this subroutine should not be called unless there are Container 
      # names to validate
      log_msg("validate_con_names: missing Container name string\n");

      return ();
   }

   my $LocConNameStr = $$ConNameStr;  
   $LocConNameStr =~ s/^'(.*)'$/$1/;  # strip single quotes
   # extract Container names into an array
   my @ConNameArr = split(/\s+/, $LocConNameStr);
  
   if (!@ConNameArr) 
   {
      # string supplied by the caller contained no Container names
      log_msg("validate_con_names: no Container names to validate\n");

      return ();
   }
  
   # string supplied by the caller contained 1 or more Container names
   if (!@$Containers) 
   {
      # report error and quit since the database is not Consolidated
      log_msg("validate_con_names: Container name(s) supplied but the DB ".
              "is not Consolidated\n");

      return ();
   }

   if ($DebugOn) 
   {
      log_msg("validate_con_names: Container name string consisted of the ".
              "following names {\n");

      foreach (@ConNameArr) 
      {
         log_msg("validate_con_names: \t$_\n");
      }

      log_msg("validate_con_names: }\n");
   }

   my %ConNameHash;   # hash of elements of @ConNameArr

   foreach (@ConNameArr) 
   {
      undef $ConNameHash{uc $_};
   }

   # array of Container names against which scripts/statements should be run
   my @ConOut = ();
   my $matched = 0; # number of elements of @$Containers found in %ConNameHash

   my $CurCon; # index into @$Containers and @$IsOpen;

   for ($CurCon = 0; $CurCon <= $#$Containers; $CurCon++) 
   {

      my $Con = uc $$Containers[$CurCon];
      my $Open = $$IsOpen[$CurCon];

      # if we have matched every element of %ConNameHash, no reason to 
      # check whether it contains $Con
      if (($matched < @ConNameArr) && exists($ConNameHash{$Con})) 
      {
         $matched++; # remember that one more element of @$Containers was found

         # remove $Con from %ConNameHash so that if we end up not matching 
         # specified Container names, we can list them as a part of error 
         delete $ConNameHash{$Con};

         if ($DebugOn) 
         {
            log_msg("validate_con_names: $$Containers[$CurCon] was matched\n");
         }
         # add matched Container name to @ConOut if caller indicated that 
         # Containers whose names were specified are to be included in the set 
         # of Containers against which scripts/statements will be run
         if (!$Exclude) 
         {
            push(@ConOut, $$Containers[$CurCon]); 

            if ($DebugOn)
            {
               log_msg("validate_con_names: Added $$Containers[$CurCon] ".
                       "to ConOut\n");
            }
         }
      } 
      else 
      {
         if ($DebugOn) 
         {
            log_msg("validate_con_names: $$Containers[$CurCon] was ".
                    "not matched\n");
         }
         # add unmatched Container name to @ConOut if caller indicated that 
         # Containers whose names were specified are to be excluded from the set 
         # of Containers against which scripts/statements will be run
         if ($Exclude)
         {
            push(@ConOut, $$Containers[$CurCon]);

            if ($DebugOn)
            {
               log_msg("validate_con_names: Added $$Containers[$CurCon] ".
                       "to ConOut\n");
            }
         }
      }
   }

   # if any of specified Container names did not get matched, report an error
   if ($matched != @ConNameArr) 
   {
      # Bug 18029946: print the list of unmatched Container names if were not 
      # told to ignore unmatched Container names or if debug flag is set
      if (!$Force || $DebugOn) 
      {
         log_msg("validate_con_names: some specified Container names do ".
                 "not refer to existing Containers:\n");
         for (keys %ConNameHash) 
         {
            log_msg("\t$_\n");
         }

         if ($Force) 
         {
            log_msg("validate_con_names: unmatched Container names ".
                    "ignored because Force is specified\n");
         }
      }

      # Bug 18029946: unless told to ignore unmatched Container names, return 
      # an empty list which will cause the caller to report an error
      if (!$Force) 
      {
         return ();
      }
   } 

   # if @ConOut is empty (which could happen if we were asked to exclude every 
   # Container or if all existing PDBs which matched caller's criteria were 
   # closed)
   if (!@ConOut) 
   {
      log_msg("validate_con_names: resulting Container list is empty\n");
      return ();
   }

   if ($DebugOn) 
   {
      log_msg("validate_con_names: resulting Container set consists of ".
              "the following Containers {\n");

      foreach (@ConOut) 
      {
         log_msg("validate_con_names:\t$_\n");
      }

      log_msg("validate_con_names: }\n");
   }

   return @ConOut;
}

#
# get_con_info - query V$CONTAINERS to get info about all Containers (currently
#                we are only getting names and open modes)
#
# parameters:
#   - connect strings - [0] used to connect to a DB, 
#                       [1] can be used to produce diagnostic message
#   - a command to create a file whose existence will indicate that the 
#     last statement of the script has executed (needed by exec_DB_script())
#   - base for a name of a "done" file (see above)
#   - reference to an array of Container names (OUT)
#   - reference to an array of indicators of whether a corresponding 
#     Container is open (OUT)
#   - indicator of whether to produce debugging info
#
sub get_con_info (\@$$\@\@$) 
{
   my ($myConnect, $DoneCmd, $DoneFilePathBase, $ConNames, $IsOpen,
       $debugOn) = @_;

   # NOTE: it is important that we do not fetch data from dictionary views 
   #       (e.g. DBA_PDBS) because views may yet to be created if this 
   #       procedure is used when running catalog.sql
   # NOTE: since a non-CDB may also have a row in CONTAINER$ with con_id#==0,
   #       we must avoid fetching a CONTAINER$ row with con_id==0 when looking 
   #       for Container names 
   #
   # Bug 18070841: "SET LINES" was added to make sure that both the PDB name 
   #               and the open_mode are on the same line.  500 is way bigger 
   #               than what is really needed, but it does not hurt anything.
   #
   # Bug 20307059: append mode to XXXXXX<container-name> to ensure that the 
   #               two do not get split across multiple lines
  
   my @GetConInfoStmts = (
       "connect $myConnect->[0]\n",
       "set echo off\n",
       "set heading off\n",
       "set lines 500\n",
       "select \'XXXXXX\' || v.name || ".
       "       decode(v.open_mode, 'MOUNTED', ' N', ' Y') ".
       "  from v\$containers v ".
       "  where v.con_id > 0 ".
       "  order by v.con_id\n/\n");
  
   # each row consists of a Container name and Y/N of whether it is open
   my $rows_ref;
   my $spool_ref;

   ($rows_ref, $spool_ref) =
      exec_DB_script(@GetConInfoStmts, "XXXXXX",
                     $DoneCmd, $DoneFilePathBase, "sqlplus");
  
   if (!@$rows_ref || $#$rows_ref < 0) 
   {
      # Container info could not be obtained
      print_exec_DB_script_output("get_con_info", $spool_ref, 1);
      return 1;
   }
    
   # split each row into a Container name and an indicator of whether 
   # it'is open
   for my $row ( @$rows_ref ) 
   {
      my ($name, $open) = split /\s+/, $row;
      push @$ConNames, $name;
      push @$IsOpen, $open;
   }
  
   return 0;
}   

#
# get_pdb_connect_string - get PDBs connect strings 
#
# parameters:
#   - connect string for sqlplus
#   - connect string template to build RMAN Connection String for PDBs 
#   - connect string template to build RMAN Connection String for PDBs, 
#     password hidden 
#   - a command to create a file whose existence will indicate that the 
#     last statement of the script has executed (needed by exec_DB_script())
#   - base for a name of a "done" file (see above)
#   - Array of Container names to be processed (IN)
#   - reference to an array of PDB Connect Strings (OUT)
#   - reference to an array of PDB Connect Strings that hides password (OUT)
#   - indicator of whether to produce debugging info
#
sub get_pdb_connect_string (\@$$$$$$\@\@\@$) 
{
   my ($myConnect, $rmanconn, $rmanconnDiag, $hostname, $port, 
       $DoneCmd, $DoneFilePathBase, $Containers, $PDBConnString, 
       $PDBConnStringDiag, $debugOn) = @_;

   my $rmanconnTemplate;
   my $rmanconnTemplateDiag;
   my @QuoteContainers; 
   my @Link;
   my $ContainerArray; 
   my $StringTemplate;
   my $TcpTemplate    = "(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)".
                        "(HOST=%s)(PORT=%s))".
                        "(CONNECT_DATA=(SERVICE_NAME=%s)))";

   my $IpcTemplate    = "(DESCRIPTION=(ADDRESS=(PROTOCOL=ipc)".
                        "(KEY=%s))".
                        "(CONNECT_DATA=(SERVICE_NAME=%s)))";

   if (!@$Containers || $#$Containers < 0)
   {
      log_msg("get_pdb_connect_string: invalid Containers Array\n");
      return 1;
   }

   if ($hostname and $port)
   {
      $StringTemplate = $TcpTemplate;
   }
   elsif ($ENV{ORACLE_SID})
   {
      $StringTemplate = $IpcTemplate;
   }
   else
   {
      log_msg("get_pdb_connect_string: Neither Host/Port nor ".
              "ORACLE_SID set\n");
      return 1;
   }

   foreach (@$Containers)
   {
      my $element = "\'"."$_"."\'";
      push @QuoteContainers, $element; 
   }
   
   # To construct array like ('PDB1', 'PDB2'...)
   $ContainerArray = join(', ', @QuoteContainers);

   my @GetPdbConnStmts = (
       "connect $myConnect->[0]\n",
       "set echo off\n",
       "set heading off\n",
       "set lines 800\n",
       "select \'XXXXXX\' || max(name) name ".
       "from v\$active_services where ".
       "con_name in ($ContainerArray) group by con_name ".
       "order by max(con_id)\n/\n");

       #"select \'XXXXXX\' || s.name from cdb_service\$ s, v\$pdbs p where ".
       #"s.con_id# = p.con_id and bitand(s.flags, 128) = 128 and p.name ".
       #"in ($ContainerArray)\n/\n");

   # each row consists of a PDB's service name 
   my ($rows_ref, $Spool_ref) =
      exec_DB_script(@GetPdbConnStmts, "XXXXXX",
                     $DoneCmd, $DoneFilePathBase, "sqlplus");

   if (!@$rows_ref || $#$rows_ref < 0) 
   {
      print_exec_DB_script_output("get_pdb_connect_string", $Spool_ref, 1);
      return 1;
   }

   for my $row ( @$rows_ref ) 
   {
      my $ConnString;
      if ($hostname and $port)
      {
         $ConnString = sprintf($StringTemplate, $hostname, $port, $row);
      }
      else
      {
         $ConnString = sprintf($StringTemplate, $ENV{ORACLE_SID}, $row);
      }
      push @Link, $ConnString;
   }

   if ($#Link ne $#$Containers)
   {
      log_msg("get_pdb_connect_string: invalid PDB Connecting Strings\n");
      return 1;
   }

   # If rmanconnTemplate is like user/passwd, append @%s to it, otherwise 
   # we are good to go
   if (index($rmanconn, "@%s") == -1)
   {
      $rmanconnTemplate = $rmanconn."@%s"; 
      $rmanconnTemplateDiag = $rmanconnDiag."@%s"; 
   }
   else
   {
      $rmanconnTemplate = $rmanconn;
      $rmanconnTemplateDiag = $rmanconnDiag;
   }

   foreach (@Link)
   {
      push @$PDBConnString, sprintf($rmanconnTemplate, $_);
      push @$PDBConnStringDiag, sprintf($rmanconnTemplateDiag, $_);
   }

   return 0;
}


##############################################################################
# End of Helper Function Section
##############################################################################


# subroutines which may be invoked by callers outside this file and variables 
# that need to persist across such invocations
{
   # indicator of whether rmanInit() is called.
   my $rman_InitDone;

   # DebugOn indicator in a variable that will persist across calls
   my $rman_DebugOn;

   # Verbose indicator in a variable that will persist across calls
   my $rman_Verbose;

   # indicator whether to use all RAC instacnes 
   # that will persist across calls
   my $rman_AllInstances;

   # hash mapping names of instances to
   # - number of RMAN processes that can be allocated for that instance
   #   ($rman_InstProcMap{$inst}->{NUM_PROCS})
   # - offset of the id of the first such process in @catcon_ProcIds
   #   ($rman_InstProcMa {$inst}->{FIRST_PROC})
   # if we are processing PDBs using all available instances
   my %rman_InstProcMap;

   # Databae hostname needed to construct connection string for each PDBs
   my $rman_hostname;

   # Databae port number needed to construct connection string for each PDBs
   my $rman_port;

   # an indicator of whether we are being invoked from a GUI tool which on 
   # Windows means that the passwords and hidden parameters need not be hidden
   my $rman_GUI;

   # indicator of whether non-existent and closed PDBs should be ignored when 
   # constructing a list of Containers against which to run scripts
   my $rman_IgnoreInaccessiblePDBs;

   # name of a Root of a Federation against members of which script(s) are 
   # to be run;
   my $rman_FedRoot;

   # name of EZ connect strings that include all instances provided by caller
   my $rman_EZConnect;

   # Number of RMAN processes
   my $rman_NumProcesses;

   # array of file handle ref for RMAN processes
   my @rman_FileHandles;
 
   # array of RMAN proc ids
   my @rman_ProcIds;

   # set source directory
   my $rman_SrcDir;

   # set LogDir
   my $rman_LogDir;

   # base for paths of log files
   my $rman_LogFilePathBase;

   # password used when connecting to the database to run sqlplus/rman script(s)
   my $rman_UserPass;

   # name of the Root if operating on a Consolidated DB
   my $rman_Root;

   # if running against PDB only
   my $rman_Rootonly;

   # empty if connected to a non-CDB; same as @rman_Containers if the 
   # user did not specify -c or -e flag, otherwise, consists of 
   # names of Containers explicitly included (-c) or not explicitly 
   # excluded (-e);
   # NOTE:
   #   by default, rmanExec will run scripts/statements in all Containers 
   #   whose names are found in rman_Containers;
   my @rman_AllContainers;

   # names of ALL containers of a CDB if connected to one; empty if connected 
   # to a non-CDB;
   my @rman_Containers;

   # indicator of whether each PDB is invoked by RMAN in available processes.
   # 0 indicates never used, 1 indicates ready to accept scirpts/statemtns, 
   # 2 indicates currently running scripts, 3 means job completed.
   my @rman_PDBStatus;

   # Array of length ProcNum, to map a process to the offset of PDBStatus,
   # used to swith processes to a unprocessed PDB
   my @rman_PDB_Pointer;

   # indicators (Y or N) of whether a corresponding Container in 
   # @rman_AllContainers is open
   my @rman_IsConOpen;

   # Final connection string to use 
   my $rman_ConnectString_SQL;
   my $rman_ConnectString_RMAN;

   # Final connection string to use, hide password for logs 
   my $rman_ConnectStringDiag_SQL;
   my $rman_ConnectStringDiag_RMAN;

   # connect string used for SQL queries, may contain %s placeholders
   my $rman_UserConnectString_SQL;

   # connect string used for SQL queries, may contain %s placeholders
   # password hiddent for logs
   my $rman_UserConnectStringDiag_SQL;

   # connect string used when running rman script(s) or rman statement(s)
   # may contain %s placeholders
   my $rman_UserConnectString_RMAN;

   # connect string used when running rman script(s) or rman statement(s)
   # may contain %s placeholders, password hidden for logs
   my $rman_UserConnectStringDiag_RMAN;

   # list of PDB connect string to open rman
   my @rman_PDBConnectString_RMAN;
   my @rman_PDBConnectStringDiag_RMAN;

   # this hash will be used to save signal handlers in effect before rmanInit 
   # was invoked so that they can be restored at the end of rmanWrapup
   my %rman_SaveSig;

   # are we running on Windows?
   my $rman_Windows;
   my $rman_DoneCmd_SQL;
   my $rman_DoneCmd_RMAN;

   # environment variable containing user password
   my $USER_PASSWD_ENV_TAG = "RMANUSERPASSWD";

   # environment variable containing EZConnect Strings for instances to be 
   # used to run scripts
   my $EZCONNECT_ENV_TAG = "RMANEZCONNECT";

   #
   # rmanRootonly- set a flag indicating whether we should run scripts only in 
   # PDBs, otherwise run scripts in all containers including CDB$ROOT and 
   # CDB$SEED. 
   #
   # Parameters:
   #   - value to assign to $rman_Rootonly
   #
   sub rmanRootonly($) 
   {
     my ($Rootonly) = @_;

     $rman_Rootonly= $Rootonly;
   }

   #
   # rmanIgnore - set a flag indicating whether we should ignore closed or 
   # non-existent PDBs when constructing a list of Containers against which to 
   # run scripts
   #
   # Parameters:
   #   - value to assign to $rman_IgnoreInaccessiblePDBs
   #
   sub rmanIgnore($) 
   {
     my ($Ignore) = @_;

     # remember whether the caller wants us to ignore closed and 
     # non-existent PDBs 
     $rman_IgnoreInaccessiblePDBs = $Ignore;
   }

   #
   # rmanFedRoot - store name of a Root of a Federation against members of 
   #                 which script(s) are to be run
   #
   # Parameters:
   #   - value to assign to $rman_FedRoot
   #
   sub rmanFedRoot ($) 
   {
     my ($FedRoot) = @_;

     $rman_FedRoot = $FedRoot;
   }

   # 
   # rmanEZConnect - store a string consisting of EZConnect strings 
   # corresponding to RAC instances to be used to run scripts
   # 
   # Parameters:
   #   - value to assign to $rman_EZConnect
   # 
   sub rmanEZConnect ($) 
   {
     my ($EZConnect) = @_;
    
     $rman_EZConnect = $EZConnect;
   } 

   # 
   # rmanVerbose- set rman_Verbose parameter 
   # 
   # Parameters:
   #   - value to assign to $rman_Verbose
   # 
   sub rmanVerbose ($) 
   {
     my ($Verbose) = @_;
    
     $rman_Verbose = $Verbose;
   } 

   # 
   # rmanAllInst- set rman_AllInstances parameter 
   # 
   # Parameters:
   #   - value to assign to $rman_AllInstances
   # 
   sub rmanAllInst($) 
   {
     my ($AllInst) = @_;
    
     $rman_AllInstances = $AllInst;
   } 

   # 
   # rmanHostPort- store a string consisting of Database Hostname and Ports 
   # 
   # Parameters:
   #   - value to assign to $rman_hostname, $rman_port
   # 
   sub rmanHostPort($$$) 
   {
     my ($hostname, $port, $debug) = @_;
    
     if ($hostname and $port)
     {
        $rman_hostname = $hostname;
        $rman_port = $port;

        if ($debug)
        {
           print STDERR "rman.pl: host name and port number specified, ".
                        "use TCP connection to containers ".
                        "if the database is consolidated\n";
        }
     }
     elsif(!$hostname and !$port)
     {
        $rman_hostname = '';
        $rman_port = '';

        if ($debug)
        {
           print STDERR "rman.pl: neither host name nor ".
                        "port number specified, ".
                        "use IPC connection to containers ".
                        "if the database is consolidated\n";
        }
     }
     else
     {
        $rman_hostname = '';
        $rman_port = '';
        die "rman.pl: only one of host name and port number specified, ".
            "NOT permitted\n";
     }
   } 

   #
   # rmanInit - initialize and validate rman static vars and start 
   #            RMAN processes
   #    
   # Parameters:
   #   - user name, optionally with password, supplied by the caller; may be 
   #     undefined (default / AS SYSDBA)
   #   - directory containing rman script(s) to be run; may be undefined
   #   - directory to use for spool and log files; may be undefined
   #   - base for spool log file name; must be specified
   #
   #   - indicator if only running in Root
   #   - container(s) (names separated by one or more spaces) in which to run 
   #     rman script(s) and rman statement(s) (i.e. skip containers not 
   #     referenced in this list)
   #   - container(s) (names separated by one or more spaces) in which NOT to 
   #     run rman script(s) and rman statement(s) (i.e. skip containers
   #     referenced in this list)
   #     NOTE: the above 2 args are mutually exclusive; if neither is defined, 
   #       rman script(s) and rman statement(s) will be run in the 
   #       non-Consolidated Database or all Containers of a Consolidated 
   #       Database
   #
   #   - number of processes to be used to run rman script(s) and rman 
   #     statement(s);  may be undefined (default value will be computed based 
   #     on host's hardware characteristics, number of concurrent sessions, 
   #     and whether the subroutine is getting invoked interactively)
   #   - external degree of parallelism, i.e. if more than one script using 
   #     rman will be invoked simultaneously on a given host, this parameter 
   #     should contain a number of such simultaneous invocations; 
   #     may be undefined;
   #     will be used to determine number of processes to start;
   #     this parameter MUST be undefined or set to 0 if the preceding 
   #     parameter (number of processes) is non-zero
   #   - indicator of whether to produce debugging messages; defaults to FALSE
   #   - an indicator of whether we are being called from a GUI tool which on 
   #     Windows means that the passwords and hidden parameters need not be 
   #     hidden
   sub rmanInit ($$$$$$$$$$$) 
   {
      my ($User, $SrcDir, $LogDir, $LogBase, $RootOnly,
          $ConNamesIncl, $ConNamesExcl,
          $NumProcesses, $ExtParallelDegree,
          $DebugOn, $GUI) = @_;

      if ($rman_InitDone) 
      {
         # if rmanInit has already been called 
         print STDERR  <<msg;
rmanInit: script execution state has already been initialized; call 
    rmanWrapUp before invoking rmanInit
msg
         return 1;
      }

      # make STDERR "hot" so diagnostic and error message output does not get 
      # buffered
      select((select(STDERR), $|=1)[0]);

      # set Log File Path Base
      $rman_LogFilePathBase =
         set_log_file_base_path ($LogDir,$LogBase, $DebugOn);

      # export $RMANOUT to catcon.pm
      catconImportRmanVariables($RMANOUT, $DebugOn); 

      # check to make sure we have a Log File Path Base
      if (!$rman_LogFilePathBase) 
      {
         # use STDERR because RMANOUT is yet to be opened
         print STDERR "rmanInit: Unexpected error returned ".
                      "by set_log_file_base_path\n";
         return 1;
      }

      # Set Log directory
      $rman_LogDir = $LogDir;
         
      my $ts = TimeStamp();

      if ($DebugOn || $rman_Verbose)
      {
         log_msg("rmanInit: base for log and spool file ".
                 "names = $rman_LogFilePathBase\n");
      }
 
      # save DebugOn indicator in a variable that will persist across calls
      $rman_DebugOn = $DebugOn;

      my $UserDiag;
      # Hide password for log output
      if ($User =~ /(.*)\/(.*)/)
      {
         $UserDiag = $1."/######";
      }
      else
      {
         $UserDiag = $User;
      }

      if ($rman_DebugOn)
      {
         log_msg <<rmanInit_DEBUG;
  running rmanInit(User              = $UserDiag, 
                   SrcDir            = $SrcDir, 
                   LogDir            = $LogDir, 
                   LogBase           = $LogBase,
                   ConNamesIncl      = $ConNamesIncl, 
                   ConNamesExcl      = $ConNamesExcl, 
                   NumProcesses      = $NumProcesses,
                   ExtParallelDegree = $ExtParallelDegree,
                   Debug             = $DebugOn,
                   GUI               = $GUI)\t\t($ts)

rmanInit_DEBUG
      }

      my $UnixDoneCmd_SQL = "\nhost sqlplus -v >";
      my $WindowsDoneCmd_SQL = "\nhost sqlplus/nolog -v >";

      # RMAN requires quotes in host commands
      my $UnixDoneCmd_RMAN = qq#host "sqlplus -v > %s";#;
      my $WindowsDoneCmd_RMAN = qq#host "sqlplus/nolog -v > %s";#;
    
      # will contain an indicator of whether a DB is a CDB (v$database.cdb)
      my $IsCDB;

      # base for log file names must be supplied
      if (!$LogBase)
      {
         log_msg("rmanInit: Base for log file names must be supplied");
         return 1;
      }

      # if caller indicated that we are to start a negative number of processes,
      # it is meaningless, so replace it with 0
      if ($NumProcesses < 0)
      {
         $NumProcesses = 0;
      }

      if ($NumProcesses && $ExtParallelDegree)
      {
         log_msg <<msg;
rmanInit: you may not specify both the number of processes ($NumProcesses) 
    and the external degree of parallelism ($ExtParallelDegree)
msg
         return 1;
      }
      
      if ($NumProcesses > 0)
      {
         $rman_NumProcesses = $NumProcesses;
      }

      if ($rman_AllInstances)
      {
         if (!$rman_hostname)
         {
            log_msg <<msg;
rmanInit: you may not specify allinstances option without specifying 
    SCAN host name
msg
            return 1;
         }
      }

      # set up signal handler in case SQL/RMAN process crashes
      # before it completes its work
      # 
      # Bug 18488530: add a handler for SIGINT
      # Bug 18548396: add handlers for SIGTERM and SIGQUIT
      # 
      # save original handlers for SIGCHLD, SIGINT, SIGTERM, and SIGQUIT before 
      # resetting them
      if (exists $SIG{CHLD})
      {
         $rman_SaveSig{CHLD} = $SIG{CHLD};
      }

      if (exists $SIG{INT})
      {
         $rman_SaveSig{INT}  = $SIG{INT};
      }

      if (exists $SIG{TERM})
      {
         $rman_SaveSig{TERM} = $SIG{TERM};
      }

      if (exists $SIG{QUIT})
      {
         $rman_SaveSig{QUIT} = $SIG{QUIT};
      }

      #$SIG{CHLD} = \&rman_HandleSigchld;
      $SIG{INT} = \&rman_HandleSigINT;
      $SIG{TERM} = \&rman_HandleSigTERM;
      $SIG{QUIT} = \&rman_HandleSigQUIT;

      # figure out if we are running under Windows and set $rman_DoneCmd
      # accordingly
      $rman_DoneCmd_SQL = ($rman_Windows = ($OSNAME =~ /^MSWin/)) ? 
                           $WindowsDoneCmd_SQL : $UnixDoneCmd_SQL;

      $rman_DoneCmd_RMAN = ($rman_Windows = ($OSNAME =~ /^MSWin/)) ? 
                            $WindowsDoneCmd_RMAN : $UnixDoneCmd_RMAN;

      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanInit: running on $OSNAME; SQL DoneCmd = ".
                 "$rman_DoneCmd_SQL RMAN DoneCmd = $rman_DoneCmd_RMAN\n");
      }

      # unset TWO_TASK  or LOCAL if it happens to be set to ensure that CONNECT
      # which does not specify a service results in a connection to the Root
      if ($rman_Windows) 
      {
         if ($ENV{LOCAL})
         {
            if ($rman_DebugOn || $rman_Verbose) 
            {
               log_msg("rmanInit: LOCAL was set to $ENV{LOCAL} - ".
                       "unsetting it\n");
            }

            delete $ENV{LOCAL};
         }
         else
         {
            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanInit: LOCAL was not set, so there is ".
                       "not need to unset it\n");
            }
         }
      }

      if ($ENV{TWO_TASK})
      {
         if ($rman_DebugOn || $rman_Verbose) 
         {
            log_msg("rmanInit: TWO_TASK was set to $ENV{TWO_TASK} - ".
                    "unsetting it\n");
         }

         delete $ENV{TWO_TASK};
      }
      else
      {
         if ($rman_DebugOn || $rman_Verbose) 
         {
            log_msg("rmanInit: TWO_TASK was not set, so there is ".
                    "no need to unset it\n");
         }
      }

      #ezConnection String list
      my @ezConnStrings;

      if ($rman_EZConnect || $ENV{$EZCONNECT_ENV_TAG})
      {
         if ($rman_EZConnect)
         {
            @ezConnStrings = split(/\s+/,$rman_EZConnect);

            if ($#ezConnStrings < 0)
            {
               log_msg("rmanInit: EZConnect string list must contain at least ".
                       "1 string\n");
               return 1;
            }

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanInit: EZConnect strings supplied by the user:\n\t");
               log_msg(join("\n\t", @ezConnStrings)."\n");
            }
         }
         else
         {
            @ezConnStrings = split(/\s+/, $ENV{$EZCONNECT_ENV_TAG});

            if ($#ezConnStrings < 0)
            {
               log_msg("rmanInit: $EZCONNECT_ENV_TAG environment variable ".
                       "must contain at least 1 string\n");
               return 1;
            }

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanInit: EZConnect strings found in ".
                       "$EZCONNECT_ENV_TAG env var");
               log_msg(join("\n\t", @ezConnStrings));
            }
         }
      }
      else
      {
         # having $ezConnStrings[0] eq "" serves as an indicator that we will
         # use the default instance
         push @ezConnStrings, "";

         if ($rman_DebugOn || $rman_Verbose) 
         {
            log_msg("rmanInit: no EZConnect strings supplied - ".
                    "using default instance\n");
         }
      }

      # NOTE: If ezConnStrings does not represent the default instance, 
      # $rman_UserConnectString will contain a placeholder (@%s) for 
      # the EZConnect string corresponding to the instance which we will 
      # eventually pick.
      ($rman_UserConnectString_SQL, $rman_UserConnectStringDiag_SQL,
       $rman_UserConnectString_RMAN, $rman_UserConnectStringDiag_RMAN,
       $rman_UserPass) =
        get_connect_string($User, !($rman_Windows && $GUI),
                           $ENV{$USER_PASSWD_ENV_TAG},
                           $ezConnStrings[0] ne "");

      if (!$rman_UserConnectString_SQL or !$rman_UserConnectString_RMAN)
      {
         # NOTE: $rman_UserConnectString may be set to undef
         # if the caller has not supplied a value 
         # for $USER (which would lead get_connect_string() to return 
         # "/ AS SYSDBA" as the connect string while specifying instances 
         # on which scripts are to be run (by means of passing one or more 
         # EZConnect strings)
         log_msg("rmanInit: Empty user connect string returned ".
                 "by get_connect_string\n");
         return 1;
      }

      if ($rman_DebugOn) 
      {
         log_msg("rmanInit: User SQL Connect String = ".
                 "$rman_UserConnectStringDiag_SQL\n");
         log_msg("rmanInit: User RMAN Connect String = ".
                 "$rman_UserConnectStringDiag_RMAN\n");
         log_msg("rmanInit: User password ".($rman_UserPass ? "= ".
                 "######" : "not specified")."\n");
      }

      if (!valid_src_dir($rman_SrcDir = $SrcDir))
      {
         log_msg("rmanInit: Unexpected error returned by valid_src_dir\n");
         return 1;
      }

      if ($rman_DebugOn || $rman_Verbose) 
      {
         if ($rman_SrcDir)
         {
            log_msg("rmanInit: source file directory = $rman_SrcDir\n");
         }
         else 
         {
            log_msg("rmanInit: no source file directory was specified\n");
         }
      }

      # check if sqlplus and rman be in $PATH
      if (!find_in_path("sqlplus", $rman_Windows, undef))
      {
         log_msg("rmanInit: sqlplus not in PATH.\n");
         return 1;
      }

      if (!find_in_path("rman", $rman_Windows, undef))
      {
         log_msg("rmanInit: rman not in PATH.\n");
         return 1;
      }

      # prefix of "done" file name
      my $doneFileNamePrefix =
         done_file_name_prefix($rman_LogFilePathBase, $$);

      # For each EZConnect String (or "" for default instance) stored in 
      # @ezConnStrings, connect to the instances represented by the string and
      # - verify that the database is open/mounted on that instance, 
      #   if open/mounted then pick this instance.
      #   - remember that this is the instance which we will be using
      #
      # - determine whether the database is a CDB 
      # - if the EZConnect string takes us to a CDB, 
      #   - verify that it takes us to the CDB's Root
      #       - determine the set of Containers against which scripts will be 
      #         run, honoring the caller's directive regarding reporting of 
      #         errors due to some PDBs not being accessible
      #       - if Federation Root is specified then process that and filter
      #         out all needed containers
      #               
      #   - determine how many processes may be started on it
      # - else (i.e. not connecting to a CDB)
      #   - determine the number of processes that can be started on this 
      # - endif

      # EZConnect string corresponding to the instance which we picked to run 
      # scripts and its instance status and name. 
      my $ezConnToUse;
      my @connectString; #SQL connect string to get CON info
      my $instanceStatus;
      my $instanceName;

      for (my $currInst = 0; $currInst <= $#ezConnStrings; $currInst++)
      {
         my $EZConnect = $ezConnStrings[$currInst];

         if ($rman_DebugOn)
         {
            log_msg("rmanInit: considering EZConnect string ".$EZConnect."\n");
         }

         my @curconnectString;
         $curconnectString[0] =
            build_connect_string($rman_UserConnectString_SQL, 
                                 $EZConnect,
                                 0);
         $curconnectString[1] =
            build_connect_string($rman_UserConnectStringDiag_SQL, 
                                 $EZConnect,
                                 $rman_DebugOn);

         if (!$curconnectString[0])
         {
            log_msg("rmanInit: unexpected error encountered in ".
                    "build_connect_string\n");
            return 1;
         }

         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanInit: call get_instance_status_and_name\n");
         }

         # status and name of the instance (v$instance.status and 
         # v$instance.instance_name)
         my ($curinstanceStatus, $curinstanceName) =
            get_instance_status_and_name(@curconnectString, $rman_DoneCmd_SQL,
                                         $doneFileNamePrefix, "sqlplus");

         if (!$curinstanceStatus)
         {
            log_msg("rmanInit: unexpected error in ".
                    "get_instance_status_and_name\n");
            return 1;
         }
         elsif ($curinstanceStatus !~ /^(OPEN|MOUNTED|STARTED)/)
         {
            log_msg("rmanInit: database is not open, mounted or nomounted");

            if ($EZConnect eq "")
            {
               # default instance
               log_msg(" on the default instance (".$curinstanceName.")\n");
            }
            else
            {
               log_msg(" on instance ".$curinstanceName.
                       " with EZConnect string = ".$EZConnect."\n");
            }
         }
         else
         {
            $ezConnToUse   = $EZConnect;
            @connectString = @curconnectString;
            $instanceName  = $curinstanceName;
         }

         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanInit: instance $curinstanceName (EZConnect = "
                    .$EZConnect.") has status $curinstanceStatus\n");
         }

         if (defined $ezConnToUse)
         {
            if ($rman_DebugOn || $rman_Verbose)
            {

               log_msg("rmanInit: Pick instance $curinstanceName (EZConnect = "
                       .$EZConnect.") since it is $curinstanceStatus\n");
            }
            last;
         }
      }

      if (!(defined $ezConnToUse))
      {
         log_msg("rmanInit: No available instance to execute RMAN, none of ".
                 "which is open or mounted\n");
         return 2;
      }
 
      # determine whether a DB is a CDB.
      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanInit: processing instance ".
                 "- call get_CDB_indicator\n");
      }
  
      $IsCDB =
         get_CDB_indicator(@connectString, $rman_DoneCmd_SQL,
                           $doneFileNamePrefix, "sqlplus");
  
      if (!(defined $IsCDB))
      {
         log_msg("rmanInit: unexpected error in get_CDB_indicator\n");
         return 1;
      }

      if ($rman_DebugOn || $rman_Verbose) 
      {
         log_msg("rmanInit: database is ".
                 (($IsCDB eq 'NO') ? "non-" : "")."Consolidated\n");
      }

      # CDB-specific stuff (see above)
      if ($IsCDB eq "YES")
      {
         if ($rman_DebugOn || $rman_Verbose) 
         {
            log_msg("rmanInit: CDB-specific processing\n");
         }

         # make sure that the current EZConnect string will take us 
         # to the Root.
         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanInit: call get_con_id to verify that the ".
                    "current EZConnect string will take us to the Root\n");
         }

         my $conId = get_con_id(@connectString, $rman_DoneCmd_SQL,
                                $doneFileNamePrefix, "sqlplus");

         if (!(defined $conId))
         {
            # con_id could not be obtained
            log_msg("rmanInit: unexpected error in get_con_id\n");
            return 1;
         }

         if ($conId != 1)
         {
            log_msg <<msg;
rmanInit: Instance $instanceName (EZConnect string = $ezConnToUse)
    points to a Container with CON_ID of $conId instead of the Root
msg
            return 1;
         }

         # indicators of which Containers are open on this instance
         my @isConOpen;

         if (!@rman_AllContainers)
         {
            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanInit: call get_con_info to obtain ".
                       "container names and open indicators\n");
            }

            if (get_con_info(@connectString, $rman_DoneCmd_SQL,
                             $doneFileNamePrefix, @rman_AllContainers,
                             @isConOpen, $rman_DebugOn)) 
            {
               log_msg("rmanInit: unexpected error in get_con_info\n");
             return 1;
            }

            # save name of the Root (it will always be in the 0-th element of
            # @rman_AllContainers because its contents are sorted by 
            # v$containers.con_id)
            $rman_Root = $rman_AllContainers[0];

            if (!$rman_Rootonly)
            {
               # if running PDBs only, discard 
               # the first 2 elements of @rman_AllContainers (CDB$ROOT and 
               # PDB$SEED) and corresponding elements of @isConOpen
               # if the CDB does not contain any other Containers, 
               # report error
               if ($#rman_AllContainers < 2)
               {
                  log_msg("rmanInit: cannot run user scripts against ".
                          "a CDB which contains no user PDBs\n");
                  return 1;
               }

               shift @rman_AllContainers;
               shift @rman_AllContainers;

               shift @isConOpen;
               shift @isConOpen;

               if ($rman_DebugOn)
               {
                  log_msg <<msg;
rmanInit: running in PDBonly mode, so purged the first 2 entries from 
    rman_AllContainers and isConOpen
msg
               }
            }
            else
            {
               if (!$rman_Root)
               {
                  log_msg("rmanInit: cannot find Root CDB\n");
                  return 1;
               }

               @rman_AllContainers = ($rman_Root); 
               @rman_Containers = ($rman_Root); 
               @isConOpen = ($isConOpen[0]);

               if ($rman_DebugOn)
               {
                  log_msg <<msg;
rmanInit: running in Rootonly mode, so extract first entry from 
    rman_AllContainers and isConOpen
msg
               }

               # If we run only in Root CDB, then no need to filer PDBs,
               # such as finding FedRoot, include/exclude PDBs
               goto skipPDBFilter;
            }

            if ($ConNamesIncl && $ConNamesExcl)
            {
               log_msg <<msg;
rmanInit: both a list of Containers in which to run scripts and a list of 
    Containers in which NOT to run scripts were specified, which is disallowed
msg
               return 1;
            } 

            if ($rman_FedRoot)
            {
               # if Fedeeration Root name was supplied, verify that neither 
               #   inclusive nor exclusive list of Container name has been 
               #   supplied
               # - confirm that the specified Container exists and is, indeed, 
               #   a Federation Root
               # - construct a string consisting of the name of the specified 
               #   Federation Root and all Federation PDBs belonging to it and 
               #   set $ConNamesIncl to that string (as if it were specified by 
               #   the caller, so we can take advantage of existing code) 
               if ($ConNamesIncl || $ConNamesExcl)
               {
                  log_msg <<msg;
rmanInit: Federation Root name and either a list of Containers in which to 
    run scripts or a list of Containers in which NOT to run scripts were 
    specified, which is disallowed
msg
                  return 1;
               }

               if ($rman_DebugOn || $rman_Verbose)
               {
                  log_msg("rmanInit: confirm that $rman_FedRoot ".
                          "is a Federation Root\n");
               }

               for (my $curCon = 0; $curCon <= $#rman_AllContainers; $curCon++)
               {
                  if ($rman_AllContainers[$curCon] eq $rman_FedRoot) 
                  {
                     # specified Container appears to exist; next we will 
                     # - verify that the specified Container name refers to a 
                     #   Federation Root and obtain its CON ID
                     # - construct $ConNamesIncl by appending to the name of 
                     #   Federation Root names of all Federation PDBs 
                     #   belonging to this Federation Root (so we can take 
                     #   advantage of existing code handling caller-supplied 
                     #   inclusive Container name list)
                     if ($rman_DebugOn || $rman_Verbose)
                     {
                        log_msg <<msg;
rmanInit: Container specified as a Federation Root exists
    verify that it is a Federation Root
msg
                     }

                     my $IsFedRoot;
                     my $FedRootConId;

                     if (get_fed_root_info(@connectString, 
                                           $rman_DoneCmd_SQL,
                                           $doneFileNamePrefix,
                                           $rman_FedRoot, $IsFedRoot,
                                           $FedRootConId, "sqlplus"))
                     {
                        log_msg("rmanInit: unexpected error in ".
                                "get_fed_root_info\n");
                        return 1;
                     }

                     if (!$IsFedRoot)
                     {
                        log_msg <<msg;
rmanInit: Purported Federation Root $rman_FedRoot has disappeared
msg
                        return 1;
                     }

                     if ($IsFedRoot ne "YES")
                     {
                        log_msg <<msg;
rmanInit: Purported $rman_FedRoot is not Federation Root
msg
                        return 1;
                     }

                     if ($rman_DebugOn || $rman_Verbose)
                     {
                        log_msg <<msg;
rmanInit: Container $rman_FedRoot is indeed a Federation Root;
    calling get_fed_pdb_names
msg
                     }

                     my @FedPdbNames;

                     if (get_fed_pdb_names(@connectString, 
                                           $rman_DoneCmd_SQL,
                                           $doneFileNamePrefix,
                                           $FedRootConId, @FedPdbNames, 
                                           "sqlplus"))
                     {
                        log_msg("rmanInit: unexpected error in ".
                                "get_fed_pdb_names\n");
                        return 1;
                     }

                     # place Federation Root name at the beginning of 
                     # $ConNamesIncl
                     $ConNamesIncl = $rman_FedRoot;

                     # append Federation PDB names to $ConNamesIncl
                     for my $FedPdbName ( @FedPdbNames )
                     {
                        $ConNamesIncl .= " ".$FedPdbName;
                     }

                     if ($rman_DebugOn)
                     {
                        log_msg <<msg;
rmanInit: ConNamesIncl reset to names of Containers comprising 
    Federation rooted in $rman_FedRoot:
        $ConNamesIncl
msg
                     }
                  }
               }

               # if the specified Container was a Federation Root, $ConNamesIncl
               # should be set to a list consisting of the name of the 
               # Federation Root followed by the names of its Federation PDBs, 
               # but if that Container did not exist $ConNamesIncl will be 
               # still undefined
               if (!$ConNamesIncl)
               {
                  log_msg <<msg;
rmanInit: Purported Federation Root $rman_FedRoot does not exist
msg
                  return 1;
               }
            }

            if ($rman_DebugOn || $rman_Verbose) 
            {
               log_msg("rmanInit: Processing include container and ".
                       "exclude containers option\n");
            }

            if ($ConNamesExcl)
            {
               if (!(@rman_Containers =
                     validate_con_names($ConNamesExcl, 1,
                                        @rman_AllContainers, @isConOpen,
                                        $rman_IgnoreInaccessiblePDBs,
                                        $rman_DebugOn)))
               {
                  log_msg("rmanInit: Unexpected error returned ".
                          "by validate_con_names for exclusive ".
                          "Container list\n");
                  return 1;
               }
            } 
            elsif ($ConNamesIncl)
            {
               if (!(@rman_Containers =
                     validate_con_names($ConNamesIncl, 0,
                                        @rman_AllContainers, @isConOpen,
                                        $rman_IgnoreInaccessiblePDBs,
                                        $rman_DebugOn)))
               {
                  log_msg("rmanInit: Unexpected error returned ".
                          "by validate_con_names for inclusive ".
                          "Container list\n");
                  return 1;
               }
            }
            else
            {
               @rman_Containers = @rman_AllContainers;
            }

            # Record the open status of all Containers, but rman.pl at
            # present is not using it, just keep it for possible future
            # usage.
            @rman_IsConOpen = @isConOpen;
         }

skipPDBFilter:

         if (!@rman_Containers)
         {
            log_msg("rmanInit: Unable to initialize all containers, ".
                    "aborting\n");
            return 1;
         }

         if ($rman_DebugOn || $rman_Verbose) 
         {
            log_msg("rmanInit: Containers @rman_Containers will be ".
                    "processing RMAN scripts\n");
         }

         # determine the number of RMAN processes to be started.
         if (!$rman_NumProcesses)
         {
            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg <<msg;
rmanInit: call get_num_procs to determine number of processes that will
    be started on a CDB instance
msg
            }

            my $getProcsOnAllInstances =
               $IsCDB eq "YES" &&
               $rman_AllInstances &&
               index($connectString[0], "@") == -1;

            $rman_NumProcesses =
               get_num_procs($NumProcesses, $ExtParallelDegree,
                             \@connectString,
                             $rman_DoneCmd_SQL, $doneFileNamePrefix, 
                             $getProcsOnAllInstances, \%rman_InstProcMap, 
                             0, "sqlplus");

            # Reset ProcNum if it's greater than ContianerNum
            if ($rman_NumProcesses > scalar @rman_Containers)
            {
               $rman_NumProcesses = scalar @rman_Containers;
            }

            if ($rman_NumProcesses == -1)
            {
               log_msg("rmanInit: unexpected error in get_num_procs\n");
               return 1;
            }
            elsif ($rman_NumProcesses < 1)
            {
               log_msg("rmanInit: invalid number of processes ".
                       "($rman_NumProcesses) returned by get_num_procs\n");
               return 1;
            }
            elsif ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanInit: get_num_procs determined that ".
                       "$rman_NumProcesses processes can be started ".
                       "on this instance\n");
            }
         }
         else
         {
            # If User specified the number of Procs to be run
            # Reset ProcNum if it's greater than ContianerNum
            if ($rman_NumProcesses > scalar @rman_Containers)
            {
               $rman_NumProcesses = scalar @rman_Containers;
            }
         }
      }
      else
      {
         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanInit: non-CDB-specific processing\n");
         }

         # will run scripts against a non-CDB
         $rman_NumProcesses = 1;

         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanInit: Will start $rman_NumProcesses RMAN processes\n");
         }
      }

      # construct final connect string to root, including the user specified
      # instance
      $rman_ConnectString_SQL =
       build_connect_string($rman_UserConnectString_SQL, 
                            $ezConnToUse,
                            0);

      $rman_ConnectStringDiag_SQL =
       build_connect_string($rman_UserConnectStringDiag_SQL, 
                            $ezConnToUse,
                            $rman_DebugOn);
 
      # This string can be used to connect to root in RMAN, not for PDBs
      $rman_ConnectString_RMAN = 
       build_connect_string($rman_UserConnectString_RMAN, 
                            $ezConnToUse,
                            0);

      $rman_ConnectStringDiag_RMAN =
       build_connect_string($rman_UserConnectStringDiag_RMAN, 
                            $ezConnToUse,
                            $rman_DebugOn);

      if ($rman_DebugOn)
      {
         log_msg("rmanInit: Final RMAN connection string is ".
                 "$rman_ConnectStringDiag_RMAN\n");
         log_msg("rmanInit: Final SQL connection string is ".
                 "$rman_ConnectStringDiag_SQL\n");
      }

      # temporrarily push rman_UserConnectString_RMAN into 
      # rman_PDBConnectString_RMAN aray
      #if (@rman_Containers)
      #{
      #@rman_PDBConnectString_RMAN 
      #= ($rman_ConnectString_RMAN) x scalar(@rman_Containers); 
      #}
      #else
      #{
      #@rman_PDBConnectString_RMAN = ($rman_ConnectString_RMAN); 
      #}

      if (@rman_Containers and !$rman_Rootonly)
      {
         @rman_PDBConnectString_RMAN = (); 
          
         if (get_pdb_connect_string (@connectString, 
                                     $rman_UserConnectString_RMAN,
                                     $rman_UserConnectStringDiag_RMAN,
                                     $rman_hostname, $rman_port, 
                                     $rman_DoneCmd_SQL,
                                     $doneFileNamePrefix,
                                     @rman_Containers, 
                                     @rman_PDBConnectString_RMAN, 
                                     @rman_PDBConnectStringDiag_RMAN, 
                                     $rman_DebugOn)) 
         {
            log_msg("rmanInit: unexpected error in ".
                    "get_pdb_connect_string\n");
            return 1;
         }
      }
      else
      {
         @rman_PDBConnectString_RMAN = ($rman_ConnectString_RMAN); 
         @rman_PDBConnectStringDiag_RMAN = ($rman_ConnectStringDiag_RMAN); 
      }

      if (start_processes($rman_NumProcesses, $rman_LogFilePathBase,
                          @rman_FileHandles, @rman_ProcIds,
                          @rman_Containers, @rman_PDBStatus,
                          @rman_PDB_Pointer,
                          @rman_PDBConnectString_RMAN,
                          @rman_PDBConnectStringDiag_RMAN,
                          $rman_DebugOn, 1,
                          $rman_DoneCmd_RMAN))
      {
         log_msg("rmanInit: unexpected error in ".
                 "start_processes\n");
         return 1;
      }

      # remember whether we are being invoked from a GUI tool which on Windows 
      # means that the passwords and hidden parameters need not be hidden
      $rman_GUI = $GUI;

      # remember that initialization has completed successfully
      $rman_InitDone = 1;

      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanInit: initialization completed successfully (".
                 TimeStamp().")\n");
      }

      # Workaround for bug 18969473
      #if ($rman_Windows)
      #{
      #   open(STDIN, "<", "NUL") || die "open NUL: $!";
      #}
      return 0;
   }

   # rmanExec - run specified RMAN script(s) or RMAN statements
   #
   # If connected to a non-Consolidated DB, each script will be executed 
   # using one of the processes connected to the DB.
   #
   # If connected to a Consolidated DB and the caller requested that all 
   # scripts and RMAN statements be run against the Root (possibly in addition 
   # to other Containers), each script and statement will be executed in the 
   # Root using one of the processes connected to the Root.
   #
   # If connected to a Consolidated DB and were asked to run scripts and RMAN 
   # statements in one or more Containers with/without the Root, all scripts 
   # statements will be run against those PDBs in parallel.
   #
   # Parameters:
   #   - a reference to an array of RMAN script name(s) or RMAN statement(s); 
   #     script names are expected to be prefixed with @
   #   - an indicator of whether scripts or RMAN statements need to be run only 
   #     in the Root if operating on a CDB 
   #       TRUE => if operating on a CDB, run in Root only (NOT USED NOW)
   #   - a reference to a list of names of Containers in which to run scripts 
   #     and statements during this invocation of rmanExec; does not 
   #     overwrite @rman_Containers so if no value is supplied for this or 
   #     the next parameter during subsequent invocations of rmanExec, 
   #     @rman_Containers will be used
   #
   #     NOTE: This parameter should not be defined if
   #           - connected to a non-CDB
   #           - caller told us to run statements/scripts only in the Root
   #           - caller has supplied us with a list of names of Containers in 
   #             which NOT to run scripts
   #
   #   - a reference to a list of names of Containers in which NOT to run 
   #     scripts and statements during this invocation of rmanExec; does not 
   #     overwrite @rman_Containers so if no value is supplied for this or 
   #     the next parameter during subsequent invocations of rmanExec, 
   #     @rman_Containers will be used
   #
   #     NOTE: This parameter should not be defined if
   #           - connected to a non-CDB
   #           - caller told us to run statements/scripts only in the Root
   #           - caller has supplied us with a list of names of Containers in 
   #             which NOT to run scripts
   #
   #   - a query to run within a container to determine if the array should
   #     be executed in that container.  A NULL return value causes the
   #     script array to NOT be run.
 
   sub rmanExec(\@$\$\$) 
   {
      my ($StuffToRun, $RootOnly, 
          $ConNamesIncl, $ConNamesExcl) = @_;

      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanExec:\n\tScript names/RMAN statements:\n");

         foreach (@$StuffToRun)
         {
            log_msg("\t\t$_\n");
         }

         log_msg("\tRootOnly              = $RootOnly\n");

         if ($$ConNamesIncl)
         {
            log_msg("\tConNamesIncl          = $$ConNamesIncl\n");
         }
         else
         {
            log_msg("\tConNamesIncl            undefined\n");
         }

         if ($$ConNamesExcl)
         {
            log_msg("\tConNamesExcl          = $$ConNamesExcl\n");
         }
         else
         {
            log_msg("\tConNamesExcl            undefined\n");
         }

         log_msg("\t(".TimeStamp().")\n");
      }

      # there must be at least one script or statement to run
      if (!@$StuffToRun || $#$StuffToRun == -1) 
      {
         log_msg("rmanExec: At least one RMAN script name or ".
                 "RMAN statement must be supplied\n");
         return 1;
      }

      # rmanInit had better been invoked
      if (!$rman_InitDone)
      {
         log_msg("rmanExec: rmanInit has not been run\n");
         return 1;
      }

      # a number of checks if either a list of Containers in which to run 
      # scripts or a list of Containers in which NOT to run scripts specified
      if ($$ConNamesIncl || $$ConNamesExcl) 
      {
         if (!@rman_Containers)
         {
            # Container names specified even though we are running against a 
            # non-CDB
            log_msg("rmanExec: Container names specified for a non-CDB\n");
            return 1;
         }

         if ($rman_Rootonly)
         {
            # Container names specified even though we were told to run ONLY 
            # against the Root
            log_msg("rmanExec: Container names specified while ".
                    "told to run only against the Root\n");
            return 1;
         }

         # only one of the lists may be defined
         if ($$ConNamesIncl && $$ConNamesExcl)
         {
            log_msg <<msg;
rmanExec: both inclusive
        $$ConNamesIncl
    and exclusive
        $$ConNamesExcl
    Container name lists are defined
msg
            return 1;
         }
      }

      # move signal handling here because SQL*PLUS execution function is called
      # from catcon.pm, and it has changed its behavior to bounce up dead proc
      # rather than die from Sigchld, but RMAN does not need to do it at the moment,
      # so skip CHLD signaling in SQLPULS but keep it when calling RMAN script
      $SIG{CHLD} = \&rman_HandleSigchld;
      $SIG{INT} = \&rman_HandleSigINT;
      $SIG{TERM} = \&rman_HandleSigTERM;
      $SIG{QUIT} = \&rman_HandleSigQUIT;

      return rmanExec_int($StuffToRun, $RootOnly,
                          $ConNamesIncl, $ConNamesExcl);
   }

   # NOTE: rmanExec_int() should ONLY be called via rmanExec which 
   #       performs error checking to ensure that rmanExec_int is not 
   #       presented with a combination of arguments which it is not prepared 
   #       to handle
   sub rmanExec_int (\@$\$\$) 
   {
      my ($StuffToRun, $RootOnly, 
          $ConNamesIncl, $ConNamesExcl) = @_;

      # script invocations (together with parameters) and/or SQL statements 
      # which will be executed
      my @ScriptPaths;

      my $NextItem;

      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanExec: validating scripts/statements ".
                 "supplied by the caller\n");
      }

      # variable will be set to true after we encounter a script name to remind 
      # us to check for possible arguments
      my $LookForArgs = 0;

      foreach $NextItem (@$StuffToRun)
      {
         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanExec: going over StuffToRun: ".
                    "NextItem = $NextItem\n");
         }

         if ($NextItem =~ /^@/)
         {
            # leading @ implies that $NextItem contains a script name
            # name of script
            my $FileName;

            if ($rman_DebugOn || $rman_Verbose) 
            {
               log_msg("rmanExec: next script name = $NextItem\n");
            }

            # strip off the leading @ before prepending source directory to 
            # script name
            ($FileName = $NextItem) =~ s/^@//;

            # validate path of the script and add it to @ScriptPaths
            my $Path =
               validate_script_path($FileName, $rman_SrcDir, 
                                    $rman_Windows, 0);

            if (!$Path)
            {
               log_msg <<msg;
rmanExec: empty Path returned by validate_script_path for 
    SrcDir = $rman_SrcDir, FileName = $FileName
msg
               return 1;
            }

            push @ScriptPaths, "@".$Path;

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanExec: full path = $Path\n");
            }

            # Look for possible arguments to the script 
            $LookForArgs = 1;
         }
         elsif ($NextItem =~ /^--p/)
         {
            if (!$LookForArgs)
            {
               log_msg("rmanExec: unexpected script argument ".
                       "($NextItem) encountered\n");
               return 1;
            }

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanExec: processing script argument ".
                       "string ($NextItem)\n");
            }
 
            my $Arg;

            ($Arg = $NextItem) =~ s/^--p//;

            # Append argument to the preceeding script name
            $ScriptPaths[$#ScriptPaths] .= " " .$Arg;

            if ($rman_DebugOn || $rman_Verbose) 
            {
               log_msg("rmanExec: added argument to script invocation ".
                       "string\n");
            }

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanExec: script invocation string constructed ".
                       "so far:\n\t$ScriptPaths[$#ScriptPaths]\n");
            }
         }
         else
         {
            # $NextItem must contain a RMAN statement which we will copy into 
            # @ScriptPaths

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanExec: next RMAN statement = $NextItem\n");
            }

            push @ScriptPaths, $NextItem;

            # we expect no arguments following a SQL statement
            $LookForArgs = 0;
         }
      }

      # if running against a non-Consolidated Database
      #   - each script/statement will be run exactly once; 
      # else 
      #   - if running against one or more PDBs
      #     - each script/statement will be run against PDBs in parallel

      # offset into the array of process file handles
      my $CurProc = 0;

      # compute number of processes which will be used to run script/statements 
      # specified by the caller; used to determine when all processes finished 
      # their work
      my $ProcsUsed;

      my @Containers;

      @Containers = @rman_Containers;

      # if connected to a CDB, save name of a CDB Root or FedRoot
      my $CDBorFedRoot;

      if (@Containers)
      {
         $CDBorFedRoot = ($rman_FedRoot) ? $rman_FedRoot : $rman_Root;

         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanExec: run all scripts/statements against PDBs\n");
         }

         my $idx;

         $CurProc = -1;

         for ($idx= 0; $idx<= $#Containers; $idx++)
         {
            $CurProc = pickNextProc($rman_NumProcesses, $rman_NumProcesses, 
                                    $CurProc + 1, $rman_LogFilePathBase ,
                                    @rman_PDBStatus, @rman_PDB_Pointer, 
                                    @rman_PDBConnectString_RMAN, 
                                    @rman_PDBConnectStringDiag_RMAN, 
                                    @rman_FileHandles, @rman_ProcIds,
                                    $rman_DoneCmd_RMAN, $rman_Windows,
                                    $rman_DebugOn);

            if ($CurProc == -1)
            {
               log_msg("rmanExec: failed to find the available process\n");
               return -1;
            }

            my $fh = $rman_FileHandles[$CurProc];

            foreach my $FilePath (@ScriptPaths) 
            {
               if ($rman_DebugOn || $rman_Verbose)
               {
                  log_msg("rmanExec: running scripts/statements ".
                          "$FilePath in Container ".
                          "$Containers[$rman_PDB_Pointer[$CurProc]]\n");
               }

               printToRman("rmanExec", $fh, $FilePath,
                           "\n", $rman_DebugOn);
            }

            # we need a "done" file to be created after the last 
            # script or statement sent to the current Container 
            # completes 
            my $DoneFile =
            done_file_name($rman_LogFilePathBase, $rman_ProcIds[$CurProc]);

            if (! -e $DoneFile)
            {
               my $DoneCmdFile = sprintf($rman_DoneCmd_RMAN, $DoneFile);
               # "done" file does not exist - cause it to be created
               printToRman("create_done_files", $rman_FileHandles[$CurProc],
                           $DoneCmdFile, "\n", $rman_DebugOn);

               $rman_FileHandles[$CurProc]->flush;

               if ($rman_DebugOn || $rman_Verbose)
               {
               log_msg <<msg;
rmanExec: sent "$rman_DoneCmd_RMAN $DoneFile" to process 
    $CurProc (id = $rman_ProcIds[$CurProc]) to indicate its availability
msg
               }
            }
            elsif (! -f $DoneFile)
            {
               log_msg(
               qq#rmanExec: "done" file name collision: $DoneFile\n#);

               return 1;
           }
           else
           {
              if ($rman_DebugOn)
              {
                 log_msg(
                 qq#rmanExec: "done" file $DoneFile already exists\n#);
              }
           }

         }

         if (wait_for_completion($rman_NumProcesses, $rman_NumProcesses,
                                 $rman_LogFilePathBase,
                                 @rman_FileHandles, @rman_ProcIds,
                                 @rman_PDBStatus, @rman_PDB_Pointer,
                                 $rman_DoneCmd_RMAN, $rman_Windows,
                                 $rman_DebugOn) == 1)
         {
            # unexpected error was encountered, return
            log_msg("rmanExec: unexpected error in wait_for_completions\n");
            return 1;
         }
         
      }

      if (!@Containers)
      {
         if ($rman_DebugOn || $rman_Verbose)
         {
            log_msg("rmanExec: run all scripts/statements against ".
                    "a non-Consolidated Database\n");
         }

         if (scalar @rman_FileHandles != 1)
         {
            log_msg("rmanExec: Only one process is expected to be started ".
                    "in non_Consolidated Database");
            return 1;
         }

         $CurProc = pickNextProc($rman_NumProcesses, $rman_NumProcesses, 0,
                                 $rman_LogFilePathBase ,
                                 @rman_PDBStatus, @rman_PDB_Pointer, 
                                 @rman_PDBConnectString_RMAN, 
                                 @rman_PDBConnectStringDiag_RMAN, 
                                 @rman_FileHandles, @rman_ProcIds,
                                 $rman_DoneCmd_RMAN, $rman_Windows,
                                 $rman_DebugOn );

         if ($CurProc == -1)
         {
            log_msg("rmanExec: failed to find the available process\n");
            return -1;
         }

         my $fh = $rman_FileHandles[$CurProc];

         foreach my $FilePath (@ScriptPaths) 
         {
            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg("rmanExec: running scripts/statements ".
                       "$FilePath to Database\n");
            }

            printToRman("rmanExec", $fh, $FilePath,
                        "\n", $rman_DebugOn);
         }

         # we need a "done" file to be created after the last 
         # script or statement sent to the current Container 
         # completes 
         my $DoneFile =
         done_file_name($rman_LogFilePathBase, $rman_ProcIds[$CurProc]);

         if (! -e $DoneFile)
         {
            my $DoneCmdFile = sprintf($rman_DoneCmd_RMAN, $DoneFile);
            # "done" file does not exist - cause it to be created
            printToRman("create_done_files", $rman_FileHandles[$CurProc],
                        $DoneCmdFile, "\n", $rman_DebugOn);

            $rman_FileHandles[$CurProc]->flush;

            if ($rman_DebugOn || $rman_Verbose)
            {
               log_msg <<msg;
rmanExec: sent "$rman_DoneCmd_RMAN $DoneFile" to process 
    $CurProc (id = $rman_ProcIds[$CurProc]) to indicate its availability
msg
            }
         }
         elsif (! -f $DoneFile)
         {
            log_msg(
            qq#rmanExec: "done" file name collision: $DoneFile\n#);

            return 1;
         }
         else
         {
            if ($rman_DebugOn)
            {
               log_msg(
                  qq#rmanExec: "done" file $DoneFile already exists\n#);
            }
         }

         if (wait_for_completion($rman_NumProcesses, $rman_NumProcesses,
                                 $rman_LogFilePathBase,
                                 @rman_FileHandles, @rman_ProcIds,
                                 @rman_PDBStatus, @rman_PDB_Pointer,
                                 $rman_DoneCmd_RMAN, $rman_Windows,
                                 $rman_DebugOn) == 1)
         {
            # unexpected error was encountered, return
            log_msg("rmanExec: unexpected error in wait_for_completions\n");
            return 1;
         }
      }

      return 0;
   }

   #
   # rmanWrapUp - free any resources which may have been allocated by 
   #                various rman.pl subroutines
   # 
   # Returns
   #   1 if some unexpected error was encountered; 0 otherwise
   #
   sub rmanWrapUp () 
   {
      # rmanInit had better been invoked
      if (!$rman_InitDone)
      {
         log_msg("rmanWrapUp: rmanInit has not been run");
         return 1;
      }

      if ($rman_DebugOn || $rman_Verbose) 
      {
         log_msg("rmanWrapUp: about to free up all resources\n");
      }

      # end processes
      end_processes(0, $rman_NumProcesses - 1, @rman_FileHandles,
                    @rman_ProcIds, $rman_DebugOn, $rman_LogFilePathBase);

      # restore signal handlers which were reset in rmanInit
      # NOTE: if, for example, $SIG{CHLD} existed but was not defined in 
      #      rmanInit, I would have liked to undef $SIG{CHLD} here, but that 
      #      results in "Use of uninitialized value in scalar assignment",
      #      so I was forced to delete the $SIG{} element instead

      if ((exists $rman_SaveSig{CHLD}) && (defined $rman_SaveSig{CHLD}))
      {
         $SIG{CHLD}  = $rman_SaveSig{CHLD};
      } 
      else 
      {
         delete $SIG{CHLD};
      }

      if ((exists $rman_SaveSig{INT}) && (defined $rman_SaveSig{INT}))
      {
         $SIG{INT}  = $rman_SaveSig{INT};
      }
      else
      {
         delete $SIG{INT};
      }

      if ((exists $rman_SaveSig{TERM}) && (defined $rman_SaveSig{TERM}))
      {
         $SIG{TERM}  = $rman_SaveSig{TERM};
      }
      else
      {
         delete $SIG{TERM};
      }

      if ((exists $rman_SaveSig{QUIT}) && (defined $rman_SaveSig{QUIT}))
      {
         $SIG{QUIT}  = $rman_SaveSig{QUIT};
      }
      else 
      {
         delete $SIG{QUIT};
      }

      $rman_InitDone = 0;

      if ($rman_DebugOn || $rman_Verbose)
      {
         log_msg("rmanWrapUp: done\n");
      }

      return 0;
   }


   ######################################################################
   #  If one of the child process terminates, it is a fatal error
   ######################################################################
   sub rman_HandleSigchld () 
   {
      log_msg <<msg;
A process terminated prior to completion.
    Review the ${rman_LogFilePathBase}*.log files to identify the failure
msg
      $SIG{CHLD} = 'IGNORE';  # now ignore any child processes
      rmanWrapUp();
      die;
   }

   sub rman_HandleSigINT () 
   {
      log_msg("Signal INT was received.\n");

      # reregister SIGINT handler in case we are running on a system where 
      # signal(3) acts in "the old unreliable System V way", i.e. clears the 
      # signal handler
      $SIG{INT} = \&rman_HandleSigINT;
      rmanWrapUp();
      die;
   }

   sub rman_HandleSigTERM () 
   {
      log_msg("Signal TERM was received.\n");

      # reregister SIGINT handler in case we are running on a system where 
      # signal(3) acts in "the old unreliable System V way", i.e. clears the 
      # signal handler
      $SIG{TERM} = \&rman_HandleSigTERM;
      rmanWrapUp();
      die;
   }

   sub rman_HandleSigQUIT () 
   {
      log_msg("Signal QUIT was received.\n");

      # reregister SIGINT handler in case we are running on a system where 
      # signal(3) acts in "the old unreliable System V way", i.e. clears the 
      # signal handler
      $SIG{QUIT} = \&rman_HandleSigQUIT;
      rmanWrapUp();
      die;
   }

   # For internal test only
   sub rmanTest ($) 
   {
      exit 0;
   } 

}

1;

OHA YOOOO