MINI MINI MANI MO

Path : /opt/oracle/product/18c/dbhomeXE/bin/
File Upload :
Current File : //opt/oracle/product/18c/dbhomeXE/bin/bdschecksw

#!/bin/bash
#
# $Header: rdbms/src/client/datapump/dml/bigsql/bdschecksw /main/2 2017/10/26 08:32:53 richen Exp $
#
# Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      bdschecksw - Oracle Big Data Sql CHECK SoftWare utility
#
#    DESCRIPTION
#
#    NOTES
#      This script terminates with an exit code of 0 on success, 1 or above on failure.
#
#    MODIFIED   (MM/DD/YY)
#    richen      10/12/17 - Check cell ip against db /etc/hosts for BDA
#    richen      10/10/17 - Fixed problems with db_unique_name, db_name and instance_name 
#    richen      09/14/17 - Bug#26734413
#    richen      08/30/17 - IPSec check
#    richen      07/27/17 - V3.2.1
#    richen      03/02/17 - Parallel processing for Hadoop cell nodes and misc.
#                           enhancements
#    richen      01/17/17 - Added PDB container and hidden password support
#    richen      11/03/16 - Option to turn on log4j and extproc tracing during testing
#    richen      08/03/16 - Creation.
#

##====================
#  Constants
##====================

RELEASE_NO=$(/bin/grep bdsversion $ORACLE_HOME/BDSJaguar*/*/database-install.env | /bin/cut -d= -f2 | /usr/bin/uniq | /bin/sed "/'/s///g")
BDSCHECKSW_BANNER="Big Data SQL Diagnostics Collection Tool v$RELEASE_NO"

TITLE="Oracle Big Data SQL System Information for $HOSTNAME"

BASENAME=$(basename $0)
OS_TYPE=`uname`
RIGHT_NOW=$(/bin/date +"%x %r %Z")
RIGHT_NOW2=$(/bin/date +"%Y%m%d-%H:%M:%S")
TIME_STAMP="Updated on $RIGHT_NOW by OS user $USER. $BASENAME v$RELEASE_NO"

CELLINIT="/etc/oracle/cell/network-config/cellinit.ora"
HADOOP_CELLINIT="/opt/oracle/cell/cellsrv/deploy/config/cellinit.ora"
CELLINITETH="/etc/oracle/cell/network-config/celliniteth.ora"
 
DB_TABLE_CNAME=('' 'DB Node' 'Directories, Files and Datapatch' 'Database Objects' 'Extproc' 'Prerequisite Software' 'Hadoop Cluster/Hive Connectivity' 'BDSQL Sanity Test' 'Kerberos' 'TLS/SSL/IPSec')

CELL_TABLE_CNAME=('' 'Hadoop Cell Node' 'BDSQL Sanity Test' 'oracle in oinstall group' 'bdscli Quarantine' 'bdscli Alert History' 'BDSQL Online' 'Java JCE Files' 'Kerberos' 'TLS/SSL/IPSec')

DB_TABLE_CVALUE_NA="N.A."
DB_TABLE_CVALUE_PASSED="Passed"
DB_TABLE_CVALUE_FAILED="Failed"
DB_TABLE_CVALUE_ON="ON"
DB_TABLE_CVALUE_OFF="OFF"

CELL_TABLE_CVALUE_NA="N.A."
CELL_TABLE_CVALUE_PASSED="Passed"
CELL_TABLE_CVALUE_FAILED="Failed"
CELL_TABLE_CVALUE_ON="ON"
CELL_TABLE_CVALUE_OFF="OFF"

DB_NODE_RPM_PACKAGE_1="oracle-instantclient12.1-basic-12.1.0.2.0-1.x86_64"
DB_NODE_RPM_PACKAGE_2="oracle-instantclient12.1-jdbc-12.1.0.2.0-1.x86_64"
DB_NODE_RPM_PACKAGE_3="perl-XML-LibXML-1.70-5.el6.x86_64"

DB_NODE_DEFAULT_DIR=
DB_NODE_BIGDATA_CONFIG=

CELL_XT_OFFLOAD_RETRIES="cell XT granule predicate offload retries"
CELL_XT_SMART_SCAN_BYTES="cell interconnect bytes returned by XT smart scan"

DB_DYNAMIC_PROTOCOL="_skgxp_dynamic_protocol"
CELL_DYNAMIC_PROTOCOL="_cell_big_data_sql_software_only"
CELL_AUTHENTICATION="_cell_big_data_sql_enable_authentication"

cpcmd='/bin/cp'
rmcmd='/bin/rm'
lncmd='/bin/ln'
touchcmd='/bin/touch'
catcmd='/bin/cat'
echocmd='/bin/echo'
chmodcmd='/bin/chmod'
hostnamecmd='/bin/hostname'
sedcmd='/bin/sed'
sleepcmd='/bin/sleep'
killcmd='/bin/kill'
grepcmd='/bin/grep'
lscmd='/bin/ls'
pscmd='/bin/ps'
awkcmd='/bin/awk'
cutcmd='/bin/cut'
sedcmd='/bin/sed'
rpmcmd='/bin/rpm'
basenamecmd='/bin/basename'
findcmd='/bin/find'
tarcmd='/bin/tar'
gzipcmd='/bin/gzip'
headcmd='/usr/bin/head'
tailcmd='/usr/bin/tail'
expectcmd='/usr/bin/expect'
klistcmd='/usr/bin/klist'
nmapcmd='/usr/bin/nmap'
trcmd='/usr/bin/tr'
getentcmd='/usr/bin/getent'
whoamicmd='/usr/bin/whoami'
bdaclicmd='/opt/oracle/bda/bin/bdacli'
ipcmd='/sbin/ip'
ifconfigcmd='/sbin/ifconfig'
mtactlcmd=
hadoopcmd=

##====================
#  Functions
##====================

function spinning_cursor
{
    pid=$! # Process Id of the previous running command

    spin='-\|/'

    i=0
    while $killcmd -0 $pid 2>/dev/null
    do
      i=$(( (i+1) %4 ))
      printf "\r${spin:$i:1}"
      $sleepcmd .1
    done
}   # end of spinning_cursor


function spinning_cursor2
{
    pid_list=$* # a list of pids of the running commands

    spin='-\|/'

    i=0
    for pid in $pid_list
    do
        while $killcmd -0 $pid 2>/dev/null
        do
          i=$(( (i+1) %4 ))
          printf "\r${spin:$i:1}"
          $sleepcmd .1
        done
    done
}   # end of spinning_cursor


function grid_lsinventory
{
    oracle_home_saved=$ORACLE_HOME
    ld_library_path_saved=$LD_LIBRARY_PATH
    path_saved=$PATH
    export ORACLE_HOME=$gi_home
    export PATH=$gi_home/bin:$PATH
    export LD_LIBRARY_PATH=${ORACLE_HOME}/lib:${LD_LIBRARY_PATH}

    checkpoint_echo "<b>Oracle Listener Status (GI_HOME):</b>"

    if [ "$gi_user" != "" ] && [ "$gi_passwd" != "" ]
    then
      verbose_echo "Running 'lsnrctl status' as OS user '${gi_user}'..."

      # only feasible if oracle is in the sudoers file.
      if [ "$x_mode" == "1" ]; then
          run_as $gi_user $db_root_passwd "$ORACLE_HOME/bin/lsnrctl status"
      else
          run_as $gi_user $gi_passwd "$ORACLE_HOME/bin/lsnrctl status"
      fi

      if [ "$tmp" != "" ]; then
        verbose_echo $tmp
      else
        nonfatal_error_echo "Failed to run 'lsnrctl status'."
      fi
    else
      verbose_echo "Running 'lsnrctl status' as OS user '`$whoamicmd`'..."
      lsnrctl status
    fi

    $echocmd
    $echocmd "<b>Installation Inventory for GI_HOME \"${GI_HOME}\"</b>"
    $echocmd
    if [ "$ade_mode" == "" ]; then
      if [ -x ${ORACLE_HOME}/OPatch/opatch ]; then
        if [ "$gi_user" != "" ] && [ "$gi_passwd" != "" ]
        then
          verbose_echo "Running 'opatch lsinventory' as OS user '${gi_user}'..."

          # only feasible if oracle is in the sudoers file.
          if [ "$x_mode" == "1" ]; then
              run_as $gi_user $db_root_passwd "$ORACLE_HOME/OPatch/opatch lsinventory"
          else
              run_as $gi_user $gi_passwd "$ORACLE_HOME/OPatch/opatch lsinventory"
          fi

          if [ "$tmp" != "" ]; then
            verbose_echo $tmp
          else
            nonfatal_error_echo "Failed to run 'opatch lsinventory'."
          fi
        else
          verbose_echo "Running 'opatch lsinventory' as OS user '`$whoamicmd`'..."
          $echocmd "`${ORACLE_HOME}/OPatch/opatch lsinventory 2>/dev/null`"
        fi
      fi
    else
      if [ -x ${ORACLE_HOME}/opatch/OPatch/opatch ]; then
        $echocmd "You are running $BASENAME in Oarcle ADE view name \"$ADE_VIEW_NAME\"."
        $echocmd
        $echocmd "`${ORACLE_HOME}/opatch/OPatch/opatch lsinventory 2>/dev/null`"
      fi
    fi

    export ORACLE_HOME=$oracle_home_saved
    export LD_LIBRARY_PATH=$ld_library_path_saved
    export PATH=$path_saved
}   # end of grid_lsinventory


function checkpoint_echo
{
    $echocmd "<font color="GREEN">[V]</font> $1"
    $echocmd
}   # end of checkpoint_echo


function checkpoint_echo_bold
{
    $echocmd "<font color="GREEN">[V]</font> <b>$1</b>"
    $echocmd
}   # end of checkpoint_echo


function verbose_echo
{
    if [ "$verbose_mode" == "1" ]; then
      $echocmd "<font color="ORANGE">Verbose mode: </font>[O] $1"
      $echocmd
    fi
}   # end of verbose_echo


function fatal_error_echo
{
    $echocmd "<blink><font color="RED">Error! [X] $1</font></blink>"
    $echocmd
    $echocmd "<blink><font color="RED">Check this error manually and see if it's legit!</font></blink>"
    $echocmd
#   exit 2
}   # end of fatal_error_echo


function nonfatal_error_echo
{
    $echocmd "<blink><font color="ORANGE">WARNING! [X] $1</font></blink>"
    $echocmd
    $echocmd "<blink><font color="ORANGE">Check this warning manually and see if it's legit!</font></blink>"
    $echocmd
#   exit 2
}   # end of nonfatal_error_echo


function remote_command
{
    USER=$1
    HOST=$2
    PASS=$3
    CMD=$4
    
    if [ "$PASS" == "<NULL>" ]; then
        PASS=
    fi

    if [ "$#" == "5" ]; then
        KEY=$5
        RCMD=$($expectcmd -c "
    spawn ssh -t -t -t -t -o StrictHostKeyChecking=no -o ConnectTimeout=900 -o LogLevel=INFO -i $KEY $USER@$HOST $CMD
    expect {
    \"password:\" {
      send \"$PASS\r\"
      expect eof
      }
    \"\\\\$\" {
      expect eof
      }
    }")
    else
        RCMD=$($expectcmd -c "
    spawn ssh -t -t -t -t -o StrictHostKeyChecking=no -o ConnectTimeout=900 -o LogLevel=INFO $USER@$HOST $CMD
    expect {
    \"password:\" {
      send \"$PASS\r\"
      expect eof
      }
    \"\\\\$\" {
      expect eof
      }
    }")
    fi

    $echocmd "<b>Calling \"$CMD\" on cell node $HOST...</b>"
    $echocmd "$RCMD" 2>/dev/null

}   # end of remote_command


function remote_copy
{
    FROM=$1
    TO=$2
    PASS=$3

    if [ "$PASS" == "<NULL>" ]; then
        PASS=
    fi

    if [ "$#" == "4" ]; then
        KEY=$4
        RCMD=$($expectcmd -c "
    spawn scp -p -r -o StrictHostKeyChecking=no -i $KEY $FROM $TO
    expect {
    \"password:\" {
      send \"$PASS\r\"
      expect eof
      }
    \"\\\\$\" {
      expect eof
      }
    }")
    else
        RCMD=$($expectcmd -c "
    spawn scp -p -r -o StrictHostKeyChecking=no $FROM $TO
    expect {
    \"password:\" {
      send \"$PASS\r\"
      expect eof
      }
    \"\\\\$\" {
      expect eof
      }
    }")
    fi

    $echocmd "$RCMD"

}   # end of remote_copy


function get_value_by_xml_name
{
    FILE=$1
    NAME=$2

    if [ -f $FILE ]; then
        x=`$grepcmd -n $NAME $FILE | $cutcmd -d: -f1`
        y=1
        ans=$(( x + y ))
        VALUE=`$sedcmd -n "${ans}p" $FILE | $sedcmd 's/<\/value>//g' | $cutcmd -d'>' -f2`
    else
        VALUE=
    fi

    echo $VALUE

}   #end of get_value_by_xml_name


function print_dbnode_table
{
    $echocmd "<h2>Oracle Big Data SQL System Information Overview</h2>"

    db_table_result_html=('0' '1' '2' '3' '4' '5' '6' '7' '8' '9')
    var=1
    while IFS='' read -r line || [[ -n "$line" ]];
    do
        case ${line} in
          "${DB_TABLE_CVALUE_PASSED}")
            db_table_result_html[$var]="<font color="green">$line</font>"
            ;;
          "${DB_TABLE_CVALUE_FAILED}")
            db_table_result_html[$var]="<font color="red">$line</font>"
            ;;
          *)
            db_table_result_html[$var]=$line
            ;;
        esac
        ((var+=1))
    done < "$MY_ERROR_LOG_DIRECTORY/bdsql_dbnode_result.txt"

    $echocmd "<table border="1" cellpadding="0" cellspacing="0" style="border-collapse: collapse" width="800">
    <tr>
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[1]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[2]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[3]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[4]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[5]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[6]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[7]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[8]}</font></td> 
      <td height="20" width="10%">&nbsp; <font color="blue">${DB_TABLE_CNAME[9]}</font></td> 
    </tr>
    <tr>
      <td height="20" width="10%">&nbsp; <a href="#D1"><pre>$hostname</pre></td> 
      <td height="20" width="10%">&nbsp; <a href="#D2">${db_table_result_html[2]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D3">${db_table_result_html[3]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D4">${db_table_result_html[4]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D5">${db_table_result_html[5]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D6">${db_table_result_html[6]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D7">${db_table_result_html[7]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D8">${db_table_result_html[8]}</td> 
      <td height="20" width="10%">&nbsp; <a href="#D9">${db_table_result_html[9]}</td> 
    </table>"

}   # end of print_dbnode_table


function print_hadoop_cellnode_table
{
    cell_name=$1
    cell_table_result_html=('1' '2' '3' '4' '5' '6' '7' '8' '9')
    var=1
    while IFS='' read -r line || [[ -n "$line" ]];
    do
        case ${line} in
          "${CELL_TABLE_CVALUE_PASSED}")
            cell_table_result_html[$var]="<font color="green">$line</font>"
            ;;
          "${CELL_TABLE_CVALUE_FAILED}")
            cell_table_result_html[$var]="<font color="red">$line</font>"
            ;;
          *)
            cell_table_result_html[$var]=$line
            ;;
        esac
        ((var+=1))
    done < "$MY_ERROR_LOG_DIRECTORY/bdsql_${cell_name}_result.txt"

$echocmd "<table border="1" cellpadding="0" cellspacing="0" style="border-collapse: collapse" width="800">
<tr>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[1]}</font></td> 
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[2]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[3]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[4]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[5]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[6]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[7]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[8]}</font></td>
 <td height="20" width="10%">&nbsp; <font color="blue">${CELL_TABLE_CNAME[9]}</font></td>
</tr>
<tr>
 <td height="20" width="10%">&nbsp; <a href="#C1"$cell_name><pre>$cell_name</pre></td> 
 <td height="20" width="10%">&nbsp; <a href="#C2"$cell_name>${cell_table_result_html[2]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C3"$cell_name>${cell_table_result_html[3]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C4"$cell_name>${cell_table_result_html[4]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C5"$cell_name>${cell_table_result_html[5]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C6"$cell_name>${cell_table_result_html[6]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C7"$cell_name>${cell_table_result_html[7]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C8"$cell_name>${cell_table_result_html[8]}</td> 
 <td height="20" width="10%">&nbsp; <a href="#C9"$cell_name>${cell_table_result_html[9]}</td> 
</table>"
}   # end of print_hadoop_cellnode_table


function run_dbnode_bdsql_sanity_test
{
    db_table_result[7]=${DB_TABLE_CVALUE_FAILED}

    $rmcmd -f $MY_ERROR_LOG_DIRECTORY/hadoop_fs_mkdir.out > /dev/null
    $rmcmd -f $MY_ERROR_LOG_DIRECTORY/hadoop_fs_mkdir.pid > /dev/null
    $rmcmd -f $MY_ERROR_LOG_DIRECTORY/hadoop_fs_put.out > /dev/null
    $rmcmd -f $MY_ERROR_LOG_DIRECTORY/hadoop_fs_put.pid > /dev/null

    $echocmd "<h3 id="D6"><font color="blue">${DB_TABLE_CNAME[6]}</font></h3>"
    #-----------------------------------------------------------------------------------    
    # hadoop fs <sub-command> would hang if Hadoop is in trouble!
    # Use nohup...
    #-----------------------------------------------------------------------------------    

    $hadoopcmd fs -mkdir /tmp/oracle > $MY_ERROR_LOG_DIRECTORY/hadoop_fs_mkdir.out 2>&1
    checkpoint_echo_bold "$hadoopcmd fs -mkdir /tmp/oracle"
    verbose_echo "`$catcmd $MY_ERROR_LOG_DIRECTORY/hadoop_fs_mkdir.out | $grepcmd -v SLF4J`"

    tmp=$($grepcmd failed $MY_ERROR_LOG_DIRECTORY/hadoop_fs_mkdir.out | $sedcmd -n '1p')

    if [ "$tmp" != "" ]
    then
        db_table_result[6]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[6]="$tmp"
        fatal_error_echo "${db_table_action[6]}"
    else
        $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.txt
This is a test created by $BASENAME from $HOSTNAME on $RIGHT_NOW.
_EOF_

        $hadoopcmd fs -rm /tmp/oracle/bdsql_sanity_test1.txt > $MY_ERROR_LOG_DIRECTORY/hadoop_fs_rm.out 2>&1
        checkpoint_echo_bold "$hadoopcmd fs -rm /tmp/oracle/bdsql_sanity_test1.txt"
        verbose_echo "`$catcmd $MY_ERROR_LOG_DIRECTORY/hadoop_fs_rm.out | $grepcmd -v SLF4J`"

        $hadoopcmd fs -put $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.txt /tmp/oracle/bdsql_sanity_test1.txt > $MY_ERROR_LOG_DIRECTORY/hadoop_fs_put.out 2>&1
        checkpoint_echo_bold "$hadoopcmd fs -put bdsql_sanity_test1.txt /tmp/oracle/bdsql_sanity_test1.txt"
        verbose_echo "`$catcmd $MY_ERROR_LOG_DIRECTORY/hadoop_fs_put.out | $grepcmd -v SLF4J`"

        tmp=$($grepcmd failed $MY_ERROR_LOG_DIRECTORY/hadoop_fs_put.out | $sedcmd -n '1p')

        if [ "$tmp" != "" ]
        then
#           pkill -TERM $($pscmd -aef | $grepcmd $BASENAME | $grepcmd -v grep | $awkcmd '{print $2}' | $grepcmd -v $BASHPID) > /dev/null
            db_table_result[6]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[6]="$tmp"
            fatal_error_echo "${db_table_action[6]}"
        else
            db_table_result[6]=${DB_TABLE_CVALUE_PASSED}

            if [ "$pdb_container" != "" ]
            then
              $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.sql
ALTER SESSION SET CONTAINER = ${pdb_container};
DROP TABLE bds_test;
CREATE TABLE bds_test (line VARCHAR2(4000)) ORGANIZATION EXTERNAL (TYPE ORACLE_HDFS DEFAULT DIRECTORY DEFAULT_DIR access parameters (com.oracle.bigdata.debug=true) LOCATION('/tmp/oracle/bdsql_sanity_test1.txt')) REJECT LIMIT UNLIMITED;
exit;
_EOF_
            else
              $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.sql
DROP TABLE bds_test;
CREATE TABLE bds_test (line VARCHAR2(4000)) ORGANIZATION EXTERNAL (TYPE ORACLE_HDFS DEFAULT DIRECTORY DEFAULT_DIR access parameters (com.oracle.bigdata.debug=true) LOCATION('/tmp/oracle/bdsql_sanity_test1.txt')) REJECT LIMIT UNLIMITED;
exit;
_EOF_
            fi

            if [ "$pdb_container" != "" ]
            then
              $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.sql
ALTER SESSION SET CONTAINER = ${pdb_container};
set linesize 132
ALTER SESSION SET EVENTS = 'TRACE[KCFIS] MEMORY HIGHEST, DISK HIGHEST';
ALTER SESSION SET TRACEFILE_IDENTIFIER = 'bdschecksw';

SELECT * FROM bds_test;
exit;
_EOF_
            else
              $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.sql
set linesize 132
ALTER SESSION SET EVENTS = 'TRACE[KCFIS] MEMORY HIGHEST, DISK HIGHEST';
ALTER SESSION SET TRACEFILE_IDENTIFIER = 'bdschecksw';

SELECT * FROM bds_test;
exit;
_EOF_
            fi
        fi
    fi


    $echocmd "<h3 id="D7"><font color="blue">${DB_TABLE_CNAME[7]}</font></h3>"

    if [ "$sqlplus_uid_passwd" != "" ]
    then
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1 > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.err
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11 > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.err
    else
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1 > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.err
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11 > $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.err
    fi

    checkpoint_echo_bold "sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1"
    $echocmd "$catcmd ${MY_ERROR_LOG_DIRECTORY}/bdsql_sanity_test1.sql"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.sql`"
    verbose_echo "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.err`"

    checkpoint_echo_bold "sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11"
    $echocmd "$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.sql"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.sql`"
    verbose_echo "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.err`"

    tmp=`$grepcmd "by $BASENAME" $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test11.err`
    checkpoint_echo_bold "Input:  `$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.txt`"
    checkpoint_echo_bold "Output: $tmp"

    check_hs_cores 7
    if [ "$hs_core" != "" ]; then
        start_time=$(/bin/date +%s)
    fi

    if [[ $tmp == "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_sanity_test1.txt`" ]]; 
    then
        smart_scan_bytes_before=`$grepcmd "${CELL_XT_SMART_SCAN_BYTES}" $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.lst | $awkcmd '{print $9}'`

        if [ "$sqlplus_uid_passwd" != "" ]
        then
            sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_statistics > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err
        else
            sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_statistics > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err
        fi
 
        $echocmd
        checkpoint_echo "<b>Statistics:</b>"
        $echocmd "`$catcmd bdsql_statistics.lst`"

        retry_err_count_after=`$grepcmd "${CELL_XT_OFFLOAD_RETRIES}" $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err | $awkcmd '{print $7}'`
        $echocmd
        checkpoint_echo_bold "Offload retry counts: ${retry_err_count_before} (before) => ${retry_err_count_after} (after)"

        db_table_result[7]=${DB_TABLE_CVALUE_PASSED}
        if [ ${retry_err_count_before} -ne ${retry_err_count_after} ]; then
            nonfatal_error_echo "Although the sanity test passed, offload retry is observed and it may be involving fallback."
        fi

        smart_scan_bytes_after=`$grepcmd "${CELL_XT_SMART_SCAN_BYTES}" $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err | $awkcmd '{print $9}'`
        checkpoint_echo_bold "Smart scan counts: ${smart_scan_bytes_before} (before) => ${smart_scan_bytes_after} (after)"

        if [ ${smart_scan_bytes_before} -eq ${smart_scan_bytes_after} ]; then
            fatal_error_echo "cell interconnect bytes returned by XT smart scan did not grow."
            db_table_result[7]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[7]="Check if database init parameter _external_table_smart_scan = OFF?"
        fi
    else 
        db_table_result[7]=${DB_TABLE_CVALUE_FAILED}
    fi

    echo ${db_table_result[7]} > $MY_ERROR_LOG_DIRECTORY/.db_node_sanity

    if [ "${db_table_result[7]}" == "${DB_TABLE_CVALUE_FAILED}" ]
    then
        #
        # Sanity test failed! Let's collect trace files generated by the test.
        #
        my_timestamp="`/bin/date +%Y%m%d%H%M`"

        # 1. tar *bdschecksw.trc and alert_<sid>.log in $ORACLE_BASE/diag/rdbms/<db_sid>/db_sid>/trace
        tmp=`cd $ORACLE_BASE/diag/rdbms/$db_name/$instance_name/trace; $findcmd . -maxdepth 1 -cmin -10 -type f | $grepcmd "bdschecksw\.trc" | $tarcmd cf ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar -T -`
        tmp=`cd $ORACLE_BASE/diag/rdbms/$db_name/$instance_name/trace; $tarcmd -r -f ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar ./alert_$instance_name.log`

        # 2. tar *.trc in $ORACLE_BASE/diag/crs/$hostname/crs/trace or $ORACLE_HOME/log/$hostname
        if [ "${GI_HOME}" != "" ]; then
            tmp=`cd $ORACLE_BASE/diag/crs/$hostname/crs/trace; $findcmd . -maxdepth 1 -cmin -10 -type f | $grepcmd "\.trc" | $tarcmd rf ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar -T -`
        else
            tmp=`cd $ORACLE_HOME/log; $findcmd . -maxdepth 1 -cmin -10 -type f | $grepcmd "\.trc" | $tarcmd rf ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar -T -`
        fi

        # 3. gzip bdsql_sanity_test1_${my_timestamp}.tar
        tmp=`$gzipcmd ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar`
        fatal_error_echo "Sanity test failed! (Send ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar.gz back to Oracle for further analysis)"
        db_table_action[7]="Check ${MY_LOG_DIRECTORY}/bdsql_sanity_test1_${my_timestamp}.tar.gz for great details"
    fi

    check_hs_cores 7
    if [ "$hs_core" != "" ]; then
        start_time=$(/bin/date +%s)
    fi

}   # end of run_dbnode_bdsql_sanity_test


function give_me_a_name_value
{
    local IFS
    local line
    local name
    local value="_NOT_FOUND_"

    while IFS='' read -r line || [[ -n "$line" ]];
     do
         if [[ $line == *"$2"* ]]; then
             name=`$echocmd $line | $sedcmd 's/ //g'`
             read -r line
             value=`$echocmd $line | $sedcmd 's/ //g'`
             break;
         fi
     done < "$1"
    $echocmd "$value"

}   # end of give_me_a_name_value


function turn_on_tracing
{
    checkpoint_echo "<b>Turn on bigdata extproc (including HS) and log4j tracing!</b>"

    #--------------------------------------------------------------------------
    # On DB/Compute node, turn on tracing and re-start MTA extproc.
    #
    # 1. ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
    #    e.g.
    #    bigdata.extproc.debug=true
    # 
    # 2. ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
    #    e.g.
    #    log4j.logger.oracle.hadoop.sql=DEBUG
    # 
    # 3. $ORACLE_HOME/hs/admin/init<mta_sid>.ora
    #    e.g.
    #    TRACE_LEVEL=1
    #--------------------------------------------------------------------------

    if [ -f "${DB_NODE_BIGDATA_CONFIG}/bigdata.properties" ]; then
        $cpcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties.bdschecksw 2>/dev/null
        tmp=`$grepcmd -i "bigdata.extproc.debug=" ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties`
        tmp2=`$echocmd $tmp | $trcmd [:upper:] [:lower:] | sed 's/ //g'`

        if [ "$tmp" == "" ] 
        then
            $echocmd "bigdata.extproc.debug=true" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
        else
            if [[ ${tmp2:0:1} == "#" ]]
            then
                $grepcmd -v "$tmp" ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties.bdschecksw > ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
                $echocmd "bigdata.extproc.debug=true" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
            elif [ "$tmp2" != *"true"* ] 
            then
                $grepcmd -v "$tmp" ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties.bdschecksw > ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
                $echocmd "bigdata.extproc.debug=true" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties
            fi
        fi
    fi

    if [ -f "${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties" ]; then
        $cpcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties.bdschecksw 2>/dev/null
        tmp=`$grepcmd -i "log4j.logger.oracle.hadoop.sql=" ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties`
        tmp2=`$echocmd $tmp | $trcmd [:upper:] [:lower:] | sed 's/ //g'`

        if [ "$tmp" == "" ]
        then
            $echocmd "log4j.logger.oracle.hadoop.sql=DEBUG" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
        else
            if [[ ${tmp2:0:1} == "#" ]]
            then
                $grepcmd -v "$tmp" ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties.bdschecksw > ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
                $echocmd "log4j.logger.oracle.hadoop.sql=DEBUG" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
            elif [ "$tmp2" != *"debug"* ] 
            then
                $grepcmd -v "$tmp" ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties.bdschecksw > ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
                $echocmd "log4j.logger.oracle.hadoop.sql=DEBUG" >> ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties
            fi
        fi
    fi

    if [ -f "$mta_init_file_pathname" ]; then
        $cpcmd $mta_init_file_pathname tmp_mta_init_file 2>/dev/null
        tmp=`$grepcmd -i trace_level $mta_init_file_pathname`
        tmp2=`$echocmd $tmp | $trcmd [:upper:] [:lower:] | sed 's/ //g'`

        if [ "$tmp" == "" ]
        then
            $echocmd "TRACE_LEVEL=1" >> $mta_init_file_pathname
        elif [[ ${tmp2:0:1} == "#" ]]
        then
            $grepcmd -v "$tmp" tmp_mta_init_file > $mta_init_file_pathname
            $echocmd "TRACE_LEVEL=1" >> $mta_init_file_pathname
        else 
            if [ "$tmp2" != *"=1"* ]; then
                $grepcmd -v "$tmp" tmp_mta_init_file > $mta_init_file_pathname
                $echocmd "TRACE_LEVEL=1" >> $mta_init_file_pathname
            fi
        fi
    fi
    mta_sid_name=`$basenamecmd $mta_init_file_pathname | $cutcmd -d. -f1 | $sedcmd 's/init//'`
    checkpoint_echo "<b>mtactl stop ${mta_sid_name}</b>"
    $mtactlcmd stop ${mta_sid_name}
    checkpoint_echo "<b>mtactl start ${mta_sid_name}</b>"
    $mtactlcmd start ${mta_sid_name}
    checkpoint_echo "<b>Sleep up to 90 seconds to allow listener registration!</b>"

    tmp="`$mtactlcmd status ${mta_sid_name} | grep service_register | grep ${mta_sid_name} 2>/dev/null`"
    ans=0
    while [[ "$tmp" == "" ]]
    do
      ans=$(( ans + 3 ))
      $sleepcmd 3
      verbose_echo "slept $ans seconds already..."
      if [ $ans -gt 90 ]; then
        nonfatal_error_echo "WARNING! Slept over 90 seconds for listener registration already."
        break;
      fi
      tmp="`$mtactlcmd status ${mta_sid_name} | grep service_register | grep ${mta_sid_name} 2>/dev/null`"
    done

    checkpoint_echo "<b>mtactl status ${mta_sid_name}</b>"
    tmp=`$mtactlcmd status ${mta_sid_name}`
    checkpoint_echo "$tmp"

}   # end of turn_on_tracing


function turn_off_tracing
{
    checkpoint_echo "<b>Turn off or restore bigdata extproc (including HS) and log4j configuration for tracing!</b>"
    #--------------------------------------------------------------------------
    # On DB/Compute node, turn off tracing and re-start MTA extproc.
    # Restore configuration files.
    #--------------------------------------------------------------------------

    $cpcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties.bdschecksw ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties 2>/dev/null
    $rmcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties.bdschecksw 2>/dev/null
    $cpcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties.bdschecksw ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties 2>/dev/null
    $rmcmd ${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties.bdschecksw 2>/dev/null
    $cpcmd tmp_mta_init_file ${mta_init_file_pathname} 2>/dev/null
    $rmcmd tmp_mta_init_file 2>/dev/null

    checkpoint_echo "<b>mtactl stop ${mta_sid_name}</b>"
    $mtactlcmd stop ${mta_sid_name}
    checkpoint_echo "<b>mtactl start ${mta_sid_name}</b>"
    $mtactlcmd start ${mta_sid_name}
#   checkpoint_echo "<b>mtactl status ${mta_sid_name}</b>"
#   tmp=`$mtactlcmd status ${mta_sid_name}`
#   checkpoint_echo "$tmp"

}   # end of turn_off_tracing


function check_database_initparams
{
    if [ "${pdb_container}" != "" ]; then
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.sql
set echo off
set heading off
set serveroutput off
set linesize 132

ALTER SESSION SET CONTAINER = ${pdb_container};
_EOF_
    else
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.sql
set echo off
set heading off
set serveroutput off
set linesize 132

_EOF_
    fi

    $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.sql

spool $MY_ERROR_LOG_DIRECTORY/bdsql_desc_dbms_hadoop
--
-- 0. Run desc dbms_hadoop and check 'GET_USER_MAP' to determine whether 
--    datapatch has been executed or not
--
desc dbms_hadoop
spool off

spool $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams
--
-- 1. Check _external_table_scan and _bigdata_external_table
--

SELECT a.ksppinm "Parameter",b.ksppstvl "Session Value",c.ksppstvl "Instance Value"
FROM
x\$ksppi a,x\$ksppcv b,x\$ksppsv c
WHERE
a.indx = b.indx
AND a.indx = c.indx
AND a.ksppinm LIKE '/_%' escape '/'
AND (a.ksppinm like '_external_table_smart_scan'
OR a.ksppinm like '_bigdata_external_table');

spool off
exit;
_EOF_

    if [ "$sqlplus_uid_passwd" != "" ]
    then
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams > $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.err
    else
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams > $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.err
    fi

    checkpoint_echo "<b>Database Init Parameters (session and instance values):</b>"
    $catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_database_initparams.lst

}   # end of check_database_initparams


function check_database_objects
{
    db_table_result[3]=${DB_TABLE_CVALUE_PASSED}

    if [ "${pdb_container}" != "" ]; then
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_database_objects.sql
set echo off
set heading off
set serveroutput off
set linesize 132

ALTER SESSION SET CONTAINER = ${pdb_container};
_EOF_
    else
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_database_objects.sql
set echo off
set heading off
set serveroutput off
set linesize 132
_EOF_
    fi

    $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_database_objects.sql
set echo off
set heading off
set serveroutput off
set linesize 132

spool bdsql_database_objects
--
-- 1. Check directory objects: DEFAULT_DIR, ORACLE_BIGDATA_CONFIG and ORA_BIGDATA_CL_<cluster>
--

select DIRECTORY_NAME, DIRECTORY_PATH
from all_directories
where DIRECTORY_NAME in ('DEFAULT_DIR', 'ORACLE_BIGDATA_CONFIG')
or DIRECTORY_NAME like 'ORA_BIGDATA_CL_%';

spool off
spool bdsql_database_links
--
-- 2. Check database links: 
--

select DB_LINK, HOST
from all_db_links
where DB_LINK like 'BDSQL\$_DEFAULT_CLUSTER%'
or DB_LINK like 'BDSQL\$_HADOOP_CL_%';

spool off

set echo on
set serveroutput on

spool $MY_ERROR_LOG_DIRECTORY/bdsql_dictionary_views
--
-- 3. Check data dictionary views: 
--

select count(*) from user_hive_tables;
select count(*) from all_hive_tables;
select count(*) from dba_hive_tables;

select count(*) from user_hive_databases;
select count(*) from all_hive_databases;
select count(*) from dba_hive_databases;

select count(*) from user_hive_columns;
select count(*) from all_hive_columns;
select count(*) from dba_hive_columns;

select CELL_PATH from v\$cell where CELL_TYPE like 'BDSQL';

spool off

set echo off
set serveroutput off

spool bdsql_functions
--
-- 4. Check functions/procedures for Hive data dictionary: 
--

spool off
spool $MY_ERROR_LOG_DIRECTORY/bdsql_tables
--
-- 5. Check tables:
--

desc SYS.HIVE_URI$;

spool off
spool $MY_ERROR_LOG_DIRECTORY/bdsql_statistics
--
-- 6. Check statistics:
--

select stat.name, sys.value
from v\$sysstat sys, v\$statname stat
where sys.statistic# = stat.statistic#
and stat.name like 'cell%XT%';

exit;
_EOF_

    if [ "${pdb_container}" != "" ]; then
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.sql
set echo off
set heading off
set serveroutput off
set linesize 132

ALTER SESSION SET CONTAINER = ${pdb_container};
_EOF_
    else
      $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.sql
set echo off
set heading off
set serveroutput off
set linesize 132

_EOF_
    fi

    $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.sql
spool bdsql_statistics

select stat.name, sys.value
from v\$sysstat sys, v\$statname stat
where sys.statistic# = stat.statistic#
and stat.name like 'cell%XT%';

exit;
_EOF_


    if [ "$sqlplus_uid_passwd" != "" ]
    then
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_database_objects > $MY_ERROR_LOG_DIRECTORY/bdsql_database_objects.err
    else
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_database_objects > $MY_ERROR_LOG_DIRECTORY/bdsql_database_objects.err
    fi

    retry_err_count_before=`$grepcmd "${CELL_XT_OFFLOAD_RETRIES}" ${MY_ERROR_LOG_DIRECTORY}/bdsql_database_objects.err | $awkcmd '{print $7}'`

    checkpoint_echo "<b>Database Objects:</b>"

    file="bdsql_database_objects.lst"
    name="DEFAULT_DIR"
    value=$(give_me_a_name_value $file $name)
    if [ "$value" != "_NOT_FOUND_" ]; then
        $echocmd "$name = $value"
    else
        db_table_result[3]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[3]="Database object $name is not found!"
        fatal_error_echo "${db_table_action[3]}"
    fi

    name="ORACLE_BIGDATA_CONFIG"
    value=$(give_me_a_name_value $file $name)
    if [ "$value" != "_NOT_FOUND_" ]; then
        $echocmd "$name = $value"
    else
        db_table_result[3]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[3]="Database object $name is not found!"
        fatal_error_echo "${db_table_action[3]}"
    fi

    name="ORA_BIGDATA_CL_${bdsql_cluster_name}"
    value=$(give_me_a_name_value $file $name)
    if [ "$value" != "_NOT_FOUND_" ]; then
        $echocmd "$name = $value"
    else
        db_table_result[3]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[3]="Database object $name is not found!"
        fatal_error_echo "${db_table_action[3]}"
    fi
    $rmcmd -f $file > /dev/null

    $echocmd
    checkpoint_echo "<b>Database Links:</b>"

    file="bdsql_database_links.lst"
    name="BDSQL\$_DEFAULT_CLUSTER"
    value=$(give_me_a_name_value $file $name)
    if [ "$value" != "_NOT_FOUND_" ]; then
        $echocmd "$name = $value"
    else
        db_table_result[3]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[3]="Database link $name is not found!"
        fatal_error_echo "${db_table_action[3]}"
    fi

    name="BDSQL\$_HADOOP_CL_"
    value=$(give_me_a_name_value $file $name)
    if [ "$value" != "_NOT_FOUND_" ]; then
        $echocmd "$name = $value"
    fi
    $rmcmd -f $file > /dev/null

    file="$MY_ERROR_LOG_DIRECTORY/bdsql_tables.lst"

    $echocmd
    checkpoint_echo "<b>Dictionary Views:</b>"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_dictionary_views.lst`"

    ans=$($catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_dictionary_views.lst | $grepcmd "ORA-")
    if [ "$ans" != "" ]; then
        db_table_result[3]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[3]="Failed to query BDS dictionary views!"
        fatal_error_echo "${db_table_action[3]}"
    fi

    check_hs_cores 3

    file="bdsql_functions.lst"
    $rmcmd -f $file > /dev/null

    $echocmd
    checkpoint_echo "<b>Statistics:</b>"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.lst`"

    check_hs_cores 3

    file="$MY_ERROR_LOG_DIRECTORY/bdsql_dictionary_views.lst"
    start_time=$(/bin/date +%s)

}   # end of check_database_objects


function check_database_extproc
{
    # check mtactl 
    if [ ! -x $mtactlcmd ]; then
        db_table_result[4]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[4]="$mtactlcmd is not found!"
        fatal_error_echo "${db_table_action[4]}"
    fi

    if [ ! -f ${mta_init_file_pathname} ]; then
        db_table_result[4]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[4]="${mta_init_file_pathname} is not found!"
        fatal_error_echo "${db_table_action[4]}"
    fi

    if [ "$ade_mode" == "" ]; then
        mta_init_file=`basename $oracle_home/hs/admin/init*_${db_unique_name}_${bdsql_cluster_name}.ora`
    else
        mta_init_file=`basename $oracle_home/hs/admin/initBDSQL_${bdsql_cluster_name}.ora`
    fi

    mta_sid_name=`$echocmd $mta_init_file | $cutcmd -d. -f1 | $sedcmd 's/init//'`
    mta_uid_name=`$pscmd -aef | $grepcmd extproc${mta_sid_name} | $grepcmd -v grep | $awkcmd '{print $1}'`
    mta_pid=`$pscmd -aef | $grepcmd extproc${mta_sid_name} | $grepcmd -v grep | $awkcmd '{print $2}'`

    if [ "${mta_uid_name}" == "" ]; then
        db_table_result[4]=${DB_TABLE_CVALUE_FAILED}
        fatal_error_echo "MTA extproc${mta_sid_name} is not found!"
        db_table_action[4]="Bring up MTA extproc${mta_sid_name} via \"crsctl start resource ${mta_sid_name}\" or mtactl manually."
        fatal_error_echo "${db_table_action[4]}"
    elif [ "${mta_uid_name}" != "oracle" ]; then
        nonfatal_error_echo "WARNING! You can ignore this warning if Kerberos is not used for your Big Data SQL system."
        nonfatal_error_echo "         Your MTA extproc${mta_sid_name} was started up by Oracle GRID OS user ${mta_uid_name} which is not oracle."
        nonfatal_error_echo "         When configuring Kerberos on your Big Data SQL system with two prinples (i.e. ${mta_uid_name} and oracle),"
        nonfatal_error_echo "         make sure to run kinit and grant read permission of Kerberos cache files to group oinstall for all involved princiles."
    fi

    tmp=`$mtactlcmd status ${mta_sid_name}`
    checkpoint_echo "<b>mtactl status ${mta_sid_name}</b>"
    checkpoint_echo "$tmp"

    if [[ $tmp == *"service_register"* ]]; 
      then
        db_table_result[4]=${DB_TABLE_CVALUE_PASSED}
      else
        if [[ $tmp == *"service_update"* ]]; 
        then
            db_table_result[4]=${DB_TABLE_CVALUE_PASSED}
        else
            if [[ $tmp == *"establish"* ]]; 
            then
                db_table_result[4]=${DB_TABLE_CVALUE_PASSED}
            else
                db_table_result[4]=${DB_TABLE_CVALUE_FAILED}
                db_table_action[4]="MTA process \"extproc${mta_sid_name} -mt\" is not found!"
                fatal_error_echo "${db_table_action[4]}"
            fi
        fi
    fi

    if [ -f /proc/${mta_pid}/environ ]; then
        checkpoint_echo "<b>/proc/${mta_pid}/environ: </b>"
        $catcmd /proc/${mta_pid}/environ
    fi

#   db_val[2]="${DB_TABLE_CVALUE_PASSED}"

}   # end of check_database_extproc


function check_bdsql_files
{
    db_table_result[2]=${DB_TABLE_CVALUE_PASSED}

    #----------------------------------------
    # Check datapatch if BDSQL 3.2 and above
    #----------------------------------------
    if [ "$db_node_bigdata_home_32" != "" ]
    then
        checkpoint_echo "<b>Describe dbms_hadoop:</b>"
        $catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_desc_dbms_hadoop.lst

        tmp=
        tmp=$($grepcmd -i GET_USER_MAP $MY_ERROR_LOG_DIRECTORY/bdsql_desc_dbms_hadoop.lst)

        if [ "$tmp" == "" ]
        then
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="It appears that you do not have the proper maintenance level or you have not applied datapatch."
            fatal_error_echo "${db_table_action[2]}"
            return
        else
            $echocmd "NOTE: GET_USER_MAP is found! (i.e. datapatch was applied)"
            $echocmd
        fi
    fi

    ##------------------------------------------------
    # DB node check point 1: Key directories and files 
    ##------------------------------------------------
    # Check cellinit.ora first

    if [ "$ade_mode" == "" ]; then
      # Determine the effective cellinit (i.e. cellinit.ora or celliniteth.ora)
      if [ -f $CELLINITETH ]
      then
          my_cellinit=$CELLINITETH
      elif [ -f $CELLINIT ]
      then
          my_cellinit=$CELLINIT
      else
          my_cellinit=""
      fi

      if [ "$my_cellinit" != "" ]
      then
        tmp=`$catcmd $my_cellinit > ${MY_ERROR_LOG_DIRECTORY}/cellinit.dbhome`
        checkpoint_echo_bold "$my_cellinit: `$catcmd $my_cellinit`"
#
#       ipaddress* = grep ipaddress /etc/oracle/cell/network-config/celliniteth.ora | cut -d= -f1
#       subet mask = grep ipaddress /etc/oracle/cell/network-config/celliniteth.ora | cut -d= -f2 | cut -d/ -f2
#
        ipaddress_lower=
        subnet_mask=
        ipaddress_lower=$($grepcmd -i ipaddress /etc/oracle/cell/network-config/celliniteth.ora | $cutcmd -d= -f1 | $trcmd [:upper:] [:lower:])
        subnet_mask=$($grepcmd -i ipaddress /etc/oracle/cell/network-config/celliniteth.ora | $cutcmd -d= -f2 | $cutcmd -d/ -f2)
        if [[ "$ipaddress_lower" == "ipaddress"* ]]; then
            $echocmd $subnet_mask > ${MY_ERROR_LOG_DIRECTORY}/.$ipaddress_lower
            checkpoint_echo "Subnet Mask: $subnet_mask"
        else
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            fatal_error_echo "Subnet Mask is not found!"
        fi

        tcpudp_mode="$($grepcmd $DB_DYNAMIC_PROTOCOL $my_cellinit | $cutcmd -d= -f2 | $trcmd -d '[blank]')"

        if [ "$tcpudp_mode" != "" ]; then
            checkpoint_echo "$my_cellinit: $DB_DYNAMIC_PROTOCOL=$tcpudp_mode"
            if [ "$tcpudp_mode" == "2" ]; then
                $echocmd "NOTE: For TCP/UDP connection, you must have $CELL_DYNAMIC_PROTOCOL=TRUE in"
                $echocmd "      $HADOOP_CELLINIT on Hadoop Data/Cell nodes. By default, InfiniBand is used."
                $echocmd
            else
                db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                db_table_action[2]="$DB_DYNAMIC_PROTOCOL in $my_cellinit has a bad value!"
                fatal_error_echo "${db_table_action[2]}"
            fi
        fi

        tmp=`$ifconfigcmd -a 2>/dev/null`
        verbose_echo "$tmp"

        case ${OS_TYPE} in
          Linux)
            ip_address=`$grepcmd ipaddress1 $my_cellinit | $cutcmd -d= -f2 | $cutcmd -d/ -f1`
            checkpoint_echo "<b>Cellinit IP address in $my_cellinit: $ip_address</b>"
            ip_tmp="`$ipcmd addr show | $grepcmd $ip_address`"
            ;;
          SunOS)
            ip_address=`$ifconfigcmd -a | $awkcmd 'BEGIN { count=0; } { if ( $1 ~ /inet/ ) { count++; if( count==2 ) { print $2; } } }'`
            checkpoint_echo "<b>Host IP address: $ip_address</b>"
            ip_tmp=`$grepcmd $ip_address $my_cellinit | $cutcmd -d= -f2 | $cutcmd -d/ -f1`
            checkpoint_echo "<b>Cellinit IP address in $my_cellinit: $ip_tmp</b>"

            x="`$ifconfigcmd -a | $grepcmd -n $ip_address | $cutcmd -d: -f1`"
            y=1
            ans=$(( x - y ))
            ip_tmp=`$ifconfigcmd -a | $sedcmd -n "${ans}p"`
            ;;
          *)
            ip_tmp=""
            ;;
        esac

        case "$ip_tmp" in
          *"bondib"*)
            checkpoint_echo "<b>Cellinit IP address is bonded to InfiniBand: $ip_tmp</b>"
            ;;
          *"bondeth"*)
            checkpoint_echo "<b>Cellinit IP address is bonded to Ethernet: $ip_tmp</b>"
            ;;
          *)
            checkpoint_echo "<b>Cellinit IP address: $ip_tmp</b>"
            ;;
        esac

        if [ "${ip_tmp}" == "" ];  then
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="Cellinit IP address in $my_cellinit doesn't match any of your IP address(es)!"
            fatal_error_echo "${db_table_action[2]}"
        fi
      else
        db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[2]="Neither $CELLINIT nor $CELLINITETH exist!"
        fatal_error_echo "${db_table_action[2]}"
      fi
    fi

    if [ -d "$oracle_home/bigdatasql" ]
    then
        checkpoint_echo "<b>$oracle_home/bigdatasql:</b>"
        if [ -f "${DB_NODE_BIGDATA_CONFIG}/bigdata.properties" ]
        then
            if [ "${bdsql_cluster_name}" != "" ]; 
            then
                if [ -f "$oracle_home/bigdatasql/hadoop_${bdsql_cluster_name}.env" ]; 
                then
#                   hadoop_env_file=$oracle_home/bigdatasql/hadoop_${bdsql_cluster_name}.env
#                   . $hadoop_env_file 2>/dev/null
                    if [ -f "${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties" ]; 
                    then
                        if [ -d "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}" ]; 
                        then
                            if [ -d "${DB_NODE_DEFAULT_DIR}" ]; 
                            then
                                verbose_echo "${DB_NODE_DEFAULT_DIR}."
                            else
                                db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                                db_table_action[2]="${DB_NODE_DEFAULT_DIR} doesn't exist!"
                                fatal_error_echo "${db_table_action[2]}"
                            fi
                            verbose_echo "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}."
                        else
                            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                            db_table_action[2]="${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name} doesn't exist!"
                            fatal_error_echo "${db_table_action[2]}"
                        fi
                        verbose_echo "${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties."
                    else
                        db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                        db_table_action[2]="${DB_NODE_BIGDATA_CONFIG}/bigdata-log4j.properties doesn't exist!"
                        fatal_error_echo "${db_table_action[2]}"
                    fi
                    verbose_echo "$oracle_home/bigdatasql/hadoop_${bdsql_cluster_name}.env."
                else
                    db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                    db_table_action[2]="$oracle_home/bigdatasql/hadoop_${bdsql_cluster_name}.env doesn't exist!"
                    fatal_error_echo "${db_table_action[2]}"
                fi
                verbose_echo "${bdsql_cluster_name}."
            else
                db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
                db_table_action[2]="${bdsql_cluster_name} doesn't exist!"
                fatal_error_echo "${db_table_action[2]}"
            fi
            verbose_echo "${DB_NODE_BIGDATA_CONFIG}/bigdata.properties."
        else
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="${DB_NODE_BIGDATA_CONFIG}/bigdata.properties doesn't exist!"
            fatal_error_echo "${db_table_action[2]}"
        fi
        verbose_echo "$oracle_home/bigdatasql."
    else
        db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[2]="$oracle_home/bigdatasql doesn't exist!"
        fatal_error_echo "${db_table_action[2]}"
    fi

    checkpoint_echo "<b>$oracle_home/rdbms/admin:</b>"

    if [ -d "$oracle_home/rdbms/admin" ]; then 
        if [ -f "$oracle_home/rdbms/admin/cathive.sql" ]; then
            verbose_echo "$oracle_home/rdbms/admin/cathive.sql"
        else
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="Find out why $oracle_home/rdbms/admin/cathive.sql doesn't exist!"
            fatal_error_echo "${db_table_action[2]}"
        fi
        if [ -f "$oracle_home/rdbms/admin/dbmshadp.sql" ]; then
            verbose_echo "$oracle_home/rdbms/admin/dbmshadp.sql"
        else
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="Find out why $oracle_home/rdbms/admin/dbmshadp.sql doesn't exist!"
            fatal_error_echo "${db_table_action[2]}"
        fi
        if [ -f "$oracle_home/lib/libkubsagt12.so" ]; then
            verbose_echo "$oracle_home/lib/libkubsagt12.so"
        else
            db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
            db_table_action[2]="Find out why $oracle_home/lib/libkubsagt12.so doesn't exist!"
            fatal_error_echo "${db_table_action[2]}"
        fi
    else
        db_table_result[2]=${DB_TABLE_CVALUE_FAILED}
        db_table_action[2]="Find out why $oracle_home/rdbms/admin doesn't exist!"
        fatal_error_echo "${db_table_action[2]}"
    fi

}   # end of check_bdsql_files


function check_prerequisites
{
    # DB_NODE_RPM_PACKAGE_1, DB_NODE_RPM_PACKAGE_2 & DB_NODE_RPM_PACKAGE_3

    db_table_result[5]=${DB_TABLE_CVALUE_FAILED}

    jdk_tmp=$(cd $oracle_home/bigdatasql; $lscmd -d jdk* 2>/dev/null | $grepcmd -v gz | $tailcmd -1)
    if [ "$jdk_tmp" != "" ]; then
        checkpoint_echo "$jdk_tmp"
    fi

    hadoop_tmp=$(cd $oracle_home/bigdatasql; $lscmd -d hadoop-* 2>/dev/null | $grepcmd -v gz | $tailcmd -1)
    if [ "$hadoop_tmp" != "" ]; then
        checkpoint_echo "$hadoop_tmp"
    fi

    hive_tmp=$(cd $oracle_home/bigdatasql; $lscmd -d hive-* 2>/dev/null | $grepcmd -v gz | $tailcmd -1)
    if [ "$hive_tmp" != "" ]; then
        checkpoint_echo "$hive_tmp"
    fi

    if [ "${OS_TYPE}" == "Linux" ]; then
    tmp=`$rpmcmd -qa | $grepcmd "${DB_NODE_RPM_PACKAGE_1}\|${DB_NODE_RPM_PACKAGE_2}\|${DB_NODE_RPM_PACKAGE_3}" 2>/dev/null`

    if [ "$tmp" != "" ]; then
        checkpoint_echo "$tmp"
    fi

    if [ "$bda_mode" == "1" ]; then
      if [ "`$echocmd $tmp | $grepcmd -e $DB_NODE_RPM_PACKAGE_1`" != "" ]; 
      then
        verbose_echo "RPM package $DB_NODE_RPM_PACKAGE_1."
        if [ "`$echocmd $tmp | $grepcmd -e $DB_NODE_RPM_PACKAGE_2`" != "" ]; 
        then
            verbose_echo "RPM package $DB_NODE_RPM_PACKAGE_2."
            if [ "`$echocmd $tmp | $grepcmd -e $DB_NODE_RPM_PACKAGE_3`" != "" ]; 
            then
                db_table_result[5]=${DB_TABLE_CVALUE_PASSED}
                verbose_echo "RPM package $DB_NODE_RPM_PACKAGE_3."
            else
                db_table_action[5]="$DB_NODE_RPM_PACKAGE_3 is not installed."
                fatal_error_echo "${db_table_action[5]}"
            fi
        else
            db_table_action[5]="$DB_NODE_RPM_PACKAGE_2 is not installed."
            fatal_error_echo "${db_table_action[5]}"
        fi
      else
        if [ "$ade_mode" == "" ]; then
            db_table_action[5]="$DB_NODE_RPM_PACKAGE_1 is not installed."
            fatal_error_echo "${db_table_action[5]}"
        else
            db_table_result[5]=${DB_TABLE_CVALUE_PASSED}
        fi
      fi
    else
        db_table_result[5]=${DB_TABLE_CVALUE_PASSED}
    fi
    else
        db_table_result[5]=${DB_TABLE_CVALUE_PASSED}
    fi

}   # end of check_prerequisites


function check_kerberos_tickets
{
    tmp=`$klistcmd -e 2>/dev/null`

    checkpoint_echo_bold "Kerberos Tickets:"
    verbose_echo "$tmp"

    if [ "$tmp" == "" ]; then
        db_table_action[8]="Double check that if there is indeed no Kerberos tickets in use."
        fatal_error_echo "${db_table_action[8]}"
    fi

    if [ -f "/etc/krb5.conf" ]; then
        checkpoint_echo_bold "/etc/krb5.conf:"
        verbose_echo "`$catcmd /etc/krb5.conf`"
    fi
    
    tmp=`get_value_by_xml_name ${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/core-site.xml hadoop.security.authentication`
    checkpoint_echo_bold "core-site.xml: hadoop.security.authentication=$tmp"

    tmp=`get_value_by_xml_name ${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/core-site.xml hadoop.security.authorization`
    checkpoint_echo_bold "core-site.xml: hadoop.security.authorization=$tmp"

    case "$tmp" in
      *"true"*)
        db_table_result[8]="$DB_TABLE_CVALUE_ON"
        ;;
      *"false"*)
        db_table_result[8]="$DB_TABLE_CVALUE_OFF"
        ;;
      *)
        db_table_result[8]="$DB_TABLE_CVALUE_NA"
        ;;
    esac

}   # end of check_kerberos_tickets


function check_tls_ssl_service
{
    checkpoint_echo_bold "DB/Compute Node TLS/SSL Service:"

    if [ -x $nmapcmd ]; then
        tmp=`$nmapcmd -F -sV localhost 2>/dev/null`

        verbose_echo "$tmp"

        if [ "`$echocmd $tmp | $grepcmd -i openssl 2>/dev/null`" == "" ]; then
            db_table_action[9]="Double check that if there is indeed no TLS/SSL service configured on your host."
            nonfatal_error_echo "${db_table_action[9]}"
        fi
    fi
    
    tmp=`get_value_by_xml_name ${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/core-site.xml hadoop.ssl.enabled`
    checkpoint_echo_bold "core-site.xml: hadoop.ssl.enabled=$tmp"

    case "$tmp" in
      *"true"*)
        db_table_result[9]=$DB_TABLE_CVALUE_ON
        ;;
      *"false"*)
        db_table_result[9]=$DB_TABLE_CVALUE_OFF
        ;;
      *)
        db_table_result[9]=$DB_TABLE_CVALUE_NA
        ;;
    esac

}   # end of check_tls_ssl_service


function check_db_ipsec
{
    checkpoint_echo_bold "DB/Compute Node IPSec Check:"

    db_table_result[9]=$DB_TABLE_CVALUE_PASSED
    tmp=`$catcmd /etc/ipsec.conf | sed "/^[ \t]*#/ d" | grep protostack | tr -d '[\t]'`
    if [ "$tmp" == "protostack=netkey" ]; then
        checkpoint_echo_bold "/etc/ipsec.conf has '$tmp' properly defined."
    else
        fatal_error_echo "/etc/ipsec.conf does not have 'protostack=netkey'!"
        db_table_result[9]=$DB_TABLE_CVALUE_FAILED
        return
    fi

    tmp=`$catcmd /etc/ipsec.conf | sed "/^[ \t]*#/ d" | grep include | awk '{print $2}'`
    if [ "$tmp" == "/etc/ipsec.d/*.conf" ]; then
        checkpoint_echo_bold "/etc/ipsec.conf has '$tmp' properly defined."
    else
        fatal_error_echo "/etc/ipsec.conf does not include '/etc/ipsec.d/*.conf'!"
        db_table_result[9]=$DB_TABLE_CVALUE_FAILED
        return
    fi

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_1.sh
#!/bin/bash
/bin/ls -1 /etc/ipsec.d/$bdsql_cluster_name*.conf | /bin/grep $HOSTNAME
_EOF_

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_2.sh
#!/bin/bash
/sbin/service ipsec status
_EOF_

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_3.sh
#!/bin/bash
tmp="\`/usr/bin/lsb_release -rs | /bin/cut -f1 -d.\`"

case \$tmp in
  5)
    /usr/sbin/ipsec whack --status | /bin/awk '/IPsec SA established/ && /admin/ {print \$3}' | /bin/cut '-d"' -f2
    ;;
  *)
    /usr/sbin/ipsec status | /bin/awk '/IPsec SA established/ && /admin/ {print \$3}' | /bin/cut '-d"' -f2
    ;;
esac
_EOF_
    $chmodcmd 755 $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_*.sh
    tmp=
    tmp=$(run_as root $db_root_passwd $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_1.sh)
    $echocmd $tmp
    if [ "$tmp" != "" ]; then
        checkpoint_echo_bold "/etc/ipsec.d has '$tmp' properly defined."
    else
        fatal_error_echo "/etc/ipsec.d does not have a connection configuration file for '$HOSTNAME'!"
        db_table_result[9]=$DB_TABLE_CVALUE_FAILED
        return
    fi

    checkpoint_echo_bold "/sbin/service ipsec status"
    run_as root $db_root_passwd $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_2.sh
    $echocmd

    checkpoint_echo_bold "/usr/sbin/ipsec status"
    run_as root $db_root_passwd $MY_ERROR_LOG_DIRECTORY/check_db_ipsec_3.sh
    $echocmd

}   # end of check_db_ipsec


function check_cell_ipsec
{
    checkpoint_echo_bold "Hadoop/Cell Node IPSec Check:"
    cell_table_result[9]=$CELL_TABLE_CVALUE_PASSED

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/check_${new_cell_name}_ipsec_3.sh
#!/bin/bash
tmp="\`/usr/bin/lsb_release -rs | /bin/cut -f1 -d.\`"

case \$tmp in
  5)
    /usr/sbin/ipsec whack --status | /bin/awk '/IPsec SA established/ {print \$3}' | /bin/cut '-d"' -f2
    ;;
  *)
    /usr/sbin/ipsec status | /bin/awk '/IPsec SA established/ {print \$3}' | /bin/cut '-d"' -f2
    ;;
esac
_EOF_

    $chmodcmd 755 $MY_ERROR_LOG_DIRECTORY/check_${new_cell_name}_ipsec_*.sh

    tmp=
    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/check_${new_cell_name}_ipsec_3.sh root@${cell_name}:/tmp/check_${new_cell_name}_ipsec_3.sh $cell_root_passwd $identitykey`
    checkpoint_echo_bold "remote_copy ${MY_ERROR_LOG_DIRECTORY}/check_${new_cell_name}_ipsec_3.sh root@${cell_name}:/tmp/check_${new_cell_name}_ipsec_3.sh $identitykey"
    verbose_echo "$tmp"

    command="/tmp/check_${new_cell_name}_ipsec_3.sh"
    checkpoint_echo_bold "remote_command root $cell_name $command $identitykey"
    checkpoint_echo "`$catcmd ${MY_ERROR_LOG_DIRECTORY}/check_${new_cell_name}_ipsec_3.sh`"

    tmp=`remote_command root $cell_name $cell_root_passwd $command $identitykey`
    checkpoint_echo "$tmp"
    tmp2=
    tmp2=`$echocmd "$tmp" | $sedcmd -e 's/\r/\n/g' | $grepcmd -e $HOSTNAME -e ${new_cell_name} | $awkcmd '{print $1}'`

    if [ "$tmp2" == "" ]; then
        cell_table_result[9]=$CELL_TABLE_CVALUE_FAILED
        fatal_error_echo "IPSec active connection not found!"
    else
        checkpoint_echo "IPSec active connection: $tmp2"
    fi

}   # end of check_cell_ipsec


function print_dbnode_table_json
{
$echocmd "\"${hostname}\":["

for i in {2..8}
do
  $echocmd "  {"
  $echocmd "    \"check\":\"${DB_TABLE_CNAME[$i]}\","
  $echocmd "    \"result\":\"${db_table_result[$i]}\","
  $echocmd "    \"action\":\"${db_table_action[$i]}\""
  $echocmd "  },"
done

  $echocmd "  {"
  $echocmd "    \"check\":\"${DB_TABLE_CNAME[9]}\","
  $echocmd "    \"result\":\"${db_table_result[9]}\","
  $echocmd "    \"action\":\"${db_table_action[9]}\""
  $echocmd "  }"
$echocmd "]"

}   # end of print_dbnode_table_json


function print_cellnode_table_json
{
echo "\"${cell_name}\":["

for i in {2..8}
do
  $echocmd "  {"
  $echocmd "    \"check\":\"${CELL_TABLE_CNAME[$i]}\","
  $echocmd "    \"result\":\"${cell_table_result[$i]}\","
  $echocmd "    \"action\":\"${cell_table_action[$i]}\""
  $echocmd "  },"
done

  $echocmd "  {"
  $echocmd "    \"check\":\"${CELL_TABLE_CNAME[9]}\","
  $echocmd "    \"result\":\"${cell_table_result[9]}\","
  $echocmd "    \"action\":\"${cell_table_action[9]}\""
  $echocmd "  }"
$echocmd "];"

}   # end of print_cellnode_table_json


function check_dbnode
{
    # Clean up $MY_ERROR_LOG_DIRECTORY/*
    $rmcmd -f ${MY_ERROR_LOG_DIRECTORY}/* > /dev/null

    $echocmd "<h3>Database/Compute Node: </h3>"
    $echocmd "<pre>"

    ##------------------------------------------------
    # DB node check point 1: DB Node
    ##------------------------------------------------
    $echocmd "<h3 id="D1"><font color="blue">${DB_TABLE_CNAME[1]}</font></h3>"

    # Check if BDA
    if [ -x $bdaclicmd ]; then
        $echocmd "<b>BDA Server Name       </b>= `$bdaclicmd getinfo server_name`"
        $echocmd "<b>BDA Server OS Version </b>= Oracle Linux `$bdaclicmd getinfo server_os_version`"
        $echocmd "<b>BDA Rack Name         </b>= `$bdaclicmd getinfo rack_name`"
        $echocmd "<b>BDA Rack Serial NO.   </b>= `$bdaclicmd getinfo rack_serial_number`"
    fi

    $echocmd
    host_ip=`$getentcmd hosts $HOSTNAME | awk '{print $1}'`
    $echocmd "<b>Database Node = $hostname ($host_ip)</b>"

    case ${OS_TYPE} in
      Linux)
        tmp="`$ipcmd addr show | $grepcmd $host_ip`"
        ;;
      SunOS)
        x="`$ifconfigcmd -a | $grepcmd -n $host_ip | $cutcmd -d: -f1`"
        y=1
        ans=$(( x - y ))
        tmp=`$ifconfigcmd -a | $sedcmd -n "${ans}p"`
        ;;
      *)
        ;;
    esac

    case "$tmp" in
      *"bondib"*)
        $echocmd "<b>Host IP address $host_ip is bonded to InfiniBand</b>"
        ;;
      *"bondeth"*)
        $echocmd "<b>Host IP address $host_ip is bonded to Ethernet</b>"
        ;;
      *"bondmgt"*)
        $echocmd "<b>Host IP address $host_ip is bonded to a management group</b>"
        ;;
      *)
        $echocmd "<b>Host IP address is $host_ip</b>"
        ;;
    esac

    $echocmd
    $echocmd "<b>BDSQL ORACLE_HOME     </b>= $oracle_home"
    $echocmd "<b>BDSQL ORACLE_SID      </b>= $oracle_sid"
    if [ "${pdb_container}" != "" ]; then
      $echocmd "<b>BDSQL PDB CONTAINER   </b>= $pdb_container"
    fi
    $echocmd "<b>BDSQL HOME            </b>= $oracle_home/bigdatasql"
    $echocmd "<b>BDSQL Cluster         </b>= ${bdsql_cluster_name}"
    $echocmd "<b>BDSQL Default Cluster </b>= $bigdata_cluster_default"
    $echocmd "<b>BDSQL Diskmon Trace   </b>= $oracle_base/diag/crs/$hostname/crs/trace/diskmon.trc"
    $echocmd "<b>BDSQL Trace Log       </b>= $oracle_home/bigdatasql/log"

    if [ "$ade_mode" == "" ]; then
        $echocmd "<b>BDSQL Extproc Init    </b>= $oracle_home/hs/admin/initbds_${db_unique_name}_${bdsql_cluster_name}.ora"
    else
        $echocmd "<b>BDSQL Extproc Init    </b>= $oracle_home/hs/admin/initBDSQL_${bdsql_cluster_name}.ora"
    fi
    $echocmd "<b>BDSQL Extproc Log     </b>= $oracle_home/hs/log"
    $echocmd 
    $echocmd "<b>Hadoop Env File       </b>= $hadoop_env_file"
    $echocmd "<b>HADOOP_HOME           </b>= $HADOOP_HOME"
    $echocmd "<b>HADOOP_CONF_DIR       </b>= $HADOOP_CONF_DIR"
    $echocmd "<b>HADOOP_OPTS           </b>= $HADOOP_OPTS"
    $echocmd
    $echocmd "<b>HIVE_HOME             </b>= $HIVE_HOME"
    $echocmd "<b>HIVE_CONF_DIR         </b>= $HIVE_CONF_DIR"
    $echocmd
    $echocmd "<b>JAVA_HOME             </b>= $JAVA_HOME"

    if [ "${GI_HOME}" != "" ]; then
        $echocmd
        $echocmd "<b>CRS/GI_HOME*          </b>= ${GI_HOME}"
        $echocmd "<b>TNS_ADMIN             </b>= ${TNS_ADMIN}"
        $echocmd "<b>DISKMON Process*      </b>= $($pscmd -aef | $grepcmd diskmon | $grepcmd -v grep)"
        $echocmd
#       $echocmd "<b>Installation Inventory for GI_HOME \"${GI_HOME}\"</b>"
#       $echocmd
        grid_lsinventory
    fi

    checkpoint_echo "<b>Oracle Listener Status (ORACLE_HOME):</b>"
    lsnrctl status

    #----------------------------------------------------------------
    # Opatch Inventory
    #----------------------------------------------------------------
    $echocmd
    $echocmd "<b>Installation Inventory for ORACLE_HOME \"${ORACLE_HOME}\"</b>"
    $echocmd
    if [ "$ade_mode" == "" ]; then
      if [ -x $oracle_home/OPatch/opatch ]; then
        $echocmd "`$oracle_home/OPatch/opatch lsinventory 2>/dev/null`"
      fi
    else
      if [ -x $oracle_home/opatch/OPatch/opatch ]; then
        $echocmd "You are running $BASENAME in Oarcle ADE view name \"$ADE_VIEW_NAME\"."
        $echocmd
        $echocmd "`$oracle_home/opatch/OPatch/opatch lsinventory 2>/dev/null`"
      fi
    fi

    if [ "$trace_mode" == "1" ]; then
        turn_on_tracing
    fi

    $echocmd
    check_database_initparams

    ##------------------------------------------------
    # DB node check point 2: Key directories and files 
    ##------------------------------------------------
    $echocmd "<h3 id="D2"><font color="blue">${DB_TABLE_CNAME[2]}</font></h3>"
    check_bdsql_files

    ##-------------------------------------------------
    # DB node check point 3: Database objects
    ##-------------------------------------------------
    $echocmd "<h3 id="D3"><font color="blue">${DB_TABLE_CNAME[3]}</font></h3>"
    check_database_objects

    ##-------------------------------------------------
    # DB node check point 4: Extproc
    ##-------------------------------------------------
    $echocmd "<h3 id="D4"><font color="blue">${DB_TABLE_CNAME[4]}</font></h3>"
    check_database_extproc

    ##-------------------------------------------------
    # DB node check point 5: Hadoop client
    ##-------------------------------------------------
    $echocmd "<h3 id="D5"><font color="blue">${DB_TABLE_CNAME[5]}</font></h3>"
    if [ "$ade_mode" == "" ]; then
        check_prerequisites
    else
        $echocmd "<b>N.A.</b>"
        $echocmd "You are running $BASENAME in Oarcle ADE view name \"$ADE_VIEW_NAME\"."
    fi

    ##-------------------------------------------------
    # DB node check point 6,7: Sanity test
    ##-------------------------------------------------
    run_dbnode_bdsql_sanity_test

    ##-------------------------------------------------
    # DB node check point 8: Kerberos tickets
    ##-------------------------------------------------
    $echocmd "<h3 id="D8"><font color="blue">${DB_TABLE_CNAME[8]}</font></h3>"
    check_kerberos_tickets

    ##-------------------------------------------------
    # DB node check point 9: TLS/SSL/IPSec service
    ##-------------------------------------------------
    $echocmd "<h3 id="D9"><font color="blue">${DB_TABLE_CNAME[9]}</font></h3>"
    check_tls_ssl_service

    # ipsec check for extensive mode only
    if [ "$x_mode" == "1" ]; then
        check_db_ipsec
    fi

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_dbnode_result.txt
${db_table_result[1]}
${db_table_result[2]}
${db_table_result[3]}
${db_table_result[4]}
${db_table_result[5]}
${db_table_result[6]}
${db_table_result[7]}
${db_table_result[8]}
${db_table_result[9]}
_EOF_
    $echocmd "</pre>"

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_dbnode_result.json
    $(print_dbnode_table_json)
_EOF_

}   # end of check_dbnode


function run_cellnode_bdsql_sanity_test
{
    cell_name=$1
    first_cell_name=$2
    new_cell_name="`echo $cell_name | sed 's/\./_/g'`"

    #----------------------
    # Before block comment
    #----------------------
    if [ ]; then
    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.txt
This is a test created by $BASENAME from ${new_cell_name} on $RIGHT_NOW.
_EOF_

    $hadoopcmd fs -rm /tmp/oracle/${BASENAME}_${new_cell_name}_test.txt > $MY_ERROR_LOG_DIRECTORY/hadoop_${new_cell_name}_fs_rm.out 2>&1
    checkpoint_echo_bold "$hadoopcmd fs -rm /tmp/oracle/${BASENAME}_${new_cell_name}_test.txt"
    verbose_echo "$MY_ERROR_LOG_DIRECTORY/hadoop_${new_cell_name}_fs_rm.out"

    $hadoopcmd fs -put $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.txt /tmp/oracle/${BASENAME}_${new_cell_name}_test.txt > $MY_ERROR_LOG_DIRECTORY/hadoop_${new_cell_name}_fs_put.out 2>&1
    checkpoint_echo_bold "$hadoopcmd fs -put bdsql_${new_cell_name}_sanity_test.txt /tmp/oracle/${BASENAME}_${new_cell_name}_test.txt"
    verbose_echo "$MY_ERROR_LOG_DIRECTORY/hadoop_${new_cell_name}_fs_put.out"

    tmp=$($grepcmd failed $MY_ERROR_LOG_DIRECTORY/hadoop_${new_cell_name}_fs_put.out | $sedcmd -n '1p')

    if [ "$tmp" != "" ]
    then
        cell_table_action[2]="$tmp"
        fatal_error_echo "${cell_table_action[2]}"
    else
        $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_1.sh
#!/bin/bash
hive -e "drop table if exists bds_test_${new_cell_name}_hive"
_EOF_

        $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_2.sh
#!/bin/bash
hive -e "create table bds_test_${new_cell_name}_hive(line string)"
_EOF_

        $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_3.sh
#!/bin/bash
hive -e "load data inpath '/tmp/oracle/${BASENAME}_${new_cell_name}_test.txt' overwrite into table bds_test_${new_cell_name}_hive"
_EOF_

        $chmodcmd 755 $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_*.sh

        if [ "$pdb_container" != "" ]
        then
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.sql
ALTER SESSION SET CONTAINER = ${pdb_container};
DROP TABLE bds_test_${new_cell_name}_db;
exit;
_EOF_
        else
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.sql
DROP TABLE bds_test_${new_cell_name}_db;
exit;
_EOF_
        fi

        if [ "$pdb_container" != "" ]
        then
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql
set linesize 132
set serveroutput on

ALTER SESSION SET CONTAINER = ${pdb_container};
_EOF_
        else
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql
set linesize 132
set serveroutput on

_EOF_
        fi

        $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql
DECLARE
   DDLtxt VARCHAR2(4000);
BEGIN
   dbms_hadoop.create_extddl_for_hive(
     CLUSTER_ID=>'${bdsql_cluster_name}',
     DB_NAME=>'default',
     HIVE_TABLE_NAME=>'bds_test_${new_cell_name}_hive',
     HIVE_PARTITION=>FALSE,
     TABLE_NAME=>'bds_test_${new_cell_name}_db',
     PERFORM_DDL=>TRUE,
     TEXT_OF_DDL=>DDLtxt
   );
   dbms_output.put_line(DDLtxt);
END;
/
exit;
_EOF_

        if [ "$pdb_container" != "" ]
        then
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.sql
set linesize 132
ALTER SESSION SET CONTAINER = ${pdb_container};
SELECT * FROM bds_test_${new_cell_name}_db;
exit;
_EOF_
        else
          $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.sql
set linesize 132
SELECT * FROM bds_test_${new_cell_name}_db;
exit;
_EOF_
        fi
    fi

    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_1.sh $username@${cell_name}:setup_${new_cell_name}_1.sh $password $identitykey`
    verbose_echo "$tmp"

    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_2.sh $username@${cell_name}:setup_${new_cell_name}_2.sh $password $identitykey`
    verbose_echo "$tmp"

    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_3.sh $username@${cell_name}:setup_${new_cell_name}_3.sh $password $identitykey`
    verbose_echo "$tmp"

    command="./setup_${new_cell_name}_1.sh"
    checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
    tmp=`remote_command $username $cell_name $password $command $identitykey`
    checkpoint_echo "`$catcmd ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_1.sh`"
    verbose_echo "$tmp"

    command="./setup_${new_cell_name}_2.sh"
    checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
    tmp=`remote_command $username $cell_name $password $command $identitykey`
    checkpoint_echo "`$catcmd ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_2.sh`"
    verbose_echo "$tmp"

    command="./setup_${new_cell_name}_3.sh"
    checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
    tmp=`remote_command $username $cell_name $password $command $identitykey`
    checkpoint_echo "`$catcmd ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_3.sh`"
    verbose_echo "$tmp"
    fi
    #----------------------
    # After block comment
    #----------------------

    my_timestamp="`/bin/date +%Y%m%d%H%M`"

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_kill.sh
#!/bin/bash
pid="\`/bin/ps -aef | /bin/grep bdsqloflsrv | /bin/grep -v grep | /bin/awk '{print \$2}'\`"
echo "/bin/kill -12 \$pid" > /tmp/setup_${new_cell_name}_kill.log
/bin/kill -12 \$pid
_EOF_
    $chmodcmd 755 $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_*.sh
    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_kill.sh $username@${cell_name}:setup_${new_cell_name}_kill.sh $password $identitykey`
    verbose_echo "$tmp"

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_tar.sh
#!/bin/bash
cd \$ADR_BASE
/bin/find . -cmin -10 -type f | /bin/grep trc | /bin/tar -czf /tmp/bdsql_${new_cell_name}_sanity_adr_${my_timestamp}.tar.gz -T - >> /tmp/setup_${new_cell_name}_tar.log
cd /opt/oracle/bd_cell/log/deploy
/bin/find . -cmin -10 -type f | /bin/grep trc | /bin/tar -czf /tmp/bdsql_${new_cell_name}_sanity_test_${my_timestamp}.tar.gz -T - >> /tmp/setup_${new_cell_name}_tar.log
_EOF_
    $chmodcmd 755 $MY_ERROR_LOG_DIRECTORY/setup_${new_cell_name}_*.sh
    tmp=`remote_copy ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_tar.sh $username@${cell_name}:setup_${new_cell_name}_tar.sh $password $identitykey`
    verbose_echo "$tmp"

    #----------------------
    # Before block comment
    #----------------------
    if [ ]; then
    if [ "$sqlplus_uid_passwd" != "" ]
    then
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.sql > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.err
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.err
    else
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0 > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.err
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.err
    fi

    checkpoint_echo_bold "sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0"
    verbose_echo "$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.sql"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.sql`"

    tmp=`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test0.err`
    verbose_echo "$tmp"

    checkpoint_echo_bold "sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test"
    verbose_echo "$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.sql`"

    tmp=`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.err`
    verbose_echo "$tmp"

    command="./setup_${new_cell_name}_3.sh"
    checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
    tmp=`remote_command $username $cell_name $password $command $identitykey`
    checkpoint_echo "`$catcmd ${MY_ERROR_LOG_DIRECTORY}/setup_${new_cell_name}_3.sh`"
    verbose_echo "$tmp"

    $sleepcmd 5

    if [ "$sqlplus_uid_passwd" != "" ]
    then
        sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.sql > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.err
    else
        sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1 > $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.err
    fi

    checkpoint_echo_bold "sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1"
    verbose_echo "$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.sql"
    $echocmd "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.sql`"

    tmp=`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.err`
    verbose_echo "$tmp"

    tmp=`$grepcmd " from ${new_cell_name}" $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.err`
    checkpoint_echo_bold "Input:  `$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.txt`"
    checkpoint_echo_bold "Output: $tmp"
    fi
    #----------------------
    # After block comment
    #----------------------

    #----------------------
    # Before block comment
    #----------------------
    if [ ]; then
    if [[ $tmp == "`$catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.txt`" ]]; 
    then
        declare ${cell_name}_retry_err_count_before=`$grepcmd "${CELL_XT_OFFLOAD_RETRIES}" $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err | $awkcmd '{print $7}'`
        if [ "$sqlplus_uid_passwd" != "" ]
        then
            sqlplus $sqlplus_uid_passwd @$MY_ERROR_LOG_DIRECTORY/bdsql_statistics > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err
        else
            sqlplus system/manager @$MY_ERROR_LOG_DIRECTORY/bdsql_statistics > $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err
        fi
 
        $echocmd
        checkpoint_echo "<b>Statistics:</b>"
        $echocmd "`$catcmd bdsql_statistics.lst`"

        declare ${cell_name}_retry_err_count_after=`$grepcmd "${CELL_XT_OFFLOAD_RETRIES}" $MY_ERROR_LOG_DIRECTORY/bdsql_statistics.err | $awkcmd '{print $7}'`
        $echocmd
        checkpoint_echo_bold "Offload retry counts: ${cell_name}_retry_err_count_before (before) => ${cell_name}_retry_err_count_after (after)"

#       if [ ${cell_name}_retry_err_count_before -ne ${cell_name}_retry_err_count_after ]; then
#       if [ ${!${cell_name}_retry_err_count_before} -ne ${!${cell_name}_retry_err_count_after} ]; then
#       if [ $(tmpvar1=${cell_name}_retry_err_count_before && echo ${!tmpvar1}) -ne $(tmpvar2=${cell_name}_retry_err_count_afte && echo ${!tmpvar2}) ]; then
#           nonfatal_error_echo "Although the sanity test passed, offload retry is observed and it may be involving fallback."
#       fi
#       $rmcmd -f $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test.* $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test1.*
    else 
        $echocmd
    fi
    fi
    #----------------------
    # After block comment
    #----------------------

#       if [ "$cell_name" == "$first_cell_name" ]; then
#           cell_table_result[2]=$CELL_TABLE_CVALUE_FAILED
#       fi
#       fatal_error_echo "Sanity test failed! (See $MY_ERROR_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_test*.* for great details)"

    db_table_result[7]="`$catcmd $MY_ERROR_LOG_DIRECTORY/.db_node_sanity`"

    if [ "${db_table_result[7]}" != "${DB_TABLE_CVALUE_PASSED}" ]; then
        cell_table_result[2]=$CELL_TABLE_CVALUE_FAILED
        # Run kill -12 <pid_of_bdsqloflsrv> to trigger user-defined signal for additional tracing
        command="./setup_${new_cell_name}_kill.sh"
        checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
        tmp=`remote_command $username $cell_name $password $command $identitykey`
        verbose_echo "$tmp"
        $sleepcmd 10
        command="./setup_${new_cell_name}_tar.sh"
        checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
        tmp=`remote_command $username $cell_name $password $command $identitykey`
        verbose_echo "$tmp"

#       command="cd \$ADR_BASE; $findcmd . -cmin -10 -type f | $grepcmd trc | $tarcmd -czf /tmp/bdsql_${new_cell_name}_sanity_adr_${my_timestamp}.tar.gz -T -"
#       checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
#       tmp=`remote_command $username $cell_name $password $command $identitykey`
#       verbose_echo "$tmp"
        tmp=`remote_copy $username@${cell_name}:/tmp/bdsql_${new_cell_name}_sanity_adr_${my_timestamp}.tar.gz $MY_LOG_DIRECTORY $password $identitykey`
        verbose_echo "$tmp"

#       command="cd /opt/oracle/bd_cell/log/deploy; $findcmd . -cmin -10 -type f | $grepcmd trc | $tarcmd -czf /tmp/bdsql_${new_cell_name}_sanity_test_${my_timestamp}.tar.gz -T -"
#       checkpoint_echo_bold "remote_command $username $cell_name $command $identitykey"
#       tmp=`remote_command $username $cell_name $password $command $identitykey`
#       verbose_echo "$tmp"
        tmp=`remote_copy $username@${cell_name}:/tmp/bdsql_${new_cell_name}_sanity_test_${my_timestamp}.tar.gz $MY_LOG_DIRECTORY $password $identitykey`
        verbose_echo "$tmp"

        cell_table_action[2]="Check $MY_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_*_${my_timestamp}.tar.gz for great details"
        fatal_error_echo "Sanity test failed! (Send $MY_LOG_DIRECTORY/bdsql_${new_cell_name}_sanity_*_${my_timestamp}.tar.gz back to Oracle for further analysis)"
    fi

}   # end of run_cellnode_bdsql_sanity_test


function check_hadoop_cellnode_part1
{
    $echocmd > ${report_fullpath_basename}_CBODY.html
    $echocmd "<h3>Hadoop Data/Cell Node:</h3>" >> ${report_fullpath_basename}_CBODY.html
    $echocmd "<pre>" >> ${report_fullpath_basename}_CBODY.html

    if [ "$ade_mode" == "" ]; then
        $echocmd "<b>BDSQL Cluster            </b>= ${bdsql_cluster_name}" >> ${report_fullpath_basename}_CBODY.html
        $echocmd "<b>Hadoop BDS HOME          </b>= /opt/oracle/bigdatasql" >> ${report_fullpath_basename}_CBODY.html
        $echocmd "<b>Hadoop BDS Prop.         </b>= /opt/oracle/bigdatasql/bdcell-12.1/bigdata.properties" >> ${report_fullpath_basename}_CBODY.html
        $echocmd "<b>Hadoop BDS Log4J Prop.   </b>= /opt/oracle/bigdatasql/bdcell-12.1/bigdata-log4j.properties" >> ${report_fullpath_basename}_CBODY.html
        $echocmd "<b>Hadoop BDS Log4J Log     </b>= /opt/oracle/bigdatasql/bdcell-12.1/log" >> ${report_fullpath_basename}_CBODY.html
        $echocmd "<b>Hadoop BDS Offload Trace </b>= /var/log/oracle/diag/bdsql/cell/SYS_*/trace" >> ${report_fullpath_basename}_CBODY.html
        $echocmd >> ${report_fullpath_basename}_CBODY.html
    fi

    if [ "$cells" == "" ]; then
        # Check bigdata.hosts in bigdata.properties
        cells="`$grepcmd bigdata.hosts ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties| $cutcmd -d= -f2 | $sedcmd 's/,/ /g'`"

        if [ "$ade_mode" == "" ]; then
            if [ "$cells" == "" ]; then
              if [ -f "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/topology.map" ]; then
                while IFS='' read -r line || [[ -n "$line" ]];
                do
                    if [[ $line == *"node name"* ]]; then
                        name=`$echocmd $line | $cutcmd -d\" -f2 | $cutcmd -d. -f1`
                        cells="$cells $name"
                        read -r line
                    fi
                done < "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/topology.map"
                checkpoint_echo_bold "Cells are derived from ${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/topology.map: $cells" >> ${report_fullpath_basename}_CBODY.html
            else
                checkpoint_echo_bold "Cells are derived from ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties: $cells" >> ${report_fullpath_basename}_CBODY.html
              fi
            fi
        else
            $echocmd "<b>BDSQL Cluster            </b>= ${bdsql_cluster_name}" >> ${report_fullpath_basename}_CBODY.html
            $echocmd  >> ${report_fullpath_basename}_CBODY.html
            $echocmd "You are running $BASENAME in Oarcle ADE view name \"$ADE_VIEW_NAME\"." >> ${report_fullpath_basename}_CBODY.html
            $echocmd "Hadoop cluster cell nodes cannot be derived from ${DB_NODE_BIGDATA_CONFIG} in ADE view." >> ${report_fullpath_basename}_CBODY.html
            $echocmd "You need to specify your Hadoop cluster cell nodes with the -c option." >> ${report_fullpath_basename}_CBODY.html
            $echocmd  >> ${report_fullpath_basename}_CBODY.html
            $echocmd "e.g." >> ${report_fullpath_basename}_CBODY.html
            $echocmd "$BASENAME -d /app/oracle/product/12.1.0/dbhome_1 -s orcl -q sys -c \"hadoopc01 hadoopc02 hadoopc03\" -u oracle -v" >> ${report_fullpath_basename}_CBODY.html
        fi
    else
        checkpoint_echo_bold "Cells are derived from command line input: $cells" >> ${report_fullpath_basename}_CBODY.html
        cells=`$echocmd $cells | sed 's/,/ /g'`
    fi
    $echocmd "</pre>" >> ${report_fullpath_basename}_CBODY.html

    if [ "$cells" != "" ]; then
        $rmcmd -f $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json > /dev/null

        $catcmd <<- _EOF2_ > $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json
"${bdsql_cluster_name}":[
_EOF2_
    fi
    
    MY_CELLS=$cells

} # end of check_hadoop_cellnode_part1 


function check_hadoop_cellnode_part2
{ 
    cell_name=$1
    first_cell_name=$2
    #-------------------------------------------------
    # Convert FQDN to short name if not an ip address
    #-------------------------------------------------
    ip=`echo $cell_name | sed 's/[0-9]*//g'`

    if [ "$ip" != "..." ]; then
      cell_name=`echo $cell_name | $cutcmd -d. -f1`
    fi
    $echocmd "<h4 id="C1"$cell_name>Hadoop Cell Node $cell_name:</h4>"
    $echocmd "<pre>"

    # Check if BDA
    if [ "$bda_mode" == "1" ]; then
        tmp=`remote_command $username $cell_name $password "$bdaclicmd getinfo cluster_name" $identitykey`
        $echocmd "<b>BDA Cluster Name      </b>= $tmp"
        tmp=`remote_command $username $cell_name $password "$bdaclicmd getinfo cluster_type" $identitykey`
        $echocmd "<b>BDA Cluster Type      </b>= $tmp"
        tmp=`remote_command $username $cell_name $password "$bdaclicmd getinfo cluster_version" $identitykey`
        $echocmd "<b>BDA Cluster Version   </b>= $tmp"
    fi
    $echocmd

    $echocmd "<h3 id="C2"$cell_name><font color="blue">${CELL_TABLE_CNAME[2]}</font></h3>"
    cell_table_result[2]=$CELL_TABLE_CVALUE_PASSED

    tmp=`remote_command $username $cell_name $password "$catcmd /opt/oracle/bd_cell/cellsrv/deploy/config/cell_disk_config.xml" $identitykey > ${MY_ERROR_LOG_DIRECTORY}/cell_disk_config.xml.${cell_name}`

    checkpoint_echo "$HADOOP_CELLINIT: "
    $echocmd "<b>Hadoop Cell Init File </b>= $HADOOP_CELLINIT"
    tmp=`remote_command $username $cell_name $password "$catcmd $HADOOP_CELLINIT" $identitykey > ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name}`
    $catcmd ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name}

    ipaddress_lower=
    subnet_mask=
    ipaddress_lower=$($grepcmd -i ipaddress ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name} | $cutcmd -d= -f1 | $trcmd [:upper:] [:lower:])
    subnet_mask=$($grepcmd -i ipaddress ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name} | $cutcmd -d= -f2 | $cutcmd -d/ -f2)
    $rmcmd -f ${MY_ERROR_LOG_DIRECTORY}/.${ipaddress_lower} > /dev/null
    if [[ "$ipaddress_lower" == "ipaddress"* ]]; then
        $echocmd $subnet_mask > ${MY_ERROR_LOG_DIRECTORY}/.$ipaddress_lower
        checkpoint_echo "Cell Subnet Mask: $subnet_mask"
        if [ "$subnet_mask" != "$($catcmd ${MY_ERROR_LOG_DIRECTORY}/.$ipaddress_lower)" ]; then
            cell_table_result[2]=$CELL_TABLE_CVALUE_FAILED
            fatal_error_echo "The cell node subnet mask is different from the db node subnet mask!"
        fi
    else
        cell_table_result[2]=$CELL_TABLE_CVALUE_FAILED
        fatal_error_echo "Cell Subnet Mask is not found!"
    fi

    # If BDA, check cell node ip address against db node /etc/hosts
    if [ "$bda_mode" == "1" ]; then
        tmp=`$grepcmd $ipaddress_lower /etc/hosts | awk '{print $1}'`

        if [[ "$tmp" == "$ipaddress_lower" ]]; then
            checkpoint_echo "Cell node IP address \"$tmp\" is defined in /etc/hosts."
        else
            cell_table_result[2]=$CELL_TABLE_CVALUE_FAILED
            fatal_error_echo "Cell node IP address \"$ipaddress_lower\" is not defined in /etc/hosts!"
        fi
    fi

    bds_software_only="$($catcmd ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name} | $grepcmd $CELL_DYNAMIC_PROTOCOL | $cutcmd -d= -f2 | $trcmd -d '[blank]' | $trcmd [:upper:] [:lower:])"

    if [ "$bds_software_only" != "" ]; then
        checkpoint_echo "$HADOOP_CELLINIT: $CELL_DYNAMIC_PROTOCOL=$bds_software_only"

        case "$bds_software_only" in
          *"true"*)
            my_tcpudp_mode="$($grepcmd $DB_DYNAMIC_PROTOCOL ${MY_ERROR_LOG_DIRECTORY}/cellinit.dbhome | $cutcmd -d= -f2 | $trcmd -d '[blank]')"
            if [ "$my_tcpudp_mode" == "2" ]; then
              checkpoint_echo "You are using TCP/UDP connection!"
            else
              $echocmd "NOTE: By default, InfiniBand is used."
              $echocmd "      For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in"
              $echocmd "      $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in"
              $echocmd "      $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              $echocmd
              cell_table_action[2]="For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              fatal_error_echo "${cell_table_action[2]}"
            fi
            ;;
          *"false"*)
            if [ "$my_tcpudp_mode" == "2" ]; then
              $echocmd "NOTE: By default, InfiniBand is used."
              $echocmd "      For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in"
              $echocmd "      $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in"
              $echocmd "      $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              $echocmd
              cell_table_action[2]="For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              fatal_error_echo "${cell_table_action[2]}"
            fi
            ;;
          *)
            if [ "$my_tcpudp_mode" == "2" ]; then
              $echocmd "NOTE: By default, InfiniBand is used."
              $echocmd "      For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in"
              $echocmd "      $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in"
              $echocmd "      $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              $echocmd
              cell_table_action[2]="For TCP/UDP connection, you must have $DB_DYNAMIC_PROTOCOL=2 in $CELLINIT on you DB/Compute node and $CELL_DYNAMIC_PROTOCOL=true in $HADOOP_CELLINIT on Hadoop Data/Cell nodes."
              fatal_error_echo "${cell_table_action[2]}"
            fi
            ;;
        esac
    fi

    # CELL_AUTHENTICATION:
    # Check _cell_big_data_sql_enable_authentication in /opt/oracle/bd_cell/cellsrv/deploy/config/cellinit.ora
    #
    # If _cell_big_data_sql_enable_authentication=TRUE, 
    #    the guid/key value pair in /opt/oracle/bd_cell/cellsrv/deploy/config/cell_disk_config.xml on cell node 
    #    must match $OH/admin/<dbname>/pfile/bdsql_cellkey.ora of db node
    #
    bds_enable_authentication="$($catcmd ${MY_ERROR_LOG_DIRECTORY}/cellinit.${cell_name} | $grepcmd $CELL_AUTHENTICATION | $cutcmd -d= -f2 | $trcmd -d '[blank]' | $trcmd [:upper:] [:lower:])"

    if [ "$bds_enable_authentication" != "" ]; then
        checkpoint_echo "$HADOOP_CELLINIT: $CELL_AUTHENTICATION=$bds_enable_authentication"
        $echocmd "NOTE: If _cell_big_data_sql_enable_authentication=TRUE,"
        $echocmd "      the guid/key value pair in /opt/oracle/bd_cell/cellsrv/deploy/config/cell_disk_config.xml on cell node"
        $echocmd "      must match the guid/key in ${oracle_home}/admin/${db_unique_name}/pfile/bdsql_cellkey.ora on database node."
        $echocmd

        case "$bds_enable_authentication" in
          *"true"*)
            my_db_guid="`$grepcmd -i guid $ORACLE_HOME/admin/${db_unique_name}/pfile/bdsql_cellkey.ora | $cutcmd -d= -f2`"
            my_db_key="`$grepcmd -i key $ORACLE_HOME/admin/${db_unique_name}/pfile/bdsql_cellkey.ora | $cutcmd -d= -f2`"
            checkpoint_echo_bold "${oracle_home}/admin/${db_unique_name}/pfile/bdsql_cellkey.ora:"
            $echocmd "GUID = $my_db_guid"
            $echocmd

            checkpoint_echo_bold "${MY_ERROR_LOG_DIRECTORY}/cell_disk_config.xml.${cell_name}:"
            x="`$grepcmd -n $my_db_guid ${MY_ERROR_LOG_DIRECTORY}/cell_disk_config.xml.${cell_name} | $grepcmd oracle.ossmgmt.ms.core.MSClient | $cutcmd -d: -f1`"
            y=1
            ans=$(( x + y ))
            my_cell_key="`sed -n ${ans}p ${MY_ERROR_LOG_DIRECTORY}/cell_disk_config.xml.${cell_name} | sed 's/<\/Attribute>//g' | awk '{print $3}' | $cutcmd -d'"' -f2`"
            $echocmd "GUID = $my_db_guid"

#---------------------------------------------------------------          
#           Temporarily commented out for now.
#---------------------------------------------------------------          
#           if [ "${my_cell_key}" == "${my_db_key}" ]; then
#             checkpoint_echo_bold "GUID KEYS ARE IDENTICAL!"
#           else
#             checkpoint_echo_bold "GUID KEYS ARE DIFFERENT!"
#             fatal_error_echo "GUID KEYS ARE DIFFERENT!"
#           fi
            ;;
        esac
    fi

    tmp=`remote_command $username $cell_name $password "$ifconfigcmd -a 2>/dev/null" $identitykey`
    verbose_echo "<b>Output of ifconfig    </b>= $tmp"
    $echocmd 
    $echocmd "<b>Hadoop BDS Cell Trace </b>= /var/log/oracle/diag/bdsql/cell/$cell_name/trace"

#   $echocmd "<h3 id="C2"$cell_name><font color="blue">${CELL_TABLE_CNAME[2]}</font></h3>"

#   if [ "${cell_name}" == "${first_cell_name}" ]; then
      run_cellnode_bdsql_sanity_test ${cell_name} ${first_cell_name}
#   fi

    $echocmd "<h3 id="C3"$cell_name><font color="blue">${CELL_TABLE_CNAME[3]}</font></h3>"
    id_tmp=`remote_command $username $cell_name $password "id -ng oracle" $identitykey`
    checkpoint_echo "$id_tmp"

    if [[ $id_tmp == *"oinstall"* ]]; 
      then
        cell_table_result[3]=$CELL_TABLE_CVALUE_PASSED
      else
        cell_table_result[3]=$CELL_TABLE_CVALUE_FAILED
        cell_table_action[3]="Make sure that Oracle installation owner (oracle) belongs to Oracle inventory group (oinstall)."
        fatal_error_echo "${cell_table_action[3]}"
    fi

    $echocmd "<h3 id="C6"$cell_name><font color="blue">${CELL_TABLE_CNAME[6]}</font></h3>"
    tmp=`remote_command $username $cell_name $password "bdscli -e list bdsql" $identitykey`
    checkpoint_echo "$tmp"

    if [[ $tmp == *"online"* ]]; 
      then
        cell_table_result[6]=$CELL_TABLE_CVALUE_PASSED
      else
        cell_table_result[6]=$CELL_TABLE_CVALUE_FAILED
        cell_table_action[6]="Run bdscli with an appropriate sub-command to bring up Oracle Big Data SQL service."
        fatal_error_echo "${cell_table_action[6]}"
    fi

    tmp=`remote_command $username $cell_name $password "$pscmd -aef | $grepcmd bds | $grepcmd -v grep" $identitykey`
    checkpoint_echo "$tmp"
    ans=$($echocmd $tmp | $grepcmd bdsqloflsrv)

    if [ "$ans" == "" ]; then
       cell_table_result[6]=$CELL_TABLE_CVALUE_FAILED
       fatal_error_echo "Error! bdsqloflsrv is not up and running."
    fi

    $echocmd "<h3 id="C4"$cell_name><font color="blue">${CELL_TABLE_CNAME[4]}</font></h3>"
    tmp=`remote_command $username $cell_name $password "bdscli -e list quarantine" $identitykey`
    checkpoint_echo "$tmp"

    case "$tmp" in
      *"command not found"*)
        cell_table_result[4]=$CELL_TABLE_CVALUE_FAILED
        cell_table_action[4]="Make sure that you have access to bdscli."
        fatal_error_echo "${cell_table_action[4]}"
        ;;
      *"Permission denied"*)
        cell_table_result[4]=$CELL_TABLE_CVALUE_FAILED
        cell_table_action[4]="Make sure that you have permission to run bdscli."
        fatal_error_echo "${cell_table_action[4]}"
        ;;
      "")
        cell_table_result[4]=$CELL_TABLE_CVALUE_FAILED
        cell_table_action[4]="Double check that if there is indeed no quarantine."
        fatal_error_echo "${cell_table_action[4]}"
        ;;
      *)
        cell_table_result[4]=$CELL_TABLE_CVALUE_PASSED
        ;;
    esac

    $echocmd "<h3 id="C5"$cell_name><font color="blue">${CELL_TABLE_CNAME[5]}</font></h3>"
    tmp=`remote_command $username $cell_name $password "bdscli -e list alerthistory | $tailcmd -10" $identitykey`
    checkpoint_echo "$tmp"

    case "$tmp" in
      *"_1"*)
        cell_table_result[5]=$CELL_TABLE_CVALUE_PASSED
        ;;
      *)
        cell_table_result[5]=$CELL_TABLE_CVALUE_NA
        cell_table_action[5]="Double check that if there is indeed no alter history."
        fatal_error_echo "${cell_table_action[5]}"
        ;;
    esac

    $echocmd "<h3 id="C7"$cell_name><font color="blue">${CELL_TABLE_CNAME[7]}</font></h3>"
    jce_tmp=`remote_command $username $cell_name $password "$lscmd -l /usr/java/default/jre/lib/security/*.jar" $identitykey`
    checkpoint_echo "$jce_tmp"

    case "$jce_tmp" in
      *"_policy.jar"*)
        cell_table_result[7]=$CELL_TABLE_CVALUE_PASSED
        ;;
      *)
        cell_table_result[7]=$CELL_TABLE_CVALUE_NA
        cell_table_action[7]="Double check your Big Data SQL installation log and make sure that JCE is properly installed."
        fatal_error_echo "${cell_table_action[7]}"
        ;;
    esac
    verbose_echo " If you are using CentOS or Red Hat Enterprise Linux 5.5 or later, which use AES-256 encryption by default for tickets, you must install the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy File on all cluster and Hadoop user machines. For JCE Policy File installation instructions, see the README.txt file included in the jce_policy-x.zip file.

Alternatively, you can configure Kerberos to not use AES-256 by removing aes256-cts:normal from the supported_enctypes field of the kdc.conf or krb5.conf file. Note that after changing the kdc.conf file, you'll need to restart both the KDC and the kadmin server for those changes to take affect. You may also need to recreate or change the password of the relevant principals, including potentially the Ticket Granting Ticket principal (krbtgt/REALM@REALM). If AES-256 is still used after all of those steps, it's because the aes256-cts:normal setting existed when the Kerberos database was created. To fix this, create a new Kerberos database and then restart both the KDC and the kadmin server."

    jce_tmp=`remote_command $username $cell_name $password "$lscmd -l ${JAVA_HOME}/jre/lib/security/*.jar" $identitykey`
    checkpoint_echo "$jce_tmp"

    $echocmd "<h3 id="C8"$cell_name><font color="blue">${CELL_TABLE_CNAME[8]}</font></h3>"
    tmp=`remote_command $username $cell_name $password "$klistcmd -e" $identitykey`
    checkpoint_echo "$tmp"

    case "$tmp" in
      *"No credentials cache found"*)
        fatal_error_echo "No credentials cache found!"
        cell_table_action[8]="Double check that if there is indeed no Kerberos tickets in use."
        fatal_error_echo "${cell_table_action[8]}"
        ;;
      *"No route to host"*)
        fatal_error_echo "Failed to connect to host!"
        cell_table_action[8]="Double check that this Hadoop cluster node is accessible."
        fatal_error_echo "${cell_table_action[8]}"
        ;;
      *)
        ;;
    esac

    tmp=`remote_copy $username@${cell_name}:/opt/oracle/bigdatasql/bdcell-hadoopconf/core-site.xml ${MY_ERROR_LOG_DIRECTORY}/${cell_name}_core-site.xml $password $identitykey`

    tmp=`get_value_by_xml_name $MY_ERROR_LOG_DIRECTORY/${cell_name}_core-site.xml hadoop.security.authentication`
    checkpoint_echo_bold "core-site.xml: hadoop.security.authentication=$tmp"

    tmp=`get_value_by_xml_name $MY_ERROR_LOG_DIRECTORY/${cell_name}_core-site.xml hadoop.security.authorization`
    checkpoint_echo_bold "core-site.xml: hadoop.security.authorization=$tmp"

    case "$tmp" in
      *"true"*)
        cell_table_result[8]=$CELL_TABLE_CVALUE_ON
        ;;
      *"false"*)
        cell_table_result[8]=$CELL_TABLE_CVALUE_OFF
        ;;
      *)
        if ["${db_table_result[8]}" == "${DB_TABLE_CVALUE_ON}" ]; then
            cell_table_result[8]=${CELL_TABLE_CVALUE_FAILED}
        else
            cell_table_result[8]=${CELL_TABLE_CVALUE_NA}
        fi
        ;;
    esac

    tmp=`remote_command $username $cell_name $password "$catcmd /var/kerberos/krb5kdc/kdc.conf" $identitykey`
    checkpoint_echo "$tmp"

    tmp=`remote_command $username $cell_name $password "$catcmd /etc/krb5.conf" $identitykey`
    checkpoint_echo "$tmp"

    tmp=`remote_command $username $cell_name $password "$tailcmd -10 /var/log/krb5kdc.log" $identitykey`
    checkpoint_echo "$tmp"

    $echocmd "<h3 id="C9"$cell_name><font color="blue">${CELL_TABLE_CNAME[9]}</font></h3>"
    tmp=`remote_command $username $cell_name $password "$nmapcmd -F -sV localhost 2>/dev/null" $identitykey`
    checkpoint_echo "$tmp"

    if [ "`$echocmd $tmp | $grepcmd -i openssl 2>/dev/null`" == "" ]; then
        nonfatal_error_echo "No TLS/SSL service found!"
        cell_table_action[9]="Double check that if there is indeed no TLS/SSL service configured on your host."
        nonfatal_error_echo "${cell_table_action[9]}"
    fi

    tmp=`get_value_by_xml_name $MY_ERROR_LOG_DIRECTORY/${cell_name}_core-site.xml hadoop.ssl.enabled`
    checkpoint_echo_bold "core-site.xml: hadoop.ssl.enabled=$tmp"

    case "$tmp" in
      *"true"*)
        cell_table_result[9]=$CELL_TABLE_CVALUE_ON
        ;;
      *"false"*)
        cell_table_result[9]=$CELL_TABLE_CVALUE_OFF
        ;;
      *)
        cell_table_result[9]=$CELL_TABLE_CVALUE_NA
        ;;
    esac

    if [ "$trace_mode" == "1" ]; then
        turn_off_tracing
    fi

    if [ "$x_mode" == "1" ]; then
        check_cell_ipsec ${cell_name}
    fi

    $catcmd <<- _EOF_ > $MY_ERROR_LOG_DIRECTORY/bdsql_${cell_name}_result.txt
${cell_table_result[1]}
${cell_table_result[2]}
${cell_table_result[3]}
${cell_table_result[4]}
${cell_table_result[5]}
${cell_table_result[6]}
${cell_table_result[7]}
${cell_table_result[8]}
${cell_table_result[9]}
_EOF_

    print_hadoop_cellnode_table $cell_name > ${report_fullpath_basename}_CTABLE_${cell_name}.html

    $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json
    $(print_cellnode_table_json)
_EOF_

    $echocmd "</pre>"

} # end of check_hadoop_cellnode_part2


#function check_hadoop_cellnode
#{
#    $catcmd <<- _EOF_ > ${report_fullpath_basename}_CBODY_${1}.html 
#    $(check_hadoop_cellnode_part2) ${1}
#_EOF_
#
#}


function check_hadoop_cellnodes_in_parallel
{
    check_hadoop_cellnode_part1

#   $echocmd "Debug: MY_CELLS = $MY_CELLS"

    first_cell_name=`$echocmd $MY_CELLS | $awkcmd '{print $1}'`
    pid_list=
    for my_cell_name in $MY_CELLS
    do
        $echocmd "Checking Hadoop cell node [$my_cell_name] in parallel..."
        nohup `check_hadoop_cellnode_part2 ${my_cell_name} ${first_cell_name} > ${report_fullpath_basename}_CBODY_${my_cell_name}.html` 2>${MY_ERROR_LOG_DIRECTORY}/check_cellnode_${my_cell_name}.err &
        pid_list="$pid_list $!"
    done

    spinning_cursor2 $pid_list

    if [ "$MY_CELLS" != "" ]; then
        $catcmd <<- _EOF_ >> $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json
]
_EOF_
    fi

}   # end of check_hadoop_cellnodes_in_parallel


function check_hs_cores
{
    hs_core=
    for i in ${oracle_home}/rdbms/log/hs_core_*
    do
      if [ -d $i ]; then
        hs_core=$i
        end_time=$(/usr/bin/stat -c "%Z" $i)
    
        if [[ ${start_time} < ${end_time} ]]; then
          #--------------------------
          # New hs_core_* is found!
          #--------------------------
          db_table_result[$1]=${DB_TABLE_CVALUE_FAILED}
          fatal_error_echo "A new HS core file is found at $i"
          fatal_error_echo "If error persists, contact Oracle Worldwide Customer support."
          cat $i/hs_core_trace_*.trc | sed -e '/Binary Stack Dump/,$d'
        fi
      fi
    done
}


function prompt_for_user_input
{
    unset PASSWORD
    unset CHARCOUNT

    stty -echo

    CHARCOUNT=0
    while IFS= read -p "$PROMPT" -r -s -n 1 CHAR
    do
        # Enter - accept password
        if [[ $CHAR == $'\0' ]] ; then
            break
        fi
        # Backspace
        if [[ $CHAR == $'\177' ]] || [[ $CHAR == $'\b' ]] ; then
            if [ $CHARCOUNT -gt 0 ] ; then
                CHARCOUNT=$((CHARCOUNT-1))
                PROMPT=$'\b \b'
                PASSWORD="${PASSWORD%?}"
            else
                PROMPT=''
            fi
        else
            CHARCOUNT=$((CHARCOUNT+1))
            PROMPT='*'
            PASSWORD+="$CHAR"
        fi
    done

    stty echo
    echo $PASSWORD
}


function check_bigdata_config
{
    if [ -d "${oracle_home}/bigdatasql/bigdata_config" ]; then
        db_node_bigdata_home_31="$oracle_home/bigdatasql"
    else
        if [ -d "${oracle_home}/bigdatasql/databases/${db_unique_name}/bigdata_config" ]; then
            db_node_bigdata_home_32="$oracle_home/bigdatasql/databases/${db_unique_name}"
        fi
    fi

    if [ "$db_node_bigdata_home_31" != "" ] && [ "$db_node_bigdata_home_32" != "" ]
    then # dup config
        DB_NODE_BIGDATA_CONFIG=
        $echocmd "Warning! Duplicate bigdata_config."
        $echocmd "1. ${db_node_bigdata_home_31}/bigdata_config"
        $echocmd "2. ${db_node_bigdata_home_32}/bigdata_config"
    elif [ "$db_node_bigdata_home_32" != "" ]
    then
        DB_NODE_BIGDATA_CONFIG="${db_node_bigdata_home_32}/bigdata_config"
        DB_NODE_DEFAULT_DIR="${db_node_bigdata_home_32}/default_dir"
    elif [ "$db_node_bigdata_home_31" != "" ]
    then
        DB_NODE_BIGDATA_CONFIG="${db_node_bigdata_home_31}/bigdata_config"
        DB_NODE_DEFAULT_DIR="${db_node_bigdata_home_31}/default_dir"
    else
        DB_NODE_BIGDATA_CONFIG=
        $echocmd "Warning! bigdata_config is not found."
    fi
}


function run_as
{
    os_user=$1
    os_passwd=$2
    os_command=$3

    echo $os_passwd | /usr/bin/sudo -S -E -u $os_user $os_command 2>/dev/null
}


check_the_dbnode()
{
    $catcmd <<- _EOF_ > ${report_fullpath_basename}_DBBODY.html
    $(check_dbnode)
_EOF_
}


print_report_body()
{
    $echocmd
    $echocmd "Checking Database/Compute node [$hostname]..."
    nohup `check_the_dbnode` 2>${MY_ERROR_LOG_DIRECTORY}/check_the_dbnode.err &
    spinning_cursor
    $echocmd

    $catcmd <<- _EOF_ > ${report_fullpath_basename}_DBTABLE.html
    $(print_dbnode_table)
_EOF_

    check_hadoop_cellnodes_in_parallel

}   # end of print_report_body


print_report_head()
{
    $catcmd <<- _EOF_ > ${report_fullpath_basename}_HEAD.html
<html>
    <head>
    <title>$TITLE</title>
    </head>
    <body>
        <h1>$TITLE</h1>
        <p>$TIME_STAMP</p>
_EOF_

}   # end of print_report_head


print_report_tail()
{
    $catcmd <<- _EOF_ > ${report_fullpath_basename}_TAIL.html
    </body>
</html>
_EOF_

}   # end of print_report_tail


generate_report()
{
    $catcmd ${report_fullpath_basename}_HEAD.html > ${html_filename}
    $rmcmd -f ${report_fullpath_basename}_HEAD.html > /dev/null
    $catcmd ${report_fullpath_basename}_DBTABLE.html >> ${html_filename}
    $rmcmd -f ${report_fullpath_basename}_DBTABLE.html > /dev/null

    for my_cell in ${MY_CELLS}
    do
        $catcmd ${report_fullpath_basename}_CTABLE_${my_cell}.html >> ${html_filename}
#       $rmcmd -f ${report_fullpath_basename}_CTABLE_${my_cell}.html 
    done

    $catcmd ${report_fullpath_basename}_DBBODY.html >> ${html_filename}
    $rmcmd -f ${report_fullpath_basename}_DBBODY.html > /dev/null

    $catcmd ${report_fullpath_basename}_CBODY.html >> ${html_filename}
    $rmcmd -f ${report_fullpath_basename}_CBODY.html > /dev/null

    for my_cell in ${MY_CELLS}
    do
        $catcmd ${report_fullpath_basename}_CBODY_${my_cell}.html >> ${html_filename}
        $rmcmd -f ${report_fullpath_basename}_CBODY_${my_cell}.html > /dev/null
    done

    $catcmd ${report_fullpath_basename}_TAIL.html >> ${html_filename}
    $rmcmd -f ${report_fullpath_basename}_TAIL.html > /dev/null

    $catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_dbnode_result.json > $json_filename
    $rmcmd -f $MY_ERROR_LOG_DIRECTORY/bdsql_dbnode_result.json > /dev/null

    if [ -f ${MY_ERROR_LOG_DIRECTORY}/bdsql_cluster_result.json ]; then
        $catcmd $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json >> $json_filename
        $rmcmd -f $MY_ERROR_LOG_DIRECTORY/bdsql_cluster_result.json > /dev/null
    fi

}   # end of generate_report


bds_check_sw()
{
    print_report_head

    print_report_body

    print_report_tail

    generate_report

}   # end of bds_check_sw


usage()
{
    $echocmd
    $echocmd "Usage: $BASENAME [OPTION]..."
    $echocmd $BDSCHECKSW_BANNER
    $echocmd 
    $echocmd "  -h, --help                 display this help and exit"
    $echocmd "  -d, --dbhome=ORACLE_HOME   DB node ORACLE_HOME (mandatory if \$ORACLE_HOME is not set)"
    $echocmd "  -s, --sid=ORACLE_SID       DB node ORACLE_SID (mandatory if \$ORACLE_SID is not set)"
    $echocmd "  -p, --pdb=PDB_CONTAINER    DB node PDB container name"
    $echocmd "  -g, --gihome=GI_HOME       DB node GI_HOME (mandatory if \$GI_HOME is not set)"
    $echocmd "  -y, --giuser               DB node GI_HOME administrator name"
    $echocmd "  -q, --sqlplus              DB node Oracle username (mandatory)"
    $echocmd "  -c, --cell                 Hadoop cluster cell nodes (mandatory for Hortonworks)"
    $echocmd "  -u, --uname                Hadoop cluster node username to run remote commands (mandatory)"
    $echocmd "  -k, --key                  Secure Shell (SSH) identity key file"
    $echocmd "  -r, --cluster              Hadoop cluster name"
    $echocmd "  -f, --file=FILE            redirect output to FILE"
    $echocmd "  -i, --interactive          enter command line arguments interactively"
    $echocmd "  -t, --trace                turn on extproc and log4j tracing during test execution"
    $echocmd "  -x, --extensive            extensive mode (root privilege is required)"
    $echocmd "  -v, --verbose              verbose mode (recommended for great details in the report)"
    $echocmd 
    $echocmd "Exit status:"
    $echocmd " 0  if OK,"
    $echocmd " 1  if minor proglems (e.g., no response in interactive mode,),"
    $echocmd " 2  if serious trouble (e.g., invalid command-line argument)."
    $echocmd 
}   # end of usage


##=============================================================================
#  Main Program
#
#  The bdschecksw main program starts from here. 
#
#  1. Assign default values to global variables
#  2. Check if BDA and if ADE ?
#  3. Process command line options
#  4. Process command line options interactively if necessary
#  5. Conduct config sanity checks for the pre-requisites of bdschecksw
#  6. Call the main routine bds_check_sw
#
#      bds_check_sw
#          print_report_head
#          print_report_body
#          print_report_tail
#          generate_report
#
#  For the main routine bds_check_sw, what it does are:
#  1. print_report_head - pre-process and save the report header
#  2. print_report_body - pre-process and save the report body
#  3. print_report_tail - pre-process and save the report tail
#  4. generate_report   - process and consolidate the entire report 
#
#  The core sub-routine is print_report_body:
#
#          print_report_body
#              check_the_dbnode
#                  check_dbnode
#                      check_database_initparams
#                      check_bdsql_files
#                      check_database_objects
#                      check_database_extproc
#                      check_prerequisites
#                      run_dbnode_bdsql_sanity_test
#              check_hadoop_cellnodes_in_parallel
#                  check_hadoop_cellnode_part1 ... once
#                  check_hadoop_cellnode_part2 ... in a loop
#                      run_cellnode_bdsql_sanity_test
#
##=============================================================================


##----------------------------------------------
#  1. Assign default values to global variables
##----------------------------------------------
oracle_home=$ORACLE_HOME
oracle_base=$ORACLE_BASE
oracle_sid=$ORACLE_SID
gi_home=$GI_HOME
gi_user=
gi_passwd=
pdb_container=
crs_home=
html_filename=~/${BASENAME}_${RIGHT_NOW2}.html
html_filename_link=~/${BASENAME}.html
text_filename=~/${BASENAME}_${RIGHT_NOW2}.txt
text_filename_link=~/${BASENAME}.txt
interactive=
x_mode=
db_root_passwd=
cell_root_passwd=
trace_mode=
verbose_mode=
bda_mode=
ade_mode=
tcpudp_mode=
bds_software_only=
bds_enable_authentication=
username=
password=
identitykey=
cells=
sqlplus_uid=
sqlplus_passwd=
sqlplus_uid_passwd=
db_node_bigdata_home_31=
db_node_bigdata_home_32=
bigdata_cluster_default=
cluster_name=
bdsql_cluster_name=
hostname=`$hostnamecmd|$sedcmd "s/\..*//"`
retry_err_count_before=
retry_err_count_after=
smart_scan_bytes_before=
smart_scan_bytes_after=
db_table_result=('N.A' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.')
cell_table_result=('N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.')
db_table_action=('N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.')
cell_table_action=('N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.' 'N.A.')
start_time=$(/bin/date +%s)
end_time=
hs_core=
db_unique_name=
db_name=
instance_name=


#
# NOTE: In theory, there should be only one bigdata_config but in case that 
#       users installed BDS 3.2 without de-install BDS 3.1, we need to 
#       accomodate this scenario. The real bigdata_config in use is the 
#       ORACLE_BIGDATA_CONFIG in all_directories table.
#       It's critical to identify bigdata_config before we can derive the 
#       bigdata.cluster.default from bigdata_config/bigdata.properties.
#


##----------------------------------------------
#  2. Check if BDA and if ADE ?
##----------------------------------------------
# Check if BDA
if [ -x $bdaclicmd ]; then
    bda_mode=1
fi

# Check if ADE (additional customization might be needed if ade_mode == 1)
if [ "$ADE_VIEW_NAME" != "" ]; then
    ade_mode=1
    if [ ! -e $oracle_home/bigdatasql ]; then
        $lncmd -s $T_WORK $oracle_home/bigdatasql > /dev/null
    fi
    if [ ! -d $T_WORK/log ]; then
        mkdir $T_WORK/log
    fi
    if [ ! -d $T_WORK/default_dir ]; then
        mkdir $T_WORK/default_dir
    fi
fi


##---------------------------------
#  3. Process command line options
##---------------------------------
while [[ $# -gt 0 ]]
do
  key="$1"

  case $key in
    -d|--dbhome)
      oracle_home="$2"
      shift # past argument
      ;;
    -s|--sid)
      oracle_sid="$2"
      shift # past argument
      ;;
    -p|--pdb)
      pdb_container="$2"
      shift # past argument
      ;;
    -u|--username)
      username="$2"
      shift # past argument
      ;;
    -g|--gihome)
      gi_home="$2"
      shift # past argument
      ;;
    -x|--extensive)
      x_mode=1
      shift # past argument
      ;;
    -y|--giuser)
      gi_user="$2"
      shift # past argument
      ;;
    -k|--key)
      identitykey="$2"
      shift # past argument
      ;;
    -q|--sqlplus)
      sqlplus_uid="$2"
      shift # past argument
      ;;
    -r|--cluster)
      cluster_name="$2"
      shift # past argument
      ;;
    -c|--cell)
      cells="$2"
      shift # past argument
      ;;
    -i|--interactive)
      interactive=1
      $echocmd "interactive=$interactive"
      ;;
    -t|--trace)
      trace_mode=1
      ;;
    -v|--verbose)
      verbose_mode=1
      ;;
    -h|--help)
      usage
      exit
      ;;
    *) # unknown option
      usage
      exit 2
      ;;
  esac
  shift # past argument or value
done


if [[ $oracle_home == "" ]];
then
    interactive=1
else
    if [[ $oracle_sid == "" ]];
    then
        interactive=1
    else
        if [[ $username == "" ]];
        then
            interactive=1
        else
            if [[ $sqlplus_uid == "" ]];
            then
                interactive=1
            fi
        fi
    fi
fi

if [ "$password" == "" ]; then
    if [[ $identitykey == "" ]];
    then
        interactive=1
    fi
fi

##------------------------------------------------------------
#  4. Process command line options interactively if necessary
##------------------------------------------------------------
while [[ "$interactive" == "1" ]]
do
#if [ "$interactive" == "1" ]; then

    response=

    if [ ! -d "$oracle_home" ]; then 
        $echocmd
        $echocmd "Error! Invalid ORACLE_HOME=\"$oracle_home\""
        $echocmd
        oracle_home=

        $echocmd -n "Enter DB Node ORACLE_HOME > "
        read response
        if [ -n "$response" ]; then
            oracle_home=$response
        fi
        if [ ! -d "$oracle_home" ]; then 
            oracle_home=
            continue;
        fi
    else
        interactive=
    fi

    if [ "$oracle_sid" == "" ]; then 
        $echocmd -n "Enter DB Node ORACLE_SID > "
        read response
        if [ -n "$response" ]; then
            oracle_sid=$response
        fi
        if [ "$oracle_sid" == "" ]; then 
            $echocmd
            $echocmd "Error! Invalid ORACLE_SID=\"$oracle_sid\""
            $echocmd
            interactive=1
            continue;
        else
            interactive=
        fi
    else
        interactive=
    fi

    if [ "${sqlplus_uid}" == "" ]; then 
        $echocmd -n "Enter DB Node SQLPlus user name> "
        read response
        if [ -n "$response" ]; then
            sqlplus_uid=$response
            interactive=
        fi
        if [ "${sqlplus_uid}" == "" ]; then 
            $echocmd
            $echocmd "Error! Invalid SQLPlus user name=\"${sqlplus_uid}\""
            $echocmd
            interactive=1
            continue;
        else
            interactive=
        fi
    else
        interactive=
    fi

    if [ "$username" == "" ]; then 
        $echocmd -n "Enter Hadoop Node username > "
        read response
        if [ -n "$response" ]; then
            username=$response
        fi
        if [ "$username" == "" ]; then 
            echo
            echo "Error! Invalid username=\"$username\""
            echo
            interactive=1
            continue;
        else
            interactive=
        fi
    else
        interactive=
    fi

#   $echocmd -n "Enter name of output file [$html_filename] > "
#   read response
#   if [ -n "$response" ]; then
#       html_filename=$response
#       text_filename="$response.txt"
#       interactive=
#   fi

    if [ -f $html_filename ]; then
        $echocmd -n "Output file exists. Overwrite? (y/n) > "
        read response
        if [ "$response" != "y" ]; then
            $echocmd "Exiting program."
            exit 1
        fi
        interactive=
    fi
#fi
done # while loop


# Prompt users for root passwords if extensive mode
if [ "$x_mode" == "1" ]; then
    unset db_root_passwd
    $echocmd
    $echocmd -n "Enter sudo password for DB node OS user '`$whoamicmd`': "
    db_root_passwd=$(prompt_for_user_input)
    $echocmd

    tmp="$(run_as root $db_root_passwd $whoamicmd)"

    if [ "$tmp" != "root" ]; then
        exit 1
    else
        $echocmd "sudo password has been verified!"
    fi

    $echocmd -n "Enter root password for Hadoop cluster node OS user '$username': "
    cell_root_passwd=$(prompt_for_user_input)
    $echocmd
fi

#----------------------
# Before block comment
#----------------------
if [ ]; then
if [ "$x_mode" == "1" ] || [ "$gi_user" != "" ]; then
    unset cell_root_passwd
    $echocmd
    $echocmd -n "Enter password for Cell node OS user \"root\": "
    cell_root_passwd=$(prompt_for_user_input)
    $echocmd
fi
fi
#----------------------
# After block comment
#----------------------


# Prompt users for sqlplus password
echo
unset sqlplus_passwd;
echo -n "Enter password for DB node SQLPlus user \"${sqlplus_uid}\": "
sqlplus_passwd=$(prompt_for_user_input)
echo

if [ "$sqlplus_uid" == "sys" ] || [ "$sqlplus_uid" == "system" ]; then
    sqlplus_uid_passwd="${sqlplus_uid}/${sqlplus_passwd} as sysdba"
else
    sqlplus_uid_passwd=${sqlplus_uid}/${sqlplus_passwd}
fi

# Verify sqlplus password. The bdschecksw simply exit if password is wrong.
sqlplus -s -L /NOLOG <<EOF
whenever sqlerror exit 1
whenever oserror exit 1
CONNECT $sqlplus_uid_passwd
exit
EOF

if [ $? -ne 0 ];
then
    echo "Failed to connect to Oracle!"
    exit 1
else
    echo -n "The password is successfully authenticated!"
    echo
    echo
fi


# Once sqlplus password is verified, we can run queries against the DB node.
#

$rmcmd -f /tmp/db_unique_name.lst > /dev/null
$rmcmd -f /tmp/db_name.lst > /dev/null
$rmcmd -f /tmp/instance_name.lst > /dev/null

sqlplus -s -L /NOLOG <<EOF >/dev/null
set heading off
set echo off
set serveroutput off
CONNECT $sqlplus_uid_passwd
spool /tmp/db_unique_name
show parameter db_unique_name
spool off
spool /tmp/db_name
show parameter db_name
spool off
spool /tmp/instance_name
show parameter instance_name
spool off
exit
EOF

db_unique_name="`$grepcmd db_unique_name /tmp/db_unique_name.lst | $awkcmd '{print $3}'`"
db_name="`$grepcmd db_name /tmp/db_name.lst | $awkcmd '{print $3}'`"
instance_name="`$grepcmd instance_name /tmp/instance_name.lst | $awkcmd '{print $3}'`"

if [[ "$db_unique_name" == "" ]] || [[ "$db_name" == "" ]] || [[ "$instance_name" == "" ]] ; then
    echo DB_UNIQUE_NAME=$db_unique_name
    echo DB_NAME=$db_name
    echo INSTANCE_NAME=$instance_name
    echo "Error! DB_UNIQUE_NAME or DB_NAME or INSTANCE_NAME is NULL!"
    exit 1
fi

check_bigdata_config
bigdata_cluster_default="`$grepcmd bigdata.cluster.default ${DB_NODE_BIGDATA_CONFIG}/bigdata.properties| $cutcmd -d= -f2`"

if [ -f $oracle_home/bigdatasql/hadoop_${cluster_name}.env ]
then
    bdsql_cluster_name="$cluster_name"
else
    bdsql_cluster_name="$bigdata_cluster_default"
fi


# Prompt users for remote login password for cell nodes
unset password;
echo -n "Enter password for Hadoop cluster node OS user \"${username}\": "
password=$(prompt_for_user_input)
echo

if [ "$password" == "" ]; then
    password="<NULL>"
fi

if [ "$cells" == "" ]; then
  if [ -f "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/topology.map" ]; then
    while IFS='' read -r line || [[ -n "$line" ]];
    do
        if [[ $line == *"node name"* ]]; then
            name=`$echocmd $line | $cutcmd -d\" -f2 | $cutcmd -d. -f1`
            cells="$cells $name"
            read -r line
        fi
    done < "${DB_NODE_BIGDATA_CONFIG}/${bdsql_cluster_name}/topology.map"
  fi
fi

for cn in `$echocmd $cells`
do
   tmp=`remote_command $username $cn $password "/bin/uname -n" $identitykey`
   if [[ "$tmp" != *"Permission denied"* ]]
   then
      echo -n "The password for $username@$cn is successfully authenticated!"
      echo
   else
      echo -n "Error! Authentication failed for $username@$cn."
      echo
      exit 1
   fi
done


if [ "$ade_mode" == "1" ]; then
    crs_home=
else
    if [ -f /etc/oracle/olr.loc ]; then
        crs_home="`$grepcmd crs_home /etc/oracle/olr.loc | $cutcmd -d= -f2 2>/dev/null`"
    else
        crs_home=
    fi
fi


# Prompt users for gi_passwd if gi_user is specified
if [ "$x_mode" != "1" ] && [ "$gi_user" != "" ]; then
    unset gi_passwd;
    echo -n "Enter password for DB node OS user \"${gi_user}\": "
    gi_passwd=$(prompt_for_user_input)
    echo
fi


##---------------------------------------------------------------------
#  5. Conduct config sanity checks for the pre-requisites of bdschecksw
##---------------------------------------------------------------------
if [ -d "$oracle_home" ] 
then
    if [ -d "$oracle_home/bigdatasql" ] 
    then
        export ORACLE_HOME=$oracle_home
        export ORACLE_SID=$oracle_sid
        export PATH=$oracle_home/bin:$PATH
        export LD_LIBRARY_PATH=$oracle_home/lib:${LD_LIBRARY_PATH}

        if [ -d "$gi_home" ]; then 
          export GI_HOME=$gi_home
          export TNS_ADMIN=$gi_home/network/admin
        else
          if [ -d "$crs_home" ]; then 
            export GI_HOME=$crs_home
            export TNS_ADMIN=$crs_home/network/admin
            oracle_base=$(cd $crs_home/../../..; pwd)
            if [ -d "$oracle_base" ]; then 
                export ORACLE_BASE=$oracle_base
            fi
          fi
        fi
    else
        $echocmd
        $echocmd "Error! \"$oracle_home/bigdatasql\" doesn't exist."
        usage
        exit 2
    fi
else
    $echocmd
    $echocmd "Error! Invalid ORACLE_HOME=\"$oracle_home\""
    usage
    exit 2
fi

if [ "$oracle_base" == "" ]; then
    export ORACLE_BASE="$ORACLE_HOME/../../.."
fi

if [ "$ade_mode" == "" ]; then
    mta_init_file_pathname=`$lscmd $oracle_home/hs/admin/init*_${db_unique_name}_${bdsql_cluster_name}.ora`
else
    mta_init_file_pathname=`$lscmd $oracle_home/hs/admin/initBDSQL_${bdsql_cluster_name}.ora`
fi

hadoop_env_file=$oracle_home/bigdatasql/hadoop_${bdsql_cluster_name}.env

if [ ! -f ${hadoop_env_file} ]; then
    $echocmd
    $echocmd "Error! \"${hadoop_env_file}\" doesn't exist."
    usage
    exit 2
fi

if [ "$ade_mode" == "1" ]; then
    if [ ! -e $T_WORK/hadoop_${bdsql_cluster_name}.env ]; then
        $lncmd -s $T_WORK/hadoop_env.sh $T_WORK/hadoop_${bdsql_cluster_name}.env > /dev/null
    fi
    if [ ! -e $oracle_home/bin/mtactl ]; then
        $lncmd -s $oracle_home/rdbms/bin/mtactl $oracle_home/bin/mtactl > /dev/null
    fi
fi
mtactlcmd=${oracle_home}/bin/mtactl

if [ -d "$oracle_home/bigdatasql/log" ] 
then
    MY_LOG_DIRECTORY=$oracle_home/bigdatasql/log
else
    MY_LOG_DIRECTORY="~"
fi

if [ ! -d "${MY_LOG_DIRECTORY}/.${BASENAME}" ]; then
    mkdir "${MY_LOG_DIRECTORY}/.${BASENAME}"
fi
MY_ERROR_LOG_DIRECTORY=${MY_LOG_DIRECTORY}/.${BASENAME}

# Verify cell node remote login password. The bdschecksw simply exit if password is wrong.
. ${hadoop_env_file} > /dev/null 2>&1

hadoopcmd=${HADOOP_HOME}/bin/hadoop

$hadoopcmd fs -ls / > $MY_ERROR_LOG_DIRECTORY/hadoop_fs_ls.out 2>&1 &
$sleepcmd 3

tmp=$($grepcmd failed $MY_ERROR_LOG_DIRECTORY/hadoop_fs_ls.out | $sedcmd -n '1p')
if [ "$tmp" != "" ]; then
    $echocmd
    $echocmd "Fatal Hadoop Error! Check ${MY_ERROR_LOG_DIRECTORY}/hadoop_fs_ls.out for great details."
    $echocmd
    exit 2
fi


MY_CELLS=

report_fullpath_basename=${MY_LOG_DIRECTORY}/${BASENAME}

html_filename="${report_fullpath_basename}_${RIGHT_NOW2}.html"
html_filename_link="${report_fullpath_basename}.html"
text_filename="${report_fullpath_basename}_${RIGHT_NOW2}.txt"
text_filename_link="${report_fullpath_basename}.txt"
json_filename="${report_fullpath_basename}_${RIGHT_NOW2}.json"
json_filename_link="${report_fullpath_basename}.json"

$echocmd 
$echocmd $BDSCHECKSW_BANNER
$echocmd 
$echocmd "output html file = file://${html_filename}"

# Port specific changes
case ${OS_TYPE} in
  Linux)
    if [ -x /usr/bin/links ]; then
      $echocmd "output text file = file://${text_filename}"
    fi
    ;;
  SunOS)
      ifconfigcmd='/usr/sbin/ifconfig'
    ;;
  *)
    ;;
esac

$echocmd "output json file = file://${json_filename}"


##---------------------------------------
#  6. Call the main routine bds_check_sw
##---------------------------------------

if [ "$html_filename" == "" ]; 
then
    bds_check_sw
else
    bds_check_sw

    $rmcmd -f $html_filename_link > /dev/null
    $lncmd -s $html_filename $html_filename_link

    $rmcmd -f $json_filename_link > /dev/null
    $lncmd -s $json_filename $json_filename_link

    case ${OS_TYPE} in
      Linux)
        if [ -x /usr/bin/links ]; then
          $rmcmd -f $text_filename_link > /dev/null
          /usr/bin/links -dump $html_filename > $text_filename 
          $lncmd -s $text_filename $text_filename_link 
        fi
        ;;
      SunOS)
        ;;
      *)
        ;;
    esac
fi



OHA YOOOO