# DCexl.ctl:315:Collects Oracle Exalogic Information
# $Id: DCexl.ctl,v 1.3 2013/10/30 07:18:32 RDA Exp $
# ARCS: $Header: /home/cvs/cvs/RDA_8/src/scripting/lib/collect/OFM/DCexl.ctl,v 1.3 2013/10/30 07:18:32 RDA Exp $
#
# Change History
# 20130611  KRA  Improve user validation.

=head1 NAME

OFM:DCexl - Collects Oracle Exalogic Information

=head1 DESCRIPTION

This module collects the Oracle Exalogic-related diagnostic information.
F<ssh> can prompt you for a password to connect to each node. Therefore, RDA
waits per default 30 seconds before starting the data collection on the next
node. The data collection can take up to 13 minutes with the default timeout
value.

The following reports can be generated and are regrouped under
C<Oracle Exalogic>:

=cut

use Buffer,Mrc,Remote

echo tput('bold'),'Processing OFM.EXL module ...',tput('off')

# Initialization
var $PAUSE        = ${N_NODE_WAIT_PWD:30}
var $COLLECT_NODE = ${B_COLLECT_NODE:true}
var $TAIL         = ${N_TAIL:1000}
var $TIMEOUT      = ${DFT.N_TIMEOUT:0}
var $TYPE         = ${T_TYPE:'_All_'}
var $USERID       = nvl(isUser(${T_NODE_LOGIN},true),'oracle')

var $TOC = '%TOC%'
var $TOP = '[[#Top][Back to top]]'
toc '%TITLE("1:Oracle Exalogic (',replace($TYPE,'_','',true),')")%'

=head2 Multi-run Collections

Collects Oracle Exalogic-related rack wide information from both current user
and super user.

=cut

toc '%TITLE("2:Rack Information")%'

collect OFM:MCexlr|EXLR()

toc '%UNTITLE%'

# Collect from remote nodes
debug ' Inside EXL module, getting node information'
var %nod = ()
var $nod{${RDA.T_HOST}} = 1
if $COLLECT_NODE
{# Define parsing macros
 macro set_done
 {var ($buf,$nod) = @arg
  import %acc
  keep %acc

  var $acc{$nod} = '|Collection Completed| '
 }

 # Define configuration checks macro
 macro config_checks
 {var ($buf,$dat) = @arg
  import $TOC,$TOP
  keep $TOC,$TOP

  var ($lvl,$lnk,$ttl,$rep) = split('\|',$dat,4)
  output F,$rep
  write '---+!! ',$ttl
  write $TOC

  # limits.conf check
  var $ref = getRemoteBuffer('LIMIT',true)
  write '---+ ',encode('limits.conf')
  write '---## Information Taken from ',encode('/etc/security/limits.conf')
  write '|*Check*|*Result*|'
  loop $rec (['hard nofile 65536',\
              '^\*\s*hard\s+nofile\s+65536'],\
             ['soft nofile 65536',\
              '^\*\s*soft\s+nofile\s+65536'],\
             ['hard memlock unlimited',\
              '^\*\s*hard\s+memlock\s+unlimited'],\
             ['soft memlock unlimited',\
              '^\*\s*soft\s+memlock\s+unlimited'],\
             ['hard core unlimited',\
              '^\*\s*hard\s+core\s+unlimited'],\
             ['soft core unlimited',\
              '^\*\s*soft\s+core\s+unlimited'])
  {var ($chk,$pat) = @{$rec}
   if $ref->grep($pat,'f')
    write '|``',$chk,'``|%BLUE%PASSED%ENDCOLOR%|'
   else
    write '|``',$chk,'``|%RED%FAILED%ENDCOLOR%|'
  }
  write $TOP

  # openib.conf check
  var $ref = getRemoteBuffer('OPENIB',true)
  write '---+ ',encode('openib.conf')
  write '---## Information Taken from ',encode('/etc/infiniband/openib.conf')
  write '|*Check*|*Result*|'
  loop $rec (['IPOIB_LOAD=yes',\
              '^IPOIB_LOAD\s*=\s*yes'],\
             ['MLX4_EN_LOAD=no',\
              '^MLX4_EN_LOAD\s*=\s*no'],\
             ['SDP_LOAD=yes',\
              '^SDP_LOAD\s*=\s*yes'])
  {var ($chk,$pat) = @{$rec}
   if $ref->grep($pat,'f')
    write '|``',$chk,'``|%BLUE%PASSED%ENDCOLOR%|'
   else
    write '|``',$chk,'``|%RED%FAILED%ENDCOLOR%|'
  }
  write $TOP

  # recv_poll check
  var $ref = getRemoteBuffer('RECV',true)
  write '---+ ',encode('recv_poll')
  write '---## Information Taken from ',\
    encode('/sys/module/ib_sdp/parameters/recv_poll')
  write '|*Check*|*Result*|'
  if $ref->grep('^0','f')
   write '|``0``|%BLUE%PASSED%ENDCOLOR%|'
  else
   write '|``0``|%RED%FAILED%ENDCOLOR%|'
  write $TOP

  # S05openibd check
  var $ref = getRemoteBuffer('IBD',true)
  write '---+ S05openibd'
  write '|*Check*|*Result*|'
  if $ref->grep('\/etc\/rc3\.d\/S05openibd\s+\-\>','f')
   write '|``/etc/rc3.d/S05openibd``|%BLUE%PASSED%ENDCOLOR%|'
  else
   write '|``/etc/rc3.d/S05openibd``|%RED%FAILED%ENDCOLOR%|'
  write $TOP

  # modprobe.conf check
  var $ref = getRemoteBuffer('MODPROBE',true)
  write '---+ ',encode('modprobe.conf')
  write '---## Information Taken from ',encode('/etc/modprobe.conf')
  write '|*Check*|*Result*|'
  if $ref->grep('^ib_sdp\s+\/bin\/true','f')
   write '|``ib_sdp /bin/true``|%RED%FAILED%ENDCOLOR%|'
  else
   write '|``ib_sdp /bin/true``|%BLUE%PASSED%ENDCOLOR%|'
  loop $rec (['alias net-pf-27 ib_sdp',\
              '^alias\s+net-pf-27\s+ib_sdp'],\
             ['options ib_ipoib lro=Y',\
              '^options\s+ib_ipoib\s+lro=Y'],\
             ['options ib_sdp sdp_zcopy_thresh=0 recv_poll=0',\
              '^options\s+ib_sdp\s+sdp_zcopy_thresh=0\s+recv_poll=0'],\
             ['options mlx4_vnic tx_rings_num=8 rx_rings_num=8',\
              '^options mlx4_vnic\s+tx_rings_num=8\s+rx_rings_num=8'])
  {var ($chk,$pat) = @{$rec}
   if $ref->grep($pat,'f')
    write '|``',$chk,'``|%BLUE%PASSED%ENDCOLOR%|'
   else
    write '|``',$chk,'``|%RED%FAILED%ENDCOLOR%|'
  }
  write $TOP

  # libsdp.conf check
  var $ref = getRemoteBuffer('LIBSDP',true)
  write '---+ ',encode('libsdp.conf')
  write '---## Information Taken from ',encode('/etc/libsdp.conf')
  write '|*Check*|*Result*|'
  loop $rec (['use tcp server * :',\
              '^use\s+tcp\s+server\s+\*\s+:'],\
             ['use tcp client * :',\
              '^use\s+tcp\s+client\s+\*\s+:'])
  {var ($chk,$pat) = @{$rec}
   if $ref->grep($pat,'f')
    write '|``',$chk,'``|%BLUE%PASSED%ENDCOLOR%|'
   else
    write '|``',$chk,'``|%RED%FAILED%ENDCOLOR%|'
  }
  write $TOP

  # sysctl.conf check
  var $ref = getRemoteBuffer('SYSCTL',true)
  write '---+ ',encode('sysctl.conf')
  write '---## Information Taken from ',encode('/etc/sysctl.conf')
  write '|*Check*|*Result*|'
  loop $rec (['net.ipv4.tcp_rmem = 16777216 16777216 16777216',\
              '^net\.ipv4\.tcp_rmem\s*=\s*16777216\s+16777216\s+16777216'],\
             ['net.ipv4.tcp_wmem = 16777216 16777216 16777216',\
              '^net\.ipv4\.tcp_wmem\s*=\s*16777216\s+16777216\s+16777216'],\
             ['net.ipv4.tcp_mem = 16777216 16777216 16777216',\
              '^net\.ipv4\.tcp_mem\s*=\s*16777216\s+16777216\s+16777216'],\
             ['net.core.optmem_max = 16777216',\
              '^net\.core\.optmem_max\s*=\s*16777216'],\
             ['net.core.rmem_max=16777216',\
              '^net\.core\.rmem_max\s*=\s*16777216'],\
             ['net.core.wmem_max=16777216',\
              '^net\.core\.wmem_max\s*=\s*16777216'],\
             ['net.core.rmem_default=16777216',\
              '^net\.core\.rmem_default\s*=\s*16777216'],\
             ['net.core.wmem_default=16777216',\
              '^net\.core\.wmem_default\s*=\s*16777216'],\
             ['net.ipv4.ip_local_port_range = 9000 65500',\
              '^net\.ipv4\.ip_local_port_range\s*=\s*9000\s+65500'],\
             ['vm.nr_hugepages = 10000',\
              '^vm\.nr_hugepages\s*=\s*10000'],\
             ['fs.file-max=262144',\
              '^fs\.file-max\s*=\s*262144'])
  {var ($chk,$pat) = @{$rec}
   if $ref->grep($pat,'f')
    write '|``',$chk,'``|%BLUE%PASSED%ENDCOLOR%|'
   else
    write '|``',$chk,'``|%RED%FAILED%ENDCOLOR%|'
  }
  write $TOP
  toc $lvl,':[[',getFile(),'][rda_report][',$lnk,']]'
 }

 # Definition for exec_command function
 set $exe_cmd
 {exec_command()
 "{ LIM="$1"
 "  shift 1
 "  if [ "$LIM" -gt 0 ]
 "  then
 "    "$@" 2>/dev/null &
 "    PID=$!
 "    while [ $LIM -gt 0 ]
 "    do
 "      /bin/sleep 2
 "      if /bin/ps -l -p $PID 2>/dev/null | \
              /bin/egrep " $PID  *$$ " >/dev/null 2>&1
 "      then
 "        LIM=`/usr/bin/expr $LIM - 2`
 "      else
 "        return
 "      fi
 "    done
 "    if /bin/ps -l -p $PID 2>/dev/null | \
            /bin/egrep " $PID  *$$ " >/dev/null 2>&1
 "    then
 "      /bin/kill -15 $PID 2>/dev/null
 "      /bin/sleep 5
 "      /bin/kill -9 $PID 2>/dev/null
 "      echo "---# RDA:SET TIMEOUT:***Execution Timeout***"
 "    fi
 "  else
 "    "$@" 2>/dev/null
 "  fi
 "}
 }

 # Definition for exec_subshell_command function
 set $exe_sub
 {exec_subshell_command()
 "{ LIM="$1"
 "  shift 1
 "  if [ "$LIM" -gt 0 ]
 "  then
 "    "$@" 2>/dev/null &
 "    PID=$!
 "    SHP=`/bin/ps -l -p $PID 2>/dev/null | \
           /bin/sed -e "/$PID/{s/.*$PID  *\([0-9][0-9]*\).*/\1/
 "         p
 "         }
 "         d"`
 "    while [ $LIM -gt 0 ]
 "    do
 "      /bin/sleep 2
 "      if /bin/ps -l -p $PID 2>/dev/null | \
           /bin/egrep " $PID  *$SHP " >/dev/null 2>&1
 "      then
 "        LIM=`/usr/bin/expr $LIM - 2`
 "      else
 "        return
 "      fi
 "    done
 "    if /bin/ps -l -p $PID 2>/dev/null | \
         /bin/egrep " $PID  *$SHP " >/dev/null 2>&1
 "    then
 "      /bin/kill -15 $PID 2>/dev/null
 "      /bin/sleep 5
 "      /bin/kill -9 $PID 2>/dev/null
 "      echo "---# RDA:SET TIMEOUT:***Execution Timeout***"
 "    fi
 "  else
 "    "$@" 2>/dev/null
 "  fi
 "}
 }

 # Node section begin marker
 set $job_node_begin
 {echo "---# RDA:BEGIN SECTION:2:${NODE} Information"
 }

 # Node section end marker
 set $job_node_end
 {echo "---# RDA:END SECTION"
 }

 # Job to mark the end of a node collection
 set $job_end
 {echo "---# RDA:BEGIN"
 "echo "Collection Ended"
 "echo "---# RDA:END MACRO set_done:${NODE}"
 "
 }

=head2 config_checks - Configuration Checks

Performs Oracle Exalogic configuration checks.

=cut

 set $cfg_checks
 {echo "---# RDA:BEGIN CAPTURE:LIMIT"
 "/bin/cat /etc/security/limits.conf
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:OPENIB"
 "/bin/cat /etc/infiniband/openib.conf
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:RECV"
 "/bin/cat /sys/module/ib_sdp/parameters/recv_poll
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:IBD"
 "/bin/ls -l /etc/rc3.d/S05openibd
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:MODPROBE"
 "/bin/cat /etc/modprobe.conf
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:LIBSDP"
 "/bin/cat /etc/libsdp.conf
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN CAPTURE:SYSCTL"
 "/bin/cat /etc/sysctl.conf
 "echo "---# RDA:END CAPTURE"
 "echo "---# RDA:BEGIN"
 "echo "Configuration Collection Ended"
 "echo "---# RDA:END MACRO config_checks:3|Configuration Checks|\
             Oracle Exalogic Configuration Checks|config_checks"
 }

=head2 ibcheckstate - InfiniBand State Check

Collects the port physical state information using the
F</usr/sbin/ibcheckstate> command (requires root access).

=cut

 set $cfg_ib_chkst
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /usr/sbin/ibcheckstate
 "echo "---# RDA:END REPORT:3|InfiniBand State Check|InfiniBand State Check|\
            Using: /usr/sbin/ibcheckstate|ibcheckstate"
 }

=head2 prtconf - Port Configuration

Collects the port error counters and configuration information using the
C<ibqueryerrors.pl -rR -s LinkDowned,RcvSwRelayErrors,XmtDiscards,XmtWait>
command (requires root access and applicable for C<IbFabric> collection).

=cut

 set $cfg_ib_prtconf
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /usr/sbin/ibqueryerrors.pl -rR -s \
    LinkDowned,RcvSwRelayErrors,XmtDiscards,XmtWait
 "echo "---# RDA:END REPORT:3|Port Configuration|Port Configuration|\
            Using: /usr/sbin/ibqueryerrors.pl -rR -s LinkDowned,\
            RcvSwRelayErrors,XmtDiscards,XmtWait|prtconf"
 }

=head2 ipmitool - IPMI Information

Collects the Intelligent Platform Management Interface (IPMI) information using
the C</usr/bin/ipmitool sel elist> command (requires root access and applicable
for C<DriveFailure> collection).

=cut

 set $cfg_ipmi
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sel elist
 "echo "---# RDA:END REPORT:3|IPMI Information|Intelligent Platform Management \
            Interface (IPMI) Information|Using: /usr/bin/ipmitool sel elist|\
            ipmitool"
 }

=head2 MegaCli64 Information

Collects the output of the following variants of the
F</opt/MegaRAID/MegaCli/MegaCli64> command (requires root access and applicable
for C<Raid> collection).

=head3 adpallinfo_aall - Internal RAID Card Check

Gets the C<-AdpAllInfo -aALL> information.

=head3 adpbbucmd_a0 - MegaRAID Battery Check

Gets the C<-AdpBbuCmd -GetBbuStatus -a0> information.

=head3 adpeventlog_aall - MegaRAID Events

Gets the C<-AdpEventLog -GetEvents -f E<lt>filE<gt> -aALL> information.

=head3 ldpdinfo_a0 - SSD Disk Information

Gets the C<-LdPdInfo -a0> information.

=cut

 set $cfg_cli64
 {echo "---# RDA:BEGIN SECTION:3:MegaCli64 Information"
 "
 "echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL
 "echo "---# RDA:END REPORT:4|Internal RAID Card Check|Internal RAID Card \
            Check|Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL|\
            adpallinfo_aall"
 "
 "echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd \
    -GetBbuStatus -a0
 "echo "---# RDA:END REPORT:4|MegaRAID Battery Check|MegaRAID Battery Check|\
            Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpBbuCmd -GetBbuStatus \
            -a0|adpbbucmd_a0"
 "
 "exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog \
    -GetEvents -f /tmp/MegaRda$$ -aALL
 "echo "---# RDA:BEGIN"
 "/usr/bin/tail -${TAIL} /tmp/MegaRda$$
 "echo "---# RDA:END REPORT:4|MegaRAID Events|MegaRAID Events|\
            Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog \
            -GetEvents -f /tmp/MegaRda$$ -aALL|adpeventlog_aall"
 "/bin/rm -f /tmp/MegaRda$$
 "
 "echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -a0
 "echo "---# RDA:END REPORT:4|SSD Disk Information|SSD Disk Information|\
            Using: /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -a0|ldpdinfo_a0"
 "
 "echo "---# RDA:END SECTION"
 }

=head2 lspci - PCI Devices Configuration

Collects the Peripheral Component Interconnect (PCI) devices information using
the F</sbin/lspci> command (applicable for C<DriveFailure> collection).

=cut

 set $cfg_pci
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /sbin/lspci
 "echo "---# RDA:END REPORT:3|PCI Devices Configuration|PCI Devices \
            Configuration Information|Using: /sbin/lspci|lspci"
 }

=head2 prdserial - Product Serial Number

Gets the product serial number (requires root access and applicable for
C<DriveFailure> collection).

=cut

 set $cfg_prdserial
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli \
   'show /SYS product_serial_number'
 "echo "---# RDA:END REPORT:3|Product Serial Number|Product Serial Number|\
            Using: ipmitool sunoem cli 'show /SYS product_serial_number'|\
            prdserial"
 }

=head2 temp - Temperature Information

Collects the Inlet Ambient Temperature information (requires root access and
applicable for C<Temp> collection).

=cut

 set $cfg_temp
 {echo "---# RDA:BEGIN"
 "exec_subshell_command ${TIMEOUT:-0} /usr/bin/ipmitool sensor | grep AMB
 "echo "---# RDA:END REPORT:3|Temperature Information|Inlet Ambient \
            Temperature Information|Using: /usr/bin/ipmitool sensor|temp"
 }

=head2 tools_version - Tools Version

Collects the output of the C<rpm -qi exalogic.tools> command.

=cut

 set $cfg_tools
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /bin/rpm -qi exalogic.tools
 "echo "---# RDA:END REPORT:3|Tools Version|Tools Version Information|\
            Using: /bin/rpm -qi exalogic.tools|tools_version"
 }

=head2 vmstat - Virtual Memory Statistics

Collects the output of the C<vmstat 10 3> command.

=cut

 set $cfg_vmstat
 {echo "---# RDA:BEGIN"
 "exec_command ${TIMEOUT:-0} /usr/bin/vmstat 10 3
 "echo "---# RDA:END REPORT:3|Virtual Memory Statistics|Virtual Memory \
            Statistics|Using: /usr/bin/vmstat 10 3|vmstat"
 }

 # Define the collection code
 if compare('eq',$USERID,'root')
  var $cod = check($TYPE,\
    '^_DriveFailure_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,$cfg_ipmi,$cfg_pci,$cfg_prdserial,\
      $job_node_end,$job_end),\
    '^_IbFabric_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,$cfg_ib_prtconf,$job_node_end,\
      $job_end),\
    '^_Raid_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,$cfg_cli64,$job_node_end,$job_end),\
    '^_Temp_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,$cfg_temp,$job_node_end,$job_end),\
    '^_All_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,\
      $cfg_checks,$cfg_cli64,$cfg_ib_chkst,$cfg_ib_prtconf,$cfg_ipmi,$cfg_pci,\
      $cfg_prdserial,$cfg_temp,$cfg_tools,$cfg_vmstat,$job_node_end,$job_end),\
    '')
 else
  var $cod = check($TYPE,\
    '^_DriveFailure_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,$cfg_pci,$job_node_end,$job_end),\
    '^_All_',join("\012\012",\
      $exe_cmd,$exe_sub,$job_node_begin,\
      $cfg_checks,$cfg_pci,$cfg_tools,$cfg_vmstat,$job_node_end,$job_end),\
    '')

 # Collect remote node information in threads
 if length($cod)
 {var ($cnt,$pwd,@nod,%acc,%buf,%slp,%tim,%tmp) = (0)

  # Initialize the remote sessions
  loop $nod (@{T_NODES})
  {next compare('eq',$nod,${RDA.T_HOST})
   var $nod{$nod} = true
   call addRemoteSession('EXL',$nod,$USERID)
   if and(defined($pwd),not(hasPassword('host',$nod,$USERID)))
    call setPassword('host',$nod,$USERID,$pwd)
   if needPassword('EXL')
   {var $pwd = setPassword('host',$nod,$USERID,\
      askPassword("Enter ${VAR.USERID} at ${VAR.nod} password:",''))
    if needPassword('EXL')
    {var $acc{$nod} = '|Skipped due to connection problems| '
     var $tim{$nod} = 0
     call endRemoteSession('EXL')
     delete $nod{$nod}
     next
    }
   }
   var $slp{$nod} = cond(needPause('EXL'),$PAUSE,1)
   call endRemoteSession('EXL')
   call push(@nod,$nod)
  }

  # Collect node specific information using remote session
  var $sta = ${STA.${CUR.K_NAME}.NODE/i}->clear
  call $[REM]->set_timeout($dur = expr('*',25,$TIMEOUT))
  loop $nod (@nod)
  {var $acc{$nod} = '|Collection Incomplete| '
   var $tim{$nod} = 0
   var $tmp{$nod} = concat('R_EXIT',incr($cnt))
   thread($cod,$nod,$sta,$tmp{$nod},$buf{$nod} = getTemp($tmp{$nod}))
   {var ($cod,$nod,$sta,$key,$res) = @arg
    import $TAIL,$TIMEOUT,$USERID

    debug '  - Getting node information from ',$nod
    call addRemoteSession('EXL',$nod,$USERID)
    var $ret = rcollect('EXL',$cod,$res,\
      {NODE => $nod,\
       TAIL => $TAIL,\
       TIMEOUT => $TIMEOUT\
      })
    if !hasRemoteTimeout('EXL')
     call $sta->set_value($key,$ret,'Remote collection exit code')
    call endRemoteSession('EXL')
    debug '  - Data collection ended for ',$nod
   }
   sleep $slp{$nod}
  }
  sleep 3
  echo 'The remote node data collection can take more than ',$dur,\
       ' seconds. Do not interrupt the data collection.'
  wait $dur
 }
}

# Generate the node reports from both local and remote
loop $nod (keys(%nod))
{if compare('eq',$nod,${RDA.T_HOST})
 {toc '%TITLE("2:',$nod,' Information")%'
  collect OFM:MCexlh|EXLH()
  toc '%UNTITLE%'
 }
 elsif length($cod)
 {if $sta->get_first($tmp{$nod})
   var $acc{$nod} = concat('|Connection error (',last,')| ')
  else
  {call writeRemoteResult($buf = new('Buffer','R',$buf{$nod}),$nod)
   call $buf->close
  }
  call unlinkTemp($tmp{$nod})
 }
}

=head2 summary - Node Collection Summary

Displays the node list and their accessibility.

=cut

if length($cod)
{debug ' Inside EXL module, producing the node collection summary'
 report summary
 prefix
 {write '---+ Node Collection Summary'
  write '|*Node*|*Collection Status*| *Execution Timeout*|'
 }
 loop $key (keys(%acc))
  write '|',$key,$acc{$key},$tim{$key},'|'
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][Node Collection Summary]]'
 else
  echo 'No nodes found'
}

toc '%UNTITLE%'

=head1 SEE ALSO

L<OFM:MCexlh|collect::OFM:MCexlh>,
L<OFM:MCexlr|collect::OFM:MCexlr>

=begin credits

=over 10

=item RDA 4.25:  Chenna Gaddam, Wes Root.

=item RDA 4.26:  Chenna Gaddam.

=item RDA 4.27:  Chenna Gaddam.

=item RDA 4.28:  Chenna Gaddam.

=back

=end credits

=head1 COPYRIGHT NOTICE

Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.

=head1 TRADEMARK NOTICE

Oracle and Java are registered trademarks of Oracle and/or its
affiliates. Other names may be trademarks of their respective owners.

=cut