# DCexa.ctl:280:Collects Exadata Information
# $Id: DCexa.ctl,v 1.7 2015/02/06 14:09:30 RDA Exp $
# ARCS: $Header: /home/cvs/cvs/RDA_8/src/scripting/lib/collect/DB/DCexa.ctl,v 1.7 2015/02/06 14:09:30 RDA Exp $
#
# Change History
# 20150204  KRA  Extend 'FlashDrives' collection.

=head1 NAME

DB:DCexa - Collects Exadata Information

=head1 DESCRIPTION

This module collects the Exadata-related diagnostic information. F<ssh> can
prompt you for a password to connect to each cell. Therefore, RDA waits per
default 30 seconds before starting the data collection on the next cell. The
data collection can take up to 13 minutes with the default timeout value.

The following reports can be generated and are regrouped under
C<Oracle Exadata>:

=cut

use Buffer,Mrc,Remote

echo tput('bold'),'Processing DB.EXA module ...',tput('off')

# Initialization
var $ALERT_TAIL     = ${N_ALERT_TAIL:30000}
var $CELL_PAUSE     = ${N_CELL_WAIT_PWD:30}
var $CELL_USERID    = nvl(isUser(${T_CELL_LOGIN},true),'celladmin')
var $COLLECT_CELL   = ${B_COLLECT_CELL}
var $MSODL_TAIL     = ${N_MSODL_TAIL:30000}
var $TIMEOUT        = ${DFT.N_TIMEOUT:0}
var $TRACE_TAIL     = ${N_TRACE_TAIL:1000}
var $TYPE           = ${T_TYPE:'_All_'}

var $TOC = '%TOC%'
var $TOP = '[[#Top][Back to top]]'
toc '%TITLE("1:Oracle Exadata (',replace($TYPE,'_','',true),')")%'

# Validate the settings
if match($TYPE,'^_RacInstance_')
 var $TYPE = '_All_'

# Load the common macros
run DB:DBinfo()
run RDA:library()

# Definition for exec_command function
set $exe_cmd
{exec_command()
"{ LIM="$1"
"  shift 1
"  if [ "$LIM" -gt 0 ]
"  then
"    "$@" 2>/dev/null &
"    PID=$!
"    while [ $LIM -gt 0 ]
"    do
"      /bin/sleep 2
"      if /bin/ps -l -p $PID 2>/dev/null | \
             /bin/egrep " $PID  *$$ " >/dev/null 2>&1
"      then
"        LIM=`/usr/bin/expr $LIM - 2`
"      else
"        return
"      fi
"    done
"    if /bin/ps -l -p $PID 2>/dev/null | \
           /bin/egrep " $PID  *$$ " >/dev/null 2>&1
"    then
"      /bin/kill -15 $PID 2>/dev/null
"      /bin/sleep 5
"      /bin/kill -9 $PID 2>/dev/null
"      echo "---# RDA:SET TIMEOUT:***Execution Timeout***"
"    fi
"  else
"    "$@" 2>/dev/null
"  fi
"}
}

# Definition for exec_subshell_command function
set $exe_sub
{exec_subshell_command()
"{ LIM="$1"
"  shift 1
"  if [ "$LIM" -gt 0 ]
"  then
"    "$@" 2>/dev/null &
"    PID=$!
"    SHP=`/bin/ps -l -p $PID 2>/dev/null | \
          /bin/sed -e "/$PID/{s/.*$PID  *\([0-9][0-9]*\).*/\1/
"         p
"         }
"         d"`
"    while [ $LIM -gt 0 ]
"    do
"      /bin/sleep 2
"      if /bin/ps -l -p $PID 2>/dev/null | \
          /bin/egrep " $PID  *$SHP " >/dev/null 2>&1
"      then
"        LIM=`/usr/bin/expr $LIM - 2`
"      else
"        return
"      fi
"    done
"    if /bin/ps -l -p $PID 2>/dev/null | \
        /bin/egrep " $PID  *$SHP " >/dev/null 2>&1
"    then
"      /bin/kill -15 $PID 2>/dev/null
"      /bin/sleep 5
"      /bin/kill -9 $PID 2>/dev/null
"      echo "---# RDA:SET TIMEOUT:***Execution Timeout***"
"    fi
"  else
"    "$@" 2>/dev/null
"  fi
"}
}

# Definition for tail_command function
set $exe_tail
{tail_command()
"{ LIM="$1"
"  MAX="$2"
"  shift 2
"  if [ "$LIM" -gt 0 ]
"  then
"    "$@" 2>/dev/null | /usr/bin/tail -${MAX} &
"    PID=$!
"    while [ $LIM -gt 0 ]
"    do
"      /bin/sleep 2
"      if /bin/ps -l -p $PID 2>/dev/null | \
             /bin/egrep " $PID  *$$ " >/dev/null 2>&1
"      then
"        LIM=`/usr/bin/expr $LIM - 2`
"      else
"        return
"      fi
"    done
"    if /bin/ps -l -p $PID 2>/dev/null | \
           /bin/egrep " $PID  *$$ " >/dev/null 2>&1
"    then
"      /bin/kill -15 $PID 2>/dev/null
"      /bin/sleep 5
"      /bin/kill -9 $PID 2>/dev/null
"      echo "---# RDA:SET TIMEOUT:***Execution Timeout***"
"    fi
"  else
"    "$@" 2>/dev/null | /usr/bin/tail -${MAX}
"  fi
"}
}

=head1 COMPUTE NODE COLLECTIONS

=head2 no_db_access - No Database Access

Reports database access issue.

=cut

toc '%TITLE("2:Compute Node")%'

# All collection type
if match($TYPE,'^_All_$')
{call setSqlTarget(${SET.DB.DB.I_DB})
 if testSql()
 {report no_db_access
  write 'Cannot connect to the database ',' (',getSqlMessage(),')'
  toc '3:[[',getFile(),'][rda_report][No Database Access]]'
 }

 # Get database version
 var $ver = get_db_version()

=head2 dbinfo - Database Information

Gets Exadata Database information (available for Oracle Database 11g and later).

=cut

 report dbinfo
 var $TTL = '---+!! Database Information'
 if match($ver,'^(11|12)')
 {debug ' Inside Exadata module, gathering Database information'
  var @TTL = ('',\
              '---+ Database Instance Parameters Relative to Exadata',\
              '---+ Exadata Cells Statistics',\
              '---+ Data Sent Back to Host After Smart Scan',\
              '---+ Filtering Efficiency',\
              '---+ Tablespaces Storage Option',\
              '---+ V$Cell',\
              '---+ V$Cell_State',\
              '---+ V$Cell_Thread_History',\
              '---+ V$Cell_Request_Totals')
  var @HDR = ('',\
              '|*Name*|*Value*|',\
              '|*Name*|*Value*|',\
              '| *Size*|',\
              '| *Efficiency*|',\
              '|*Tablespace Name*|*Predicate Evaluation*|')
  set $sql
  {SELECT '|' ||
  "       name || ' |' ||
  "       value|| ' |'
  " FROM v$parameter
  " WHERE UPPER(name) LIKE 'CELL_%';
  "PROMPT ___Macro_separator(2)___
  "SELECT '|' ||
  "       name || ' |' ||
  "       value || ' |'
  " FROM v$sysstat
  " WHERE UPPER(name) LIKE 'CELL%';
  "PROMPT ___Macro_separator(3)___
  "SELECT '| ' ||
  "       (a.value-(b.value-c.value)) || '|'
  " FROM v$sysstat a,v$sysstat b,v$sysstat c
  " WHERE a.name = 'cell physical IO interconnect bytes'
  "   AND b.name = 'physical IO disk bytes'
  "   AND c.name = 'cell physical IO bytes eligible for predicate offload';
  "PROMPT ___Macro_separator(4)___
  "SELECT '| ' ||
  "       DECODE(c.value,0,'Not calculated',
  "                        (a.value-(b.value-c.value)) / c.value) || '|'
  " FROM v$sysstat a,v$sysstat b,v$sysstat c
  " WHERE a.name = 'cell physical IO interconnect bytes'
  "   AND b.name = 'physical IO disk bytes'
  "   AND c.name = 'cell physical IO bytes eligible for predicate offload';
  "PROMPT ___Macro_separator(5)___
  "SELECT '|' ||
  "       tablespace_name || ' |' ||
  "       predicate_evaluation || ' |'
  " FROM dba_tablespaces
  " WHERE predicate_evaluation = 'STORAGE';
  }
  var ($HDR[6],$col6) = getSqlColumns('RDA','','V$CELL')
  call clearSqlColumns('RDA')
  if $col6
  {append $sql
   {PROMPT ___Macro_separator(6)___
   "SELECT :1
   " FROM v$cell;
   }
  }
  var ($HDR[7],$col7) = getSqlColumns('RDA','','V$CELL_STATE')
  call clearSqlColumns('RDA')
  if $col7
  {append $sql
   {PROMPT ___Macro_separator(7)___
   "SELECT :2
   " FROM v$cell_state;
   }
  }
  var ($HDR[8],$col8) = getSqlColumns('RDA','','V$CELL_THREAD_HISTORY')
  call clearSqlColumns('RDA')
  if $col8
  {append $sql
   {PROMPT ___Macro_separator(8)___
   "SELECT :3
   " FROM (SELECT *
   "        FROM v$cell_thread_history
   "        WHERE snapshot_time > SYSDATE - 1
   "        ORDER BY snapshot_time DESC)
   " WHERE ROWNUM < 1000;
   }
  }
  var ($HDR[9],$col9) = getSqlColumns('RDA','','V$CELL_REQUEST_TOTALS')
  call clearSqlColumns('RDA')
  if $col9
  {append $sql
   {PROMPT ___Macro_separator(9)___
   "SELECT :4
   " FROM (SELECT *
   "        FROM v$cell_request_totals
   "        WHERE snapshot_time > SYSDATE - 1
   "        ORDER BY snapshot_time DESC)
   " WHERE ROWNUM < 1000;
   }
  }
  call separator(1)
  call writeSql(bindSql($sql,$col6,$col7,$col8,$col9))
  call separator(0,'Database Information',3)
 }

=head2 Configuration Files

Gathers configuration files in the following directory structure:

=over 4

=item o F</etc/oracle/cell/network-config>

=back

=cut

 debug ' Inside Exadata module, collecting the configuration files'
 pretoc '3:Configuration Files'
 call sort_files(4,undef,\
                 grepDir('/etc/oracle/cell/network-config','^\.+$','vr'))
 unpretoc
}

=head2 Multi-run Collections

Collects Exadata-related information from both current user and super user.

=cut

collect DB:MCexa|EXA()

toc '%UNTITLE%'

=head1 CELL NODE COLLECTIONS

Collects Exadata-related information from the cells.

=cut

if !$COLLECT_CELL
 return

debug ' Inside Exadata module, getting information from the remote cells'
var %tb_cmd = (\
  'alerthist',   'cellcli -e list alerthistory detail',\
  'celldet',     'cellcli -e list cell detail',\
  'celldisk',    'cellcli -e list celldisk detail',\
  'flashcache',  'cellcli -e list flashcache detail',\
  'flashdisk',   'cellcli -e list physicaldisk where disktype=flashdisk \
                  detail',\
  'griddisk',    'cellcli -e list griddisk detail',\
  'iormplan',    'cellcli -e list iormplan detail',\
  'lun',         'cellcli -e list lun detail',\
  'metdef',      'cellcli -e list metricdefinition detail',\
  'metcurd',     'cellcli -e list metriccurrent detail',\
  'methist',     'cellcli -e list metrichistory detail | tail -500',\
  'metric',      'cellcli -e list metriccurrent CD_IO_TM_W_SM_RQ where \
                  metricObjectName like "FD.*"',\
  'phydisk',     'cellcli -e list physicaldisk detail',\
  'phydisk_fail','cellcli -e list physicaldisk where status!=normal',\
  'threshold',   'cellcli -e list threshold detail')

var %tb_ttl = (\
  'alerthist',   'Alert History',\
  'celldet',     'Cell Detail',\
  'celldisk',    'Cell Disk Detail',\
  'flashcache',  'Flash Cache Detail',\
  'flashdisk',   'Flash Disk Detail',\
  'griddisk',    'Grid Disk Detail',\
  'iormplan',    'Iormplan Detail',\
  'lun',         'LUN Detail',\
  'metdef',      'Metric Definition Detail',\
  'metcurd',     'Metric Current Detail',\
  'methist',     'Metric History Detail',\
  'metric',      'Metric Current Flash Drive Detail',\
  'phydisk',     'Physical Disk Detail',\
  'phydisk_fail','Physical Disk Fail Detail',\
  'threshold',   'Threshold Detail')

# Define parsing macros
macro write_cell
{var ($buf,$dat) = @arg
 var ($cel,$key) = split('\|',$dat,3)
 import $TOC,$TOP,%rpt,%tim,%tb_cmd,%tb_ttl
 keep $TOC,$TOP,%rpt,%tim,%tb_cmd,%tb_ttl

 # Create the report on the first use
 if exists($rpt{$key})
  var $rpt = $rpt{$key}
 else
 {var $rpt{$key} = $rpt = $[OUT]->add_report('c',$key,0)
  title {$rpt} '---+!! ',$tb_ttl{$key}
  title {$rpt} '---## Using: ',$tb_cmd{$key}
  title {$rpt} $TOC
 }

 # Add the cell contribution
 prefix $rpt
 {write {$rpt} '---+ ',$cel,' Information'
  write {$rpt} '|*Name*|*Value*|'
 }

 if $buf->grep('\*\*\*Execution Timeout\*\*\*','f')
  incr $tim{$cel}
 else
 {loop $lin ($buf->grep('\S'))
  {if !match($lin,'^(CELL-\d+|HTTP transport error):')
    write {$rpt} '|',replace(replace(replace(trim($lin),'\*','&#042;',true),\
                                     '\|','&#x7C;',true),\
                             ':\s*',' |'),' |'
  }
 }
 if $rpt->has_output(true)
  write {$rpt} $TOP
}

macro write_list
{var ($buf,$dat) = @arg
 var ($dir,$rpt,$lim) = split('\|',$dat,3)

 var $rpt = $[OUT]->add_report('F',$rpt,0)
 prefix $rpt
 {write {$rpt} '---+ Latest 10 Trace Files from ',$dir
  write {$rpt} '   * Last ',$lim,' trace file lines collected'
  write {$rpt} '   * Links point to files that have been collected in their \
                     original format. Opening them directly in your browser \
                     can present risks. To prevent them, access the file \
                     outside the browser or use the link to save them and use \
                     an adequate viewer.'
  write {$rpt} '|*Trace File Name*|'
 }
 while ?$lnk = $buf->get_line
  write {$rpt} '|',$lnk,'|'
 if $rpt->is_created(true)
  toc '5:[[',$rpt->get_report,'][rda_report][Latest Trace Files]]'
 end $rpt
}

macro write_crash_list
{var ($buf,$rpt) = @arg

 var $rpt = $[OUT]->add_report('F',$rpt,0)
 prefix $rpt
 {write {$rpt} '---+ Crash Files'
  write {$rpt} '|*Filename*|'
 }
 while ?$lnk = $buf->get_line
  write {$rpt} '|',$lnk,'|'
 if $rpt->is_created(true)
  toc '5:[[',$rpt->get_report,'][rda_report][Crash Files]]'
 end $rpt
}

macro write_dev_list
{var ($buf,$rpt) = @arg

 var $rpt = $[OUT]->add_report('F',$rpt,0)
 prefix $rpt
 {write {$rpt} '---+ Aurasmart Details'
  write {$rpt} '|*Device*|'
 }
 while ?$lnk = $buf->get_line
  write {$rpt} '|',$lnk,'|'
 if $rpt->is_created(true)
  toc '4:[[',$rpt->get_report,'][rda_report][Aurasmart Details]]'
 end $rpt
}

macro set_done
{var ($buf,$cel) = @arg
 import %acc
 keep %acc

 var $acc{$cel} = '|Collection Completed| '
}

# Cell section begin marker
set $job_cell_begin
{echo "---# RDA:BEGIN SECTION:3:${CELL} Information"
}

# Cell section end marker
set $job_cell_end
{echo "---# RDA:END SECTION"
}

# Job to mark the end of a cell collection
set $job_end
{echo "---# RDA:BEGIN"
"echo "Collection Ended"
"echo "---# RDA:END MACRO set_done:${CELL}"
"
}

=head2 scripts_aura - Scripts Aura

Collects the Aura information using the F<scripts_aura.sh> script from the
cells (applicable for C<FailedDrives>, and C<FlashDrives> collections).

=cut

set $cfg_aura
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /bin/sh /opt/oracle/cell/cellsrv/deploy/scripts/\
    unix/hwadapter/diskadp/scripts_aura.sh
"echo "---# RDA:END REPORT:4|Scripts Aura|Scripts Aura Information|\
           Using: /opt/oracle/cell/cellsrv/deploy/scripts/unix/hwadapter/\
                  diskadp/scripts_aura.sh|scripts_aura"
}

=for stopwords Aurasmart

=head2 aurasmart - Aurasmart Details

Collects the devices list using C<aurasmart -D E<lt>devE<gt> -N>, and
C<aurasmart -v -d E<lt>devE<gt> -N> from the cells (applicable for
C<FailedDrives> collection).

The device list is derived from the F<lsscsi> command output.

=cut

set $cfg_aurasmart
{echo "---# RDA:BEGIN LIST"
"for dev in `/usr/bin/lsscsi | /bin/egrep MARVELL | /bin/awk '{print $7}'`
"do
"  echo "---# RDA:BEGIN"
"  exec_command ${TIMEOUT:-0} /usr/bin/aurasmart -D $dev -N
"  exec_command ${TIMEOUT:-0} /usr/bin/aurasmart -v -d $dev -N
"  echo "---# RDA:END DATA:$dev"
"done
"echo "---# RDA:END LIST write_dev_list:aurasmart"
}

=head2 bios - BIOS Information

Gets the BIOS information from the cells (applicable for C<FailedDrives>, and
C<FlashDrives> collections).

=cut

set $cfg_bios
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli 'show /SYS/MB/BIOS'
"echo "---# RDA:END REPORT:4|BIOS Information|BIOS Information|\
           Using: ipmitool sunoem cli 'show /SYS/MB/BIOS'|bios"
}

=for stopwords cellinit

=head2 cellinit - cellinit.ora

Collects F<cellinit.ora> file from the cells (applicable for C<FailedDrives>
collection).

=cut

set $cfg_cellinit
{fil="/opt/oracle/cell/cellsrv/deploy/config/cellinit.ora"
"echo "---# RDA:BEGIN"
"/bin/cat $fil
"echo "---# RDA:END REPORT:4|cellinit.ora|Information from cellinit.ora|\
            Using: cat $fil|cellinit"
}

=head2 alerthist - Alert History

Gets the C<alerthistory detail> information (applicable for C<FailedDrives>
collection).

=head2 celldet - Cell Detail

Gets the C<cell detail> information (applicable for C<FailedDrives>
collection).

=head2 celldisk - Cell Disk Detail

Gets the C<celldisk detail> information (applicable for C<FailedDrives>
collection).

=head2 flashcache - Flash Cache Detail

Gets the C<flashcache detail> information for Sun (applicable for
C<FailedDrives> collection).

=head2 flashdisk - Flash Disk Detail

Gets the C<physicaldisk where disktype=flashdisk detail> information
(applicable for C<FlashDrives> collection).

=head2 griddisk - Grid Disk Detail

Gets the C<griddisk detail> information (applicable for C<FailedDrives>
collection).

=for stopwords Iormplan

=head2 iormplan - Iormplan Detail

Gets the C<iormplan detail> information (applicable for C<FailedDrives>
collection).

=head2 lun - LUN Detail

Gets the C<lun detail> information (applicable for C<FailedDrives>, and
C<FlashDrives> collection).

=head2 metdef - Metric Definition Detail

Gets the C<metricdefinition detail> information (applicable for C<FailedDrives>
collection).

=head2 metcurd - Metric Current Detail

Gets the C<metriccurrent detail> information (applicable for C<FailedDrives>
and C<SickCell> collections).

=head2 methist - Metric History Detail

Gets the C<metrichistory detail> information (applicable for C<FailedDrives>
and C<SickCell> collections).

=head2 metric - Metric Current Flash Drive Detail

Gets the C<metriccurrent CD_IO_TM_W_SM_RQ> information (applicable for
C<FlashDrives> collections).

=head2 phydisk - Physical Disk Detail

Gets the C<physicaldisk detail> information (applicable for C<FailedDrives>,
C<FlashDrives>, and C<SickCell> collections).

=head2 phydisk_fail - Physical Disk Fail Detail

Gets the C<physicaldisk where status!=normal> information (applicable for
C<FailedDrives> collection).

=head2 threshold - Threshold Detail

Gets the C<threshold detail> information (applicable for C<FailedDrives>
collection).

=cut

set $cfg_cli_all
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   alerthistory detail
"echo "---# RDA:END MACRO write_cell:${CELL}|alerthist"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list cell \
   detail
"echo "---# RDA:END MACRO write_cell:${CELL}|celldet"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   celldisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|celldisk"
"
"if exec_subshell_command 10 /opt/oracle/cell/cellsrv/bin/cellcli -e list cell \
    attributes makeModel | /bin/egrep " SUN " >/dev/null 2>&1
"then
"  echo "---# RDA:BEGIN"
"  exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
     flashcache detail
"  echo "---# RDA:END MACRO write_cell:${CELL}|flashcache"
"fi
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk where disktype=flashdisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|flashdisk"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   griddisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|griddisk"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   iormplan detail
"echo "---# RDA:END MACRO write_cell:${CELL}|iormplan"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list lun \
   detail
"echo "---# RDA:END MACRO write_cell:${CELL}|lun"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metricdefinition detail
"echo "---# RDA:END MACRO write_cell:${CELL}|metdef"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metriccurrent detail
"echo "---# RDA:END MACRO write_cell:${CELL}|metcurd"
"
"echo "---# RDA:BEGIN"
"tail_command ${TIMEOUT:-0} 500 /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metrichistory detail
"echo "---# RDA:END MACRO write_cell:${CELL}|methist"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|phydisk"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk where status!=normal
"echo "---# RDA:END MACRO write_cell:${CELL}|phydisk_fail"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   threshold detail
"echo "---# RDA:END MACRO write_cell:${CELL}|threshold"
"echo "---# RDA:END PARSE"
}

set $cfg_cli_cellfail
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list cell \
   detail
"echo "---# RDA:END MACRO write_cell:${CELL}|celldet"
"echo "---# RDA:END PARSE"
}

set $cfg_cli_flash
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   alerthistory detail
"echo "---# RDA:END MACRO write_cell:${CELL}|alerthist"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   celldisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|celldisk"
"
"if exec_subshell_command 10 /opt/oracle/cell/cellsrv/bin/cellcli -e list cell \
     attributes makeModel | /bin/egrep " SUN " >/dev/null 2>&1
"then
"  echo "---# RDA:BEGIN"
"  exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
     flashcache detail
"  echo "---# RDA:END MACRO write_cell:${CELL}|flashcache"
"fi
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk where disktype=flashdisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|flashdisk"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list lun \
   detail
"echo "---# RDA:END MACRO write_cell:${CELL}|lun"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metriccurrent CD_IO_TM_W_SM_RQ where metricObjectName like 'FD.*'
"echo "---# RDA:END MACRO write_cell:${CELL}|metric"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|phydisk"
"echo "---# RDA:END PARSE"
}

set $cfg_cli_sick
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metriccurrent detail
"echo "---# RDA:END MACRO write_cell:${CELL}|metcurd"
"
"echo "---# RDA:BEGIN"
"tail_command ${TIMEOUT:-0} 500 /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   metrichistory detail
"echo "---# RDA:END MACRO write_cell:${CELL}|methist"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle/cell/cellsrv/bin/cellcli -e list \
   physicaldisk detail
"echo "---# RDA:END MACRO write_cell:${CELL}|phydisk"
"echo "---# RDA:END PARSE"
}

=head2 MegaCli64 Information

Collects the output of the following variants of the
F</opt/MegaRAID/MegaCli/MegaCli64> command from the cells (applicable for
C<FailedDrives> collection).

=head3 adpallinfo_aall - ADP All Information

Gets the C<-AdpAllInfo -aALL> information.

=head3 adpbbucmd_aall - BBU Command

Gets the C<-adpbbucmd -aALL> information.

=head3 adpeventlog_aall - ADP Events

Gets the C<-AdpEventLog -GetEvents -f E<lt>filE<gt> -aALL> information.

=head3 cfgdsply_aall - Configuration Display

Gets the C<-cfgdsply -aALL> information.

=for stopwords FwTerm

=head3 fwtermlog_aall - FwTerm Log

Gets the C<-fwtermlog -dsply -aALL> information.

=head3 ldinfo_aall - LD Information

Gets the C<-LDInfo -LALL -aALL> information.

=for stopwords LdPd

=head3 ldpdinfo_aall - LdPd Information

Gets the C<-LdPdInfo -aALL> information.

=head3 pdlist_aall - PD List

Gets the C<-PDList -aALL> information.

=head3 pdlist_short_aall - PD Short List

Gets the C<-PDList -aALL> information.

=cut

set $cfg_cli64_all
{echo "---# RDA:BEGIN SECTION:4:MegaCli64 Information"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL
"echo "---# RDA:END REPORT:5|ADP All Information|ADP All Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL|\
           adpallinfo_aall"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -adpbbucmd -aALL
"echo "---# RDA:END REPORT:5|BBU Command|BBU Command Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -adpbbucmd -aALL|\
           adpbbucmd_aall"
"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog \
   -GetEvents -f /tmp/MegaRda$$ -aALL
"echo "---# RDA:BEGIN"
"/usr/bin/tail -${TAIL_T} /tmp/MegaRda$$
"echo "---# RDA:END REPORT:5|ADP Events|ADP Events|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpEventLog \
           -GetEvents -f /tmp/MegaRda$$ -aALL|adpeventlog_aall"
"/bin/rm -f /tmp/MegaRda$$
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -cfgdsply -aALL
"echo "---# RDA:END REPORT:5|Configuration Display|Configuration Display|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -cfgdsply -aALL|cfgdsply_aall"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog \
   -dsply -aALL
"echo "---# RDA:END REPORT:5|FwTerm Log|FwTerm Log Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog -dsply -aALL|\
           fwtermlog_aall"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -LALL -aALL
"echo "---# RDA:END REPORT:5|LD Information|LD Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -LALL -aALL|\
           ldinfo_aall"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL
"echo "---# RDA:END REPORT:5|LdPd Information|LdPd Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL|ldpdinfo_aall"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -PDList -aALL
"echo "---# RDA:END REPORT:5|PD List|PD List|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -PDList -aALL|pdlist_aall"
"
"echo "---# RDA:BEGIN"
"exec_subshell_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -PDList \
  -aALL | /bin/awk '/Slot Number/ { counter += 1; slot[counter] = $3 } \
            /Device Id/ { device[counter] = $3 } \
            /Firmware state/ { state_drive[counter] = $3 } \
            /Inquiry/ { name_drive[counter] = $3 " " $4 " " $5 " " $6 } \
            END { for (i=1; i<=counter; i+=1) \
                  printf ( "Slot %02d Device %02d (%s) status is: %s \n", \
                           slot[i], device[i], name_drive[i], \
                           state_drive[i]); }'
"echo "---# RDA:END REPORT:5|PD Short List|PD Short List|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -PDList -aALL|\
           pdlist_short_aall"
"
"echo "---# RDA:END SECTION"
}

=head2 MegaCli64 Information

Collects the output of the following variants of the
F</opt/MegaRAID/MegaCli/MegaCli64> command from the cells which are sick
(applicable for C<SickCell> collection).

=head3 adpbbucmd_prop_sick - Battery Controller Properties

Gets the C<-adpbbucmd -GetBbuProperties -a0> information.

=head3 adpbbucmd_sick - Battery Controller Status

Gets the C<-adpbbucmd -GetBbuStatus -a0> information.

=head3 adpallinfo_sick - Controller Setting

Gets the C<-AdpAllInfo -a0> information.

=head3 ldpdinfo_sick - Disk Attributes

Gets the C<-LdPdInfo -aALL> information.

=head3 ldinfo_sick - Drives Performance

Gets the C<-LDInfo -LALL -aALL> information.

=head3 fwtermlog_sick - Firmware Logs

Gets the C<-fwtermlog -dsply -a0> information.

=head3 pdlist_sick - Physical Disk Information

Gets the C<-PDList -a0> information.

=cut

set $cfg_cli64_sick
{echo "---# RDA:BEGIN SECTION:4:MegaCli64 Information"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -adpbbucmd \
   -GetBbuProperties -a0
"echo "---# RDA:END REPORT:5|Battery Controller Properties|Battery \
           Controller Properties Information|Using: \
           /opt/MegaRAID/MegaCli/MegaCli64 -adpbbucmd -GetBbuProperties \
           -a0|adpbbucmd_prop_sick"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -adpbbucmd \
   -GetBbuStatus -a0
"echo "---# RDA:END REPORT:5|Battery Controller Status|Battery Controller \
           Status Information|Using: /opt/MegaRAID/MegaCli/MegaCli64 \
           -adpbbucmd -GetBbuStatus -a0|adpbbucmd_sick"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -a0
"echo "---# RDA:END REPORT:5|Controller Setting|Controller Setting \
           Information|Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo \
           -a0|adpallinfo_sick"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL
"echo "---# RDA:END REPORT:5|Disk Attributes|Disk Attributes Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -LdPdInfo -aALL|\
           ldpdinfo_sick"
"
"echo "---# RDA:BEGIN"
"exec_subshell_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo \
   -LALL -aALL | /bin/egrep 'Current Cache Policy'
"echo "---# RDA:END REPORT:5|Drives Performance|Drives Performance \
           Information|Using: /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo \
           -LALL -aALL|ldinfo_sick"
"
"echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog \
   -dsply -a0
"echo "---# RDA:END REPORT:5|Firmware Logs|Firmware Logs Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog -dsply -a0|\
           fwtermlog_sick"
"
"echo "---# RDA:BEGIN"
"exec_subshell_command ${TIMEOUT:-0} /opt/MegaRAID/MegaCli/MegaCli64 -PDList \
    -a0 | /bin/egrep Inquiry
"echo "---# RDA:END REPORT:5|Physical Disk Information|Physical Disk \
           Information|Using: /opt/MegaRAID/MegaCli/MegaCli64 -PDList \
           -a0|pdlist_sick"
"
"echo "---# RDA:END SECTION"
}

=head2 get_dsk_dev - Disk Devices

Collects the Disk Devices information using the
C</opt/oracle/cell/cellsrv/deploy/scripts/unix/hwadapter/diskadp/
get_disk_devices.pl 5042> script from the cells (applicable for C<FailedDrives>
collection).

=cut

set $cfg_disk_dev
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/perl /opt/oracle/cell/cellsrv/deploy/\
   scripts/unix/hwadapter/diskadp/get_disk_devices.pl 5042
"echo "---# RDA:END REPORT:4|Disk Devices|Disk Devices Information|\
           Using: perl /opt/oracle/cell/cellsrv/deploy/scripts/unix/\
                       hwadapter/diskadp/get_disk_devices.pl 5042|get_dsk_dev"
}

=head2 fdisk - Disk Partition Information

Collects the disk partition information using the C</sbin/fdisk -l> command
from the cells (applicable for C<FailedDrives> collection).

=cut

set $cfg_fdisk
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /sbin/fdisk -l
"echo "---# RDA:END REPORT:4|Disk Partition Information|Disk Partition \
           Information|Using: /sbin/fdisk -l|fdisk"
}

=for stopwords Dom

=head2 flash_dom - Flash Dom

Collects the Flash Dom information using the C</usr/bin/flash_dom -l> command
from the cells (applicable for C<FlashDrives> collection).

=cut

set $cfg_flash_dom
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/flash_dom -l
"echo "---# RDA:END REPORT:4|Flash Dom|Flash Dom Information|\
           Using: /usr/bin/flash_dom -l|flash_dom"
}

=head2 fs_sta - File System Space Status

Collects the file system space status information from the cells (applicable
for C<SickCell> collection).

=cut

set $cfg_fs_sta
{echo "---# RDA:BEGIN"
"exec_subshell_command ${TIMEOUT:-0} /usr/sbin/lsof -S 2 | /bin/egrep deleted
"echo "---# RDA:END REPORT:4|File System Space Status|File System Space Status \
           Information|Using: lsof -S 2|fs_sta"
}

=head2 dns - Name Server Information

Gets the Integrated Lights Out Manager (ILOM)-related name server information
from the cells using the C<ipmitool sunoem cli 'show /SP/clients/dns'> command
(applicable for C<Ilom> collection).

=cut

set $cfg_ilom_dns
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli 'show /SP/clients/dns'
"echo "---# RDA:END REPORT:4|Name Server|Name Server Information|Using: \
           /usr/bin/ipmitool sunoem cli 'show /SP/clients/dns'|ilom_dns"
}

=head2 event - Events Information

Gets the Integrated Lights Out Manager (ILOM)-related events information from
the cells using the C<ipmitool sel list> command (applicable for C<Ilom>
collection).

=cut

set $cfg_ilom_event
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sel list
"echo "---# RDA:END REPORT:4|Events|Events Information|Using: \
           /usr/bin/ipmitool sel list|ilom_event"
}

=head2 lan - LAN Information

Gets the Integrated Lights Out Manager (ILOM)-related LAN information from the
cells using the C<ipmitool lan print> command (applicable for C<Ilom>
collection).

=cut

set $cfg_ilom_lan
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool lan print
"echo "---# RDA:END REPORT:4|LAN|LAN Information|Using: \
           /usr/bin/ipmitool lan print|ilom_lan"
}

=head2 mac - Machine Details

Gets the Integrated Lights Out Manager (ILOM)-related machine details from the
cells using the C<ipmitool mc info> command (applicable for C<Ilom> collection).

=cut

set $cfg_ilom_mac
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool mc info
"echo "---# RDA:END REPORT:4|Machine|Machine Details|Using: \
           /usr/bin/ipmitool mc info|ilom_mac"
}

=head2 network - Network Information

Gets the Integrated Lights Out Manager (ILOM)-related network information from
the cells using the C<ipmitool sunoem cli 'show /SP/network'> command
(applicable for C<Ilom> collection).

=cut

set $cfg_ilom_network
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli 'show /SP/network'
"echo "---# RDA:END REPORT:4|Network|Network Information|Using: \
           /usr/bin/ipmitool sunoem cli 'show /SP/network'|ilom_network"
}

=head2 sdr - SDR Information

Gets the Integrated Lights Out Manager (ILOM)-related SDR information from the
cells using the C<ipmitool sdr> command (applicable for C<Ilom> collection).

=cut

set $cfg_ilom_sdr
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sdr
"echo "---# RDA:END REPORT:4|SDR|SDR Information|Using: /usr/bin/ipmitool sdr|\
           ilom_sdr"
}

=head2 sensor - Sensor Information

Gets the Integrated Lights Out Manager (ILOM)-related sensor information from
the cells using the C<ipmitool sensor> command (applicable for C<Ilom>
collection).

=cut

set $cfg_ilom_sensor
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sensor
"echo "---# RDA:END REPORT:4|Sensor|Sensor Information|Using: \
           /usr/bin/ipmitool sensor|ilom_sensor"
}

=head2 sysinfo - System Information

Gets the Integrated Lights Out Manager (ILOM)-related system information from
the cells using the C<ipmitool sunoem cli 'show /SYS'> command (applicable for
C<Ilom> collection).

=cut

set $cfg_ilom_sysinfo
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli 'show /SYS'
"echo "---# RDA:END REPORT:4|System|System Information|Using: \
           /usr/bin/ipmitool sunoem cli 'show /SYS'|ilom_sysinfo"
}

=head2 imageinfo - Image Information

Gets the image information from the cells (applicable for C<FailedDrives>
collection).

=cut

set $cfg_imageinfo
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /opt/oracle.cellos/imageinfo -all
"echo "---# RDA:END REPORT:4|Image Information|Cell Image Information|\
           Using: /opt/oracle.cellos/imageinfo -all|imageinfo"
}

set $cfg_imageinfo_dft
{echo "---# RDA:BEGIN"
"echo "System Information:\n"
"exec_command ${TIMEOUT:-0} /bin/uname -a
"echo "\nMount Information:\n"
"exec_subshell_command ${TIMEOUT:-0} /bin/mount | /bin/egrep " / "
"echo "\nFiles in /opt/oracle.cellos/patch/history Directory:\n"
"exec_command ${TIMEOUT:-0} /bin/ls -alrt /opt/oracle.cellos/patch/history
"echo "---# RDA:END REPORT:4|Image Information|Cell Image Information||\
           imageinfo"
}

=head2 iostat - Input/Output Statistics

Collects the input/output statistics for devices and partitions using
the F</usr/bin/iostat> command from the cells (applicable for C<FlashDrives>
collections).

=cut

set $cfg_iostat
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/iostat -x
"echo "---# RDA:END REPORT:4|Input/Output Statistics|\
           Input/Output Statistics Information|Using: /usr/bin/iostat -x|iostat"
}

=head2 ipmitool - IPMI Information

Collects the Intelligent Platform Management Interface (IPMI) information using
the C</usr/bin/ipmitool sel elist> command from the cells (applicable for
C<CellFailure> and C<FailedDrives> collections).

=cut

set $cfg_ipmi
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sel elist
"echo "---# RDA:END REPORT:4|IPMI Information|Intelligent Platform Management \
           Interface (IPMI) Information|Using: /usr/bin/ipmitool sel elist|\
           ipmitool"
}

=head2 lspci - PCI Devices

Collects the Peripheral Component Interconnect (PCI) devices information using
the F</sbin/lspci> command from the cells (applicable for C<FailedDrives>
collection).

=cut

set $cfg_pci
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /sbin/lspci
"echo "---# RDA:END REPORT:4|PCI Devices|PCI Devices Information|\
           Using: /sbin/lspci|lspci"
}

=head2 lspci_conf - PCI Devices Configuration

Collects the Peripheral Component Interconnect (PCI) devices configuration
using the C</sbin/lspci -xxxx> command from the cells (applicable for
C<FailedDrives> collection).

=cut

set $cfg_pci_conf
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /sbin/lspci -xxxx
"echo "---# RDA:END REPORT:4|PCI Devices Configuration|PCI Devices \
           Configuration Information|Using: /sbin/lspci -xxxx|lspci_conf"
}

=head2 prdserial - Product Serial Number

Gets the product serial number from the cells (applicable for C<FlashDrives>
collection).

=cut

set $cfg_prdserial
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli \
  'show /SYS product_serial_number'
"echo "---# RDA:END REPORT:4|Product Serial Number|Product Serial Number|\
           Using: ipmitool sunoem cli 'show /SYS product_serial_number'|\
           prdserial"
}

=head2 rackserial - Rack Serial Number

Gets the rack serial number from the cells (applicable for C<FailedDrives>
collection).

=cut

set $cfg_racserial
{echo "---# RDA:BEGIN"
"exec_subshell_command ${TIMEOUT:-0} /usr/sbin/dmidecode -t 1 | \
   /bin/egrep " I|:" | /usr/bin/head -5 | /bin/egrep -v ": *$"; \
   /usr/bin/ipmitool sunoem cli \
   "show /SP system_description system_identifier" | /bin/egrep "="
"echo "---# RDA:END REPORT:4|Rack Serial Number|Rack Serial Number|\
   Using: dmidecode -t 1; \
   ipmitool sunoem cli 'show /SP system_description system_identifier'|\
   rackserial"
}

=head2 raid_status - RAID Status

Gathers Redundant Array of Inexpensive Disks (RAID)-related information from
the cells (applicable for C<FailedDrives> collection).

=cut

set $cfg_raid
{MEGACLI=/opt/MegaRAID/MegaCli/MegaCli64
"echo "---# RDA:BEGIN"
"echo "Controller a0: `$MEGACLI -AdpAllInfo -a0 | \
       /bin/egrep "Product Name" | /bin/cut -d: -f2`"
"echo "No of Physical disks online : `$MEGACLI PDList -a0 | \
       /bin/egrep Online | /usr/bin/wc -l`"
"echo `$MEGACLI -AdpAllInfo -a0 | /bin/egrep "Degrade"`
"echo `$MEGACLI -AdpAllInfo -a0 | /bin/egrep "Failed Disks"`
"echo "---# RDA:END REPORT:4|RAID Status|RAID Status Information|\
           Using: /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -a0|raid_status"
}

=head2 rdmadev - RDMA Devices

Collects the RDMA devices information using the F</usr/bin/ibv_devinfo> command
from the cells.

=cut

set $cfg_rdmadev
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ibv_devinfo
"echo "---# RDA:END REPORT:4|RDMA Devices|RDMA Devices Information|\
           Using: /usr/bin/ibv_devinfo|rdmadev"
}

=head2 lsscsi - SCSI Devices

Collects the Small Computer System Interface (SCSI) devices information using
the F</usr/bin/lsscsi> command from the cells (applicable for C<FailedDrives>,
and C<FlashDrives> collections).

=cut

set $cfg_scsi
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/lsscsi
"echo "---# RDA:END REPORT:4|SCSI Devices|SCSI Devices Information|\
           Using: /usr/bin/lsscsi|lsscsi"
}

=head2 serial - Serial

Collects the Serial number information using the
F</var/log/cellos/SerialNumbers> file (applicable for C<FailedDrives>
collection).

=cut

set $cfg_serial
{fil="/var/log/cellos/SerialNumbers"
"echo "---# RDA:BEGIN"
"/bin/cat $fil
"echo "---# RDA:END REPORT:4|Serial|Information from SerialNumbers|\
            Using: cat $fil|serial"
}

=head2 serial_console - Serial Console History

Collects the serial console history information using the
C<ipmitool sunoem cli 'show /SP/console/history'> command from the cells
(applicable for C<CellFailure> collection).

=cut

set $cfg_serial_con
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli \
                            'show /SP/console/history'
"echo "---# RDA:END REPORT:4|Serial Console History|Serial Console History|\
           Using: ipmitool sunoem cli 'show /SP/console/history'|serial_console"
}

=head2 temp - Temperature Information

Gets the temperature information from the cells (applicable for C<FlashDrives>
collection).

=cut

set $cfg_temp
{echo "---# RDA:BEGIN"
"exec_command ${TIMEOUT:-0} /usr/bin/ipmitool sunoem cli 'show /SYS/T_AMB value'
"echo "---# RDA:END REPORT:4|Temperature Information|Temperature Information|\
           Using: ipmitool sunoem cli 'show /SYS/T_AMB value'|temp"
}

=head2 alert - alert.log

Collects alert.log from the cells (applicable for C<CellFailure>,
C<FailedDrives> and C<SickCell> collections).

=head2 alert_brownout - alert.log

Collects alert.log from the cells (applicable for C<CellBrownout> collection).

=head2 alertxml_brownout - log.xml

Collects log.xml from the cells (applicable for C<CellBrownout> collection).

=head2 msodl - ms-odl.trc

Collects ms-odl.trc from the cells.

=head2 tracelist - Trace File List

Collects the list of trace files.

=head2 tracefiles - Latest Trace Files

Collects the last lines of the trace files (applicable for C<FailedDrives> and
C<SickCell> collections).

=cut

set $log_alert
{for dir in `/bin/ls /opt/oracle/cell/log/diag/asm/cell`
"do
"  dir="/opt/oracle/cell/log/diag/asm/cell/$dir/trace"
"  if [ -d $dir ]
"  then
"    echo "---# RDA:BEGIN SECTION:4:$dir"
"
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_A} "$dir/alert.log"
"    echo "---# RDA:END REPORT:5|alert.log|Information from alert.log|\
                    Using: tail -${TAIL_A} $dir/alert.log|alert"
"
"    echo "---# RDA:END SECTION"
"  fi
"done
}

set $log_all
{for dir in `/bin/ls /opt/oracle/cell/log/diag/asm/cell`
"do
"  dir="/opt/oracle/cell/log/diag/asm/cell/$dir/trace"
"  if [ -d $dir ]
"  then
"    echo "---# RDA:BEGIN SECTION:4:$dir"
"
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_A} "$dir/alert.log"
"    echo "---# RDA:END REPORT:5|alert.log|Information from alert.log|\
                    Using: tail -${TAIL_A} $dir/alert.log|alert"
"
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_M} "$dir/ms-odl.trc"
"    echo "---# RDA:END REPORT:5|ms-odl.trc|Information from ms-odl.trc|\
                    Using: tail -${TAIL_M} $dir/ms-odl.trc|msodl"
"
"    echo "---# RDA:BEGIN"
"    /bin/ls -alt $dir
"    echo "---# RDA:END REPORT:5|Trace File List|Trace File List|\
                    From $dir|tracelist"
"
"    echo "---# RDA:BEGIN LIST"
"    for fil in `/bin/ls -t $dir | /bin/egrep -v alert.log | \
                 /bin/egrep -v ms-odl.trc | /usr/bin/head -10`
"    do
"      echo "---# RDA:BEGIN"
"      /usr/bin/tail -${TAIL_T} "$dir/$fil"
"      echo "---# RDA:END DATA:$fil"
"    done
"    echo "---# RDA:END LIST write_list:$dir|tracefiles|${TAIL_T}"
"
"    echo "---# RDA:END SECTION"
"  fi
"done
}

set $log_cell_brownout
{cel=`cellcli -e list cell attributes name`
"cel=`echo "$cel" | /bin/sed -e 's/^[ \t]*//' -e 's/[ \t]*$//'`
"for dir in `/bin/ls /opt/oracle/cell/log/diag/asm/cell`
"do
"  if [ "$cel" = "$dir" ]
"  then
"    top="/opt/oracle/cell/log/diag/asm/cell/$dir"
"    dir="$top/alert"
"    if [ -d $dir ]
"    then
"      echo "---# RDA:BEGIN SECTION:4:$dir"
"      echo "---# RDA:BEGIN"
"      /usr/bin/tail -${TAIL_A} "$dir/log.xml"
"      echo "---# RDA:END REPORT:5|log.xml|Information from log.xml|\
                      Using: tail -${TAIL_A} $dir/log.xml|alertxml_brownout"
"      echo "---# RDA:END SECTION"
"    fi
"    dir="$top/trace"
"    if [ -d $dir ]
"    then
"      echo "---# RDA:BEGIN SECTION:4:$dir"
"      echo "---# RDA:BEGIN"
"      /usr/bin/tail -${TAIL_A} "$dir/alert.log"
"      echo "---# RDA:END REPORT:5|alert.log|Information from alert.log|\
                      Using: tail -${TAIL_A} $dir/alert.log|alert_brownout"
"      echo "---# RDA:END SECTION"
"    fi
"  fi
"done
}

set $log_db
{for dir in `/bin/ls /opt/oracle/cell/log/diag/asm/cell`
"do
"  dir="/opt/oracle/cell/log/diag/asm/cell/$dir/trace"
"  if [ -d $dir ]
"  then
"    echo "---# RDA:BEGIN SECTION:4:$dir"
"
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_A} "$dir/alert.log"
"    echo "---# RDA:END REPORT:5|alert.log|Information from alert.log|\
                    Using: tail -${TAIL_A} $dir/alert.log|alert"
"
"    echo "---# RDA:BEGIN LIST"
"    for fil in `/bin/ls -t $dir | /bin/egrep -v alert.log | \
                 /bin/egrep -v ms-odl.trc | /usr/bin/head -10`
"    do
"      echo "---# RDA:BEGIN"
"      /usr/bin/tail -${TAIL_T} "$dir/$fil"
"      echo "---# RDA:END DATA:$fil"
"    done
"    echo "---# RDA:END LIST write_list:$dir|tracefiles|${TAIL_T}"
"
"    echo "---# RDA:END SECTION"
"  fi
"done
}

=head2 celllogs - Latest Trace Files

Collects the last lines of the C<cellsrv>-related trace files (applicable for
C<CellFailure> and C<SickCell> collections).

=cut

set $log_cell
{dir="/opt/oracle/cell/cellsrv/deploy/log"
"if [ -d $dir ]
"then
"  pid=`/bin/ps -ef | /bin/egrep "/cellsrv " | /bin/egrep -v grep | \
         /bin/awk '{print $2}'`
"  echo "---# RDA:BEGIN SECTION:4:$dir"
"
"  echo "---# RDA:BEGIN LIST"
"  for fil in `/bin/ls $dir/cellsrv$pid.trc; \
               /bin/ls $dir/cellsrv_pid.lst; \
               /bin/ls $dir/ms.*`
"  do
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_T} "$fil"
"    echo "---# RDA:END DATA:`basename $fil`"
"  done
"  echo "---# RDA:END LIST write_list:$dir|celllogs|${TAIL_T}"
"
"  echo "---# RDA:END SECTION"
"fi
}

=head2 crashfiles - Crash Files

Collects crash files from the cells (applicable for C<CellFailure> collection).

=cut

set $log_crash
{dir="/var/log/oracle/crashfiles"
"if [ -d $dir ]
"then
"  echo "---# RDA:BEGIN SECTION:4:$dir"
"  echo "---# RDA:BEGIN LIST"
"  for fil in `/bin/ls $dir/*.log`
"  do
"    echo "---# RDA:BEGIN"
"    /usr/bin/tail -${TAIL_T} "$fil"
"    echo "---# RDA:END DATA:$fil"
"  done
"  echo "---# RDA:END LIST write_crash_list:crashfiles"
"  echo "---# RDA:END SECTION"
"fi
}

=head2 system_error_log - System Error Log Data

Collects system error log data from the cells (applicable for C<FailedDrives>
collection).

=cut

set $log_dmesg
{echo "---# RDA:BEGIN"
"tail_command ${TIMEOUT:-0} ${TAIL_T} /bin/dmesg
"echo "---# RDA:END REPORT:4|System Error Log Data|Last ${TAIL_T} Lines of \
           System Error Log Data|Using: /bin/dmesg|system_error_log"
}

=head2 sys_err_log - System Error Log

Collects system error log from the cells (applicable for C<CellFailure>
collection).

=cut

set $log_mesg
{echo "---# RDA:BEGIN"
"/usr/bin/tail -${TAIL_T} /var/log/messages
"echo "---# RDA:END REPORT:4|System Error Log|Last ${TAIL_T} Lines of \
           System Error Log|/var/log/messages|sys_err_log"
}

=head2 mslogs - MS Log Files

Collects the last lines of the C<ms*> log files (applicable for
C<FailedDrives> and C<FlashDrives> collections).

=cut

set $log_mslogs
{for dir in `/bin/ls /var/log/oracle/diag/asm/cell`
"do
"  dir="/var/log/oracle/diag/asm/cell/$dir/trace"
"  if [ -d $dir ]
"  then
"    echo "---# RDA:BEGIN SECTION:4:$dir"
"    echo "---# RDA:BEGIN LIST"
"    for fil in `/bin/ls $dir/ms*`
"    do
"      echo "---# RDA:BEGIN"
"      /usr/bin/tail -${TAIL_M} "$fil"
"      echo "---# RDA:END DATA:`basename $fil`"
"    done
"    echo "---# RDA:END LIST write_list:$dir|mslogs|${TAIL_M}"
"    echo "---# RDA:END SECTION"
"  fi
"done
}

# Define the base collection code
if compare('eq',$CELL_USERID,'root')
 var $cod = check($TYPE,\
   '^_CellBrownout_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$job_cell_begin,\
     $cfg_rdmadev,$log_cell_brownout,$job_cell_end,$job_end),\
   '^_CellFailure_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_cellfail,\
     $job_cell_begin,$cfg_ipmi,$cfg_rdmadev,$cfg_serial_con,$log_crash,\
     $log_mesg,$log_all,$log_cell,$job_cell_end,$job_end),\
   '^_FailedDrives_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_all,\
     $job_cell_begin,$cfg_aura,$cfg_aurasmart,$cfg_bios,$cfg_cellinit,\
     $cfg_cli64_all,$cfg_disk_dev,$cfg_fdisk,$cfg_imageinfo,$cfg_ipmi,\
     $cfg_pci,$cfg_pci_conf,$cfg_racserial,$cfg_raid,$cfg_rdmadev,$cfg_scsi,\
     $cfg_serial,$log_alert,$log_dmesg,$log_mslogs,$job_cell_end,$job_end),\
   '^_FlashDrives_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_flash,\
     $job_cell_begin,$cfg_aura,$cfg_bios,$cfg_flash_dom,$cfg_iostat,\
     $cfg_prdserial,$cfg_rdmadev,$cfg_scsi,$cfg_temp,$log_mslogs,\
     $job_cell_end,$job_end),\
   '^_Ilom_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$job_cell_begin,\
     $cfg_ilom_dns,$cfg_ilom_event,$cfg_ilom_lan,$cfg_ilom_mac,\
     $cfg_ilom_network,$cfg_ilom_sdr,$cfg_ilom_sensor,$cfg_ilom_sysinfo,\
     $cfg_rdmadev,$job_cell_end,$job_end),\
   '^_SickCell_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_sick,\
     $job_cell_begin,$cfg_cli64_sick,$cfg_fs_sta,$cfg_rdmadev,$log_cell,\
     $log_db,$job_cell_end,$job_end),\
   '^_All_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_all,\
     $job_cell_begin,$cfg_aura,$cfg_aurasmart,$cfg_bios,$cfg_cellinit,\
     $cfg_cli64_all,$cfg_disk_dev,$cfg_fdisk,$cfg_flash_dom,$cfg_fs_sta,\
     $cfg_imageinfo,$cfg_ipmi,$cfg_pci,$cfg_pci_conf,$cfg_prdserial,\
     $cfg_racserial,$cfg_raid,$cfg_rdmadev,$cfg_scsi,$cfg_serial,$cfg_temp,\
     $log_all,$log_cell,$log_dmesg,$log_mesg,$job_cell_end,$job_end),\
   '')
else
 var $cod = check($TYPE,\
   '^_CellBrownout_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$job_cell_begin,\
     $cfg_rdmadev,$log_cell_brownout,$job_cell_end,$job_end),\
   '^_CellFailure_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_cellfail,\
     $job_cell_begin,$cfg_rdmadev,$log_crash,$log_all,$log_cell,$job_cell_end,\
     $job_end),\
   '^_FailedDrives_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_all,\
     $job_cell_begin,$cfg_aura,$cfg_cellinit,$cfg_fdisk,$cfg_imageinfo_dft,\
     $cfg_pci,$cfg_pci_conf,$cfg_rdmadev,$cfg_scsi,$cfg_serial,$log_alert,\
     $log_dmesg,$log_mslogs,$job_cell_end,$job_end),\
   '^_FlashDrives_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_flash,\
     $job_cell_begin,$cfg_aura,$cfg_iostat,$cfg_rdmadev,$cfg_scsi,$log_mslogs,\
     $job_cell_end,$job_end),\
   '^_SickCell_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_sick,\
     $job_cell_begin,$cfg_rdmadev,$log_cell,$log_db,$job_cell_end,$job_end),\
   '^_All_',join("\012\012",\
     $exe_cmd,$exe_sub,$exe_tail,$cfg_cli_all,\
     $job_cell_begin,$cfg_aura,$cfg_cellinit,$cfg_fdisk,$cfg_imageinfo_dft,\
     $cfg_pci,$cfg_pci_conf,$cfg_rdmadev,$cfg_scsi,$cfg_serial,$log_all,\
     $log_dmesg,$job_cell_end,$job_end),\
   '')

# Collect cell information in threads
if length($cod)
{pretoc '2:Cell Nodes'
 var ($cnt,$pwd,@cel,%acc,%buf,%tim,%tmp,%rpt,%slp) = (0)

 # Initialize the remote sessions
 loop $cel (@{T_CELLS})
 {next !match($cel,'^[\w\-\.]+$')
  call addRemoteSession('EXA',$cel,$CELL_USERID)
  if and(defined($pwd),not(hasPassword('host',$cel,$CELL_USERID)))
   call setPassword('host',$cel,$CELL_USERID,$pwd)
  if needPassword('EXA')
  {var $pwd = setPassword('host',$cel,$CELL_USERID,\
     askPassword("Enter ${VAR.CELL_USERID} at ${VAR.cel} password:",''))
   if needPassword('EXA')
   {var $acc{$cel} = '|Skipped due to connection problems| '
    var $tim{$cel} = 0
    call endRemoteSession('EXA')
    next
   }
  }
  var $slp{$cel} = cond(needPause('EXA'),$CELL_PAUSE,1)
  call endRemoteSession('EXA')
  call push(@cel,$cel)
 }

 # Collect cell specific information using remote session
 var $sta = ${STA.${CUR.K_NAME}.CELL/i}->clear
 call $[REM]->set_timeout($dur = expr('*',25,$TIMEOUT))
 loop $cel (@cel)
 {var $acc{$cel} = '|Collection Incomplete| '
  var $tim{$cel} = 0
  var $tmp{$cel} = concat('R_EXIT',incr($cnt))
  thread($cod,$cel,$sta,$tmp{$cel},$buf{$cel} = getTemp($tmp{$cel}))
  {var ($cod,$cel,$sta,$key,$res) = @arg
   import $ALERT_TAIL,$CELL_USERID,$MSODL_TAIL,$TRACE_TAIL,$TIMEOUT

   debug '  - Getting cell information from ',$cel
   call addRemoteSession('EXA',$cel,$CELL_USERID)
   var $ret = rcollect('EXA',$cod,$res,\
     {CELL => $cel,\
      TAIL_A  => $ALERT_TAIL,\
      TAIL_M  => $MSODL_TAIL,\
      TAIL_T  => $TRACE_TAIL,\
      TIMEOUT => $TIMEOUT\
     })
   if !hasRemoteTimeout('EXA')
    call $sta->set_value($key,$ret,'Remote collection exit code')
   call endRemoteSession('EXA')
   debug '  - Data collection ended for ',$cel
  }
  sleep $slp{$cel}
 }
 sleep 3
 echo 'The cell data collection can take more than ',$dur,\
      ' seconds. Do not interrupt the data collection.'
 wait $dur

 # Generate the cell detail reports
 loop $cel (@cel)
 {if $sta->get_first($tmp{$cel})
   var $acc{$cel} = concat('|Connection error (',last,')| ')
  else
   call writeRemoteResult($buf{$cel} = new('Buffer','R',$buf{$cel}),$cel)
 }
 loop $key (keys(%rpt))
 {var $rpt = $rpt{$key}
  if $rpt->is_created(true)
   toc '3:[[',$rpt->get_report,'][rda_report][',$tb_ttl{$key},']]'
  end $rpt
 }

 # Generate the remaining reports
 loop $cel (@cel)
 {if ref($buf{$cel})
  {call writeRemoteResult($buf{$cel},$cel)
   call $buf{$cel}->close
  }
  call unlinkTemp($tmp{$cel})
 }

=head2 summary - Cell Collection Summary

Displays the cell list and their accessibility.

=cut

 debug ' Inside Exadata module, producing the cell collection summary'
 report summary
 prefix
 {write '---+ Cell Collection Summary'
  write '|*Cell*|*Collection Status*| *Execution Timeout*|'
 }
 loop $key (keys(%acc))
  write '|',$key,$acc{$key},$tim{$key},'|'
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][Cell Collection Summary]]'
 else
  echo 'No cells found'

 unpretoc
}

toc '%UNTITLE%'

=head1 SEE ALSO

L<DB:DBinfo|collect::DB:DBinfo>,
L<DB:MCexa|collect::DB:MCexa>,
L<RDA:library|collect::RDA:library>

=begin credits

=over 10

=item RDA 4.16:  Jean-Marc Gaudron.

=item RDA 4.19:  Jean-Marc Gaudron.

=item RDA 4.20:  Dan Norris.

=item RDA 4.25:  Jean-Marc Gaudron, Deepak Malik, Sathyanarayanan Mohankalyan,
Ericka Washington.

=item RDA 4.26:  Jean-Marc Gaudron, Sathyanarayanan Mohankalyan,
Ericka Washington.

=item RDA 4.27:  Douglas Utzig.

=item RDA 4.28:  Rajan Santhanam, Ericka Washington.

=item RDA 8.02:  Mark Richwine.

=item RDA 8.07:  Ericka Washington.

=back

=end credits

=head1 COPYRIGHT NOTICE

Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.

=head1 TRADEMARK NOTICE

Oracle and Java are registered trademarks of Oracle and/or its
affiliates. Other names may be trademarks of their respective owners.

=cut