#!/bin/sh
# metachk: sanity-check DiskSuite configuration and status, version 2
# Module: the whole program
#
version="metachk 2.3.3 16may2003"
#
# Copyright (c) 1998, 2001, 2002, 2003 Mike Spooner
#----------------------------------------------------------------------
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#----------------------------------------------------------------------
#
# 
# Description:	checks DiskSuite metadevices and state-database
#		for various error conditions. Warning and error
#		messages are emitted to stderr.
#		Returns 2 if any errors, 1 if warnings, else 0.
#
#		Error-messages describe storage-component faults
#		that have almost certainly already caused
#		data-loss.
#		Error-conditions should be fixed as soon as possible.
#
#		Warning-messages describe component failures or
#		misconfiguration that may subsequently compromise
#		data-integrity (eg: a one-way mirror!).
#		
#
# Implementation/Design Notes:
#
#	Nearly all parts of this program expects slice-names to be shorn
#	of any /dev/dsk/ or /dev/rdsk/ prefix.
#
#	The subroutines and most of the rest of the program expect 
#	metadevice names and hot-spare-pool names to be DISKSET QUALIFIED
#	eg: "blue/d10", rather than "-s blue d10".
#
#	This program makes extensive use of the information reported
#	by "metastat". Unfortunately, metastat is "locked out" whilst
#	manual or automatic reconfigurations are in progress or if a
#	state-database replica is being "probed" after a failure.
#
#	Thus it is expensive and potentially extremely slow to to be
#	invoking metastat (or metadb) lots and lots of times. Instead this
#	program calls metadb and metastat once or twice for each diskset,
#	and caches the information for subsequent extensive use. The exception
#	is the nodestat() function, so use it as little as possible!
#
#	There is nothing CPU-dependant in this program, not even
#	indirectly via CPU-specific Solaris features. This is as it
#	should be.
#
#	The program should also be capable of dealing with the multiple
#	different block-device naming conventions (eg: cXtXdXsX and cXdXsX).
#	The program will have to be changed if other additional
#	naming conventions are introduced by future releases of Solaris.
#
#	Although there are many funky new versions (XPG4) and ways to use
#	eg: nawk, grep, sed, etc in recent releases of Solaris, and wacky new
#	utilities too, this program always restricts itself to the
#	forms and facilities available on ALL public releases of Solaris
#	from 2.1 onwards.
#
#	There are also additional facilities available in newer releases
#	of DiskSuite (UFS logging, disksets, RAID-5). However, where
#	necessary (disksets!), this program explicitly tests for the
#	availability of the facility, and does not use it if not present.
#	Again, the intent is to run unmodified against all public releases
#	of DiskSuite from 2.0.1 onwards, at least for the forseeable future.
#
#	Finally, the program must be capable of running in single-user
#	mode, so that it can be used to diagnose faults in the state-database
#	for the local diskset that may be preventing the system from reaching
#	multi-user mode.
#
#
# Caveats:
#
#	Currently, it is not possible to detect failures of unprotected
#	components (eg: a raw slice or concat/stripe that is not part of
#	a mirror or raid5).
#
#	Worse, a UFS filesystem housed on a concat/stripe made up of
#	components with different geometries has a peculiar risk: if a
#	component goes "offline" (eg: disk drive switched off or falls off
#	the SCSI bus), then I/O is attempted, and then the failed component
#	comes back online (eg: drive switched on, or SCSI bus reset),
#	the system will usually panic - the pending I/Os are resubmitted
#	and the metadevice driver indicates I/O completion but fails to
#	actually perform the I/O (it thinks that the device-geometry is
#	"wrong"). The "missing" I/O causes the UFS driver to panic.
#	This is a bug in DiskSuite, all versions from 2.0.1 through 4.0.
#	It may also apply to version 4.1.
#
#
# Intended Platforms:
#
#	Solaris 2.1 and later (all available architectures)
#	DiskSuite 2.0.1 and later
#
#
# Tested Platforms:
#
#	SPARC: Solaris 2.1 with DiskSuite 2.0.1
#	SPARC: Solaris 2.2 with DiskSuite 2.0.1, 3.0
#	SPARC: Solaris 2.3 with DiskSuite 2.0.1, 3.0, 4.0
#	SPARC: Solaris 2.4, 2.5, 2.5.1 with DiskSuite 2.0.1, 3.0, 4.0, 4.1
#	x86: Solaris 2.4 with DiskSuite 4.1
#	SPARC: Solaris 2.6 with DiskSuite 4.0, 4.1
#	SPARC: Solaris 7 (32-bit and 64-bit) with DiskSuite 4.1, 4.2
#	x86: Solaris 7 with DiskSuite 4.2
#	SPARC: Solaris 8 (32-bit and 64-bit) with DiskSuite 4.2.1
#	x86: Solaris 8 with DiskSuite 4.2.1
#	SPARC: Solaris 9 (64-bit) with DiskSuite 4.2.1
#
# Contributors:
#
#	Fixes for the checking of references to non-existant hotspare pools
#	were donated by Anthony Tonns on 09oct2002.
#

DEBUG=no


########################################################################
#
# Subroutines:
#
########################################################################

# print maximum of two numbers to stdout
#
max()
	{
	if [ "$1" -gt "$2" ] ; then
		echo "$1"
	else
		echo "$2"
	fi
	}

# print message to stdout
#
remark()
	{
	/bin/echo "`basename $0`: $*"
	}

# print error message to stderr, and set error-level
#
error()
	{
	remark 'error:' $* 1>&2
	S=`max 3 $S`
	}

# print warning message to stderr, and set error-level
#
warning()
	{
	remark 'warning:' $* 1>&2
	S=`max 2 $S`
	}

# strip "#-to-end-of-line" comments and perform line-continuations,
# from named files (or stdin if none) to stdout...
#
splice()
	{

	nawk '
		{ \
			# strip "#-to-end-of-line" comments:
			sub("#.*$", "", $0);
			
			# force trailing backslash to be a separate field:
			sub("\\$", " &", $NF);
			
			if (!length($0))	# empty line
				{
				# discard it
				}
			else if ($NF == "\\")	# continued
				{
				# print it without a newline,
				# but with a trailing tab...
				#
				# NOTE: when checking if it already has
				# trailing whitespace, we cannot use match(),
				# because field NF-1 may contain what would
				# be an invalid regular-expression which would
				# make nawk bomb out - so we use index(), which
				# always interprets arguments as literal
				# strings.
				#
				x = NF - 1
				if (index(" ", $x) || index("\t", $x))
					$NF = "";	# already got one
				else
					$NF = "\t";
				printf("%s", $0);
				}
			else			# not continued
				{
				# print it with a newline:
				printf("%s\n", $0);
				}
		}
	' $*
	}

# reformat output of metahs -i" into tabular format
#
fmthst()
	{
	cat $* | nawk '
		function putrec(p, d, s)
			{
			if (match(s, "[Ii]n$") == 1)
				s = "In-use";
			printf("%s:%s:%s\n", p, d, s);
			flush = 0;
			}
		BEGIN		{ flush = 0; }
		/hsp[0-9]/	{ sub(":", "", $1); pool = $1 ; flush = 1; }
		/.*/	{ \
			if (length($0) && !match($0, "hsp[0-9]"))
				{
				gsub("/dev/dsk/", "", $0);
				gsub("[()]", "", $2);
				putrec(pool, $1, $2);
				}
			}
		END	{ if (flush) putrec(pool, "", ""); }
		'
	}
		
# reformat output of "metastat -p" into
# type-parent-children table,
# from named files (or stdin if none) to stdout...
#
fmttpc()
	{
	splice $* | nawk '
		function refhsp(f)
			{
			# parse metaoptions into referenced-hotspare-pool field
			for (i = f; i < NF; ++i)
				{
				if ($i == "-h")
					{
					++i;
					printf("%s", $i);
					break;
					}
				}
			}
			
		function complex(type)
			{
			printf("%s:%s:", type, $1);
			sep=""
			for (f = 3; f <= NF; ++f)
				{
				if (match($f,
					"[-+_a-zA-Z0-9/]*d[0-9][0-9]*"))
					{
					printf("%s%s", sep, $f);
					sep=" ";
					}
				else
					break;
				}
			printf(":");
			refhsp(f);
			printf(":\n");
			}
		
		function simplex()
			{
			printf("concat/stripe:%s:", $1);
			sep="";
			f = 2;
			nstripes = $f;
			++f;
			for (s = 1; s <= nstripes; ++s)
				{
				width = $f;
				++f;
				for (i = 1; i <= width; ++i)
					{
					printf("%s%s", sep, $f);
					sep=" ";
					++f;
					}
				# skip the stripe-column-interleave spec, if any
				if ( $f == "-i" )
					f += 2;
				}
			printf(":");
			refhsp(f);
			printf(":\n");
			}
		
		/^[-+_a-zA-Z0-9\/]*d[0-9][0-9]*[ \t]/ { \
			gsub("/dev/dsk/", "", $0);
			gsub("/dev/md/dsk/", "", $0);
			if (index($2, "-m") == 1)
				complex("mirror");
			else if (index($2, "-r") == 1)
				complex("raid5");
			else if (index($2, "-t") == 1)
				complex("trans");
			else
				simplex();
		}
	' | sort
	}


# output space-seperated input words each on a separate line...
# from named files (or stdin if none) to stdout...
#
words()
	{
	cat $* | tr -s ' ' '\012'
	}

# convert slice-names to disk-names
# from named files (or stdin if none) to stdout...
#
disknm()
	{
	# NOTE: there are two slice-name formats under Solaris -
	#	cXtXdXsX for SCSI, IPI, SMD, SSA, etc, etc, etc.
	#	cXdXsX for local IDE, etc.
	#
	sed \
		-e's/\(c[0-9][0-9]*d[0-9][0-9]*\)s[0-9][0-9]*/\1/' \
		-e's/\(c[0-9][0-9]*t[0-9][0-9]*d[0-9][0-9]*\)s[0-9][0-9]*/\1/'
	}


########################################################################
#
# Main Program:
#
########################################################################
	
# initialise variables, etc.
#
S=0
UID=`id | cut -d= -f2 | cut -d'(' -f1`
exec 9<&0	# remember original stdin

clean="rm -f /tmp/$$ /tmp/$$.*"
trap "$clean" 0 1 2 3 13 15
$clean


# parse command-line flags:
#
USAGE="usage: `basename $0` [ -m -version ]"
usage()
	{
	echo "$USAGE" 1>&2
	exit 1
	}
allow1waymirrors=''
while [ 1 = 1 ] ; do
	case "$1" in
	-m)
		allow1waymirrors=yes
		shift
		;;
	-version)
		echo "$version"
		exit 0
		;;
	-*)
		usage
		;;
	*)
		break
		;;
	esac
done


# if DiskSuite is not installed or activated, then there can be no faults!
# Note that we must cater for both the locations used by DS 4.2.1 and those
# used by earlier versions of DS.
#
ETC=''
for dir in /etc/opt/SUNWmd /etc/lvm ; do
	if [ -f "$dir"/mddb.cf ] ; then
		ETC="$dir"
		break
	fi
done
[ -z "$ETC" ] && exit $S
BIN=''
for dir in /usr/opt/SUNWmd/sbin /usr/sbin ; do
	if [ -x "$dir"/metastat ] ; then
		BIN="$dir"
		PATH=${BIN}:${PATH}
		export PATH
		break
	fi
done
[ -z "$BIN" ] && exit $S


##############################################################################
#
# PHASE ONE:	for all owned disksets (including the private local one),
#		check status of state-databases and collect metadevice
#		and hotspare configuration of the diskset.
#
##############################################################################

# NOTE: releases of DiskSuite before v4.0 only provide a local diskset
# (they do not have a metaset command), thus we can use the metaset command
# and the "-s set" options to the other commands, ONLY if metaset actually
# exists and is executable.
#
if [ -x $BIN/metaset ] ; then
	# NOTE: the state-database for the local diskset also holds
	# locators for any shared disksets, so we must check the local
	# diskset FIRST...
	#
	sets="local `metaset 2>/dev/null | grep 'Set name' | cut -d= -f2 | cut -d, -f1`"
else
	sets='local'
fi

for set in $sets ; do
	if [ "$set" = 'local' ] ; then		# local diskset
		prefix=''
		select=''
	else				# shared diskset
		prefix="$set/"
		select="-s $set"
		# skip checks if we do not own the diskset:
		metaset $select -o 2>/dev/null || continue
	fi

	# check DiskSuite active configuration database:
	#
	if metadb $select 2>/dev/null | tail +2 | sort -t/ +1.0 > /tmp/$$ ; then
		count=`wc -l < /tmp/$$`
		n=1
	
		# for each metadevice state-database replica...
		#
		while [ "$n" -le "$count" ] ; do
			line=`head -$n /tmp/$$ | tail -1`
			dev="/`echo $line | cut -d/ -f2-`"
			bits="`echo $line | cut -d/ -f1`"
	
			# check replica for faults:
			#
			# IMPORTANT NOTE: the case tests are organised in
			# order of severity, to ensure that only the most
			# appropriate message is output, rather than
			# multiple redundant messages.
			#
			case $bits in
				*R*)
					warning "$set diskset: state database" \
						"on $dev" \
						'has read error'
					;;
				*W*)
					warning "$set diskset: state database" \
						"on $dev" \
						'has write error'
					;;
				*M*|*D*|*F*)
					warning "$set diskset: state database" \
						"on $dev" \
						'has data error'
					;;
				*S*)
					warning "$set diskset: state database" \
						"on $dev" \
						'is too small' 
					;;
			esac
	
			# check that replica is up-to-date:
			#
			# NOTE: we do this even if another fault has already
			# been reported, because it is information needed to
			# assess the immediacy of the other possible faults.
			#
			echo $bits | fgrep -v u >/dev/null 2>&1 &&
				warning "$set diskset: state database on $dev" \
					'is out of date'
	
			n=`expr $n + 1`
		done

		# and cache the metadevice configuration for this diskset...
		#
		# NOTE: we discard error-messages output by metastat and metahs,
		# they whinge if you have a "null" configuration (no metadevice
		# state-databases, or no hotspare-pools).
		#
		metastat $select 2>/dev/null >> /tmp/$$.ms
		metastat $select -p 2>/dev/null | fmttpc >>/tmp/$$.tpc
		metahs $select -i 2>/dev/null | fmthst >> /tmp/$$.hst
	else
		error "$set diskset: state database unreachable" \
			'(manual intervention needed IMMEDIATELY)'

		# when the local diskset state database is unavailable,
		# there is no point in continuing...
		#
		[ "$set" = "local" ] && exit $S
	fi

	rm -f /tmp/$$
done


# save parse results when debugging:
#
if [ "$DEBUG" = "yes" ] ; then
	[ -d /tmp/mc ] || mkdir /tmp/mc
	fmttpc $ETC/md.tab > /tmp/mc/tab
	cp /tmp/$$.tpc /tmp/mc/tpc
	cp /tmp/$$.ms /tmp/mc/ms
	cp /tmp/$$.hst /tmp/mc/hst
fi


##############################################################################
#
# Define those functions that utilise the cached metastat information,
# now that the cache has been populated.
#
##############################################################################

# print the "type" of a metadevice.
#
# Argument is the name of a metadevice.
#
metatype()
	{
	[ -n "$1" ] && grep "^[^:]*:${1}:" /tmp/$$.tpc | cut -d: -f1
	}

# print a space-seperated list of children of the specified metadevice node.
#
# Argument 1 is the name of the parent metadevice.
#
children()
	{
	[ -n "$1" ] && grep "^[^:]*:${1}:" /tmp/$$.tpc | cut -d: -f3
	}

# print the name of the parent of a slice or metadevice from the
# metadevice-tree.
# If there is no parent, nothing is printed.
#
# Argument 1 is the name of the presumed child, shorn of any diskset prefix.
#
parent()
	{
	if [ -n "$1" ] ; then
		grep "^[^:]*:[^:]*:${1}[ :]" /tmp/$$.tpc | cut -d: -f2
		grep "^[^:]*:[^:]*:[^:]* ${1}[ :]" /tmp/$$.tpc | cut -d: -f2
	fi
	}

# print list of metadevices in the specified state...
#
# Argument is a regular-expression specifying a set of states
# eg: "Needs maintenance" or "Okay"
#
matchstate()
	{
	[ -n "$1" ] && nawk '
		/^d[0-9][0-9]*:/ { node = $1; gsub(":", "", node); }
		/State: '"$1"'/ { printf("%s\n", node); }
		' /tmp/$$.ms
	}

# output list of all disks that the specified metadevice is currently using
# (including any currently attached hot-spares)...
#
# Argument is a diskset-qualified metadevice name.
#
# NOTE: the output may list a disk more than once: use sort and uniq
#	to post-process the output (sometimes we want to discard
#	the duplicates, other times we want to discard everything else!).
#
# WOW:	recursive descent - neat, huh?
#
disks_used()
	{
	for child in `children $1` ; do
		case $child in
			c[0-9]*)	echo $child | disknm ;;
			*)		disks_used $child ;;
		esac
	done
	}


#############################################################################
#
# PHASE TWO:	check metadevices' configuration for data-danger
#
#############################################################################

# check for empty hotspare pools:
#
grep '^hsp[0-9][0-9]*::' /tmp/$$.hst > /tmp/$$
exec 0</tmp/$$
while read x ; do
	s="`echo $x | cut -d: -f1`"
	warning "hotspare pool $s is empty"
done
exec 0<&9


# check for mirrors or raid5s with multiple
# components on the same disk:
#
for type in mirror raid5 ; do
	for md in `grep "^${type}:" /tmp/$$.tpc | cut -d: -f2` ; do
		s="`disks_used $md | sort | uniq -d`"
		for d in $s ; do
			warning "$type $md has multiple" \
				"components on single disk $d"
		done
	done
done


# check for trans with protected (mirrored or RAID-5) master but
# unprotected (non-mirror and non-RAID-5) log.
#
# NOTE: careful to ignore logless trans metadevices.
#
grep '^trans:[^:]*:[^ :]*d[0-9][0-9]* [^ :]*d[0-9][0-9]*:' /tmp/$$.tpc > /tmp/$$
exec 0</tmp/$$
while read x ; do
	md="`echo $x | cut -d: -f2`"
	master="`echo $x | cut -d: -f3 | awk '{ print $1 }'`"
	log="`echo $x | cut -d: -f3 | awk '{ print $2 }'`"

	tl="`metatype $log`"
	##remark "check trans $md: master $master, log $log ($tl)"
	if [ "$tl" != mirror ] && [ "$tl" != raid5 ] ; then
		tm="`metatype $master`"
		if [ "$tm" = mirror ] || [ "$tm" = raid5 ] ; then
			warning "trans $md: master $master ($tm)" \
				"has unprotected log $log ($tl)"
		fi
	fi
	rm -f /tmp/$$.[12]
done
exec 0<&9


# check for one-way mirrors:
#
if [ -z "$allow1waymirrors" ] ; then
	s="`grep '^mirror:[^:]*:[^ :]*d[0-9][0-9]*:' /tmp/$$.tpc | cut -d: -f2`"
	for md in $s ; do
		warning "mirror $md has only a single submirror"
	done
fi


# check for logless trans:
#
s="`grep '^trans:[^:]*:[^ :]*d[0-9][0-9]*:' /tmp/$$.tpc | cut -d: -f2`"
for md in $s ; do
	warning "trans $md has no logging device"
done


# check for mirror or RAID-5 that references a non-existant hotspare-pool:
#
grep '^[^:]*:[^:]*:[^:]*:hsp' /tmp/$$.tpc > /tmp/$$
exec 0</tmp/$$
while read x ; do
	s="`echo $x | cut -d: -f4`"
	if grep "^${s}:" /tmp/$$.hst >/dev/null ; then
		:
	else
		md="`echo $x | cut -d: -f2`"
		type="`echo $x | cut -d: -f1`"
		warning "$type $md references non-existant hotspare-pool $s"
	fi
done
exec 0<&9


#############################################################################
#
# PHASE THREE:	check metadevices' status for data-danger
#
#############################################################################

# check for metadevices and components in "last errored" or
# "hard error" state...(lost or corrupted data is implied)...
#
# This detects:
#
#	multiple overlapping failed components of submirror
#	multiple failed components of raid5
#
for md in `matchstate "[Ll]ast [Ee]rr"` ; do
	error "`metatype $md` $md errored (data may be lost)"
done
for md in `matchstate "[Hh]ard [Ee]rr"` ; do
	error "`metatype $md` $md errored (data lost)"
done


# check for metadevices and components in "maintenance" state:
# This detects:
#
#	offlined submirror
#	failed components of submirror or raid5
#	detached trans log
#
for md in `matchstate "Needs maintenance"` ; do
	warning "`metatype $md` $md needs maintenance"
done


# check for 'in-use' hot-spares:
#
if grep ':[Ii]n.[Uu]se' /tmp/$$.hst >/tmp/$$ ; then
	exec 0</tmp/$$
	while read x ; do
		pool="`echo $x | cut -d: -f1`"
		slice="`echo $x | cut -d: -f2`"
		# find out who is using it...
		s="`grep '^[^:]*:[^:]*:[^:]*'${slice} /tmp/$$.tpc | head -1`"
		md="`echo $s | cut -d: -f2`"
		mt="`echo $s | cut -d: -f1`"
		if [ -n "$s" ] ; then
			warning "$mt $md is using hotspare $slice from pool $pool"
		else	# "should not happen"
			warning "hotspare $slice from pool $pool is in use"
		fi
	done
	exec 0<&9
fi
rm -f /tmp/$$


# check for 'broken' hotspares:
#
if grep ':[Bb]roken' /tmp/$$.hst >/tmp/$$ ; then
	exec 0</tmp/$$
	while read x ; do
		pool="`echo $x | cut -d: -f1`"
		slice="`echo $x | cut -d: -f2`"
		warning "hotspare $slice in pool $pool is broken"
	done
	exec 0<&9
fi
rm -f /tmp/$$


#############################################################################
#
# PHASE FOUR:	check md.tab is up to date
#
#############################################################################

ok="yes"	# until proven otherwise...

# non-mirrors first...
#
splice $ETC/md.tab | fmttpc > /tmp/$$
grep -v '^mirror:' /tmp/$$.tpc > /tmp/$$.1
grep -v '^mirror:' /tmp/$$ > /tmp/$$.2
if cmp /tmp/$$.1 /tmp/$$.2 >/dev/null 2>&1; then
	#
	# non-mirrors are OK, now check the mirrors.
	# We must allow for mirrors in md.tab to be defined
	# as one-way mirrors, even when they are currently
	# configured as multi-way mirrors - ie: according to
	# conventional safety measures.
	#
	# Thus the test against each mirror is: for each
	# submirror in md.tab, make sure that active configuration
	# includes at least that sunmirror, BUT NOT VICE VERSA.
	#
	grep '^mirror:' /tmp/$$ > /tmp/$$.1
	exec 0</tmp/$$.1
	while read x ; do
		md="`echo $x | cut -d: -f2`"
		echo "$x" | cut -d: -f3 >/tmp/$$.2
		for sm in `cat /tmp/$$.2` ; do
			grep "^mirror:${md}:" /tmp/$$.tpc |
				cut -d: -f3 | words > /tmp/$$.3
			if grep "^${sm}$" /tmp/$$.3 >/dev/null ; then
				:
			else
				ok="no"
				break
			fi
		done
		rm -f /tmp/$$.[23]
		[ "$ok" = "no" ] && break
	done
	exec 0<&9
else
	ok=no
fi
rm -f /tmp/$$.[12] /tmp/$$

if [ "$ok" = "no" ] ; then
	warning "template file (md.tab) does not match" \
		"active configuration"
	continue
fi



############################################################################
#
# PHASE FIVE:	check that md.tab is not dangerous
#
############################################################################

# check that mirrors are defined as one-way in md.tab
#
fmttpc $ETC/md.tab | grep '^mirror:' | \
	grep -v '^mirror:[^:]*:[^ :]*d[0-9][0-9]*:' > /tmp/$$
exec 0</tmp/$$
while read x ; do
	md="`echo $x | cut -d: -f2`"
	warning "$md is defined as multi-way mirror in template file (md.tab)"
done
exec 0<&9

exit $S
