findpkgs

#!/bin/bash
#
# findpkgs
#

# Use unset in case these have been set elsewhere.
unset BLD && declare BLD=""
unset UND && declare UND=""
unset OFF && declare OFF=""

# Contlol-C exit code
# see http://www.tldp.org/LDP/abs/html/exitcodes.html
unset CTLC_EXIT &&  declare -i CTLC_EXIT=130

# Other exit codes
declare -i EXIT_OK=0
declare -i EXIT_INVARG=1
declare -i EXIT_INVFIL=2
declare -i EXIT_INVDIR=3

declare -a exitmsgary=(
""
"Invalid number of arguments."
" is an invalid filename."
" is an invalid directory name."
)

exitme() {
	local exitval=$1
	local strarg=""
	local exitmsg

	if ([ $exitval -ne $EXIT_OK ] && [ $exitval -ne $CTLC_EXIT ]); then
		[ $# -eq 2 ] && strarg=$2
		[ ${#exitmsgary[@]} -gt $exitval ] \
			&& exitmsg="${exitmsgary[$exitval]}"

		echo -e "$BLD$strarg$exitmsg$OFF"

		[ $exitval -ne 0 ] && echo -e \
			"Type$BLD findpkgs -h$OFF for help."
	fi

	exit $exitval
}

# run if user hits control-c
#
control_c()
{
	echo -en "
Ctrl-c detected
Cleaning up and exiting.
"
	exitme $CTLC_EXIT
}


declare -i optcount=0

declare usagestr=$(
cat <<EOF

$(basename $0) [options] pkglist topdir

Description:

  This script will crawl the RHEL7 and RHE8 latest builds and look for
  packages listed in the pkglist argument. The links to the top of the
  most recent builds are...

  RHEL7
  http://download.eng.bos.redhat.com/rel-eng/latest-RHEL-7/compose
  http://download.eng.bos.redhat.com/rel-eng/latest-Supp-7-RHEL-7/compose
  http://download.devel.redhat.com/brewroot/repos/rhel-7.0-build/latest/x86_64/pkglist

  RHEL8
  http://download-node-02.eng.bos.redhat.com/rel-eng/latest-RHEL-8/compose
  http://download.eng.bos.redhat.com/rel-eng/latest-Supp-8.0-RHEL-8/compose
  http://download.devel.redhat.com/brewroot/repos/rhel-8.0-build/latest/x86_64/pkglist

Arguments:
  pkglist - This is a file containing a list of packages to seek.
            The format of the package list is a colon-separated list as
            follows.
            package : maintainer : rhel : [rhel]
            The optional rhel as a fourth field will appear if the package
            is found in both RHEL7 and RHEL8.
            Use the getpkglist script to create this list.
  topdir  - The directory into which the RHEL7 and RHEL8 distro trees are
            created by the -b option and accessed by the rest of the script.

Options:
  -b  - Builds the RHEL7 and RHEL8 distro directory trees.
        This is only needed once, or if you need to refresh the distro trees.
  -h  - Prints this help text to the terminal.

\0
EOF
)

usage() {
	echo -en "$usagestr"
	exitme 0
}

declare b_buildtrees=false
declare b_arches=false

declare rh7link="http://download.eng.bos.redhat.com/rel-eng/latest-RHEL-7/compose"
declare su7link="http://download.eng.bos.redhat.com/rel-eng/latest-Supp-7-RHEL-7/compose"
declare br7link="http://download.devel.redhat.com/brewroot/repos/rhel-7.0-build/latest/x86_64/pkglist"

declare rh8link="http://download-node-02.eng.bos.redhat.com/rel-eng/latest-RHEL-8/compose"
declare su8link="http://download.eng.bos.redhat.com/rel-eng/latest-Supp-8.0-RHEL-8/compose"
declare br8link="http://download.eng.bos.redhat.com/brewroot/repos/brew-rhel-8-build/latest/x86_64/pkglist"

declare pkg="os/Packages"
declare topdir=		# dir where rh7 and rh8 subdirs are created
declare -i tdindex		# index to first dir after topdir/rh[7|8], which is where the
			# : path to the packages starts.
declare -a rh8stream=(
"Supplementary"
"AppStream"
"BaseOS"
"CRB"
"HighAvailability"
"NFV"
"RT"
"ResilientStorage"
)
declare -i rh8streamsiz=${#rh8stream[@]}

declare -a rh7stream=(
"Supplementary"
"Client-optional"
"Client"
"ComputeNode-optional"
"ComputeNode"
"Server-NFV"
"Server-RT"
"Server-SAP"
"Server-SAPHANA"
"Server-optional"
"Server"
"Workstation-optional"
"Workstation"
)
declare -i rh7streamsiz=${#rh7stream[@]}

declare -a rh7arch=(
"x86_64"
"ppc64"
"ppc64le"
"s390x"
)
declare -i rh7archsiz=${#rh7arch[@]}

declare -a rh8arch=(
"x86_64"
"ppc64le"
"s390x"
"aarch64"
)
declare -i rh8archsiz=${#rh8arch[@]}

declare hcmd="elinks -dump -no-references"
declare -a farray	# array of discovered or not found packages

# ui_strindex string-1 string-2
#
# Return zero-based index of the first instance of string-2 in string-1
# Returns -1 if string-2 is not found in string-1
#
function ui_strindex {
	local x="${1%%$2*}"
	[[ $x = $1 ]] && echo -1 || echo ${#x}
}

# ui_strtok - tokenizes a string and returns an array
#
# $1 - the string to tokenize
# $2 - the string of delimiters
# $3 - returns the tokenized string as an array
#
# examples:
#
# ui_strtok kernel-pegas-4.11.0-2.el7.x86_64 ".-" array
#
# ${array[@]} will contain all the tokens separated by delimiters "."
# and "-", and the delimiters will be replaced by spaces.
#
ui_strtok() {
	IFS="$2" read -r -a $3 <<< "$1"
}

# get_pkgname - extract the n from nvr
#
# Tokenize the package name.
# There are packages with names like this..
#
#   java-1.6.0-openjdk-demo-1.6.0.41-1.13.13.1.el7_3.x86_64.rpm
#
# ..where splitting on the last hyphen isn't good enough to isolate the
# package name.
#
# The last token is automatically dropped, since it's the distro and arch.
# It's the second to last token that must be examined. If it begins with a
# number, it will be dropped and all preceding tokens will be recombined to
# the package name.
#
get_pkgname(){
    local p="$1"
    local pary
    local pn
    local j
    local k
    local f

    ui_strtok "$p" "-" pary

    # We will always use the first token.
    #
    pn=${pary[0]}

    # Get the index to the second to last token. If its first char is a
    # number, then drop it and set the index to the previous token.
    #
    k=$((${#pary[@]} - 2))
    f=${pary[k]}
    [[ ${f:0:1} =~ ^-?[0-9]+$ ]] && let --k

    # Now get the remaining tokens.
    #
    for ((j = 1; j <= k; ++j)); do
	    pn="$pn""-""${pary[j]}"
    done

    eval $2="$pn"
}

clean_pkglist() {
	local path=$1
	local dir=$(dirname $path)
	local name
	local lary

	echo $path
	> $dir/tmp

	while read line; do
		[[ $line == *".rpm"* ]] || continue
		ui_strtok "$line" "] " lary
		line=${lary[3]}
		get_pkgname "$line" name
		echo $name >> $dir/tmp
	done < $path

	mv -f $dir/tmp $path
}

# getpkglist - acquire a list of packages at the url
#
# Globals
#   topdir    - top directory for the RHEL7 and RHEL8 trees
#
# Arguments
#   outsubdir - the rh7 or rh8 path in the repo at the link
#   stream    - Server, Client, Workstation, etc.
#   arch      - x86_64, ppc, etc
#   link      - url for the given stream
#   dir       - directory into wich the packages go
#
getpkglist() {
	local link="$1"
	local outsubdir="$2"

	[ -d $outsubdir ] || mkdir -p $outsubdir

	$hcmd $link > $outsubdir/pkglist
	clean_pkglist "$outsubdir/pkglist"
}

clean_brpkglist() {
	local path="$1"
	local name
	local tfil=/tmp/tfil

	echo $path
	> $tfil

	while read line; do
		[[ $line == *".rpm"* ]] || continue
		line=$(basename $line)
		get_pkgname "$line" name
		echo $name >> $tfil
	done < $path

	mv -f $tfil $path
}

get_brpkglist() {
	# topdir is global
	local distro="$1"
	dir="$topdir/rh$distro/BUILDROOT"
	local link

	[ $distro -eq 7 ] && link="$br7link"
	[ $distro -eq 8 ] && link="$br8link"

	[ -d $dir ] || mkdir -p $dir

	$hcmd $link > $dir/pkglist
	clean_brpkglist "$dir/pkglist"
}

# buildtrees() - build the RHEL7 and RHEL8 trees from the distros
#
# Globals
#   pkg          - os/Packages
#   topdir       - top directory for RHEL trees
#   rh7link      - url of rh7 distro tree
#   br7link      - url of rh7 BUILDROOOT
#   su7link      - url of rh7 Supplementary
#   rh7stream    - array of streams in RHEL7
#   rh7streamsiz - number of elements in the the rh7stream array
#   rh8link      - url of rh8 distro tree
#   br8link      - url of rh8 BUILDROOT
#   su8link      - url of rh8 Supplemental
#   rh8stream    - array of streams in RHEL8
#   rh8streamsiz - number of elements in the rh8stream array
#
buildtrees() {
	# topdir is global

	local j	        # - iterating variables
	local k         # - :
	local stream    # - extracted from given stream array
	local arch      # - extracted from given arch array
	local urlsubdir # - composite subdir of the url distro tree
	local outsubdir # - composite subdir of the destination dir
	local link      # - copmosite of appropriate link and urlsubdir

	echo "RHEL7"
	echo "-----"

	get_brpkglist 7

	# loop through streams
	#
	for ((j = 0; j < $rh7streamsiz; ++j)); do

		stream=${rh7stream[$j]}

		if [[ $stream == "Server" ]]          || \
		   [[ $stream == "Server-optional" ]]; then

			# loop through arches
			#
			for ((k = 0; k < $rh7archsiz; ++k)); do
				arch=${rh7arch[$k]}
				link=$rh7link/$stream/$arch/$pkg
				outsubdir=$topdir/rh7/$stream/$arch
				getpkglist $link $outsubdir
			done

		elif [[ $stream == "Supplementary" ]]; then

			# loop through arches
			#
			for ((k = 0; k < $rh7archsiz; ++k)); do
				arch=${rh7arch[$k]}
				link=$su7link/Server/$arch/$pkg
				outsubdir=$topdir/rh7/$stream/$arch
				getpkglist $link $outsubdir
			done
		else
			# Other streams only support arch x86_64
			#
			link=$rh7link/$stream/x86_64/$pkg
			outsubdir=$topdir/rh7/$stream/x86_64
			getpkglist $link $outsubdir
		fi
	done

	echo

	echo "RHEL8"
	echo "-----"

	get_brpkglist 8

	# Loop through streams
	#
	for ((j = 0; j < $rh8streamsiz; ++j)); do

		stream=${rh8stream[$j]}

		if [[ $stream == "AppStream" ]] || \
		   [[ $stream == "BaseOS" ]]    || \
		   [[ $stream == "CRB" ]]; then

			# Loop through arches
			#
			for ((k = 0; k < $rh8archsiz; ++k)); do
				arch=${rh8arch[k]}
				link=$rh8link/$stream/$arch/$pkg
				outsubdir=$topdir/rh8/$stream/$arch
				getpkglist $link $outsubdir
			done

		elif [[ $stream == "HighAvailability" ]] ||
		     [[ $stream == "ResilientStorage" ]]; then

			# Loop through arches
			#
			for ((k = 0; k < ($rh8archsiz - 1); ++k)); do
				arch=${rh8arch[k]}
				link=$rh8link/$stream/$arch/$pkg
				outsubdir=$topdir/rh8/$stream/$arch
				getpkglist $link $outsubdir
			done

		elif [[ $stream == "Supplementary" ]]; then

			# Loop through arches
			#
			for ((k = 0; k < $rh8archsiz; ++k)); do
				arch=${rh8arch[k]}
				link=$su8link/$stream/$arch/$pkg
				outsubdir=$topdir/rh8/$stream/$arch
				getpkglist $link $outsubdir
			done
		else
			getpkglist 8 $stream "x86_64" $rh8link
		fi
	done
}

# extract_pkgname
#
# Extracts the package name from the line, which is the second
# field after the colon in each line formatted as the following
# example.
#
#  ../tmp/rh8/Server/pkglist:mpich-3.0
#
#
extract_pkgname() {
	echo "$1" | awk -F ":" '{print $2}'
}

# extract_strmname()
#
# Extracts the name of the stream, e.g. Server, Workstation, from
# a line formatted as the following example.
#
#  ../tmp/rh8/Server/pkglist:mpich-3.0
#             ^^^^^^
#            Stream name
#
# Globals
#   tdindex
#
extract_strmname() {

	# tdindex is 0-based, but the fields in cut are 1-based,
	# ergo the +1.
	#
	echo "$1" | cut -d'/' -f$((tdindex+1))
}

# extract_rhel() - determines whether pkg line is from rhel7 or rhel8
#
# Assumes that pline is formatted as in the following example
#   topdir/rhel7/Server/pkglist:mpich
#
# Globals
#   tdindex - index to the top of the path for the distro
#
# Arguments
#   $1 - pline, path line
#
extract_rhel() {
	local pline="$1"
	local rheldir

	rheldir=$(echo "$pline" | cut -d'/' -f$tdindex)
	echo ${rheldir: -1}
}

# rm_arches() - remove the arch component of the path in each string of pkgstr
#
# Globals
#   tdindex - 0-based index to the stream component, e.g. Workstation, Server,
#             in each string of the string-of-strings for the packages.
#
rm_arches() {
	local pkgstr="$1"	# string of strings generated by grep for the package
	local retstr		# string of strings that will be returned
	local findex=$((tdindex+2))	# index in each path string to the arch

	# The BUILDROOT directory does not have an arch component
	#
	retstr=$(
		while read line; do
			[[ $line == *BUILDROOT* ]] \
				&& { echo "$line "; continue;} \
				|| echo $(echo $line | cut --complement -d'/' -f$findex)
		done <<< "$pkgstr"
		)
	echo "$retstr"
}

# sort_pkgs - sort the string-of-strings for ordered parsing
#
# First sort unique on the stream component, e.g. Server, Workstation, so we
# have only one package per stream.
# Then sort on the package so that they are all grouped for orderly parsing.
#
# Globals
#   tdindex - 0-based index to the stream component of the path to the package
#
sort_pkgs() {
	local pkgstr="$1"	# string-of-strings from the grep pkg
	local tmpstr		# scratch
	local retstr		# returned string-of-strings

	tmpstr=$(echo "$pkgstr" | sort -u -t'/' -k $tdindex)
	retstr=$(echo "$tmpstr" | sort -t':' -k2)

	echo "$retstr"
}

# check_fstate - determine whether a package was not found in one or both rhels
#
# Globals
#   farray - array of strings containing the name/maintainer of the package
#            and/or its hyphenated ancillaries and whether the package was
#            found in one or both or none of the rhel trees.
# Arguments
#   $1 - the fstate registerd by the loop in fill_farray, a two bit truth
#        table that indicates whether and where the package was found.
#
check_fstate() {
	case $1 in
		0 ) farray+=("    RHEL-8 NOT FOUND" "    RHEL-7 NOT FOUND") ;;
		1 ) farray+=("    RHEL-8 NOT FOUND") ;;
		2 ) farray+=("    RHEL-7 NOT FOUND") ;;
	esac
}

# fill_farray - fills an array of hyphenated packages for a given distro
#
# Arguments
#   pkgstr - This is a string generated by grepping for hyphenated packages.
#            It is comprised of strings separated by IFS, one string for
#            each hyphenated package discovered. Additionally there can be
#            more than one genus of hypenated package, e.g. mpich-devel
#            mpich-doc, mpich-autoload, etc.
# Globals
#   tdindex - 0-based index to path of each package on each line of pkgstr
#   farray  - an array intended to contain the strings extrackted from
#             pkgstr to be printed to the screen.
#
fill_farray() {
	local pkgstr="$1"	# sorted package string-of-strings
	local mnt="$2"		# maintainer
	local distro		# the current distro in the loop
	local apkg		# package name extracted from each line
	local prevpkg		# previous package name in the loop
	local pdir		# dir path to the package
	local tstr		# scratch string to be added to array
	local line		# string captured in each loop pass
	local b_done=false	# bool to detect last line
	local fstate=3		# bits: 1 rhel7 only, 2 rhel8 only, 3 both
	local fsos=6		# offset from fstate value to rhel value

	until $b_done; do

		read line || b_done=true

		# Only extract the info if we actually were able to read
		# a line.
		$b_done || {
			pdir=$(extract_strmname "$line") # location of pkg
			apkg=$(extract_pkgname "$line")  # name of pkg
			distro=$(extract_rhel "$line")   # 7 or 8
		}

		# If the package name changes, then we're on to a new
		# package.
		#
		if [[ $apkg != $prevpkg ]]; then

			$b_done || {
				check_fstate $fstate
				tstr=$(printf "\n%-32s%s\n" $apkg $mnt)
				farray+=("$tstr")
			}

			fstate=$((distro - $fsos))
		fi

		$b_done || {
			tstr=$(printf "    RHEL-$distro %s\n" $pdir)
			farray+=("$tstr")
			prevpkg="$apkg"
			prevrhel=$distro
			fstate=$((fstate | (distro-$fsos)))
		}
	done <<< "$pkgstr"

	check_fstate $fstate
}

# normalize_pkgstr() - not interested in hypenated variants of the package.
#
# For example, AppStream/x86_64/pkglist:alsa-plugins-pulseaudio will be reduced
# to AppStream/x86_64/pkglist:alsa-plugins
#
# $1 - the generic package name
# $2 - the string of strings containing all the packages that start with the
#      generic package name.
#
normalize_pkgstr()
{
	local pkg="$1"
	local pkgstr="$2"
	local pkgstr
	local retstr
	local idx

	retstr=$(
		while read line; do
			idx=$(ui_strindex "$line" "$pkg")
			echo "${line:0:idx}""$pkg"
		done <<< "$pkgstr"
		)
	echo "$retstr"
}

# find_pkgs() - use grep to find the packages in the rhel7 and rhel8 trees
#
# Globals
#   topdir - the location of the rhel7 and rhel8 trees
#   farray - the "found" array contains whether and where the pkg was
#            discovered in the rhel7 and rhel8 trees.
#
#  $1 - the generic package name
#  $2 - the maintainer of the package
#
find_pkgs() {
	local pkg="$1"
	local mnt="$2"
	local fee	# grept packages in distros
	local foo	# grept packages normalized to pkg
	local fie	# fee with the arches stripped from the paths
	local foe	# fie sorted unique on paths and then pkg
	local stat	# exit status of commands

	fee=$(grep -r -m1 "^$pkg" $topdir)
	stat=$?
	(($stat != 0)) && {
		tstr=$(printf "\n%-32s%s\n" $pkg $mnt)
		farray+=("$tstr")
		check_fstate 0
		return 1
	}

	foo=$(normalize_pkgstr "$pkg" "$fee")
	fie=$(rm_arches "$foo")		# remove arch component of the path
	foe=$(sort_pkgs "$fie") 	# sort unique on paths and pkg

	fill_farray "$foe" $mnt
	return 0
}

dump_farray() {
	local farysize=${#farray[@]}
	local k

	for ((k = 0; k < farysize; ++k)); do
		echo "${farray[k]}"
	done
}

# parse_pkglist - seek each of the packages in the pkglist
#
# Globals
#   farray	- array of discovered or not found packages
#
parse_pkglist() {
	local pkglist="$1"
	local line
	local lary
	local pkg
	local mnt

	while read line; do
		ui_strtok "$line" ":" lary
		pkg="${lary[0]}"
		mnt="${lary[1]}"
		farray=()
		find_pkgs "$pkg" $mnt	# find generic package name
		dump_farray
	done < $pkglist
}

main() {
        # Trap for control-c
        trap control_c SIGINT

	declare pkglist
	declare args=2

	local td
	local dary
	local darysize

	while getopts abh OPTION; do
    	    case "$OPTION" in
		h ) usage
		    exitme 0
		    ;;
		a ) b_arches=true
		    let ++optcount
		    ;;
		b ) b_buildtrees=true
		    let ++optcount
		    args=2
		    ;;
		* ) echo "unrecognized option -$OPTION"
		    echo -e "$usagestr"
		    exit 127
	    esac
	done

	shift $optcount
	[ $# -eq $args ] || exitme $EXIT_INVARG

	pkglist="$1"
	[ -f $pkglist ] || exitme $EXIT_INVFIL

	topdir="$2"
	[ -d $topdir ]  || exitme $EXIT_INVDIR

	# get the global index to the beginning of the path to the packages
	#
	td=${topdir%/}  # topdir stripped of any trailing slash
	ui_strtok "$td" "/" dary
	darysize=${#dary[@]}
	tdindex=$((darysize+1))	# index to 2nd token past the topdir is the path,
				# : because the 1st token is rh[7|8]

	$b_buildtrees && buildtrees $topdir
	parse_pkglist $pkglist

	exitme $EXIT_OK
}

main $@

exitme $EXIT_OK