xml-check

#!/usr/bin/env bash

# Check the input files for well-formedness and schema errors, produce a feedback file if appropriate

WORKSPACE=${WORKSPACE:-/work/space}
INPUTDIR=${1:-input}
SRCDIR=${2:-src}
FILEMASK=${3}*
DATASETS=${4:-iati-datasets}
PREFIX="XML check"
API=${API:-none}

VALLOGL=100
VALLOGS=50

mkdir -p \
  $WORKSPACE/tmp/xmltest \
  $WORKSPACE/tmp/xmltestlog \
  $WORKSPACE/tmp/xmlschemalog \
  $WORKSPACE/tmp/iatifeedback \
  $WORKSPACE/tmp/feedback \
  $WORKSPACE/input \
  $WORKSPACE/dest

cd $WORKSPACE/$INPUTDIR
find . -name "${FILEMASK}*" -type f -print | while read FILE; do

  NAME=`basename "$FILE"`
  MD5="${NAME%.xml}"
  
  cd $WORKSPACE/$INPUTDIR
  xmllint --recover --output "$WORKSPACE/tmp/xmltest/$NAME" "$NAME" 2> "$WORKSPACE/tmp/xmltestlog/$NAME"
  LL=`cat "$WORKSPACE/tmp/xmltestlog/$NAME" | wc -l`
  if [[ $LL -gt 100 ]]; then
    echo "$PREFIX $MD5: limiting XML validation log of $LL lines to $VALLOGL lines"
    sed -i "2001,$((LL-$VALLOGS))c...skipped $((LL-$VALLOGL)) lines..." "$WORKSPACE/tmp/xmltestlog/$NAME" 
  fi
  
  if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
    mv "$WORKSPACE/tmp/xmltest/$NAME" $WORKSPACE/$SRCDIR
  else
    cp --preserve "$WORKSPACE/$INPUTDIR/$NAME" $WORKSPACE/$SRCDIR
    rm "$WORKSPACE/tmp/xmltest/$NAME"
  fi

  cd $WORKSPACE/$SRCDIR
  XMLROOT=`xmllint --xpath "local-name(/*)" "$NAME" 2> /dev/null`
  XMLATTR=`xmllint --xpath "string(/*/@version)" "$NAME" 2> /dev/null`
  
  if [[ "$XMLROOT." == "iati-activities." || "$XMLROOT." == "iati-organisations." ]]; then
    if [[ $XMLATTR. =~ 1\.0[1-5]\. || $XMLATTR. =~ 2\.0[1-3]\. ]]; then
      VERSION=$XMLATTR
    elif [[ $XMLATTR. =~ 1\..* ]]; then
      VERSION=1.05
    else
      VERSION=2.03
    fi
    
    if xmllint --noout --schema /home/lib/iati-rulesets/lib/schemata/$VERSION/$XMLROOT-schema.xsd "$NAME" 2> "$WORKSPACE/tmp/xmlschemalog/$NAME"; then
      rm "$WORKSPACE/tmp/xmlschemalog/$NAME"
      if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
        echo '<recovered-iati-file/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
        echo "$PREFIX $MD5: recovered-iati-file"
        
        if [[ "$API" != "none" ]]; then
          APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"recovered-iati-file\", \"iati-version\": \"$VERSION\"}"

          curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
            -d "$APIDATA" \
            "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
        fi        
      else
        echo '<iati-file/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
        echo "$PREFIX $MD5: iati-file"
        
        if [[ "$API" != "none" ]]; then
          APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"iati-file\", \"iati-version\": \"$VERSION\"}"

          curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
            -d "$APIDATA" \
            "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
        fi        
      fi
    else
      LL=`cat "$WORKSPACE/tmp/xmlschemalog/$NAME" | wc -l`
      if [[ $LL -gt 4000 ]]; then
        echo "$PREFIX $MD5: limiting schema validation log of $LL lines to $VALLOGL lines"
        sed -i "2001,$((LL-$VALLOGS))c...skipped $((LL-$VALLOGL)) lines..." "$WORKSPACE/tmp/xmlschemalog/$NAME" 
      fi

      if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
        echo '<recovered-iati-file-with-schema-errors/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
        echo "$PREFIX $MD5: recovered-iati-file-with-schema-errors"

        if [[ "$API" != "none" ]]; then
          APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"recovered-iati-file-with-schema-errors\", \"iati-version\": \"$VERSION\"}"

          curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
            -d "$APIDATA" \
            "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
        fi        
      else
        echo '<iati-file-with-schema-errors/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
        echo "$PREFIX $MD5: iati-file-with-schema-errors"

        if [[ "$API" != "none" ]]; then
          APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"iati-file-with-schema-errors\", \"iati-version\": \"$VERSION\"}"

          curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
            -d "$APIDATA" \
            "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
        fi
      fi       
    fi
    
  
  elif [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
     
    echo '<not-an-xml-file/>' > "$WORKSPACE/tmp/feedback/$NAME"
    echo "$PREFIX $MD5: not-an-xml-file"
    rm "$WORKSPACE/src/$NAME"
    
    if [[ "$API" != "none" ]]; then
      APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"not-an-xml-file\"}"

      curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
        -d "$APIDATA" \
        "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
    fi    
  else
    echo '<not-an-iati-file/>' > "$WORKSPACE/tmp/feedback/$NAME"
    echo "$PREFIX $MD5: not-an-iati-file"
    rm "$WORKSPACE/src/$NAME"

    if [[ "$API" != "none" ]]; then
      APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"not-an-iati-file\"}"

      curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
        -d "$APIDATA" \
        "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
    fi
  fi 

done