loop-unprocessed-testfiles

#!/bin/sh

PREFIX="Loop unprocessed testfiles"
API=${API:-http://validator-api/api/v1}

./update-data.sh

# loop forever
while :
do

  # request a file to be processed

  # Refresh a report after 48 hours (JSON is the last bit updated in the processing
  # so we check for this to select incomplete or obsolete data)
  TDone=`date -d "48 hours ago" --iso-8601=seconds`
  # Restart processing if the last start was more than 1 hour ago
  TProc=`date -d "1 hour ago" --iso-8601=seconds`

  QUERY=`cat <<EOT
{
  "limit": 1, 
  "where": {
  "and": [
    {"or": [
      {"json-updated": {"exists": false}},
      {"svrl-updated": {"exists": false}}
    ]},
    {"or":[
      {"processing": {"exists": false}},
      {"processing": {"lt": "$TProc"}}
    ]}
    ]
  }
}
EOT`
  
  # Get (-G) a dataset with the query
  DATASET=`curl -sS --header 'Accept: application/xml' \
  -G --data-urlencode "filter=$QUERY" \
  "$API/iati-testdatasets"`
  
  FID=`echo $DATASET | xmllint --xpath '//fileid/text()' -` 2> "/dev/null"
  DSID=`echo $DATASET | xmllint --xpath '//id/text()' -` 2> "/dev/null"
  WSID=`echo $DATASET | xmllint --xpath '//tmpworkspaceId/text()' -` 2> "/dev/null"
  
  if [ -z $DSID ]; then
    echo "$PREFIX: Nothing found to process, waiting"
    sleep 10s
  
  else
    echo "$PREFIX: Found $DSID in workspace '$WSID' -- start processing"
    
    NOW=`date --iso-8601=seconds`
    APIDATA="{\"processing\": \"$NOW\", \"status\": \"Processing validation rules (step 2 of 3)\"}"
  
    echo "$PREFIX: update iati-testdatasets $DSID to indicate processing"
    curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
    -d "$APIDATA" \
    "$API/iati-testdatasets/update?where=%7B%22id%22%3A%22$DSID%22%7D"

    echo "$PREFIX: process $DSID in workspace $WSID"
    ./webhook-scripts/inhouse-testfiles.sh $FID
    
    ./email-notification.sh $WSID
  fi
  
done