use dedicated partition for MARC exports
[sitka/sitka-tools.git] / marc_export_custom / sitka-full-export-parallel-outlook-wrapper.sh
index b5a5dc5..1271b34 100755 (executable)
@@ -1,7 +1,14 @@
 #!/bin/bash
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
+
+CURL_USERNAME=$1
+CURL_PASSWORD=$2
+
+EXPORTDIR="/marcexport"
+
 export DIR=$(readlink -f $(dirname $0))
+NAME=outlook
 if [ -z "${MAKEOPTS}" ]; then
        echo "MAKEOPTS env variable is not set"
        echo "If you really want this to be serialized, export MAKEOPTS=-j1"
@@ -9,27 +16,53 @@ if [ -z "${MAKEOPTS}" ]; then
        exit 1
 fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
-[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d outlook_export.$T.XXXXXXX)
+[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir=${EXPORTDIR} -d ${NAME}_export.$T.XXXXXXX)
 cd $OUTDIR
-$DIR/fetch-shortcodes.sh
+echo Prep $(date -uR)/@$(date +%s)
+$DIR/fetch-shortcodes.sh shortcodes-$T.csv
 cd $DIR
-time ./sitka-full-export-parallel.sh
+echo Exporting $(date -uR)/@$(date +%s)
+export PERLOPT=-X
+time ./sitka-full-export-parallel.sh >$OUTDIR/script.log 2>&1
+rc=$?
 cd $OUTDIR
 mkdir split
-mv *split* split/
-pbzip2 -9nvk -p4 *{marc,log}
-pbzip2 -9nv -p4 split/*{marc,log}
-$DIR/stats.sh
-cp $DIR/email-template-outlook.sh $OUTDIR/
-cat >lftp.cmd <<EOF
-open ftp://ftpbcuc:bcv15c@ftp.auto-graphics.com/
-mkdir -p sitka/$T
-cd sitka/$T
-mput *csv *stats
-mput *bz2
-rels -l
-EOF
-lftp -f lftp.cmd
-chmod +x email-template-outlook.sh
-./email-template-outlook.sh "$T" "@$(date +%s)" >email.txt
-/usr/lib/sendmail -f rjohnson@sitka.bclibraries.ca <email.txt
+mv *-split* split/
+echo Compressing $(date -uR)/@$(date +%s)
+pbzip2 -9nk -p4 *{marc,log}
+pbzip2 -9n -p4 split/*{marc,log}
+echo Stats $(date -uR)/@$(date +%s)
+DELETE=1 $DIR/stats.sh >$OUTDIR/stats.log 2>&1
+# Look for zeros in the output marc
+echo Sanity $(date -uR)/@$(date +%s)
+egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
+grep_result=$?
+# halt if found
+if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
+       echo "Failed to run export! Human needed! export=$rc grep_sanity=$grep_result" 1>&2
+       cat *stats 1>&2
+       exit 1
+fi
+
+# upload fails if curl username/password not provided
+if [ -z "${CURL_PASSWORD}" ]; then
+    echo "Missing curl auth parameter(s), cannot upload to Outlook" 1>&2
+    exit 1
+fi
+
+# individually upload all required output files with curl
+for CURL_FILE in `ls $OUTDIR | egrep '(.bz2|.stats|.csv)$' | grep -v script.log.bz2` ; do
+    $DIR/outlook-curl.sh $CURL_FILE $CURL_USERNAME $CURL_PASSWORD
+    echo $CURL_FILE >> $OUTDIR/Manifest
+done
+
+cp $DIR/email-template-${NAME}.sh $OUTDIR/
+chmod +x email-template-${NAME}.sh
+./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
+/usr/lib/sendmail -t -f sitka-tier2@bc.libraries.coop <email.txt
+
+# log successful export and cleanup old exports, saving this one and the previous one
+echo "$OUTDIR" >> $DIR/sitka_full_outlook.log
+head -n -2 $DIR/sitka_full_outlook.log | while read OLDEXPORT ; do
+  [ -d "${OLDEXPORT}" ] && rm -rf "${OLDEXPORT}"
+done