Document scripts better, and squelch output.

Signed-off-by: Robin H. Johnson <rjohnson@sitka.bclibraries.ca>

diff --git a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
index 2d6f4a3..f7250b2 100755 (executable)
--- a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
+++ b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
@@ -11,16 +11,33 @@ fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ebsco_export.$T.XXXXXXX)
 cd $OUTDIR
+echo Prep $(date -uR)/@($date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR
+echo Exporting $(date -uR)/@($date +%s)
 time ./mucn-ebsco-full-export-parallel.sh
 cd $OUTDIR
 mkdir split
 mv *split* split/
-pbzip2 -9nvk -p4 *{marc,log}
-pbzip2 -9nv -p4 split/*{marc,log}
+echo Compressing $(date -uR)/@($date +%s)
+pbzip2 -9nk -p4 *{marc,log}
+pbzip2 -9n -p4 split/*{marc,log}
+echo Stats $(date -uR)/@($date +%s)
 $DIR/stats.sh
-cp $DIR/email-template-ebsco.sh $OUTDIR/
+# Look for zeros in the output marc
+echo Sanity $(date -uR)/@($date +%s)
+egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
+grep_result=$?
+# halt if found
+if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
+       echo "Failed to run export! Human needed! export=$rc grep_sanity=$grep_result" 1>&2
+       cat *stats 1>&2
+       exit 1
+fi
+echo FTP $(date -uR)/@($date +%s)
+# Two passes with mget, so the stats and csv are always there first
+# they are small and serve to show we might be uploading more, and they have
+# checksums so AutoGraphics can check the upload
 cat >lftp.cmd <<EOF
 open ftp://cat00975a:neJUJ4ez@ftp.epnet.com/
 mkdir -p mucn/$T
@@ -30,6 +47,8 @@ mput *bz2
 rels -l
 EOF
 lftp -f lftp.cmd
+echo Email $(date -uR)/@($date +%s)
+cp $DIR/email-template-ebsco.sh $OUTDIR/
 chmod +x email-template-ebsco.sh
 ./email-template-ebsco.sh "$T" "@$(date +%s)" >email.txt
 /usr/lib/sendmail -f rjohnson@sitka.bclibraries.ca <email.txt

diff --git a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
index f26cbb4..beb2848 100755 (executable)
--- a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
+++ b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
@@ -11,17 +11,22 @@ fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d outlook_export.$T.XXXXXXX)
 cd $OUTDIR
+echo Prep $(date -uR)/@($date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR
 export PERLOPT=-X
+echo Exporting $(date -uR)/@($date +%s)
 time ./sitka-full-export-parallel.sh
 rc=$?
 cd $OUTDIR
 mkdir split
 mv *split* split/
-pbzip2 -9nvk -p4 *{marc,log}
-pbzip2 -9nv -p4 split/*{marc,log}
+echo Compressing $(date -uR)/@($date +%s)
+pbzip2 -9nk -p4 *{marc,log}
+pbzip2 -9n -p4 split/*{marc,log}
+echo Stats $(date -uR)/@($date +%s)
 $DIR/stats.sh
+echo Sanity $(date -uR)/@($date +%s)
 # Look for zeros in the output marc
 egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
 grep_result=$?
@@ -31,7 +36,10 @@ if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
        cat *stats 1>&2
        exit 1
 fi
-cp $DIR/email-template-outlook.sh $OUTDIR/
+echo FTP $(date -uR)/@($date +%s)
+# Two passes with mget, so the stats and csv are always there first
+# they are small and serve to show we might be uploading more, and they have
+# checksums so AutoGraphics can check the upload
 cat >lftp.cmd <<EOF
 open ftp://ftpbcuc:bcv15c@ftp.auto-graphics.com/
 mkdir -p sitka/$T
@@ -41,6 +49,8 @@ mput *bz2
 rels -l
 EOF
 lftp -f lftp.cmd
+echo Email $(date -uR)/@($date +%s)
+cp $DIR/email-template-outlook.sh $OUTDIR/
 chmod +x email-template-outlook.sh
 ./email-template-outlook.sh "$T" "@$(date +%s)" >email.txt
 /usr/lib/sendmail -f rjohnson@sitka.bclibraries.ca <email.txt

diff --git a/marc_export_custom/stats.sh b/marc_export_custom/stats.sh
index add09b5..32018c4 100755 (executable)
--- a/marc_export_custom/stats.sh
+++ b/marc_export_custom/stats.sh
@@ -10,7 +10,7 @@ for i in $files; do
                bunzip2 -k $j
        fi
        if [ -f $i -a ! -f $j ]; then
-               pbzip2 -9kv -p4 $i
+               pbzip2 -9k -p4 $i
        fi
        if [ ! -f $i -o ! -f $j ]; then
                echo "Missing $i or $j"