install-eg.sh: install custom dojo build from upstream

[sitka/sitka-tools.git] / marc_export_custom / mucn-full-export-parallel-ebsco-wrapper.sh

#!/bin/bash
# This is meant to be fire-and-forget
# Only use it if you trust the output!
export DIR=$(readlink -f $(dirname $0))
NAME=ebsco
if [ -z "${MAKEOPTS}" ]; then
	echo "MAKEOPTS env variable is not set"
	echo "If you really want this to be serialized, export MAKEOPTS=-j1"
	echo "Otherwise set it about 1.5x your number of CPUs!"
	exit 1
fi
[ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)
cd $OUTDIR
echo Prep $(date -uR)/@$(date +%s)
$DIR/fetch-shortcodes.sh
cd $DIR
echo Exporting $(date -uR)/@$(date +%s)
export PERLOPT=-X
time ./mucn-${NAME}-full-export-parallel.sh >$OUTDIR/script.log 2>&1
rc=$?
cd $OUTDIR
mkdir split
mv *split* split/
echo Compressing $(date -uR)/@$(date +%s)
pbzip2 -9nk -p4 *{marc,log}
pbzip2 -9n -p4 split/*{marc,log}
echo Stats $(date -uR)/@$(date +%s)
DELETE=0 $DIR/stats.sh >$OUTDIR/stats.log 2>&1
# Look for zeros in the output marc
echo Sanity $(date -uR)/@$(date +%s)
egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
grep_result=$?
# halt if found
if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
	echo "Failed to run export! Human needed! export=$rc grep_sanity=$grep_result" 1>&2
	cat *stats 1>&2
	exit 1
fi
echo FTP $(date -uR)/@$(date +%s)
# Two passes with mget, so the stats and csv are always there first
# they are small and serve to show we might be uploading more, and they have
# checksums so AutoGraphics can check the upload
# EBSCO wants the raw .marc, they do not support any compression
FTPSITE=ftp://cat00975a:neJUJ4ez@ftp.epnet.com/
FTPDIR=full/mucn/$T
cat >lftp.cmd <<EOF
open $FTPSITE
mkdir -p $FTPDIR
cd $FTPDIR
mput *csv *stats
mput *bz2 *marc
rels -l
EOF
lftp -f lftp.cmd
echo Email $(date -uR)/@$(date +%s)
cp $DIR/email-template-${NAME}.sh $OUTDIR/
chmod +x email-template-${NAME}.sh
./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
/usr/lib/sendmail -t -f sysop@bc.libraries.coop <email.txt