use dedicated partition for MARC exports
[sitka/sitka-tools.git] / marc_export_custom / mucn-full-export-parallel-ebsco-wrapper.sh
CommitLineData
e11c7cc6
RJ
1#!/bin/bash
2# This is meant to be fire-and-forget
3# Only use it if you trust the output!
489af5f5
JD
4
5EXPORTDIR="/marcexport"
6
e11c7cc6 7export DIR=$(readlink -f $(dirname $0))
d10c96dc 8NAME=ebsco
e11c7cc6
RJ
9if [ -z "${MAKEOPTS}" ]; then
10 echo "MAKEOPTS env variable is not set"
11 echo "If you really want this to be serialized, export MAKEOPTS=-j1"
12 echo "Otherwise set it about 1.5x your number of CPUs!"
13 exit 1
14fi
15[ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
489af5f5 16[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir=${EXPORTDIR} -d ${NAME}_export.$T.XXXXXXX)
e11c7cc6 17cd $OUTDIR
5e85b8ab 18echo Prep $(date -uR)/@$(date +%s)
ec4760e5 19$DIR/fetch-shortcodes.sh shortcodes.csv
e11c7cc6 20cd $DIR
5e85b8ab 21echo Exporting $(date -uR)/@$(date +%s)
e270fa90 22export PERLOPT=-X
ff03d26c 23time ./mucn-${NAME}-full-export-parallel.sh >$OUTDIR/script.log 2>&1
d10c96dc 24rc=$?
e11c7cc6
RJ
25cd $OUTDIR
26mkdir split
3f5a4340 27mv *-split* split/
5e85b8ab 28echo Compressing $(date -uR)/@$(date +%s)
2ec1922a
RJ
29pbzip2 -9nk -p4 *{marc,log}
30pbzip2 -9n -p4 split/*{marc,log}
5e85b8ab 31echo Stats $(date -uR)/@$(date +%s)
ff03d26c 32DELETE=0 $DIR/stats.sh >$OUTDIR/stats.log 2>&1
2ec1922a 33# Look for zeros in the output marc
5e85b8ab 34echo Sanity $(date -uR)/@$(date +%s)
2ec1922a
RJ
35egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
36grep_result=$?
37# halt if found
38if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
39 echo "Failed to run export! Human needed! export=$rc grep_sanity=$grep_result" 1>&2
40 cat *stats 1>&2
41 exit 1
42fi
5e85b8ab 43echo FTP $(date -uR)/@$(date +%s)
2ec1922a
RJ
44# Two passes with mget, so the stats and csv are always there first
45# they are small and serve to show we might be uploading more, and they have
46# checksums so AutoGraphics can check the upload
d10c96dc
RJ
47# EBSCO wants the raw .marc, they do not support any compression
48FTPSITE=ftp://cat00975a:neJUJ4ez@ftp.epnet.com/
8dad87fc 49FTPDIR=full/mucn/$T
e11c7cc6 50cat >lftp.cmd <<EOF
d10c96dc 51open $FTPSITE
8dad87fc
RJ
52mkdir -p $FTPDIR
53cd $FTPDIR
e11c7cc6 54mput *csv *stats
8dad87fc 55mput *bz2 *marc
e11c7cc6
RJ
56rels -l
57EOF
58lftp -f lftp.cmd
5e85b8ab 59echo Email $(date -uR)/@$(date +%s)
d10c96dc
RJ
60cp $DIR/email-template-${NAME}.sh $OUTDIR/
61chmod +x email-template-${NAME}.sh
62./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
0e3cccc0 63/usr/lib/sendmail -t -f sitka-tier2@bc.libraries.coop <email.txt
6e15b2f5 64
65# log successful export and cleanup old exports, saving this one and the previous one
66echo "$OUTDIR" >> $DIR/mucn_ebsco.log
67head -n -2 $DIR/mucn_ebsco.log | while read OLDEXPORT ; do
68 [ -d "${OLDEXPORT}" ] && rm -rf "${OLDEXPORT}"
69done