Refactor export scripts and take my email address off them.

Signed-off-by: Robin H. Johnson <rjohnson@sitka.bclibraries.ca>

diff --git a/marc_export_custom/email-template-ebsco.sh b/marc_export_custom/email-template-ebsco.sh
index 3956814..abde63e 100755 (executable)
--- a/marc_export_custom/email-template-ebsco.sh
+++ b/marc_export_custom/email-template-ebsco.sh
@@ -12,10 +12,10 @@ fi
 
 cat <<EOF
 Date: $(date -uR)
 
 cat <<EOF
 Date: $(date -uR)

-From: "Robin H. Johnson" <rjohnson@sitka.bclibraries.ca>
+From: "SITKA sysop" <sysop@bc.libraries.coop>

 Subject: UCN EBSCO export $(date +%Y/%m/%d -d "$t1") notification
 To: "Daniel Mertens" <danielmertens@ebsco.com>, "Kathy Ellerton" <kellerton@ucn.ca>, "Jessica Carmody" <jcarmody@EBSCO.COM>
 Subject: UCN EBSCO export $(date +%Y/%m/%d -d "$t1") notification
 To: "Daniel Mertens" <danielmertens@ebsco.com>, "Kathy Ellerton" <kellerton@ucn.ca>, "Jessica Carmody" <jcarmody@EBSCO.COM>

-Cc: "Robin H. Johnson" <rjohnson@sitka.bclibraries.ca>, "Sharon Herbert" <sherbert@sitka.bclibraries.ca>
+Cc: "SITKA sysop" <sysop@bc.libraries.coop>, "Sharon Herbert" <sherbert@sitka.bclibraries.ca>

 
 Full automated UCN EBSCO export
 
 
 Full automated UCN EBSCO export
 

diff --git a/marc_export_custom/email-template-outlook.sh b/marc_export_custom/email-template-outlook.sh
index 418f35f..fd3d5f6 100755 (executable)
--- a/marc_export_custom/email-template-outlook.sh
+++ b/marc_export_custom/email-template-outlook.sh
@@ -13,10 +13,10 @@ fi
 # illsupport@bclibrary.ca goes to Mari.Martin@gov.bc.ca and Bob.Yuan@gov.bc.ca
 cat <<EOF
 Date: $(date -uR)
 # illsupport@bclibrary.ca goes to Mari.Martin@gov.bc.ca and Bob.Yuan@gov.bc.ca
 cat <<EOF
 Date: $(date -uR)

-From: "Robin H. Johnson" <rjohnson@sitka.bclibraries.ca>
+From: "SITKA sysop" <sysop@bc.libraries.coop>

 Subject: SITKA Outlook export $(date +%Y/%m/%d -d "$t1") (includes ELN media/serials)
 To: "BC Public Libraries" <illsupport@bclibrary.ca>, "Sunni Nishimura" <sunnin@eln.bc.ca>, "Ramiro Castillo" <rcr@auto-graphics.com>
 Subject: SITKA Outlook export $(date +%Y/%m/%d -d "$t1") (includes ELN media/serials)
 To: "BC Public Libraries" <illsupport@bclibrary.ca>, "Sunni Nishimura" <sunnin@eln.bc.ca>, "Ramiro Castillo" <rcr@auto-graphics.com>

-Cc: "Robin H. Johnson" <rjohnson@sitka.bclibraries.ca>, "Sharon Herbert" <sherbert@sitka.bclibraries.ca>
+Cc: "SITKA sysop" <sysop@bc.libraries.coop>, "Sharon Herbert" <sherbert@sitka.bclibraries.ca>

 
 Full automated Outlook export
 
 
 Full automated Outlook export
 

diff --git a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
index 01789f8..5c0a9d6 100755 (executable)
--- a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
+++ b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
@@ -2,6 +2,7 @@
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
 export DIR=$(readlink -f $(dirname $0))
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
 export DIR=$(readlink -f $(dirname $0))

+NAME=ebsco

 if [ -z "${MAKEOPTS}" ]; then
        echo "MAKEOPTS env variable is not set"
        echo "If you really want this to be serialized, export MAKEOPTS=-j1"
 if [ -z "${MAKEOPTS}" ]; then
        echo "MAKEOPTS env variable is not set"
        echo "If you really want this to be serialized, export MAKEOPTS=-j1"


@@ -9,14 +10,15 @@ if [ -z "${MAKEOPTS}" ]; then
        exit 1
 fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
        exit 1
 fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)

-[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ebsco_export.$T.XXXXXXX)
+[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)

 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X

-time ./mucn-ebsco-full-export-parallel.sh
+time ./mucn-${NAME}-full-export-parallel.sh >script.log 2>&1
+rc=$?

 cd $OUTDIR
 mkdir split
 mv *split* split/
 cd $OUTDIR
 mkdir split
 mv *split* split/


@@ -24,7 +26,7 @@ echo Compressing $(date -uR)/@$(date +%s)
 pbzip2 -9nk -p4 *{marc,log}
 pbzip2 -9n -p4 split/*{marc,log}
 echo Stats $(date -uR)/@$(date +%s)
 pbzip2 -9nk -p4 *{marc,log}
 pbzip2 -9n -p4 split/*{marc,log}
 echo Stats $(date -uR)/@$(date +%s)

-DELETE=0 $DIR/stats.sh
+DELETE=0 $DIR/stats.sh >stats.log 2>&1

 # Look for zeros in the output marc
 echo Sanity $(date -uR)/@$(date +%s)
 egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
 # Look for zeros in the output marc
 echo Sanity $(date -uR)/@$(date +%s)
 egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats


@@ -39,9 +41,11 @@ echo FTP $(date -uR)/@$(date +%s)
 # Two passes with mget, so the stats and csv are always there first
 # they are small and serve to show we might be uploading more, and they have
 # checksums so AutoGraphics can check the upload
 # Two passes with mget, so the stats and csv are always there first
 # they are small and serve to show we might be uploading more, and they have
 # checksums so AutoGraphics can check the upload

+# EBSCO wants the raw .marc, they do not support any compression
+FTPSITE=ftp://cat00975a:neJUJ4ez@ftp.epnet.com/

 FTPDIR=full/mucn/$T
 cat >lftp.cmd <<EOF
 FTPDIR=full/mucn/$T
 cat >lftp.cmd <<EOF

-open ftp://cat00975a:neJUJ4ez@ftp.epnet.com/
+open $FTPSITE

 mkdir -p $FTPDIR
 cd $FTPDIR
 mput *csv *stats
 mkdir -p $FTPDIR
 cd $FTPDIR
 mput *csv *stats


@@ -50,7 +54,7 @@ rels -l
 EOF
 lftp -f lftp.cmd
 echo Email $(date -uR)/@$(date +%s)
 EOF
 lftp -f lftp.cmd
 echo Email $(date -uR)/@$(date +%s)

-cp $DIR/email-template-ebsco.sh $OUTDIR/
-chmod +x email-template-ebsco.sh
-./email-template-ebsco.sh "$T" "@$(date +%s)" >email.txt
-/usr/lib/sendmail -t -f rjohnson@sitka.bclibraries.ca <email.txt
+cp $DIR/email-template-${NAME}.sh $OUTDIR/
+chmod +x email-template-${NAME}.sh
+./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
+/usr/lib/sendmail -t -f sysop@bc.libraries.coop <email.txt

diff --git a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
index 4676512..363c14b 100755 (executable)
--- a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
+++ b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
@@ -2,6 +2,7 @@
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
 export DIR=$(readlink -f $(dirname $0))
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
 export DIR=$(readlink -f $(dirname $0))

+NAME=outlook

 if [ -z "${MAKEOPTS}" ]; then
        echo "MAKEOPTS env variable is not set"
        echo "If you really want this to be serialized, export MAKEOPTS=-j1"
 if [ -z "${MAKEOPTS}" ]; then
        echo "MAKEOPTS env variable is not set"
        echo "If you really want this to be serialized, export MAKEOPTS=-j1"


@@ -9,14 +10,14 @@ if [ -z "${MAKEOPTS}" ]; then
        exit 1
 fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)
        exit 1
 fi
 [ -z "${T}" ] && export T=$(date -u +%Y%m%dT%H%M%SZ)

-[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d outlook_export.$T.XXXXXXX)
+[ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)

 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 $DIR/fetch-shortcodes.sh
 cd $DIR

-export PERLOPT=-X

 echo Exporting $(date -uR)/@$(date +%s)
 echo Exporting $(date -uR)/@$(date +%s)

-time ./sitka-full-export-parallel.sh
+export PERLOPT=-X
+time ./sitka-full-export-parallel.sh >script.log 2>&1

 rc=$?
 cd $OUTDIR
 mkdir split
 rc=$?
 cd $OUTDIR
 mkdir split


@@ -25,9 +26,9 @@ echo Compressing $(date -uR)/@$(date +%s)
 pbzip2 -9nk -p4 *{marc,log}
 pbzip2 -9n -p4 split/*{marc,log}
 echo Stats $(date -uR)/@$(date +%s)
 pbzip2 -9nk -p4 *{marc,log}
 pbzip2 -9n -p4 split/*{marc,log}
 echo Stats $(date -uR)/@$(date +%s)

-$DIR/stats.sh
-echo Sanity $(date -uR)/@$(date +%s)
+DELETE=1 $DIR/stats.sh >stats.log 2>&1

 # Look for zeros in the output marc
 # Look for zeros in the output marc

+echo Sanity $(date -uR)/@$(date +%s)

 egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
 grep_result=$?
 # halt if found
 egrep -vsq '^([[:space:]]*0){2}.*marc$' *stats
 grep_result=$?
 # halt if found


@@ -40,17 +41,19 @@ echo FTP $(date -uR)/@$(date +%s)
 # Two passes with mget, so the stats and csv are always there first
 # they are small and serve to show we might be uploading more, and they have
 # checksums so AutoGraphics can check the upload
 # Two passes with mget, so the stats and csv are always there first
 # they are small and serve to show we might be uploading more, and they have
 # checksums so AutoGraphics can check the upload

+FTPSITE=ftp://ftpbcuc:bcv15c@ftp.auto-graphics.com/
+FTPDIR=sitka/$T

 cat >lftp.cmd <<EOF
 cat >lftp.cmd <<EOF

-open ftp://ftpbcuc:bcv15c@ftp.auto-graphics.com/
-mkdir -p sitka/$T
-cd sitka/$T
+open $FTPSITE
+mkdir -p $FTPDIR
+cd $FTPDIR

 mput *csv *stats
 mput *bz2
 rels -l
 EOF
 lftp -f lftp.cmd
 echo Email $(date -uR)/@$(date +%s)
 mput *csv *stats
 mput *bz2
 rels -l
 EOF
 lftp -f lftp.cmd
 echo Email $(date -uR)/@$(date +%s)

-cp $DIR/email-template-outlook.sh $OUTDIR/
-chmod +x email-template-outlook.sh
-./email-template-outlook.sh "$T" "@$(date +%s)" >email.txt
-/usr/lib/sendmail -t -f rjohnson@sitka.bclibraries.ca <email.txt
+cp $DIR/email-template-${NAME}.sh $OUTDIR/
+chmod +x email-template-${NAME}.sh
+./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
+/usr/lib/sendmail -t -f sysop@bc.libraries.coop <email.txt