Update MARC export process for Outlook, per RT#42708

Signed-off-by: Jeff Davis <jdavis@sitka.bclibraries.ca>

diff --git a/marc_export_custom/email-template-outlook.sh b/marc_export_custom/email-template-outlook.sh
index 9107550..a214f69 100755 (executable)
--- a/marc_export_custom/email-template-outlook.sh
+++ b/marc_export_custom/email-template-outlook.sh
@@ -10,6 +10,8 @@ if [ -z "$t1" -o -z "$t2" ]; then
        echo "Usage: $0 'Generate timestamp' 'Submit timestamp'" 1>&2
 fi
 
        echo "Usage: $0 'Generate timestamp' 'Submit timestamp'" 1>&2
 fi
 

+FILELIST=`cat Manifest`
+

 # illsupport@bclibrary.ca goes to Mari.Martin@gov.bc.ca and Bob.Yuan@gov.bc.ca
 cat <<EOF
 Date: $(date -uR)
 # illsupport@bclibrary.ca goes to Mari.Martin@gov.bc.ca and Bob.Yuan@gov.bc.ca
 cat <<EOF
 Date: $(date -uR)


@@ -26,7 +28,7 @@ Notes:
 (none, this was automated)
 
 Directory for uploads:
 (none, this was automated)
 
 Directory for uploads:

-sitka/$T
+sitka/

 
 Submission timestamp:
 $(date -uR -d "$t2")
 
 Submission timestamp:
 $(date -uR -d "$t2")


@@ -35,7 +37,7 @@ Generation timestamp:
 $(date -uR -d "$t1")
 
 Submission method:
 $(date -uR -d "$t1")
 
 Submission method:

-FTP
+WebDAV

 
 Compressed sizes in bytes:
 $(head -n2 *stats | awk '/.marc.bz2$/{print $9,$5}')
 
 Compressed sizes in bytes:
 $(head -n2 *stats | awk '/.marc.bz2$/{print $9,$5}')


@@ -47,6 +49,9 @@ $(find -name '*.csv' -printf '%f %s\n')
 Number of records submitted:
 $(grep -h Filename -A5 $i *stats | awk '/.marc$/{print $3,$1}')
 
 Number of records submitted:
 $(grep -h Filename -A5 $i *stats | awk '/.marc$/{print $3,$1}')
 

+Files uploaded:
+$FILELIST
+

 SHA1 checksums:
 $(egrep '^[a-f0-9]{32}' *stats -h |sort -k +2)
 $(sha1sum *.csv)
 SHA1 checksums:
 $(egrep '^[a-f0-9]{32}' *stats -h |sort -k +2)
 $(sha1sum *.csv)

diff --git a/marc_export_custom/fetch-shortcodes.sh b/marc_export_custom/fetch-shortcodes.sh
index a9c45da..48380da 100755 (executable)
--- a/marc_export_custom/fetch-shortcodes.sh
+++ b/marc_export_custom/fetch-shortcodes.sh
@@ -1,5 +1,9 @@
 #!/bin/bash 
 #!/bin/bash 

-psql -Upostgres -U evergreen evergreen -h db1.sitka.bclibraries.ca -o shortcodes.csv \
+SHORTCODES=$1
+if [ -z "${SHORTCODES}" ]; then
+    SHORTCODES="shortcodes.csv"
+fi
+psql -Upostgres -U evergreen evergreen -h prod-pg94-db2-sitka.sitka.bclibraries.ca -o $SHORTCODES \

 -c "COPY (
       SELECT
         a.shortname,a.name,b.shortname AS parent
 -c "COPY (
       SELECT
         a.shortname,a.name,b.shortname AS parent

diff --git a/marc_export_custom/marc_export_custom b/marc_export_custom/marc_export_custom
index bf9273a..5f2d80e 100755 (executable)
--- a/marc_export_custom/marc_export_custom
+++ b/marc_export_custom/marc_export_custom
@@ -272,13 +272,13 @@ sub export_record {
                 # Add new 852 fields 
                 add_bib_holdings($bib, $r);
                 # Check that at least one 852 was added
                 # Add new 852 fields 
                 add_bib_holdings($bib, $r);
                 # Check that at least one 852 was added

-                @marc852 = $r->field('852');
-                @marc856 = $r->field('856');
+                my @marc852 = $r->field('852');
+                my @marc856 = $r->field('856');

                 # If not, we should NOT add this item to the export 
                 # ... but we may still want the record if it has an 856
                 return unless ( @marc852 || ($with_ebooks && @marc856) );
             } else {
                 # If not, we should NOT add this item to the export 
                 # ... but we may still want the record if it has an 856
                 return unless ( @marc852 || ($with_ebooks && @marc856) );
             } else {

-                add_bib_holdings($bib, $r);
+                add_bib_holdings($bib, $r) if ($holdings);

             }
         }
 
             }
         }
 

diff --git a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
index 5d9601d..4bb731d 100755 (executable)
--- a/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
+++ b/marc_export_custom/mucn-full-export-parallel-ebsco-wrapper.sh
@@ -13,7 +13,7 @@ fi
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)

-$DIR/fetch-shortcodes.sh
+$DIR/fetch-shortcodes.sh shortcodes.csv

 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X
 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X

diff --git a/marc_export_custom/outlook-curl.sh b/marc_export_custom/outlook-curl.sh
new file mode 100755 (executable)
index 0000000..679a30c
--- /dev/null
+++ b/marc_export_custom/outlook-curl.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Upload a file to Auto-Graphics ShareFile system using WebDAV/curl
+# Usage: ./outlook-curl.sh <file> <username> <password>
+FILE=$1
+USERNAME=$2
+PASSWORD=$3
+LOG="outlook-curl.log"
+curl -k -u $USERNAME:$PASSWORD -T $FILE https://agcloud.auto-graphics.com/public.php/webdav/sitka/$FILE 2> $LOG

diff --git a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
index 362a8e5..b66c4f8 100755 (executable)
--- a/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
+++ b/marc_export_custom/sitka-full-export-parallel-outlook-wrapper.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!
 #!/bin/bash
 # This is meant to be fire-and-forget
 # Only use it if you trust the output!

+
+CURL_USERNAME=$1
+CURL_PASSWORD=$2
+

 export DIR=$(readlink -f $(dirname $0))
 NAME=outlook
 if [ -z "${MAKEOPTS}" ]; then
 export DIR=$(readlink -f $(dirname $0))
 NAME=outlook
 if [ -z "${MAKEOPTS}" ]; then


@@ -13,7 +17,7 @@ fi
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)
 [ -z "${OUTDIR}" ] && export OUTDIR=$(mktemp --tmpdir -d ${NAME}_export.$T.XXXXXXX)
 cd $OUTDIR
 echo Prep $(date -uR)/@$(date +%s)

-$DIR/fetch-shortcodes.sh
+$DIR/fetch-shortcodes.sh shortcodes-$T.csv

 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X
 cd $DIR
 echo Exporting $(date -uR)/@$(date +%s)
 export PERLOPT=-X


@@ -37,22 +41,19 @@ if [ $rc -ne 0 -o $grep_result -ne 0 ]; then
        cat *stats 1>&2
        exit 1
 fi
        cat *stats 1>&2
        exit 1
 fi

-echo FTP $(date -uR)/@$(date +%s)
-# Two passes with mget, so the stats and csv are always there first
-# they are small and serve to show we might be uploading more, and they have
-# checksums so AutoGraphics can check the upload
-FTPSITE=ftp://ftpbcuc:bcv15c@ftp.auto-graphics.com/
-FTPDIR=sitka/$T
-cat >lftp.cmd <<EOF
-open $FTPSITE
-mkdir -p $FTPDIR
-cd $FTPDIR
-mput *csv *stats
-mput *bz2
-rels -l
-EOF
-lftp -f lftp.cmd
-echo Email $(date -uR)/@$(date +%s)
+
+# upload fails if curl username/password not provided
+if [ -z "${CURL_PASSWORD}" ]; then
+    echo "Missing curl auth parameter(s), cannot upload to Outlook" 1>&2
+    exit 1
+fi
+
+# individually upload all required output files with curl
+for CURL_FILE in `ls $OUTDIR | egrep '(.bz2|.stats|.csv)$' | grep -v script.log.bz2` ; do
+    $DIR/outlook-curl.sh $CURL_FILE $CURL_USERNAME $CURL_PASSWORD
+    echo $CURL_FILE >> $OUTDIR/Manifest
+done
+

 cp $DIR/email-template-${NAME}.sh $OUTDIR/
 chmod +x email-template-${NAME}.sh
 ./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt
 cp $DIR/email-template-${NAME}.sh $OUTDIR/
 chmod +x email-template-${NAME}.sh
 ./email-template-${NAME}.sh "$T" "@$(date +%s)" >email.txt