Merge branch 'master' of git://git.sitka.bclibraries.ca/sitka/sitka-tools
authorroot <root@app3-2.sitka.bclibraries.ca>
Fri, 7 Oct 2011 20:58:41 +0000 (13:58 -0700)
committerroot <root@app3-2.sitka.bclibraries.ca>
Fri, 7 Oct 2011 20:58:41 +0000 (13:58 -0700)
marc_export_custom/Makefile-marc_export [new file with mode: 0644]
marc_export_custom/marc_export_custom
marc_export_custom/sitka-export-no-cotr.sh [changed mode: 0644->0755]
marc_export_custom/sitka-full-export-parallel.sh [new file with mode: 0755]
marc_export_custom/sitka-full-export.sh [changed mode: 0644->0755]

diff --git a/marc_export_custom/Makefile-marc_export b/marc_export_custom/Makefile-marc_export
new file mode 100644 (file)
index 0000000..01ae1f1
--- /dev/null
@@ -0,0 +1,4 @@
+MARC_EXPORT_BIN=./marc_export_custom
+%.marc: %.id
+       $(MARC_EXPORT_BIN) $(MARC_PARAMS) <$< --output-file $@.tmp 2>&1 | tee $@.log \
+       && mv $@.tmp $@
index d6993c8..ed05f99 100755 (executable)
@@ -49,7 +49,10 @@ GetOptions(
        'output-file=s' => \$output_file,
 );
 
-$cfg = new Config::Simple($exclusion_ini) if ($exclusion_ini);
+if ($exclusion_ini) {
+       die "exclusion ini file does not exist" unless (-r $exclusion_ini and -s $exclusion_ini);
+       $cfg = new Config::Simple($exclusion_ini) 
+}
 
 if ($help) {
 print <<"HELP";
@@ -588,5 +591,9 @@ sub add_bib_holdings {
 } # sub
 
 sub skipnote { 
-       printf(STDOUT "Skipped %s due to config: %s\n",$1,$2); 
+       my $id = shift;
+       my $note = shift;
+       my $outf = *STDERR;
+       $outf = *STDOUT if($output_file) ;
+       printf($outf "Skipped %s due to config: %s\n",$id,$note); 
 }
old mode 100644 (file)
new mode 100755 (executable)
diff --git a/marc_export_custom/sitka-full-export-parallel.sh b/marc_export_custom/sitka-full-export-parallel.sh
new file mode 100755 (executable)
index 0000000..cd89c8c
--- /dev/null
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+DIR=`dirname $0`
+#cd /srv/openils/bin
+
+PGUSER=evergreen
+PGHOST=db1.sitka.bclibraries.ca
+PSQL="psql -A -t -U $PGUSER -h $PGHOST"
+DATABASE=evergreen
+#FULL_QUERY="select distinct id from biblio.record_entry where not deleted"
+FULL_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where not bre.deleted and bre.active and not acn.deleted"
+POSTSEC_QUERY="select distinct record FROM asset.call_number where owning_lib in (select distinct id from actor.org_unit_descendants(27)) and not deleted and label != '##URI##';"
+MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --items --location SITKA --collapse_to_depth 2 --timeout 300 --force901 --encoding UTF-8"
+MARC_EXPORT_BIN="${DIR}/marc_export_custom"
+MARC_EXPORT="${MARC_EXPORT_BIN} ${MARC_PARAMS}"
+SPLIT_COUNT=50000
+
+[ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
+[ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d outlook_export.XXXXXXX)
+
+process_batch() {
+       F="$1" SQL="$2"
+       if [ ! -f ${OUTDIR}/${F}-${T}.id ]; then
+               echo "Getting IDs"
+               $PSQL -c "$SQL" $DATABASE | sort | uniq >${OUTDIR}/${F}-${T}.id
+       fi
+       if [ ! -f ${OUTDIR}/${F}.id.targets ]; then
+               echo "Splitting work"
+               split_suffix ${OUTDIR}/${F}-${T}.id ${OUTDIR}/${F}-${T}-split. .id
+               echo "Making targets"
+               find ${OUTDIR} -name "${F}-${T}-split.[0-9]*[0-9].id" \
+               | sed 's,.id$,.marc,g' \
+               >${OUTDIR}/${F}.id.targets
+       fi
+
+       make ${MAKEOPTS} -f Makefile-marc_export \
+       MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
+       MARC_PARAMS="$MARC_PARAMS --exclusion_ini ${F}.ini" \
+       $(<${OUTDIR}/${F}.id.targets)
+
+       find ${OUTDIR} -name "${F}-${T}-split*.marc" \
+       -exec cat \{} \; \
+       >${OUTDIR}/${F}-${T}.marc
+}
+
+split_suffix() {
+       INPUT="$1" OUTPUT_PREFIX="$2" OUTPUT_SUFFIX="$3"
+       split -a 7 -d -C ${SPLIT_COUNT} ${INPUT} ${OUTPUT_PREFIX}
+       find $(dirname ${OUTPUT_PREFIX}) -name "$(basename ${OUTPUT_PREFIX})*" -exec mv -f \{\} \{\}${OUTPUT_SUFFIX} \;
+}
+
+# media for BC ELN post-secondary libraries
+F=bc_eln_media SQL="$POSTSEC_QUERY"
+process_batch "$F" "${SQL}"
+
+# serials for BC ELN post-secondary libraries
+F=bc_eln_serials SQL="$POSTSEC_QUERY"
+process_batch "$F" "${SQL}"
+
+# full dump for outlook public libraries
+F=sitka_full_outlook SQL="$FULL_QUERY"
+process_batch "$F" "${SQL}"
+
old mode 100644 (file)
new mode 100755 (executable)