Add parallel tools.
authorRobin H. Johnson <rjohnson@sitka.bclibraries.ca>
Fri, 7 Oct 2011 20:57:36 +0000 (13:57 -0700)
committerRobin H. Johnson <rjohnson@sitka.bclibraries.ca>
Fri, 7 Oct 2011 20:57:36 +0000 (13:57 -0700)
Signed-off-by: Robin H. Johnson <rjohnson@sitka.bclibraries.ca>
marc_export_custom/Makefile-marc_export [new file with mode: 0644]
marc_export_custom/sitka-full-export-parallel.sh [new file with mode: 0755]

diff --git a/marc_export_custom/Makefile-marc_export b/marc_export_custom/Makefile-marc_export
new file mode 100644 (file)
index 0000000..bcd9307
--- /dev/null
@@ -0,0 +1,3 @@
+MARC_EXPORT_BIN=./marc_export_custom
+%.marc: %.id
+       $(MARC_EXPORT_BIN) $(MARC_PARAMS) <$< --output-file $@.tmp && mv $@.tmp $@
diff --git a/marc_export_custom/sitka-full-export-parallel.sh b/marc_export_custom/sitka-full-export-parallel.sh
new file mode 100755 (executable)
index 0000000..a26d4f5
--- /dev/null
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+DIR=`dirname $0`
+#cd /srv/openils/bin
+
+PGUSER=evergreen
+PGHOST=db1.sitka.bclibraries.ca
+PSQL="psql -A -t -U $PGUSER -h $PGHOST"
+DATABASE=evergreen
+#FULL_QUERY="select distinct id from biblio.record_entry where not deleted"
+FULL_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where not bre.deleted and bre.active and not acn.deleted"
+POSTSEC_QUERY="select distinct record FROM asset.call_number where owning_lib in (select distinct id from actor.org_unit_descendants(27)) and not deleted and label != '##URI##';"
+MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --items --location SITKA --collapse_to_depth 2 --timeout 300 --force901 --encoding UTF-8"
+MARC_EXPORT_BIN="${DIR}/marc_export_custom"
+MARC_EXPORT="${MARC_EXPORT_BIN} ${MARC_PARAMS}"
+SPLIT_COUNT=50000
+
+
+[ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
+[ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d outlook_export.XXXXXXX)
+
+process_batch() {
+       F="$1" SQL="$2"
+       if [ ! -f ${OUTDIR}/${F}-${T}.id ]; then
+               echo "Getting IDs"
+               $PSQL -c "$SQL" $DATABASE >${OUTDIR}/${F}-${T}.id
+       fi
+       if [ ! -f ${OUTDIR}/${F}.id.targets ]; then
+               echo "Splitting work"
+               split_suffix ${OUTDIR}/${F}-${T}.id ${OUTDIR}/${F}-${T}-split. .id
+               echo "Making targets"
+               find ${OUTDIR} -name "${F}-${T}-split.[0-9]*[0-9].id" \
+               | sed 's,.id$,.marc,g' \
+               >${OUTDIR}/${F}.id.targets
+       fi
+
+       make ${MAKEOPTS} -f Makefile-marc_export \
+       MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
+       MARC_PARAMS="$MARC_PARAMS --exclusion_ini ${F}.ini" \
+       $(<${OUTDIR}/${F}.id.targets)
+
+       find ${OUTDIR} -name "${F}-${T}-split*.marc" \
+       -exec cat \{} \; \
+       >${OUTDIR}/${F}-${T}.marc
+}
+
+split_suffix() {
+       INPUT="$1" OUTPUT_PREFIX="$2" OUTPUT_SUFFIX="$3"
+       split -a 7 -d -C ${SPLIT_COUNT} ${INPUT} ${OUTPUT_PREFIX}
+       find $(dirname ${OUTPUT_PREFIX}) -name "$(basename ${OUTPUT_PREFIX})*" -exec mv -f \{\} \{\}${OUTPUT_SUFFIX} \;
+}
+
+# full dump for outlook public libraries
+F=sitka_full_outlook SQL="$FULL_QUERY"
+#process_batch "$F" "${SQL}"
+
+# media for BC ELN post-secondary libraries
+F=bc_eln_media SQL="$POSTSEC_QUERY"
+#process_batch "$F" "${SQL}"
+
+# serials for BC ELN post-secondary libraries
+F=bc_eln_serials SQL="$POSTSEC_QUERY"
+process_batch "$F" "${SQL}"