Avoid dupes in input ids, add outlook publics.
[sitka/sitka-tools.git] / marc_export_custom / sitka-full-export-parallel.sh
CommitLineData
a54b8f42
RJ
1#!/bin/bash
2
3DIR=`dirname $0`
4#cd /srv/openils/bin
5
6PGUSER=evergreen
7PGHOST=db1.sitka.bclibraries.ca
8PSQL="psql -A -t -U $PGUSER -h $PGHOST"
9DATABASE=evergreen
10#FULL_QUERY="select distinct id from biblio.record_entry where not deleted"
11FULL_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where not bre.deleted and bre.active and not acn.deleted"
12POSTSEC_QUERY="select distinct record FROM asset.call_number where owning_lib in (select distinct id from actor.org_unit_descendants(27)) and not deleted and label != '##URI##';"
13MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --items --location SITKA --collapse_to_depth 2 --timeout 300 --force901 --encoding UTF-8"
14MARC_EXPORT_BIN="${DIR}/marc_export_custom"
15MARC_EXPORT="${MARC_EXPORT_BIN} ${MARC_PARAMS}"
16SPLIT_COUNT=50000
17
a54b8f42
RJ
18[ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
19[ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d outlook_export.XXXXXXX)
20
21process_batch() {
22 F="$1" SQL="$2"
23 if [ ! -f ${OUTDIR}/${F}-${T}.id ]; then
24 echo "Getting IDs"
6f7b4d95 25 $PSQL -c "$SQL" $DATABASE | sort | uniq >${OUTDIR}/${F}-${T}.id
a54b8f42
RJ
26 fi
27 if [ ! -f ${OUTDIR}/${F}.id.targets ]; then
28 echo "Splitting work"
29 split_suffix ${OUTDIR}/${F}-${T}.id ${OUTDIR}/${F}-${T}-split. .id
30 echo "Making targets"
31 find ${OUTDIR} -name "${F}-${T}-split.[0-9]*[0-9].id" \
32 | sed 's,.id$,.marc,g' \
33 >${OUTDIR}/${F}.id.targets
34 fi
35
36 make ${MAKEOPTS} -f Makefile-marc_export \
37 MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
38 MARC_PARAMS="$MARC_PARAMS --exclusion_ini ${F}.ini" \
39 $(<${OUTDIR}/${F}.id.targets)
40
41 find ${OUTDIR} -name "${F}-${T}-split*.marc" \
42 -exec cat \{} \; \
43 >${OUTDIR}/${F}-${T}.marc
44}
45
46split_suffix() {
47 INPUT="$1" OUTPUT_PREFIX="$2" OUTPUT_SUFFIX="$3"
48 split -a 7 -d -C ${SPLIT_COUNT} ${INPUT} ${OUTPUT_PREFIX}
49 find $(dirname ${OUTPUT_PREFIX}) -name "$(basename ${OUTPUT_PREFIX})*" -exec mv -f \{\} \{\}${OUTPUT_SUFFIX} \;
50}
51
a54b8f42
RJ
52# media for BC ELN post-secondary libraries
53F=bc_eln_media SQL="$POSTSEC_QUERY"
6f7b4d95 54process_batch "$F" "${SQL}"
a54b8f42
RJ
55
56# serials for BC ELN post-secondary libraries
57F=bc_eln_serials SQL="$POSTSEC_QUERY"
58process_batch "$F" "${SQL}"
6f7b4d95
RJ
59
60# full dump for outlook public libraries
61F=sitka_full_outlook SQL="$FULL_QUERY"
62process_batch "$F" "${SQL}"
63