add parallel script for BSLW (no holdings, with separate TNRD export)
[sitka/sitka-tools.git] / marc_export_custom / sitka-full-export-bslw.sh
1 #!/bin/bash
2
3 DIR=`dirname $0`
4 #cd /srv/openils/bin
5
6 PGUSER=evergreen
7 PGHOST=db1.sitka.bclibraries.ca
8 PSQL="psql -A -t -U $PGUSER -h $PGHOST"
9 DATABASE=evergreen
10 FULL_QUERY="select id from biblio.record_entry where not deleted"
11 TNRD_QUERY="select distinct record FROM asset.call_number where owning_lib in (select distinct id from actor.org_unit_descendants(50)) and not deleted;"
12 MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --timeout 300 --force901 --encoding UTF-8"
13 MARC_EXPORT_BIN="${DIR}/marc_export_custom"
14 MARC_EXPORT="${MARC_EXPORT_BIN} ${MARC_PARAMS}"
15
16 [ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
17 [ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d backstage_export.XXXXXXX)
18
19 process_batch() {
20         local f="$1" sql="$2" split_count="$3"
21         if [ ! -f ${OUTDIR}/${f}-${T}.id ]; then
22                 echo "Getting IDs"
23                 $PSQL -c "$sql" $DATABASE | sort | uniq >${OUTDIR}/${f}-${T}.id
24         fi
25         if [ ! -f ${OUTDIR}/${f}.id.targets ]; then
26                 echo "Splitting work"
27                 split_suffix ${OUTDIR}/${f}-${T}.id ${OUTDIR}/${f}-${T}-split. .id $split_count
28                 echo "Making targets"
29                 find ${OUTDIR} -name "${f}-${T}-split.[0-9]*[0-9].id" \
30                 | sed 's,.id$,.marc,g' \
31                 >${OUTDIR}/${f}.id.targets
32         fi
33
34         make ${MAKEOPTS} -f Makefile-marc_export \
35         MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
36         MARC_PARAMS="$MARC_PARAMS" \
37         $(<${OUTDIR}/${f}.id.targets)
38
39         find ${OUTDIR} -name "${f}-${T}-split*.marc" \
40         -exec cat \{} \; \
41         >${OUTDIR}/${f}-${T}.marc
42 }
43
44 split_suffix() {
45         local input="$1" output_prefix="$2" output_suffix="$3" split_count="$4"
46         split -a 7 -d -C ${split_count} ${input} ${output_prefix}
47         find $(dirname ${output_prefix}) -name "$(basename ${output_prefix})*" -exec mv -f \{\} \{\}${output_suffix} \;
48 }
49
50 # TNRD dump
51 export SPLIT_COUNT=10000
52 F=tnrd SQL="$TNRD_QUERY"
53 process_batch "$F" "${SQL}" $SPLIT_COUNT
54
55 export SPLIT_COUNT=100000
56 # full dump for all libraries
57 F=sitka_full_backstage SQL="$FULL_QUERY"
58 process_batch "$F" "${SQL}" $SPLIT_COUNT
59