typofix on installing Bootstrap dependencies
[sitka/sitka-tools.git] / marc_export_custom / sitka-full-export-parallel.sh
CommitLineData
a54b8f42
RJ
1#!/bin/bash
2
e38e56a8 3DIR=$(readlink -f $(dirname $0))
a54b8f42
RJ
4#cd /srv/openils/bin
5
6PGUSER=evergreen
278f5edb 7PGHOST=prod-pg96-db2-sitka
a54b8f42
RJ
8PSQL="psql -A -t -U $PGUSER -h $PGHOST"
9DATABASE=evergreen
10#FULL_QUERY="select distinct id from biblio.record_entry where not deleted"
01ccafec
MB
11#FULL_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where not bre.deleted and bre.active and not acn.deleted"
12FULL_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where not bre.deleted and bre.active and not acn.deleted and acn.label != '##URI##' AND acn.owning_lib IN (SELECT id FROM actor.org_unit_descendants(21) UNION SELECT id FROM actor.org_unit_descendants(600) )"
77d28950 13NONEXPORTED_QUERY="select distinct acn.record FROM asset.call_number acn join biblio.record_entry bre on (acn.record = bre.id) where bre.deleted or not bre.active or acn.deleted"
01ccafec 14MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --items --location SITKA --collapse_to_depth 3 --timeout 300 --force901 --encoding UTF-8 --onlyholdings --quiet --field-852-4b-type circ"
a54b8f42 15MARC_EXPORT_BIN="${DIR}/marc_export_custom"
e5fb4602
RJ
16# this lets us inject perl opts
17MARC_EXPORT="perl ${PERLOPT} -- ${MARC_EXPORT_BIN} ${MARC_PARAMS}"
a54b8f42 18
a54b8f42
RJ
19[ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
20[ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d outlook_export.XXXXXXX)
21
22process_batch() {
77d28950
RJ
23 local f="$1" sql="$2" split_count="$3"
24 if [ ! -f ${OUTDIR}/${f}-${T}.id ]; then
a54b8f42 25 echo "Getting IDs"
77d28950 26 $PSQL -c "$sql" $DATABASE | sort | uniq >${OUTDIR}/${f}-${T}.id
a54b8f42 27 fi
77d28950 28 if [ ! -f ${OUTDIR}/${f}.id.targets ]; then
a54b8f42 29 echo "Splitting work"
77d28950 30 split_suffix ${OUTDIR}/${f}-${T}.id ${OUTDIR}/${f}-${T}-split. .id $split_count
a54b8f42 31 echo "Making targets"
77d28950 32 find ${OUTDIR} -name "${f}-${T}-split.[0-9]*[0-9].id" \
a54b8f42 33 | sed 's,.id$,.marc,g' \
77d28950 34 >${OUTDIR}/${f}.id.targets
a54b8f42
RJ
35 fi
36
37 make ${MAKEOPTS} -f Makefile-marc_export \
38 MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
77d28950
RJ
39 MARC_PARAMS="$MARC_PARAMS --exclusion_ini ${f}.ini" \
40 $(<${OUTDIR}/${f}.id.targets)
368b2926 41 make_rc=$?
a54b8f42 42
77d28950 43 find ${OUTDIR} -name "${f}-${T}-split*.marc" \
a54b8f42 44 -exec cat \{} \; \
77d28950 45 >${OUTDIR}/${f}-${T}.marc
368b2926 46 return $make_rc
a54b8f42
RJ
47}
48
49split_suffix() {
77d28950
RJ
50 local input="$1" output_prefix="$2" output_suffix="$3" split_count="$4"
51 split -a 7 -d -C ${split_count} ${input} ${output_prefix}
52 find $(dirname ${output_prefix}) -name "$(basename ${output_prefix})*" -exec mv -f \{\} \{\}${output_suffix} \;
a54b8f42
RJ
53}
54
77d28950 55export SPLIT_COUNT=100000
6f7b4d95
RJ
56# full dump for outlook public libraries
57F=sitka_full_outlook SQL="$FULL_QUERY"
77d28950 58process_batch "$F" "${SQL}" $SPLIT_COUNT
fe093894 59rc=$?
368b2926 60
368b2926 61exit $rc