install-eg.sh: fix setup of non-prod Apache includes
[sitka/sitka-tools.git] / marc_export_custom / mwpl-full-export-parallel.sh
CommitLineData
a2926055
JD
1#!/bin/bash
2
3DIR=$(readlink -f $(dirname $0))
4#cd /srv/openils/bin
5
6PGUSER=evergreen
286b73a7 7PGHOST=prod-pg94-db2-sitka.sitka.bclibraries.ca
a2926055
JD
8PSQL="psql -A -t -U $PGUSER -h $PGHOST"
9DATABASE=evergreen
10MWPL_QUERY="select distinct record FROM asset.call_number where owning_lib in (select distinct id from actor.org_unit_descendants(1020)) and not deleted and label != '##URI##';"
11MARC_PARAMS="--config /srv/openils/conf/opensrf_core.xml --location MWPL --timeout 300 --force901 --encoding UTF-8 --quiet"
12MARC_EXPORT_BIN="${DIR}/marc_export_custom"
e270fa90
RJ
13# this lets us inject perl opts
14MARC_EXPORT="perl ${PERLOPT} -- ${MARC_EXPORT_BIN} ${MARC_PARAMS}"
a2926055
JD
15
16[ -z "${T}" ] && T=$(date -u +%Y%m%dT%H%M%SZ)
17[ -z "${OUTDIR}" ] && OUTDIR=$(mktemp --tmpdir -d outlook_export.XXXXXXX)
18
19process_batch() {
20 local f="$1" sql="$2" split_count="$3"
21 if [ ! -f ${OUTDIR}/${f}-${T}.id ]; then
22 echo "Getting IDs"
23 $PSQL -c "$sql" $DATABASE | sort | uniq >${OUTDIR}/${f}-${T}.id
24 fi
25 if [ ! -f ${OUTDIR}/${f}.id.targets ]; then
26 echo "Splitting work"
27 split_suffix ${OUTDIR}/${f}-${T}.id ${OUTDIR}/${f}-${T}-split. .id $split_count
28 echo "Making targets"
29 find ${OUTDIR} -name "${f}-${T}-split.[0-9]*[0-9].id" \
30 | sed 's,.id$,.marc,g' \
31 >${OUTDIR}/${f}.id.targets
32 fi
33
34 make ${MAKEOPTS} -f Makefile-marc_export \
35 MARC_EXPORT_BIN="${MARC_EXPORT_BIN}" \
36 MARC_PARAMS="$MARC_PARAMS --exclusion_ini ${f}.ini" \
37 $(<${OUTDIR}/${f}.id.targets)
38
39 find ${OUTDIR} -name "${f}-${T}-split*.marc" \
40 -exec cat \{} \; \
41 >${OUTDIR}/${f}-${T}.marc
42}
43
44split_suffix() {
45 local input="$1" output_prefix="$2" output_suffix="$3" split_count="$4"
46 split -a 7 -d -C ${split_count} ${input} ${output_prefix}
47 find $(dirname ${output_prefix}) -name "$(basename ${output_prefix})*" -exec mv -f \{\} \{\}${output_suffix} \;
48}
49
50# media for BC ELN post-secondary libraries
51export SPLIT_COUNT=10000
52F=mwpl SQL="$MWPL_QUERY"
53process_batch "$F" "${SQL}" $SPLIT_COUNT