From: Liam Whalen Date: Mon, 16 Dec 2013 17:48:36 +0000 (-0800) Subject: [RT18107] A number of modifications X-Git-Url: https://git.sitka.bclibraries.ca/gitweb/?p=sitka%2Fsitka-tools.git;a=commitdiff_plain;h=1c47dbca5dcb121133c9e608cc84b1774ac40eca;hp=c554d43bcaecb329bb3f6c8271c845a53513b1f7 [RT18107] A number of modifications I added the ability for the scirpt to detect access_pathmap.pl. As well, temporary files are now placed in /tmp/${BRANCH}/server_list/year_month_day_hour_minute_second/. So, if I was pusing user/lwhalen/rt14111 to dev3 and dev4 then my files from the branch would be located at /tmp/user/lwhalen/rt14111/dev3_dev4/year_month_day_hour_minute_second/. If we do not delete the temporary files until space is a concern, then this will allow us to have a history of what was deployed and when it was deployed. I also added some more help file text, and a message indicating when a file could not be pushed because it does not have a mapping in the PATHMAP_FILE. Changed output to go to stderr or stdout as appropriate. This means error messages are now going to stderr, and the prompt to ensure that you mean to sync a branch with the servers is going to stdout. The server list used to create the temporary directory is sorted using the sort command in order to provide a consistent directory structure regardless of how the servers are entered on the command line. git-deploy was changed to send a host sepcific config file to access_pathmap.pl. The convention I am using is ${HOSTNAME}_pathmap.ini. If this file exist, then access_pathmap.pl is called with that file as the --config option. I also moved the logic that determins a hostname from a string to a funciton because it is now used in two places. Signed-off-by: Liam Whalen --- diff --git a/deployment/access_pathmap.pl b/deployment/access_pathmap.pl index 678accb..58b21c3 100755 --- a/deployment/access_pathmap.pl +++ b/deployment/access_pathmap.pl @@ -4,13 +4,13 @@ use warnings; use Config::Simple; use Getopt::Long; -my ($help, $config_file, $component, $srcpath); +my ($help, $config_file, $component, $srcpath, $host); GetOptions( - 'help' => \$help, # show help message and exit - 'config=s' => \$config_file, # INI file for path mappings - 'component=s' => \$component, # The component to grab the source paths for - 'srcpath=s' => \$srcpath # The source path to grap the destination path for + 'help' => \$help, # show help message and exit + 'config=s' => \$config_file, # INI file for path mappings + 'component=s' => \$component, # The component to grab the source paths for + 'srcpath=s' => \$srcpath, # The source path to grap the destination path for ); if ($help) { @@ -35,6 +35,7 @@ HELP # load config die "No config file specified\n" unless ($config_file); die "Config file does not exist\n" unless (-r $config_file and -s $config_file); + my $cfg = new Config::Simple($config_file); if (!$component) { diff --git a/deployment/git-deploy.sh b/deployment/git-deploy.sh index ef96175..8321482 100755 --- a/deployment/git-deploy.sh +++ b/deployment/git-deploy.sh @@ -1,10 +1,10 @@ #!/bin/bash #TODO decide about sql files and other odd files -#TODO make sure access_pathmap.ini is fixed so Open-ILS/xul/staff_client/ files map properly PROD_SERVERS=" app1-1 app1-2 app2-1 app2-2 app3-1 app3-2 stanns bibc-prod " REPO='.' +PATHMAP_FILE='pathmap.ini' function usage() { @@ -26,11 +26,19 @@ OPTIONS: -g The location of the git repo -r The remote to deploy from -b The branch on the remote to deploy from + -c The config file with the path mapping between the repo and the server -P Deploy to all production servers -d Actually deploy. While testing this option is needed to force the commands to execute + -s Sync the entire branch (as much as possible tt2, pm, js, maybe others?) with the server(s) + -h This help information EOF } +function get_hostname() { + HOST=$1 + echo `echo $HOST | sed -e 's/^\([^.]*\).*$/\1/'` +} + function create_tmp_file() { TMP_DIR=$1 FILENAME=$2 @@ -38,12 +46,35 @@ function create_tmp_file() { echo "$TMP_DIR/$FILENAME" } +function does_file_exist() { + FILE=$1; + if [ `ls $FILE 2> /dev/null | wc -l` -eq 1 ] + then + return 0 + else + return 1 + fi +} + function find_base_path() { RELATIVE_PATH=$1 + HOST=$2 + HOST=`get_hostname $HOST` - for COMPONENT in `access_pathmap.pl --config $PATH_MAP_FILE` + if does_file_exist ${HOST}'_'$PATHMAP_FILE + then + LOCAL_PATHMAP_FILE=${HOST}'_'$PATHMAP_FILE + elif does_file_exist $PATHMAP_FILE + then + LOCAL_PATHMAP_FILE=$PATHMAP_FILE + else + echo "$PATHMAP file could not be found!" >&2 + exit 4 + fi + + for COMPONENT in `$ACCESS_PATHMAP --config $LOCAL_PATHMAP_FILE` do - BASE_PATH=`access_pathmap.pl --config $PATH_MAP_FILE --component $COMPONENT --srcpath $RELATIVE_PATH` + BASE_PATH=`$ACCESS_PATHMAP --config $LOCAL_PATHMAP_FILE --component $COMPONENT --srcpath $RELATIVE_PATH` if [[ $BASE_PATH ]] then echo $BASE_PATH @@ -52,7 +83,6 @@ function find_base_path() { done } - function get_full_path() { RELATIVE_PATH=$1 BASE_PATH=$2 @@ -79,7 +109,7 @@ function deployfile() { RELATIVE_PATH=$4 FILENAME=$5 TMP_DIR=$6 - HOSTS=$7 + HOST=$7 PERL_FILE_RE='^.*.pm$' SQL_FILE_RE='^.*.sql$' @@ -101,31 +131,28 @@ function deployfile() { TEMPFILE=`create_tmp_file $TMP_DIR $FILENAME` git --git-dir $REPO show $REMOTE/$BRANCH:$RELATIVE_PATH/$FILENAME > $TEMPFILE - for HOST in $HOSTS - do - if [[ $PROD_SERVERS =~ " `echo $HOST | cut -d '.' -f 1` " ]] + + if [[ $PROD_SERVERS =~ " `echo $HOST | cut -d '.' -f 1` " ]] + then + echo "Cannot deploy `basename $TEMPFILE` to $HOST while testing. Please use another server" + else + COMMAND="deployfile.sh -f $TEMPFILE -p $FULL_PATH -u $USER -b $HOST" + if [[ -z "$DEPLOY" ]] then - echo "Cannot deploy `basename $TEMPFILE` to $HOST while testing. Please use another server" + echo $COMMAND else - COMMAND="deployfile.sh -f $TEMPFILE -p $FULL_PATH -u $USER -b $HOST" - if [[ -z "$DEPLOY" ]] - then - echo $COMMAND - else - $COMMAND - fi - + $COMMAND fi - done + fi } -while getopts ":g:r:b:c:Pdhs" opt +while getopts ":g:r:b:c:Pdsh" opt do case $opt in g ) REPO=$OPTARG;; r ) REMOTE=$OPTARG;; b ) BRANCH=$OPTARG;; - c ) PATH_MAP_FILE=$OPTARG;; + c ) PATHMAP_FILE=$OPTARG;; P ) PROD="P";; d ) DEPLOY="d";; s ) SYNC="s";; @@ -149,37 +176,64 @@ fi if [[ -z $HOST_LIST ]] then - echo 'You must specify a host to deploy to in order to use this script' + echo 'You must specify a host to deploy to in order to use this script' >&2 exit 1 fi if [[ ! -z $SYNC ]] then - read -p "Are you SURE you want to sync $HOST_LIST with $BRANCH (Yes/No): " ANSWER + read -p "Are you SURE you want to sync $HOST_LIST with $BRANCH (Yes/No): " ANSWER 2>&1 if [[ ! "$ANSWER" =~ [Yy]es ]] then - echo "Aborting sync" + echo "Aborting sync" >&2 exit 2; fi fi -TMP_DIR=`mktemp -d` +#This line takes the host list and separates it into individual lines, then sorts it. Next it removes anything following the hostname with get_hostname +#(hopefully the domain name), then makes it a single line again with _ instead of ' '. +#It then uses parameter substitution to remove the last character from the string, which is a _ because sort places a newline at the end of the +#last server that gets translated into an unwanted _. We need to sort the servers, so that they are used in a consistent manner for tracking +#deployment +HOST_LIST_FOR_DIR_NAME=`echo $HOST_LIST | tr ' ' '\n' | sort | while read SORTED_HOST_LIST ; do get_hostname $SORTED_HOST_LIST ; done | tr '\n' '_' | { read SORTED_HOST_LIST; echo ${SORTED_HOST_LIST%?}; }` +TMP_DIR='/tmp/'$BRANCH'/'$HOST_LIST_FOR_DIR_NAME'/'`date +%Y_%m_%d_%H_%M_%S` + +mkdir -p $TMP_DIR if [[ -z "$SYNC" ]] then LIST_BRANCH_FILES=`git --git-dir $REPO show $REMOTE/$BRANCH --name-only --oneline | awk '{if(NR!=1) {print}}'` else BRANCH_NAME=`basename $BRANCH` - LIST_BRANCH_FILES=`cd $REPO && git ls-tree -r --full-name $BRANCH_NAME | tr '\t' ' ' | cut -d ' ' -f 4` + LIST_BRANCH_FILES=`cd $REPO && git ls-tree -r --full-name $BRANCH_NAME | cut -f 2` +fi + +#Find access_pathmap.pl and if not found assume it is in current direcotry +ACCESS_PATHMAP=`which access_pathmap.pl` + +if [[ -z $ACCESS_PATHMAP ]] +then + if does_file_exist access_pathmap.pl + then + ACCESS_PATHMAP='./access_pathmap.pl' + else + echo 'Could not locate access_pathmap.pl. Either add access_pathmap.pl to your PATH or place it in the directory you are working in.' >&2 + exit 3 + fi fi for PATH_AND_FILENAME in $LIST_BRANCH_FILES do FILENAME=`basename $PATH_AND_FILENAME` RELATIVE_PATH=`dirname $PATH_AND_FILENAME` - BASE_PATH=`find_base_path $RELATIVE_PATH` - if [[ $BASE_PATH ]] - then - deployfile $REMOTE $BRANCH $BASE_PATH $RELATIVE_PATH $FILENAME $TMP_DIR "$HOST_LIST" - fi + for DEPLOY_HOST in $HOST_LIST + do + BASE_PATH=`find_base_path $RELATIVE_PATH $DEPLOY_HOST` + if [[ $BASE_PATH ]] + then + deployfile $REMOTE $BRANCH $BASE_PATH $RELATIVE_PATH $FILENAME $TMP_DIR $DEPLOY_HOST + else + echo "Could not deploy $FILENAME to $DEPLOY_HOST because no mapping could be found in $PATHMAP_FILE or any host specific pathmap files" >&2 + fi + done done