Merge branch 'master' into eg210

[sitka/sitka-tools.git] / deployment / integrity-checker.pl

#!/usr/bin/perl
use strict;
use warnings;
# On Ubuntu, you'll want the following packages:
# - libconfig-simple-perl
# - libgit-repository-perl
# - libdate-manip-perl
use Config::Simple;
use File::Find;
use File::stat;
use Date::Manip qw/ParseDate UnixDate/;
use Time::localtime;
use Git::Repository;
use Git::Repository::Command;
use Getopt::Long;
use Data::Dumper;

my ($help, $config_file, $repo_path, $all, $check_files, $hash_file, $since, $git_output, $deployed_output);
my $branch = 'HEAD';
my $remote = 'origin';
my @components;

GetOptions(
    'help'              => \$help,            # show help message and exit
    'config=s'          => \$config_file,     # INI file for path mappings
    'repo=s'            => \$repo_path,       # location of git repo
    'branch=s'          => \$branch,          # git branch/head to check against (defaults to 'HEAD')
    'remote=s'          => \$remote,          # git remote to pull from (defaults to 'origin')
    'component=s'       => \@components,      # parts of EG to be checked (each component is a block in the INI file)
    'all'               => \$all,             # check all components specified in config (overrides --component)
    'check-files'       => \$check_files,     # check deployed files
    'hash-file=s'       => \$hash_file,       # file containing git hashes (overrides --repo)
    'since=s'           => \$since,           # check only files modified since this time
    'print-git-hashes=s'      => \$git_output,      # output file for git hashes (optional)
    'print-deployed-hashes=s' => \$deployed_output  # output file for hashes of deployed files (optional)
);

if ($help) {
    print <<"HELP";
USAGE:
    $0 --config pathmap.ini --repo /path/to/evergreen.git [ --component perl [ --component tt2 ] | --all ] --check-files [ --since <date> ]
    $0 --config pathmap.ini --repo /path/to/evergreen.git [ --component perl [ --component tt2 ... ] | --all ] --print-git-hashes <git-hashes.txt>
    $0 --config pathmap.ini --repo /path/to/evergreen.git [ --component perl [ --component tt2 ... ] | --all ] --print-deployed-hashes <deployed-hashes.txt>

OPTIONS:
    --help
        Show help message and exit.
    --config
        Location of INI file for path mappings.
    --repo
        Location of git repo (overridden by --hash-file).
    --branch
        Git branch (head) to check against (defaults to HEAD).
    --remote
        Git remote to pull from (defaults to origin).
    --component
        Parts of EG to be checked (each component is a block in the config file).
        You can use this option multiple times: --component perl --component web
    --all
        Check all components specified in config file (overrides --component).
    --check-files
        Get file hashes from git, then check deployed files to see if they match.
    --hash-file
        File containing file hashes from git.  If you use this option, you don't
        need to specify a git repo using the --repo option.
        Use case: pull hashes from git once, copy the resulting file to multiple
        servers, then check the deployed code against the file instead of pulling
        hashes from git individually on each server.
    --since
        Only calculate hashes if file has been modified since the specified time.
    --print-git-hashes
        Get file hashes from git repo and print/append to the specified file.
        Can be used with --check-files and --print-deployed-hashes.
    --print-deployed-hashes
        Print git-like hashes for deployed files.
        Can be used with --check-files and --print-git-hashes.

HELP
    exit;
}

# specify all possible components (--all option);
@components = split('\n', `./access_pathmap.pl --config $config_file`) if ($all);

if ($git_output) {
    open (GITOUTPUT, '>>', $git_output) or die "Could not open $git_output: $!\n";
}
if ($deployed_output) {
    open (DEPLOYEDOUTPUT, '>>', $deployed_output) or die "Could not open $deployed_output: $!\n";
}

my %git_hashes;

# optionally read in git hashes from file
if ($hash_file) {
    open (HASHFILE, '<', $hash_file) or die "Could not open $hash_file: $!\n";
    while (<HASHFILE>) {
        chomp;
        my ($hash, $file) = split(/\s+/, $_, 2);
        $git_hashes{$file} = $hash;
    }
    close HASHFILE;
}

foreach my $component (@components) {
    my @paths = split('\n', `./access_pathmap.pl --config $config_file --component $component`);

    # if no hash file was supplied, grab git hashes from repo
    if (!$hash_file) {

        # load git repo
        die "No repo specified\n" unless ($repo_path);
        $repo_path =~ s|/$||;
        $repo_path = "$repo_path/.git" unless ($repo_path =~ /\.git$/);
        my $repo = Git::Repository->new( git_dir => $repo_path ) or die "Could not load git repo $repo_path: $!\n";

        # ensure git repo is up-to-date
        if ($branch ne 'HEAD') {
            $repo->run( 'pull' => $remote );
            $repo->run( 'checkout' => $branch ); # TODO: is this necessary?
        }

        # get hashes from git
        foreach my $srcpath (@paths) {
            # use git-ls-tree to traverse the file tree starting at $srcpath
            # e.g. `git ls-tree -r HEAD Open-ILS/src/perlmods/lib`
            my @tree = $repo->run( 'ls-tree' => '-r', $branch, $srcpath );
            foreach my $file (@tree) {
                my ($mode, $type, $hash, $filename) = split(/\s+/, $file, 4);
                $git_hashes{$filename} = $hash;
                print GITOUTPUT $hash, "\t", $filename, "\n" if ($git_output);
            }
        }
    }

    # check deployed files
    if ($check_files || $deployed_output) {
        foreach my $srcpath (@paths) {
            my $destpath = `./access_pathmap.pl --config $config_file --component $component --srcpath $srcpath`;

            # for each file in the destination path, push the file's absolute path to @files;
            # output will include symlinked files, but will not include directories
            # clear @files for each time through loop
            my @files = ();
            find( { wanted => sub { push @files, $_ if -f }, follow => 1, no_chdir => 1 }, $destpath );

            foreach my $file (@files) {
                if ($since) {
                    # convert $since to seconds since epoch
                    my $since_ts = UnixDate($since, '%s');
                    # get $file modification time as seconds since epoch
                    my $file_ts = ctime(stat($file)->mtime);

                    next unless $file_ts > $since_ts;
                }

                # you can calculate what the git hash would be
                # for any file using `git hash-object <file>`;
                # you don't even need to be in a git repo to run it!
                my $hash = Git::Repository->run( 'hash-object', $file );

                if ($check_files) {
                    my $srcfile = $file;
                    $srcfile =~ s|^$destpath|$srcpath|;

                    if (!$git_hashes{$srcfile}) {
                        print "untracked\t$file\n";
                    } elsif ($git_hashes{$srcfile} ne $hash) {
                        print "modified\t$file\n";
                    }
                }
                print DEPLOYEDOUTPUT $hash, "\t", $file, "\n" if ($deployed_output);

            }
        }
    }
}

close (GITOUTPUT) if ($git_output);
close (DEPLOYEDOUTPUT) if ($deployed_output);