Download_log_files

The idea of “download_log_files.pl” script is to have a simple method to access multiple servers that run the same web application (load balanced environment) and pull log files to the same location.

The reason this file was created, is because I am required to analyze the log files from a web application that runs in multiple servers and can not modify the way the logs are created or handle by the application. By using this script I can automate the process of moving the logs from the servers into a server I have more permissions to work with.

The script will perform an SSH connection to a set list of servers, and pull the log archive created that morning by the log rotation process in each server. Then, it will extract the http log file from each of the archives and dump into a file that can later be used by awstats.

#!/usr/bin/perl



##################################

# Andres  Felipe                 #

# afelotreyu @ gmail.com         #

# Date:         Aug  22  2011    #

##################################

use strict;

use vars qw/ $REVISION $VERSION $DIR $PROG $Extension /;

$REVISION='0.01';

$VERSION="3.0 (Build $REVISION)";

($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1;



# Use specific CPAN packages

use Net::SFTP;

use POSIX qw(strftime);



### Variable declaration

my ($user,$pass,@wr0);

my ($re,$re1,$re2,$re3,$re4,$re5,$re6,$re7,$re8);

my %log_out;

my ($wkday,$month,$day,$time,$year);

my $time = localtime;



($wkday,$month,$day,$time,$year) = split(/\s+/, localtime);



#### This lines allow to manually set a day to download

#$day="6";

#$month="Feb";



#####

# This is just to make sure Username and password arguments are provided

for (0..@ARGV-1) {

        if ($ARGV[$_] =~ /^-/) {

                if ($ARGV[$_] =~ /u=/i) { $user = $ARGV[$_]; $user =~ s/-u=//; }

                elsif ($ARGV[$_] =~ /p=/i) { $pass = $ARGV[$_]; $pass =~ s/-p=//; }

        }

}

if ((!$user) || (!$pass)) { print "\n Error please provide username and password \n"; exit(1); }



#####

# create a little globals hash

my %GLOBAL = ( 

        #'server' => ['server1'],

        'server' => ['server1','server2','server3']

        'fqdn' => 'somedomain.local',

        'remdir' => '/some/webapp/logs/archives',

        'locdir' => '/tmp/logs/awstats',

        'user' => "$user",

        'pass' => "$pass",

);



#$ctr_dirs = $#{$GLOBAL{'dirs'}}+1;



#### Regular expressions

#########################################################################

$re1='((?:(?:[0-1][0-9])|(?:[2][0-3])|(?:[0-9])):(?:[0-5][0-9])(?::[0-5][0-9])?)';      # HourMinuteSec 1

$re2='.*?';             # Non-greedy match 

$re3='\s*?';            # White Space match

$re4="$re2($month)$re3($day)$re3$re1$re2(\.zip)";       # Match '* Month Day Hour * .zip'

#####



#### Ok lets get started



my $i=0;

my $count_files;

my ($serv,$host,$local_dir,$call,$file);

my ($dir,$tmp,$k,$v);

my $sftp;

my (@files,$stdout,$stderr,$exit);

my @stdout;

my %ssh_args = ( 'user'=>$GLOBAL{'user'}, 'password'=>$GLOBAL{'pass'}, );



#####

# start the loop of servers.

foreach (@{$GLOBAL{'server'}}) {

        $sftp = '';

        @files ='';

        $i=0;

        # create variables for ssh and sftp

        $serv = $_;

        $host = $serv.".".$GLOBAL{'fqdn'};

        $local_dir = $GLOBAL{locdir}."/".$serv."/";

        print "Working with host : ". $host ."\n";

        

        ## First delete all content from local directory

        $call = 'rm -fr '.$local_dir.'/*';      

        system($call);

        

        eval{ 

                #$sftp = Net::SFTP->new($host, %ssh_args);

                ## Make SSH connection to server

                $sftp = Net::SFTP->new($host, %ssh_args); };

        if ($@) { print "SFTP connection failed:\n  $@\n"; }

        if (! $sftp) {

                print "I can't connect to $host! \n\n";

        }

        else{

                print "Connected to host $host!\n";

                # This gets a list of all zip files created for today

                # From the logdir Variable

                # sftp->ls($remotedir);

                $sftp->ls($GLOBAL{remdir}, sub { 

                        $tmp = $_[0]->{longname};

                        if ($tmp =~ m/$re4/ ) { 

                                $files[$i] = $_[0]->{filename}; 

                                $i++;

                                #######

                                print $tmp."\n"; # Be nice and print on screen for now

                        }

                });



                # Do we have any files to download?

                if ($files[0]) { # We have at least one

                        for ($i=0; $i < = $#files; $i++) {

                                print "Downloading file $files[$i] \n";

                                ## Sftp->get ( remotedir, localdir, function )

                                $sftp->get($GLOBAL{remdir}."/".$files[$i], $local_dir.$files[$i], \&callback);

                                print "\n Download Completed \n";

                        }

        

                        undef $sftp;

        

                        print "Now we extract the access files \n";

                        for ($i=0; $i < = $#files; $i++) {

                                ## Build The command

                                ## Note here we get unzip to pipe output to a file we append information to.

                                $call="unzip -p ". $local_dir.$files[$i] ." \"*logs\/httpd\/access_log\" >> ". $local_dir ."access_log";

                                print $call."\n";

                                system($call);

                        } # Match .zip

                }

                else {

                        print "No Files on Server for this date \n";

                }

        

                print "Done with this host. \n\n\n";



                $call = "chown -R root:apache $local_dir";

                system($call);

        } # Closing else 

}



sub callback {

        my($sftp, $data, $offset, $size) = @_;

        print "Read $offset / $size bytes\r";

    }



## Great work, lets close this script

exit(0);

I hope it can help others, if not as reference then at least for them to build something else.